LLVM  9.0.0svn
AMDGPULibCalls.cpp
Go to the documentation of this file.
1 //===- AMDGPULibCalls.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file does AMD library function optimizations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #define DEBUG_TYPE "amdgpu-simplifylib"
15 
16 #include "AMDGPU.h"
17 #include "AMDGPULibFunc.h"
19 #include "llvm/Analysis/Loads.h"
20 #include "llvm/ADT/StringSet.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/IR/Constants.h"
23 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/IR/Instructions.h"
25 #include "llvm/IR/IRBuilder.h"
26 #include "llvm/IR/Function.h"
27 #include "llvm/IR/LLVMContext.h"
28 #include "llvm/IR/Module.h"
30 #include "llvm/Support/Debug.h"
33 #include <vector>
34 #include <cmath>
35 
36 using namespace llvm;
37 
38 static cl::opt<bool> EnablePreLink("amdgpu-prelink",
39  cl::desc("Enable pre-link mode optimizations"),
40  cl::init(false),
41  cl::Hidden);
42 
43 static cl::list<std::string> UseNative("amdgpu-use-native",
44  cl::desc("Comma separated list of functions to replace with native, or all"),
46  cl::Hidden);
47 
48 #define MATH_PI 3.14159265358979323846264338327950288419716939937511
49 #define MATH_E 2.71828182845904523536028747135266249775724709369996
50 #define MATH_SQRT2 1.41421356237309504880168872420969807856967187537695
51 
52 #define MATH_LOG2E 1.4426950408889634073599246810018921374266459541529859
53 #define MATH_LOG10E 0.4342944819032518276511289189166050822943970058036665
54 // Value of log2(10)
55 #define MATH_LOG2_10 3.3219280948873623478703194294893901758648313930245806
56 // Value of 1 / log2(10)
57 #define MATH_RLOG2_10 0.3010299956639811952137388947244930267681898814621085
58 // Value of 1 / M_LOG2E_F = 1 / log2(e)
59 #define MATH_RLOG2_E 0.6931471805599453094172321214581765680755001343602552
60 
61 namespace llvm {
62 
64 private:
65 
67 
68  // -fuse-native.
69  bool AllNative = false;
70 
71  bool useNativeFunc(const StringRef F) const;
72 
73  // Return a pointer (pointer expr) to the function if function defintion with
74  // "FuncName" exists. It may create a new function prototype in pre-link mode.
75  FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
76 
77  // Replace a normal function with its native version.
78  bool replaceWithNative(CallInst *CI, const FuncInfo &FInfo);
79 
80  bool parseFunctionName(const StringRef& FMangledName,
81  FuncInfo *FInfo=nullptr /*out*/);
82 
83  bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
84 
85  /* Specialized optimizations */
86 
87  // recip (half or native)
88  bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
89 
90  // divide (half or native)
91  bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
92 
93  // pow/powr/pown
94  bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
95 
96  // rootn
97  bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
98 
99  // fma/mad
100  bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
101 
102  // -fuse-native for sincos
103  bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
104 
105  // evaluate calls if calls' arguments are constants.
106  bool evaluateScalarMathFunc(FuncInfo &FInfo, double& Res0,
107  double& Res1, Constant *copr0, Constant *copr1, Constant *copr2);
108  bool evaluateCall(CallInst *aCI, FuncInfo &FInfo);
109 
110  // exp
111  bool fold_exp(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
112 
113  // exp2
114  bool fold_exp2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
115 
116  // exp10
117  bool fold_exp10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
118 
119  // log
120  bool fold_log(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
121 
122  // log2
123  bool fold_log2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
124 
125  // log10
126  bool fold_log10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
127 
128  // sqrt
129  bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
130 
131  // sin/cos
132  bool fold_sincos(CallInst * CI, IRBuilder<> &B, AliasAnalysis * AA);
133 
134  // __read_pipe/__write_pipe
135  bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, FuncInfo &FInfo);
136 
137  // Get insertion point at entry.
138  BasicBlock::iterator getEntryIns(CallInst * UI);
139  // Insert an Alloc instruction.
140  AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix);
141  // Get a scalar native builtin signle argument FP function
142  FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
143 
144 protected:
146 
147  bool isUnsafeMath(const CallInst *CI) const;
148 
149  void replaceCall(Value *With) {
150  CI->replaceAllUsesWith(With);
151  CI->eraseFromParent();
152  }
153 
154 public:
155  bool fold(CallInst *CI, AliasAnalysis *AA = nullptr);
156 
157  void initNativeFuncs();
158 
159  // Replace a normal math function call with that native version
160  bool useNative(CallInst *CI);
161 };
162 
163 } // end llvm namespace
164 
165 namespace {
166 
167  class AMDGPUSimplifyLibCalls : public FunctionPass {
168 
169  AMDGPULibCalls Simplifier;
170 
171  const TargetOptions Options;
172 
173  public:
174  static char ID; // Pass identification
175 
176  AMDGPUSimplifyLibCalls(const TargetOptions &Opt = TargetOptions())
177  : FunctionPass(ID), Options(Opt) {
179  }
180 
181  void getAnalysisUsage(AnalysisUsage &AU) const override {
183  }
184 
185  bool runOnFunction(Function &M) override;
186  };
187 
188  class AMDGPUUseNativeCalls : public FunctionPass {
189 
190  AMDGPULibCalls Simplifier;
191 
192  public:
193  static char ID; // Pass identification
194 
195  AMDGPUUseNativeCalls() : FunctionPass(ID) {
197  Simplifier.initNativeFuncs();
198  }
199 
200  bool runOnFunction(Function &F) override;
201  };
202 
203 } // end anonymous namespace.
204 
206 char AMDGPUUseNativeCalls::ID = 0;
207 
208 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
209  "Simplify well-known AMD library calls", false, false)
211 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
212  "Simplify well-known AMD library calls", false, false)
213 
214 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative",
215  "Replace builtin math calls with that native versions.",
216  false, false)
217 
218 template <typename IRB>
219 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
220  const Twine &Name = "") {
221  CallInst *R = B.CreateCall(Callee, Arg, Name);
222  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
223  R->setCallingConv(F->getCallingConv());
224  return R;
225 }
226 
227 template <typename IRB>
228 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
229  Value *Arg2, const Twine &Name = "") {
230  CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
231  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
232  R->setCallingConv(F->getCallingConv());
233  return R;
234 }
235 
236 // Data structures for table-driven optimizations.
237 // FuncTbl works for both f32 and f64 functions with 1 input argument
238 
239 struct TableEntry {
240  double result;
241  double input;
242 };
243 
244 /* a list of {result, input} */
245 static const TableEntry tbl_acos[] = {
246  {MATH_PI/2.0, 0.0},
247  {MATH_PI/2.0, -0.0},
248  {0.0, 1.0},
249  {MATH_PI, -1.0}
250 };
251 static const TableEntry tbl_acosh[] = {
252  {0.0, 1.0}
253 };
254 static const TableEntry tbl_acospi[] = {
255  {0.5, 0.0},
256  {0.5, -0.0},
257  {0.0, 1.0},
258  {1.0, -1.0}
259 };
260 static const TableEntry tbl_asin[] = {
261  {0.0, 0.0},
262  {-0.0, -0.0},
263  {MATH_PI/2.0, 1.0},
264  {-MATH_PI/2.0, -1.0}
265 };
266 static const TableEntry tbl_asinh[] = {
267  {0.0, 0.0},
268  {-0.0, -0.0}
269 };
270 static const TableEntry tbl_asinpi[] = {
271  {0.0, 0.0},
272  {-0.0, -0.0},
273  {0.5, 1.0},
274  {-0.5, -1.0}
275 };
276 static const TableEntry tbl_atan[] = {
277  {0.0, 0.0},
278  {-0.0, -0.0},
279  {MATH_PI/4.0, 1.0},
280  {-MATH_PI/4.0, -1.0}
281 };
282 static const TableEntry tbl_atanh[] = {
283  {0.0, 0.0},
284  {-0.0, -0.0}
285 };
286 static const TableEntry tbl_atanpi[] = {
287  {0.0, 0.0},
288  {-0.0, -0.0},
289  {0.25, 1.0},
290  {-0.25, -1.0}
291 };
292 static const TableEntry tbl_cbrt[] = {
293  {0.0, 0.0},
294  {-0.0, -0.0},
295  {1.0, 1.0},
296  {-1.0, -1.0},
297 };
298 static const TableEntry tbl_cos[] = {
299  {1.0, 0.0},
300  {1.0, -0.0}
301 };
302 static const TableEntry tbl_cosh[] = {
303  {1.0, 0.0},
304  {1.0, -0.0}
305 };
306 static const TableEntry tbl_cospi[] = {
307  {1.0, 0.0},
308  {1.0, -0.0}
309 };
310 static const TableEntry tbl_erfc[] = {
311  {1.0, 0.0},
312  {1.0, -0.0}
313 };
314 static const TableEntry tbl_erf[] = {
315  {0.0, 0.0},
316  {-0.0, -0.0}
317 };
318 static const TableEntry tbl_exp[] = {
319  {1.0, 0.0},
320  {1.0, -0.0},
321  {MATH_E, 1.0}
322 };
323 static const TableEntry tbl_exp2[] = {
324  {1.0, 0.0},
325  {1.0, -0.0},
326  {2.0, 1.0}
327 };
328 static const TableEntry tbl_exp10[] = {
329  {1.0, 0.0},
330  {1.0, -0.0},
331  {10.0, 1.0}
332 };
333 static const TableEntry tbl_expm1[] = {
334  {0.0, 0.0},
335  {-0.0, -0.0}
336 };
337 static const TableEntry tbl_log[] = {
338  {0.0, 1.0},
339  {1.0, MATH_E}
340 };
341 static const TableEntry tbl_log2[] = {
342  {0.0, 1.0},
343  {1.0, 2.0}
344 };
345 static const TableEntry tbl_log10[] = {
346  {0.0, 1.0},
347  {1.0, 10.0}
348 };
349 static const TableEntry tbl_rsqrt[] = {
350  {1.0, 1.0},
351  {1.0/MATH_SQRT2, 2.0}
352 };
353 static const TableEntry tbl_sin[] = {
354  {0.0, 0.0},
355  {-0.0, -0.0}
356 };
357 static const TableEntry tbl_sinh[] = {
358  {0.0, 0.0},
359  {-0.0, -0.0}
360 };
361 static const TableEntry tbl_sinpi[] = {
362  {0.0, 0.0},
363  {-0.0, -0.0}
364 };
365 static const TableEntry tbl_sqrt[] = {
366  {0.0, 0.0},
367  {1.0, 1.0},
368  {MATH_SQRT2, 2.0}
369 };
370 static const TableEntry tbl_tan[] = {
371  {0.0, 0.0},
372  {-0.0, -0.0}
373 };
374 static const TableEntry tbl_tanh[] = {
375  {0.0, 0.0},
376  {-0.0, -0.0}
377 };
378 static const TableEntry tbl_tanpi[] = {
379  {0.0, 0.0},
380  {-0.0, -0.0}
381 };
382 static const TableEntry tbl_tgamma[] = {
383  {1.0, 1.0},
384  {1.0, 2.0},
385  {2.0, 3.0},
386  {6.0, 4.0}
387 };
388 
390  switch(id) {
406  return true;
407  default:;
408  }
409  return false;
410 }
411 
412 struct TableRef {
413  size_t size;
414  const TableEntry *table; // variable size: from 0 to (size - 1)
415 
416  TableRef() : size(0), table(nullptr) {}
417 
418  template <size_t N>
419  TableRef(const TableEntry (&tbl)[N]) : size(N), table(&tbl[0]) {}
420 };
421 
423  switch(id) {
424  case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos);
425  case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh);
426  case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi);
427  case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin);
428  case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh);
429  case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi);
430  case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan);
431  case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh);
432  case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi);
433  case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt);
435  case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos);
436  case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh);
437  case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi);
438  case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc);
439  case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf);
440  case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp);
442  case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2);
443  case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10);
444  case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1);
445  case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log);
447  case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2);
448  case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10);
450  case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt);
452  case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin);
453  case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh);
454  case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi);
456  case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt);
457  case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan);
458  case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh);
459  case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi);
460  case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma);
461  default:;
462  }
463  return TableRef();
464 }
465 
466 static inline int getVecSize(const AMDGPULibFunc& FInfo) {
467  return FInfo.getLeads()[0].VectorSize;
468 }
469 
470 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
471  return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
472 }
473 
474 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
475  // If we are doing PreLinkOpt, the function is external. So it is safe to
476  // use getOrInsertFunction() at this stage.
477 
479  : AMDGPULibFunc::getFunction(M, fInfo);
480 }
481 
482 bool AMDGPULibCalls::parseFunctionName(const StringRef& FMangledName,
483  FuncInfo *FInfo) {
484  return AMDGPULibFunc::parse(FMangledName, *FInfo);
485 }
486 
488  if (auto Op = dyn_cast<FPMathOperator>(CI))
489  if (Op->isFast())
490  return true;
491  const Function *F = CI->getParent()->getParent();
492  Attribute Attr = F->getFnAttribute("unsafe-fp-math");
493  return Attr.getValueAsString() == "true";
494 }
495 
496 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
497  return AllNative ||
498  std::find(UseNative.begin(), UseNative.end(), F) != UseNative.end();
499 }
500 
502  AllNative = useNativeFunc("all") ||
503  (UseNative.getNumOccurrences() && UseNative.size() == 1 &&
504  UseNative.begin()->empty());
505 }
506 
507 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
508  bool native_sin = useNativeFunc("sin");
509  bool native_cos = useNativeFunc("cos");
510 
511  if (native_sin && native_cos) {
512  Module *M = aCI->getModule();
513  Value *opr0 = aCI->getArgOperand(0);
514 
515  AMDGPULibFunc nf;
516  nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
517  nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
518 
521  FunctionCallee sinExpr = getFunction(M, nf);
522 
525  FunctionCallee cosExpr = getFunction(M, nf);
526  if (sinExpr && cosExpr) {
527  Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI);
528  Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI);
529  new StoreInst(cosval, aCI->getArgOperand(1), aCI);
530 
531  DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
532  << " with native version of sin/cos");
533 
534  replaceCall(sinval);
535  return true;
536  }
537  }
538  return false;
539 }
540 
542  CI = aCI;
543  Function *Callee = aCI->getCalledFunction();
544 
545  FuncInfo FInfo;
546  if (!parseFunctionName(Callee->getName(), &FInfo) || !FInfo.isMangled() ||
547  FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
548  getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) ||
549  !(AllNative || useNativeFunc(FInfo.getName()))) {
550  return false;
551  }
552 
553  if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
554  return sincosUseNative(aCI, FInfo);
555 
557  FunctionCallee F = getFunction(aCI->getModule(), FInfo);
558  if (!F)
559  return false;
560 
561  aCI->setCalledFunction(F);
562  DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
563  << " with native version");
564  return true;
565 }
566 
567 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
568 // builtin, with appended type size and alignment arguments, where 2 or 4
569 // indicates the original number of arguments. The library has optimized version
570 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
571 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
572 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
573 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
574 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
575  FuncInfo &FInfo) {
576  auto *Callee = CI->getCalledFunction();
577  if (!Callee->isDeclaration())
578  return false;
579 
580  assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
581  auto *M = Callee->getParent();
582  auto &Ctx = M->getContext();
583  std::string Name = Callee->getName();
584  auto NumArg = CI->getNumArgOperands();
585  if (NumArg != 4 && NumArg != 6)
586  return false;
587  auto *PacketSize = CI->getArgOperand(NumArg - 2);
588  auto *PacketAlign = CI->getArgOperand(NumArg - 1);
589  if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign))
590  return false;
591  unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue();
592  unsigned Align = cast<ConstantInt>(PacketAlign)->getZExtValue();
593  if (Size != Align || !isPowerOf2_32(Size))
594  return false;
595 
596  Type *PtrElemTy;
597  if (Size <= 8)
598  PtrElemTy = Type::getIntNTy(Ctx, Size * 8);
599  else
600  PtrElemTy = VectorType::get(Type::getInt64Ty(Ctx), Size / 8);
601  unsigned PtrArgLoc = CI->getNumArgOperands() - 3;
602  auto PtrArg = CI->getArgOperand(PtrArgLoc);
603  unsigned PtrArgAS = PtrArg->getType()->getPointerAddressSpace();
604  auto *PtrTy = llvm::PointerType::get(PtrElemTy, PtrArgAS);
605 
607  for (unsigned I = 0; I != PtrArgLoc; ++I)
608  ArgTys.push_back(CI->getArgOperand(I)->getType());
609  ArgTys.push_back(PtrTy);
610 
611  Name = Name + "_" + std::to_string(Size);
612  auto *FTy = FunctionType::get(Callee->getReturnType(),
613  ArrayRef<Type *>(ArgTys), false);
614  AMDGPULibFunc NewLibFunc(Name, FTy);
616  if (!F)
617  return false;
618 
619  auto *BCast = B.CreatePointerCast(PtrArg, PtrTy);
621  for (unsigned I = 0; I != PtrArgLoc; ++I)
622  Args.push_back(CI->getArgOperand(I));
623  Args.push_back(BCast);
624 
625  auto *NCI = B.CreateCall(F, Args);
626  NCI->setAttributes(CI->getAttributes());
627  CI->replaceAllUsesWith(NCI);
628  CI->dropAllReferences();
629  CI->eraseFromParent();
630 
631  return true;
632 }
633 
634 // This function returns false if no change; return true otherwise.
636  this->CI = CI;
637  Function *Callee = CI->getCalledFunction();
638 
639  // Ignore indirect calls.
640  if (Callee == 0) return false;
641 
642  FuncInfo FInfo;
643  if (!parseFunctionName(Callee->getName(), &FInfo))
644  return false;
645 
646  // Further check the number of arguments to see if they match.
647  if (CI->getNumArgOperands() != FInfo.getNumArgs())
648  return false;
649 
650  BasicBlock *BB = CI->getParent();
652  IRBuilder<> B(Context);
653 
654  // Set the builder to the instruction after the call.
655  B.SetInsertPoint(BB, CI->getIterator());
656 
657  // Copy fast flags from the original call.
658  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI))
659  B.setFastMathFlags(FPOp->getFastMathFlags());
660 
661  if (TDOFold(CI, FInfo))
662  return true;
663 
664  // Under unsafe-math, evaluate calls if possible.
665  // According to Brian Sumner, we can do this for all f32 function calls
666  // using host's double function calls.
667  if (isUnsafeMath(CI) && evaluateCall(CI, FInfo))
668  return true;
669 
670  // Specilized optimizations for each function call
671  switch (FInfo.getId()) {
673  // skip vector function
674  assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
675  FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
676  "recip must be an either native or half function");
677  return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo);
678 
680  // skip vector function
681  assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
682  FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
683  "divide must be an either native or half function");
684  return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo);
685 
689  return fold_pow(CI, B, FInfo);
690 
692  // skip vector function
693  return (getVecSize(FInfo) != 1) ? false : fold_rootn(CI, B, FInfo);
694 
698  // skip vector function
699  return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo);
700 
702  return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo);
705  if ((getArgType(FInfo) == AMDGPULibFunc::F32 ||
706  getArgType(FInfo) == AMDGPULibFunc::F64)
707  && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX))
708  return fold_sincos(CI, B, AA);
709 
710  break;
715  return fold_read_write_pipe(CI, B, FInfo);
716 
717  default:
718  break;
719  }
720 
721  return false;
722 }
723 
724 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
725  // Table-Driven optimization
726  const TableRef tr = getOptTable(FInfo.getId());
727  if (tr.size==0)
728  return false;
729 
730  int const sz = (int)tr.size;
731  const TableEntry * const ftbl = tr.table;
732  Value *opr0 = CI->getArgOperand(0);
733 
734  if (getVecSize(FInfo) > 1) {
735  if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) {
737  for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) {
738  ConstantFP *eltval = dyn_cast<ConstantFP>(
739  CV->getElementAsConstant((unsigned)eltNo));
740  assert(eltval && "Non-FP arguments in math function!");
741  bool found = false;
742  for (int i=0; i < sz; ++i) {
743  if (eltval->isExactlyValue(ftbl[i].input)) {
744  DVal.push_back(ftbl[i].result);
745  found = true;
746  break;
747  }
748  }
749  if (!found) {
750  // This vector constants not handled yet.
751  return false;
752  }
753  }
754  LLVMContext &context = CI->getParent()->getParent()->getContext();
755  Constant *nval;
756  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
758  for (unsigned i = 0; i < DVal.size(); ++i) {
759  FVal.push_back((float)DVal[i]);
760  }
761  ArrayRef<float> tmp(FVal);
762  nval = ConstantDataVector::get(context, tmp);
763  } else { // F64
764  ArrayRef<double> tmp(DVal);
765  nval = ConstantDataVector::get(context, tmp);
766  }
767  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
768  replaceCall(nval);
769  return true;
770  }
771  } else {
772  // Scalar version
773  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
774  for (int i = 0; i < sz; ++i) {
775  if (CF->isExactlyValue(ftbl[i].input)) {
776  Value *nval = ConstantFP::get(CF->getType(), ftbl[i].result);
777  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
778  replaceCall(nval);
779  return true;
780  }
781  }
782  }
783  }
784 
785  return false;
786 }
787 
788 bool AMDGPULibCalls::replaceWithNative(CallInst *CI, const FuncInfo &FInfo) {
789  Module *M = CI->getModule();
790  if (getArgType(FInfo) != AMDGPULibFunc::F32 ||
791  FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
792  !HasNative(FInfo.getId()))
793  return false;
794 
795  AMDGPULibFunc nf = FInfo;
797  if (FunctionCallee FPExpr = getFunction(M, nf)) {
798  LLVM_DEBUG(dbgs() << "AMDIC: " << *CI << " ---> ");
799 
800  CI->setCalledFunction(FPExpr);
801 
802  LLVM_DEBUG(dbgs() << *CI << '\n');
803 
804  return true;
805  }
806  return false;
807 }
808 
809 // [native_]half_recip(c) ==> 1.0/c
810 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B,
811  const FuncInfo &FInfo) {
812  Value *opr0 = CI->getArgOperand(0);
813  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
814  // Just create a normal div. Later, InstCombine will be able
815  // to compute the divide into a constant (avoid check float infinity
816  // or subnormal at this point).
817  Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0),
818  opr0,
819  "recip2div");
820  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
821  replaceCall(nval);
822  return true;
823  }
824  return false;
825 }
826 
827 // [native_]half_divide(x, c) ==> x/c
828 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B,
829  const FuncInfo &FInfo) {
830  Value *opr0 = CI->getArgOperand(0);
831  Value *opr1 = CI->getArgOperand(1);
832  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
833  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
834 
835  if ((CF0 && CF1) || // both are constants
836  (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32)))
837  // CF1 is constant && f32 divide
838  {
839  Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0),
840  opr1, "__div2recip");
841  Value *nval = B.CreateFMul(opr0, nval1, "__div2mul");
842  replaceCall(nval);
843  return true;
844  }
845  return false;
846 }
847 
848 namespace llvm {
849 static double log2(double V) {
850 #if _XOPEN_SOURCE >= 600 || _ISOC99_SOURCE || _POSIX_C_SOURCE >= 200112L
851  return ::log2(V);
852 #else
853  return log(V) / 0.693147180559945309417;
854 #endif
855 }
856 }
857 
858 bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
859  const FuncInfo &FInfo) {
860  assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
861  FInfo.getId() == AMDGPULibFunc::EI_POWR ||
862  FInfo.getId() == AMDGPULibFunc::EI_POWN) &&
863  "fold_pow: encounter a wrong function call");
864 
865  Value *opr0, *opr1;
866  ConstantFP *CF;
867  ConstantInt *CINT;
868  ConstantAggregateZero *CZero;
869  Type *eltType;
870 
871  opr0 = CI->getArgOperand(0);
872  opr1 = CI->getArgOperand(1);
873  CZero = dyn_cast<ConstantAggregateZero>(opr1);
874  if (getVecSize(FInfo) == 1) {
875  eltType = opr0->getType();
876  CF = dyn_cast<ConstantFP>(opr1);
877  CINT = dyn_cast<ConstantInt>(opr1);
878  } else {
879  VectorType *VTy = dyn_cast<VectorType>(opr0->getType());
880  assert(VTy && "Oprand of vector function should be of vectortype");
881  eltType = VTy->getElementType();
883 
884  // Now, only Handle vector const whose elements have the same value.
885  CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr;
886  CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr;
887  }
888 
889  // No unsafe math , no constant argument, do nothing
890  if (!isUnsafeMath(CI) && !CF && !CINT && !CZero)
891  return false;
892 
893  // 0x1111111 means that we don't do anything for this call.
894  int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
895 
896  if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) {
897  // pow/powr/pown(x, 0) == 1
898  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n");
899  Constant *cnval = ConstantFP::get(eltType, 1.0);
900  if (getVecSize(FInfo) > 1) {
901  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
902  }
903  replaceCall(cnval);
904  return true;
905  }
906  if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) {
907  // pow/powr/pown(x, 1.0) = x
908  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
909  replaceCall(opr0);
910  return true;
911  }
912  if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) {
913  // pow/powr/pown(x, 2.0) = x*x
914  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " << *opr0
915  << "\n");
916  Value *nval = B.CreateFMul(opr0, opr0, "__pow2");
917  replaceCall(nval);
918  return true;
919  }
920  if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) {
921  // pow/powr/pown(x, -1.0) = 1.0/x
922  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1 / " << *opr0 << "\n");
923  Constant *cnval = ConstantFP::get(eltType, 1.0);
924  if (getVecSize(FInfo) > 1) {
925  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
926  }
927  Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip");
928  replaceCall(nval);
929  return true;
930  }
931 
932  Module *M = CI->getModule();
933  if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) {
934  // pow[r](x, [-]0.5) = sqrt(x)
935  bool issqrt = CF->isExactlyValue(0.5);
936  if (FunctionCallee FPExpr =
937  getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
939  FInfo))) {
940  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
941  << FInfo.getName().c_str() << "(" << *opr0 << ")\n");
942  Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt"
943  : "__pow2rsqrt");
944  replaceCall(nval);
945  return true;
946  }
947  }
948 
949  if (!isUnsafeMath(CI))
950  return false;
951 
952  // Unsafe Math optimization
953 
954  // Remember that ci_opr1 is set if opr1 is integral
955  if (CF) {
956  double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
957  ? (double)CF->getValueAPF().convertToFloat()
958  : CF->getValueAPF().convertToDouble();
959  int ival = (int)dval;
960  if ((double)ival == dval) {
961  ci_opr1 = ival;
962  } else
963  ci_opr1 = 0x11111111;
964  }
965 
966  // pow/powr/pown(x, c) = [1/](x*x*..x); where
967  // trunc(c) == c && the number of x == c && |c| <= 12
968  unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;
969  if (abs_opr1 <= 12) {
970  Constant *cnval;
971  Value *nval;
972  if (abs_opr1 == 0) {
973  cnval = ConstantFP::get(eltType, 1.0);
974  if (getVecSize(FInfo) > 1) {
975  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
976  }
977  nval = cnval;
978  } else {
979  Value *valx2 = nullptr;
980  nval = nullptr;
981  while (abs_opr1 > 0) {
982  valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0;
983  if (abs_opr1 & 1) {
984  nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2;
985  }
986  abs_opr1 >>= 1;
987  }
988  }
989 
990  if (ci_opr1 < 0) {
991  cnval = ConstantFP::get(eltType, 1.0);
992  if (getVecSize(FInfo) > 1) {
993  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
994  }
995  nval = B.CreateFDiv(cnval, nval, "__1powprod");
996  }
997  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
998  << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
999  << ")\n");
1000  replaceCall(nval);
1001  return true;
1002  }
1003 
1004  // powr ---> exp2(y * log2(x))
1005  // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
1006  FunctionCallee ExpExpr =
1007  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
1008  if (!ExpExpr)
1009  return false;
1010 
1011  bool needlog = false;
1012  bool needabs = false;
1013  bool needcopysign = false;
1014  Constant *cnval = nullptr;
1015  if (getVecSize(FInfo) == 1) {
1016  CF = dyn_cast<ConstantFP>(opr0);
1017 
1018  if (CF) {
1019  double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1020  ? (double)CF->getValueAPF().convertToFloat()
1021  : CF->getValueAPF().convertToDouble();
1022 
1023  V = log2(std::abs(V));
1024  cnval = ConstantFP::get(eltType, V);
1025  needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) &&
1026  CF->isNegative();
1027  } else {
1028  needlog = true;
1029  needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1030  (!CF || CF->isNegative());
1031  }
1032  } else {
1034 
1035  if (!CDV) {
1036  needlog = true;
1037  needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR;
1038  } else {
1039  assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1040  "Wrong vector size detected");
1041 
1043  for (int i=0; i < getVecSize(FInfo); ++i) {
1044  double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1045  ? (double)CDV->getElementAsFloat(i)
1046  : CDV->getElementAsDouble(i);
1047  if (V < 0.0) needcopysign = true;
1048  V = log2(std::abs(V));
1049  DVal.push_back(V);
1050  }
1051  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1052  SmallVector<float, 0> FVal;
1053  for (unsigned i=0; i < DVal.size(); ++i) {
1054  FVal.push_back((float)DVal[i]);
1055  }
1056  ArrayRef<float> tmp(FVal);
1057  cnval = ConstantDataVector::get(M->getContext(), tmp);
1058  } else {
1059  ArrayRef<double> tmp(DVal);
1060  cnval = ConstantDataVector::get(M->getContext(), tmp);
1061  }
1062  }
1063  }
1064 
1065  if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) {
1066  // We cannot handle corner cases for a general pow() function, give up
1067  // unless y is a constant integral value. Then proceed as if it were pown.
1068  if (getVecSize(FInfo) == 1) {
1069  if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) {
1070  double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1071  ? (double)CF->getValueAPF().convertToFloat()
1072  : CF->getValueAPF().convertToDouble();
1073  if (y != (double)(int64_t)y)
1074  return false;
1075  } else
1076  return false;
1077  } else {
1078  if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) {
1079  for (int i=0; i < getVecSize(FInfo); ++i) {
1080  double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1081  ? (double)CDV->getElementAsFloat(i)
1082  : CDV->getElementAsDouble(i);
1083  if (y != (double)(int64_t)y)
1084  return false;
1085  }
1086  } else
1087  return false;
1088  }
1089  }
1090 
1091  Value *nval;
1092  if (needabs) {
1093  FunctionCallee AbsExpr =
1094  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo));
1095  if (!AbsExpr)
1096  return false;
1097  nval = CreateCallEx(B, AbsExpr, opr0, "__fabs");
1098  } else {
1099  nval = cnval ? cnval : opr0;
1100  }
1101  if (needlog) {
1102  FunctionCallee LogExpr =
1103  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1104  if (!LogExpr)
1105  return false;
1106  nval = CreateCallEx(B,LogExpr, nval, "__log2");
1107  }
1108 
1109  if (FInfo.getId() == AMDGPULibFunc::EI_POWN) {
1110  // convert int(32) to fp(f32 or f64)
1111  opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F");
1112  }
1113  nval = B.CreateFMul(opr1, nval, "__ylogx");
1114  nval = CreateCallEx(B,ExpExpr, nval, "__exp2");
1115 
1116  if (needcopysign) {
1117  Value *opr_n;
1118  Type* rTy = opr0->getType();
1119  Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty();
1120  Type *nTy = nTyS;
1121  if (const VectorType *vTy = dyn_cast<VectorType>(rTy))
1122  nTy = VectorType::get(nTyS, vTy->getNumElements());
1123  unsigned size = nTy->getScalarSizeInBits();
1124  opr_n = CI->getArgOperand(1);
1125  if (opr_n->getType()->isIntegerTy())
1126  opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou");
1127  else
1128  opr_n = B.CreateFPToSI(opr1, nTy, "__ytou");
1129 
1130  Value *sign = B.CreateShl(opr_n, size-1, "__yeven");
1131  sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign");
1132  nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign);
1133  nval = B.CreateBitCast(nval, opr0->getType());
1134  }
1135 
1136  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1137  << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1138  replaceCall(nval);
1139 
1140  return true;
1141 }
1142 
1143 bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
1144  const FuncInfo &FInfo) {
1145  Value *opr0 = CI->getArgOperand(0);
1146  Value *opr1 = CI->getArgOperand(1);
1147 
1148  ConstantInt *CINT = dyn_cast<ConstantInt>(opr1);
1149  if (!CINT) {
1150  return false;
1151  }
1152  int ci_opr1 = (int)CINT->getSExtValue();
1153  if (ci_opr1 == 1) { // rootn(x, 1) = x
1154  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
1155  replaceCall(opr0);
1156  return true;
1157  }
1158  if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x)
1159  std::vector<const Type*> ParamsTys;
1160  ParamsTys.push_back(opr0->getType());
1161  Module *M = CI->getModule();
1162  if (FunctionCallee FPExpr =
1163  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
1164  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n");
1165  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt");
1166  replaceCall(nval);
1167  return true;
1168  }
1169  } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1170  Module *M = CI->getModule();
1171  if (FunctionCallee FPExpr =
1172  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1173  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n");
1174  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt");
1175  replaceCall(nval);
1176  return true;
1177  }
1178  } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1179  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1.0 / " << *opr0 << "\n");
1180  Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0),
1181  opr0,
1182  "__rootn2div");
1183  replaceCall(nval);
1184  return true;
1185  } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x)
1186  std::vector<const Type*> ParamsTys;
1187  ParamsTys.push_back(opr0->getType());
1188  Module *M = CI->getModule();
1189  if (FunctionCallee FPExpr =
1190  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) {
1191  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0
1192  << ")\n");
1193  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt");
1194  replaceCall(nval);
1195  return true;
1196  }
1197  }
1198  return false;
1199 }
1200 
1201 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B,
1202  const FuncInfo &FInfo) {
1203  Value *opr0 = CI->getArgOperand(0);
1204  Value *opr1 = CI->getArgOperand(1);
1205  Value *opr2 = CI->getArgOperand(2);
1206 
1207  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
1208  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
1209  if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) {
1210  // fma/mad(a, b, c) = c if a=0 || b=0
1211  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n");
1212  replaceCall(opr2);
1213  return true;
1214  }
1215  if (CF0 && CF0->isExactlyValue(1.0f)) {
1216  // fma/mad(a, b, c) = b+c if a=1
1217  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2
1218  << "\n");
1219  Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd");
1220  replaceCall(nval);
1221  return true;
1222  }
1223  if (CF1 && CF1->isExactlyValue(1.0f)) {
1224  // fma/mad(a, b, c) = a+c if b=1
1225  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2
1226  << "\n");
1227  Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd");
1228  replaceCall(nval);
1229  return true;
1230  }
1231  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) {
1232  if (CF->isZero()) {
1233  // fma/mad(a, b, c) = a*b if c=0
1234  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * "
1235  << *opr1 << "\n");
1236  Value *nval = B.CreateFMul(opr0, opr1, "fmamul");
1237  replaceCall(nval);
1238  return true;
1239  }
1240  }
1241 
1242  return false;
1243 }
1244 
1245 // Get a scalar native builtin signle argument FP function
1246 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
1247  const FuncInfo &FInfo) {
1248  if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))
1249  return nullptr;
1250  FuncInfo nf = FInfo;
1251  nf.setPrefix(AMDGPULibFunc::NATIVE);
1252  return getFunction(M, nf);
1253 }
1254 
1255 // fold sqrt -> native_sqrt (x)
1256 bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B,
1257  const FuncInfo &FInfo) {
1258  if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) &&
1259  (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) {
1260  if (FunctionCallee FPExpr = getNativeFunction(
1262  Value *opr0 = CI->getArgOperand(0);
1263  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1264  << "sqrt(" << *opr0 << ")\n");
1265  Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt");
1266  replaceCall(nval);
1267  return true;
1268  }
1269  }
1270  return false;
1271 }
1272 
1273 // fold sin, cos -> sincos.
1274 bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
1275  AliasAnalysis *AA) {
1276  AMDGPULibFunc fInfo;
1277  if (!AMDGPULibFunc::parse(CI->getCalledFunction()->getName(), fInfo))
1278  return false;
1279 
1280  assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1281  fInfo.getId() == AMDGPULibFunc::EI_COS);
1282  bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1283 
1284  Value *CArgVal = CI->getArgOperand(0);
1285  BasicBlock * const CBB = CI->getParent();
1286 
1287  int const MaxScan = 30;
1288 
1289  { // fold in load value.
1290  LoadInst *LI = dyn_cast<LoadInst>(CArgVal);
1291  if (LI && LI->getParent() == CBB) {
1292  BasicBlock::iterator BBI = LI->getIterator();
1293  Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA);
1294  if (AvailableVal) {
1295  CArgVal->replaceAllUsesWith(AvailableVal);
1296  if (CArgVal->getNumUses() == 0)
1297  LI->eraseFromParent();
1298  CArgVal = CI->getArgOperand(0);
1299  }
1300  }
1301  }
1302 
1303  Module *M = CI->getModule();
1305  std::string const PairName = fInfo.mangle();
1306 
1307  CallInst *UI = nullptr;
1308  for (User* U : CArgVal->users()) {
1309  CallInst *XI = dyn_cast_or_null<CallInst>(U);
1310  if (!XI || XI == CI || XI->getParent() != CBB)
1311  continue;
1312 
1313  Function *UCallee = XI->getCalledFunction();
1314  if (!UCallee || !UCallee->getName().equals(PairName))
1315  continue;
1316 
1317  BasicBlock::iterator BBI = CI->getIterator();
1318  if (BBI == CI->getParent()->begin())
1319  break;
1320  --BBI;
1321  for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) {
1322  if (cast<Instruction>(BBI) == XI) {
1323  UI = XI;
1324  break;
1325  }
1326  }
1327  if (UI) break;
1328  }
1329 
1330  if (!UI) return false;
1331 
1332  // Merge the sin and cos.
1333 
1334  // for OpenCL 2.0 we have only generic implementation of sincos
1335  // function.
1338  FunctionCallee Fsincos = getFunction(M, nf);
1339  if (!Fsincos) return false;
1340 
1342  AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_");
1343  B.SetInsertPoint(UI);
1344 
1345  Value *P = Alloc;
1346  Type *PTy = Fsincos.getFunctionType()->getParamType(1);
1347  // The allocaInst allocates the memory in private address space. This need
1348  // to be bitcasted to point to the address space of cos pointer type.
1349  // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1351  P = B.CreateAddrSpaceCast(Alloc, PTy);
1352  CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P);
1353 
1354  LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with "
1355  << *Call << "\n");
1356 
1357  if (!isSin) { // CI->cos, UI->sin
1358  B.SetInsertPoint(&*ItOld);
1359  UI->replaceAllUsesWith(&*Call);
1360  Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1361  CI->replaceAllUsesWith(Reload);
1362  UI->eraseFromParent();
1363  CI->eraseFromParent();
1364  } else { // CI->sin, UI->cos
1365  Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1366  UI->replaceAllUsesWith(Reload);
1367  CI->replaceAllUsesWith(Call);
1368  UI->eraseFromParent();
1369  CI->eraseFromParent();
1370  }
1371  return true;
1372 }
1373 
1374 // Get insertion point at entry.
1375 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) {
1376  Function * Func = UI->getParent()->getParent();
1377  BasicBlock * BB = &Func->getEntryBlock();
1378  assert(BB && "Entry block not found!");
1379  BasicBlock::iterator ItNew = BB->begin();
1380  return ItNew;
1381 }
1382 
1383 // Insert a AllocsInst at the beginning of function entry block.
1384 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B,
1385  const char *prefix) {
1386  BasicBlock::iterator ItNew = getEntryIns(UI);
1387  Function *UCallee = UI->getCalledFunction();
1388  Type *RetType = UCallee->getReturnType();
1389  B.SetInsertPoint(&*ItNew);
1390  AllocaInst *Alloc = B.CreateAlloca(RetType, 0,
1391  std::string(prefix) + UI->getName());
1392  Alloc->setAlignment(UCallee->getParent()->getDataLayout()
1393  .getTypeAllocSize(RetType));
1394  return Alloc;
1395 }
1396 
1397 bool AMDGPULibCalls::evaluateScalarMathFunc(FuncInfo &FInfo,
1398  double& Res0, double& Res1,
1399  Constant *copr0, Constant *copr1,
1400  Constant *copr2) {
1401  // By default, opr0/opr1/opr3 holds values of float/double type.
1402  // If they are not float/double, each function has to its
1403  // operand separately.
1404  double opr0=0.0, opr1=0.0, opr2=0.0;
1405  ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0);
1406  ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1);
1407  ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2);
1408  if (fpopr0) {
1409  opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1410  ? fpopr0->getValueAPF().convertToDouble()
1411  : (double)fpopr0->getValueAPF().convertToFloat();
1412  }
1413 
1414  if (fpopr1) {
1415  opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1416  ? fpopr1->getValueAPF().convertToDouble()
1417  : (double)fpopr1->getValueAPF().convertToFloat();
1418  }
1419 
1420  if (fpopr2) {
1421  opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1422  ? fpopr2->getValueAPF().convertToDouble()
1423  : (double)fpopr2->getValueAPF().convertToFloat();
1424  }
1425 
1426  switch (FInfo.getId()) {
1427  default : return false;
1428 
1430  Res0 = acos(opr0);
1431  return true;
1432 
1434  // acosh(x) == log(x + sqrt(x*x - 1))
1435  Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0));
1436  return true;
1437 
1439  Res0 = acos(opr0) / MATH_PI;
1440  return true;
1441 
1443  Res0 = asin(opr0);
1444  return true;
1445 
1447  // asinh(x) == log(x + sqrt(x*x + 1))
1448  Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0));
1449  return true;
1450 
1452  Res0 = asin(opr0) / MATH_PI;
1453  return true;
1454 
1456  Res0 = atan(opr0);
1457  return true;
1458 
1460  // atanh(x) == (log(x+1) - log(x-1))/2;
1461  Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0;
1462  return true;
1463 
1465  Res0 = atan(opr0) / MATH_PI;
1466  return true;
1467 
1469  Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0);
1470  return true;
1471 
1472  case AMDGPULibFunc::EI_COS:
1473  Res0 = cos(opr0);
1474  return true;
1475 
1477  Res0 = cosh(opr0);
1478  return true;
1479 
1481  Res0 = cos(MATH_PI * opr0);
1482  return true;
1483 
1484  case AMDGPULibFunc::EI_EXP:
1485  Res0 = exp(opr0);
1486  return true;
1487 
1489  Res0 = pow(2.0, opr0);
1490  return true;
1491 
1493  Res0 = pow(10.0, opr0);
1494  return true;
1495 
1497  Res0 = exp(opr0) - 1.0;
1498  return true;
1499 
1500  case AMDGPULibFunc::EI_LOG:
1501  Res0 = log(opr0);
1502  return true;
1503 
1505  Res0 = log(opr0) / log(2.0);
1506  return true;
1507 
1509  Res0 = log(opr0) / log(10.0);
1510  return true;
1511 
1513  Res0 = 1.0 / sqrt(opr0);
1514  return true;
1515 
1516  case AMDGPULibFunc::EI_SIN:
1517  Res0 = sin(opr0);
1518  return true;
1519 
1521  Res0 = sinh(opr0);
1522  return true;
1523 
1525  Res0 = sin(MATH_PI * opr0);
1526  return true;
1527 
1529  Res0 = sqrt(opr0);
1530  return true;
1531 
1532  case AMDGPULibFunc::EI_TAN:
1533  Res0 = tan(opr0);
1534  return true;
1535 
1537  Res0 = tanh(opr0);
1538  return true;
1539 
1541  Res0 = tan(MATH_PI * opr0);
1542  return true;
1543 
1545  Res0 = 1.0 / opr0;
1546  return true;
1547 
1548  // two-arg functions
1550  Res0 = opr0 / opr1;
1551  return true;
1552 
1553  case AMDGPULibFunc::EI_POW:
1555  Res0 = pow(opr0, opr1);
1556  return true;
1557 
1558  case AMDGPULibFunc::EI_POWN: {
1559  if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1560  double val = (double)iopr1->getSExtValue();
1561  Res0 = pow(opr0, val);
1562  return true;
1563  }
1564  return false;
1565  }
1566 
1567  case AMDGPULibFunc::EI_ROOTN: {
1568  if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1569  double val = (double)iopr1->getSExtValue();
1570  Res0 = pow(opr0, 1.0 / val);
1571  return true;
1572  }
1573  return false;
1574  }
1575 
1576  // with ptr arg
1578  Res0 = sin(opr0);
1579  Res1 = cos(opr0);
1580  return true;
1581 
1582  // three-arg functions
1583  case AMDGPULibFunc::EI_FMA:
1584  case AMDGPULibFunc::EI_MAD:
1585  Res0 = opr0 * opr1 + opr2;
1586  return true;
1587  }
1588 
1589  return false;
1590 }
1591 
1592 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, FuncInfo &FInfo) {
1593  int numArgs = (int)aCI->getNumArgOperands();
1594  if (numArgs > 3)
1595  return false;
1596 
1597  Constant *copr0 = nullptr;
1598  Constant *copr1 = nullptr;
1599  Constant *copr2 = nullptr;
1600  if (numArgs > 0) {
1601  if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr)
1602  return false;
1603  }
1604 
1605  if (numArgs > 1) {
1606  if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) {
1607  if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
1608  return false;
1609  }
1610  }
1611 
1612  if (numArgs > 2) {
1613  if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr)
1614  return false;
1615  }
1616 
1617  // At this point, all arguments to aCI are constants.
1618 
1619  // max vector size is 16, and sincos will generate two results.
1620  double DVal0[16], DVal1[16];
1621  bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS);
1622  if (getVecSize(FInfo) == 1) {
1623  if (!evaluateScalarMathFunc(FInfo, DVal0[0],
1624  DVal1[0], copr0, copr1, copr2)) {
1625  return false;
1626  }
1627  } else {
1628  ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0);
1629  ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1);
1630  ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2);
1631  for (int i=0; i < getVecSize(FInfo); ++i) {
1632  Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr;
1633  Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr;
1634  Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr;
1635  if (!evaluateScalarMathFunc(FInfo, DVal0[i],
1636  DVal1[i], celt0, celt1, celt2)) {
1637  return false;
1638  }
1639  }
1640  }
1641 
1642  LLVMContext &context = CI->getParent()->getParent()->getContext();
1643  Constant *nval0, *nval1;
1644  if (getVecSize(FInfo) == 1) {
1645  nval0 = ConstantFP::get(CI->getType(), DVal0[0]);
1646  if (hasTwoResults)
1647  nval1 = ConstantFP::get(CI->getType(), DVal1[0]);
1648  } else {
1649  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1650  SmallVector <float, 0> FVal0, FVal1;
1651  for (int i=0; i < getVecSize(FInfo); ++i)
1652  FVal0.push_back((float)DVal0[i]);
1653  ArrayRef<float> tmp0(FVal0);
1654  nval0 = ConstantDataVector::get(context, tmp0);
1655  if (hasTwoResults) {
1656  for (int i=0; i < getVecSize(FInfo); ++i)
1657  FVal1.push_back((float)DVal1[i]);
1658  ArrayRef<float> tmp1(FVal1);
1659  nval1 = ConstantDataVector::get(context, tmp1);
1660  }
1661  } else {
1662  ArrayRef<double> tmp0(DVal0);
1663  nval0 = ConstantDataVector::get(context, tmp0);
1664  if (hasTwoResults) {
1665  ArrayRef<double> tmp1(DVal1);
1666  nval1 = ConstantDataVector::get(context, tmp1);
1667  }
1668  }
1669  }
1670 
1671  if (hasTwoResults) {
1672  // sincos
1673  assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS &&
1674  "math function with ptr arg not supported yet");
1675  new StoreInst(nval1, aCI->getArgOperand(1), aCI);
1676  }
1677 
1678  replaceCall(nval0);
1679  return true;
1680 }
1681 
1682 // Public interface to the Simplify LibCalls pass.
1684  return new AMDGPUSimplifyLibCalls(Opt);
1685 }
1686 
1688  return new AMDGPUUseNativeCalls();
1689 }
1690 
1691 static bool setFastFlags(Function &F, const TargetOptions &Options) {
1692  AttrBuilder B;
1693 
1694  if (Options.UnsafeFPMath || Options.NoInfsFPMath)
1695  B.addAttribute("no-infs-fp-math", "true");
1696  if (Options.UnsafeFPMath || Options.NoNaNsFPMath)
1697  B.addAttribute("no-nans-fp-math", "true");
1698  if (Options.UnsafeFPMath) {
1699  B.addAttribute("less-precise-fpmad", "true");
1700  B.addAttribute("unsafe-fp-math", "true");
1701  }
1702 
1703  if (!B.hasAttributes())
1704  return false;
1705 
1707 
1708  return true;
1709 }
1710 
1712  if (skipFunction(F))
1713  return false;
1714 
1715  bool Changed = false;
1716  auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1717 
1718  LLVM_DEBUG(dbgs() << "AMDIC: process function ";
1719  F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
1720 
1721  if (!EnablePreLink)
1722  Changed |= setFastFlags(F, Options);
1723 
1724  for (auto &BB : F) {
1725  for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1726  // Ignore non-calls.
1727  CallInst *CI = dyn_cast<CallInst>(I);
1728  ++I;
1729  if (!CI) continue;
1730 
1731  // Ignore indirect calls.
1732  Function *Callee = CI->getCalledFunction();
1733  if (Callee == 0) continue;
1734 
1735  LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n";
1736  dbgs().flush());
1737  if(Simplifier.fold(CI, AA))
1738  Changed = true;
1739  }
1740  }
1741  return Changed;
1742 }
1743 
1745  if (skipFunction(F) || UseNative.empty())
1746  return false;
1747 
1748  bool Changed = false;
1749  for (auto &BB : F) {
1750  for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1751  // Ignore non-calls.
1752  CallInst *CI = dyn_cast<CallInst>(I);
1753  ++I;
1754  if (!CI) continue;
1755 
1756  // Ignore indirect calls.
1757  Function *Callee = CI->getCalledFunction();
1758  if (Callee == 0) continue;
1759 
1760  if(Simplifier.useNative(CI))
1761  Changed = true;
1762  }
1763  }
1764  return Changed;
1765 }
A vector constant whose element type is a simple 1/2/4/8-byte integer or float/double, and whose elements are just simple data values (i.e.
Definition: Constants.h:761
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
static const TableEntry tbl_sqrt[]
static const TableEntry tbl_log10[]
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & Context
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Definition: IRBuilder.h:1379
void dropAllReferences()
Drop all references to operands.
Definition: User.h:294
EFuncId getId() const
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1772
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:134
Constant * getElementAsConstant(unsigned i) const
Return a Constant for a specified index&#39;s element.
Definition: Constants.cpp:2751
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ty, Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool...
Definition: IRBuilder.h:1392
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:65
void setAlignment(unsigned Align)
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
static const TableEntry tbl_cospi[]
amdgpu simplifylib
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:164
static const TableEntry tbl_cbrt[]
static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr)
void initializeAMDGPUSimplifyLibCallsPass(PassRegistry &)
This class represents a function call, abstracting a target machine&#39;s calling convention.
Address space for private memory.
Definition: AMDGPU.h:259
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space...
Definition: Type.cpp:629
bool isUnsafeMath(const CallInst *CI) const
float convertToFloat() const
Definition: APFloat.h:1097
F(f)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:534
An instruction for reading from memory.
Definition: Instructions.h:167
static const TableEntry tbl_erf[]
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:176
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
void setPrefix(ENamePrefix PFX)
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:32
static const TableEntry tbl_acosh[]
static FunctionCallee getOrInsertFunction(llvm::Module *M, const AMDGPULibFunc &fInfo)
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:268
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:64
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:346
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1218
static const TableEntry tbl_atanh[]
std::string getName() const
Get unmangled name for mangled library function and name for unmangled library function.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
bool hasAttributes() const
Return true if the builder has IR-level attributes.
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:351
Value * FindAvailableLoadedValue(LoadInst *Load, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan=DefMaxInstsToScan, AliasAnalysis *AA=nullptr, bool *IsLoadCSE=nullptr, unsigned *NumScanedInst=nullptr)
Scan backwards to see if we have the value of the given load available locally within a small number ...
Definition: Loads.cpp:328
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:244
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:196
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:742
Param * getLeads()
Get leading parameters for mangled lib functions.
static const TableEntry tbl_sinh[]
unsigned getNumArgs() const
INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", "Simplify well-known AMD library calls", false, false) INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls
All zero aggregate value.
Definition: Constants.h:340
void setId(EFuncId Id)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1767
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
amdgpu Simplify well known AMD library false
static const TableEntry tbl_sinpi[]
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition: InstrTypes.h:1300
bool isNegative() const
Return true if the sign bit is set.
Definition: Constants.h:308
static const TableEntry tbl_asinh[]
static const TableEntry tbl_tanpi[]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1744
An instruction for storing to memory.
Definition: Instructions.h:320
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1777
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:126
static const TableEntry tbl_log2[]
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1217
static AMDGPULibFunc::EType getArgType(const AMDGPULibFunc &FInfo)
const BasicBlock & getEntryBlock() const
Definition: Function.h:645
static bool runOnFunction(Function &F, bool PostInlining)
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
TableRef(const TableEntry(&tbl)[N])
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
bool fold(CallInst *CI, AliasAnalysis *AA=nullptr)
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1282
static const TableEntry tbl_exp2[]
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
static Function * getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo)
Constant * getSplatValue() const
If this is a splat constant, meaning that all of the elements have the same value, return that value.
Definition: Constants.cpp:2788
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
static cl::opt< bool > EnablePreLink("amdgpu-prelink", cl::desc("Enable pre-link mode optimizations"), cl::init(false), cl::Hidden)
double convertToDouble() const
Definition: APFloat.h:1096
float getElementAsFloat(unsigned i) const
If this is an sequential container of floats, return the specified element as a float.
Definition: Constants.cpp:2739
static const TableEntry tbl_atan[]
static const TableEntry tbl_sin[]
bool isMangled() const
Represent the analysis usage information of a pass.
static const TableEntry tbl_rsqrt[]
static const TableEntry tbl_acospi[]
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1325
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:296
Wrapper class for AMDGPULIbFuncImpl.
static const TableEntry tbl_expm1[]
self_iterator getIterator()
Definition: ilist_node.h:81
static int getVecSize(const AMDGPULibFunc &FInfo)
static Constant * getSplat(unsigned NumElts, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
Definition: Constants.cpp:2628
unsigned NoInfsFPMath
NoInfsFPMath - This flag is enabled when the -enable-no-infs-fp-math flag is specified on the command...
static double log2(double V)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:196
static unsigned getEPtrKindFromAddrSpace(unsigned AS)
size_t size() const
Definition: SmallVector.h:52
void initializeAMDGPUUseNativeCallsPass(PassRegistry &)
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1206
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4289
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
static const TableEntry tbl_atanpi[]
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
const APFloat & getValueAPF() const
Definition: Constants.h:302
static const TableEntry tbl_exp10[]
Iterator for intrusive lists based on ilist_node.
TargetPassConfig.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
static const TableEntry tbl_asinpi[]
static bool setFastFlags(Function &F, const TargetOptions &Options)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
Definition: Type.cpp:129
Address space for flat memory.
Definition: AMDGPU.h:253
ENamePrefix getPrefix() const
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:245
Module.h This file contains the declarations for the Module class.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:179
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:694
const TableEntry * table
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1343
static const TableEntry tbl_log[]
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Class to represent vector types.
Definition: DerivedTypes.h:424
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
static const TableEntry tbl_cosh[]
amdgpu Simplify well known AMD library false FunctionCallee Callee
iterator_range< user_iterator > users()
Definition: Value.h:399
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:89
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1138
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1813
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:160
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:469
unsigned getNumUses() const
This method computes the number of uses of this Value.
Definition: Value.cpp:160
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1216
static const TableEntry tbl_exp[]
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:194
static const TableEntry tbl_tanh[]
bool isZero() const
Return true if the value is positive or negative zero.
Definition: Constants.h:305
double getElementAsDouble(unsigned i) const
If this is an sequential container of doubles, return the specified element as a double.
Definition: Constants.cpp:2745
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1264
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
static TableRef getOptTable(AMDGPULibFunc::EFuncId id)
#define N
#define MATH_PI
FunctionPass * createAMDGPUSimplifyLibCallsPass(const TargetOptions &)
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1212
static const TableEntry tbl_asin[]
Value * CreateFPToSI(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1736
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1347
FunctionType * getFunctionType()
Definition: DerivedTypes.h:181
static cl::list< std::string > UseNative("amdgpu-use-native", cl::desc("Comma separated list of functions to replace with native, or all"), cl::CommaSeparated, cl::ValueOptional, cl::Hidden)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2009
#define MATH_SQRT2
static const TableEntry tbl_tan[]
static const TableEntry tbl_cos[]
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1248
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1199
static bool HasNative(AMDGPULibFunc::EFuncId id)
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1299
const std::string to_string(const T &Value)
Definition: ScopedPrinter.h:61
void replaceCall(Value *With)
void addAttributes(unsigned i, const AttrBuilder &Attrs)
adds the attributes to the list of attributes.
Definition: Function.cpp:383
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
FunctionPass * createAMDGPUUseNativeCallsPass()
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:565
LLVM Value Representation.
Definition: Value.h:72
amdgpu Simplify well known AMD library calls
static VectorType * get(Type *ElementType, unsigned NumElements)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:605
static const TableEntry tbl_erfc[]
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:219
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:121
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:333
std::string mangle() const
Type * getElementType() const
Definition: DerivedTypes.h:391
bool isExactlyValue(const APFloat &V) const
We don&#39;t rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: Constants.cpp:833
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
unsigned getNumElements() const
Return the number of elements in the array or vector.
Definition: Constants.cpp:2434
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:156
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
Definition: Constants.cpp:2573
#define LLVM_DEBUG(X)
Definition: Debug.h:122
bool isDoubleTy() const
Return true if this is &#39;double&#39;, a 64-bit IEEE fp type.
Definition: Type.h:149
static const TableEntry tbl_tgamma[]
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
bool useNative(CallInst *CI)
static const TableEntry tbl_acos[]
const BasicBlock * getParent() const
Definition: Instruction.h:66
an instruction to allocate memory on the stack
Definition: Instructions.h:59
#define MATH_E