LLVM 24.0.0git
SimplifyLibCalls.cpp
Go to the documentation of this file.
1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APSInt.h"
20#include "llvm/Analysis/Loads.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/MDBuilder.h"
32#include "llvm/IR/Module.h"
44
45#include <cmath>
46
47using namespace llvm;
48using namespace PatternMatch;
49
50#define DEBUG_TYPE "simplify-lib-calls"
51
52static cl::opt<bool>
53 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
54 cl::init(false),
55 cl::desc("Enable unsafe double to float "
56 "shrinking for math lib calls"));
57
58// Enable conversion of operator new calls with a MemProf hot or cold hint
59// to an operator new call that takes a hot/cold hint. Off by default since
60// not all allocators currently support this extension.
61static cl::opt<bool>
62 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false),
63 cl::desc("Enable hot/cold operator new library calls"));
70 "optimize-existing-hot-cold-new", cl::Hidden,
72 "Enable optimization of existing hot/cold operator new library calls"),
76 "Do not optimize existing hot/cold operator new library calls"),
78 "Only optimize existing hot/cold operator new library calls "
79 "if determined to be cold"),
82 "Always optimize existing hot/cold operator new library calls"),
85 "Always optimize existing hot/cold operator new library calls")),
88 "optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(false),
89 cl::desc("Enable transformation of nobuiltin operator new library calls"));
91 "min-existing-hot-cold-new-hint", cl::Hidden, cl::init(false),
92 cl::desc("Take the minimum of compiler hint and existing hint when "
93 "optimizing existing hot/cold operator new library calls"));
94
95namespace llvm {
97} // namespace llvm
98
99namespace {
100
101// Specialized parser to ensure the hint is an 8 bit value (we can't specify
102// uint8_t to opt<> as that is interpreted to mean that we are passing a char
103// option with a specific set of values.
104struct HotColdHintParser : public cl::parser<unsigned> {
105 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
106
107 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
108 if (Arg.getAsInteger(0, Value))
109 return O.error("'" + Arg + "' value invalid for uint argument!");
110
111 if (Value > 255)
112 return O.error("'" + Arg + "' value must be in the range [0, 255]!");
113
114 return false;
115 }
116};
117
118} // end anonymous namespace
119
120// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
121// and 255 is the hottest. Default to 1 value away from the coldest and hottest
122// hints, so that the compiler hinted allocations are slightly less strong than
123// manually inserted hints at the two extremes.
125 "cold-new-hint-value", cl::Hidden, cl::init(1),
126 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
128 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(128),
129 cl::desc("Value to pass to hot/cold operator new for "
130 "notcold (warm) allocation"));
132 "hot-new-hint-value", cl::Hidden, cl::init(254),
133 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
135 "ambiguous-new-hint-value", cl::Hidden, cl::init(222),
136 cl::desc(
137 "Value to pass to hot/cold operator new for ambiguous allocation"));
138
139//===----------------------------------------------------------------------===//
140// Helper Functions
141//===----------------------------------------------------------------------===//
142
143static bool ignoreCallingConv(LibFunc Func) {
144 return Func == LibFunc_abs || Func == LibFunc_labs ||
145 Func == LibFunc_llabs || Func == LibFunc_strlen;
146}
147
148/// Return true if it is only used in equality comparisons with With.
150 for (User *U : V->users()) {
151 if (ICmpInst *IC = dyn_cast<ICmpInst>(U))
152 if (IC->isEquality() && IC->getOperand(1) == With)
153 continue;
154 // Unknown instruction.
155 return false;
156 }
157 return true;
158}
159
161 return any_of(CI->operands(), [](const Use &OI) {
162 return OI->getType()->isFloatingPointTy();
163 });
164}
165
166static bool callHasFP128Argument(const CallInst *CI) {
167 return any_of(CI->operands(), [](const Use &OI) {
168 return OI->getType()->isFP128Ty();
169 });
170}
171
172// Convert the entire string Str representing an integer in Base, up to
173// the terminating nul if present, to a constant according to the rules
174// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
175// return the result, otherwise null.
176// The function assumes the string is encoded in ASCII and carefully
177// avoids converting sequences (including "") that the corresponding
178// library call might fail and set errno for.
179static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
180 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
181 if (Base < 2 || Base > 36)
182 if (Base != 0)
183 // Fail for an invalid base (required by POSIX).
184 return nullptr;
185
186 // Current offset into the original string to reflect in EndPtr.
187 size_t Offset = 0;
188 // Strip leading whitespace.
189 for ( ; Offset != Str.size(); ++Offset)
190 if (!isSpace((unsigned char)Str[Offset])) {
191 Str = Str.substr(Offset);
192 break;
193 }
194
195 if (Str.empty())
196 // Fail for empty subject sequences (POSIX allows but doesn't require
197 // strtol[l]/strtoul[l] to fail with EINVAL).
198 return nullptr;
199
200 // Strip but remember the sign.
201 bool Negate = Str[0] == '-';
202 if (Str[0] == '-' || Str[0] == '+') {
203 Str = Str.drop_front();
204 if (Str.empty())
205 // Fail for a sign with nothing after it.
206 return nullptr;
207 ++Offset;
208 }
209
210 // Set Max to the absolute value of the minimum (for signed), or
211 // to the maximum (for unsigned) value representable in the type.
212 Type *RetTy = CI->getType();
213 unsigned NBits = RetTy->getPrimitiveSizeInBits();
214 uint64_t Max = AsSigned && Negate ? 1 : 0;
215 Max += AsSigned ? maxIntN(NBits) : maxUIntN(NBits);
216
217 // Autodetect Base if it's zero and consume the "0x" prefix.
218 if (Str.size() > 1) {
219 if (Str[0] == '0') {
220 if (toUpper((unsigned char)Str[1]) == 'X') {
221 if (Str.size() == 2 || (Base && Base != 16))
222 // Fail if Base doesn't allow the "0x" prefix or for the prefix
223 // alone that implementations like BSD set errno to EINVAL for.
224 return nullptr;
225
226 Str = Str.drop_front(2);
227 Offset += 2;
228 Base = 16;
229 }
230 else if (Base == 0)
231 Base = 8;
232 } else if (Base == 0)
233 Base = 10;
234 }
235 else if (Base == 0)
236 Base = 10;
237
238 // Convert the rest of the subject sequence, not including the sign,
239 // to its uint64_t representation (this assumes the source character
240 // set is ASCII).
241 uint64_t Result = 0;
242 for (unsigned i = 0; i != Str.size(); ++i) {
243 unsigned char DigVal = Str[i];
244 if (isDigit(DigVal))
245 DigVal = DigVal - '0';
246 else {
247 DigVal = toUpper(DigVal);
248 if (isAlpha(DigVal))
249 DigVal = DigVal - 'A' + 10;
250 else
251 return nullptr;
252 }
253
254 if (DigVal >= Base)
255 // Fail if the digit is not valid in the Base.
256 return nullptr;
257
258 // Add the digit and fail if the result is not representable in
259 // the (unsigned form of the) destination type.
260 bool VFlow;
261 Result = SaturatingMultiplyAdd(Result, Base, (uint64_t)DigVal, &VFlow);
262 if (VFlow || Result > Max)
263 return nullptr;
264 }
265
266 if (EndPtr) {
267 // Store the pointer to the end.
268 Value *Off = B.getInt64(Offset + Str.size());
269 Value *StrBeg = CI->getArgOperand(0);
270 Value *StrEnd = B.CreateInBoundsGEP(B.getInt8Ty(), StrBeg, Off, "endptr");
271 B.CreateStore(StrEnd, EndPtr);
272 }
273
274 if (Negate) {
275 // Unsigned negation doesn't overflow.
276 Result = -Result;
277 // For unsigned numbers, discard sign bits.
278 if (!AsSigned)
279 Result &= maxUIntN(NBits);
280 }
281
282 return ConstantInt::get(RetTy, Result, AsSigned);
283}
284
286 for (User *U : V->users()) {
287 if (ICmpInst *IC = dyn_cast<ICmpInst>(U))
288 if (Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
289 if (C->isNullValue())
290 continue;
291 // Unknown instruction.
292 return false;
293 }
294 return true;
295}
296
297static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
298 const SimplifyQuery &SQ) {
300 return false;
301
302 if (!isDereferenceablePointer(Str, APInt(64, Len), SQ))
303 return false;
304
305 if (CI->getFunction()->hasFnAttribute(Attribute::SanitizeMemory))
306 return false;
307
308 return true;
309}
310
312 ArrayRef<unsigned> ArgNos,
313 uint64_t DereferenceableBytes) {
314 const Function *F = CI->getCaller();
315 if (!F)
316 return;
317 for (unsigned ArgNo : ArgNos) {
318 uint64_t DerefBytes = DereferenceableBytes;
319 unsigned AS = CI->getArgOperand(ArgNo)->getType()->getPointerAddressSpace();
320 if (!llvm::NullPointerIsDefined(F, AS) ||
321 CI->paramHasAttr(ArgNo, Attribute::NonNull))
322 DerefBytes = std::max(CI->getParamDereferenceableOrNullBytes(ArgNo),
323 DereferenceableBytes);
324
325 if (CI->getParamDereferenceableBytes(ArgNo) < DerefBytes) {
326 CI->removeParamAttr(ArgNo, Attribute::Dereferenceable);
327 if (!llvm::NullPointerIsDefined(F, AS) ||
328 CI->paramHasAttr(ArgNo, Attribute::NonNull))
329 CI->removeParamAttr(ArgNo, Attribute::DereferenceableOrNull);
331 CI->getContext(), DerefBytes));
332 }
333 }
334}
335
337 ArrayRef<unsigned> ArgNos,
338 uint64_t CharSize = 1) {
339 Function *F = CI->getCaller();
340 if (!F)
341 return;
342
343 for (unsigned ArgNo : ArgNos) {
344 if (!CI->paramHasAttr(ArgNo, Attribute::NoUndef))
345 CI->addParamAttr(ArgNo, Attribute::NoUndef);
346
347 if (!CI->paramHasAttr(ArgNo, Attribute::NonNull)) {
348 unsigned AS =
351 continue;
352 CI->addParamAttr(ArgNo, Attribute::NonNull);
353 }
354
355 annotateDereferenceableBytes(CI, ArgNo, CharSize);
356 }
357}
358
360 Value *Size, const DataLayout &DL) {
363 annotateDereferenceableBytes(CI, ArgNos, LenC->getZExtValue());
364 } else if (isKnownNonZero(Size, DL)) {
366 uint64_t X, Y;
367 uint64_t DerefMin = 1;
369 DerefMin = std::min(X, Y);
370 annotateDereferenceableBytes(CI, ArgNos, DerefMin);
371 }
372 }
373}
374
375// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
376// easier chaining. Calls to emit* and B.createCall should probably be wrapped
377// in this function when New is created to replace Old. Callers should take
378// care to check Old.isMustTailCall() if they aren't replacing Old directly
379// with New.
380static Value *copyFlags(const CallInst &Old, Value *New) {
381 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
382 assert(!Old.isNoTailCall() && "do not copy notail call flags");
383 if (auto *NewCI = dyn_cast_or_null<CallInst>(New))
384 NewCI->setTailCallKind(Old.getTailCallKind());
385 return New;
386}
387
388static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
389 NewCI->setAttributes(AttributeList::get(
390 NewCI->getContext(), {NewCI->getAttributes(), Old.getAttributes()}));
391 if (NewCI->getType()->isVoidTy()) {
392 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
393 NewCI->removeParamAttr(I, Attribute::Returned);
394 }
395 NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(
396 NewCI->getType(), NewCI->getRetAttributes()));
397 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
398 NewCI->removeParamAttrs(
399 I, AttributeFuncs::typeIncompatible(NewCI->getArgOperand(I)->getType(),
400 NewCI->getParamAttributes(I)));
401
402 return copyFlags(Old, NewCI);
403}
404
405// Helper to avoid truncating the length if size_t is 32-bits.
407 return Len >= Str.size() ? Str : Str.substr(0, Len);
408}
409
410//===----------------------------------------------------------------------===//
411// String and Memory Library Call Optimizations
412//===----------------------------------------------------------------------===//
413
414Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
415 // Extract some information from the instruction
416 Value *Dst = CI->getArgOperand(0);
417 Value *Src = CI->getArgOperand(1);
419
420 // See if we can get the length of the input string.
422 if (Len)
424 else
425 return nullptr;
426 --Len; // Unbias length.
427
428 // Handle the simple, do-nothing case: strcat(x, "") -> x
429 if (Len == 0)
430 return Dst;
431
432 return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, Len, B));
433}
434
435Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
436 IRBuilderBase &B) {
437 // We need to find the end of the destination string. That's where the
438 // memory is to be moved to. We just generate a call to strlen.
439 Value *DstLen = emitStrLen(Dst, B, DL, TLI);
440 if (!DstLen)
441 return nullptr;
442
443 // Now that we have the destination's length, we must index into the
444 // destination's pointer to get the actual memcpy destination (end of
445 // the string .. we're concatenating).
446 Value *CpyDst = B.CreateInBoundsGEP(B.getInt8Ty(), Dst, DstLen, "endptr");
447
448 // We have enough information to now generate the memcpy call to do the
449 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
450 B.CreateMemCpy(CpyDst, Align(1), Src, Align(1),
451 TLI->getAsSizeT(Len + 1, *B.getModule()));
452 return Dst;
453}
454
455Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
456 // Extract some information from the instruction.
457 Value *Dst = CI->getArgOperand(0);
458 Value *Src = CI->getArgOperand(1);
459 Value *Size = CI->getArgOperand(2);
462 if (isKnownNonZero(Size, DL))
464
465 // We don't do anything if length is not constant.
466 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size);
467 if (LengthArg) {
468 Len = LengthArg->getZExtValue();
469 // strncat(x, c, 0) -> x
470 if (!Len)
471 return Dst;
472 } else {
473 return nullptr;
474 }
475
476 // See if we can get the length of the input string.
477 uint64_t SrcLen = GetStringLength(Src);
478 if (SrcLen) {
479 annotateDereferenceableBytes(CI, 1, SrcLen);
480 --SrcLen; // Unbias length.
481 } else {
482 return nullptr;
483 }
484
485 // strncat(x, "", c) -> x
486 if (SrcLen == 0)
487 return Dst;
488
489 // We don't optimize this case.
490 if (Len < SrcLen)
491 return nullptr;
492
493 // strncat(x, s, c) -> strcat(x, s)
494 // s is constant so the strcat can be optimized further.
495 return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, SrcLen, B));
496}
497
498// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
499// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
500// is that either S is dereferenceable or the value of N is nonzero.
502 IRBuilderBase &B, const DataLayout &DL)
503{
504 Value *Src = CI->getArgOperand(0);
505 Value *CharVal = CI->getArgOperand(1);
506
507 // Fold memchr(A, C, N) == A to N && *A == C.
508 Type *CharTy = B.getInt8Ty();
509 Value *Char0 = B.CreateLoad(CharTy, Src);
510 CharVal = B.CreateTrunc(CharVal, CharTy);
511 Value *Cmp = B.CreateICmpEQ(Char0, CharVal, "char0cmp");
512
513 if (NBytes) {
514 Value *Zero = ConstantInt::get(NBytes->getType(), 0);
515 Value *And = B.CreateICmpNE(NBytes, Zero);
516 Cmp = B.CreateLogicalAnd(And, Cmp);
517 // The and above is based on the byte count and the query, neither of which
518 // we know without value profiling, so mark the profile as unknown.
519 if (auto *SI = dyn_cast<SelectInst>(Cmp))
521 }
522
523 Value *NullPtr = Constant::getNullValue(CI->getType());
524 return B.CreateSelect(Cmp, Src, NullPtr);
525}
526
527Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
528 Value *SrcStr = CI->getArgOperand(0);
529 Value *CharVal = CI->getArgOperand(1);
531
532 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
533 return memChrToCharCompare(CI, nullptr, B, DL);
534
535 // If the second operand is non-constant, see if we can compute the length
536 // of the input string and turn this into memchr.
537 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
538 if (!CharC) {
539 uint64_t Len = GetStringLength(SrcStr);
540 if (Len)
542 else
543 return nullptr;
544
546 FunctionType *FT = Callee->getFunctionType();
547 unsigned IntBits = TLI->getIntSize();
548 if (!FT->getParamType(1)->isIntegerTy(IntBits)) // memchr needs 'int'.
549 return nullptr;
550
551 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
552 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
553 return copyFlags(*CI,
554 emitMemChr(SrcStr, CharVal, // include nul.
555 ConstantInt::get(SizeTTy, Len), B,
556 DL, TLI));
557 }
558
559 if (CharC->isZero()) {
560 Value *NullPtr = Constant::getNullValue(CI->getType());
561 if (isOnlyUsedInEqualityComparison(CI, NullPtr))
562 // Pre-empt the transformation to strlen below and fold
563 // strchr(A, '\0') == null to false.
564 return B.CreateIntToPtr(B.getTrue(), CI->getType());
565 }
566
567 // Otherwise, the character is a constant, see if the first argument is
568 // a string literal. If so, we can constant fold.
569 StringRef Str;
570 if (!getConstantStringInfo(SrcStr, Str)) {
571 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
572 if (Value *StrLen = emitStrLen(SrcStr, B, DL, TLI))
573 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, StrLen, "strchr");
574 return nullptr;
575 }
576
577 // Compute the offset, make sure to handle the case when we're searching for
578 // zero (a weird way to spell strlen).
579 size_t I = (0xFF & CharC->getSExtValue()) == 0
580 ? Str.size()
581 : Str.find(CharC->getSExtValue());
582 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
583 return Constant::getNullValue(CI->getType());
584
585 // strchr(s+n,c) -> gep(s+n+i,c)
586 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(I), "strchr");
587}
588
589Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
590 Value *SrcStr = CI->getArgOperand(0);
591 Value *CharVal = CI->getArgOperand(1);
592 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
594
595 StringRef Str;
596 if (!getConstantStringInfo(SrcStr, Str)) {
597 // strrchr(s, 0) -> strchr(s, 0)
598 if (CharC && CharC->isZero())
599 return copyFlags(*CI, emitStrChr(SrcStr, '\0', B, TLI));
600 return nullptr;
601 }
602
603 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
604 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
605
606 // Try to expand strrchr to the memrchr nonstandard extension if it's
607 // available, or simply fail otherwise.
608 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
609 Value *Size = ConstantInt::get(SizeTTy, NBytes);
610 return copyFlags(*CI, emitMemRChr(SrcStr, CharVal, Size, B, DL, TLI));
611}
612
613Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
614 Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
615 if (Str1P == Str2P) // strcmp(x,x) -> 0
616 return ConstantInt::get(CI->getType(), 0);
617
618 StringRef Str1, Str2;
619 bool HasStr1 = getConstantStringInfo(Str1P, Str1);
620 bool HasStr2 = getConstantStringInfo(Str2P, Str2);
621
622 // strcmp(x, y) -> cnst (if both x and y are constant strings)
623 if (HasStr1 && HasStr2)
624 return ConstantInt::getSigned(CI->getType(),
625 std::clamp(Str1.compare(Str2), -1, 1));
626
627 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
628 return B.CreateNeg(B.CreateZExt(
629 B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
630
631 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
632 return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
633 CI->getType());
634
635 // strcmp(P, "x") -> memcmp(P, "x", 2)
636 uint64_t Len1 = GetStringLength(Str1P);
637 if (Len1)
638 annotateDereferenceableBytes(CI, 0, Len1);
639 uint64_t Len2 = GetStringLength(Str2P);
640 if (Len2)
641 annotateDereferenceableBytes(CI, 1, Len2);
642
643 if (Len1 && Len2) {
644 return copyFlags(
645 *CI, emitMemCmp(Str1P, Str2P,
646 TLI->getAsSizeT(std::min(Len1, Len2), *CI->getModule()),
647 B, DL, TLI));
648 }
649
650 // strcmp to memcmp
651 SimplifyQuery SQ(DL, TLI, DT, AC, CI);
652 if (!HasStr1 && HasStr2) {
653 if (canTransformToMemCmp(CI, Str1P, Len2, SQ))
654 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
655 TLI->getAsSizeT(Len2, *CI->getModule()),
656 B, DL, TLI));
657 } else if (HasStr1 && !HasStr2) {
658 if (canTransformToMemCmp(CI, Str2P, Len1, SQ))
659 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
660 TLI->getAsSizeT(Len1, *CI->getModule()),
661 B, DL, TLI));
662 }
663
665 return nullptr;
666}
667
668// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
669// arrays LHS and RHS and nonconstant Size.
671 Value *Size, bool StrNCmp,
672 IRBuilderBase &B, const DataLayout &DL);
673
674Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
675 Value *Str1P = CI->getArgOperand(0);
676 Value *Str2P = CI->getArgOperand(1);
677 Value *Size = CI->getArgOperand(2);
678 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
679 return ConstantInt::get(CI->getType(), 0);
680
681 if (isKnownNonZero(Size, DL))
683 // Get the length argument if it is constant.
685 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size))
686 Length = LengthArg->getZExtValue();
687 else
688 return optimizeMemCmpVarSize(CI, Str1P, Str2P, Size, true, B, DL);
689
690 if (Length == 0) // strncmp(x,y,0) -> 0
691 return ConstantInt::get(CI->getType(), 0);
692
693 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
694 return copyFlags(*CI, emitMemCmp(Str1P, Str2P, Size, B, DL, TLI));
695
696 StringRef Str1, Str2;
697 bool HasStr1 = getConstantStringInfo(Str1P, Str1);
698 bool HasStr2 = getConstantStringInfo(Str2P, Str2);
699
700 // strncmp(x, y) -> cnst (if both x and y are constant strings)
701 if (HasStr1 && HasStr2) {
702 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
703 StringRef SubStr1 = substr(Str1, Length);
704 StringRef SubStr2 = substr(Str2, Length);
705 return ConstantInt::getSigned(CI->getType(),
706 std::clamp(SubStr1.compare(SubStr2), -1, 1));
707 }
708
709 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
710 return B.CreateNeg(B.CreateZExt(
711 B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
712
713 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
714 return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
715 CI->getType());
716
717 uint64_t Len1 = GetStringLength(Str1P);
718 if (Len1)
719 annotateDereferenceableBytes(CI, 0, Len1);
720 uint64_t Len2 = GetStringLength(Str2P);
721 if (Len2)
722 annotateDereferenceableBytes(CI, 1, Len2);
723
724 // strncmp to memcmp
725 if (!HasStr1 && HasStr2) {
726 Len2 = std::min(Len2, Length);
727 if (canTransformToMemCmp(CI, Str1P, Len2, DL))
728 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
729 TLI->getAsSizeT(Len2, *CI->getModule()),
730 B, DL, TLI));
731 } else if (HasStr1 && !HasStr2) {
732 Len1 = std::min(Len1, Length);
733 if (canTransformToMemCmp(CI, Str2P, Len1, DL))
734 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
735 TLI->getAsSizeT(Len1, *CI->getModule()),
736 B, DL, TLI));
737 }
738
739 return nullptr;
740}
741
742Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
743 Value *Src = CI->getArgOperand(0);
744 ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
745 uint64_t SrcLen = GetStringLength(Src);
746 if (SrcLen && Size) {
747 annotateDereferenceableBytes(CI, 0, SrcLen);
748 if (SrcLen <= Size->getZExtValue() + 1)
749 return copyFlags(*CI, emitStrDup(Src, B, TLI));
750 }
751
752 return nullptr;
753}
754
755Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
756 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
757 if (Dst == Src) // strcpy(x,x) -> x
758 return Src;
759
761 // See if we can get the length of the input string.
763 if (Len)
765 else
766 return nullptr;
767
768 // We have enough information to now generate the memcpy call to do the
769 // copy for us. Make a memcpy to copy the nul byte with align = 1.
770 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
771 TLI->getAsSizeT(Len, *CI->getModule()));
772 mergeAttributesAndFlags(NewCI, *CI);
773 return Dst;
774}
775
776Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
777 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
778
779 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
780 if (CI->use_empty())
781 return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
782
783 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
784 Value *StrLen = emitStrLen(Src, B, DL, TLI);
785 return StrLen ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, StrLen) : nullptr;
786 }
787
788 // See if we can get the length of the input string.
790 if (Len)
792 else
793 return nullptr;
794
795 Value *LenV = TLI->getAsSizeT(Len, *CI->getModule());
796 Value *DstEnd = B.CreateInBoundsGEP(
797 B.getInt8Ty(), Dst, TLI->getAsSizeT(Len - 1, *CI->getModule()));
798
799 // We have enough information to now generate the memcpy call to do the
800 // copy for us. Make a memcpy to copy the nul byte with align = 1.
801 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1), LenV);
802 mergeAttributesAndFlags(NewCI, *CI);
803 return DstEnd;
804}
805
806// Optimize a call to size_t strlcpy(char*, const char*, size_t).
807
808Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
809 Value *Size = CI->getArgOperand(2);
810 if (isKnownNonZero(Size, DL))
811 // Like snprintf, the function stores into the destination only when
812 // the size argument is nonzero.
814 // The function reads the source argument regardless of Size (it returns
815 // its length).
817
818 uint64_t NBytes;
819 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Size))
820 NBytes = SizeC->getZExtValue();
821 else
822 return nullptr;
823
824 Value *Dst = CI->getArgOperand(0);
825 Value *Src = CI->getArgOperand(1);
826 if (NBytes <= 1) {
827 if (NBytes == 1)
828 // For a call to strlcpy(D, S, 1) first store a nul in *D.
829 B.CreateStore(B.getInt8(0), Dst);
830
831 // Transform strlcpy(D, S, 0) to a call to strlen(S).
832 return copyFlags(*CI, emitStrLen(Src, B, DL, TLI));
833 }
834
835 // Try to determine the length of the source, substituting its size
836 // when it's not nul-terminated (as it's required to be) to avoid
837 // reading past its end.
838 StringRef Str;
839 if (!getConstantStringInfo(Src, Str, /*TrimAtNul=*/false))
840 return nullptr;
841
842 uint64_t SrcLen = Str.find('\0');
843 // Set if the terminating nul should be copied by the call to memcpy
844 // below.
845 bool NulTerm = SrcLen < NBytes;
846
847 if (NulTerm)
848 // Overwrite NBytes with the number of bytes to copy, including
849 // the terminating nul.
850 NBytes = SrcLen + 1;
851 else {
852 // Set the length of the source for the function to return to its
853 // size, and cap NBytes at the same.
854 SrcLen = std::min(SrcLen, uint64_t(Str.size()));
855 NBytes = std::min(NBytes - 1, SrcLen);
856 }
857
858 if (SrcLen == 0) {
859 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
860 B.CreateStore(B.getInt8(0), Dst);
861 return ConstantInt::get(CI->getType(), 0);
862 }
863
864 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
865 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
866 // D[N' - 1] if necessary.
867 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
868 TLI->getAsSizeT(NBytes, *CI->getModule()));
869 mergeAttributesAndFlags(NewCI, *CI);
870
871 if (!NulTerm) {
872 Value *EndOff = ConstantInt::get(CI->getType(), NBytes);
873 Value *EndPtr = B.CreateInBoundsGEP(B.getInt8Ty(), Dst, EndOff);
874 B.CreateStore(B.getInt8(0), EndPtr);
875 }
876
877 // Like snprintf, strlcpy returns the number of nonzero bytes that would
878 // have been copied if the bound had been sufficiently big (which in this
879 // case is strlen(Src)).
880 return ConstantInt::get(CI->getType(), SrcLen);
881}
882
883// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
884// otherwise.
885Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
886 IRBuilderBase &B) {
887 Value *Dst = CI->getArgOperand(0);
888 Value *Src = CI->getArgOperand(1);
889 Value *Size = CI->getArgOperand(2);
890
891 if (isKnownNonZero(Size, DL)) {
892 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
893 // only when N is nonzero.
896 }
897
898 // If the "bound" argument is known set N to it. Otherwise set it to
899 // UINT64_MAX and handle it later.
901 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Size))
902 N = SizeC->getZExtValue();
903
904 if (N == 0)
905 // Fold st{p,r}ncpy(D, S, 0) to D.
906 return Dst;
907
908 if (N == 1) {
909 Type *CharTy = B.getInt8Ty();
910 Value *CharVal = B.CreateLoad(CharTy, Src, "stxncpy.char0");
911 B.CreateStore(CharVal, Dst);
912 if (!RetEnd)
913 // Transform strncpy(D, S, 1) to return (*D = *S), D.
914 return Dst;
915
916 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
917 Value *ZeroChar = ConstantInt::get(CharTy, 0);
918 Value *Cmp = B.CreateICmpEQ(CharVal, ZeroChar, "stpncpy.char0cmp");
919
920 Value *Off1 = B.getInt32(1);
921 Value *EndPtr = B.CreateInBoundsGEP(CharTy, Dst, Off1, "stpncpy.end");
922 return B.CreateSelect(Cmp, Dst, EndPtr, "stpncpy.sel");
923 }
924
925 // If the length of the input string is known set SrcLen to it.
926 uint64_t SrcLen = GetStringLength(Src);
927 if (SrcLen)
928 annotateDereferenceableBytes(CI, 1, SrcLen);
929 else
930 return nullptr;
931
932 --SrcLen; // Unbias length.
933
934 if (SrcLen == 0) {
935 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
936 Align MemSetAlign =
937 CI->getAttributes().getParamAttrs(0).getAlignment().valueOrOne();
938 CallInst *NewCI = B.CreateMemSet(Dst, B.getInt8('\0'), Size, MemSetAlign);
939 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(0));
940 NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
941 CI->getContext(), 0, ArgAttrs));
942 copyFlags(*CI, NewCI);
943 return Dst;
944 }
945
946 if (N > SrcLen + 1) {
947 if (N > 128)
948 // Bail if N is large or unknown.
949 return nullptr;
950
951 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
952 StringRef Str;
953 if (!getConstantStringInfo(Src, Str))
954 return nullptr;
955 std::string SrcStr = Str.str();
956 // Create a bigger, nul-padded array with the same length, SrcLen,
957 // as the original string.
958 SrcStr.resize(N, '\0');
959 Src = B.CreateGlobalString(SrcStr, "str", /*AddressSpace=*/0,
960 /*M=*/nullptr, /*AddNull=*/false);
961 }
962
963 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
964 // S and N are constant.
965 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
966 TLI->getAsSizeT(N, *CI->getModule()));
967 mergeAttributesAndFlags(NewCI, *CI);
968 if (!RetEnd)
969 return Dst;
970
971 // stpncpy(D, S, N) returns the address of the first null in D if it writes
972 // one, otherwise D + N.
973 Value *Off = B.getInt64(std::min(SrcLen, N));
974 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, Off, "endptr");
975}
976
977Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
978 unsigned CharSize,
979 Value *Bound) {
980 Value *Src = CI->getArgOperand(0);
981 Type *CharTy = B.getIntNTy(CharSize);
982
984 (!Bound || isKnownNonZero(Bound, DL))) {
985 // Fold strlen:
986 // strlen(x) != 0 --> *x != 0
987 // strlen(x) == 0 --> *x == 0
988 // and likewise strnlen with constant N > 0:
989 // strnlen(x, N) != 0 --> *x != 0
990 // strnlen(x, N) == 0 --> *x == 0
991 return B.CreateZExt(B.CreateLoad(CharTy, Src, "char0"),
992 CI->getType());
993 }
994
995 if (Bound) {
996 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Bound)) {
997 if (BoundCst->isZero())
998 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
999 return ConstantInt::get(CI->getType(), 0);
1000
1001 if (BoundCst->isOne()) {
1002 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
1003 Value *CharVal = B.CreateLoad(CharTy, Src, "strnlen.char0");
1004 Value *ZeroChar = ConstantInt::get(CharTy, 0);
1005 Value *Cmp = B.CreateICmpNE(CharVal, ZeroChar, "strnlen.char0cmp");
1006 return B.CreateZExt(Cmp, CI->getType());
1007 }
1008 }
1009 }
1010
1011 if (uint64_t Len = GetStringLength(Src, CharSize)) {
1012 Value *LenC = ConstantInt::get(CI->getType(), Len - 1);
1013 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
1014 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
1015 if (Bound)
1016 return B.CreateBinaryIntrinsic(Intrinsic::umin, LenC, Bound);
1017 return LenC;
1018 }
1019
1020 if (Bound)
1021 // Punt for strnlen for now.
1022 return nullptr;
1023
1024 // If s is a constant pointer pointing to a string literal, we can fold
1025 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
1026 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
1027 // We only try to simplify strlen when the pointer s points to an array
1028 // of CharSize elements. Otherwise, we would need to scale the offset x before
1029 // doing the subtraction. This will make the optimization more complex, and
1030 // it's not very useful because calling strlen for a pointer of other types is
1031 // very uncommon.
1032 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Src)) {
1033 unsigned BW = DL.getIndexTypeSizeInBits(GEP->getType());
1034 SmallMapVector<Value *, APInt, 4> VarOffsets;
1035 APInt ConstOffset(BW, 0);
1036 assert(CharSize % 8 == 0 && "Expected a multiple of 8 sized CharSize");
1037 // Check the gep is a single variable offset.
1038 if (!GEP->collectOffset(DL, BW, VarOffsets, ConstOffset) ||
1039 VarOffsets.size() != 1 || ConstOffset != 0 ||
1040 VarOffsets.begin()->second != CharSize / 8)
1041 return nullptr;
1042
1043 ConstantDataArraySlice Slice;
1044 if (getConstantDataArrayInfo(GEP->getOperand(0), Slice, CharSize)) {
1045 uint64_t NullTermIdx;
1046 if (Slice.Array == nullptr) {
1047 NullTermIdx = 0;
1048 } else {
1049 NullTermIdx = ~((uint64_t)0);
1050 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
1051 if (Slice.Array->getElementAsInteger(I + Slice.Offset) == 0) {
1052 NullTermIdx = I;
1053 break;
1054 }
1055 }
1056 // If the string does not have '\0', leave it to strlen to compute
1057 // its length.
1058 if (NullTermIdx == ~((uint64_t)0))
1059 return nullptr;
1060 }
1061
1062 Value *Offset = VarOffsets.begin()->first;
1063 KnownBits Known = computeKnownBits(Offset, DL, nullptr, CI, nullptr);
1064
1065 // If Offset is not provably in the range [0, NullTermIdx], we can still
1066 // optimize if we can prove that the program has undefined behavior when
1067 // Offset is outside that range. That is the case when GEP->getOperand(0)
1068 // is a pointer to an object whose memory extent is NullTermIdx+1.
1069 if ((Known.isNonNegative() && Known.getMaxValue().ule(NullTermIdx)) ||
1070 (isa<GlobalVariable>(GEP->getOperand(0)) &&
1071 NullTermIdx == Slice.Length - 1)) {
1072 Offset = B.CreateSExtOrTrunc(Offset, CI->getType());
1073 return B.CreateSub(ConstantInt::get(CI->getType(), NullTermIdx),
1074 Offset);
1075 }
1076 }
1077 }
1078
1079 // strlen(x?"foo":"bars") --> x ? 3 : 4
1080 if (SelectInst *SI = dyn_cast<SelectInst>(Src)) {
1081 uint64_t LenTrue = GetStringLength(SI->getTrueValue(), CharSize);
1082 uint64_t LenFalse = GetStringLength(SI->getFalseValue(), CharSize);
1083 if (LenTrue && LenFalse) {
1084 ORE.emit([&]() {
1085 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1086 << "folded strlen(select) to select of constants";
1087 });
1088 return B.CreateSelect(SI->getCondition(),
1089 ConstantInt::get(CI->getType(), LenTrue - 1),
1090 ConstantInt::get(CI->getType(), LenFalse - 1), "",
1091 ProfcheckDisableMetadataFixes ? nullptr : SI);
1092 }
1093 }
1094
1095 return nullptr;
1096}
1097
1098Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1099 if (Value *V = optimizeStringLength(CI, B, 8))
1100 return V;
1102 return nullptr;
1103}
1104
1105Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1106 Value *Bound = CI->getArgOperand(1);
1107 if (Value *V = optimizeStringLength(CI, B, 8, Bound))
1108 return V;
1109
1110 if (isKnownNonZero(Bound, DL))
1112 return nullptr;
1113}
1114
1115Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1116 Module &M = *CI->getModule();
1117 unsigned WCharSize = M.getWCharSize();
1118
1119 if (Value *V = optimizeStringLength(CI, B, WCharSize * 8))
1120 return V;
1121 // At least the terminating wide character is read.
1122 annotateNonNullNoUndefBasedOnAccess(CI, 0, WCharSize);
1123 return nullptr;
1124}
1125
1126Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1127 StringRef S1, S2;
1128 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1129 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1130
1131 // strpbrk(s, "") -> nullptr
1132 // strpbrk("", s) -> nullptr
1133 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1134 return Constant::getNullValue(CI->getType());
1135
1136 // Constant folding.
1137 if (HasS1 && HasS2) {
1138 size_t I = S1.find_first_of(S2);
1139 if (I == StringRef::npos) // No match.
1140 return Constant::getNullValue(CI->getType());
1141
1142 return B.CreateInBoundsGEP(B.getInt8Ty(), CI->getArgOperand(0),
1143 B.getInt64(I), "strpbrk");
1144 }
1145
1146 // strpbrk(s, "a") -> strchr(s, 'a')
1147 if (HasS2 && S2.size() == 1)
1148 return copyFlags(*CI, emitStrChr(CI->getArgOperand(0), S2[0], B, TLI));
1149
1150 return nullptr;
1151}
1152
1153Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1154 Value *EndPtr = CI->getArgOperand(1);
1155 if (isa<ConstantPointerNull>(EndPtr)) {
1156 // With a null EndPtr, this function won't capture the main argument.
1157 // It would be readonly too, except that it still may write to errno.
1160 }
1161
1162 return nullptr;
1163}
1164
1165Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1166 StringRef S1, S2;
1167 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1168 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1169
1170 // strspn(s, "") -> 0
1171 // strspn("", s) -> 0
1172 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1173 return Constant::getNullValue(CI->getType());
1174
1175 // Constant folding.
1176 if (HasS1 && HasS2) {
1177 size_t Pos = S1.find_first_not_of(S2);
1178 if (Pos == StringRef::npos)
1179 Pos = S1.size();
1180 return ConstantInt::get(CI->getType(), Pos);
1181 }
1182
1183 return nullptr;
1184}
1185
1186Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1187 StringRef S1, S2;
1188 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1189 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1190
1191 // strcspn("", s) -> 0
1192 if (HasS1 && S1.empty())
1193 return Constant::getNullValue(CI->getType());
1194
1195 // Constant folding.
1196 if (HasS1 && HasS2) {
1197 size_t Pos = S1.find_first_of(S2);
1198 if (Pos == StringRef::npos)
1199 Pos = S1.size();
1200 return ConstantInt::get(CI->getType(), Pos);
1201 }
1202
1203 // strcspn(s, "") -> strlen(s)
1204 if (HasS2 && S2.empty())
1205 return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B, DL, TLI));
1206
1207 return nullptr;
1208}
1209
1210Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1211 // fold strstr(x, x) -> x.
1212 if (CI->getArgOperand(0) == CI->getArgOperand(1))
1213 return CI->getArgOperand(0);
1214
1215 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1217 Value *StrLen = emitStrLen(CI->getArgOperand(1), B, DL, TLI);
1218 if (!StrLen)
1219 return nullptr;
1220 Value *StrNCmp = emitStrNCmp(CI->getArgOperand(0), CI->getArgOperand(1),
1221 StrLen, B, DL, TLI);
1222 if (!StrNCmp)
1223 return nullptr;
1224 for (User *U : llvm::make_early_inc_range(CI->users())) {
1225 ICmpInst *Old = cast<ICmpInst>(U);
1226 Value *Cmp =
1227 B.CreateICmp(Old->getPredicate(), StrNCmp,
1228 ConstantInt::getNullValue(StrNCmp->getType()), "cmp");
1229 replaceAllUsesWith(Old, Cmp);
1230 }
1231 return CI;
1232 }
1233
1234 // See if either input string is a constant string.
1235 StringRef SearchStr, ToFindStr;
1236 bool HasStr1 = getConstantStringInfo(CI->getArgOperand(0), SearchStr);
1237 bool HasStr2 = getConstantStringInfo(CI->getArgOperand(1), ToFindStr);
1238
1239 // fold strstr(x, "") -> x.
1240 if (HasStr2 && ToFindStr.empty())
1241 return CI->getArgOperand(0);
1242
1243 // If both strings are known, constant fold it.
1244 if (HasStr1 && HasStr2) {
1245 size_t Offset = SearchStr.find(ToFindStr);
1246
1247 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1248 return Constant::getNullValue(CI->getType());
1249
1250 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1251 return B.CreateConstInBoundsGEP1_64(B.getInt8Ty(), CI->getArgOperand(0),
1252 Offset, "strstr");
1253 }
1254
1255 // fold strstr(x, "y") -> strchr(x, 'y').
1256 if (HasStr2 && ToFindStr.size() == 1) {
1257 return emitStrChr(CI->getArgOperand(0), ToFindStr[0], B, TLI);
1258 }
1259
1261 return nullptr;
1262}
1263
1264Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1265 Value *SrcStr = CI->getArgOperand(0);
1266 Value *Size = CI->getArgOperand(2);
1268 Value *CharVal = CI->getArgOperand(1);
1269 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1270 Value *NullPtr = Constant::getNullValue(CI->getType());
1271
1272 if (LenC) {
1273 if (LenC->isZero())
1274 // Fold memrchr(x, y, 0) --> null.
1275 return NullPtr;
1276
1277 if (LenC->isOne()) {
1278 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1279 // constant or otherwise.
1280 Value *Val = B.CreateLoad(B.getInt8Ty(), SrcStr, "memrchr.char0");
1281 // Slice off the character's high end bits.
1282 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1283 Value *Cmp = B.CreateICmpEQ(Val, CharVal, "memrchr.char0cmp");
1284 return B.CreateSelectWithUnknownProfile(Cmp, SrcStr, NullPtr, DEBUG_TYPE,
1285 "memrchr.sel");
1286 }
1287 }
1288
1289 StringRef Str;
1290 if (!getConstantStringInfo(SrcStr, Str, /*TrimAtNul=*/false))
1291 return nullptr;
1292
1293 if (Str.size() == 0)
1294 // If the array is empty fold memrchr(A, C, N) to null for any value
1295 // of C and N on the basis that the only valid value of N is zero
1296 // (otherwise the call is undefined).
1297 return NullPtr;
1298
1299 uint64_t EndOff = UINT64_MAX;
1300 if (LenC) {
1301 EndOff = LenC->getZExtValue();
1302 if (Str.size() < EndOff)
1303 // Punt out-of-bounds accesses to sanitizers and/or libc.
1304 return nullptr;
1305 }
1306
1307 if (ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal)) {
1308 // Fold memrchr(S, C, N) for a constant C.
1309 size_t Pos = Str.rfind(CharC->getZExtValue(), EndOff);
1310 if (Pos == StringRef::npos)
1311 // When the character is not in the source array fold the result
1312 // to null regardless of Size.
1313 return NullPtr;
1314
1315 if (LenC)
1316 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1317 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(Pos));
1318
1319 if (Str.find(Str[Pos]) == Pos) {
1320 // When there is just a single occurrence of C in S, i.e., the one
1321 // in Str[Pos], fold
1322 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1323 // for nonconstant N.
1324 Value *Cmp = B.CreateICmpULE(Size, ConstantInt::get(Size->getType(), Pos),
1325 "memrchr.cmp");
1326 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr,
1327 B.getInt64(Pos), "memrchr.ptr_plus");
1328 return B.CreateSelectWithUnknownProfile(Cmp, NullPtr, SrcPlus, DEBUG_TYPE,
1329 "memrchr.sel");
1330 }
1331 }
1332
1333 // Truncate the string to search at most EndOff characters.
1334 Str = Str.substr(0, EndOff);
1335 if (Str.find_first_not_of(Str[0]) != StringRef::npos)
1336 return nullptr;
1337
1338 // If the source array consists of all equal characters, then for any
1339 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1340 // N != 0 && *S == C ? S + N - 1 : null
1341 Type *SizeTy = Size->getType();
1342 Type *Int8Ty = B.getInt8Ty();
1343 Value *NNeZ = B.CreateICmpNE(Size, ConstantInt::get(SizeTy, 0));
1344 // Slice off the sought character's high end bits.
1345 CharVal = B.CreateTrunc(CharVal, Int8Ty);
1346 Value *CEqS0 =
1347 B.CreateICmpEQ(ConstantInt::get(Int8Ty, (unsigned char)Str[0]), CharVal);
1348 Value *And = B.CreateLogicalAnd(NNeZ, CEqS0);
1349 if (auto *AndSI = dyn_cast<SelectInst>(And))
1351 Value *SizeM1 = B.CreateSub(Size, ConstantInt::get(SizeTy, 1));
1352 Value *SrcPlus =
1353 B.CreateInBoundsGEP(Int8Ty, SrcStr, SizeM1, "memrchr.ptr_plus");
1354 return B.CreateSelectWithUnknownProfile(And, SrcPlus, NullPtr, DEBUG_TYPE,
1355 "memrchr.sel");
1356}
1357
1358Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1359 Value *SrcStr = CI->getArgOperand(0);
1360 Value *Size = CI->getArgOperand(2);
1361
1362 if (isKnownNonZero(Size, DL)) {
1364 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
1365 return memChrToCharCompare(CI, Size, B, DL);
1366 }
1367
1368 Value *CharVal = CI->getArgOperand(1);
1369 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
1370 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1371 Value *NullPtr = Constant::getNullValue(CI->getType());
1372
1373 // memchr(x, y, 0) -> null
1374 if (LenC) {
1375 if (LenC->isZero())
1376 return NullPtr;
1377
1378 if (LenC->isOne()) {
1379 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1380 // constant or otherwise.
1381 Value *Val = B.CreateLoad(B.getInt8Ty(), SrcStr, "memchr.char0");
1382 // Slice off the character's high end bits.
1383 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1384 Value *Cmp = B.CreateICmpEQ(Val, CharVal, "memchr.char0cmp");
1385 // The condition depends on the value of the string being equal to the
1386 // query, neither of which we know without value profiling, so mark the
1387 // profile unknown.
1388 return B.CreateSelectWithUnknownProfile(Cmp, SrcStr, NullPtr, DEBUG_TYPE,
1389 "memchr.sel");
1390 }
1391 }
1392
1393 StringRef Str;
1394 if (!getConstantStringInfo(SrcStr, Str, /*TrimAtNul=*/false))
1395 return nullptr;
1396
1397 if (CharC) {
1398 size_t Pos = Str.find(CharC->getZExtValue());
1399 if (Pos == StringRef::npos)
1400 // When the character is not in the source array fold the result
1401 // to null regardless of Size.
1402 return NullPtr;
1403
1404 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1405 // When the constant Size is less than or equal to the character
1406 // position also fold the result to null.
1407 Value *Cmp = B.CreateICmpULE(Size, ConstantInt::get(Size->getType(), Pos),
1408 "memchr.cmp");
1409 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(Pos),
1410 "memchr.ptr");
1411 // The condition is dependent upon the value of n, which we cannot infer
1412 // without value profiling, so mark the profile unknown.
1413 return B.CreateSelectWithUnknownProfile(Cmp, NullPtr, SrcPlus, DEBUG_TYPE);
1414 }
1415
1416 if (Str.size() == 0)
1417 // If the array is empty fold memchr(A, C, N) to null for any value
1418 // of C and N on the basis that the only valid value of N is zero
1419 // (otherwise the call is undefined).
1420 return NullPtr;
1421
1422 if (LenC)
1423 Str = substr(Str, LenC->getZExtValue());
1424
1425 size_t Pos = Str.find_first_not_of(Str[0]);
1426 if (Pos == StringRef::npos
1427 || Str.find_first_not_of(Str[Pos], Pos) == StringRef::npos) {
1428 // If the source array consists of at most two consecutive sequences
1429 // of the same characters, then for any C and N (whether in bounds or
1430 // not), fold memchr(S, C, N) to
1431 // N != 0 && *S == C ? S : null
1432 // or for the two sequences to:
1433 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1434 // ^Sel2 ^Sel1 are denoted above.
1435 // The latter makes it also possible to fold strchr() calls with strings
1436 // of the same characters.
1437 Type *SizeTy = Size->getType();
1438 Type *Int8Ty = B.getInt8Ty();
1439
1440 // Slice off the sought character's high end bits.
1441 CharVal = B.CreateTrunc(CharVal, Int8Ty);
1442
1443 Value *Sel1 = NullPtr;
1444 if (Pos != StringRef::npos) {
1445 // Handle two consecutive sequences of the same characters.
1446 Value *PosVal = ConstantInt::get(SizeTy, Pos);
1447 Value *StrPos = ConstantInt::get(Int8Ty, (unsigned char)Str[Pos]);
1448 Value *CEqSPos = B.CreateICmpEQ(CharVal, StrPos);
1449 Value *NGtPos = B.CreateICmp(ICmpInst::ICMP_UGT, Size, PosVal);
1450 Value *And = B.CreateAnd(CEqSPos, NGtPos);
1451 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, PosVal);
1452 // The condition depends on the value of the query and size, neither of
1453 // which we know without value profiling, so mark the profile unknown.
1454 Sel1 = B.CreateSelectWithUnknownProfile(And, SrcPlus, NullPtr, DEBUG_TYPE,
1455 "memchr.sel1");
1456 }
1457
1458 Value *Str0 = ConstantInt::get(Int8Ty, (unsigned char)Str[0]);
1459 Value *CEqS0 = B.CreateICmpEQ(Str0, CharVal);
1460 Value *NNeZ = B.CreateICmpNE(Size, ConstantInt::get(SizeTy, 0));
1461 Value *And = B.CreateAnd(NNeZ, CEqS0);
1462 // The condition depends on the value of the query and size, neither of
1463 // which we know without value profiling, so mark the profile unknown.
1464 return B.CreateSelectWithUnknownProfile(And, SrcStr, Sel1, DEBUG_TYPE,
1465 "memchr.sel2");
1466 }
1467
1468 if (!LenC) {
1469 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
1470 // S is dereferenceable so it's safe to load from it and fold
1471 // memchr(S, C, N) == S to N && *S == C for any C and N.
1472 // TODO: This is safe even for nonconstant S.
1473 return memChrToCharCompare(CI, Size, B, DL);
1474
1475 // From now on we need a constant length and constant array.
1476 return nullptr;
1477 }
1478
1479 bool OptForSize = llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
1481
1482 // If the char is variable but the input str and length are not we can turn
1483 // this memchr call into a simple bit field test. Of course this only works
1484 // when the return value is only checked against null.
1485 //
1486 // It would be really nice to reuse switch lowering here but we can't change
1487 // the CFG at this point.
1488 //
1489 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1490 // != 0
1491 // after bounds check.
1492 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CI))
1493 return nullptr;
1494
1495 unsigned char Max =
1496 *std::max_element(reinterpret_cast<const unsigned char *>(Str.begin()),
1497 reinterpret_cast<const unsigned char *>(Str.end()));
1498
1499 // Make sure the bit field we're about to create fits in a register on the
1500 // target.
1501 // FIXME: On a 64 bit architecture this prevents us from using the
1502 // interesting range of alpha ascii chars. We could do better by emitting
1503 // two bitfields or shifting the range by 64 if no lower chars are used.
1504 if (!DL.fitsInLegalInteger(Max + 1)) {
1505 // Build chain of ORs
1506 // Transform:
1507 // memchr("abcd", C, 4) != nullptr
1508 // to:
1509 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1510 std::string SortedStr = Str.str();
1511 llvm::sort(SortedStr);
1512 // Compute the number of of non-contiguous ranges.
1513 unsigned NonContRanges = 1;
1514 for (size_t i = 1; i < SortedStr.size(); ++i) {
1515 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1516 NonContRanges++;
1517 }
1518 }
1519
1520 // Restrict this optimization to profitable cases with one or two range
1521 // checks.
1522 if (NonContRanges > 2)
1523 return nullptr;
1524
1525 // Slice off the character's high end bits.
1526 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1527
1528 SmallVector<Value *> CharCompares;
1529 for (unsigned char C : SortedStr)
1530 CharCompares.push_back(B.CreateICmpEQ(CharVal, B.getInt8(C)));
1531
1532 return B.CreateIntToPtr(B.CreateOr(CharCompares), CI->getType());
1533 }
1534
1535 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1536 // creating unnecessary illegal types.
1537 unsigned char Width = NextPowerOf2(std::max((unsigned char)7, Max));
1538
1539 // Now build the bit field.
1540 APInt Bitfield(Width, 0);
1541 for (char C : Str)
1542 Bitfield.setBit((unsigned char)C);
1543 Value *BitfieldC = B.getInt(Bitfield);
1544
1545 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1546 Value *C = B.CreateZExtOrTrunc(CharVal, BitfieldC->getType());
1547 C = B.CreateAnd(C, B.getIntN(Width, 0xFF));
1548
1549 // First check that the bit field access is within bounds.
1550 Value *Bounds = B.CreateICmp(ICmpInst::ICMP_ULT, C, B.getIntN(Width, Width),
1551 "memchr.bounds");
1552
1553 // Create code that checks if the given bit is set in the field.
1554 Value *Shl = B.CreateShl(B.getIntN(Width, 1ULL), C);
1555 Value *Bits = B.CreateIsNotNull(B.CreateAnd(Shl, BitfieldC), "memchr.bits");
1556
1557 // Finally merge both checks and cast to pointer type. The inttoptr
1558 // implicitly zexts the i1 to intptr type.
1559 Value *Memchr = B.CreateLogicalAnd(Bounds, Bits, "memchr");
1560 // We construct an and between the value of the memory and the bytes to search
1561 // for. We cannot infer how often this would be true without value profiling
1562 // for the query, so mark the profile unknown.
1563 if (auto *SI = dyn_cast<SelectInst>(Memchr))
1565 return B.CreateIntToPtr(Memchr, CI->getType());
1566}
1567
1568// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1569// arrays LHS and RHS and nonconstant Size.
1571 Value *Size, bool StrNCmp,
1572 IRBuilderBase &B, const DataLayout &DL) {
1573 if (LHS == RHS) // memcmp(s,s,x) -> 0
1574 return Constant::getNullValue(CI->getType());
1575
1576 StringRef LStr, RStr;
1577 if (!getConstantStringInfo(LHS, LStr, /*TrimAtNul=*/false) ||
1578 !getConstantStringInfo(RHS, RStr, /*TrimAtNul=*/false))
1579 return nullptr;
1580
1581 // If the contents of both constant arrays are known, fold a call to
1582 // memcmp(A, B, N) to
1583 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1584 // where Pos is the first mismatch between A and B, determined below.
1585
1586 uint64_t Pos = 0;
1587 Value *Zero = ConstantInt::get(CI->getType(), 0);
1588 for (uint64_t MinSize = std::min(LStr.size(), RStr.size()); ; ++Pos) {
1589 if (Pos == MinSize ||
1590 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1591 // One array is a leading part of the other of equal or greater
1592 // size, or for strncmp, the arrays are equal strings.
1593 // Fold the result to zero. Size is assumed to be in bounds, since
1594 // otherwise the call would be undefined.
1595 return Zero;
1596 }
1597
1598 if (LStr[Pos] != RStr[Pos])
1599 break;
1600 }
1601
1602 // Normalize the result.
1603 typedef unsigned char UChar;
1604 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1605 Value *MaxSize = ConstantInt::get(Size->getType(), Pos);
1606 Value *Cmp = B.CreateICmp(ICmpInst::ICMP_ULE, Size, MaxSize);
1607 Value *Res = ConstantInt::getSigned(CI->getType(), IRes);
1608 return B.CreateSelect(Cmp, Zero, Res);
1609}
1610
1611// Optimize a memcmp call CI with constant size Len.
1613 uint64_t Len, IRBuilderBase &B,
1614 const DataLayout &DL) {
1615 if (Len == 0) // memcmp(s1,s2,0) -> 0
1616 return Constant::getNullValue(CI->getType());
1617
1618 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1619 if (Len == 1) {
1620 Value *LHSV = B.CreateZExt(B.CreateLoad(B.getInt8Ty(), LHS, "lhsc"),
1621 CI->getType(), "lhsv");
1622 Value *RHSV = B.CreateZExt(B.CreateLoad(B.getInt8Ty(), RHS, "rhsc"),
1623 CI->getType(), "rhsv");
1624 return B.CreateSub(LHSV, RHSV, "chardiff");
1625 }
1626
1627 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1628 // TODO: The case where both inputs are constants does not need to be limited
1629 // to legal integers or equality comparison. See block below this.
1630 if (DL.isLegalInteger(Len * 8) && isOnlyUsedInZeroEqualityComparison(CI)) {
1631 IntegerType *IntType = IntegerType::get(CI->getContext(), Len * 8);
1632 Align PrefAlignment = DL.getPrefTypeAlign(IntType);
1633
1634 // First, see if we can fold either argument to a constant.
1635 Value *LHSV = nullptr;
1636 if (auto *LHSC = dyn_cast<Constant>(LHS))
1637 LHSV = ConstantFoldLoadFromConstPtr(LHSC, IntType, DL);
1638
1639 Value *RHSV = nullptr;
1640 if (auto *RHSC = dyn_cast<Constant>(RHS))
1641 RHSV = ConstantFoldLoadFromConstPtr(RHSC, IntType, DL);
1642
1643 // Don't generate unaligned loads. If either source is constant data,
1644 // alignment doesn't matter for that source because there is no load.
1645 if ((LHSV || getKnownAlignment(LHS, DL, CI) >= PrefAlignment) &&
1646 (RHSV || getKnownAlignment(RHS, DL, CI) >= PrefAlignment)) {
1647 if (!LHSV)
1648 LHSV = B.CreateLoad(IntType, LHS, "lhsv");
1649 if (!RHSV)
1650 RHSV = B.CreateLoad(IntType, RHS, "rhsv");
1651 return B.CreateZExt(B.CreateICmpNE(LHSV, RHSV), CI->getType(), "memcmp");
1652 }
1653 }
1654
1655 return nullptr;
1656}
1657
1658// Most simplifications for memcmp also apply to bcmp.
1659Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1660 IRBuilderBase &B) {
1661 Value *LHS = CI->getArgOperand(0), *RHS = CI->getArgOperand(1);
1662 Value *Size = CI->getArgOperand(2);
1663
1664 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1665
1666 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, false, B, DL))
1667 return Res;
1668
1669 // Handle constant Size.
1670 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1671 if (!LenC)
1672 return nullptr;
1673
1674 return optimizeMemCmpConstantSize(CI, LHS, RHS, LenC->getZExtValue(), B, DL);
1675}
1676
1677Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1678 Module *M = CI->getModule();
1679 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1680 return V;
1681
1682 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1683 // bcmp can be more efficient than memcmp because it only has to know that
1684 // there is a difference, not how different one is to the other.
1685 if (isLibFuncEmittable(M, TLI, LibFunc_bcmp) &&
1687 Value *LHS = CI->getArgOperand(0);
1688 Value *RHS = CI->getArgOperand(1);
1689 Value *Size = CI->getArgOperand(2);
1690 return copyFlags(*CI, emitBCmp(LHS, RHS, Size, B, DL, TLI));
1691 }
1692
1693 return nullptr;
1694}
1695
1696Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1697 return optimizeMemCmpBCmpCommon(CI, B);
1698}
1699
1700Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1701 Value *Size = CI->getArgOperand(2);
1702 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1703 if (isa<IntrinsicInst>(CI))
1704 return nullptr;
1705
1706 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1707 CallInst *NewCI = B.CreateMemCpy(CI->getArgOperand(0), Align(1),
1708 CI->getArgOperand(1), Align(1), Size);
1709 mergeAttributesAndFlags(NewCI, *CI);
1710 return CI->getArgOperand(0);
1711}
1712
1713Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1714 Value *Dst = CI->getArgOperand(0);
1715 Value *Src = CI->getArgOperand(1);
1716 ConstantInt *StopChar = dyn_cast<ConstantInt>(CI->getArgOperand(2));
1717 ConstantInt *N = dyn_cast<ConstantInt>(CI->getArgOperand(3));
1718 StringRef SrcStr;
1719 if (CI->use_empty() && Dst == Src)
1720 return Dst;
1721 // memccpy(d, s, c, 0) -> nullptr
1722 if (N) {
1723 if (N->isNullValue())
1724 return Constant::getNullValue(CI->getType());
1725 if (!getConstantStringInfo(Src, SrcStr, /*TrimAtNul=*/false) ||
1726 // TODO: Handle zeroinitializer.
1727 !StopChar)
1728 return nullptr;
1729 } else {
1730 return nullptr;
1731 }
1732
1733 // Wrap arg 'c' of type int to char
1734 size_t Pos = SrcStr.find(StopChar->getSExtValue() & 0xFF);
1735 if (Pos == StringRef::npos) {
1736 if (N->getZExtValue() <= SrcStr.size()) {
1737 copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1),
1738 CI->getArgOperand(3)));
1739 return Constant::getNullValue(CI->getType());
1740 }
1741 return nullptr;
1742 }
1743
1744 Value *NewN =
1745 ConstantInt::get(N->getType(), std::min(uint64_t(Pos + 1), N->getZExtValue()));
1746 // memccpy -> llvm.memcpy
1747 copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1), NewN));
1748 return Pos + 1 <= N->getZExtValue()
1749 ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, NewN)
1751}
1752
1753Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1754 Value *Dst = CI->getArgOperand(0);
1755 Value *N = CI->getArgOperand(2);
1756 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1757 CallInst *NewCI =
1758 B.CreateMemCpy(Dst, Align(1), CI->getArgOperand(1), Align(1), N);
1759 // Propagate attributes, but memcpy has no return value, so make sure that
1760 // any return attributes are compliant.
1761 // TODO: Attach return value attributes to the 1st operand to preserve them?
1762 mergeAttributesAndFlags(NewCI, *CI);
1763 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, N);
1764}
1765
1766Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1767 Value *Size = CI->getArgOperand(2);
1768 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1769 if (isa<IntrinsicInst>(CI))
1770 return nullptr;
1771
1772 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1773 CallInst *NewCI = B.CreateMemMove(CI->getArgOperand(0), Align(1),
1774 CI->getArgOperand(1), Align(1), Size);
1775 mergeAttributesAndFlags(NewCI, *CI);
1776 return CI->getArgOperand(0);
1777}
1778
1779Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1780 Value *Size = CI->getArgOperand(2);
1782 if (isa<IntrinsicInst>(CI))
1783 return nullptr;
1784
1785 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1786 Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
1787 CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val, Size, Align(1));
1788 mergeAttributesAndFlags(NewCI, *CI);
1789 return CI->getArgOperand(0);
1790}
1791
1792Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1794 Value *Malloc = emitMalloc(CI->getArgOperand(1), B, DL, TLI);
1795 if (auto *MallocCI = dyn_cast_or_null<CallInst>(Malloc))
1796 if (MDNode *MD = CI->getMetadata(LLVMContext::MD_alloc_token))
1797 MallocCI->setMetadata(LLVMContext::MD_alloc_token, MD);
1798 return copyFlags(*CI, Malloc);
1799 }
1800
1801 return nullptr;
1802}
1803
1804// Optionally allow optimization of nobuiltin calls to operator new and its
1805// variants.
1806Value *LibCallSimplifier::maybeOptimizeNoBuiltinOperatorNew(CallInst *CI,
1807 IRBuilderBase &B) {
1808 if (!OptimizeHotColdNew)
1809 return nullptr;
1811 if (!Callee)
1812 return nullptr;
1813 LibFunc Func = TLI->getLibFunc(*Callee);
1814 if (Func == NotLibFunc)
1815 return nullptr;
1816 switch (Func) {
1817 case LibFunc_Znwm:
1818 case LibFunc_ZnwmRKSt9nothrow_t:
1819 case LibFunc_ZnwmSt11align_val_t:
1820 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1821 case LibFunc_Znam:
1822 case LibFunc_ZnamRKSt9nothrow_t:
1823 case LibFunc_ZnamSt11align_val_t:
1824 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1825 case LibFunc_size_returning_new:
1826 case LibFunc_size_returning_new_aligned:
1827 // By default normal operator new calls (not already passing a hot_cold_t
1828 // parameter) are not mutated if the call is not marked builtin. Optionally
1829 // enable that in cases where it is known to be safe.
1831 return nullptr;
1832 break;
1833 case LibFunc_Znwm12__hot_cold_t:
1834 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1835 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1836 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1837 case LibFunc_Znam12__hot_cold_t:
1838 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1839 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1840 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1841 case LibFunc_size_returning_new_hot_cold:
1842 case LibFunc_size_returning_new_aligned_hot_cold:
1843 // If the nobuiltin call already passes a hot_cold_t parameter, allow update
1844 // of that parameter when enabled.
1846 return nullptr;
1847 break;
1848 default:
1849 return nullptr;
1850 }
1851 return optimizeNew(CI, B, Func);
1852}
1853
1854// When enabled, replace operator new() calls marked with a hot or cold memprof
1855// attribute with an operator new() call that takes a __hot_cold_t parameter.
1856// Currently this is supported by the open source version of tcmalloc, see:
1857// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1858Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1859 LibFunc &Func) {
1860 if (!OptimizeHotColdNew)
1861 return nullptr;
1862
1863 uint8_t HotCold;
1864 bool IsCold = false;
1865 if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "cold") {
1866 HotCold = ColdNewHintValue;
1867 IsCold = true;
1868 } else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() ==
1869 "notcold")
1870 HotCold = NotColdNewHintValue;
1871 else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "hot")
1872 HotCold = HotNewHintValue;
1873 else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() ==
1874 "ambiguous")
1875 HotCold = AmbiguousNewHintValue;
1876 else
1877 return nullptr;
1878
1879 bool ShouldOptimizeExistingHotColdNew =
1882 IsCold);
1883
1884 Value *HotColdVal = B.getInt8(HotCold);
1885 auto getHotColdHintForExisting = [&](uint8_t HotCold) -> Value * {
1886 // If not taking the minimum, simply use the compiler hint value.
1888 return HotColdVal;
1889 Value *ExistingHint = CI->getArgOperand(CI->arg_size() - 1);
1890 if (ExistingHint->getType() != B.getInt8Ty())
1891 ExistingHint = B.CreateTruncOrBitCast(ExistingHint, B.getInt8Ty());
1892 // Emit a umin intrinsic to take the minimum of the existing hint and the
1893 // compiler hint. When the existing hint is a compile-time constant, the
1894 // IRBuilder folder will automatically constant-fold this into a constant.
1895 return B.CreateBinaryIntrinsic(Intrinsic::umin, ExistingHint, HotColdVal);
1896 };
1897
1898 // For calls that already pass a hot/cold hint, only update the hint if
1899 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1900 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1901 // Note that in cases where we decide it is "notcold", it might be slightly
1902 // better to replace the hinted call with a non hinted call, to avoid the
1903 // extra parameter and the if condition check of the hint value in the
1904 // allocator. This can be considered in the future.
1905 Value *NewCall = nullptr;
1906 switch (Func) {
1907 case LibFunc_Znwm12__hot_cold_t:
1908 if (ShouldOptimizeExistingHotColdNew)
1909 NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
1910 LibFunc_Znwm12__hot_cold_t,
1911 getHotColdHintForExisting(HotCold));
1912 break;
1913 case LibFunc_Znwm:
1914 NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
1915 LibFunc_Znwm12__hot_cold_t, HotColdVal);
1916 break;
1917 case LibFunc_Znam12__hot_cold_t:
1918 if (ShouldOptimizeExistingHotColdNew)
1919 NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
1920 LibFunc_Znam12__hot_cold_t,
1921 getHotColdHintForExisting(HotCold));
1922 break;
1923 case LibFunc_Znam:
1924 NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
1925 LibFunc_Znam12__hot_cold_t, HotColdVal);
1926 break;
1927 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1928 if (ShouldOptimizeExistingHotColdNew)
1929 NewCall =
1931 TLI, LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1932 getHotColdHintForExisting(HotCold));
1933 break;
1934 case LibFunc_ZnwmRKSt9nothrow_t:
1935 NewCall = emitHotColdNewNoThrow(
1936 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1937 LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotColdVal);
1938 break;
1939 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1940 if (ShouldOptimizeExistingHotColdNew)
1941 NewCall =
1943 TLI, LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1944 getHotColdHintForExisting(HotCold));
1945 break;
1946 case LibFunc_ZnamRKSt9nothrow_t:
1947 NewCall = emitHotColdNewNoThrow(
1948 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1949 LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotColdVal);
1950 break;
1951 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1952 if (ShouldOptimizeExistingHotColdNew)
1953 NewCall =
1955 TLI, LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1956 getHotColdHintForExisting(HotCold));
1957 break;
1958 case LibFunc_ZnwmSt11align_val_t:
1959 NewCall = emitHotColdNewAligned(
1960 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1961 LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotColdVal);
1962 break;
1963 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1964 if (ShouldOptimizeExistingHotColdNew)
1965 NewCall =
1967 TLI, LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1968 getHotColdHintForExisting(HotCold));
1969 break;
1970 case LibFunc_ZnamSt11align_val_t:
1971 NewCall = emitHotColdNewAligned(
1972 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1973 LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotColdVal);
1974 break;
1975 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1976 if (ShouldOptimizeExistingHotColdNew)
1978 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1979 TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1980 getHotColdHintForExisting(HotCold));
1981 break;
1982 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1984 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1985 TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1986 HotColdVal);
1987 break;
1988 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1989 if (ShouldOptimizeExistingHotColdNew)
1991 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1992 TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1993 getHotColdHintForExisting(HotCold));
1994 break;
1995 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1997 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1998 TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1999 HotColdVal);
2000 break;
2001 case LibFunc_size_returning_new:
2002 NewCall = emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
2003 LibFunc_size_returning_new_hot_cold,
2004 HotColdVal, CI->getType());
2005 break;
2006 case LibFunc_size_returning_new_hot_cold:
2007 if (ShouldOptimizeExistingHotColdNew)
2009 CI->getArgOperand(0), B, TLI, LibFunc_size_returning_new_hot_cold,
2010 getHotColdHintForExisting(HotCold), CI->getType());
2011 break;
2012 case LibFunc_size_returning_new_aligned:
2014 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
2015 LibFunc_size_returning_new_aligned_hot_cold, HotColdVal, CI->getType());
2016 break;
2017 case LibFunc_size_returning_new_aligned_hot_cold:
2018 if (ShouldOptimizeExistingHotColdNew)
2020 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
2021 LibFunc_size_returning_new_aligned_hot_cold,
2022 getHotColdHintForExisting(HotCold), CI->getType());
2023 break;
2024 default:
2025 return nullptr;
2026 }
2027
2028 if (auto *NewCI = dyn_cast_or_null<Instruction>(NewCall))
2029 NewCI->copyMetadata(*CI);
2030
2031 return NewCall;
2032}
2033
2034//===----------------------------------------------------------------------===//
2035// Math Library Optimizations
2036//===----------------------------------------------------------------------===//
2037
2038/// Preserve the accuracy requirement of \p Old on the replacement \p New.
2039static void copyFPMath(const CallInst &Old, Value *New) {
2040 if (auto *NewI = dyn_cast<Instruction>(New))
2041 if (MDNode *MD = Old.getMetadata(LLVMContext::MD_fpmath))
2042 NewI->setMetadata(LLVMContext::MD_fpmath, MD);
2043}
2044
2045// Replace a libcall \p CI with a call to intrinsic \p IID
2047 Intrinsic::ID IID) {
2048 Value *NewCall = B.CreateUnaryIntrinsic(IID, CI->getArgOperand(0), CI);
2049 NewCall->takeName(CI);
2050 copyFPMath(*CI, NewCall);
2051 return copyFlags(*CI, NewCall);
2052}
2053
2055 Intrinsic::ID IID) {
2056 Value *NewCall = B.CreateBinaryIntrinsic(IID, CI->getArgOperand(0),
2057 CI->getArgOperand(1), CI);
2058 NewCall->takeName(CI);
2059 copyFPMath(*CI, NewCall);
2060 return copyFlags(*CI, NewCall);
2061}
2062
2063/// Return a variant of Val with float type.
2064/// Currently this works in two cases: If Val is an FPExtension of a float
2065/// value to something bigger, simply return the operand.
2066/// If Val is a ConstantFP but can be converted to a float ConstantFP without
2067/// loss of precision do so.
2069 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
2070 Value *Op = Cast->getOperand(0);
2071 if (Op->getType()->isFloatTy())
2072 return Op;
2073 }
2074 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
2075 APFloat F = Const->getValueAPF();
2076 bool losesInfo;
2078 &losesInfo);
2079 if (!losesInfo)
2080 return ConstantFP::get(Const->getContext(), F);
2081 }
2082 return nullptr;
2083}
2084
2085/// Shrink double -> float functions.
2087 bool isBinary, const TargetLibraryInfo *TLI,
2088 bool isPrecise = false) {
2089 Function *CalleeFn = CI->getCalledFunction();
2090 if (!CI->getType()->isDoubleTy() || !CalleeFn)
2091 return nullptr;
2092
2093 // If not all the uses of the function are converted to float, then bail out.
2094 // This matters if the precision of the result is more important than the
2095 // precision of the arguments.
2096 if (isPrecise)
2097 for (User *U : CI->users()) {
2099 if (!Cast || !Cast->getType()->isFloatTy())
2100 return nullptr;
2101 }
2102
2103 // If this is something like 'g((double) float)', convert to 'gf(float)'.
2104 Value *V[2];
2106 V[1] = isBinary ? valueHasFloatPrecision(CI->getArgOperand(1)) : nullptr;
2107 if (!V[0] || (isBinary && !V[1]))
2108 return nullptr;
2109
2110 // If call isn't an intrinsic, check that it isn't within a function with the
2111 // same name as the float version of this call, otherwise the result is an
2112 // infinite loop. For example, from MinGW-w64:
2113 //
2114 // float expf(float val) { return (float) exp((double) val); }
2115 StringRef CalleeName = CalleeFn->getName();
2116 bool IsIntrinsic = CalleeFn->isIntrinsic();
2117 if (!IsIntrinsic) {
2118 StringRef CallerName = CI->getFunction()->getName();
2119 if (CallerName.ends_with('f') &&
2120 CallerName.size() == (CalleeName.size() + 1) &&
2121 CallerName.starts_with(CalleeName))
2122 return nullptr;
2123 }
2124
2125 // Propagate the math semantics from the current function to the new function.
2127 B.setFastMathFlags(CI->getFastMathFlags());
2128
2129 // g((double) float) -> (double) gf(float)
2130 Value *R;
2131 if (IsIntrinsic) {
2132 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
2133 R = isBinary ? B.CreateIntrinsic(IID, B.getFloatTy(), V)
2134 : B.CreateIntrinsic(IID, B.getFloatTy(), V[0]);
2135 } else {
2136 AttributeList CallsiteAttrs = CI->getAttributes();
2137 R = isBinary
2138 ? emitBinaryFloatFnCall(V[0], V[1], TLI, CalleeName, B,
2139 CallsiteAttrs)
2140 : emitUnaryFloatFnCall(V[0], TLI, CalleeName, B, CallsiteAttrs);
2141 }
2142 return B.CreateFPExt(R, B.getDoubleTy());
2143}
2144
2145/// Shrink double -> float for unary functions.
2147 const TargetLibraryInfo *TLI,
2148 bool isPrecise = false) {
2149 return optimizeDoubleFP(CI, B, false, TLI, isPrecise);
2150}
2151
2152/// Shrink double -> float for binary functions.
2154 const TargetLibraryInfo *TLI,
2155 bool isPrecise = false) {
2156 return optimizeDoubleFP(CI, B, true, TLI, isPrecise);
2157}
2158
2159/// Shrink double -> float for llvm.sincos.
2161 auto *RetTy = dyn_cast<StructType>(CI->getType());
2162 if (!RetTy || RetTy->getNumElements() != 2 ||
2163 !RetTy->getElementType(0)->getScalarType()->isDoubleTy())
2164 return nullptr;
2165
2167 if (!X)
2168 if (auto *Ext = dyn_cast<FPExtInst>(CI->getArgOperand(0)))
2169 if (Ext->getOperand(0)->getType()->getScalarType()->isFloatTy())
2170 X = Ext->getOperand(0);
2171 if (!X)
2172 return nullptr;
2173
2174 for (User *U : CI->users()) {
2175 auto *EV = dyn_cast<ExtractValueInst>(U);
2176 if (!EV)
2177 return nullptr;
2178 for (User *EVU : EV->users()) {
2179 auto *Cast = dyn_cast<FPTruncInst>(EVU);
2180 if (!Cast || !Cast->getType()->getScalarType()->isFloatTy())
2181 return nullptr;
2182 }
2183 }
2184
2186 B.setFastMathFlags(CI->getFastMathFlags());
2187
2188 Value *NewCall = B.CreateIntrinsic(Intrinsic::sincos, X->getType(), X);
2189 cast<Instruction>(NewCall)->setMetadata(
2190 LLVMContext::MD_fpmath, CI->getMetadata(LLVMContext::MD_fpmath));
2191 Value *Res = PoisonValue::get(RetTy);
2192 for (unsigned I = 0; I != 2; ++I) {
2193 Value *Ext = B.CreateFPExt(B.CreateExtractValue(NewCall, I),
2194 RetTy->getElementType(I));
2195 Res = B.CreateInsertValue(Res, Ext, I);
2196 }
2197 return Res;
2198}
2199
2200// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
2201Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
2202 Value *Real, *Imag;
2203
2204 if (CI->arg_size() == 1) {
2205
2206 if (!CI->isFast())
2207 return nullptr;
2208
2209 Value *Op = CI->getArgOperand(0);
2210 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
2211
2212 Real = B.CreateExtractValue(Op, 0, "real");
2213 Imag = B.CreateExtractValue(Op, 1, "imag");
2214
2215 } else {
2216 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
2217
2218 Real = CI->getArgOperand(0);
2219 Imag = CI->getArgOperand(1);
2220
2221 // if real or imaginary part is zero, simplify to abs(cimag(z))
2222 // or abs(creal(z))
2223 Value *AbsOp = nullptr;
2224 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Real)) {
2225 if (ConstReal->isZero())
2226 AbsOp = Imag;
2227
2228 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Imag)) {
2229 if (ConstImag->isZero())
2230 AbsOp = Real;
2231 }
2232
2233 if (AbsOp)
2234 return copyFlags(*CI, B.CreateFAbs(AbsOp, CI, "cabs"));
2235
2236 if (!CI->isFast())
2237 return nullptr;
2238 }
2239
2240 // Propagate fast-math flags from the existing call to new instructions.
2241 Value *RealReal = B.CreateFMulFMF(Real, Real, CI);
2242 Value *ImagImag = B.CreateFMulFMF(Imag, Imag, CI);
2243 return copyFlags(
2244 *CI, B.CreateUnaryIntrinsic(Intrinsic::sqrt,
2245 B.CreateFAddFMF(RealReal, ImagImag, CI), CI,
2246 "cabs"));
2247}
2248
2249// Return a properly extended integer (DstWidth bits wide) if the operation is
2250// an itofp.
2251static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2252 if (isa<SIToFPInst>(I2F) || isa<UIToFPInst>(I2F)) {
2253 Value *Op = cast<Instruction>(I2F)->getOperand(0);
2254 // Make sure that the exponent fits inside an "int" of size DstWidth,
2255 // thus avoiding any range issues that FP has not.
2256 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2257 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(I2F))) {
2258 Type *IntTy = Op->getType()->getWithNewBitWidth(DstWidth);
2259 return isa<SIToFPInst>(I2F) ? B.CreateSExt(Op, IntTy)
2260 : B.CreateZExt(Op, IntTy);
2261 }
2262 }
2263
2264 return nullptr;
2265}
2266
2267/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2268/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2269/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2270Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2271 Module *M = Pow->getModule();
2272 Value *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1);
2273 Type *Ty = Pow->getType();
2274 bool Ignored;
2275
2276 // Evaluate special cases related to a nested function as the base.
2277
2278 // pow(exp(x), y) -> exp(x * y)
2279 // pow(exp2(x), y) -> exp2(x * y)
2280 // If exp{,2}() is used only once, it is better to fold two transcendental
2281 // math functions into one. If used again, exp{,2}() would still have to be
2282 // called with the original argument, then keep both original transcendental
2283 // functions. However, this transformation is only safe with fully relaxed
2284 // math semantics, since, besides rounding differences, it changes overflow
2285 // and underflow behavior quite dramatically. For example:
2286 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2287 // Whereas:
2288 // exp(1000 * 0.001) = exp(1)
2289 // TODO: Loosen the requirement for fully relaxed math semantics.
2290 // TODO: Handle exp10() when more targets have it available.
2291 CallInst *BaseFn = dyn_cast<CallInst>(Base);
2292 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2293 Function *CalleeFn = BaseFn->getCalledFunction();
2294 LibFunc LibFn =
2295 CalleeFn ? TLI->getLibFunc(CalleeFn->getName()) : NotLibFunc;
2296 if (isLibFuncEmittable(M, TLI, LibFn)) {
2297 StringRef ExpName;
2299 Value *ExpFn;
2300 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2301
2302 switch (LibFn) {
2303 default:
2304 return nullptr;
2305 case LibFunc_expf:
2306 case LibFunc_exp:
2307 case LibFunc_expl:
2308 ExpName = TLI->getName(LibFunc_exp);
2309 ID = Intrinsic::exp;
2310 LibFnFloat = LibFunc_expf;
2311 LibFnDouble = LibFunc_exp;
2312 LibFnLongDouble = LibFunc_expl;
2313 break;
2314 case LibFunc_exp2f:
2315 case LibFunc_exp2:
2316 case LibFunc_exp2l:
2317 ExpName = TLI->getName(LibFunc_exp2);
2318 ID = Intrinsic::exp2;
2319 LibFnFloat = LibFunc_exp2f;
2320 LibFnDouble = LibFunc_exp2;
2321 LibFnLongDouble = LibFunc_exp2l;
2322 break;
2323 }
2324
2325 // Create new exp{,2}() with the product as its argument.
2326 Value *FMul = B.CreateFMul(BaseFn->getArgOperand(0), Expo, "mul");
2327 ExpFn = BaseFn->doesNotAccessMemory()
2328 ? B.CreateUnaryIntrinsic(ID, FMul, nullptr, ExpName)
2329 : emitUnaryFloatFnCall(FMul, TLI, LibFnDouble, LibFnFloat,
2330 LibFnLongDouble, B,
2331 BaseFn->getAttributes());
2332
2333 // Since the new exp{,2}() is different from the original one, dead code
2334 // elimination cannot be trusted to remove it, since it may have side
2335 // effects (e.g., errno). When the only consumer for the original
2336 // exp{,2}() is pow(), then it has to be explicitly erased.
2337 substituteInParent(BaseFn, ExpFn);
2338 return ExpFn;
2339 }
2340 }
2341
2342 // Evaluate special cases related to a constant base.
2343
2344 const APFloat *BaseF;
2345 if (!match(Base, m_APFloat(BaseF)))
2346 return nullptr;
2347
2348 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2349
2350 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2351
2352 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2353 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(2.0) &&
2354 (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo)) &&
2355 (UseIntrinsic ||
2356 hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl))) {
2357
2358 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2359 // just directly use the original integer type.
2360 if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize())) {
2361 Constant *One = ConstantFP::get(Ty, 1.0);
2362
2363 if (UseIntrinsic) {
2364 return copyFlags(*Pow, B.CreateIntrinsic(Intrinsic::ldexp,
2365 {Ty, ExpoI->getType()},
2366 {One, ExpoI}, Pow, "exp2"));
2367 }
2368
2370 One, ExpoI, TLI, LibFunc_ldexp, LibFunc_ldexpf,
2371 LibFunc_ldexpl, B, NoAttrs));
2372 }
2373 }
2374
2375 // pow(2.0 ** n, x) -> exp2(n * x)
2376 if (hasFloatFn(M, TLI, Ty, LibFunc_exp2, LibFunc_exp2f, LibFunc_exp2l)) {
2377 APFloat BaseR = APFloat(1.0);
2378 BaseR.convert(BaseF->getSemantics(), APFloat::rmTowardZero, &Ignored);
2379 BaseR = BaseR / *BaseF;
2380 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2381 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2382 APSInt NI(64, false);
2383 if ((IsInteger || IsReciprocal) &&
2384 NF->convertToInteger(NI, APFloat::rmTowardZero, &Ignored) ==
2385 APFloat::opOK &&
2386 NI > 1 && NI.isPowerOf2()) {
2387 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2388 Value *FMul = B.CreateFMul(Expo, ConstantFP::get(Ty, N), "mul");
2389 if (Pow->doesNotAccessMemory())
2390 return copyFlags(*Pow, B.CreateUnaryIntrinsic(Intrinsic::exp2, FMul,
2391 nullptr, "exp2"));
2392 else
2393 return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
2394 LibFunc_exp2f,
2395 LibFunc_exp2l, B, NoAttrs));
2396 }
2397 }
2398
2399 // pow(10.0, x) -> exp10(x)
2400 if (BaseF->isExactlyValue(10.0) &&
2401 hasFloatFn(M, TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l)) {
2402
2403 if (Pow->doesNotAccessMemory()) {
2404 return B.CreateIntrinsic(Intrinsic::exp10, {Ty}, {Expo}, Pow, "exp10", {},
2405 [Pow](CallInst *CI) { CI->copyIRFlags(Pow); });
2406 }
2407
2408 return copyFlags(*Pow, emitUnaryFloatFnCall(Expo, TLI, LibFunc_exp10,
2409 LibFunc_exp10f, LibFunc_exp10l,
2410 B, NoAttrs));
2411 }
2412
2413 // pow(x, y) -> exp2(log2(x) * y)
2414 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2415 !BaseF->isNegative()) {
2416 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2417 // Luckily optimizePow has already handled the x == 1 case.
2418 assert(!match(Base, m_FPOne()) &&
2419 "pow(1.0, y) should have been simplified earlier!");
2420
2421 Value *Log = nullptr;
2422 if (Ty->isFloatTy())
2423 Log = ConstantFP::get(Ty, std::log2(BaseF->convertToFloat()));
2424 else if (Ty->isDoubleTy())
2425 Log = ConstantFP::get(Ty, std::log2(BaseF->convertToDouble()));
2426
2427 if (Log) {
2428 Value *FMul = B.CreateFMul(Log, Expo, "mul");
2429 if (Pow->doesNotAccessMemory())
2430 return copyFlags(*Pow, B.CreateUnaryIntrinsic(Intrinsic::exp2, FMul,
2431 nullptr, "exp2"));
2432 else if (hasFloatFn(M, TLI, Ty, LibFunc_exp2, LibFunc_exp2f,
2433 LibFunc_exp2l))
2434 return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
2435 LibFunc_exp2f,
2436 LibFunc_exp2l, B, NoAttrs));
2437 }
2438 }
2439
2440 return nullptr;
2441}
2442
2443static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2444 Module *M, IRBuilderBase &B,
2445 const TargetLibraryInfo *TLI) {
2446 // If errno is never set, then use the intrinsic for sqrt().
2447 if (NoErrno)
2448 return B.CreateUnaryIntrinsic(Intrinsic::sqrt, V, nullptr, "sqrt");
2449
2450 // Otherwise, use the libcall for sqrt().
2451 if (hasFloatFn(M, TLI, V->getType(), LibFunc_sqrt, LibFunc_sqrtf,
2452 LibFunc_sqrtl))
2453 // TODO: We also should check that the target can in fact lower the sqrt()
2454 // libcall. We currently have no way to ask this question, so we ask if
2455 // the target has a sqrt() libcall, which is not exactly the same.
2456 return emitUnaryFloatFnCall(V, TLI, LibFunc_sqrt, LibFunc_sqrtf,
2457 LibFunc_sqrtl, B, Attrs);
2458
2459 return nullptr;
2460}
2461
2462/// Use square root in place of pow(x, +/-0.5).
2463Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2464 Value *Sqrt, *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1);
2465 Module *Mod = Pow->getModule();
2466 Type *Ty = Pow->getType();
2467
2468 const APFloat *ExpoF;
2469 if (!match(Expo, m_APFloat(ExpoF)) ||
2470 (!ExpoF->isExactlyValue(0.5) && !ExpoF->isExactlyValue(-0.5)))
2471 return nullptr;
2472
2473 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2474 // so that requires fast-math-flags (afn or reassoc).
2475 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2476 return nullptr;
2477
2478 // If we have a pow() library call (accesses memory) and we can't guarantee
2479 // that the base is not an infinity, give up:
2480 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2481 // errno), but sqrt(-Inf) is required by various standards to set errno.
2482 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2484 Base, SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2485 return nullptr;
2486
2487 Sqrt = getSqrtCall(Base, AttributeList(), Pow->doesNotAccessMemory(), Mod, B,
2488 TLI);
2489 if (!Sqrt)
2490 return nullptr;
2491
2492 // Handle signed zero base by expanding to fabs(sqrt(x)).
2493 if (!Pow->hasNoSignedZeros())
2494 Sqrt = B.CreateFAbs(Sqrt, nullptr, "abs");
2495
2496 Sqrt = copyFlags(*Pow, Sqrt);
2497
2498 // Handle non finite base by expanding to
2499 // (x == -infinity ? +infinity : sqrt(x)).
2500 if (!Pow->hasNoInfs()) {
2501 Value *PosInf = ConstantFP::getInfinity(Ty),
2502 *NegInf = ConstantFP::getInfinity(Ty, true);
2503 Value *FCmp = B.CreateFCmpOEQ(Base, NegInf, "isinf");
2504 Sqrt = B.CreateSelect(FCmp, PosInf, Sqrt);
2505 // We assume that the case where x == -infinity is unlikely, so we assign
2506 // unlikely branch weights to that arm of the select.
2508 if (auto *SqrtSI = dyn_cast<SelectInst>(Sqrt))
2510 *SqrtSI,
2512 /*IsExpected=*/false);
2513 }
2514 }
2515
2516 // If the exponent is negative, then get the reciprocal.
2517 if (ExpoF->isNegative())
2518 Sqrt = B.CreateFDiv(ConstantFP::get(Ty, 1.0), Sqrt, "reciprocal");
2519
2520 return Sqrt;
2521}
2522
2524 IRBuilderBase &B) {
2525 Value *Args[] = {Base, Expo};
2526 Type *Types[] = {Base->getType(), Expo->getType()};
2527 return B.CreateIntrinsic(Intrinsic::powi, Types, Args);
2528}
2529
2530Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2531 Value *Base = Pow->getArgOperand(0);
2532 Value *Expo = Pow->getArgOperand(1);
2533 Function *Callee = Pow->getCalledFunction();
2534 StringRef Name = Callee->getName();
2535 Type *Ty = Pow->getType();
2536 Module *M = Pow->getModule();
2537 bool AllowApprox = Pow->hasApproxFunc();
2538 bool Ignored;
2539
2540 // Propagate the math semantics from the call to any created instructions.
2541 IRBuilderBase::FastMathFlagGuard Guard(B);
2542 B.setFastMathFlags(Pow->getFastMathFlags());
2543 // Evaluate special cases related to the base.
2544
2545 // pow(1.0, x) -> 1.0
2546 if (match(Base, m_FPOne()))
2547 return Base;
2548
2549 if (Value *Exp = replacePowWithExp(Pow, B))
2550 return Exp;
2551
2552 // Evaluate special cases related to the exponent.
2553
2554 // pow(x, -1.0) -> 1.0 / x
2555 if (match(Expo, m_SpecificFP(-1.0)))
2556 return B.CreateFDiv(ConstantFP::get(Ty, 1.0), Base, "reciprocal");
2557
2558 // pow(x, +/-0.0) -> 1.0
2559 if (match(Expo, m_AnyZeroFP()))
2560 return ConstantFP::get(Ty, 1.0);
2561
2562 // pow(x, 1.0) -> x
2563 if (match(Expo, m_FPOne()))
2564 return Base;
2565
2566 // pow(x, 2.0) -> x * x
2567 if (match(Expo, m_SpecificFP(2.0)) && Pow->doesNotAccessMemory())
2568 return B.CreateFMul(Base, Base, "square");
2569
2570 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2571 return Sqrt;
2572
2573 // If we can approximate pow:
2574 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2575 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2576 const APFloat *ExpoF;
2577 if (AllowApprox && match(Expo, m_APFloat(ExpoF)) &&
2578 !ExpoF->isExactlyValue(0.5) && !ExpoF->isExactlyValue(-0.5)) {
2579 APFloat ExpoA(abs(*ExpoF));
2580 APFloat ExpoI(*ExpoF);
2581 Value *Sqrt = nullptr;
2582 if (!ExpoA.isInteger()) {
2583 APFloat Expo2 = ExpoA;
2584 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2585 // is no floating point exception and the result is an integer, then
2586 // ExpoA == integer + 0.5
2587 if (Expo2.add(ExpoA, APFloat::rmNearestTiesToEven) != APFloat::opOK)
2588 return nullptr;
2589
2590 if (!Expo2.isInteger())
2591 return nullptr;
2592
2593 if (ExpoI.roundToIntegral(APFloat::rmTowardNegative) !=
2595 return nullptr;
2596 if (!ExpoI.isInteger())
2597 return nullptr;
2598 ExpoF = &ExpoI;
2599
2600 Sqrt = getSqrtCall(Base, AttributeList(), Pow->doesNotAccessMemory(), M,
2601 B, TLI);
2602 if (!Sqrt)
2603 return nullptr;
2604 }
2605
2606 // 0.5 fraction is now optionally handled.
2607 // Do pow -> powi for remaining integer exponent
2608 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2609 if (ExpoF->isInteger() &&
2610 ExpoF->convertToInteger(IntExpo, APFloat::rmTowardZero, &Ignored) ==
2611 APFloat::opOK) {
2612 Value *PowI = copyFlags(
2613 *Pow,
2615 Base, ConstantInt::get(B.getIntNTy(TLI->getIntSize()), IntExpo),
2616 M, B));
2617
2618 if (PowI && Sqrt)
2619 return B.CreateFMul(PowI, Sqrt);
2620
2621 return PowI;
2622 }
2623 }
2624
2625 // powf(x, itofp(y)) -> powi(x, y)
2626 // The powi exponent must be a scalar integer, so a vector y is not usable.
2627 if (AllowApprox && !Expo->getType()->isVectorTy() &&
2628 (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo))) {
2629 if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize()))
2630 return copyFlags(*Pow, createPowWithIntegerExponent(Base, ExpoI, M, B));
2631 }
2632
2633 // Shrink pow() to powf() if the arguments are single precision,
2634 // unless the result is expected to be double precision.
2635 if (UnsafeFPShrink && Name == TLI->getName(LibFunc_pow) &&
2636 hasFloatVersion(M, Name)) {
2637 if (Value *Shrunk = optimizeBinaryDoubleFP(Pow, B, TLI, true))
2638 return Shrunk;
2639 }
2640
2641 return nullptr;
2642}
2643
2644Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2645 Module *M = CI->getModule();
2647 StringRef Name = Callee->getName();
2648 Value *Ret = nullptr;
2649 if (UnsafeFPShrink && Name == TLI->getName(LibFunc_exp2) &&
2650 hasFloatVersion(M, Name))
2651 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2652
2653 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2654 // have the libcall, emit the libcall.
2655 //
2656 // TODO: In principle we should be able to just always use the intrinsic for
2657 // any doesNotAccessMemory callsite.
2658
2659 const bool UseIntrinsic = Callee->isIntrinsic();
2660 // Bail out for vectors because the code below only expects scalars.
2661 Type *Ty = CI->getType();
2662 if (!UseIntrinsic && Ty->isVectorTy())
2663 return Ret;
2664
2665 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2666 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2667 Value *Op = CI->getArgOperand(0);
2668 if ((isa<SIToFPInst>(Op) || isa<UIToFPInst>(Op)) &&
2669 (UseIntrinsic ||
2670 hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl))) {
2671 if (Value *Exp = getIntToFPVal(Op, B, TLI->getIntSize())) {
2672 Constant *One = ConstantFP::get(Ty, 1.0);
2673
2674 if (UseIntrinsic) {
2675 return copyFlags(*CI, B.CreateIntrinsic(Intrinsic::ldexp,
2676 {Ty, Exp->getType()},
2677 {One, Exp}, CI));
2678 }
2679
2680 IRBuilderBase::FastMathFlagGuard Guard(B);
2681 B.setFastMathFlags(CI->getFastMathFlags());
2682 return copyFlags(*CI, emitBinaryFloatFnCall(
2683 One, Exp, TLI, LibFunc_ldexp, LibFunc_ldexpf,
2684 LibFunc_ldexpl, B, AttributeList()));
2685 }
2686 }
2687
2688 return Ret;
2689}
2690
2691Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B,
2692 Intrinsic::ID IID) {
2693 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2694 // the intrinsics for improved optimization (for example, vectorization).
2695 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2696 // From the C standard draft WG14/N1256:
2697 // "Ideally, fmax would be sensitive to the sign of zero, for example
2698 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2699 // might be impractical."
2700 FastMathFlags FMF = CI->getFastMathFlags();
2701 FMF.setNoSignedZeros();
2702 return copyFlags(*CI, B.CreateBinaryIntrinsic(IID, CI->getArgOperand(0),
2703 CI->getArgOperand(1), FMF));
2704}
2705
2706Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2707 Function *LogFn = Log->getCalledFunction();
2708 StringRef LogNm = LogFn->getName();
2709 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2710 Module *Mod = Log->getModule();
2711 Type *Ty = Log->getType();
2712
2713 if (UnsafeFPShrink && (LogFn->isIntrinsic() || hasFloatVersion(Mod, LogNm)))
2714 if (Value *Ret = optimizeUnaryDoubleFP(Log, B, TLI, true))
2715 return Ret;
2716
2717 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2718
2719 // This is only applicable to log(), log2(), log10().
2720 LogLb = TLI->getLibFunc(LogNm);
2721 if (LogLb != NotLibFunc) {
2722 switch (LogLb) {
2723 case LibFunc_logf:
2724 LogID = Intrinsic::log;
2725 ExpLb = LibFunc_expf;
2726 Exp2Lb = LibFunc_exp2f;
2727 Exp10Lb = LibFunc_exp10f;
2728 PowLb = LibFunc_powf;
2729 break;
2730 case LibFunc_log:
2731 LogID = Intrinsic::log;
2732 ExpLb = LibFunc_exp;
2733 Exp2Lb = LibFunc_exp2;
2734 Exp10Lb = LibFunc_exp10;
2735 PowLb = LibFunc_pow;
2736 break;
2737 case LibFunc_logl:
2738 LogID = Intrinsic::log;
2739 ExpLb = LibFunc_expl;
2740 Exp2Lb = LibFunc_exp2l;
2741 Exp10Lb = LibFunc_exp10l;
2742 PowLb = LibFunc_powl;
2743 break;
2744 case LibFunc_log2f:
2745 LogID = Intrinsic::log2;
2746 ExpLb = LibFunc_expf;
2747 Exp2Lb = LibFunc_exp2f;
2748 Exp10Lb = LibFunc_exp10f;
2749 PowLb = LibFunc_powf;
2750 break;
2751 case LibFunc_log2:
2752 LogID = Intrinsic::log2;
2753 ExpLb = LibFunc_exp;
2754 Exp2Lb = LibFunc_exp2;
2755 Exp10Lb = LibFunc_exp10;
2756 PowLb = LibFunc_pow;
2757 break;
2758 case LibFunc_log2l:
2759 LogID = Intrinsic::log2;
2760 ExpLb = LibFunc_expl;
2761 Exp2Lb = LibFunc_exp2l;
2762 Exp10Lb = LibFunc_exp10l;
2763 PowLb = LibFunc_powl;
2764 break;
2765 case LibFunc_log10f:
2766 LogID = Intrinsic::log10;
2767 ExpLb = LibFunc_expf;
2768 Exp2Lb = LibFunc_exp2f;
2769 Exp10Lb = LibFunc_exp10f;
2770 PowLb = LibFunc_powf;
2771 break;
2772 case LibFunc_log10:
2773 LogID = Intrinsic::log10;
2774 ExpLb = LibFunc_exp;
2775 Exp2Lb = LibFunc_exp2;
2776 Exp10Lb = LibFunc_exp10;
2777 PowLb = LibFunc_pow;
2778 break;
2779 case LibFunc_log10l:
2780 LogID = Intrinsic::log10;
2781 ExpLb = LibFunc_expl;
2782 Exp2Lb = LibFunc_exp2l;
2783 Exp10Lb = LibFunc_exp10l;
2784 PowLb = LibFunc_powl;
2785 break;
2786 default:
2787 return nullptr;
2788 }
2789
2790 // Convert libcall to intrinsic if the value is known > 0.
2791 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2792 if (!IsKnownNoErrno) {
2793 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2794 KnownFPClass Known = computeKnownFPClass(
2795 Log->getOperand(0),
2797 Function *F = Log->getParent()->getParent();
2798 const fltSemantics &FltSem = Ty->getScalarType()->getFltSemantics();
2799 IsKnownNoErrno =
2800 Known.cannotBeOrderedLessThanZero() &&
2801 Known.isKnownNeverLogicalZero(F->getDenormalMode(FltSem));
2802 }
2803 if (IsKnownNoErrno) {
2804 Value *NewLog = B.CreateUnaryIntrinsic(LogID, Log->getArgOperand(0), Log);
2805 if (auto *I = dyn_cast<Instruction>(NewLog)) {
2806 I->copyMetadata(*Log);
2807 return copyFlags(*Log, I);
2808 }
2809 return NewLog;
2810 }
2811 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2812 LogID == Intrinsic::log10) {
2813 if (Ty->getScalarType()->isFloatTy()) {
2814 ExpLb = LibFunc_expf;
2815 Exp2Lb = LibFunc_exp2f;
2816 Exp10Lb = LibFunc_exp10f;
2817 PowLb = LibFunc_powf;
2818 } else if (Ty->getScalarType()->isDoubleTy()) {
2819 ExpLb = LibFunc_exp;
2820 Exp2Lb = LibFunc_exp2;
2821 Exp10Lb = LibFunc_exp10;
2822 PowLb = LibFunc_pow;
2823 } else
2824 return nullptr;
2825 } else
2826 return nullptr;
2827
2828 // The earlier call must also be 'fast' in order to do these transforms.
2829 CallInst *Arg = dyn_cast<CallInst>(Log->getArgOperand(0));
2830 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2831 return nullptr;
2832
2833 IRBuilderBase::FastMathFlagGuard Guard(B);
2834 B.setFastMathFlags(FastMathFlags::getFast());
2835
2836 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2837 LibFunc ArgLb = TLI->getLibFunc(*Arg);
2838
2839 // log(pow(x,y)) -> y*log(x)
2840 AttributeList NoAttrs;
2841 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2842 Value *LogX =
2843 Log->doesNotAccessMemory()
2844 ? B.CreateUnaryIntrinsic(LogID, Arg->getOperand(0), nullptr, "log")
2845 : emitUnaryFloatFnCall(Arg->getOperand(0), TLI, LogNm, B, NoAttrs);
2846 Value *Y = Arg->getArgOperand(1);
2847 // Cast exponent to FP if integer.
2848 if (ArgID == Intrinsic::powi)
2849 Y = B.CreateSIToFP(Y, Ty, "cast");
2850 Value *MulY = B.CreateFMul(Y, LogX, "mul");
2851 // Since pow() may have side effects, e.g. errno,
2852 // dead code elimination may not be trusted to remove it.
2853 substituteInParent(Arg, MulY);
2854 return MulY;
2855 }
2856
2857 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2858 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2859 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2 ||
2860 ArgID == Intrinsic::exp10) {
2861 Constant *Eul;
2862 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2863 // FIXME: Add more precise value of e for long double.
2864 Eul = ConstantFP::get(Log->getType(), numbers::e);
2865 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2866 Eul = ConstantFP::get(Log->getType(), 2.0);
2867 else
2868 Eul = ConstantFP::get(Log->getType(), 10.0);
2869 Value *LogE = Log->doesNotAccessMemory()
2870 ? B.CreateUnaryIntrinsic(LogID, Eul, nullptr, "log")
2871 : emitUnaryFloatFnCall(Eul, TLI, LogNm, B, NoAttrs);
2872 Value *MulY = B.CreateFMul(Arg->getArgOperand(0), LogE, "mul");
2873 // Since exp() may have side effects, e.g. errno,
2874 // dead code elimination may not be trusted to remove it.
2875 substituteInParent(Arg, MulY);
2876 return MulY;
2877 }
2878
2879 return nullptr;
2880}
2881
2882// sqrt(exp(X)) -> exp(X * 0.5)
2883Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2884 if (!CI->hasAllowReassoc())
2885 return nullptr;
2886
2887 Function *SqrtFn = CI->getCalledFunction();
2888 CallInst *Arg = dyn_cast<CallInst>(CI->getArgOperand(0));
2889 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2890 return nullptr;
2891 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2892 LibFunc ArgLb = TLI->getLibFunc(*Arg);
2893
2894 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2895
2896 SqrtLb = TLI->getLibFunc(SqrtFn->getName());
2897 if (SqrtLb != NotLibFunc)
2898 switch (SqrtLb) {
2899 case LibFunc_sqrtf:
2900 ExpLb = LibFunc_expf;
2901 Exp2Lb = LibFunc_exp2f;
2902 Exp10Lb = LibFunc_exp10f;
2903 break;
2904 case LibFunc_sqrt:
2905 ExpLb = LibFunc_exp;
2906 Exp2Lb = LibFunc_exp2;
2907 Exp10Lb = LibFunc_exp10;
2908 break;
2909 case LibFunc_sqrtl:
2910 ExpLb = LibFunc_expl;
2911 Exp2Lb = LibFunc_exp2l;
2912 Exp10Lb = LibFunc_exp10l;
2913 break;
2914 default:
2915 return nullptr;
2916 }
2917 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2918 if (CI->getType()->getScalarType()->isFloatTy()) {
2919 ExpLb = LibFunc_expf;
2920 Exp2Lb = LibFunc_exp2f;
2921 Exp10Lb = LibFunc_exp10f;
2922 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2923 ExpLb = LibFunc_exp;
2924 Exp2Lb = LibFunc_exp2;
2925 Exp10Lb = LibFunc_exp10;
2926 } else
2927 return nullptr;
2928 } else
2929 return nullptr;
2930
2931 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2932 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2933 return nullptr;
2934
2935 IRBuilderBase::InsertPointGuard Guard(B);
2936 B.SetInsertPoint(Arg);
2937 auto *ExpOperand = Arg->getOperand(0);
2938 auto *FMul =
2939 B.CreateFMulFMF(ExpOperand, ConstantFP::get(ExpOperand->getType(), 0.5),
2940 CI, "merged.sqrt");
2941
2942 Arg->setOperand(0, FMul);
2943 return Arg;
2944}
2945
2946Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2947 Module *M = CI->getModule();
2949 Value *Ret = nullptr;
2950 // TODO: Once we have a way (other than checking for the existince of the
2951 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2952 // condition below.
2953 if (isLibFuncEmittable(M, TLI, LibFunc_sqrtf) &&
2954 (Callee->getName() == "sqrt" ||
2955 Callee->getIntrinsicID() == Intrinsic::sqrt))
2956 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2957
2958 if (Value *Opt = mergeSqrtToExp(CI, B))
2959 return Opt;
2960
2961 if (!CI->isFast())
2962 return Ret;
2963
2965 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2966 return Ret;
2967
2968 // We're looking for a repeated factor in a multiplication tree,
2969 // so we can do this fold: sqrt(x * x) -> fabs(x);
2970 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2971 Value *Op0 = I->getOperand(0);
2972 Value *Op1 = I->getOperand(1);
2973 Value *RepeatOp = nullptr;
2974 Value *OtherOp = nullptr;
2975 if (Op0 == Op1) {
2976 // Simple match: the operands of the multiply are identical.
2977 RepeatOp = Op0;
2978 } else {
2979 // Look for a more complicated pattern: one of the operands is itself
2980 // a multiply, so search for a common factor in that multiply.
2981 // Note: We don't bother looking any deeper than this first level or for
2982 // variations of this pattern because instcombine's visitFMUL and/or the
2983 // reassociation pass should give us this form.
2984 Value *MulOp;
2985 if (match(Op0, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
2986 cast<Instruction>(Op0)->isFast()) {
2987 // Pattern: sqrt((x * x) * z)
2988 RepeatOp = MulOp;
2989 OtherOp = Op1;
2990 } else if (match(Op1, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
2991 cast<Instruction>(Op1)->isFast()) {
2992 // Pattern: sqrt(z * (x * x))
2993 RepeatOp = MulOp;
2994 OtherOp = Op0;
2995 }
2996 }
2997 if (!RepeatOp)
2998 return Ret;
2999
3000 // Fast math flags for any created instructions should match the sqrt
3001 // and multiply.
3002
3003 // If we found a repeated factor, hoist it out of the square root and
3004 // replace it with the fabs of that factor.
3005 Value *FabsCall = B.CreateFAbs(RepeatOp, I, "fabs");
3006 if (OtherOp) {
3007 // If we found a non-repeated factor, we still need to get its square
3008 // root. We then multiply that by the value that was simplified out
3009 // of the square root calculation.
3010 Value *SqrtCall =
3011 B.CreateUnaryIntrinsic(Intrinsic::sqrt, OtherOp, I, "sqrt");
3012 return copyFlags(*CI, B.CreateFMulFMF(FabsCall, SqrtCall, I));
3013 }
3014 return copyFlags(*CI, FabsCall);
3015}
3016
3017Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
3018
3019 // fmod(x,y) sets errno if y == 0 or x == +/-inf. frem does not set errno,
3020 // so the fold is valid only when we can prove fmod wouldn't either.
3021 bool IsNoErrno = CI->hasNoNaNs();
3022 if (!IsNoErrno) {
3023 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
3024 KnownFPClass Known0 = computeKnownFPClass(CI->getOperand(0), fcInf, SQ);
3025 if (Known0.isKnownNeverInfinity()) {
3026 KnownFPClass Known1 =
3028 Function *F = CI->getParent()->getParent();
3029 const fltSemantics &FltSem =
3031 IsNoErrno = Known1.isKnownNeverLogicalZero(F->getDenormalMode(FltSem));
3032 }
3033 }
3034
3035 if (IsNoErrno)
3036 return B.CreateFRemFMF(CI->getOperand(0), CI->getOperand(1), CI);
3037 return nullptr;
3038}
3039
3040Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
3041 IRBuilderBase &B) {
3042 Module *M = CI->getModule();
3044 Value *Ret = nullptr;
3045 StringRef Name = Callee->getName();
3046 if (UnsafeFPShrink &&
3047 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
3048 Name == "asinh") &&
3049 hasFloatVersion(M, Name))
3050 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
3051
3052 Value *Op1 = CI->getArgOperand(0);
3053 auto *OpC = dyn_cast<CallInst>(Op1);
3054 if (!OpC)
3055 return Ret;
3056
3057 // Both calls must be 'fast' in order to remove them.
3058 if (!CI->isFast() || !OpC->isFast())
3059 return Ret;
3060
3061 // tan(atan(x)) -> x
3062 // atanh(tanh(x)) -> x
3063 // sinh(asinh(x)) -> x
3064 // asinh(sinh(x)) -> x
3065 // cosh(acosh(x)) -> x
3066 Function *F = OpC->getCalledFunction();
3067 LibFunc Func = F ? TLI->getLibFunc(F->getName()) : NotLibFunc;
3068 if (isLibFuncEmittable(M, TLI, Func)) {
3069 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
3070 .Case("tan", LibFunc_atan)
3071 .Case("atanh", LibFunc_tanh)
3072 .Case("sinh", LibFunc_asinh)
3073 .Case("cosh", LibFunc_acosh)
3074 .Case("tanf", LibFunc_atanf)
3075 .Case("atanhf", LibFunc_tanhf)
3076 .Case("sinhf", LibFunc_asinhf)
3077 .Case("coshf", LibFunc_acoshf)
3078 .Case("tanl", LibFunc_atanl)
3079 .Case("atanhl", LibFunc_tanhl)
3080 .Case("sinhl", LibFunc_asinhl)
3081 .Case("coshl", LibFunc_acoshl)
3082 .Case("asinh", LibFunc_sinh)
3083 .Case("asinhf", LibFunc_sinhf)
3084 .Case("asinhl", LibFunc_sinhl)
3085 .Default(NotLibFunc); // Used as error value
3086 if (Func == inverseFunc)
3087 Ret = OpC->getArgOperand(0);
3088 }
3089 return Ret;
3090}
3091
3092static bool isTrigLibCall(CallInst *CI) {
3093 // We can only hope to do anything useful if we can ignore things like errno
3094 // and floating-point exceptions.
3095 // We already checked the prototype.
3096 return CI->doesNotThrow() && CI->doesNotAccessMemory();
3097}
3098
3099static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
3100 bool UseFloat, Value *&Sin, Value *&Cos,
3101 Value *&SinCos, const TargetLibraryInfo *TLI) {
3102 Module *M = OrigCallee->getParent();
3103 Type *ArgTy = Arg->getType();
3104 Type *ResTy;
3105 StringRef Name;
3106
3107 Triple T(OrigCallee->getParent()->getTargetTriple());
3108 if (UseFloat) {
3109 Name = "__sincospif_stret";
3110
3111 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
3112 // x86_64 can't use {float, float} since that would be returned in both
3113 // xmm0 and xmm1, which isn't what a real struct would do.
3114 ResTy = T.getArch() == Triple::x86_64
3115 ? static_cast<Type *>(FixedVectorType::get(ArgTy, 2))
3116 : static_cast<Type *>(StructType::get(ArgTy, ArgTy));
3117 } else {
3118 Name = "__sincospi_stret";
3119 ResTy = StructType::get(ArgTy, ArgTy);
3120 }
3121
3122 if (!isLibFuncEmittable(M, TLI, Name))
3123 return false;
3124 LibFunc TheLibFunc = TLI->getLibFunc(Name);
3126 M, *TLI, TheLibFunc, OrigCallee->getAttributes(), ResTy, ArgTy);
3127
3128 if (Instruction *ArgInst = dyn_cast<Instruction>(Arg)) {
3129 // If the argument is an instruction, it must dominate all uses so put our
3130 // sincos call there.
3131 B.SetInsertPoint(++ArgInst->getIterator());
3132 } else {
3133 // Otherwise (e.g. for a constant) the beginning of the function is as
3134 // good a place as any.
3135 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
3136 B.SetInsertPoint(EntryBB.begin());
3137 }
3138
3139 SinCos = B.CreateCall(Callee, Arg, "sincospi");
3140
3141 if (SinCos->getType()->isStructTy()) {
3142 Sin = B.CreateExtractValue(SinCos, 0, "sinpi");
3143 Cos = B.CreateExtractValue(SinCos, 1, "cospi");
3144 } else {
3145 Sin = B.CreateExtractElement(SinCos, uint64_t{0}, "sinpi");
3146 Cos = B.CreateExtractElement(SinCos, uint64_t{1}, "cospi");
3147 }
3148
3149 return true;
3150}
3151
3152/// Flushing a denormal to +0.0 breaks f(-x) = -f(x) for odd f.
3156 return Mode.inputsMayBePositiveZero() || Mode.outputsMayBePositiveZero();
3157}
3158
3159static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
3160 IRBuilderBase &B) {
3161 Value *X;
3162 Value *Src = CI->getArgOperand(0);
3163
3164 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
3166 (IsEven || !mayFlushDenormalsToPositiveZero(CI))) {
3167 auto *Call = B.CreateCall(CI->getCalledFunction(), {X}, /*FMFSource=*/CI);
3168 auto *CallInst = copyFlags(*CI, Call);
3169 if (IsEven) {
3170 // Even function: f(-x) = f(x)
3171 return CallInst;
3172 }
3173 // Odd function: f(-x) = -f(x)
3174 return B.CreateFNegFMF(CallInst, CI);
3175 }
3176
3177 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
3178 if (IsEven && (match(Src, m_FAbs(m_Value(X))) ||
3179 match(Src, m_CopySign(m_Value(X), m_Value())))) {
3180 auto *Call = B.CreateCall(CI->getCalledFunction(), {X}, /*FMFSource=*/CI);
3181 return copyFlags(*CI, Call);
3182 }
3183
3184 return nullptr;
3185}
3186
3187Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
3188 IRBuilderBase &B) {
3189 switch (Func) {
3190 case LibFunc_cos:
3191 case LibFunc_cosf:
3192 case LibFunc_cosl:
3193
3194 case LibFunc_cosh:
3195 case LibFunc_coshf:
3196 case LibFunc_coshl:
3197 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
3198
3199 case LibFunc_cbrt:
3200 case LibFunc_cbrtf:
3201 case LibFunc_cbrtl:
3202
3203 case LibFunc_sin:
3204 case LibFunc_sinf:
3205 case LibFunc_sinl:
3206
3207 case LibFunc_sinh:
3208 case LibFunc_sinhf:
3209 case LibFunc_sinhl:
3210
3211 case LibFunc_asin:
3212 case LibFunc_asinf:
3213 case LibFunc_asinl:
3214
3215 case LibFunc_asinh:
3216 case LibFunc_asinhf:
3217 case LibFunc_asinhl:
3218
3219 case LibFunc_tan:
3220 case LibFunc_tanf:
3221 case LibFunc_tanl:
3222
3223 case LibFunc_tanh:
3224 case LibFunc_tanhf:
3225 case LibFunc_tanhl:
3226
3227 case LibFunc_atan:
3228 case LibFunc_atanf:
3229 case LibFunc_atanl:
3230
3231 case LibFunc_erf:
3232 case LibFunc_erff:
3233 case LibFunc_erfl:
3234 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
3235
3236 default:
3237 return nullptr;
3238 }
3239}
3240
3241Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
3242 // Make sure the prototype is as expected, otherwise the rest of the
3243 // function is probably invalid and likely to abort.
3244 if (!isTrigLibCall(CI))
3245 return nullptr;
3246
3247 Value *Arg = CI->getArgOperand(0);
3248 if (isa<ConstantData>(Arg))
3249 return nullptr;
3250
3253 SmallVector<CallInst *, 1> SinCosCalls;
3254
3255 bool IsFloat = Arg->getType()->isFloatTy();
3256
3257 // Look for all compatible sinpi, cospi and sincospi calls with the same
3258 // argument. If there are enough (in some sense) we can make the
3259 // substitution.
3260 Function *F = CI->getFunction();
3261 for (User *U : Arg->users())
3262 classifyArgUse(U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3263
3264 // It's only worthwhile if both sinpi and cospi are actually used.
3265 if (SinCalls.empty() || CosCalls.empty())
3266 return nullptr;
3267
3268 Value *Sin, *Cos, *SinCos;
3269 if (!insertSinCosCall(B, CI->getCalledFunction(), Arg, IsFloat, Sin, Cos,
3270 SinCos, TLI))
3271 return nullptr;
3272
3273 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3274 Value *Res) {
3275 for (CallInst *C : Calls)
3276 replaceAllUsesWith(C, Res);
3277 };
3278
3279 replaceTrigInsts(SinCalls, Sin);
3280 replaceTrigInsts(CosCalls, Cos);
3281 replaceTrigInsts(SinCosCalls, SinCos);
3282
3283 return IsSin ? Sin : Cos;
3284}
3285
3286void LibCallSimplifier::classifyArgUse(
3287 Value *Val, Function *F, bool IsFloat,
3290 SmallVectorImpl<CallInst *> &SinCosCalls) {
3291 auto *CI = dyn_cast<CallInst>(Val);
3292 if (!CI || CI->use_empty())
3293 return;
3294
3295 // Don't consider calls in other functions.
3296 if (CI->getFunction() != F)
3297 return;
3298
3299 Module *M = CI->getModule();
3301 LibFunc Func = Callee ? TLI->getLibFunc(*Callee) : NotLibFunc;
3302 if (!isLibFuncEmittable(M, TLI, Func) || !isTrigLibCall(CI))
3303 return;
3304
3305 if (IsFloat) {
3306 if (Func == LibFunc_sinpif)
3307 SinCalls.push_back(CI);
3308 else if (Func == LibFunc_cospif)
3309 CosCalls.push_back(CI);
3310 else if (Func == LibFunc_sincospif_stret)
3311 SinCosCalls.push_back(CI);
3312 } else {
3313 if (Func == LibFunc_sinpi)
3314 SinCalls.push_back(CI);
3315 else if (Func == LibFunc_cospi)
3316 CosCalls.push_back(CI);
3317 else if (Func == LibFunc_sincospi_stret)
3318 SinCosCalls.push_back(CI);
3319 }
3320}
3321
3322/// Constant folds remquo
3323Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3324 const APFloat *X, *Y;
3325 if (!match(CI->getArgOperand(0), m_APFloat(X)) ||
3326 !match(CI->getArgOperand(1), m_APFloat(Y)))
3327 return nullptr;
3328
3329 APFloat::opStatus Status;
3330 APFloat Quot = *X;
3331 Status = Quot.divide(*Y, APFloat::rmNearestTiesToEven);
3332 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3333 return nullptr;
3334 APFloat Rem = *X;
3335 if (Rem.remainder(*Y) != APFloat::opOK)
3336 return nullptr;
3337
3338 // TODO: We can only keep at least the three of the last bits of x/y
3339 unsigned IntBW = TLI->getIntSize();
3340 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3341 bool IsExact;
3342 Status =
3343 Quot.convertToInteger(QuotInt, APFloat::rmNearestTiesToEven, &IsExact);
3344 if (Status != APFloat::opOK && Status != APFloat::opInexact)
3345 return nullptr;
3346
3347 B.CreateAlignedStore(
3348 ConstantInt::getSigned(B.getIntNTy(IntBW), QuotInt.getExtValue()),
3349 CI->getArgOperand(2), CI->getParamAlign(2));
3350 return ConstantFP::get(CI->getType(), Rem);
3351}
3352
3353/// Constant folds fdim
3354Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3355 // Cannot perform the fold unless the call has attribute memory(none)
3356 if (!CI->doesNotAccessMemory())
3357 return nullptr;
3358
3359 // TODO : Handle undef values
3360 // Propagate poison if any
3361 if (isa<PoisonValue>(CI->getArgOperand(0)))
3362 return CI->getArgOperand(0);
3363 if (isa<PoisonValue>(CI->getArgOperand(1)))
3364 return CI->getArgOperand(1);
3365
3366 const APFloat *X, *Y;
3367 // Check if both values are constants
3368 if (!match(CI->getArgOperand(0), m_APFloat(X)) ||
3369 !match(CI->getArgOperand(1), m_APFloat(Y)))
3370 return nullptr;
3371
3372 // C99 fdim(x, y) = (x > y) ? x - y : +0.
3373 if (X->compare(*Y) != APFloat::cmpGreaterThan && !X->isNaN() && !Y->isNaN())
3374 return ConstantFP::getZero(CI->getType());
3375 APFloat Difference = *X;
3377 return ConstantFP::get(CI->getType(), Difference);
3378}
3379
3380//===----------------------------------------------------------------------===//
3381// Integer Library Call Optimizations
3382//===----------------------------------------------------------------------===//
3383
3384Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3385 // All variants of ffs return int which need not be 32 bits wide.
3386 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3387 Type *RetType = CI->getType();
3388 Value *Op = CI->getArgOperand(0);
3389 Type *ArgType = Op->getType();
3390 Value *V = B.CreateIntrinsic(Intrinsic::cttz, {ArgType}, {Op, B.getTrue()},
3391 nullptr, "cttz");
3392 V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1));
3393 V = B.CreateIntCast(V, RetType, false);
3394
3395 Value *Cond = B.CreateICmpNE(Op, Constant::getNullValue(ArgType));
3396 Value *S = B.CreateSelect(Cond, V, ConstantInt::get(RetType, 0));
3398 return S;
3399 if (auto *SI = dyn_cast<SelectInst>(S))
3402 /*IsExpected=*/false);
3403 return S;
3404}
3405
3406Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3407 // All variants of fls return int which need not be 32 bits wide.
3408 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3409 Value *Op = CI->getArgOperand(0);
3410 Type *ArgType = Op->getType();
3411 Value *V = B.CreateIntrinsic(Intrinsic::ctlz, {ArgType}, {Op, B.getFalse()},
3412 nullptr, "ctlz");
3413 V = B.CreateSub(ConstantInt::get(V->getType(), ArgType->getIntegerBitWidth()),
3414 V);
3415 return B.CreateIntCast(V, CI->getType(), false);
3416}
3417
3418Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3419 // abs(x) -> x <s 0 ? -x : x
3420 // The negation has 'nsw' because abs of INT_MIN is undefined.
3421 Value *X = CI->getArgOperand(0);
3422 Value *IsNeg = B.CreateIsNeg(X);
3423 Value *NegX = B.CreateNSWNeg(X, "neg");
3424 return B.CreateSelect(IsNeg, NegX, X);
3425}
3426
3427Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3428 // isdigit(c) -> (c-'0') <u 10
3429 Value *Op = CI->getArgOperand(0);
3430 Type *ArgType = Op->getType();
3431 Op = B.CreateSub(Op, ConstantInt::get(ArgType, '0'), "isdigittmp");
3432 Op = B.CreateICmpULT(Op, ConstantInt::get(ArgType, 10), "isdigit");
3433 return B.CreateZExt(Op, CI->getType());
3434}
3435
3436Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3437 // isascii(c) -> c <u 128
3438 Value *Op = CI->getArgOperand(0);
3439 Type *ArgType = Op->getType();
3440 Op = B.CreateICmpULT(Op, ConstantInt::get(ArgType, 128), "isascii");
3441 return B.CreateZExt(Op, CI->getType());
3442}
3443
3444Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3445 // toascii(c) -> c & 0x7f
3446 return B.CreateAnd(CI->getArgOperand(0),
3447 ConstantInt::get(CI->getType(), 0x7F));
3448}
3449
3450// Fold calls to atoi, atol, and atoll.
3451Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3452 StringRef Str;
3453 if (!getConstantStringInfo(CI->getArgOperand(0), Str))
3454 return nullptr;
3455
3456 return convertStrToInt(CI, Str, nullptr, 10, /*AsSigned=*/true, B);
3457}
3458
3459// Fold calls to strtol, strtoll, strtoul, and strtoull.
3460Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3461 bool AsSigned) {
3462 Value *EndPtr = CI->getArgOperand(1);
3463 if (isa<ConstantPointerNull>(EndPtr)) {
3464 // With a null EndPtr, this function won't capture the main argument.
3465 // It would be readonly too, except that it still may write to errno.
3468 EndPtr = nullptr;
3469 } else if (!isKnownNonZero(EndPtr, DL))
3470 return nullptr;
3471
3472 StringRef Str;
3473 if (!getConstantStringInfo(CI->getArgOperand(0), Str))
3474 return nullptr;
3475
3476 if (ConstantInt *CInt = dyn_cast<ConstantInt>(CI->getArgOperand(2))) {
3477 return convertStrToInt(CI, Str, EndPtr, CInt->getSExtValue(), AsSigned, B);
3478 }
3479
3480 return nullptr;
3481}
3482
3483//===----------------------------------------------------------------------===//
3484// Formatting and IO Library Call Optimizations
3485//===----------------------------------------------------------------------===//
3486
3487static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3488
3489Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3490 int StreamArg) {
3492 // Error reporting calls should be cold, mark them as such.
3493 // This applies even to non-builtin calls: it is only a hint and applies to
3494 // functions that the frontend might not understand as builtins.
3495
3496 // This heuristic was suggested in:
3497 // Improving Static Branch Prediction in a Compiler
3498 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3499 // Proceedings of PACT'98, Oct. 1998, IEEE
3500 if (!CI->hasFnAttr(Attribute::Cold) &&
3501 isReportingError(Callee, CI, StreamArg)) {
3502 CI->addFnAttr(Attribute::Cold);
3503 }
3504
3505 return nullptr;
3506}
3507
3508static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3509 if (!Callee || !Callee->isDeclaration())
3510 return false;
3511
3512 if (StreamArg < 0)
3513 return true;
3514
3515 // These functions might be considered cold, but only if their stream
3516 // argument is stderr.
3517
3518 if (StreamArg >= (int)CI->arg_size())
3519 return false;
3520 LoadInst *LI = dyn_cast<LoadInst>(CI->getArgOperand(StreamArg));
3521 if (!LI)
3522 return false;
3524 if (!GV || !GV->isDeclaration())
3525 return false;
3526 return GV->getName() == "stderr";
3527}
3528
3529Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3530 // Check for a fixed format string.
3531 StringRef FormatStr;
3532 if (!getConstantStringInfo(CI->getArgOperand(0), FormatStr))
3533 return nullptr;
3534
3535 // Empty format string -> noop.
3536 if (FormatStr.empty()) // Tolerate printf's declared void.
3537 return CI->use_empty() ? (Value *)CI : ConstantInt::get(CI->getType(), 0);
3538
3539 // Do not do any of the following transformations if the printf return value
3540 // is used, in general the printf return value is not compatible with either
3541 // putchar() or puts().
3542 if (!CI->use_empty())
3543 return nullptr;
3544
3545 Type *IntTy = CI->getType();
3546 // printf("x") -> putchar('x'), even for "%" and "%%".
3547 if (FormatStr.size() == 1 || FormatStr == "%%") {
3548 // Convert the character to unsigned char before passing it to putchar
3549 // to avoid host-specific sign extension in the IR. Putchar converts
3550 // it to unsigned char regardless.
3551 Value *IntChar = ConstantInt::get(IntTy, (unsigned char)FormatStr[0]);
3552 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
3553 }
3554
3555 // Try to remove call or emit putchar/puts.
3556 if (FormatStr == "%s" && CI->arg_size() > 1) {
3557 StringRef OperandStr;
3558 if (!getConstantStringInfo(CI->getOperand(1), OperandStr))
3559 return nullptr;
3560 // printf("%s", "") --> NOP
3561 if (OperandStr.empty())
3562 return (Value *)CI;
3563 // printf("%s", "a") --> putchar('a')
3564 if (OperandStr.size() == 1) {
3565 // Convert the character to unsigned char before passing it to putchar
3566 // to avoid host-specific sign extension in the IR. Putchar converts
3567 // it to unsigned char regardless.
3568 Value *IntChar = ConstantInt::get(IntTy, (unsigned char)OperandStr[0]);
3569 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
3570 }
3571 // printf("%s", str"\n") --> puts(str)
3572 if (OperandStr.back() == '\n') {
3573 if (!isLibFuncEmittable(CI->getModule(), TLI, LibFunc_puts))
3574 return nullptr;
3575 OperandStr = OperandStr.drop_back();
3576 Value *GV = B.CreateGlobalString(OperandStr, "str");
3577 return copyFlags(*CI, emitPutS(GV, B, TLI));
3578 }
3579 return nullptr;
3580 }
3581
3582 // printf("foo\n") --> puts("foo")
3583 if (FormatStr.back() == '\n' &&
3584 !FormatStr.contains('%')) { // No format characters.
3585 if (!isLibFuncEmittable(CI->getModule(), TLI, LibFunc_puts))
3586 return nullptr;
3587 // Create a string literal with no \n on it. We expect the constant merge
3588 // pass to be run after this pass, to merge duplicate strings.
3589 FormatStr = FormatStr.drop_back();
3590 Value *GV = B.CreateGlobalString(FormatStr, "str");
3591 return copyFlags(*CI, emitPutS(GV, B, TLI));
3592 }
3593
3594 // Optimize specific format strings.
3595 // printf("%c", chr) --> putchar(chr)
3596 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3597 CI->getArgOperand(1)->getType()->isIntegerTy()) {
3598 // Convert the argument to the type expected by putchar, i.e., int, which
3599 // need not be 32 bits wide but which is the same as printf's return type.
3600 Value *IntChar = B.CreateIntCast(CI->getArgOperand(1), IntTy, false);
3601 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
3602 }
3603
3604 // printf("%s\n", str) --> puts(str)
3605 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3606 CI->getArgOperand(1)->getType()->isPointerTy())
3607 return copyFlags(*CI, emitPutS(CI->getArgOperand(1), B, TLI));
3608 return nullptr;
3609}
3610
3611Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3612
3613 Module *M = CI->getModule();
3615 FunctionType *FT = Callee->getFunctionType();
3616 if (Value *V = optimizePrintFString(CI, B)) {
3617 return V;
3618 }
3619
3621
3622 // printf(format, ...) -> iprintf(format, ...) if no floating point
3623 // arguments.
3624 if (isLibFuncEmittable(M, TLI, LibFunc_iprintf) &&
3626 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_iprintf, FT,
3627 Callee->getAttributes());
3628 CallInst *New = cast<CallInst>(CI->clone());
3629 New->setCalledFunction(IPrintFFn);
3630 B.Insert(New);
3631 return New;
3632 }
3633
3634 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3635 // arguments.
3636 if (isLibFuncEmittable(M, TLI, LibFunc_small_printf) &&
3637 !callHasFP128Argument(CI)) {
3638 auto SmallPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_small_printf, FT,
3639 Callee->getAttributes());
3640 CallInst *New = cast<CallInst>(CI->clone());
3641 New->setCalledFunction(SmallPrintFFn);
3642 B.Insert(New);
3643 return New;
3644 }
3645
3646 return nullptr;
3647}
3648
3649Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3650 IRBuilderBase &B) {
3651 // Check for a fixed format string.
3652 StringRef FormatStr;
3653 if (!getConstantStringInfo(CI->getArgOperand(1), FormatStr))
3654 return nullptr;
3655
3656 // If we just have a format string (nothing else crazy) transform it.
3657 Value *Dest = CI->getArgOperand(0);
3658 if (CI->arg_size() == 2) {
3659 // Make sure there's no % in the constant array. We could try to handle
3660 // %% -> % in the future if we cared.
3661 if (FormatStr.contains('%'))
3662 return nullptr; // we found a format specifier, bail out.
3663
3664 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3665 B.CreateMemCpy(Dest, Align(1), CI->getArgOperand(1), Align(1),
3666 // Copy the null byte.
3667 TLI->getAsSizeT(FormatStr.size() + 1, *CI->getModule()));
3668 return ConstantInt::get(CI->getType(), FormatStr.size());
3669 }
3670
3671 // The remaining optimizations require the format string to be "%s" or "%c"
3672 // and have an extra operand.
3673 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3674 return nullptr;
3675
3676 // Decode the second character of the format string.
3677 if (FormatStr[1] == 'c') {
3678 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3679 if (!CI->getArgOperand(2)->getType()->isIntegerTy())
3680 return nullptr;
3681 Value *V = B.CreateTrunc(CI->getArgOperand(2), B.getInt8Ty(), "char");
3682 Value *Ptr = Dest;
3683 B.CreateStore(V, Ptr);
3684 Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Ptr, B.getInt32(1), "nul");
3685 B.CreateStore(B.getInt8(0), Ptr);
3686
3687 return ConstantInt::get(CI->getType(), 1);
3688 }
3689
3690 if (FormatStr[1] == 's') {
3691 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3692 // strlen(str)+1)
3693 if (!CI->getArgOperand(2)->getType()->isPointerTy())
3694 return nullptr;
3695
3696 if (CI->use_empty())
3697 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3698 return copyFlags(*CI, emitStrCpy(Dest, CI->getArgOperand(2), B, TLI));
3699
3700 uint64_t SrcLen = GetStringLength(CI->getArgOperand(2));
3701 if (SrcLen) {
3702 B.CreateMemCpy(Dest, Align(1), CI->getArgOperand(2), Align(1),
3703 TLI->getAsSizeT(SrcLen, *CI->getModule()));
3704 // Returns total number of characters written without null-character.
3705 return ConstantInt::get(CI->getType(), SrcLen - 1);
3706 } else if (Value *V = emitStpCpy(Dest, CI->getArgOperand(2), B, TLI)) {
3707 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3708 Value *PtrDiff = B.CreatePtrDiff(V, Dest);
3709 return B.CreateIntCast(PtrDiff, CI->getType(), false);
3710 }
3711
3712 if (llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
3714 return nullptr;
3715
3716 Value *Len = emitStrLen(CI->getArgOperand(2), B, DL, TLI);
3717 if (!Len)
3718 return nullptr;
3719 Value *IncLen =
3720 B.CreateAdd(Len, ConstantInt::get(Len->getType(), 1), "leninc");
3721 B.CreateMemCpy(Dest, Align(1), CI->getArgOperand(2), Align(1), IncLen);
3722
3723 // The sprintf result is the unincremented number of bytes in the string.
3724 return B.CreateIntCast(Len, CI->getType(), false);
3725 }
3726 return nullptr;
3727}
3728
3729Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3730 Module *M = CI->getModule();
3732 FunctionType *FT = Callee->getFunctionType();
3733 if (Value *V = optimizeSPrintFString(CI, B)) {
3734 return V;
3735 }
3736
3738
3739 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3740 // point arguments.
3741 if (isLibFuncEmittable(M, TLI, LibFunc_siprintf) &&
3743 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_siprintf,
3744 FT, Callee->getAttributes());
3745 CallInst *New = cast<CallInst>(CI->clone());
3746 New->setCalledFunction(SIPrintFFn);
3747 B.Insert(New);
3748 return New;
3749 }
3750
3751 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3752 // floating point arguments.
3753 if (isLibFuncEmittable(M, TLI, LibFunc_small_sprintf) &&
3754 !callHasFP128Argument(CI)) {
3755 auto SmallSPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_small_sprintf, FT,
3756 Callee->getAttributes());
3757 CallInst *New = cast<CallInst>(CI->clone());
3758 New->setCalledFunction(SmallSPrintFFn);
3759 B.Insert(New);
3760 return New;
3761 }
3762
3763 return nullptr;
3764}
3765
3766// Transform an snprintf call CI with the bound N to format the string Str
3767// either to a call to memcpy, or to single character a store, or to nothing,
3768// and fold the result to a constant. A nonnull StrArg refers to the string
3769// argument being formatted. Otherwise the call is one with N < 2 and
3770// the "%c" directive to format a single character.
3771Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3772 StringRef Str, uint64_t N,
3773 IRBuilderBase &B) {
3774 assert(StrArg || (N < 2 && Str.size() == 1));
3775
3776 unsigned IntBits = TLI->getIntSize();
3777 uint64_t IntMax = maxIntN(IntBits);
3778 if (Str.size() > IntMax)
3779 // Bail if the string is longer than INT_MAX. POSIX requires
3780 // implementations to set errno to EOVERFLOW in this case, in
3781 // addition to when N is larger than that (checked by the caller).
3782 return nullptr;
3783
3784 Value *StrLen = ConstantInt::get(CI->getType(), Str.size());
3785 if (N == 0)
3786 return StrLen;
3787
3788 // Set to the number of bytes to copy fron StrArg which is also
3789 // the offset of the terinating nul.
3790 uint64_t NCopy;
3791 if (N > Str.size())
3792 // Copy the full string, including the terminating nul (which must
3793 // be present regardless of the bound).
3794 NCopy = Str.size() + 1;
3795 else
3796 NCopy = N - 1;
3797
3798 Value *DstArg = CI->getArgOperand(0);
3799 if (NCopy && StrArg)
3800 // Transform the call to lvm.memcpy(dst, fmt, N).
3801 copyFlags(*CI, B.CreateMemCpy(DstArg, Align(1), StrArg, Align(1),
3802 TLI->getAsSizeT(NCopy, *CI->getModule())));
3803
3804 if (N > Str.size())
3805 // Return early when the whole format string, including the final nul,
3806 // has been copied.
3807 return StrLen;
3808
3809 // Otherwise, when truncating the string append a terminating nul.
3810 Type *Int8Ty = B.getInt8Ty();
3811 Value *NulOff = B.getIntN(IntBits, NCopy);
3812 Value *DstEnd = B.CreateInBoundsGEP(Int8Ty, DstArg, NulOff, "endptr");
3813 B.CreateStore(ConstantInt::get(Int8Ty, 0), DstEnd);
3814 return StrLen;
3815}
3816
3817Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3818 IRBuilderBase &B) {
3819 // Check for size
3820 ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
3821 if (!Size)
3822 return nullptr;
3823
3824 uint64_t N = Size->getZExtValue();
3825 uint64_t IntMax = maxIntN(TLI->getIntSize());
3826 if (N > IntMax)
3827 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3828 // to set errno to EOVERFLOW in this case.
3829 return nullptr;
3830
3831 Value *DstArg = CI->getArgOperand(0);
3832 Value *FmtArg = CI->getArgOperand(2);
3833
3834 // Check for a fixed format string.
3835 StringRef FormatStr;
3836 if (!getConstantStringInfo(FmtArg, FormatStr))
3837 return nullptr;
3838
3839 // If we just have a format string (nothing else crazy) transform it.
3840 if (CI->arg_size() == 3) {
3841 if (FormatStr.contains('%'))
3842 // Bail if the format string contains a directive and there are
3843 // no arguments. We could handle "%%" in the future.
3844 return nullptr;
3845
3846 return emitSnPrintfMemCpy(CI, FmtArg, FormatStr, N, B);
3847 }
3848
3849 // The remaining optimizations require the format string to be "%s" or "%c"
3850 // and have an extra operand.
3851 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3852 return nullptr;
3853
3854 // Decode the second character of the format string.
3855 if (FormatStr[1] == 'c') {
3856 if (N <= 1) {
3857 // Use an arbitary string of length 1 to transform the call into
3858 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3859 // to one.
3860 StringRef CharStr("*");
3861 return emitSnPrintfMemCpy(CI, nullptr, CharStr, N, B);
3862 }
3863
3864 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3865 if (!CI->getArgOperand(3)->getType()->isIntegerTy())
3866 return nullptr;
3867 Value *V = B.CreateTrunc(CI->getArgOperand(3), B.getInt8Ty(), "char");
3868 Value *Ptr = DstArg;
3869 B.CreateStore(V, Ptr);
3870 Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Ptr, B.getInt32(1), "nul");
3871 B.CreateStore(B.getInt8(0), Ptr);
3872 return ConstantInt::get(CI->getType(), 1);
3873 }
3874
3875 if (FormatStr[1] != 's')
3876 return nullptr;
3877
3878 Value *StrArg = CI->getArgOperand(3);
3879 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3880 StringRef Str;
3881 if (!getConstantStringInfo(StrArg, Str))
3882 return nullptr;
3883
3884 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3885}
3886
3887Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3888 if (Value *V = optimizeSnPrintFString(CI, B)) {
3889 return V;
3890 }
3891
3892 if (isKnownNonZero(CI->getOperand(1), DL))
3894 return nullptr;
3895}
3896
3897Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3898 IRBuilderBase &B) {
3899 optimizeErrorReporting(CI, B, 0);
3900
3901 // All the optimizations depend on the format string.
3902 StringRef FormatStr;
3903 if (!getConstantStringInfo(CI->getArgOperand(1), FormatStr))
3904 return nullptr;
3905
3906 // Do not do any of the following transformations if the fprintf return
3907 // value is used, in general the fprintf return value is not compatible
3908 // with fwrite(), fputc() or fputs().
3909 if (!CI->use_empty())
3910 return nullptr;
3911
3912 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3913 if (CI->arg_size() == 2) {
3914 // Could handle %% -> % if we cared.
3915 if (FormatStr.contains('%'))
3916 return nullptr; // We found a format specifier.
3917
3918 return copyFlags(
3919 *CI, emitFWrite(CI->getArgOperand(1),
3920 TLI->getAsSizeT(FormatStr.size(), *CI->getModule()),
3921 CI->getArgOperand(0), B, DL, TLI));
3922 }
3923
3924 // The remaining optimizations require the format string to be "%s" or "%c"
3925 // and have an extra operand.
3926 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3927 return nullptr;
3928
3929 // Decode the second character of the format string.
3930 if (FormatStr[1] == 'c') {
3931 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3932 if (!CI->getArgOperand(2)->getType()->isIntegerTy())
3933 return nullptr;
3934 Type *IntTy = B.getIntNTy(TLI->getIntSize());
3935 Value *V = B.CreateIntCast(CI->getArgOperand(2), IntTy, /*isSigned*/ true,
3936 "chari");
3937 return copyFlags(*CI, emitFPutC(V, CI->getArgOperand(0), B, TLI));
3938 }
3939
3940 if (FormatStr[1] == 's') {
3941 // fprintf(F, "%s", str) --> fputs(str, F)
3942 if (!CI->getArgOperand(2)->getType()->isPointerTy())
3943 return nullptr;
3944 return copyFlags(
3945 *CI, emitFPutS(CI->getArgOperand(2), CI->getArgOperand(0), B, TLI));
3946 }
3947 return nullptr;
3948}
3949
3950Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3951 Module *M = CI->getModule();
3953 FunctionType *FT = Callee->getFunctionType();
3954 if (Value *V = optimizeFPrintFString(CI, B)) {
3955 return V;
3956 }
3957
3958 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3959 // floating point arguments.
3960 if (isLibFuncEmittable(M, TLI, LibFunc_fiprintf) &&
3962 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_fiprintf,
3963 FT, Callee->getAttributes());
3964 CallInst *New = cast<CallInst>(CI->clone());
3965 New->setCalledFunction(FIPrintFFn);
3966 B.Insert(New);
3967 return New;
3968 }
3969
3970 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3971 // 128-bit floating point arguments.
3972 if (isLibFuncEmittable(M, TLI, LibFunc_small_fprintf) &&
3973 !callHasFP128Argument(CI)) {
3974 auto SmallFPrintFFn =
3975 getOrInsertLibFunc(M, *TLI, LibFunc_small_fprintf, FT,
3976 Callee->getAttributes());
3977 CallInst *New = cast<CallInst>(CI->clone());
3978 New->setCalledFunction(SmallFPrintFFn);
3979 B.Insert(New);
3980 return New;
3981 }
3982
3983 return nullptr;
3984}
3985
3986Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3987 optimizeErrorReporting(CI, B, 3);
3988
3989 // Get the element size and count.
3990 ConstantInt *SizeC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
3991 ConstantInt *CountC = dyn_cast<ConstantInt>(CI->getArgOperand(2));
3992 if (SizeC && CountC) {
3993 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3994
3995 // If this is writing zero records, remove the call (it's a noop).
3996 if (Bytes == 0)
3997 return ConstantInt::get(CI->getType(), 0);
3998
3999 // If this is writing one byte, turn it into fputc.
4000 // This optimisation is only valid, if the return value is unused.
4001 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
4002 Value *Char = B.CreateLoad(B.getInt8Ty(), CI->getArgOperand(0), "char");
4003 Type *IntTy = B.getIntNTy(TLI->getIntSize());
4004 Value *Cast = B.CreateIntCast(Char, IntTy, /*isSigned*/ true, "chari");
4005 Value *NewCI = emitFPutC(Cast, CI->getArgOperand(3), B, TLI);
4006 return NewCI ? ConstantInt::get(CI->getType(), 1) : nullptr;
4007 }
4008 }
4009
4010 return nullptr;
4011}
4012
4013Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
4014 optimizeErrorReporting(CI, B, 1);
4015
4016 // Don't rewrite fputs to fwrite when optimising for size because fwrite
4017 // requires more arguments and thus extra MOVs are required.
4018 if (llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
4020 return nullptr;
4021
4022 // We can't optimize if return value is used.
4023 if (!CI->use_empty())
4024 return nullptr;
4025
4026 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
4028 if (!Len)
4029 return nullptr;
4030
4031 // Known to have no uses (see above).
4032 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
4033 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
4034 return copyFlags(
4035 *CI,
4037 ConstantInt::get(SizeTTy, Len - 1),
4038 CI->getArgOperand(1), B, DL, TLI));
4039}
4040
4041Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
4043 if (!CI->use_empty())
4044 return nullptr;
4045
4046 // Check for a constant string.
4047 // puts("") -> putchar('\n')
4048 StringRef Str;
4049 if (getConstantStringInfo(CI->getArgOperand(0), Str) && Str.empty()) {
4050 // putchar takes an argument of the same type as puts returns, i.e.,
4051 // int, which need not be 32 bits wide.
4052 Type *IntTy = CI->getType();
4053 return copyFlags(*CI, emitPutChar(ConstantInt::get(IntTy, '\n'), B, TLI));
4054 }
4055
4056 return nullptr;
4057}
4058
4059Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
4060
4061 // Mark 'exit' as cold if its not exit(0) (success).
4062 const APInt *C;
4063 if (!CI->hasFnAttr(Attribute::Cold) &&
4064 match(CI->getArgOperand(0), m_APInt(C)) && !C->isZero()) {
4065 CI->addFnAttr(Attribute::Cold);
4066 }
4067 return nullptr;
4068}
4069
4070Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
4071 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
4072 return copyFlags(*CI, B.CreateMemMove(CI->getArgOperand(1), Align(1),
4073 CI->getArgOperand(0), Align(1),
4074 CI->getArgOperand(2)));
4075}
4076
4077bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
4078 SmallString<20> FloatFuncName = FuncName;
4079 FloatFuncName += 'f';
4080 return isLibFuncEmittable(M, TLI, FloatFuncName);
4081}
4082
4083Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
4084 IRBuilderBase &Builder) {
4085 Module *M = CI->getModule();
4087 LibFunc Func = TLI->getLibFunc(*Callee);
4088
4089 // Check for string/memory library functions.
4090 if (isLibFuncEmittable(M, TLI, Func)) {
4091 // Make sure we never change the calling convention.
4092 assert(
4093 (ignoreCallingConv(Func) ||
4095 "Optimizing string/memory libcall would change the calling convention");
4096 switch (Func) {
4097 case LibFunc_strcat:
4098 return optimizeStrCat(CI, Builder);
4099 case LibFunc_strncat:
4100 return optimizeStrNCat(CI, Builder);
4101 case LibFunc_strchr:
4102 return optimizeStrChr(CI, Builder);
4103 case LibFunc_strrchr:
4104 return optimizeStrRChr(CI, Builder);
4105 case LibFunc_strcmp:
4106 return optimizeStrCmp(CI, Builder);
4107 case LibFunc_strncmp:
4108 return optimizeStrNCmp(CI, Builder);
4109 case LibFunc_strcpy:
4110 return optimizeStrCpy(CI, Builder);
4111 case LibFunc_stpcpy:
4112 return optimizeStpCpy(CI, Builder);
4113 case LibFunc_strlcpy:
4114 return optimizeStrLCpy(CI, Builder);
4115 case LibFunc_stpncpy:
4116 return optimizeStringNCpy(CI, /*RetEnd=*/true, Builder);
4117 case LibFunc_strncpy:
4118 return optimizeStringNCpy(CI, /*RetEnd=*/false, Builder);
4119 case LibFunc_strlen:
4120 return optimizeStrLen(CI, Builder);
4121 case LibFunc_strnlen:
4122 return optimizeStrNLen(CI, Builder);
4123 case LibFunc_strpbrk:
4124 return optimizeStrPBrk(CI, Builder);
4125 case LibFunc_strndup:
4126 return optimizeStrNDup(CI, Builder);
4127 case LibFunc_strtol:
4128 case LibFunc_strtod:
4129 case LibFunc_strtof:
4130 case LibFunc_strtoul:
4131 case LibFunc_strtoll:
4132 case LibFunc_strtold:
4133 case LibFunc_strtoull:
4134 return optimizeStrTo(CI, Builder);
4135 case LibFunc_strspn:
4136 return optimizeStrSpn(CI, Builder);
4137 case LibFunc_strcspn:
4138 return optimizeStrCSpn(CI, Builder);
4139 case LibFunc_strstr:
4140 return optimizeStrStr(CI, Builder);
4141 case LibFunc_memchr:
4142 return optimizeMemChr(CI, Builder);
4143 case LibFunc_memrchr:
4144 return optimizeMemRChr(CI, Builder);
4145 case LibFunc_bcmp:
4146 return optimizeBCmp(CI, Builder);
4147 case LibFunc_memcmp:
4148 return optimizeMemCmp(CI, Builder);
4149 case LibFunc_memcpy:
4150 return optimizeMemCpy(CI, Builder);
4151 case LibFunc_memccpy:
4152 return optimizeMemCCpy(CI, Builder);
4153 case LibFunc_mempcpy:
4154 return optimizeMemPCpy(CI, Builder);
4155 case LibFunc_memmove:
4156 return optimizeMemMove(CI, Builder);
4157 case LibFunc_memset:
4158 return optimizeMemSet(CI, Builder);
4159 case LibFunc_realloc:
4160 return optimizeRealloc(CI, Builder);
4161 case LibFunc_wcslen:
4162 return optimizeWcslen(CI, Builder);
4163 case LibFunc_bcopy:
4164 return optimizeBCopy(CI, Builder);
4165 case LibFunc_Znwm:
4166 case LibFunc_ZnwmRKSt9nothrow_t:
4167 case LibFunc_ZnwmSt11align_val_t:
4168 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
4169 case LibFunc_Znam:
4170 case LibFunc_ZnamRKSt9nothrow_t:
4171 case LibFunc_ZnamSt11align_val_t:
4172 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
4173 case LibFunc_Znwm12__hot_cold_t:
4174 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
4175 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
4176 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4177 case LibFunc_Znam12__hot_cold_t:
4178 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
4179 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
4180 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
4181 case LibFunc_size_returning_new:
4182 case LibFunc_size_returning_new_hot_cold:
4183 case LibFunc_size_returning_new_aligned:
4184 case LibFunc_size_returning_new_aligned_hot_cold:
4185 return optimizeNew(CI, Builder, Func);
4186 default:
4187 break;
4188 }
4189 }
4190 return nullptr;
4191}
4192
4193/// Constant folding nan/nanf/nanl.
4195 StringRef CharSeq;
4196 if (!getConstantStringInfo(CI->getArgOperand(0), CharSeq))
4197 return nullptr;
4198
4199 APInt Fill;
4200 // Treat empty strings as if they were zero.
4201 if (CharSeq.empty())
4202 Fill = APInt(32, 0);
4203 else if (CharSeq.getAsInteger(0, Fill))
4204 return nullptr;
4205
4206 return ConstantFP::getQNaN(CI->getType(), /*Negative=*/false, &Fill);
4207}
4208
4209Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
4210 LibFunc Func,
4211 IRBuilderBase &Builder) {
4212 const Module *M = CI->getModule();
4213
4214 // Don't optimize calls that require strict floating point semantics.
4215 if (CI->isStrictFP())
4216 return nullptr;
4217
4218 if (Value *V = optimizeSymmetric(CI, Func, Builder))
4219 return V;
4220
4221 switch (Func) {
4222 case LibFunc_sinpif:
4223 case LibFunc_sinpi:
4224 return optimizeSinCosPi(CI, /*IsSin*/true, Builder);
4225 case LibFunc_cospif:
4226 case LibFunc_cospi:
4227 return optimizeSinCosPi(CI, /*IsSin*/false, Builder);
4228 case LibFunc_sinf:
4229 case LibFunc_sinl:
4230 if (CI->doesNotAccessMemory())
4231 return replaceUnaryCall(CI, Builder, Intrinsic::sin);
4232 return nullptr;
4233 case LibFunc_cosf:
4234 case LibFunc_cosl:
4235 if (CI->doesNotAccessMemory())
4236 return replaceUnaryCall(CI, Builder, Intrinsic::cos);
4237 return nullptr;
4238 case LibFunc_powf:
4239 case LibFunc_pow:
4240 case LibFunc_powl:
4241 return optimizePow(CI, Builder);
4242 case LibFunc_exp2l:
4243 case LibFunc_exp2:
4244 case LibFunc_exp2f:
4245 return optimizeExp2(CI, Builder);
4246 case LibFunc_ldexp:
4247 case LibFunc_ldexpf:
4248 case LibFunc_ldexpl:
4249 case LibFunc_scalbn:
4250 case LibFunc_scalbnf:
4251 case LibFunc_scalbnl:
4252 // LLVM floating-point types have radix 2, so scalbn is equivalent to
4253 // ldexp. Do not replace a libcall that may set errno.
4254 if (CI->doesNotAccessMemory()) {
4255 Value *NewCall =
4256 Builder.CreateLdexp(CI->getArgOperand(0), CI->getArgOperand(1), CI);
4257 NewCall->takeName(CI);
4258 return copyFlags(*CI, NewCall);
4259 }
4260 return nullptr;
4261 case LibFunc_fabsf:
4262 case LibFunc_fabs:
4263 case LibFunc_fabsl:
4264 return replaceUnaryCall(CI, Builder, Intrinsic::fabs);
4265 case LibFunc_sqrtf:
4266 case LibFunc_sqrt:
4267 case LibFunc_sqrtl:
4268 if (Value *V = optimizeSqrt(CI, Builder))
4269 return V;
4270 if (CI->doesNotAccessMemory())
4271 return replaceUnaryCall(CI, Builder, Intrinsic::sqrt);
4272 return nullptr;
4273 case LibFunc_fmod:
4274 case LibFunc_fmodf:
4275 case LibFunc_fmodl:
4276 return optimizeFMod(CI, Builder);
4277 case LibFunc_logf:
4278 case LibFunc_log:
4279 case LibFunc_logl:
4280 case LibFunc_log10f:
4281 case LibFunc_log10:
4282 case LibFunc_log10l:
4283 case LibFunc_log1pf:
4284 case LibFunc_log1p:
4285 case LibFunc_log1pl:
4286 case LibFunc_log2f:
4287 case LibFunc_log2:
4288 case LibFunc_log2l:
4289 case LibFunc_logbf:
4290 case LibFunc_logb:
4291 case LibFunc_logbl:
4292 return optimizeLog(CI, Builder);
4293 case LibFunc_tan:
4294 case LibFunc_tanf:
4295 case LibFunc_tanl:
4296 case LibFunc_sinh:
4297 case LibFunc_sinhf:
4298 case LibFunc_sinhl:
4299 case LibFunc_asinh:
4300 case LibFunc_asinhf:
4301 case LibFunc_asinhl:
4302 case LibFunc_cosh:
4303 case LibFunc_coshf:
4304 case LibFunc_coshl:
4305 case LibFunc_atanh:
4306 case LibFunc_atanhf:
4307 case LibFunc_atanhl:
4308 return optimizeTrigInversionPairs(CI, Builder);
4309 case LibFunc_ceilf:
4310 case LibFunc_ceil:
4311 case LibFunc_ceill:
4312 return replaceUnaryCall(CI, Builder, Intrinsic::ceil);
4313 case LibFunc_floorf:
4314 case LibFunc_floor:
4315 case LibFunc_floorl:
4316 return replaceUnaryCall(CI, Builder, Intrinsic::floor);
4317 case LibFunc_roundf:
4318 case LibFunc_round:
4319 case LibFunc_roundl:
4320 return replaceUnaryCall(CI, Builder, Intrinsic::round);
4321 case LibFunc_roundevenf:
4322 case LibFunc_roundeven:
4323 case LibFunc_roundevenl:
4324 return replaceUnaryCall(CI, Builder, Intrinsic::roundeven);
4325 case LibFunc_nearbyintf:
4326 case LibFunc_nearbyint:
4327 case LibFunc_nearbyintl:
4328 return replaceUnaryCall(CI, Builder, Intrinsic::nearbyint);
4329 case LibFunc_rintf:
4330 case LibFunc_rint:
4331 case LibFunc_rintl:
4332 return replaceUnaryCall(CI, Builder, Intrinsic::rint);
4333 case LibFunc_truncf:
4334 case LibFunc_trunc:
4335 case LibFunc_truncl:
4336 return replaceUnaryCall(CI, Builder, Intrinsic::trunc);
4337 case LibFunc_sin:
4338 case LibFunc_cos:
4339 if (UnsafeFPShrink &&
4340 hasFloatVersion(M, CI->getCalledFunction()->getName()))
4341 if (Value *V = optimizeUnaryDoubleFP(CI, Builder, TLI, true))
4342 return V;
4343 if (CI->doesNotAccessMemory())
4344 return replaceUnaryCall(
4345 CI, Builder, Func == LibFunc_sin ? Intrinsic::sin : Intrinsic::cos);
4346 return nullptr;
4347 case LibFunc_acos:
4348 case LibFunc_acosh:
4349 case LibFunc_asin:
4350 case LibFunc_atan:
4351 case LibFunc_cbrt:
4352 case LibFunc_exp:
4353 case LibFunc_exp10:
4354 case LibFunc_expm1:
4355 case LibFunc_tanh:
4356 if (UnsafeFPShrink && hasFloatVersion(M, CI->getCalledFunction()->getName()))
4357 return optimizeUnaryDoubleFP(CI, Builder, TLI, true);
4358 return nullptr;
4359 case LibFunc_copysign:
4360 if (hasFloatVersion(M, CI->getCalledFunction()->getName()))
4361 if (Value *V = optimizeBinaryDoubleFP(CI, Builder, TLI))
4362 return V;
4363 [[fallthrough]];
4364 case LibFunc_copysignf:
4365 case LibFunc_copysignl:
4366 return replaceBinaryCall(CI, Builder, Intrinsic::copysign);
4367 case LibFunc_fdim:
4368 case LibFunc_fdimf:
4369 case LibFunc_fdiml:
4370 return optimizeFdim(CI, Builder);
4371 case LibFunc_fminf:
4372 case LibFunc_fmin:
4373 case LibFunc_fminl:
4374 return optimizeFMinFMax(CI, Builder, Intrinsic::minnum);
4375 case LibFunc_fmaxf:
4376 case LibFunc_fmax:
4377 case LibFunc_fmaxl:
4378 return optimizeFMinFMax(CI, Builder, Intrinsic::maxnum);
4379 case LibFunc_fminimum_numf:
4380 case LibFunc_fminimum_num:
4381 case LibFunc_fminimum_numl:
4382 return replaceBinaryCall(CI, Builder, Intrinsic::minimumnum);
4383 case LibFunc_fmaximum_numf:
4384 case LibFunc_fmaximum_num:
4385 case LibFunc_fmaximum_numl:
4386 return replaceBinaryCall(CI, Builder, Intrinsic::maximumnum);
4387 case LibFunc_cabs:
4388 case LibFunc_cabsf:
4389 case LibFunc_cabsl:
4390 return optimizeCAbs(CI, Builder);
4391 case LibFunc_remquo:
4392 case LibFunc_remquof:
4393 case LibFunc_remquol:
4394 return optimizeRemquo(CI, Builder);
4395 case LibFunc_nan:
4396 case LibFunc_nanf:
4397 case LibFunc_nanl:
4398 return optimizeNaN(CI);
4399 default:
4400 return nullptr;
4401 }
4402}
4403
4405 Module *M = CI->getModule();
4406 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4407
4408 Function *Callee = CI->getCalledFunction();
4409 LibFunc Func = TLI->getLibFunc(*Callee);
4410
4411 // Most calls are neither intrinsics nor known library functions. Bail out
4412 // before doing the (comparatively expensive) set-up below. This also covers
4413 // nobuiltin calls, since operator new calls are always library functions.
4414 if (Func == NotLibFunc && !isa<IntrinsicInst>(CI))
4415 return nullptr;
4416
4417 // TODO: Split out the code below that operates on FP calls so that
4418 // we can all non-FP calls with the StrictFP attribute to be
4419 // optimized.
4420 if (CI->isNoBuiltin()) {
4421 // Optionally update operator new calls.
4422 return maybeOptimizeNoBuiltinOperatorNew(CI, Builder);
4423 }
4424
4425 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4426
4428 CI->getOperandBundlesAsDefs(OpBundles);
4429
4431 Builder.setDefaultOperandBundles(OpBundles);
4432
4433 // Command-line parameter overrides instruction attribute.
4434 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4435 // used by the intrinsic optimizations.
4436 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4437 UnsafeFPShrink = EnableUnsafeFPShrink;
4438 else if (isa<FPMathOperator>(CI) && CI->isFast())
4439 UnsafeFPShrink = true;
4440
4441 // First, check for intrinsics.
4443 if (!IsCallingConvC)
4444 return nullptr;
4445 // The FP intrinsics have corresponding constrained versions so we don't
4446 // need to check for the StrictFP attribute here.
4447 switch (II->getIntrinsicID()) {
4448 case Intrinsic::pow:
4449 return optimizePow(CI, Builder);
4450 case Intrinsic::exp2:
4451 return optimizeExp2(CI, Builder);
4452 case Intrinsic::log:
4453 case Intrinsic::log2:
4454 case Intrinsic::log10:
4455 return optimizeLog(CI, Builder);
4456 case Intrinsic::sqrt:
4457 return optimizeSqrt(CI, Builder);
4458 case Intrinsic::memset:
4459 return optimizeMemSet(CI, Builder);
4460 case Intrinsic::memcpy:
4461 return optimizeMemCpy(CI, Builder);
4462 case Intrinsic::memmove:
4463 return optimizeMemMove(CI, Builder);
4464 case Intrinsic::sin:
4465 case Intrinsic::cos:
4466 if (UnsafeFPShrink)
4467 return optimizeUnaryDoubleFP(CI, Builder, TLI, /*isPrecise=*/true);
4468 return nullptr;
4469 case Intrinsic::sincos:
4470 if (UnsafeFPShrink)
4471 return optimizeSinCosDoubleFP(CI, Builder);
4472 return nullptr;
4473 default:
4474 return nullptr;
4475 }
4476 }
4477
4478 // Also try to simplify calls to fortified library functions.
4479 if (Value *SimplifiedFortifiedCI =
4480 FortifiedSimplifier.optimizeCall(CI, Builder))
4481 return SimplifiedFortifiedCI;
4482
4483 // Then check for known library functions.
4484 if (isLibFuncEmittable(M, TLI, Func)) {
4485 // We never change the calling convention.
4486 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4487 return nullptr;
4488 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4489 return V;
4490 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4491 return V;
4492 switch (Func) {
4493 case LibFunc_ffs:
4494 case LibFunc_ffsl:
4495 case LibFunc_ffsll:
4496 return optimizeFFS(CI, Builder);
4497 case LibFunc_fls:
4498 case LibFunc_flsl:
4499 case LibFunc_flsll:
4500 return optimizeFls(CI, Builder);
4501 case LibFunc_abs:
4502 case LibFunc_labs:
4503 case LibFunc_llabs:
4504 return optimizeAbs(CI, Builder);
4505 case LibFunc_isdigit:
4506 return optimizeIsDigit(CI, Builder);
4507 case LibFunc_isascii:
4508 return optimizeIsAscii(CI, Builder);
4509 case LibFunc_toascii:
4510 return optimizeToAscii(CI, Builder);
4511 case LibFunc_atoi:
4512 case LibFunc_atol:
4513 case LibFunc_atoll:
4514 return optimizeAtoi(CI, Builder);
4515 case LibFunc_strtol:
4516 case LibFunc_strtoll:
4517 return optimizeStrToInt(CI, Builder, /*AsSigned=*/true);
4518 case LibFunc_strtoul:
4519 case LibFunc_strtoull:
4520 return optimizeStrToInt(CI, Builder, /*AsSigned=*/false);
4521 case LibFunc_printf:
4522 return optimizePrintF(CI, Builder);
4523 case LibFunc_sprintf:
4524 return optimizeSPrintF(CI, Builder);
4525 case LibFunc_snprintf:
4526 return optimizeSnPrintF(CI, Builder);
4527 case LibFunc_fprintf:
4528 return optimizeFPrintF(CI, Builder);
4529 case LibFunc_fwrite:
4530 return optimizeFWrite(CI, Builder);
4531 case LibFunc_fputs:
4532 return optimizeFPuts(CI, Builder);
4533 case LibFunc_puts:
4534 return optimizePuts(CI, Builder);
4535 case LibFunc_perror:
4536 return optimizeErrorReporting(CI, Builder);
4537 case LibFunc_vfprintf:
4538 case LibFunc_fiprintf:
4539 return optimizeErrorReporting(CI, Builder, 0);
4540 case LibFunc_exit:
4541 case LibFunc_Exit:
4542 return optimizeExit(CI);
4543 default:
4544 return nullptr;
4545 }
4546 }
4547 return nullptr;
4548}
4549
4551 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4554 function_ref<void(Instruction *, Value *)> Replacer,
4555 function_ref<void(Instruction *)> Eraser)
4556 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4557 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4558
4559void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4560 // Indirect through the replacer used in this instance.
4561 Replacer(I, With);
4562}
4563
4564void LibCallSimplifier::eraseFromParent(Instruction *I) {
4565 Eraser(I);
4566}
4567
4568// TODO:
4569// Additional cases that we need to add to this file:
4570//
4571// cbrt:
4572// * cbrt(expN(X)) -> expN(x/3)
4573// * cbrt(sqrt(x)) -> pow(x,1/6)
4574// * cbrt(cbrt(x)) -> pow(x,1/9)
4575//
4576// exp, expf, expl:
4577// * exp(log(x)) -> x
4578//
4579// log, logf, logl:
4580// * log(exp(x)) -> x
4581// * log(exp(y)) -> y*log(e)
4582// * log(exp10(y)) -> y*log(10)
4583// * log(sqrt(x)) -> 0.5*log(x)
4584//
4585// pow, powf, powl:
4586// * pow(sqrt(x),y) -> pow(x,y*0.5)
4587// * pow(pow(x,y),z)-> pow(x,y*z)
4588//
4589// signbit:
4590// * signbit(cnst) -> cnst'
4591// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4592//
4593// sqrt, sqrtf, sqrtl:
4594// * sqrt(expN(x)) -> expN(x*0.5)
4595// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4596// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4597//
4598
4599//===----------------------------------------------------------------------===//
4600// Fortified Library Call Optimizations
4601//===----------------------------------------------------------------------===//
4602
4603bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4604 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4605 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4606 // If this function takes a flag argument, the implementation may use it to
4607 // perform extra checks. Don't fold into the non-checking variant.
4608 if (FlagOp) {
4609 ConstantInt *Flag = dyn_cast<ConstantInt>(CI->getArgOperand(*FlagOp));
4610 if (!Flag || !Flag->isZero())
4611 return false;
4612 }
4613
4614 if (SizeOp && CI->getArgOperand(ObjSizeOp) == CI->getArgOperand(*SizeOp))
4615 return true;
4616
4617 if (ConstantInt *ObjSizeCI =
4618 dyn_cast<ConstantInt>(CI->getArgOperand(ObjSizeOp))) {
4619 if (ObjSizeCI->isMinusOne())
4620 return true;
4621 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4622 if (OnlyLowerUnknownSize)
4623 return false;
4624 if (StrOp) {
4626 // If the length is 0 we don't know how long it is and so we can't
4627 // remove the check.
4628 if (Len)
4629 annotateDereferenceableBytes(CI, *StrOp, Len);
4630 else
4631 return false;
4632 return ObjSizeCI->getZExtValue() >= Len;
4633 }
4634
4635 if (SizeOp) {
4636 if (ConstantInt *SizeCI =
4638 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4639 }
4640 }
4641 return false;
4642}
4643
4644Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4645 IRBuilderBase &B) {
4646 if (isFortifiedCallFoldable(CI, 3, 2)) {
4647 CallInst *NewCI =
4648 B.CreateMemCpy(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
4649 Align(1), CI->getArgOperand(2));
4650 mergeAttributesAndFlags(NewCI, *CI);
4651 return CI->getArgOperand(0);
4652 }
4653 return nullptr;
4654}
4655
4656Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4657 IRBuilderBase &B) {
4658 if (isFortifiedCallFoldable(CI, 3, 2)) {
4659 CallInst *NewCI =
4660 B.CreateMemMove(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
4661 Align(1), CI->getArgOperand(2));
4662 mergeAttributesAndFlags(NewCI, *CI);
4663 return CI->getArgOperand(0);
4664 }
4665 return nullptr;
4666}
4667
4668Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4669 IRBuilderBase &B) {
4670 if (isFortifiedCallFoldable(CI, 3, 2)) {
4671 Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
4672 CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val,
4673 CI->getArgOperand(2), Align(1));
4674 mergeAttributesAndFlags(NewCI, *CI);
4675 return CI->getArgOperand(0);
4676 }
4677 return nullptr;
4678}
4679
4680Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4681 IRBuilderBase &B) {
4682 const DataLayout &DL = CI->getDataLayout();
4683 if (isFortifiedCallFoldable(CI, 3, 2))
4684 if (Value *Call = emitMemPCpy(CI->getArgOperand(0), CI->getArgOperand(1),
4685 CI->getArgOperand(2), B, DL, TLI)) {
4687 }
4688 return nullptr;
4689}
4690
4691Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4693 LibFunc Func) {
4694 const DataLayout &DL = CI->getDataLayout();
4695 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1),
4696 *ObjSize = CI->getArgOperand(2);
4697
4698 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4699 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4700 Value *StrLen = emitStrLen(Src, B, DL, TLI);
4701 return StrLen ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, StrLen) : nullptr;
4702 }
4703
4704 // If a) we don't have any length information, or b) we know this will
4705 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4706 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4707 // TODO: It might be nice to get a maximum length out of the possible
4708 // string lengths for varying.
4709 if (isFortifiedCallFoldable(CI, 2, std::nullopt, 1)) {
4710 if (Func == LibFunc_strcpy_chk)
4711 return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
4712 else
4713 return copyFlags(*CI, emitStpCpy(Dst, Src, B, TLI));
4714 }
4715
4716 if (OnlyLowerUnknownSize)
4717 return nullptr;
4718
4719 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4721 if (Len)
4722 annotateDereferenceableBytes(CI, 1, Len);
4723 else
4724 return nullptr;
4725
4726 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
4727 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
4728 Value *LenV = ConstantInt::get(SizeTTy, Len);
4729 Value *Ret = emitMemCpyChk(Dst, Src, LenV, ObjSize, B, DL, TLI);
4730 // If the function was an __stpcpy_chk, and we were able to fold it into
4731 // a __memcpy_chk, we still need to return the correct end pointer.
4732 if (Ret && Func == LibFunc_stpcpy_chk)
4733 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst,
4734 ConstantInt::get(SizeTTy, Len - 1));
4735 return copyFlags(*CI, cast<CallInst>(Ret));
4736}
4737
4738Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4739 IRBuilderBase &B) {
4740 if (isFortifiedCallFoldable(CI, 1, std::nullopt, 0))
4741 return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B,
4742 CI->getDataLayout(), TLI));
4743 return nullptr;
4744}
4745
4746Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4748 LibFunc Func) {
4749 if (isFortifiedCallFoldable(CI, 3, 2)) {
4750 if (Func == LibFunc_strncpy_chk)
4751 return copyFlags(*CI,
4753 CI->getArgOperand(2), B, TLI));
4754 else
4755 return copyFlags(*CI,
4757 CI->getArgOperand(2), B, TLI));
4758 }
4759
4760 return nullptr;
4761}
4762
4763Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4764 IRBuilderBase &B) {
4765 if (isFortifiedCallFoldable(CI, 4, 3))
4766 return copyFlags(
4767 *CI, emitMemCCpy(CI->getArgOperand(0), CI->getArgOperand(1),
4768 CI->getArgOperand(2), CI->getArgOperand(3), B, TLI));
4769
4770 return nullptr;
4771}
4772
4773Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4774 IRBuilderBase &B) {
4775 if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2)) {
4776 SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 5));
4777 return copyFlags(*CI,
4779 CI->getArgOperand(4), VariadicArgs, B, TLI));
4780 }
4781
4782 return nullptr;
4783}
4784
4785Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4786 IRBuilderBase &B) {
4787 if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1)) {
4788 SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 4));
4789 return copyFlags(*CI,
4791 VariadicArgs, B, TLI));
4792 }
4793
4794 return nullptr;
4795}
4796
4797Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4798 IRBuilderBase &B) {
4799 if (isFortifiedCallFoldable(CI, 2))
4800 return copyFlags(
4801 *CI, emitStrCat(CI->getArgOperand(0), CI->getArgOperand(1), B, TLI));
4802
4803 return nullptr;
4804}
4805
4806Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4807 IRBuilderBase &B) {
4808 if (isFortifiedCallFoldable(CI, 3))
4809 return copyFlags(*CI,
4811 CI->getArgOperand(2), B, TLI));
4812
4813 return nullptr;
4814}
4815
4816Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4817 IRBuilderBase &B) {
4818 if (isFortifiedCallFoldable(CI, 3))
4819 return copyFlags(*CI,
4821 CI->getArgOperand(2), B, TLI));
4822
4823 return nullptr;
4824}
4825
4826Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4827 IRBuilderBase &B) {
4828 if (isFortifiedCallFoldable(CI, 3))
4829 return copyFlags(*CI,
4831 CI->getArgOperand(2), B, TLI));
4832
4833 return nullptr;
4834}
4835
4836Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4837 IRBuilderBase &B) {
4838 if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2))
4839 return copyFlags(
4840 *CI, emitVSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
4841 CI->getArgOperand(4), CI->getArgOperand(5), B, TLI));
4842
4843 return nullptr;
4844}
4845
4846Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4847 IRBuilderBase &B) {
4848 if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1))
4849 return copyFlags(*CI,
4851 CI->getArgOperand(4), B, TLI));
4852
4853 return nullptr;
4854}
4855
4857 IRBuilderBase &Builder) {
4858 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4859 // Some clang users checked for _chk libcall availability using:
4860 // __has_builtin(__builtin___memcpy_chk)
4861 // When compiling with -fno-builtin, this is always true.
4862 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4863 // end up with fortified libcalls, which isn't acceptable in a freestanding
4864 // environment which only provides their non-fortified counterparts.
4865 //
4866 // Until we change clang and/or teach external users to check for availability
4867 // differently, disregard the "nobuiltin" attribute and TLI::has.
4868 //
4869 // PR23093.
4870
4871 Function *Callee = CI->getCalledFunction();
4872 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4873
4875 CI->getOperandBundlesAsDefs(OpBundles);
4876
4878 Builder.setDefaultOperandBundles(OpBundles);
4879
4880 // First, check that this is a known library functions and that the prototype
4881 // is correct.
4882 LibFunc Func = TLI->getLibFunc(*Callee);
4883 if (Func == NotLibFunc)
4884 return nullptr;
4885
4886 // We never change the calling convention.
4887 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4888 return nullptr;
4889
4890 switch (Func) {
4891 case LibFunc_memcpy_chk:
4892 return optimizeMemCpyChk(CI, Builder);
4893 case LibFunc_mempcpy_chk:
4894 return optimizeMemPCpyChk(CI, Builder);
4895 case LibFunc_memmove_chk:
4896 return optimizeMemMoveChk(CI, Builder);
4897 case LibFunc_memset_chk:
4898 return optimizeMemSetChk(CI, Builder);
4899 case LibFunc_stpcpy_chk:
4900 case LibFunc_strcpy_chk:
4901 return optimizeStrpCpyChk(CI, Builder, Func);
4902 case LibFunc_strlen_chk:
4903 return optimizeStrLenChk(CI, Builder);
4904 case LibFunc_stpncpy_chk:
4905 case LibFunc_strncpy_chk:
4906 return optimizeStrpNCpyChk(CI, Builder, Func);
4907 case LibFunc_memccpy_chk:
4908 return optimizeMemCCpyChk(CI, Builder);
4909 case LibFunc_snprintf_chk:
4910 return optimizeSNPrintfChk(CI, Builder);
4911 case LibFunc_sprintf_chk:
4912 return optimizeSPrintfChk(CI, Builder);
4913 case LibFunc_strcat_chk:
4914 return optimizeStrCatChk(CI, Builder);
4915 case LibFunc_strlcat_chk:
4916 return optimizeStrLCat(CI, Builder);
4917 case LibFunc_strncat_chk:
4918 return optimizeStrNCatChk(CI, Builder);
4919 case LibFunc_strlcpy_chk:
4920 return optimizeStrLCpyChk(CI, Builder);
4921 case LibFunc_vsnprintf_chk:
4922 return optimizeVSNPrintfChk(CI, Builder);
4923 case LibFunc_vsprintf_chk:
4924 return optimizeVSPrintfChk(CI, Builder);
4925 default:
4926 break;
4927 }
4928 return nullptr;
4929}
4930
4932 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4933 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
constexpr LLT S1
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
#define DEBUG_TYPE
Hexagon Common GEP
Module.h This file contains the declarations for the Module class.
static bool mayFlushDenormalsToPositiveZero(const CallInst *CI)
Flushing a denormal to +0.0 breaks f(-x) = -f(x) for odd f.
static llvm::Error parse(GsymDataExtractor &Data, uint64_t BaseAddr, LineEntryCallback const &Callback)
Definition LineTable.cpp:54
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define T
uint64_t IntrinsicInst * II
static bool isBinary(MachineInstr &MI)
if(PassOpts->AAPipeline)
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
static bool isOnlyUsedInEqualityComparison(Value *V, Value *With)
Return true if it is only used in equality comparisons with With.
static Value * optimizeSinCosDoubleFP(CallInst *CI, IRBuilderBase &B)
Shrink double -> float for llvm.sincos.
static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef< unsigned > ArgNos, Value *Size, const DataLayout &DL)
static cl::opt< unsigned, false, HotColdHintParser > ColdNewHintValue("cold-new-hint-value", cl::Hidden, cl::init(1), cl::desc("Value to pass to hot/cold operator new for cold allocation"))
static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg, bool UseFloat, Value *&Sin, Value *&Cos, Value *&SinCos, const TargetLibraryInfo *TLI)
static Value * mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old)
static cl::opt< bool > OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false), cl::desc("Enable hot/cold operator new library calls"))
static Value * optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float for binary functions.
static cl::opt< OptimizeExistingHotColdNewKind > OptimizeExistingHotColdNew("optimize-existing-hot-cold-new", cl::Hidden, cl::desc("Enable optimization of existing hot/cold operator new library calls"), cl::values(clEnumValN(OptimizeExistingHotColdNewKind::None, "none", "Do not optimize existing hot/cold operator new library calls"), clEnumValN(OptimizeExistingHotColdNewKind::Cold, "cold", "Only optimize existing hot/cold operator new library calls " "if determined to be cold"), clEnumValN(OptimizeExistingHotColdNewKind::Always, "always", "Always optimize existing hot/cold operator new library calls"), clEnumValN(OptimizeExistingHotColdNewKind::Always, "", "Always optimize existing hot/cold operator new library calls")), cl::init(OptimizeExistingHotColdNewKind::None), cl::ValueOptional)
static cl::opt< bool > MinExistingHotColdNewHint("min-existing-hot-cold-new-hint", cl::Hidden, cl::init(false), cl::desc("Take the minimum of compiler hint and existing hint when " "optimizing existing hot/cold operator new library calls"))
static bool ignoreCallingConv(LibFunc Func)
static void annotateDereferenceableBytes(CallInst *CI, ArrayRef< unsigned > ArgNos, uint64_t DereferenceableBytes)
static void copyFPMath(const CallInst &Old, Value *New)
Preserve the accuracy requirement of Old on the replacement New.
static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg)
static Value * optimizeDoubleFP(CallInst *CI, IRBuilderBase &B, bool isBinary, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float functions.
static Value * optimizeSymmetricCall(CallInst *CI, bool IsEven, IRBuilderBase &B)
static Value * getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno, Module *M, IRBuilderBase &B, const TargetLibraryInfo *TLI)
static Value * replaceBinaryCall(CallInst *CI, IRBuilderBase &B, Intrinsic::ID IID)
static Value * valueHasFloatPrecision(Value *Val)
Return a variant of Val with float type.
static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI, ArrayRef< unsigned > ArgNos, uint64_t CharSize=1)
static Value * optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS, uint64_t Len, IRBuilderBase &B, const DataLayout &DL)
static Value * createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M, IRBuilderBase &B)
static Value * convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr, uint64_t Base, bool AsSigned, IRBuilderBase &B)
static Value * memChrToCharCompare(CallInst *CI, Value *NBytes, IRBuilderBase &B, const DataLayout &DL)
static Value * copyFlags(const CallInst &Old, Value *New)
static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len, const SimplifyQuery &SQ)
static StringRef substr(StringRef Str, uint64_t Len)
static cl::opt< unsigned, false, HotColdHintParser > HotNewHintValue("hot-new-hint-value", cl::Hidden, cl::init(254), cl::desc("Value to pass to hot/cold operator new for hot allocation"))
static bool isTrigLibCall(CallInst *CI)
static Value * optimizeNaN(CallInst *CI)
Constant folding nan/nanf/nanl.
static bool isOnlyUsedInComparisonWithZero(Value *V)
static Value * replaceUnaryCall(CallInst *CI, IRBuilderBase &B, Intrinsic::ID IID)
static bool callHasFloatingPointArgument(const CallInst *CI)
static Value * optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float for unary functions.
static bool callHasFP128Argument(const CallInst *CI)
static cl::opt< bool > OptimizeNoBuiltinHotColdNew("optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(false), cl::desc("Enable transformation of nobuiltin operator new library calls"))
static cl::opt< unsigned, false, HotColdHintParser > AmbiguousNewHintValue("ambiguous-new-hint-value", cl::Hidden, cl::init(222), cl::desc("Value to pass to hot/cold operator new for ambiguous allocation"))
static Value * optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS, Value *Size, bool StrNCmp, IRBuilderBase &B, const DataLayout &DL)
static Value * getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth)
static cl::opt< bool > EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden, cl::init(false), cl::desc("Enable unsafe double to float " "shrinking for math lib calls"))
static cl::opt< unsigned, false, HotColdHintParser > NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(128), cl::desc("Value to pass to hot/cold operator new for " "notcold (warm) allocation"))
OptimizeExistingHotColdNewKind
This file defines the SmallString class.
This file contains some functions that are useful when dealing with strings.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Value * RHS
Value * LHS
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:364
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
opStatus
IEEE-754R 7: Default exception handling.
Definition APFloat.h:377
opStatus divide(const APFloat &RHS, roundingMode RM)
Definition APFloat.h:1312
bool isFiniteNonZero() const
Definition APFloat.h:1593
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition APFloat.cpp:6045
opStatus subtract(const APFloat &RHS, roundingMode RM)
Definition APFloat.h:1294
bool isNegative() const
Definition APFloat.h:1583
LLVM_ABI double convertToDouble() const
Converts this APFloat to host double value.
Definition APFloat.cpp:6278
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
Definition APFloat.h:1566
opStatus add(const APFloat &RHS, roundingMode RM)
Definition APFloat.h:1285
const fltSemantics & getSemantics() const
Definition APFloat.h:1591
LLVM_ABI float convertToFloat() const
Converts this APFloat to host float value.
Definition APFloat.cpp:6306
opStatus remainder(const APFloat &RHS)
Definition APFloat.h:1321
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition APFloat.h:1436
bool isInteger() const
Definition APFloat.h:1600
Class for arbitrary precision integers.
Definition APInt.h:78
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
A cache of @llvm.assume calls within a function.
static LLVM_ABI Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithCaptureInfo(LLVMContext &Context, CaptureInfo CI)
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
void removeParamAttrs(unsigned ArgNo, const AttributeMask &AttrsToRemove)
Removes the attributes from the given argument.
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool doesNotAccessMemory(unsigned OpNo) const
void removeRetAttrs(const AttributeMask &AttrsToRemove)
Removes the attributes from the return value.
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
bool isStrictFP() const
Determine if the call requires strict floating point semantics.
AttributeSet getParamAttributes(unsigned ArgNo) const
Return the param attributes for this call.
uint64_t getParamDereferenceableBytes(unsigned i) const
Extract the number of dereferenceable bytes for a call or parameter (0=unknown).
LLVM_ABI bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
AttributeSet getRetAttributes() const
Return the return attributes for this call.
void setAttributes(AttributeList A)
Set the attributes for this call.
bool doesNotThrow() const
Determine if the call cannot unwind.
Value * getArgOperand(unsigned i) const
uint64_t getParamDereferenceableOrNullBytes(unsigned i) const
Extract the number of dereferenceable_or_null bytes for a parameter (0=unknown).
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
bool isNoTailCall() const
TailCallKind getTailCallKind() const
bool isMustTailCall() const
static CaptureInfo none()
Create CaptureInfo that does not capture any components of the pointer.
Definition ModRef.h:427
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
Predicate getPredicate() const
Return the predicate for this instruction.
Definition InstrTypes.h:828
LLVM_ABI uint64_t getElementAsInteger(uint64_t i) const
If this is a sequential container of integers (of any size), return the specified element in the low ...
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
static LLVM_ABI ConstantFP * getQNaN(Type *Ty, bool Negative=false, APInt *Payload=nullptr)
static LLVM_ABI ConstantFP * getInfinity(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition Constants.h:225
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:174
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
This class represents an extension of floating point types.
This class represents a truncation of floating point types.
void setNoSignedZeros(bool B=true)
Definition FMF.h:84
static FastMathFlags getFast()
Definition FMF.h:50
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
LLVM_ABI FortifiedLibCallSimplifier(const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize=false)
LLVM_ABI Value * optimizeCall(CallInst *CI, IRBuilderBase &B)
Take the given call instruction and return a more optimal value to replace the instruction with or 0 ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:329
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition Function.h:252
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
Definition Function.cpp:807
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition Globals.cpp:408
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
Value * CreateLdexp(Value *Src, Value *Exp, FMFSource FMFSource={}, const Twine &Name="")
Create call to the ldexp intrinsic.
Definition IRBuilder.h:1086
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI bool hasNoNaNs() const LLVM_READONLY
Determine whether the no-NaNs flag is set.
LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI bool isFast() const LLVM_READONLY
Determine whether all fast-math-flags are set.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
iterator_range< user_iterator > users()
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
A wrapper class for inspecting calls to intrinsic functions.
bool isKnownNeverInfinity() const
Return true if it's known this can never be an infinity.
static constexpr FPClassTest OrderedLessThanZeroMask
LLVM_ABI bool isKnownNeverLogicalZero(DenormalMode Mode) const
Return true if it's known this can never be interpreted as a zero.
LLVM_ABI LibCallSimplifier(const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT, DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, function_ref< void(Instruction *, Value *)> Replacer=&replaceAllUsesWithDefault, function_ref< void(Instruction *)> Eraser=&eraseFromParentDefault)
LLVM_ABI Value * optimizeCall(CallInst *CI, IRBuilderBase &B)
optimizeCall - Take the given call instruction and return a more optimal value to replace the instruc...
An instruction for reading from memory.
Value * getPointerOperand()
static constexpr uint32_t kLikelyBranchWeight
The weight for a branch taken with high probability.
Definition MDBuilder.h:46
static constexpr uint32_t kUnlikelyBranchWeight
The weight for a branch taken with low probability.
Definition MDBuilder.h:53
Metadata node.
Definition Metadata.h:1081
iterator begin()
Definition MapVector.h:68
size_type size() const
Definition MapVector.h:59
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition Module.h:328
The optimization diagnostic interface.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Analysis providing profile information.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
static constexpr size_t npos
Definition StringRef.h:58
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition StringRef.h:490
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
char back() const
Get the last character in the string.
Definition StringRef.h:153
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition StringRef.h:446
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
Definition StringRef.h:270
int compare(StringRef RHS) const
Compare two strings; the result is negative, zero, or positive if this string is lexicographically le...
Definition StringRef.h:177
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:462
static LLVM_ABI bool isCallingConvCCompatible(CallBase *CI)
Returns true if call site / callee has cdecl-compatible calling conventions.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:292
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:271
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition Type.h:158
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:96
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:267
void setOperand(unsigned i, Value *Val)
Definition User.h:212
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
bool use_empty() const
Definition Value.h:348
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
CallInst * Call
#define UINT64_MAX
Definition DataTypes.h:77
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::FMul > m_FMul(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
match_deferred< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
specific_fpval m_SpecificFP(double V)
Match a specific floating point value or vector with all elements equal to the value.
auto m_CopySign(const Opnd0 &Op0, const Opnd1 &Op1)
auto m_Value()
Match an arbitrary value and ignore it.
specific_fpval m_FPOne()
Match a float 1.0 or vector with all elements equal to 1.0.
FNeg_match< OpTy, true > m_FNegOrFSubNegZero(const OpTy &X)
Match fneg X or fsub -0.0, X.
auto m_FAbs(const Opnd0 &Op0)
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
constexpr double e
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
LLVM_ABI Value * emitHotColdSizeReturningNewAligned(Value *Num, Value *Align, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, Value *HotCold, Type *RetTy)
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
LLVM_ABI Value * emitUnaryFloatFnCall(Value *Op, const TargetLibraryInfo *TLI, StringRef Name, IRBuilderBase &B, const AttributeList &Attrs)
Emit a call to the unary function named 'Name' (e.g.
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
LLVM_ABI Value * emitStrChr(Value *Ptr, char C, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strchr function to the builder, for the specified pointer and character.
constexpr uint64_t maxUIntN(uint64_t N)
Gets the maximum value for a N-bit unsigned integer.
Definition MathExtras.h:208
LLVM_ABI Value * emitPutChar(Value *Char, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the putchar function. This assumes that Char is an 'int'.
LLVM_ABI Value * emitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the __memcpy_chk function to the builder.
LLVM_ABI Value * emitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strncpy function to the builder, for the specified pointer arguments and length.
LLVM_ABI bool isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not an infinity or if the floating-point vector val...
@ Known
Known to have no common set bits.
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition APFloat.h:1736
LLVM_ABI bool getConstantStringInfo(const Value *V, StringRef &Str, bool TrimAtNul=true)
This function computes the length of a null-terminated C string pointed to by V.
LLVM_ABI Value * emitSPrintf(Value *Dest, Value *Fmt, ArrayRef< Value * > VariadicArgs, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the sprintf function.
LLVM_ABI bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
LLVM_ABI Value * emitMemRChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the memrchr function, analogously to emitMemChr.
LLVM_ABI Value * emitStrLCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strlcat function.
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI bool hasFloatFn(const Module *M, const TargetLibraryInfo *TLI, Type *Ty, LibFunc DoubleFn, LibFunc FloatFn, LibFunc LongDoubleFn)
Check whether the overloaded floating point function corresponding to Ty is available.
LLVM_ABI Value * emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, Value *HotCold)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI Value * emitStrNCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strncat function.
LLVM_ABI bool isLibFuncEmittable(const Module *M, const TargetLibraryInfo *TLI, LibFunc TheLibFunc)
Check whether the library function is available on target and also that it in the current Module is a...
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
LLVM_ABI Value * emitVSNPrintf(Value *Dest, Value *Size, Value *Fmt, Value *VAList, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the vsnprintf function.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI Value * emitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the strncmp function to the builder.
LLVM_ABI Value * emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the memcmp function.
LLVM_ABI Value * emitBinaryFloatFnCall(Value *Op1, Value *Op2, const TargetLibraryInfo *TLI, StringRef Name, IRBuilderBase &B, const AttributeList &Attrs)
Emit a call to the binary function named 'Name' (e.g.
bool isAlpha(char C)
Checks if character C is a valid letter as classified by "C" locale.
LLVM_ABI Value * emitFPutS(Value *Str, Value *File, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the fputs function.
LLVM_ABI Value * emitStrDup(Value *Ptr, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strdup function to the builder, for the specified pointer.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_ABI Value * emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, Value *HotCold)
LLVM_ABI Value * emitHotColdNewAlignedNoThrow(Value *Num, Value *Align, Value *NoThrow, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, Value *HotCold)
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
LLVM_ABI Value * emitBCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the bcmp function.
bool isDigit(char C)
Checks if character C is one of the 10 decimal digits.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition MathExtras.h:684
LLVM_ABI uint64_t GetStringLength(const Value *V, unsigned CharSize=8)
If we can compute the length of the string pointed to by the specified pointer, return 'len+1'.
LLVM_ABI FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI, LibFunc TheLibFunc, FunctionType *T, AttributeList AttributeList)
Calls getOrInsertFunction() and then makes sure to add mandatory argument attributes.
LLVM_ABI Value * emitStrLen(Value *Ptr, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the strlen function to the builder, for the specified pointer.
LLVM_ABI Value * emitFPutC(Value *Char, Value *File, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the fputc function.
LLVM_ABI Value * emitStpNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the stpncpy function to the builder, for the specified pointer arguments and length.
LLVM_ABI Value * emitStrCat(Value *Dest, Value *Src, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strcat function.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI Value * emitVSPrintf(Value *Dest, Value *Fmt, Value *VAList, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the vsprintf function.
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
LLVM_ABI Value * emitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the fwrite function.
LLVM_ABI Value * emitSNPrintf(Value *Dest, Value *Size, Value *Fmt, ArrayRef< Value * > Args, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the snprintf function.
@ Mod
The access may modify the value stored in memory.
Definition ModRef.h:34
LLVM_ABI Value * emitStpCpy(Value *Dst, Value *Src, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the stpcpy function to the builder, for the specified pointer arguments.
@ FMul
Product of floats.
@ And
Bitwise or logical AND of integers.
char toUpper(char x)
Returns the corresponding uppercase character if x is lowercase.
LLVM_ABI bool isOnlyUsedInZeroEqualityComparison(const Instruction *CtxI)
DWARFExpression::Operation Op
@ NearestTiesToEven
roundTiesToEven.
constexpr int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
Definition MathExtras.h:233
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI Value * emitMalloc(Value *Num, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the malloc function.
LLVM_ABI Value * emitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the memchr function.
bool isSpace(char C)
Checks whether character C is whitespace in the "C" locale.
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition Local.h:240
LLVM_ABI Value * emitPutS(Value *Str, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the puts function. This assumes that Str is some pointer.
LLVM_ABI Value * emitMemCCpy(Value *Ptr1, Value *Ptr2, Value *Val, Value *Len, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the memccpy function.
LLVM_ABI Value * emitHotColdNew(Value *Num, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, Value *HotCold)
Emit a call to the hot/cold operator new function.
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
LLVM_ABI Value * emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, Value *HotCold, Type *RetTy)
LLVM_ABI bool isDereferenceablePointer(const Value *V, Type *Ty, const SimplifyQuery &Q, bool IgnoreFree=false)
Equivalent to isDereferenceableAndAlignedPointer with an alignment of 1.
Definition Loads.cpp:266
LLVM_ABI Value * emitStrLCpy(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strlcpy function.
LLVM_ABI Value * emitStrCpy(Value *Dst, Value *Src, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strcpy function to the builder, for the specified pointer arguments.
@ Always
Always emit .debug_str_offsets talbes as DWARF64 for testing.
Definition DWP.h:32
LLVM_ABI Value * emitMemPCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the mempcpy function.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:368
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
const ConstantDataArray * Array
ConstantDataArray pointer.
Represent subnormal handling kind for floating point instruction inputs and outputs.
Matching combinators.