LLVM 20.0.0git
SimplifyLibCalls.cpp
Go to the documentation of this file.
1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can't be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/APSInt.h"
19#include "llvm/Analysis/Loads.h"
24#include "llvm/IR/DataLayout.h"
25#include "llvm/IR/Function.h"
26#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/Module.h"
39
40#include <cmath>
41
42using namespace llvm;
43using namespace PatternMatch;
44
45static cl::opt<bool>
46 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
47 cl::init(false),
48 cl::desc("Enable unsafe double to float "
49 "shrinking for math lib calls"));
50
51// Enable conversion of operator new calls with a MemProf hot or cold hint
52// to an operator new call that takes a hot/cold hint. Off by default since
53// not all allocators currently support this extension.
54static cl::opt<bool>
55 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false),
56 cl::desc("Enable hot/cold operator new library calls"));
58 "optimize-existing-hot-cold-new", cl::Hidden, cl::init(false),
60 "Enable optimization of existing hot/cold operator new library calls"));
61
62namespace {
63
64// Specialized parser to ensure the hint is an 8 bit value (we can't specify
65// uint8_t to opt<> as that is interpreted to mean that we are passing a char
66// option with a specific set of values.
67struct HotColdHintParser : public cl::parser<unsigned> {
68 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
69
70 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
71 if (Arg.getAsInteger(0, Value))
72 return O.error("'" + Arg + "' value invalid for uint argument!");
73
74 if (Value > 255)
75 return O.error("'" + Arg + "' value must be in the range [0, 255]!");
76
77 return false;
78 }
79};
80
81} // end anonymous namespace
82
83// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
84// and 255 is the hottest. Default to 1 value away from the coldest and hottest
85// hints, so that the compiler hinted allocations are slightly less strong than
86// manually inserted hints at the two extremes.
88 "cold-new-hint-value", cl::Hidden, cl::init(1),
89 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
91 NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(128),
92 cl::desc("Value to pass to hot/cold operator new for "
93 "notcold (warm) allocation"));
95 "hot-new-hint-value", cl::Hidden, cl::init(254),
96 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
97
98//===----------------------------------------------------------------------===//
99// Helper Functions
100//===----------------------------------------------------------------------===//
101
102static bool ignoreCallingConv(LibFunc Func) {
103 return Func == LibFunc_abs || Func == LibFunc_labs ||
104 Func == LibFunc_llabs || Func == LibFunc_strlen;
105}
106
107/// Return true if it is only used in equality comparisons with With.
109 for (User *U : V->users()) {
110 if (ICmpInst *IC = dyn_cast<ICmpInst>(U))
111 if (IC->isEquality() && IC->getOperand(1) == With)
112 continue;
113 // Unknown instruction.
114 return false;
115 }
116 return true;
117}
118
120 return any_of(CI->operands(), [](const Use &OI) {
121 return OI->getType()->isFloatingPointTy();
122 });
123}
124
125static bool callHasFP128Argument(const CallInst *CI) {
126 return any_of(CI->operands(), [](const Use &OI) {
127 return OI->getType()->isFP128Ty();
128 });
129}
130
131// Convert the entire string Str representing an integer in Base, up to
132// the terminating nul if present, to a constant according to the rules
133// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
134// return the result, otherwise null.
135// The function assumes the string is encoded in ASCII and carefully
136// avoids converting sequences (including "") that the corresponding
137// library call might fail and set errno for.
138static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
139 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
140 if (Base < 2 || Base > 36)
141 if (Base != 0)
142 // Fail for an invalid base (required by POSIX).
143 return nullptr;
144
145 // Current offset into the original string to reflect in EndPtr.
146 size_t Offset = 0;
147 // Strip leading whitespace.
148 for ( ; Offset != Str.size(); ++Offset)
149 if (!isSpace((unsigned char)Str[Offset])) {
150 Str = Str.substr(Offset);
151 break;
152 }
153
154 if (Str.empty())
155 // Fail for empty subject sequences (POSIX allows but doesn't require
156 // strtol[l]/strtoul[l] to fail with EINVAL).
157 return nullptr;
158
159 // Strip but remember the sign.
160 bool Negate = Str[0] == '-';
161 if (Str[0] == '-' || Str[0] == '+') {
162 Str = Str.drop_front();
163 if (Str.empty())
164 // Fail for a sign with nothing after it.
165 return nullptr;
166 ++Offset;
167 }
168
169 // Set Max to the absolute value of the minimum (for signed), or
170 // to the maximum (for unsigned) value representable in the type.
171 Type *RetTy = CI->getType();
172 unsigned NBits = RetTy->getPrimitiveSizeInBits();
173 uint64_t Max = AsSigned && Negate ? 1 : 0;
174 Max += AsSigned ? maxIntN(NBits) : maxUIntN(NBits);
175
176 // Autodetect Base if it's zero and consume the "0x" prefix.
177 if (Str.size() > 1) {
178 if (Str[0] == '0') {
179 if (toUpper((unsigned char)Str[1]) == 'X') {
180 if (Str.size() == 2 || (Base && Base != 16))
181 // Fail if Base doesn't allow the "0x" prefix or for the prefix
182 // alone that implementations like BSD set errno to EINVAL for.
183 return nullptr;
184
185 Str = Str.drop_front(2);
186 Offset += 2;
187 Base = 16;
188 }
189 else if (Base == 0)
190 Base = 8;
191 } else if (Base == 0)
192 Base = 10;
193 }
194 else if (Base == 0)
195 Base = 10;
196
197 // Convert the rest of the subject sequence, not including the sign,
198 // to its uint64_t representation (this assumes the source character
199 // set is ASCII).
200 uint64_t Result = 0;
201 for (unsigned i = 0; i != Str.size(); ++i) {
202 unsigned char DigVal = Str[i];
203 if (isDigit(DigVal))
204 DigVal = DigVal - '0';
205 else {
206 DigVal = toUpper(DigVal);
207 if (isAlpha(DigVal))
208 DigVal = DigVal - 'A' + 10;
209 else
210 return nullptr;
211 }
212
213 if (DigVal >= Base)
214 // Fail if the digit is not valid in the Base.
215 return nullptr;
216
217 // Add the digit and fail if the result is not representable in
218 // the (unsigned form of the) destination type.
219 bool VFlow;
220 Result = SaturatingMultiplyAdd(Result, Base, (uint64_t)DigVal, &VFlow);
221 if (VFlow || Result > Max)
222 return nullptr;
223 }
224
225 if (EndPtr) {
226 // Store the pointer to the end.
227 Value *Off = B.getInt64(Offset + Str.size());
228 Value *StrBeg = CI->getArgOperand(0);
229 Value *StrEnd = B.CreateInBoundsGEP(B.getInt8Ty(), StrBeg, Off, "endptr");
230 B.CreateStore(StrEnd, EndPtr);
231 }
232
233 if (Negate)
234 // Unsigned negation doesn't overflow.
235 Result = -Result;
236
237 return ConstantInt::get(RetTy, Result);
238}
239
241 for (User *U : V->users()) {
242 if (ICmpInst *IC = dyn_cast<ICmpInst>(U))
243 if (Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
244 if (C->isNullValue())
245 continue;
246 // Unknown instruction.
247 return false;
248 }
249 return true;
250}
251
252static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
253 const DataLayout &DL) {
255 return false;
256
257 if (!isDereferenceableAndAlignedPointer(Str, Align(1), APInt(64, Len), DL))
258 return false;
259
260 if (CI->getFunction()->hasFnAttribute(Attribute::SanitizeMemory))
261 return false;
262
263 return true;
264}
265
267 ArrayRef<unsigned> ArgNos,
268 uint64_t DereferenceableBytes) {
269 const Function *F = CI->getCaller();
270 if (!F)
271 return;
272 for (unsigned ArgNo : ArgNos) {
273 uint64_t DerefBytes = DereferenceableBytes;
274 unsigned AS = CI->getArgOperand(ArgNo)->getType()->getPointerAddressSpace();
275 if (!llvm::NullPointerIsDefined(F, AS) ||
276 CI->paramHasAttr(ArgNo, Attribute::NonNull))
277 DerefBytes = std::max(CI->getParamDereferenceableOrNullBytes(ArgNo),
278 DereferenceableBytes);
279
280 if (CI->getParamDereferenceableBytes(ArgNo) < DerefBytes) {
281 CI->removeParamAttr(ArgNo, Attribute::Dereferenceable);
282 if (!llvm::NullPointerIsDefined(F, AS) ||
283 CI->paramHasAttr(ArgNo, Attribute::NonNull))
284 CI->removeParamAttr(ArgNo, Attribute::DereferenceableOrNull);
286 CI->getContext(), DerefBytes));
287 }
288 }
289}
290
292 ArrayRef<unsigned> ArgNos) {
293 Function *F = CI->getCaller();
294 if (!F)
295 return;
296
297 for (unsigned ArgNo : ArgNos) {
298 if (!CI->paramHasAttr(ArgNo, Attribute::NoUndef))
299 CI->addParamAttr(ArgNo, Attribute::NoUndef);
300
301 if (!CI->paramHasAttr(ArgNo, Attribute::NonNull)) {
302 unsigned AS =
305 continue;
306 CI->addParamAttr(ArgNo, Attribute::NonNull);
307 }
308
309 annotateDereferenceableBytes(CI, ArgNo, 1);
310 }
311}
312
314 Value *Size, const DataLayout &DL) {
315 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size)) {
317 annotateDereferenceableBytes(CI, ArgNos, LenC->getZExtValue());
318 } else if (isKnownNonZero(Size, DL)) {
320 const APInt *X, *Y;
321 uint64_t DerefMin = 1;
322 if (match(Size, m_Select(m_Value(), m_APInt(X), m_APInt(Y)))) {
323 DerefMin = std::min(X->getZExtValue(), Y->getZExtValue());
324 annotateDereferenceableBytes(CI, ArgNos, DerefMin);
325 }
326 }
327}
328
329// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
330// easier chaining. Calls to emit* and B.createCall should probably be wrapped
331// in this function when New is created to replace Old. Callers should take
332// care to check Old.isMustTailCall() if they aren't replacing Old directly
333// with New.
334static Value *copyFlags(const CallInst &Old, Value *New) {
335 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
336 assert(!Old.isNoTailCall() && "do not copy notail call flags");
337 if (auto *NewCI = dyn_cast_or_null<CallInst>(New))
338 NewCI->setTailCallKind(Old.getTailCallKind());
339 return New;
340}
341
342static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
344 NewCI->getContext(), {NewCI->getAttributes(), Old.getAttributes()}));
346 NewCI->getType(), NewCI->getRetAttributes()));
347 for (unsigned I = 0; I < NewCI->arg_size(); ++I)
348 NewCI->removeParamAttrs(
350 NewCI->getParamAttributes(I)));
351
352 return copyFlags(Old, NewCI);
353}
354
355// Helper to avoid truncating the length if size_t is 32-bits.
357 return Len >= Str.size() ? Str : Str.substr(0, Len);
358}
359
360//===----------------------------------------------------------------------===//
361// String and Memory Library Call Optimizations
362//===----------------------------------------------------------------------===//
363
364Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
365 // Extract some information from the instruction
366 Value *Dst = CI->getArgOperand(0);
367 Value *Src = CI->getArgOperand(1);
369
370 // See if we can get the length of the input string.
372 if (Len)
374 else
375 return nullptr;
376 --Len; // Unbias length.
377
378 // Handle the simple, do-nothing case: strcat(x, "") -> x
379 if (Len == 0)
380 return Dst;
381
382 return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, Len, B));
383}
384
385Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
386 IRBuilderBase &B) {
387 // We need to find the end of the destination string. That's where the
388 // memory is to be moved to. We just generate a call to strlen.
389 Value *DstLen = emitStrLen(Dst, B, DL, TLI);
390 if (!DstLen)
391 return nullptr;
392
393 // Now that we have the destination's length, we must index into the
394 // destination's pointer to get the actual memcpy destination (end of
395 // the string .. we're concatenating).
396 Value *CpyDst = B.CreateInBoundsGEP(B.getInt8Ty(), Dst, DstLen, "endptr");
397
398 // We have enough information to now generate the memcpy call to do the
399 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
400 B.CreateMemCpy(CpyDst, Align(1), Src, Align(1),
401 TLI->getAsSizeT(Len + 1, *B.GetInsertBlock()->getModule()));
402 return Dst;
403}
404
405Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
406 // Extract some information from the instruction.
407 Value *Dst = CI->getArgOperand(0);
408 Value *Src = CI->getArgOperand(1);
409 Value *Size = CI->getArgOperand(2);
412 if (isKnownNonZero(Size, DL))
414
415 // We don't do anything if length is not constant.
416 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size);
417 if (LengthArg) {
418 Len = LengthArg->getZExtValue();
419 // strncat(x, c, 0) -> x
420 if (!Len)
421 return Dst;
422 } else {
423 return nullptr;
424 }
425
426 // See if we can get the length of the input string.
427 uint64_t SrcLen = GetStringLength(Src);
428 if (SrcLen) {
429 annotateDereferenceableBytes(CI, 1, SrcLen);
430 --SrcLen; // Unbias length.
431 } else {
432 return nullptr;
433 }
434
435 // strncat(x, "", c) -> x
436 if (SrcLen == 0)
437 return Dst;
438
439 // We don't optimize this case.
440 if (Len < SrcLen)
441 return nullptr;
442
443 // strncat(x, s, c) -> strcat(x, s)
444 // s is constant so the strcat can be optimized further.
445 return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, SrcLen, B));
446}
447
448// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
449// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
450// is that either S is dereferenceable or the value of N is nonzero.
452 IRBuilderBase &B, const DataLayout &DL)
453{
454 Value *Src = CI->getArgOperand(0);
455 Value *CharVal = CI->getArgOperand(1);
456
457 // Fold memchr(A, C, N) == A to N && *A == C.
458 Type *CharTy = B.getInt8Ty();
459 Value *Char0 = B.CreateLoad(CharTy, Src);
460 CharVal = B.CreateTrunc(CharVal, CharTy);
461 Value *Cmp = B.CreateICmpEQ(Char0, CharVal, "char0cmp");
462
463 if (NBytes) {
464 Value *Zero = ConstantInt::get(NBytes->getType(), 0);
465 Value *And = B.CreateICmpNE(NBytes, Zero);
466 Cmp = B.CreateLogicalAnd(And, Cmp);
467 }
468
469 Value *NullPtr = Constant::getNullValue(CI->getType());
470 return B.CreateSelect(Cmp, Src, NullPtr);
471}
472
473Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
474 Value *SrcStr = CI->getArgOperand(0);
475 Value *CharVal = CI->getArgOperand(1);
477
478 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
479 return memChrToCharCompare(CI, nullptr, B, DL);
480
481 // If the second operand is non-constant, see if we can compute the length
482 // of the input string and turn this into memchr.
483 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
484 if (!CharC) {
485 uint64_t Len = GetStringLength(SrcStr);
486 if (Len)
488 else
489 return nullptr;
490
492 FunctionType *FT = Callee->getFunctionType();
493 unsigned IntBits = TLI->getIntSize();
494 if (!FT->getParamType(1)->isIntegerTy(IntBits)) // memchr needs 'int'.
495 return nullptr;
496
497 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
498 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
499 return copyFlags(*CI,
500 emitMemChr(SrcStr, CharVal, // include nul.
501 ConstantInt::get(SizeTTy, Len), B,
502 DL, TLI));
503 }
504
505 if (CharC->isZero()) {
506 Value *NullPtr = Constant::getNullValue(CI->getType());
507 if (isOnlyUsedInEqualityComparison(CI, NullPtr))
508 // Pre-empt the transformation to strlen below and fold
509 // strchr(A, '\0') == null to false.
510 return B.CreateIntToPtr(B.getTrue(), CI->getType());
511 }
512
513 // Otherwise, the character is a constant, see if the first argument is
514 // a string literal. If so, we can constant fold.
515 StringRef Str;
516 if (!getConstantStringInfo(SrcStr, Str)) {
517 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
518 if (Value *StrLen = emitStrLen(SrcStr, B, DL, TLI))
519 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, StrLen, "strchr");
520 return nullptr;
521 }
522
523 // Compute the offset, make sure to handle the case when we're searching for
524 // zero (a weird way to spell strlen).
525 size_t I = (0xFF & CharC->getSExtValue()) == 0
526 ? Str.size()
527 : Str.find(CharC->getSExtValue());
528 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
529 return Constant::getNullValue(CI->getType());
530
531 // strchr(s+n,c) -> gep(s+n+i,c)
532 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(I), "strchr");
533}
534
535Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
536 Value *SrcStr = CI->getArgOperand(0);
537 Value *CharVal = CI->getArgOperand(1);
538 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
540
541 StringRef Str;
542 if (!getConstantStringInfo(SrcStr, Str)) {
543 // strrchr(s, 0) -> strchr(s, 0)
544 if (CharC && CharC->isZero())
545 return copyFlags(*CI, emitStrChr(SrcStr, '\0', B, TLI));
546 return nullptr;
547 }
548
549 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
550 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
551
552 // Try to expand strrchr to the memrchr nonstandard extension if it's
553 // available, or simply fail otherwise.
554 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
555 Value *Size = ConstantInt::get(SizeTTy, NBytes);
556 return copyFlags(*CI, emitMemRChr(SrcStr, CharVal, Size, B, DL, TLI));
557}
558
559Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
560 Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
561 if (Str1P == Str2P) // strcmp(x,x) -> 0
562 return ConstantInt::get(CI->getType(), 0);
563
564 StringRef Str1, Str2;
565 bool HasStr1 = getConstantStringInfo(Str1P, Str1);
566 bool HasStr2 = getConstantStringInfo(Str2P, Str2);
567
568 // strcmp(x, y) -> cnst (if both x and y are constant strings)
569 if (HasStr1 && HasStr2)
570 return ConstantInt::get(CI->getType(),
571 std::clamp(Str1.compare(Str2), -1, 1));
572
573 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
574 return B.CreateNeg(B.CreateZExt(
575 B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
576
577 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
578 return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
579 CI->getType());
580
581 // strcmp(P, "x") -> memcmp(P, "x", 2)
582 uint64_t Len1 = GetStringLength(Str1P);
583 if (Len1)
584 annotateDereferenceableBytes(CI, 0, Len1);
585 uint64_t Len2 = GetStringLength(Str2P);
586 if (Len2)
587 annotateDereferenceableBytes(CI, 1, Len2);
588
589 if (Len1 && Len2) {
590 return copyFlags(
591 *CI, emitMemCmp(Str1P, Str2P,
592 TLI->getAsSizeT(std::min(Len1, Len2), *CI->getModule()),
593 B, DL, TLI));
594 }
595
596 // strcmp to memcmp
597 if (!HasStr1 && HasStr2) {
598 if (canTransformToMemCmp(CI, Str1P, Len2, DL))
599 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
600 TLI->getAsSizeT(Len2, *CI->getModule()),
601 B, DL, TLI));
602 } else if (HasStr1 && !HasStr2) {
603 if (canTransformToMemCmp(CI, Str2P, Len1, DL))
604 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
605 TLI->getAsSizeT(Len1, *CI->getModule()),
606 B, DL, TLI));
607 }
608
610 return nullptr;
611}
612
613// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
614// arrays LHS and RHS and nonconstant Size.
615static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
616 Value *Size, bool StrNCmp,
617 IRBuilderBase &B, const DataLayout &DL);
618
619Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
620 Value *Str1P = CI->getArgOperand(0);
621 Value *Str2P = CI->getArgOperand(1);
622 Value *Size = CI->getArgOperand(2);
623 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
624 return ConstantInt::get(CI->getType(), 0);
625
626 if (isKnownNonZero(Size, DL))
628 // Get the length argument if it is constant.
630 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size))
631 Length = LengthArg->getZExtValue();
632 else
633 return optimizeMemCmpVarSize(CI, Str1P, Str2P, Size, true, B, DL);
634
635 if (Length == 0) // strncmp(x,y,0) -> 0
636 return ConstantInt::get(CI->getType(), 0);
637
638 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
639 return copyFlags(*CI, emitMemCmp(Str1P, Str2P, Size, B, DL, TLI));
640
641 StringRef Str1, Str2;
642 bool HasStr1 = getConstantStringInfo(Str1P, Str1);
643 bool HasStr2 = getConstantStringInfo(Str2P, Str2);
644
645 // strncmp(x, y) -> cnst (if both x and y are constant strings)
646 if (HasStr1 && HasStr2) {
647 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
648 StringRef SubStr1 = substr(Str1, Length);
649 StringRef SubStr2 = substr(Str2, Length);
650 return ConstantInt::get(CI->getType(),
651 std::clamp(SubStr1.compare(SubStr2), -1, 1));
652 }
653
654 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
655 return B.CreateNeg(B.CreateZExt(
656 B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
657
658 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
659 return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
660 CI->getType());
661
662 uint64_t Len1 = GetStringLength(Str1P);
663 if (Len1)
664 annotateDereferenceableBytes(CI, 0, Len1);
665 uint64_t Len2 = GetStringLength(Str2P);
666 if (Len2)
667 annotateDereferenceableBytes(CI, 1, Len2);
668
669 // strncmp to memcmp
670 if (!HasStr1 && HasStr2) {
671 Len2 = std::min(Len2, Length);
672 if (canTransformToMemCmp(CI, Str1P, Len2, DL))
673 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
674 TLI->getAsSizeT(Len2, *CI->getModule()),
675 B, DL, TLI));
676 } else if (HasStr1 && !HasStr2) {
677 Len1 = std::min(Len1, Length);
678 if (canTransformToMemCmp(CI, Str2P, Len1, DL))
679 return copyFlags(*CI, emitMemCmp(Str1P, Str2P,
680 TLI->getAsSizeT(Len1, *CI->getModule()),
681 B, DL, TLI));
682 }
683
684 return nullptr;
685}
686
687Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
688 Value *Src = CI->getArgOperand(0);
689 ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
690 uint64_t SrcLen = GetStringLength(Src);
691 if (SrcLen && Size) {
692 annotateDereferenceableBytes(CI, 0, SrcLen);
693 if (SrcLen <= Size->getZExtValue() + 1)
694 return copyFlags(*CI, emitStrDup(Src, B, TLI));
695 }
696
697 return nullptr;
698}
699
700Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
701 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
702 if (Dst == Src) // strcpy(x,x) -> x
703 return Src;
704
706 // See if we can get the length of the input string.
708 if (Len)
710 else
711 return nullptr;
712
713 // We have enough information to now generate the memcpy call to do the
714 // copy for us. Make a memcpy to copy the nul byte with align = 1.
715 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
716 TLI->getAsSizeT(Len, *CI->getModule()));
717 mergeAttributesAndFlags(NewCI, *CI);
718 return Dst;
719}
720
721Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
722 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
723
724 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
725 if (CI->use_empty())
726 return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
727
728 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
729 Value *StrLen = emitStrLen(Src, B, DL, TLI);
730 return StrLen ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, StrLen) : nullptr;
731 }
732
733 // See if we can get the length of the input string.
735 if (Len)
737 else
738 return nullptr;
739
740 Value *LenV = TLI->getAsSizeT(Len, *CI->getModule());
741 Value *DstEnd = B.CreateInBoundsGEP(
742 B.getInt8Ty(), Dst, TLI->getAsSizeT(Len - 1, *CI->getModule()));
743
744 // We have enough information to now generate the memcpy call to do the
745 // copy for us. Make a memcpy to copy the nul byte with align = 1.
746 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1), LenV);
747 mergeAttributesAndFlags(NewCI, *CI);
748 return DstEnd;
749}
750
751// Optimize a call to size_t strlcpy(char*, const char*, size_t).
752
753Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
754 Value *Size = CI->getArgOperand(2);
755 if (isKnownNonZero(Size, DL))
756 // Like snprintf, the function stores into the destination only when
757 // the size argument is nonzero.
759 // The function reads the source argument regardless of Size (it returns
760 // its length).
762
763 uint64_t NBytes;
764 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Size))
765 NBytes = SizeC->getZExtValue();
766 else
767 return nullptr;
768
769 Value *Dst = CI->getArgOperand(0);
770 Value *Src = CI->getArgOperand(1);
771 if (NBytes <= 1) {
772 if (NBytes == 1)
773 // For a call to strlcpy(D, S, 1) first store a nul in *D.
774 B.CreateStore(B.getInt8(0), Dst);
775
776 // Transform strlcpy(D, S, 0) to a call to strlen(S).
777 return copyFlags(*CI, emitStrLen(Src, B, DL, TLI));
778 }
779
780 // Try to determine the length of the source, substituting its size
781 // when it's not nul-terminated (as it's required to be) to avoid
782 // reading past its end.
783 StringRef Str;
784 if (!getConstantStringInfo(Src, Str, /*TrimAtNul=*/false))
785 return nullptr;
786
787 uint64_t SrcLen = Str.find('\0');
788 // Set if the terminating nul should be copied by the call to memcpy
789 // below.
790 bool NulTerm = SrcLen < NBytes;
791
792 if (NulTerm)
793 // Overwrite NBytes with the number of bytes to copy, including
794 // the terminating nul.
795 NBytes = SrcLen + 1;
796 else {
797 // Set the length of the source for the function to return to its
798 // size, and cap NBytes at the same.
799 SrcLen = std::min(SrcLen, uint64_t(Str.size()));
800 NBytes = std::min(NBytes - 1, SrcLen);
801 }
802
803 if (SrcLen == 0) {
804 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
805 B.CreateStore(B.getInt8(0), Dst);
806 return ConstantInt::get(CI->getType(), 0);
807 }
808
809 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
810 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
811 // D[N' - 1] if necessary.
812 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
813 TLI->getAsSizeT(NBytes, *CI->getModule()));
814 mergeAttributesAndFlags(NewCI, *CI);
815
816 if (!NulTerm) {
817 Value *EndOff = ConstantInt::get(CI->getType(), NBytes);
818 Value *EndPtr = B.CreateInBoundsGEP(B.getInt8Ty(), Dst, EndOff);
819 B.CreateStore(B.getInt8(0), EndPtr);
820 }
821
822 // Like snprintf, strlcpy returns the number of nonzero bytes that would
823 // have been copied if the bound had been sufficiently big (which in this
824 // case is strlen(Src)).
825 return ConstantInt::get(CI->getType(), SrcLen);
826}
827
828// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
829// otherwise.
830Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
831 IRBuilderBase &B) {
832 Value *Dst = CI->getArgOperand(0);
833 Value *Src = CI->getArgOperand(1);
834 Value *Size = CI->getArgOperand(2);
835
836 if (isKnownNonZero(Size, DL)) {
837 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
838 // only when N is nonzero.
841 }
842
843 // If the "bound" argument is known set N to it. Otherwise set it to
844 // UINT64_MAX and handle it later.
846 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Size))
847 N = SizeC->getZExtValue();
848
849 if (N == 0)
850 // Fold st{p,r}ncpy(D, S, 0) to D.
851 return Dst;
852
853 if (N == 1) {
854 Type *CharTy = B.getInt8Ty();
855 Value *CharVal = B.CreateLoad(CharTy, Src, "stxncpy.char0");
856 B.CreateStore(CharVal, Dst);
857 if (!RetEnd)
858 // Transform strncpy(D, S, 1) to return (*D = *S), D.
859 return Dst;
860
861 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
862 Value *ZeroChar = ConstantInt::get(CharTy, 0);
863 Value *Cmp = B.CreateICmpEQ(CharVal, ZeroChar, "stpncpy.char0cmp");
864
865 Value *Off1 = B.getInt32(1);
866 Value *EndPtr = B.CreateInBoundsGEP(CharTy, Dst, Off1, "stpncpy.end");
867 return B.CreateSelect(Cmp, Dst, EndPtr, "stpncpy.sel");
868 }
869
870 // If the length of the input string is known set SrcLen to it.
871 uint64_t SrcLen = GetStringLength(Src);
872 if (SrcLen)
873 annotateDereferenceableBytes(CI, 1, SrcLen);
874 else
875 return nullptr;
876
877 --SrcLen; // Unbias length.
878
879 if (SrcLen == 0) {
880 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
881 Align MemSetAlign =
883 CallInst *NewCI = B.CreateMemSet(Dst, B.getInt8('\0'), Size, MemSetAlign);
884 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(0));
886 CI->getContext(), 0, ArgAttrs));
887 copyFlags(*CI, NewCI);
888 return Dst;
889 }
890
891 if (N > SrcLen + 1) {
892 if (N > 128)
893 // Bail if N is large or unknown.
894 return nullptr;
895
896 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
897 StringRef Str;
898 if (!getConstantStringInfo(Src, Str))
899 return nullptr;
900 std::string SrcStr = Str.str();
901 // Create a bigger, nul-padded array with the same length, SrcLen,
902 // as the original string.
903 SrcStr.resize(N, '\0');
904 Src = B.CreateGlobalString(SrcStr, "str", /*AddressSpace=*/0,
905 /*M=*/nullptr, /*AddNull=*/false);
906 }
907
908 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
909 // S and N are constant.
910 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
911 TLI->getAsSizeT(N, *CI->getModule()));
912 mergeAttributesAndFlags(NewCI, *CI);
913 if (!RetEnd)
914 return Dst;
915
916 // stpncpy(D, S, N) returns the address of the first null in D if it writes
917 // one, otherwise D + N.
918 Value *Off = B.getInt64(std::min(SrcLen, N));
919 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, Off, "endptr");
920}
921
922Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
923 unsigned CharSize,
924 Value *Bound) {
925 Value *Src = CI->getArgOperand(0);
926 Type *CharTy = B.getIntNTy(CharSize);
927
929 (!Bound || isKnownNonZero(Bound, DL))) {
930 // Fold strlen:
931 // strlen(x) != 0 --> *x != 0
932 // strlen(x) == 0 --> *x == 0
933 // and likewise strnlen with constant N > 0:
934 // strnlen(x, N) != 0 --> *x != 0
935 // strnlen(x, N) == 0 --> *x == 0
936 return B.CreateZExt(B.CreateLoad(CharTy, Src, "char0"),
937 CI->getType());
938 }
939
940 if (Bound) {
941 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Bound)) {
942 if (BoundCst->isZero())
943 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
944 return ConstantInt::get(CI->getType(), 0);
945
946 if (BoundCst->isOne()) {
947 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
948 Value *CharVal = B.CreateLoad(CharTy, Src, "strnlen.char0");
949 Value *ZeroChar = ConstantInt::get(CharTy, 0);
950 Value *Cmp = B.CreateICmpNE(CharVal, ZeroChar, "strnlen.char0cmp");
951 return B.CreateZExt(Cmp, CI->getType());
952 }
953 }
954 }
955
956 if (uint64_t Len = GetStringLength(Src, CharSize)) {
957 Value *LenC = ConstantInt::get(CI->getType(), Len - 1);
958 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
959 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
960 if (Bound)
961 return B.CreateBinaryIntrinsic(Intrinsic::umin, LenC, Bound);
962 return LenC;
963 }
964
965 if (Bound)
966 // Punt for strnlen for now.
967 return nullptr;
968
969 // If s is a constant pointer pointing to a string literal, we can fold
970 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
971 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
972 // We only try to simplify strlen when the pointer s points to an array
973 // of CharSize elements. Otherwise, we would need to scale the offset x before
974 // doing the subtraction. This will make the optimization more complex, and
975 // it's not very useful because calling strlen for a pointer of other types is
976 // very uncommon.
977 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Src)) {
978 // TODO: Handle subobjects.
979 if (!isGEPBasedOnPointerToString(GEP, CharSize))
980 return nullptr;
981
983 if (getConstantDataArrayInfo(GEP->getOperand(0), Slice, CharSize)) {
984 uint64_t NullTermIdx;
985 if (Slice.Array == nullptr) {
986 NullTermIdx = 0;
987 } else {
988 NullTermIdx = ~((uint64_t)0);
989 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
990 if (Slice.Array->getElementAsInteger(I + Slice.Offset) == 0) {
991 NullTermIdx = I;
992 break;
993 }
994 }
995 // If the string does not have '\0', leave it to strlen to compute
996 // its length.
997 if (NullTermIdx == ~((uint64_t)0))
998 return nullptr;
999 }
1000
1001 Value *Offset = GEP->getOperand(2);
1002 KnownBits Known = computeKnownBits(Offset, DL, 0, nullptr, CI, nullptr);
1003 uint64_t ArrSize =
1004 cast<ArrayType>(GEP->getSourceElementType())->getNumElements();
1005
1006 // If Offset is not provably in the range [0, NullTermIdx], we can still
1007 // optimize if we can prove that the program has undefined behavior when
1008 // Offset is outside that range. That is the case when GEP->getOperand(0)
1009 // is a pointer to an object whose memory extent is NullTermIdx+1.
1010 if ((Known.isNonNegative() && Known.getMaxValue().ule(NullTermIdx)) ||
1011 (isa<GlobalVariable>(GEP->getOperand(0)) &&
1012 NullTermIdx == ArrSize - 1)) {
1013 Offset = B.CreateSExtOrTrunc(Offset, CI->getType());
1014 return B.CreateSub(ConstantInt::get(CI->getType(), NullTermIdx),
1015 Offset);
1016 }
1017 }
1018 }
1019
1020 // strlen(x?"foo":"bars") --> x ? 3 : 4
1021 if (SelectInst *SI = dyn_cast<SelectInst>(Src)) {
1022 uint64_t LenTrue = GetStringLength(SI->getTrueValue(), CharSize);
1023 uint64_t LenFalse = GetStringLength(SI->getFalseValue(), CharSize);
1024 if (LenTrue && LenFalse) {
1025 ORE.emit([&]() {
1026 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1027 << "folded strlen(select) to select of constants";
1028 });
1029 return B.CreateSelect(SI->getCondition(),
1030 ConstantInt::get(CI->getType(), LenTrue - 1),
1031 ConstantInt::get(CI->getType(), LenFalse - 1));
1032 }
1033 }
1034
1035 return nullptr;
1036}
1037
1038Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1039 if (Value *V = optimizeStringLength(CI, B, 8))
1040 return V;
1042 return nullptr;
1043}
1044
1045Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1046 Value *Bound = CI->getArgOperand(1);
1047 if (Value *V = optimizeStringLength(CI, B, 8, Bound))
1048 return V;
1049
1050 if (isKnownNonZero(Bound, DL))
1052 return nullptr;
1053}
1054
1055Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1056 Module &M = *CI->getModule();
1057 unsigned WCharSize = TLI->getWCharSize(M) * 8;
1058 // We cannot perform this optimization without wchar_size metadata.
1059 if (WCharSize == 0)
1060 return nullptr;
1061
1062 return optimizeStringLength(CI, B, WCharSize);
1063}
1064
1065Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1066 StringRef S1, S2;
1067 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1068 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1069
1070 // strpbrk(s, "") -> nullptr
1071 // strpbrk("", s) -> nullptr
1072 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1073 return Constant::getNullValue(CI->getType());
1074
1075 // Constant folding.
1076 if (HasS1 && HasS2) {
1077 size_t I = S1.find_first_of(S2);
1078 if (I == StringRef::npos) // No match.
1079 return Constant::getNullValue(CI->getType());
1080
1081 return B.CreateInBoundsGEP(B.getInt8Ty(), CI->getArgOperand(0),
1082 B.getInt64(I), "strpbrk");
1083 }
1084
1085 // strpbrk(s, "a") -> strchr(s, 'a')
1086 if (HasS2 && S2.size() == 1)
1087 return copyFlags(*CI, emitStrChr(CI->getArgOperand(0), S2[0], B, TLI));
1088
1089 return nullptr;
1090}
1091
1092Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1093 Value *EndPtr = CI->getArgOperand(1);
1094 if (isa<ConstantPointerNull>(EndPtr)) {
1095 // With a null EndPtr, this function won't capture the main argument.
1096 // It would be readonly too, except that it still may write to errno.
1097 CI->addParamAttr(0, Attribute::NoCapture);
1098 }
1099
1100 return nullptr;
1101}
1102
1103Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1104 StringRef S1, S2;
1105 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1106 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1107
1108 // strspn(s, "") -> 0
1109 // strspn("", s) -> 0
1110 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1111 return Constant::getNullValue(CI->getType());
1112
1113 // Constant folding.
1114 if (HasS1 && HasS2) {
1115 size_t Pos = S1.find_first_not_of(S2);
1116 if (Pos == StringRef::npos)
1117 Pos = S1.size();
1118 return ConstantInt::get(CI->getType(), Pos);
1119 }
1120
1121 return nullptr;
1122}
1123
1124Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1125 StringRef S1, S2;
1126 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1127 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1128
1129 // strcspn("", s) -> 0
1130 if (HasS1 && S1.empty())
1131 return Constant::getNullValue(CI->getType());
1132
1133 // Constant folding.
1134 if (HasS1 && HasS2) {
1135 size_t Pos = S1.find_first_of(S2);
1136 if (Pos == StringRef::npos)
1137 Pos = S1.size();
1138 return ConstantInt::get(CI->getType(), Pos);
1139 }
1140
1141 // strcspn(s, "") -> strlen(s)
1142 if (HasS2 && S2.empty())
1143 return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B, DL, TLI));
1144
1145 return nullptr;
1146}
1147
1148Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1149 // fold strstr(x, x) -> x.
1150 if (CI->getArgOperand(0) == CI->getArgOperand(1))
1151 return CI->getArgOperand(0);
1152
1153 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1155 Value *StrLen = emitStrLen(CI->getArgOperand(1), B, DL, TLI);
1156 if (!StrLen)
1157 return nullptr;
1158 Value *StrNCmp = emitStrNCmp(CI->getArgOperand(0), CI->getArgOperand(1),
1159 StrLen, B, DL, TLI);
1160 if (!StrNCmp)
1161 return nullptr;
1162 for (User *U : llvm::make_early_inc_range(CI->users())) {
1163 ICmpInst *Old = cast<ICmpInst>(U);
1164 Value *Cmp =
1165 B.CreateICmp(Old->getPredicate(), StrNCmp,
1166 ConstantInt::getNullValue(StrNCmp->getType()), "cmp");
1167 replaceAllUsesWith(Old, Cmp);
1168 }
1169 return CI;
1170 }
1171
1172 // See if either input string is a constant string.
1173 StringRef SearchStr, ToFindStr;
1174 bool HasStr1 = getConstantStringInfo(CI->getArgOperand(0), SearchStr);
1175 bool HasStr2 = getConstantStringInfo(CI->getArgOperand(1), ToFindStr);
1176
1177 // fold strstr(x, "") -> x.
1178 if (HasStr2 && ToFindStr.empty())
1179 return CI->getArgOperand(0);
1180
1181 // If both strings are known, constant fold it.
1182 if (HasStr1 && HasStr2) {
1183 size_t Offset = SearchStr.find(ToFindStr);
1184
1185 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1186 return Constant::getNullValue(CI->getType());
1187
1188 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1189 return B.CreateConstInBoundsGEP1_64(B.getInt8Ty(), CI->getArgOperand(0),
1190 Offset, "strstr");
1191 }
1192
1193 // fold strstr(x, "y") -> strchr(x, 'y').
1194 if (HasStr2 && ToFindStr.size() == 1) {
1195 return emitStrChr(CI->getArgOperand(0), ToFindStr[0], B, TLI);
1196 }
1197
1199 return nullptr;
1200}
1201
1202Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1203 Value *SrcStr = CI->getArgOperand(0);
1204 Value *Size = CI->getArgOperand(2);
1206 Value *CharVal = CI->getArgOperand(1);
1207 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1208 Value *NullPtr = Constant::getNullValue(CI->getType());
1209
1210 if (LenC) {
1211 if (LenC->isZero())
1212 // Fold memrchr(x, y, 0) --> null.
1213 return NullPtr;
1214
1215 if (LenC->isOne()) {
1216 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1217 // constant or otherwise.
1218 Value *Val = B.CreateLoad(B.getInt8Ty(), SrcStr, "memrchr.char0");
1219 // Slice off the character's high end bits.
1220 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1221 Value *Cmp = B.CreateICmpEQ(Val, CharVal, "memrchr.char0cmp");
1222 return B.CreateSelect(Cmp, SrcStr, NullPtr, "memrchr.sel");
1223 }
1224 }
1225
1226 StringRef Str;
1227 if (!getConstantStringInfo(SrcStr, Str, /*TrimAtNul=*/false))
1228 return nullptr;
1229
1230 if (Str.size() == 0)
1231 // If the array is empty fold memrchr(A, C, N) to null for any value
1232 // of C and N on the basis that the only valid value of N is zero
1233 // (otherwise the call is undefined).
1234 return NullPtr;
1235
1236 uint64_t EndOff = UINT64_MAX;
1237 if (LenC) {
1238 EndOff = LenC->getZExtValue();
1239 if (Str.size() < EndOff)
1240 // Punt out-of-bounds accesses to sanitizers and/or libc.
1241 return nullptr;
1242 }
1243
1244 if (ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal)) {
1245 // Fold memrchr(S, C, N) for a constant C.
1246 size_t Pos = Str.rfind(CharC->getZExtValue(), EndOff);
1247 if (Pos == StringRef::npos)
1248 // When the character is not in the source array fold the result
1249 // to null regardless of Size.
1250 return NullPtr;
1251
1252 if (LenC)
1253 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1254 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(Pos));
1255
1256 if (Str.find(Str[Pos]) == Pos) {
1257 // When there is just a single occurrence of C in S, i.e., the one
1258 // in Str[Pos], fold
1259 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1260 // for nonconstant N.
1261 Value *Cmp = B.CreateICmpULE(Size, ConstantInt::get(Size->getType(), Pos),
1262 "memrchr.cmp");
1263 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr,
1264 B.getInt64(Pos), "memrchr.ptr_plus");
1265 return B.CreateSelect(Cmp, NullPtr, SrcPlus, "memrchr.sel");
1266 }
1267 }
1268
1269 // Truncate the string to search at most EndOff characters.
1270 Str = Str.substr(0, EndOff);
1271 if (Str.find_first_not_of(Str[0]) != StringRef::npos)
1272 return nullptr;
1273
1274 // If the source array consists of all equal characters, then for any
1275 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1276 // N != 0 && *S == C ? S + N - 1 : null
1277 Type *SizeTy = Size->getType();
1278 Type *Int8Ty = B.getInt8Ty();
1279 Value *NNeZ = B.CreateICmpNE(Size, ConstantInt::get(SizeTy, 0));
1280 // Slice off the sought character's high end bits.
1281 CharVal = B.CreateTrunc(CharVal, Int8Ty);
1282 Value *CEqS0 = B.CreateICmpEQ(ConstantInt::get(Int8Ty, Str[0]), CharVal);
1283 Value *And = B.CreateLogicalAnd(NNeZ, CEqS0);
1284 Value *SizeM1 = B.CreateSub(Size, ConstantInt::get(SizeTy, 1));
1285 Value *SrcPlus =
1286 B.CreateInBoundsGEP(Int8Ty, SrcStr, SizeM1, "memrchr.ptr_plus");
1287 return B.CreateSelect(And, SrcPlus, NullPtr, "memrchr.sel");
1288}
1289
1290Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1291 Value *SrcStr = CI->getArgOperand(0);
1292 Value *Size = CI->getArgOperand(2);
1293
1294 if (isKnownNonZero(Size, DL)) {
1296 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
1297 return memChrToCharCompare(CI, Size, B, DL);
1298 }
1299
1300 Value *CharVal = CI->getArgOperand(1);
1301 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
1302 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1303 Value *NullPtr = Constant::getNullValue(CI->getType());
1304
1305 // memchr(x, y, 0) -> null
1306 if (LenC) {
1307 if (LenC->isZero())
1308 return NullPtr;
1309
1310 if (LenC->isOne()) {
1311 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1312 // constant or otherwise.
1313 Value *Val = B.CreateLoad(B.getInt8Ty(), SrcStr, "memchr.char0");
1314 // Slice off the character's high end bits.
1315 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1316 Value *Cmp = B.CreateICmpEQ(Val, CharVal, "memchr.char0cmp");
1317 return B.CreateSelect(Cmp, SrcStr, NullPtr, "memchr.sel");
1318 }
1319 }
1320
1321 StringRef Str;
1322 if (!getConstantStringInfo(SrcStr, Str, /*TrimAtNul=*/false))
1323 return nullptr;
1324
1325 if (CharC) {
1326 size_t Pos = Str.find(CharC->getZExtValue());
1327 if (Pos == StringRef::npos)
1328 // When the character is not in the source array fold the result
1329 // to null regardless of Size.
1330 return NullPtr;
1331
1332 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1333 // When the constant Size is less than or equal to the character
1334 // position also fold the result to null.
1335 Value *Cmp = B.CreateICmpULE(Size, ConstantInt::get(Size->getType(), Pos),
1336 "memchr.cmp");
1337 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(Pos),
1338 "memchr.ptr");
1339 return B.CreateSelect(Cmp, NullPtr, SrcPlus);
1340 }
1341
1342 if (Str.size() == 0)
1343 // If the array is empty fold memchr(A, C, N) to null for any value
1344 // of C and N on the basis that the only valid value of N is zero
1345 // (otherwise the call is undefined).
1346 return NullPtr;
1347
1348 if (LenC)
1349 Str = substr(Str, LenC->getZExtValue());
1350
1351 size_t Pos = Str.find_first_not_of(Str[0]);
1352 if (Pos == StringRef::npos
1353 || Str.find_first_not_of(Str[Pos], Pos) == StringRef::npos) {
1354 // If the source array consists of at most two consecutive sequences
1355 // of the same characters, then for any C and N (whether in bounds or
1356 // not), fold memchr(S, C, N) to
1357 // N != 0 && *S == C ? S : null
1358 // or for the two sequences to:
1359 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1360 // ^Sel2 ^Sel1 are denoted above.
1361 // The latter makes it also possible to fold strchr() calls with strings
1362 // of the same characters.
1363 Type *SizeTy = Size->getType();
1364 Type *Int8Ty = B.getInt8Ty();
1365
1366 // Slice off the sought character's high end bits.
1367 CharVal = B.CreateTrunc(CharVal, Int8Ty);
1368
1369 Value *Sel1 = NullPtr;
1370 if (Pos != StringRef::npos) {
1371 // Handle two consecutive sequences of the same characters.
1372 Value *PosVal = ConstantInt::get(SizeTy, Pos);
1373 Value *StrPos = ConstantInt::get(Int8Ty, Str[Pos]);
1374 Value *CEqSPos = B.CreateICmpEQ(CharVal, StrPos);
1375 Value *NGtPos = B.CreateICmp(ICmpInst::ICMP_UGT, Size, PosVal);
1376 Value *And = B.CreateAnd(CEqSPos, NGtPos);
1377 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, PosVal);
1378 Sel1 = B.CreateSelect(And, SrcPlus, NullPtr, "memchr.sel1");
1379 }
1380
1381 Value *Str0 = ConstantInt::get(Int8Ty, Str[0]);
1382 Value *CEqS0 = B.CreateICmpEQ(Str0, CharVal);
1383 Value *NNeZ = B.CreateICmpNE(Size, ConstantInt::get(SizeTy, 0));
1384 Value *And = B.CreateAnd(NNeZ, CEqS0);
1385 return B.CreateSelect(And, SrcStr, Sel1, "memchr.sel2");
1386 }
1387
1388 if (!LenC) {
1389 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
1390 // S is dereferenceable so it's safe to load from it and fold
1391 // memchr(S, C, N) == S to N && *S == C for any C and N.
1392 // TODO: This is safe even for nonconstant S.
1393 return memChrToCharCompare(CI, Size, B, DL);
1394
1395 // From now on we need a constant length and constant array.
1396 return nullptr;
1397 }
1398
1399 bool OptForSize = llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
1401
1402 // If the char is variable but the input str and length are not we can turn
1403 // this memchr call into a simple bit field test. Of course this only works
1404 // when the return value is only checked against null.
1405 //
1406 // It would be really nice to reuse switch lowering here but we can't change
1407 // the CFG at this point.
1408 //
1409 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1410 // != 0
1411 // after bounds check.
1412 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CI))
1413 return nullptr;
1414
1415 unsigned char Max =
1416 *std::max_element(reinterpret_cast<const unsigned char *>(Str.begin()),
1417 reinterpret_cast<const unsigned char *>(Str.end()));
1418
1419 // Make sure the bit field we're about to create fits in a register on the
1420 // target.
1421 // FIXME: On a 64 bit architecture this prevents us from using the
1422 // interesting range of alpha ascii chars. We could do better by emitting
1423 // two bitfields or shifting the range by 64 if no lower chars are used.
1424 if (!DL.fitsInLegalInteger(Max + 1)) {
1425 // Build chain of ORs
1426 // Transform:
1427 // memchr("abcd", C, 4) != nullptr
1428 // to:
1429 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1430 std::string SortedStr = Str.str();
1431 llvm::sort(SortedStr);
1432 // Compute the number of of non-contiguous ranges.
1433 unsigned NonContRanges = 1;
1434 for (size_t i = 1; i < SortedStr.size(); ++i) {
1435 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1436 NonContRanges++;
1437 }
1438 }
1439
1440 // Restrict this optimization to profitable cases with one or two range
1441 // checks.
1442 if (NonContRanges > 2)
1443 return nullptr;
1444
1445 // Slice off the character's high end bits.
1446 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1447
1448 SmallVector<Value *> CharCompares;
1449 for (unsigned char C : SortedStr)
1450 CharCompares.push_back(B.CreateICmpEQ(CharVal, B.getInt8(C)));
1451
1452 return B.CreateIntToPtr(B.CreateOr(CharCompares), CI->getType());
1453 }
1454
1455 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1456 // creating unnecessary illegal types.
1457 unsigned char Width = NextPowerOf2(std::max((unsigned char)7, Max));
1458
1459 // Now build the bit field.
1460 APInt Bitfield(Width, 0);
1461 for (char C : Str)
1462 Bitfield.setBit((unsigned char)C);
1463 Value *BitfieldC = B.getInt(Bitfield);
1464
1465 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1466 Value *C = B.CreateZExtOrTrunc(CharVal, BitfieldC->getType());
1467 C = B.CreateAnd(C, B.getIntN(Width, 0xFF));
1468
1469 // First check that the bit field access is within bounds.
1470 Value *Bounds = B.CreateICmp(ICmpInst::ICMP_ULT, C, B.getIntN(Width, Width),
1471 "memchr.bounds");
1472
1473 // Create code that checks if the given bit is set in the field.
1474 Value *Shl = B.CreateShl(B.getIntN(Width, 1ULL), C);
1475 Value *Bits = B.CreateIsNotNull(B.CreateAnd(Shl, BitfieldC), "memchr.bits");
1476
1477 // Finally merge both checks and cast to pointer type. The inttoptr
1478 // implicitly zexts the i1 to intptr type.
1479 return B.CreateIntToPtr(B.CreateLogicalAnd(Bounds, Bits, "memchr"),
1480 CI->getType());
1481}
1482
1483// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1484// arrays LHS and RHS and nonconstant Size.
1486 Value *Size, bool StrNCmp,
1487 IRBuilderBase &B, const DataLayout &DL) {
1488 if (LHS == RHS) // memcmp(s,s,x) -> 0
1489 return Constant::getNullValue(CI->getType());
1490
1491 StringRef LStr, RStr;
1492 if (!getConstantStringInfo(LHS, LStr, /*TrimAtNul=*/false) ||
1493 !getConstantStringInfo(RHS, RStr, /*TrimAtNul=*/false))
1494 return nullptr;
1495
1496 // If the contents of both constant arrays are known, fold a call to
1497 // memcmp(A, B, N) to
1498 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1499 // where Pos is the first mismatch between A and B, determined below.
1500
1501 uint64_t Pos = 0;
1502 Value *Zero = ConstantInt::get(CI->getType(), 0);
1503 for (uint64_t MinSize = std::min(LStr.size(), RStr.size()); ; ++Pos) {
1504 if (Pos == MinSize ||
1505 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1506 // One array is a leading part of the other of equal or greater
1507 // size, or for strncmp, the arrays are equal strings.
1508 // Fold the result to zero. Size is assumed to be in bounds, since
1509 // otherwise the call would be undefined.
1510 return Zero;
1511 }
1512
1513 if (LStr[Pos] != RStr[Pos])
1514 break;
1515 }
1516
1517 // Normalize the result.
1518 typedef unsigned char UChar;
1519 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1520 Value *MaxSize = ConstantInt::get(Size->getType(), Pos);
1521 Value *Cmp = B.CreateICmp(ICmpInst::ICMP_ULE, Size, MaxSize);
1522 Value *Res = ConstantInt::get(CI->getType(), IRes);
1523 return B.CreateSelect(Cmp, Zero, Res);
1524}
1525
1526// Optimize a memcmp call CI with constant size Len.
1528 uint64_t Len, IRBuilderBase &B,
1529 const DataLayout &DL) {
1530 if (Len == 0) // memcmp(s1,s2,0) -> 0
1531 return Constant::getNullValue(CI->getType());
1532
1533 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1534 if (Len == 1) {
1535 Value *LHSV = B.CreateZExt(B.CreateLoad(B.getInt8Ty(), LHS, "lhsc"),
1536 CI->getType(), "lhsv");
1537 Value *RHSV = B.CreateZExt(B.CreateLoad(B.getInt8Ty(), RHS, "rhsc"),
1538 CI->getType(), "rhsv");
1539 return B.CreateSub(LHSV, RHSV, "chardiff");
1540 }
1541
1542 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1543 // TODO: The case where both inputs are constants does not need to be limited
1544 // to legal integers or equality comparison. See block below this.
1545 if (DL.isLegalInteger(Len * 8) && isOnlyUsedInZeroEqualityComparison(CI)) {
1546 IntegerType *IntType = IntegerType::get(CI->getContext(), Len * 8);
1547 Align PrefAlignment = DL.getPrefTypeAlign(IntType);
1548
1549 // First, see if we can fold either argument to a constant.
1550 Value *LHSV = nullptr;
1551 if (auto *LHSC = dyn_cast<Constant>(LHS))
1552 LHSV = ConstantFoldLoadFromConstPtr(LHSC, IntType, DL);
1553
1554 Value *RHSV = nullptr;
1555 if (auto *RHSC = dyn_cast<Constant>(RHS))
1556 RHSV = ConstantFoldLoadFromConstPtr(RHSC, IntType, DL);
1557
1558 // Don't generate unaligned loads. If either source is constant data,
1559 // alignment doesn't matter for that source because there is no load.
1560 if ((LHSV || getKnownAlignment(LHS, DL, CI) >= PrefAlignment) &&
1561 (RHSV || getKnownAlignment(RHS, DL, CI) >= PrefAlignment)) {
1562 if (!LHSV)
1563 LHSV = B.CreateLoad(IntType, LHS, "lhsv");
1564 if (!RHSV)
1565 RHSV = B.CreateLoad(IntType, RHS, "rhsv");
1566 return B.CreateZExt(B.CreateICmpNE(LHSV, RHSV), CI->getType(), "memcmp");
1567 }
1568 }
1569
1570 return nullptr;
1571}
1572
1573// Most simplifications for memcmp also apply to bcmp.
1574Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1575 IRBuilderBase &B) {
1576 Value *LHS = CI->getArgOperand(0), *RHS = CI->getArgOperand(1);
1577 Value *Size = CI->getArgOperand(2);
1578
1579 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1580
1581 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, false, B, DL))
1582 return Res;
1583
1584 // Handle constant Size.
1585 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1586 if (!LenC)
1587 return nullptr;
1588
1589 return optimizeMemCmpConstantSize(CI, LHS, RHS, LenC->getZExtValue(), B, DL);
1590}
1591
1592Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1593 Module *M = CI->getModule();
1594 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1595 return V;
1596
1597 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1598 // bcmp can be more efficient than memcmp because it only has to know that
1599 // there is a difference, not how different one is to the other.
1600 if (isLibFuncEmittable(M, TLI, LibFunc_bcmp) &&
1602 Value *LHS = CI->getArgOperand(0);
1603 Value *RHS = CI->getArgOperand(1);
1604 Value *Size = CI->getArgOperand(2);
1605 return copyFlags(*CI, emitBCmp(LHS, RHS, Size, B, DL, TLI));
1606 }
1607
1608 return nullptr;
1609}
1610
1611Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1612 return optimizeMemCmpBCmpCommon(CI, B);
1613}
1614
1615Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1616 Value *Size = CI->getArgOperand(2);
1617 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1618 if (isa<IntrinsicInst>(CI))
1619 return nullptr;
1620
1621 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1622 CallInst *NewCI = B.CreateMemCpy(CI->getArgOperand(0), Align(1),
1623 CI->getArgOperand(1), Align(1), Size);
1624 mergeAttributesAndFlags(NewCI, *CI);
1625 return CI->getArgOperand(0);
1626}
1627
1628Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1629 Value *Dst = CI->getArgOperand(0);
1630 Value *Src = CI->getArgOperand(1);
1631 ConstantInt *StopChar = dyn_cast<ConstantInt>(CI->getArgOperand(2));
1632 ConstantInt *N = dyn_cast<ConstantInt>(CI->getArgOperand(3));
1633 StringRef SrcStr;
1634 if (CI->use_empty() && Dst == Src)
1635 return Dst;
1636 // memccpy(d, s, c, 0) -> nullptr
1637 if (N) {
1638 if (N->isNullValue())
1639 return Constant::getNullValue(CI->getType());
1640 if (!getConstantStringInfo(Src, SrcStr, /*TrimAtNul=*/false) ||
1641 // TODO: Handle zeroinitializer.
1642 !StopChar)
1643 return nullptr;
1644 } else {
1645 return nullptr;
1646 }
1647
1648 // Wrap arg 'c' of type int to char
1649 size_t Pos = SrcStr.find(StopChar->getSExtValue() & 0xFF);
1650 if (Pos == StringRef::npos) {
1651 if (N->getZExtValue() <= SrcStr.size()) {
1652 copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1),
1653 CI->getArgOperand(3)));
1654 return Constant::getNullValue(CI->getType());
1655 }
1656 return nullptr;
1657 }
1658
1659 Value *NewN =
1660 ConstantInt::get(N->getType(), std::min(uint64_t(Pos + 1), N->getZExtValue()));
1661 // memccpy -> llvm.memcpy
1662 copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1), NewN));
1663 return Pos + 1 <= N->getZExtValue()
1664 ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, NewN)
1666}
1667
1668Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1669 Value *Dst = CI->getArgOperand(0);
1670 Value *N = CI->getArgOperand(2);
1671 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1672 CallInst *NewCI =
1673 B.CreateMemCpy(Dst, Align(1), CI->getArgOperand(1), Align(1), N);
1674 // Propagate attributes, but memcpy has no return value, so make sure that
1675 // any return attributes are compliant.
1676 // TODO: Attach return value attributes to the 1st operand to preserve them?
1677 mergeAttributesAndFlags(NewCI, *CI);
1678 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, N);
1679}
1680
1681Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1682 Value *Size = CI->getArgOperand(2);
1683 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1684 if (isa<IntrinsicInst>(CI))
1685 return nullptr;
1686
1687 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1688 CallInst *NewCI = B.CreateMemMove(CI->getArgOperand(0), Align(1),
1689 CI->getArgOperand(1), Align(1), Size);
1690 mergeAttributesAndFlags(NewCI, *CI);
1691 return CI->getArgOperand(0);
1692}
1693
1694Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1695 Value *Size = CI->getArgOperand(2);
1697 if (isa<IntrinsicInst>(CI))
1698 return nullptr;
1699
1700 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1701 Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
1702 CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val, Size, Align(1));
1703 mergeAttributesAndFlags(NewCI, *CI);
1704 return CI->getArgOperand(0);
1705}
1706
1707Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1708 if (isa<ConstantPointerNull>(CI->getArgOperand(0)))
1709 return copyFlags(*CI, emitMalloc(CI->getArgOperand(1), B, DL, TLI));
1710
1711 return nullptr;
1712}
1713
1714// When enabled, replace operator new() calls marked with a hot or cold memprof
1715// attribute with an operator new() call that takes a __hot_cold_t parameter.
1716// Currently this is supported by the open source version of tcmalloc, see:
1717// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1718Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1719 LibFunc &Func) {
1720 if (!OptimizeHotColdNew)
1721 return nullptr;
1722
1723 uint8_t HotCold;
1724 if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "cold")
1725 HotCold = ColdNewHintValue;
1726 else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() ==
1727 "notcold")
1728 HotCold = NotColdNewHintValue;
1729 else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "hot")
1730 HotCold = HotNewHintValue;
1731 else
1732 return nullptr;
1733
1734 // For calls that already pass a hot/cold hint, only update the hint if
1735 // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
1736 // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
1737 // Note that in cases where we decide it is "notcold", it might be slightly
1738 // better to replace the hinted call with a non hinted call, to avoid the
1739 // extra parameter and the if condition check of the hint value in the
1740 // allocator. This can be considered in the future.
1741 switch (Func) {
1742 case LibFunc_Znwm12__hot_cold_t:
1744 return emitHotColdNew(CI->getArgOperand(0), B, TLI,
1745 LibFunc_Znwm12__hot_cold_t, HotCold);
1746 break;
1747 case LibFunc_Znwm:
1748 if (HotCold != NotColdNewHintValue)
1749 return emitHotColdNew(CI->getArgOperand(0), B, TLI,
1750 LibFunc_Znwm12__hot_cold_t, HotCold);
1751 break;
1752 case LibFunc_Znam12__hot_cold_t:
1754 return emitHotColdNew(CI->getArgOperand(0), B, TLI,
1755 LibFunc_Znam12__hot_cold_t, HotCold);
1756 break;
1757 case LibFunc_Znam:
1758 if (HotCold != NotColdNewHintValue)
1759 return emitHotColdNew(CI->getArgOperand(0), B, TLI,
1760 LibFunc_Znam12__hot_cold_t, HotCold);
1761 break;
1762 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
1764 return emitHotColdNewNoThrow(
1765 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1766 LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
1767 break;
1768 case LibFunc_ZnwmRKSt9nothrow_t:
1769 if (HotCold != NotColdNewHintValue)
1770 return emitHotColdNewNoThrow(
1771 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1772 LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
1773 break;
1774 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
1776 return emitHotColdNewNoThrow(
1777 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1778 LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
1779 break;
1780 case LibFunc_ZnamRKSt9nothrow_t:
1781 if (HotCold != NotColdNewHintValue)
1782 return emitHotColdNewNoThrow(
1783 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1784 LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
1785 break;
1786 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
1788 return emitHotColdNewAligned(
1789 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1790 LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
1791 break;
1792 case LibFunc_ZnwmSt11align_val_t:
1793 if (HotCold != NotColdNewHintValue)
1794 return emitHotColdNewAligned(
1795 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1796 LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
1797 break;
1798 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
1800 return emitHotColdNewAligned(
1801 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1802 LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
1803 break;
1804 case LibFunc_ZnamSt11align_val_t:
1805 if (HotCold != NotColdNewHintValue)
1806 return emitHotColdNewAligned(
1807 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1808 LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
1809 break;
1810 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1813 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1814 TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1815 HotCold);
1816 break;
1817 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1818 if (HotCold != NotColdNewHintValue)
1820 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1821 TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1822 HotCold);
1823 break;
1824 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
1827 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1828 TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1829 HotCold);
1830 break;
1831 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1832 if (HotCold != NotColdNewHintValue)
1834 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1835 TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
1836 HotCold);
1837 break;
1838 case LibFunc_size_returning_new:
1839 if (HotCold != NotColdNewHintValue)
1840 return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
1841 LibFunc_size_returning_new_hot_cold,
1842 HotCold);
1843 break;
1844 case LibFunc_size_returning_new_hot_cold:
1846 return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
1847 LibFunc_size_returning_new_hot_cold,
1848 HotCold);
1849 break;
1850 case LibFunc_size_returning_new_aligned:
1851 if (HotCold != NotColdNewHintValue)
1853 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1854 LibFunc_size_returning_new_aligned_hot_cold, HotCold);
1855 break;
1856 case LibFunc_size_returning_new_aligned_hot_cold:
1859 CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
1860 LibFunc_size_returning_new_aligned_hot_cold, HotCold);
1861 break;
1862 default:
1863 return nullptr;
1864 }
1865 return nullptr;
1866}
1867
1868//===----------------------------------------------------------------------===//
1869// Math Library Optimizations
1870//===----------------------------------------------------------------------===//
1871
1872// Replace a libcall \p CI with a call to intrinsic \p IID
1874 Intrinsic::ID IID) {
1875 CallInst *NewCall = B.CreateUnaryIntrinsic(IID, CI->getArgOperand(0), CI);
1876 NewCall->takeName(CI);
1877 return copyFlags(*CI, NewCall);
1878}
1879
1880/// Return a variant of Val with float type.
1881/// Currently this works in two cases: If Val is an FPExtension of a float
1882/// value to something bigger, simply return the operand.
1883/// If Val is a ConstantFP but can be converted to a float ConstantFP without
1884/// loss of precision do so.
1886 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
1887 Value *Op = Cast->getOperand(0);
1888 if (Op->getType()->isFloatTy())
1889 return Op;
1890 }
1891 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
1892 APFloat F = Const->getValueAPF();
1893 bool losesInfo;
1895 &losesInfo);
1896 if (!losesInfo)
1897 return ConstantFP::get(Const->getContext(), F);
1898 }
1899 return nullptr;
1900}
1901
1902/// Shrink double -> float functions.
1904 bool isBinary, const TargetLibraryInfo *TLI,
1905 bool isPrecise = false) {
1906 Function *CalleeFn = CI->getCalledFunction();
1907 if (!CI->getType()->isDoubleTy() || !CalleeFn)
1908 return nullptr;
1909
1910 // If not all the uses of the function are converted to float, then bail out.
1911 // This matters if the precision of the result is more important than the
1912 // precision of the arguments.
1913 if (isPrecise)
1914 for (User *U : CI->users()) {
1915 FPTruncInst *Cast = dyn_cast<FPTruncInst>(U);
1916 if (!Cast || !Cast->getType()->isFloatTy())
1917 return nullptr;
1918 }
1919
1920 // If this is something like 'g((double) float)', convert to 'gf(float)'.
1921 Value *V[2];
1923 V[1] = isBinary ? valueHasFloatPrecision(CI->getArgOperand(1)) : nullptr;
1924 if (!V[0] || (isBinary && !V[1]))
1925 return nullptr;
1926
1927 // If call isn't an intrinsic, check that it isn't within a function with the
1928 // same name as the float version of this call, otherwise the result is an
1929 // infinite loop. For example, from MinGW-w64:
1930 //
1931 // float expf(float val) { return (float) exp((double) val); }
1932 StringRef CalleeName = CalleeFn->getName();
1933 bool IsIntrinsic = CalleeFn->isIntrinsic();
1934 if (!IsIntrinsic) {
1935 StringRef CallerName = CI->getFunction()->getName();
1936 if (!CallerName.empty() && CallerName.back() == 'f' &&
1937 CallerName.size() == (CalleeName.size() + 1) &&
1938 CallerName.starts_with(CalleeName))
1939 return nullptr;
1940 }
1941
1942 // Propagate the math semantics from the current function to the new function.
1944 B.setFastMathFlags(CI->getFastMathFlags());
1945
1946 // g((double) float) -> (double) gf(float)
1947 Value *R;
1948 if (IsIntrinsic) {
1949 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
1950 R = isBinary ? B.CreateIntrinsic(IID, B.getFloatTy(), V)
1951 : B.CreateIntrinsic(IID, B.getFloatTy(), V[0]);
1952 } else {
1953 AttributeList CalleeAttrs = CalleeFn->getAttributes();
1954 R = isBinary ? emitBinaryFloatFnCall(V[0], V[1], TLI, CalleeName, B,
1955 CalleeAttrs)
1956 : emitUnaryFloatFnCall(V[0], TLI, CalleeName, B, CalleeAttrs);
1957 }
1958 return B.CreateFPExt(R, B.getDoubleTy());
1959}
1960
1961/// Shrink double -> float for unary functions.
1963 const TargetLibraryInfo *TLI,
1964 bool isPrecise = false) {
1965 return optimizeDoubleFP(CI, B, false, TLI, isPrecise);
1966}
1967
1968/// Shrink double -> float for binary functions.
1970 const TargetLibraryInfo *TLI,
1971 bool isPrecise = false) {
1972 return optimizeDoubleFP(CI, B, true, TLI, isPrecise);
1973}
1974
1975// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
1976Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
1977 Value *Real, *Imag;
1978
1979 if (CI->arg_size() == 1) {
1980
1981 if (!CI->isFast())
1982 return nullptr;
1983
1984 Value *Op = CI->getArgOperand(0);
1985 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
1986
1987 Real = B.CreateExtractValue(Op, 0, "real");
1988 Imag = B.CreateExtractValue(Op, 1, "imag");
1989
1990 } else {
1991 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
1992
1993 Real = CI->getArgOperand(0);
1994 Imag = CI->getArgOperand(1);
1995
1996 // if real or imaginary part is zero, simplify to abs(cimag(z))
1997 // or abs(creal(z))
1998 Value *AbsOp = nullptr;
1999 if (ConstantFP *ConstReal = dyn_cast<ConstantFP>(Real)) {
2000 if (ConstReal->isZero())
2001 AbsOp = Imag;
2002
2003 } else if (ConstantFP *ConstImag = dyn_cast<ConstantFP>(Imag)) {
2004 if (ConstImag->isZero())
2005 AbsOp = Real;
2006 }
2007
2008 if (AbsOp)
2009 return copyFlags(
2010 *CI, B.CreateUnaryIntrinsic(Intrinsic::fabs, AbsOp, CI, "cabs"));
2011
2012 if (!CI->isFast())
2013 return nullptr;
2014 }
2015
2016 // Propagate fast-math flags from the existing call to new instructions.
2017 Value *RealReal = B.CreateFMulFMF(Real, Real, CI);
2018 Value *ImagImag = B.CreateFMulFMF(Imag, Imag, CI);
2019 return copyFlags(
2020 *CI, B.CreateUnaryIntrinsic(Intrinsic::sqrt,
2021 B.CreateFAddFMF(RealReal, ImagImag, CI), CI,
2022 "cabs"));
2023}
2024
2025// Return a properly extended integer (DstWidth bits wide) if the operation is
2026// an itofp.
2027static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
2028 if (isa<SIToFPInst>(I2F) || isa<UIToFPInst>(I2F)) {
2029 Value *Op = cast<Instruction>(I2F)->getOperand(0);
2030 // Make sure that the exponent fits inside an "int" of size DstWidth,
2031 // thus avoiding any range issues that FP has not.
2032 unsigned BitWidth = Op->getType()->getScalarSizeInBits();
2033 if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(I2F))) {
2034 Type *IntTy = Op->getType()->getWithNewBitWidth(DstWidth);
2035 return isa<SIToFPInst>(I2F) ? B.CreateSExt(Op, IntTy)
2036 : B.CreateZExt(Op, IntTy);
2037 }
2038 }
2039
2040 return nullptr;
2041}
2042
2043/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
2044/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
2045/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
2046Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
2047 Module *M = Pow->getModule();
2048 Value *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1);
2049 Type *Ty = Pow->getType();
2050 bool Ignored;
2051
2052 // Evaluate special cases related to a nested function as the base.
2053
2054 // pow(exp(x), y) -> exp(x * y)
2055 // pow(exp2(x), y) -> exp2(x * y)
2056 // If exp{,2}() is used only once, it is better to fold two transcendental
2057 // math functions into one. If used again, exp{,2}() would still have to be
2058 // called with the original argument, then keep both original transcendental
2059 // functions. However, this transformation is only safe with fully relaxed
2060 // math semantics, since, besides rounding differences, it changes overflow
2061 // and underflow behavior quite dramatically. For example:
2062 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
2063 // Whereas:
2064 // exp(1000 * 0.001) = exp(1)
2065 // TODO: Loosen the requirement for fully relaxed math semantics.
2066 // TODO: Handle exp10() when more targets have it available.
2067 CallInst *BaseFn = dyn_cast<CallInst>(Base);
2068 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
2069 LibFunc LibFn;
2070
2071 Function *CalleeFn = BaseFn->getCalledFunction();
2072 if (CalleeFn && TLI->getLibFunc(CalleeFn->getName(), LibFn) &&
2073 isLibFuncEmittable(M, TLI, LibFn)) {
2074 StringRef ExpName;
2076 Value *ExpFn;
2077 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2078
2079 switch (LibFn) {
2080 default:
2081 return nullptr;
2082 case LibFunc_expf:
2083 case LibFunc_exp:
2084 case LibFunc_expl:
2085 ExpName = TLI->getName(LibFunc_exp);
2086 ID = Intrinsic::exp;
2087 LibFnFloat = LibFunc_expf;
2088 LibFnDouble = LibFunc_exp;
2089 LibFnLongDouble = LibFunc_expl;
2090 break;
2091 case LibFunc_exp2f:
2092 case LibFunc_exp2:
2093 case LibFunc_exp2l:
2094 ExpName = TLI->getName(LibFunc_exp2);
2095 ID = Intrinsic::exp2;
2096 LibFnFloat = LibFunc_exp2f;
2097 LibFnDouble = LibFunc_exp2;
2098 LibFnLongDouble = LibFunc_exp2l;
2099 break;
2100 }
2101
2102 // Create new exp{,2}() with the product as its argument.
2103 Value *FMul = B.CreateFMul(BaseFn->getArgOperand(0), Expo, "mul");
2104 ExpFn = BaseFn->doesNotAccessMemory()
2105 ? B.CreateUnaryIntrinsic(ID, FMul, nullptr, ExpName)
2106 : emitUnaryFloatFnCall(FMul, TLI, LibFnDouble, LibFnFloat,
2107 LibFnLongDouble, B,
2108 BaseFn->getAttributes());
2109
2110 // Since the new exp{,2}() is different from the original one, dead code
2111 // elimination cannot be trusted to remove it, since it may have side
2112 // effects (e.g., errno). When the only consumer for the original
2113 // exp{,2}() is pow(), then it has to be explicitly erased.
2114 substituteInParent(BaseFn, ExpFn);
2115 return ExpFn;
2116 }
2117 }
2118
2119 // Evaluate special cases related to a constant base.
2120
2121 const APFloat *BaseF;
2122 if (!match(Base, m_APFloat(BaseF)))
2123 return nullptr;
2124
2125 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2126
2127 const bool UseIntrinsic = Pow->doesNotAccessMemory();
2128
2129 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2130 if ((UseIntrinsic || !Ty->isVectorTy()) && BaseF->isExactlyValue(2.0) &&
2131 (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo)) &&
2132 (UseIntrinsic ||
2133 hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl))) {
2134
2135 // TODO: Shouldn't really need to depend on getIntToFPVal for intrinsic. Can
2136 // just directly use the original integer type.
2137 if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize())) {
2138 Constant *One = ConstantFP::get(Ty, 1.0);
2139
2140 if (UseIntrinsic) {
2141 return copyFlags(*Pow, B.CreateIntrinsic(Intrinsic::ldexp,
2142 {Ty, ExpoI->getType()},
2143 {One, ExpoI}, Pow, "exp2"));
2144 }
2145
2146 return copyFlags(*Pow, emitBinaryFloatFnCall(
2147 One, ExpoI, TLI, LibFunc_ldexp, LibFunc_ldexpf,
2148 LibFunc_ldexpl, B, NoAttrs));
2149 }
2150 }
2151
2152 // pow(2.0 ** n, x) -> exp2(n * x)
2153 if (hasFloatFn(M, TLI, Ty, LibFunc_exp2, LibFunc_exp2f, LibFunc_exp2l)) {
2154 APFloat BaseR = APFloat(1.0);
2155 BaseR.convert(BaseF->getSemantics(), APFloat::rmTowardZero, &Ignored);
2156 BaseR = BaseR / *BaseF;
2157 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2158 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2159 APSInt NI(64, false);
2160 if ((IsInteger || IsReciprocal) &&
2161 NF->convertToInteger(NI, APFloat::rmTowardZero, &Ignored) ==
2162 APFloat::opOK &&
2163 NI > 1 && NI.isPowerOf2()) {
2164 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2165 Value *FMul = B.CreateFMul(Expo, ConstantFP::get(Ty, N), "mul");
2166 if (Pow->doesNotAccessMemory())
2167 return copyFlags(*Pow, B.CreateUnaryIntrinsic(Intrinsic::exp2, FMul,
2168 nullptr, "exp2"));
2169 else
2170 return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
2171 LibFunc_exp2f,
2172 LibFunc_exp2l, B, NoAttrs));
2173 }
2174 }
2175
2176 // pow(10.0, x) -> exp10(x)
2177 if (BaseF->isExactlyValue(10.0) &&
2178 hasFloatFn(M, TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l)) {
2179
2180 if (Pow->doesNotAccessMemory()) {
2181 CallInst *NewExp10 =
2182 B.CreateIntrinsic(Intrinsic::exp10, {Ty}, {Expo}, Pow, "exp10");
2183 return copyFlags(*Pow, NewExp10);
2184 }
2185
2186 return copyFlags(*Pow, emitUnaryFloatFnCall(Expo, TLI, LibFunc_exp10,
2187 LibFunc_exp10f, LibFunc_exp10l,
2188 B, NoAttrs));
2189 }
2190
2191 // pow(x, y) -> exp2(log2(x) * y)
2192 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2193 !BaseF->isNegative()) {
2194 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2195 // Luckily optimizePow has already handled the x == 1 case.
2196 assert(!match(Base, m_FPOne()) &&
2197 "pow(1.0, y) should have been simplified earlier!");
2198
2199 Value *Log = nullptr;
2200 if (Ty->isFloatTy())
2201 Log = ConstantFP::get(Ty, std::log2(BaseF->convertToFloat()));
2202 else if (Ty->isDoubleTy())
2203 Log = ConstantFP::get(Ty, std::log2(BaseF->convertToDouble()));
2204
2205 if (Log) {
2206 Value *FMul = B.CreateFMul(Log, Expo, "mul");
2207 if (Pow->doesNotAccessMemory())
2208 return copyFlags(*Pow, B.CreateUnaryIntrinsic(Intrinsic::exp2, FMul,
2209 nullptr, "exp2"));
2210 else if (hasFloatFn(M, TLI, Ty, LibFunc_exp2, LibFunc_exp2f,
2211 LibFunc_exp2l))
2212 return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
2213 LibFunc_exp2f,
2214 LibFunc_exp2l, B, NoAttrs));
2215 }
2216 }
2217
2218 return nullptr;
2219}
2220
2221static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2222 Module *M, IRBuilderBase &B,
2223 const TargetLibraryInfo *TLI) {
2224 // If errno is never set, then use the intrinsic for sqrt().
2225 if (NoErrno)
2226 return B.CreateUnaryIntrinsic(Intrinsic::sqrt, V, nullptr, "sqrt");
2227
2228 // Otherwise, use the libcall for sqrt().
2229 if (hasFloatFn(M, TLI, V->getType(), LibFunc_sqrt, LibFunc_sqrtf,
2230 LibFunc_sqrtl))
2231 // TODO: We also should check that the target can in fact lower the sqrt()
2232 // libcall. We currently have no way to ask this question, so we ask if
2233 // the target has a sqrt() libcall, which is not exactly the same.
2234 return emitUnaryFloatFnCall(V, TLI, LibFunc_sqrt, LibFunc_sqrtf,
2235 LibFunc_sqrtl, B, Attrs);
2236
2237 return nullptr;
2238}
2239
2240/// Use square root in place of pow(x, +/-0.5).
2241Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2242 Value *Sqrt, *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1);
2243 Module *Mod = Pow->getModule();
2244 Type *Ty = Pow->getType();
2245
2246 const APFloat *ExpoF;
2247 if (!match(Expo, m_APFloat(ExpoF)) ||
2248 (!ExpoF->isExactlyValue(0.5) && !ExpoF->isExactlyValue(-0.5)))
2249 return nullptr;
2250
2251 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2252 // so that requires fast-math-flags (afn or reassoc).
2253 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2254 return nullptr;
2255
2256 // If we have a pow() library call (accesses memory) and we can't guarantee
2257 // that the base is not an infinity, give up:
2258 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2259 // errno), but sqrt(-Inf) is required by various standards to set errno.
2260 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2262 Base, 0, SimplifyQuery(DL, TLI, DT, AC, Pow, true, true, DC)))
2263 return nullptr;
2264
2266 TLI);
2267 if (!Sqrt)
2268 return nullptr;
2269
2270 // Handle signed zero base by expanding to fabs(sqrt(x)).
2271 if (!Pow->hasNoSignedZeros())
2272 Sqrt = B.CreateUnaryIntrinsic(Intrinsic::fabs, Sqrt, nullptr, "abs");
2273
2274 Sqrt = copyFlags(*Pow, Sqrt);
2275
2276 // Handle non finite base by expanding to
2277 // (x == -infinity ? +infinity : sqrt(x)).
2278 if (!Pow->hasNoInfs()) {
2279 Value *PosInf = ConstantFP::getInfinity(Ty),
2280 *NegInf = ConstantFP::getInfinity(Ty, true);
2281 Value *FCmp = B.CreateFCmpOEQ(Base, NegInf, "isinf");
2282 Sqrt = B.CreateSelect(FCmp, PosInf, Sqrt);
2283 }
2284
2285 // If the exponent is negative, then get the reciprocal.
2286 if (ExpoF->isNegative())
2287 Sqrt = B.CreateFDiv(ConstantFP::get(Ty, 1.0), Sqrt, "reciprocal");
2288
2289 return Sqrt;
2290}
2291
2293 IRBuilderBase &B) {
2294 Value *Args[] = {Base, Expo};
2295 Type *Types[] = {Base->getType(), Expo->getType()};
2296 return B.CreateIntrinsic(Intrinsic::powi, Types, Args);
2297}
2298
2299Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2300 Value *Base = Pow->getArgOperand(0);
2301 Value *Expo = Pow->getArgOperand(1);
2303 StringRef Name = Callee->getName();
2304 Type *Ty = Pow->getType();
2305 Module *M = Pow->getModule();
2306 bool AllowApprox = Pow->hasApproxFunc();
2307 bool Ignored;
2308
2309 // Propagate the math semantics from the call to any created instructions.
2311 B.setFastMathFlags(Pow->getFastMathFlags());
2312 // Evaluate special cases related to the base.
2313
2314 // pow(1.0, x) -> 1.0
2315 if (match(Base, m_FPOne()))
2316 return Base;
2317
2318 if (Value *Exp = replacePowWithExp(Pow, B))
2319 return Exp;
2320
2321 // Evaluate special cases related to the exponent.
2322
2323 // pow(x, -1.0) -> 1.0 / x
2324 if (match(Expo, m_SpecificFP(-1.0)))
2325 return B.CreateFDiv(ConstantFP::get(Ty, 1.0), Base, "reciprocal");
2326
2327 // pow(x, +/-0.0) -> 1.0
2328 if (match(Expo, m_AnyZeroFP()))
2329 return ConstantFP::get(Ty, 1.0);
2330
2331 // pow(x, 1.0) -> x
2332 if (match(Expo, m_FPOne()))
2333 return Base;
2334
2335 // pow(x, 2.0) -> x * x
2336 if (match(Expo, m_SpecificFP(2.0)))
2337 return B.CreateFMul(Base, Base, "square");
2338
2339 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2340 return Sqrt;
2341
2342 // If we can approximate pow:
2343 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2344 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2345 const APFloat *ExpoF;
2346 if (AllowApprox && match(Expo, m_APFloat(ExpoF)) &&
2347 !ExpoF->isExactlyValue(0.5) && !ExpoF->isExactlyValue(-0.5)) {
2348 APFloat ExpoA(abs(*ExpoF));
2349 APFloat ExpoI(*ExpoF);
2350 Value *Sqrt = nullptr;
2351 if (!ExpoA.isInteger()) {
2352 APFloat Expo2 = ExpoA;
2353 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2354 // is no floating point exception and the result is an integer, then
2355 // ExpoA == integer + 0.5
2356 if (Expo2.add(ExpoA, APFloat::rmNearestTiesToEven) != APFloat::opOK)
2357 return nullptr;
2358
2359 if (!Expo2.isInteger())
2360 return nullptr;
2361
2362 if (ExpoI.roundToIntegral(APFloat::rmTowardNegative) !=
2364 return nullptr;
2365 if (!ExpoI.isInteger())
2366 return nullptr;
2367 ExpoF = &ExpoI;
2368
2370 B, TLI);
2371 if (!Sqrt)
2372 return nullptr;
2373 }
2374
2375 // 0.5 fraction is now optionally handled.
2376 // Do pow -> powi for remaining integer exponent
2377 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2378 if (ExpoF->isInteger() &&
2379 ExpoF->convertToInteger(IntExpo, APFloat::rmTowardZero, &Ignored) ==
2380 APFloat::opOK) {
2381 Value *PowI = copyFlags(
2382 *Pow,
2384 Base, ConstantInt::get(B.getIntNTy(TLI->getIntSize()), IntExpo),
2385 M, B));
2386
2387 if (PowI && Sqrt)
2388 return B.CreateFMul(PowI, Sqrt);
2389
2390 return PowI;
2391 }
2392 }
2393
2394 // powf(x, itofp(y)) -> powi(x, y)
2395 if (AllowApprox && (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo))) {
2396 if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize()))
2397 return copyFlags(*Pow, createPowWithIntegerExponent(Base, ExpoI, M, B));
2398 }
2399
2400 // Shrink pow() to powf() if the arguments are single precision,
2401 // unless the result is expected to be double precision.
2402 if (UnsafeFPShrink && Name == TLI->getName(LibFunc_pow) &&
2403 hasFloatVersion(M, Name)) {
2404 if (Value *Shrunk = optimizeBinaryDoubleFP(Pow, B, TLI, true))
2405 return Shrunk;
2406 }
2407
2408 return nullptr;
2409}
2410
2411Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2412 Module *M = CI->getModule();
2414 StringRef Name = Callee->getName();
2415 Value *Ret = nullptr;
2416 if (UnsafeFPShrink && Name == TLI->getName(LibFunc_exp2) &&
2417 hasFloatVersion(M, Name))
2418 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2419
2420 // If we have an llvm.exp2 intrinsic, emit the llvm.ldexp intrinsic. If we
2421 // have the libcall, emit the libcall.
2422 //
2423 // TODO: In principle we should be able to just always use the intrinsic for
2424 // any doesNotAccessMemory callsite.
2425
2426 const bool UseIntrinsic = Callee->isIntrinsic();
2427 // Bail out for vectors because the code below only expects scalars.
2428 Type *Ty = CI->getType();
2429 if (!UseIntrinsic && Ty->isVectorTy())
2430 return Ret;
2431
2432 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2433 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2434 Value *Op = CI->getArgOperand(0);
2435 if ((isa<SIToFPInst>(Op) || isa<UIToFPInst>(Op)) &&
2436 (UseIntrinsic ||
2437 hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl))) {
2438 if (Value *Exp = getIntToFPVal(Op, B, TLI->getIntSize())) {
2439 Constant *One = ConstantFP::get(Ty, 1.0);
2440
2441 if (UseIntrinsic) {
2442 return copyFlags(*CI, B.CreateIntrinsic(Intrinsic::ldexp,
2443 {Ty, Exp->getType()},
2444 {One, Exp}, CI));
2445 }
2446
2448 B.setFastMathFlags(CI->getFastMathFlags());
2449 return copyFlags(*CI, emitBinaryFloatFnCall(
2450 One, Exp, TLI, LibFunc_ldexp, LibFunc_ldexpf,
2451 LibFunc_ldexpl, B, AttributeList()));
2452 }
2453 }
2454
2455 return Ret;
2456}
2457
2458Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B) {
2459 Module *M = CI->getModule();
2460
2461 // If we can shrink the call to a float function rather than a double
2462 // function, do that first.
2464 StringRef Name = Callee->getName();
2465 if ((Name == "fmin" || Name == "fmax") && hasFloatVersion(M, Name))
2466 if (Value *Ret = optimizeBinaryDoubleFP(CI, B, TLI))
2467 return Ret;
2468
2469 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2470 // the intrinsics for improved optimization (for example, vectorization).
2471 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2472 // From the C standard draft WG14/N1256:
2473 // "Ideally, fmax would be sensitive to the sign of zero, for example
2474 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2475 // might be impractical."
2476 FastMathFlags FMF = CI->getFastMathFlags();
2477 FMF.setNoSignedZeros();
2478
2479 Intrinsic::ID IID = Callee->getName().starts_with("fmin") ? Intrinsic::minnum
2480 : Intrinsic::maxnum;
2481 return copyFlags(*CI, B.CreateBinaryIntrinsic(IID, CI->getArgOperand(0),
2482 CI->getArgOperand(1), FMF));
2483}
2484
2485Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2486 Function *LogFn = Log->getCalledFunction();
2487 StringRef LogNm = LogFn->getName();
2488 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2489 Module *Mod = Log->getModule();
2490 Type *Ty = Log->getType();
2491
2492 if (UnsafeFPShrink && hasFloatVersion(Mod, LogNm))
2493 if (Value *Ret = optimizeUnaryDoubleFP(Log, B, TLI, true))
2494 return Ret;
2495
2496 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2497
2498 // This is only applicable to log(), log2(), log10().
2499 if (TLI->getLibFunc(LogNm, LogLb)) {
2500 switch (LogLb) {
2501 case LibFunc_logf:
2502 LogID = Intrinsic::log;
2503 ExpLb = LibFunc_expf;
2504 Exp2Lb = LibFunc_exp2f;
2505 Exp10Lb = LibFunc_exp10f;
2506 PowLb = LibFunc_powf;
2507 break;
2508 case LibFunc_log:
2509 LogID = Intrinsic::log;
2510 ExpLb = LibFunc_exp;
2511 Exp2Lb = LibFunc_exp2;
2512 Exp10Lb = LibFunc_exp10;
2513 PowLb = LibFunc_pow;
2514 break;
2515 case LibFunc_logl:
2516 LogID = Intrinsic::log;
2517 ExpLb = LibFunc_expl;
2518 Exp2Lb = LibFunc_exp2l;
2519 Exp10Lb = LibFunc_exp10l;
2520 PowLb = LibFunc_powl;
2521 break;
2522 case LibFunc_log2f:
2523 LogID = Intrinsic::log2;
2524 ExpLb = LibFunc_expf;
2525 Exp2Lb = LibFunc_exp2f;
2526 Exp10Lb = LibFunc_exp10f;
2527 PowLb = LibFunc_powf;
2528 break;
2529 case LibFunc_log2:
2530 LogID = Intrinsic::log2;
2531 ExpLb = LibFunc_exp;
2532 Exp2Lb = LibFunc_exp2;
2533 Exp10Lb = LibFunc_exp10;
2534 PowLb = LibFunc_pow;
2535 break;
2536 case LibFunc_log2l:
2537 LogID = Intrinsic::log2;
2538 ExpLb = LibFunc_expl;
2539 Exp2Lb = LibFunc_exp2l;
2540 Exp10Lb = LibFunc_exp10l;
2541 PowLb = LibFunc_powl;
2542 break;
2543 case LibFunc_log10f:
2544 LogID = Intrinsic::log10;
2545 ExpLb = LibFunc_expf;
2546 Exp2Lb = LibFunc_exp2f;
2547 Exp10Lb = LibFunc_exp10f;
2548 PowLb = LibFunc_powf;
2549 break;
2550 case LibFunc_log10:
2551 LogID = Intrinsic::log10;
2552 ExpLb = LibFunc_exp;
2553 Exp2Lb = LibFunc_exp2;
2554 Exp10Lb = LibFunc_exp10;
2555 PowLb = LibFunc_pow;
2556 break;
2557 case LibFunc_log10l:
2558 LogID = Intrinsic::log10;
2559 ExpLb = LibFunc_expl;
2560 Exp2Lb = LibFunc_exp2l;
2561 Exp10Lb = LibFunc_exp10l;
2562 PowLb = LibFunc_powl;
2563 break;
2564 default:
2565 return nullptr;
2566 }
2567
2568 // Convert libcall to intrinsic if the value is known > 0.
2569 bool IsKnownNoErrno = Log->hasNoNaNs() && Log->hasNoInfs();
2570 if (!IsKnownNoErrno) {
2571 SimplifyQuery SQ(DL, TLI, DT, AC, Log, true, true, DC);
2573 Log->getOperand(0),
2575 /*Depth=*/0, SQ);
2576 Function *F = Log->getParent()->getParent();
2577 IsKnownNoErrno = Known.cannotBeOrderedLessThanZero() &&
2578 Known.isKnownNeverLogicalZero(*F, Ty);
2579 }
2580 if (IsKnownNoErrno) {
2581 auto *NewLog = B.CreateUnaryIntrinsic(LogID, Log->getArgOperand(0), Log);
2582 NewLog->copyMetadata(*Log);
2583 return copyFlags(*Log, NewLog);
2584 }
2585 } else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2586 LogID == Intrinsic::log10) {
2587 if (Ty->getScalarType()->isFloatTy()) {
2588 ExpLb = LibFunc_expf;
2589 Exp2Lb = LibFunc_exp2f;
2590 Exp10Lb = LibFunc_exp10f;
2591 PowLb = LibFunc_powf;
2592 } else if (Ty->getScalarType()->isDoubleTy()) {
2593 ExpLb = LibFunc_exp;
2594 Exp2Lb = LibFunc_exp2;
2595 Exp10Lb = LibFunc_exp10;
2596 PowLb = LibFunc_pow;
2597 } else
2598 return nullptr;
2599 } else
2600 return nullptr;
2601
2602 // The earlier call must also be 'fast' in order to do these transforms.
2603 CallInst *Arg = dyn_cast<CallInst>(Log->getArgOperand(0));
2604 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2605 return nullptr;
2606
2608 B.setFastMathFlags(FastMathFlags::getFast());
2609
2610 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2611 LibFunc ArgLb = NotLibFunc;
2612 TLI->getLibFunc(*Arg, ArgLb);
2613
2614 // log(pow(x,y)) -> y*log(x)
2615 AttributeList NoAttrs;
2616 if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
2617 Value *LogX =
2618 Log->doesNotAccessMemory()
2619 ? B.CreateUnaryIntrinsic(LogID, Arg->getOperand(0), nullptr, "log")
2620 : emitUnaryFloatFnCall(Arg->getOperand(0), TLI, LogNm, B, NoAttrs);
2621 Value *Y = Arg->getArgOperand(1);
2622 // Cast exponent to FP if integer.
2623 if (ArgID == Intrinsic::powi)
2624 Y = B.CreateSIToFP(Y, Ty, "cast");
2625 Value *MulY = B.CreateFMul(Y, LogX, "mul");
2626 // Since pow() may have side effects, e.g. errno,
2627 // dead code elimination may not be trusted to remove it.
2628 substituteInParent(Arg, MulY);
2629 return MulY;
2630 }
2631
2632 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2633 // TODO: There is no exp10() intrinsic yet.
2634 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2635 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2636 Constant *Eul;
2637 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2638 // FIXME: Add more precise value of e for long double.
2639 Eul = ConstantFP::get(Log->getType(), numbers::e);
2640 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2641 Eul = ConstantFP::get(Log->getType(), 2.0);
2642 else
2643 Eul = ConstantFP::get(Log->getType(), 10.0);
2644 Value *LogE = Log->doesNotAccessMemory()
2645 ? B.CreateUnaryIntrinsic(LogID, Eul, nullptr, "log")
2646 : emitUnaryFloatFnCall(Eul, TLI, LogNm, B, NoAttrs);
2647 Value *MulY = B.CreateFMul(Arg->getArgOperand(0), LogE, "mul");
2648 // Since exp() may have side effects, e.g. errno,
2649 // dead code elimination may not be trusted to remove it.
2650 substituteInParent(Arg, MulY);
2651 return MulY;
2652 }
2653
2654 return nullptr;
2655}
2656
2657// sqrt(exp(X)) -> exp(X * 0.5)
2658Value *LibCallSimplifier::mergeSqrtToExp(CallInst *CI, IRBuilderBase &B) {
2659 if (!CI->hasAllowReassoc())
2660 return nullptr;
2661
2662 Function *SqrtFn = CI->getCalledFunction();
2663 CallInst *Arg = dyn_cast<CallInst>(CI->getArgOperand(0));
2664 if (!Arg || !Arg->hasAllowReassoc() || !Arg->hasOneUse())
2665 return nullptr;
2666 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2667 LibFunc ArgLb = NotLibFunc;
2668 TLI->getLibFunc(*Arg, ArgLb);
2669
2670 LibFunc SqrtLb, ExpLb, Exp2Lb, Exp10Lb;
2671
2672 if (TLI->getLibFunc(SqrtFn->getName(), SqrtLb))
2673 switch (SqrtLb) {
2674 case LibFunc_sqrtf:
2675 ExpLb = LibFunc_expf;
2676 Exp2Lb = LibFunc_exp2f;
2677 Exp10Lb = LibFunc_exp10f;
2678 break;
2679 case LibFunc_sqrt:
2680 ExpLb = LibFunc_exp;
2681 Exp2Lb = LibFunc_exp2;
2682 Exp10Lb = LibFunc_exp10;
2683 break;
2684 case LibFunc_sqrtl:
2685 ExpLb = LibFunc_expl;
2686 Exp2Lb = LibFunc_exp2l;
2687 Exp10Lb = LibFunc_exp10l;
2688 break;
2689 default:
2690 return nullptr;
2691 }
2692 else if (SqrtFn->getIntrinsicID() == Intrinsic::sqrt) {
2693 if (CI->getType()->getScalarType()->isFloatTy()) {
2694 ExpLb = LibFunc_expf;
2695 Exp2Lb = LibFunc_exp2f;
2696 Exp10Lb = LibFunc_exp10f;
2697 } else if (CI->getType()->getScalarType()->isDoubleTy()) {
2698 ExpLb = LibFunc_exp;
2699 Exp2Lb = LibFunc_exp2;
2700 Exp10Lb = LibFunc_exp10;
2701 } else
2702 return nullptr;
2703 } else
2704 return nullptr;
2705
2706 if (ArgLb != ExpLb && ArgLb != Exp2Lb && ArgLb != Exp10Lb &&
2707 ArgID != Intrinsic::exp && ArgID != Intrinsic::exp2)
2708 return nullptr;
2709
2711 B.SetInsertPoint(Arg);
2712 auto *ExpOperand = Arg->getOperand(0);
2713 auto *FMul =
2714 B.CreateFMulFMF(ExpOperand, ConstantFP::get(ExpOperand->getType(), 0.5),
2715 CI, "merged.sqrt");
2716
2717 Arg->setOperand(0, FMul);
2718 return Arg;
2719}
2720
2721Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2722 Module *M = CI->getModule();
2724 Value *Ret = nullptr;
2725 // TODO: Once we have a way (other than checking for the existince of the
2726 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2727 // condition below.
2728 if (isLibFuncEmittable(M, TLI, LibFunc_sqrtf) &&
2729 (Callee->getName() == "sqrt" ||
2730 Callee->getIntrinsicID() == Intrinsic::sqrt))
2731 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2732
2733 if (Value *Opt = mergeSqrtToExp(CI, B))
2734 return Opt;
2735
2736 if (!CI->isFast())
2737 return Ret;
2738
2739 Instruction *I = dyn_cast<Instruction>(CI->getArgOperand(0));
2740 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2741 return Ret;
2742
2743 // We're looking for a repeated factor in a multiplication tree,
2744 // so we can do this fold: sqrt(x * x) -> fabs(x);
2745 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2746 Value *Op0 = I->getOperand(0);
2747 Value *Op1 = I->getOperand(1);
2748 Value *RepeatOp = nullptr;
2749 Value *OtherOp = nullptr;
2750 if (Op0 == Op1) {
2751 // Simple match: the operands of the multiply are identical.
2752 RepeatOp = Op0;
2753 } else {
2754 // Look for a more complicated pattern: one of the operands is itself
2755 // a multiply, so search for a common factor in that multiply.
2756 // Note: We don't bother looking any deeper than this first level or for
2757 // variations of this pattern because instcombine's visitFMUL and/or the
2758 // reassociation pass should give us this form.
2759 Value *MulOp;
2760 if (match(Op0, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
2761 cast<Instruction>(Op0)->isFast()) {
2762 // Pattern: sqrt((x * x) * z)
2763 RepeatOp = MulOp;
2764 OtherOp = Op1;
2765 } else if (match(Op1, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
2766 cast<Instruction>(Op1)->isFast()) {
2767 // Pattern: sqrt(z * (x * x))
2768 RepeatOp = MulOp;
2769 OtherOp = Op0;
2770 }
2771 }
2772 if (!RepeatOp)
2773 return Ret;
2774
2775 // Fast math flags for any created instructions should match the sqrt
2776 // and multiply.
2777
2778 // If we found a repeated factor, hoist it out of the square root and
2779 // replace it with the fabs of that factor.
2780 Value *FabsCall =
2781 B.CreateUnaryIntrinsic(Intrinsic::fabs, RepeatOp, I, "fabs");
2782 if (OtherOp) {
2783 // If we found a non-repeated factor, we still need to get its square
2784 // root. We then multiply that by the value that was simplified out
2785 // of the square root calculation.
2786 Value *SqrtCall =
2787 B.CreateUnaryIntrinsic(Intrinsic::sqrt, OtherOp, I, "sqrt");
2788 return copyFlags(*CI, B.CreateFMulFMF(FabsCall, SqrtCall, I));
2789 }
2790 return copyFlags(*CI, FabsCall);
2791}
2792
2793Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
2794
2795 // fmod(x,y) can set errno if y == 0 or x == +/-inf, and returns Nan in those
2796 // case. If we know those do not happen, then we can convert the fmod into
2797 // frem.
2798 bool IsNoNan = CI->hasNoNaNs();
2799 if (!IsNoNan) {
2800 SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
2802 /*Depth=*/0, SQ);
2803 if (Known0.isKnownNeverInfinity()) {
2804 KnownFPClass Known1 =
2806 /*Depth=*/0, SQ);
2807 Function *F = CI->getParent()->getParent();
2808 IsNoNan = Known1.isKnownNeverLogicalZero(*F, CI->getType());
2809 }
2810 }
2811
2812 if (IsNoNan) {
2813 Value *FRem = B.CreateFRemFMF(CI->getOperand(0), CI->getOperand(1), CI);
2814 if (auto *FRemI = dyn_cast<Instruction>(FRem))
2815 FRemI->setHasNoNaNs(true);
2816 return FRem;
2817 }
2818 return nullptr;
2819}
2820
2821Value *LibCallSimplifier::optimizeTrigInversionPairs(CallInst *CI,
2822 IRBuilderBase &B) {
2823 Module *M = CI->getModule();
2825 Value *Ret = nullptr;
2826 StringRef Name = Callee->getName();
2827 if (UnsafeFPShrink &&
2828 (Name == "tan" || Name == "atanh" || Name == "sinh" || Name == "cosh" ||
2829 Name == "asinh") &&
2830 hasFloatVersion(M, Name))
2831 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2832
2833 Value *Op1 = CI->getArgOperand(0);
2834 auto *OpC = dyn_cast<CallInst>(Op1);
2835 if (!OpC)
2836 return Ret;
2837
2838 // Both calls must be 'fast' in order to remove them.
2839 if (!CI->isFast() || !OpC->isFast())
2840 return Ret;
2841
2842 // tan(atan(x)) -> x
2843 // atanh(tanh(x)) -> x
2844 // sinh(asinh(x)) -> x
2845 // asinh(sinh(x)) -> x
2846 // cosh(acosh(x)) -> x
2847 LibFunc Func;
2848 Function *F = OpC->getCalledFunction();
2849 if (F && TLI->getLibFunc(F->getName(), Func) &&
2850 isLibFuncEmittable(M, TLI, Func)) {
2851 LibFunc inverseFunc = llvm::StringSwitch<LibFunc>(Callee->getName())
2852 .Case("tan", LibFunc_atan)
2853 .Case("atanh", LibFunc_tanh)
2854 .Case("sinh", LibFunc_asinh)
2855 .Case("cosh", LibFunc_acosh)
2856 .Case("tanf", LibFunc_atanf)
2857 .Case("atanhf", LibFunc_tanhf)
2858 .Case("sinhf", LibFunc_asinhf)
2859 .Case("coshf", LibFunc_acoshf)
2860 .Case("tanl", LibFunc_atanl)
2861 .Case("atanhl", LibFunc_tanhl)
2862 .Case("sinhl", LibFunc_asinhl)
2863 .Case("coshl", LibFunc_acoshl)
2864 .Case("asinh", LibFunc_sinh)
2865 .Case("asinhf", LibFunc_sinhf)
2866 .Case("asinhl", LibFunc_sinhl)
2867 .Default(NumLibFuncs); // Used as error value
2868 if (Func == inverseFunc)
2869 Ret = OpC->getArgOperand(0);
2870 }
2871 return Ret;
2872}
2873
2874static bool isTrigLibCall(CallInst *CI) {
2875 // We can only hope to do anything useful if we can ignore things like errno
2876 // and floating-point exceptions.
2877 // We already checked the prototype.
2878 return CI->doesNotThrow() && CI->doesNotAccessMemory();
2879}
2880
2881static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
2882 bool UseFloat, Value *&Sin, Value *&Cos,
2883 Value *&SinCos, const TargetLibraryInfo *TLI) {
2884 Module *M = OrigCallee->getParent();
2885 Type *ArgTy = Arg->getType();
2886 Type *ResTy;
2888
2889 Triple T(OrigCallee->getParent()->getTargetTriple());
2890 if (UseFloat) {
2891 Name = "__sincospif_stret";
2892
2893 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
2894 // x86_64 can't use {float, float} since that would be returned in both
2895 // xmm0 and xmm1, which isn't what a real struct would do.
2896 ResTy = T.getArch() == Triple::x86_64
2897 ? static_cast<Type *>(FixedVectorType::get(ArgTy, 2))
2898 : static_cast<Type *>(StructType::get(ArgTy, ArgTy));
2899 } else {
2900 Name = "__sincospi_stret";
2901 ResTy = StructType::get(ArgTy, ArgTy);
2902 }
2903
2904 if (!isLibFuncEmittable(M, TLI, Name))
2905 return false;
2906 LibFunc TheLibFunc;
2907 TLI->getLibFunc(Name, TheLibFunc);
2909 M, *TLI, TheLibFunc, OrigCallee->getAttributes(), ResTy, ArgTy);
2910
2911 if (Instruction *ArgInst = dyn_cast<Instruction>(Arg)) {
2912 // If the argument is an instruction, it must dominate all uses so put our
2913 // sincos call there.
2914 B.SetInsertPoint(ArgInst->getParent(), ++ArgInst->getIterator());
2915 } else {
2916 // Otherwise (e.g. for a constant) the beginning of the function is as
2917 // good a place as any.
2918 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
2919 B.SetInsertPoint(&EntryBB, EntryBB.begin());
2920 }
2921
2922 SinCos = B.CreateCall(Callee, Arg, "sincospi");
2923
2924 if (SinCos->getType()->isStructTy()) {
2925 Sin = B.CreateExtractValue(SinCos, 0, "sinpi");
2926 Cos = B.CreateExtractValue(SinCos, 1, "cospi");
2927 } else {
2928 Sin = B.CreateExtractElement(SinCos, ConstantInt::get(B.getInt32Ty(), 0),
2929 "sinpi");
2930 Cos = B.CreateExtractElement(SinCos, ConstantInt::get(B.getInt32Ty(), 1),
2931 "cospi");
2932 }
2933
2934 return true;
2935}
2936
2937static Value *optimizeSymmetricCall(CallInst *CI, bool IsEven,
2938 IRBuilderBase &B) {
2939 Value *X;
2940 Value *Src = CI->getArgOperand(0);
2941
2942 if (match(Src, m_OneUse(m_FNeg(m_Value(X))))) {
2943 auto *Call = B.CreateCall(CI->getCalledFunction(), {X});
2944 Call->copyFastMathFlags(CI);
2945 auto *CallInst = copyFlags(*CI, Call);
2946 if (IsEven) {
2947 // Even function: f(-x) = f(x)
2948 return CallInst;
2949 }
2950 // Odd function: f(-x) = -f(x)
2951 return B.CreateFNegFMF(CallInst, CI);
2952 }
2953
2954 // Even function: f(abs(x)) = f(x), f(copysign(x, y)) = f(x)
2955 if (IsEven && (match(Src, m_FAbs(m_Value(X))) ||
2956 match(Src, m_CopySign(m_Value(X), m_Value())))) {
2957 auto *Call = B.CreateCall(CI->getCalledFunction(), {X});
2958 Call->copyFastMathFlags(CI);
2959 return copyFlags(*CI, Call);
2960 }
2961
2962 return nullptr;
2963}
2964
2965Value *LibCallSimplifier::optimizeSymmetric(CallInst *CI, LibFunc Func,
2966 IRBuilderBase &B) {
2967 switch (Func) {
2968 case LibFunc_cos:
2969 case LibFunc_cosf:
2970 case LibFunc_cosl:
2971 return optimizeSymmetricCall(CI, /*IsEven*/ true, B);
2972
2973 case LibFunc_sin:
2974 case LibFunc_sinf:
2975 case LibFunc_sinl:
2976
2977 case LibFunc_tan:
2978 case LibFunc_tanf:
2979 case LibFunc_tanl:
2980
2981 case LibFunc_erf:
2982 case LibFunc_erff:
2983 case LibFunc_erfl:
2984 return optimizeSymmetricCall(CI, /*IsEven*/ false, B);
2985
2986 default:
2987 return nullptr;
2988 }
2989}
2990
2991Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
2992 // Make sure the prototype is as expected, otherwise the rest of the
2993 // function is probably invalid and likely to abort.
2994 if (!isTrigLibCall(CI))
2995 return nullptr;
2996
2997 Value *Arg = CI->getArgOperand(0);
3000 SmallVector<CallInst *, 1> SinCosCalls;
3001
3002 bool IsFloat = Arg->getType()->isFloatTy();
3003
3004 // Look for all compatible sinpi, cospi and sincospi calls with the same
3005 // argument. If there are enough (in some sense) we can make the
3006 // substitution.
3007 Function *F = CI->getFunction();
3008 for (User *U : Arg->users())
3009 classifyArgUse(U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
3010
3011 // It's only worthwhile if both sinpi and cospi are actually used.
3012 if (SinCalls.empty() || CosCalls.empty())
3013 return nullptr;
3014
3015 Value *Sin, *Cos, *SinCos;
3016 if (!insertSinCosCall(B, CI->getCalledFunction(), Arg, IsFloat, Sin, Cos,
3017 SinCos, TLI))
3018 return nullptr;
3019
3020 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
3021 Value *Res) {
3022 for (CallInst *C : Calls)
3023 replaceAllUsesWith(C, Res);
3024 };
3025
3026 replaceTrigInsts(SinCalls, Sin);
3027 replaceTrigInsts(CosCalls, Cos);
3028 replaceTrigInsts(SinCosCalls, SinCos);
3029
3030 return IsSin ? Sin : Cos;
3031}
3032
3033void LibCallSimplifier::classifyArgUse(
3034 Value *Val, Function *F, bool IsFloat,
3037 SmallVectorImpl<CallInst *> &SinCosCalls) {
3038 auto *CI = dyn_cast<CallInst>(Val);
3039 if (!CI || CI->use_empty())
3040 return;
3041
3042 // Don't consider calls in other functions.
3043 if (CI->getFunction() != F)
3044 return;
3045
3046 Module *M = CI->getModule();
3048 LibFunc Func;
3049 if (!Callee || !TLI->getLibFunc(*Callee, Func) ||
3050 !isLibFuncEmittable(M, TLI, Func) ||
3051 !isTrigLibCall(CI))
3052 return;
3053
3054 if (IsFloat) {
3055 if (Func == LibFunc_sinpif)
3056 SinCalls.push_back(CI);
3057 else if (Func == LibFunc_cospif)
3058 CosCalls.push_back(CI);
3059 else if (Func == LibFunc_sincospif_stret)
3060 SinCosCalls.push_back(CI);
3061 } else {
3062 if (Func == LibFunc_sinpi)
3063 SinCalls.push_back(CI);
3064 else if (Func == LibFunc_cospi)
3065 CosCalls.push_back(CI);
3066 else if (Func == LibFunc_sincospi_stret)
3067 SinCosCalls.push_back(CI);
3068 }
3069}
3070
3071/// Constant folds remquo
3072Value *LibCallSimplifier::optimizeRemquo(CallInst *CI, IRBuilderBase &B) {
3073 const APFloat *X, *Y;
3074 if (!match(CI->getArgOperand(0), m_APFloat(X)) ||
3075 !match(CI->getArgOperand(1), m_APFloat(Y)))
3076 return nullptr;
3077
3079 APFloat Quot = *X;
3082 return nullptr;
3083 APFloat Rem = *X;
3084 if (Rem.remainder(*Y) != APFloat::opOK)
3085 return nullptr;
3086
3087 // TODO: We can only keep at least the three of the last bits of x/y
3088 unsigned IntBW = TLI->getIntSize();
3089 APSInt QuotInt(IntBW, /*isUnsigned=*/false);
3090 bool IsExact;
3091 Status =
3092 Quot.convertToInteger(QuotInt, APFloat::rmNearestTiesToEven, &IsExact);
3094 return nullptr;
3095
3096 B.CreateAlignedStore(
3097 ConstantInt::get(B.getIntNTy(IntBW), QuotInt.getExtValue()),
3098 CI->getArgOperand(2), CI->getParamAlign(2));
3099 return ConstantFP::get(CI->getType(), Rem);
3100}
3101
3102/// Constant folds fdim
3103Value *LibCallSimplifier::optimizeFdim(CallInst *CI, IRBuilderBase &B) {
3104 // Cannot perform the fold unless the call has attribute memory(none)
3105 if (!CI->doesNotAccessMemory())
3106 return nullptr;
3107
3108 // TODO : Handle undef values
3109 // Propagate poison if any
3110 if (isa<PoisonValue>(CI->getArgOperand(0)))
3111 return CI->getArgOperand(0);
3112 if (isa<PoisonValue>(CI->getArgOperand(1)))
3113 return CI->getArgOperand(1);
3114
3115 const APFloat *X, *Y;
3116 // Check if both values are constants
3117 if (!match(CI->getArgOperand(0), m_APFloat(X)) ||
3118 !match(CI->getArgOperand(1), m_APFloat(Y)))
3119 return nullptr;
3120
3121 APFloat Difference = *X;
3123
3124 APFloat MaxVal =
3125 maximum(Difference, APFloat::getZero(CI->getType()->getFltSemantics()));
3126 return ConstantFP::get(CI->getType(), MaxVal);
3127}
3128
3129//===----------------------------------------------------------------------===//
3130// Integer Library Call Optimizations
3131//===----------------------------------------------------------------------===//
3132
3133Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
3134 // All variants of ffs return int which need not be 32 bits wide.
3135 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
3136 Type *RetType = CI->getType();
3137 Value *Op = CI->getArgOperand(0);
3138 Type *ArgType = Op->getType();
3139 Value *V = B.CreateIntrinsic(Intrinsic::cttz, {ArgType}, {Op, B.getTrue()},
3140 nullptr, "cttz");
3141 V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1));
3142 V = B.CreateIntCast(V, RetType, false);
3143
3144 Value *Cond = B.CreateICmpNE(Op, Constant::getNullValue(ArgType));
3145 return B.CreateSelect(Cond, V, ConstantInt::get(RetType, 0));
3146}
3147
3148Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
3149 // All variants of fls return int which need not be 32 bits wide.
3150 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
3151 Value *Op = CI->getArgOperand(0);
3152 Type *ArgType = Op->getType();
3153 Value *V = B.CreateIntrinsic(Intrinsic::ctlz, {ArgType}, {Op, B.getFalse()},
3154 nullptr, "ctlz");
3155 V = B.CreateSub(ConstantInt::get(V->getType(), ArgType->getIntegerBitWidth()),
3156 V);
3157 return B.CreateIntCast(V, CI->getType(), false);
3158}
3159
3160Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
3161 // abs(x) -> x <s 0 ? -x : x
3162 // The negation has 'nsw' because abs of INT_MIN is undefined.
3163 Value *X = CI->getArgOperand(0);
3164 Value *IsNeg = B.CreateIsNeg(X);
3165 Value *NegX = B.CreateNSWNeg(X, "neg");
3166 return B.CreateSelect(IsNeg, NegX, X);
3167}
3168
3169Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
3170 // isdigit(c) -> (c-'0') <u 10
3171 Value *Op = CI->getArgOperand(0);
3172 Type *ArgType = Op->getType();
3173 Op = B.CreateSub(Op, ConstantInt::get(ArgType, '0'), "isdigittmp");
3174 Op = B.CreateICmpULT(Op, ConstantInt::get(ArgType, 10), "isdigit");
3175 return B.CreateZExt(Op, CI->getType());
3176}
3177
3178Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
3179 // isascii(c) -> c <u 128
3180 Value *Op = CI->getArgOperand(0);
3181 Type *ArgType = Op->getType();
3182 Op = B.CreateICmpULT(Op, ConstantInt::get(ArgType, 128), "isascii");
3183 return B.CreateZExt(Op, CI->getType());
3184}
3185
3186Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
3187 // toascii(c) -> c & 0x7f
3188 return B.CreateAnd(CI->getArgOperand(0),
3189 ConstantInt::get(CI->getType(), 0x7F));
3190}
3191
3192// Fold calls to atoi, atol, and atoll.
3193Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
3194 CI->addParamAttr(0, Attribute::NoCapture);
3195
3196 StringRef Str;
3197 if (!getConstantStringInfo(CI->getArgOperand(0), Str))
3198 return nullptr;
3199
3200 return convertStrToInt(CI, Str, nullptr, 10, /*AsSigned=*/true, B);
3201}
3202
3203// Fold calls to strtol, strtoll, strtoul, and strtoull.
3204Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
3205 bool AsSigned) {
3206 Value *EndPtr = CI->getArgOperand(1);
3207 if (isa<ConstantPointerNull>(EndPtr)) {
3208 // With a null EndPtr, this function won't capture the main argument.
3209 // It would be readonly too, except that it still may write to errno.
3210 CI->addParamAttr(0, Attribute::NoCapture);
3211 EndPtr = nullptr;
3212 } else if (!isKnownNonZero(EndPtr, DL))
3213 return nullptr;
3214
3215 StringRef Str;
3216 if (!getConstantStringInfo(CI->getArgOperand(0), Str))
3217 return nullptr;
3218
3219 if (ConstantInt *CInt = dyn_cast<ConstantInt>(CI->getArgOperand(2))) {
3220 return convertStrToInt(CI, Str, EndPtr, CInt->getSExtValue(), AsSigned, B);
3221 }
3222
3223 return nullptr;
3224}
3225
3226//===----------------------------------------------------------------------===//
3227// Formatting and IO Library Call Optimizations
3228//===----------------------------------------------------------------------===//
3229
3230static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
3231
3232Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
3233 int StreamArg) {
3235 // Error reporting calls should be cold, mark them as such.
3236 // This applies even to non-builtin calls: it is only a hint and applies to
3237 // functions that the frontend might not understand as builtins.
3238
3239 // This heuristic was suggested in:
3240 // Improving Static Branch Prediction in a Compiler
3241 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
3242 // Proceedings of PACT'98, Oct. 1998, IEEE
3243 if (!CI->hasFnAttr(Attribute::Cold) &&
3244 isReportingError(Callee, CI, StreamArg)) {
3245 CI->addFnAttr(Attribute::Cold);
3246 }
3247
3248 return nullptr;
3249}
3250
3251static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
3252 if (!Callee || !Callee->isDeclaration())
3253 return false;
3254
3255 if (StreamArg < 0)
3256 return true;
3257
3258 // These functions might be considered cold, but only if their stream
3259 // argument is stderr.
3260
3261 if (StreamArg >= (int)CI->arg_size())
3262 return false;
3263 LoadInst *LI = dyn_cast<LoadInst>(CI->getArgOperand(StreamArg));
3264 if (!LI)
3265 return false;
3266 GlobalVariable *GV = dyn_cast<GlobalVariable>(LI->getPointerOperand());
3267 if (!GV || !GV->isDeclaration())
3268 return false;
3269 return GV->getName() == "stderr";
3270}
3271
3272Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
3273 // Check for a fixed format string.
3274 StringRef FormatStr;
3275 if (!getConstantStringInfo(CI->getArgOperand(0), FormatStr))
3276 return nullptr;
3277
3278 // Empty format string -> noop.
3279 if (FormatStr.empty()) // Tolerate printf's declared void.
3280 return CI->use_empty() ? (Value *)CI : ConstantInt::get(CI->getType(), 0);
3281
3282 // Do not do any of the following transformations if the printf return value
3283 // is used, in general the printf return value is not compatible with either
3284 // putchar() or puts().
3285 if (!CI->use_empty())
3286 return nullptr;
3287
3288 Type *IntTy = CI->getType();
3289 // printf("x") -> putchar('x'), even for "%" and "%%".
3290 if (FormatStr.size() == 1 || FormatStr == "%%") {
3291 // Convert the character to unsigned char before passing it to putchar
3292 // to avoid host-specific sign extension in the IR. Putchar converts
3293 // it to unsigned char regardless.
3294 Value *IntChar = ConstantInt::get(IntTy, (unsigned char)FormatStr[0]);
3295 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
3296 }
3297
3298 // Try to remove call or emit putchar/puts.
3299 if (FormatStr == "%s" && CI->arg_size() > 1) {
3300 StringRef OperandStr;
3301 if (!getConstantStringInfo(CI->getOperand(1), OperandStr))
3302 return nullptr;
3303 // printf("%s", "") --> NOP
3304 if (OperandStr.empty())
3305 return (Value *)CI;
3306 // printf("%s", "a") --> putchar('a')
3307 if (OperandStr.size() == 1) {
3308 // Convert the character to unsigned char before passing it to putchar
3309 // to avoid host-specific sign extension in the IR. Putchar converts
3310 // it to unsigned char regardless.
3311 Value *IntChar = ConstantInt::get(IntTy, (unsigned char)OperandStr[0]);
3312 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
3313 }
3314 // printf("%s", str"\n") --> puts(str)
3315 if (OperandStr.back() == '\n') {
3316 OperandStr = OperandStr.drop_back();
3317 Value *GV = B.CreateGlobalString(OperandStr, "str");
3318 return copyFlags(*CI, emitPutS(GV, B, TLI));
3319 }
3320 return nullptr;
3321 }
3322
3323 // printf("foo\n") --> puts("foo")
3324 if (FormatStr.back() == '\n' &&
3325 !FormatStr.contains('%')) { // No format characters.
3326 // Create a string literal with no \n on it. We expect the constant merge
3327 // pass to be run after this pass, to merge duplicate strings.
3328 FormatStr = FormatStr.drop_back();
3329 Value *GV = B.CreateGlobalString(FormatStr, "str");
3330 return copyFlags(*CI, emitPutS(GV, B, TLI));
3331 }
3332
3333 // Optimize specific format strings.
3334 // printf("%c", chr) --> putchar(chr)
3335 if (FormatStr == "%c" && CI->arg_size() > 1 &&
3336 CI->getArgOperand(1)->getType()->isIntegerTy()) {
3337 // Convert the argument to the type expected by putchar, i.e., int, which
3338 // need not be 32 bits wide but which is the same as printf's return type.
3339 Value *IntChar = B.CreateIntCast(CI->getArgOperand(1), IntTy, false);
3340 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
3341 }
3342
3343 // printf("%s\n", str) --> puts(str)
3344 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3345 CI->getArgOperand(1)->getType()->isPointerTy())
3346 return copyFlags(*CI, emitPutS(CI->getArgOperand(1), B, TLI));
3347 return nullptr;
3348}
3349
3350Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3351
3352 Module *M = CI->getModule();
3354 FunctionType *FT = Callee->getFunctionType();
3355 if (Value *V = optimizePrintFString(CI, B)) {
3356 return V;
3357 }
3358
3360
3361 // printf(format, ...) -> iprintf(format, ...) if no floating point
3362 // arguments.
3363 if (isLibFuncEmittable(M, TLI, LibFunc_iprintf) &&
3365 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_iprintf, FT,
3366 Callee->getAttributes());
3367 CallInst *New = cast<CallInst>(CI->clone());
3368 New->setCalledFunction(IPrintFFn);
3369 B.Insert(New);
3370 return New;
3371 }
3372
3373 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3374 // arguments.
3375 if (isLibFuncEmittable(M, TLI, LibFunc_small_printf) &&
3376 !callHasFP128Argument(CI)) {
3377 auto SmallPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_small_printf, FT,
3378 Callee->getAttributes());
3379 CallInst *New = cast<CallInst>(CI->clone());
3380 New->setCalledFunction(SmallPrintFFn);
3381 B.Insert(New);
3382 return New;
3383 }
3384
3385 return nullptr;
3386}
3387
3388Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3389 IRBuilderBase &B) {
3390 // Check for a fixed format string.
3391 StringRef FormatStr;
3392 if (!getConstantStringInfo(CI->getArgOperand(1), FormatStr))
3393 return nullptr;
3394
3395 // If we just have a format string (nothing else crazy) transform it.
3396 Value *Dest = CI->getArgOperand(0);
3397 if (CI->arg_size() == 2) {
3398 // Make sure there's no % in the constant array. We could try to handle
3399 // %% -> % in the future if we cared.
3400 if (FormatStr.contains('%'))
3401 return nullptr; // we found a format specifier, bail out.
3402
3403 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3404 B.CreateMemCpy(Dest, Align(1), CI->getArgOperand(1), Align(1),
3405 // Copy the null byte.
3406 TLI->getAsSizeT(FormatStr.size() + 1, *CI->getModule()));
3407 return ConstantInt::get(CI->getType(), FormatStr.size());
3408 }
3409
3410 // The remaining optimizations require the format string to be "%s" or "%c"
3411 // and have an extra operand.
3412 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3413 return nullptr;
3414
3415 // Decode the second character of the format string.
3416 if (FormatStr[1] == 'c') {
3417 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3418 if (!CI->getArgOperand(2)->getType()->isIntegerTy())
3419 return nullptr;
3420 Value *V = B.CreateTrunc(CI->getArgOperand(2), B.getInt8Ty(), "char");
3421 Value *Ptr = Dest;
3422 B.CreateStore(V, Ptr);
3423 Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Ptr, B.getInt32(1), "nul");
3424 B.CreateStore(B.getInt8(0), Ptr);
3425
3426 return ConstantInt::get(CI->getType(), 1);
3427 }
3428
3429 if (FormatStr[1] == 's') {
3430 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3431 // strlen(str)+1)
3432 if (!CI->getArgOperand(2)->getType()->isPointerTy())
3433 return nullptr;
3434
3435 if (CI->use_empty())
3436 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3437 return copyFlags(*CI, emitStrCpy(Dest, CI->getArgOperand(2), B, TLI));
3438
3439 uint64_t SrcLen = GetStringLength(CI->getArgOperand(2));
3440 if (SrcLen) {
3441 B.CreateMemCpy(Dest, Align(1), CI->getArgOperand(2), Align(1),
3442 TLI->getAsSizeT(SrcLen, *CI->getModule()));
3443 // Returns total number of characters written without null-character.
3444 return ConstantInt::get(CI->getType(), SrcLen - 1);
3445 } else if (Value *V = emitStpCpy(Dest, CI->getArgOperand(2), B, TLI)) {
3446 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3447 Value *PtrDiff = B.CreatePtrDiff(B.getInt8Ty(), V, Dest);
3448 return B.CreateIntCast(PtrDiff, CI->getType(), false);
3449 }
3450
3451 if (llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
3453 return nullptr;
3454
3455 Value *Len = emitStrLen(CI->getArgOperand(2), B, DL, TLI);
3456 if (!Len)
3457 return nullptr;
3458 Value *IncLen =
3459 B.CreateAdd(Len, ConstantInt::get(Len->getType(), 1), "leninc");
3460 B.CreateMemCpy(Dest, Align(1), CI->getArgOperand(2), Align(1), IncLen);
3461
3462 // The sprintf result is the unincremented number of bytes in the string.
3463 return B.CreateIntCast(Len, CI->getType(), false);
3464 }
3465 return nullptr;
3466}
3467
3468Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3469 Module *M = CI->getModule();
3471 FunctionType *FT = Callee->getFunctionType();
3472 if (Value *V = optimizeSPrintFString(CI, B)) {
3473 return V;
3474 }
3475
3477
3478 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3479 // point arguments.
3480 if (isLibFuncEmittable(M, TLI, LibFunc_siprintf) &&
3482 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_siprintf,
3483 FT, Callee->getAttributes());
3484 CallInst *New = cast<CallInst>(CI->clone());
3485 New->setCalledFunction(SIPrintFFn);
3486 B.Insert(New);
3487 return New;
3488 }
3489
3490 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3491 // floating point arguments.
3492 if (isLibFuncEmittable(M, TLI, LibFunc_small_sprintf) &&
3493 !callHasFP128Argument(CI)) {
3494 auto SmallSPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_small_sprintf, FT,
3495 Callee->getAttributes());
3496 CallInst *New = cast<CallInst>(CI->clone());
3497 New->setCalledFunction(SmallSPrintFFn);
3498 B.Insert(New);
3499 return New;
3500 }
3501
3502 return nullptr;
3503}
3504
3505// Transform an snprintf call CI with the bound N to format the string Str
3506// either to a call to memcpy, or to single character a store, or to nothing,
3507// and fold the result to a constant. A nonnull StrArg refers to the string
3508// argument being formatted. Otherwise the call is one with N < 2 and
3509// the "%c" directive to format a single character.
3510Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3511 StringRef Str, uint64_t N,
3512 IRBuilderBase &B) {
3513 assert(StrArg || (N < 2 && Str.size() == 1));
3514
3515 unsigned IntBits = TLI->getIntSize();
3516 uint64_t IntMax = maxIntN(IntBits);
3517 if (Str.size() > IntMax)
3518 // Bail if the string is longer than INT_MAX. POSIX requires
3519 // implementations to set errno to EOVERFLOW in this case, in
3520 // addition to when N is larger than that (checked by the caller).
3521 return nullptr;
3522
3523 Value *StrLen = ConstantInt::get(CI->getType(), Str.size());
3524 if (N == 0)
3525 return StrLen;
3526
3527 // Set to the number of bytes to copy fron StrArg which is also
3528 // the offset of the terinating nul.
3529 uint64_t NCopy;
3530 if (N > Str.size())
3531 // Copy the full string, including the terminating nul (which must
3532 // be present regardless of the bound).
3533 NCopy = Str.size() + 1;
3534 else
3535 NCopy = N - 1;
3536
3537 Value *DstArg = CI->getArgOperand(0);
3538 if (NCopy && StrArg)
3539 // Transform the call to lvm.memcpy(dst, fmt, N).
3540 copyFlags(*CI, B.CreateMemCpy(DstArg, Align(1), StrArg, Align(1),
3541 TLI->getAsSizeT(NCopy, *CI->getModule())));
3542
3543 if (N > Str.size())
3544 // Return early when the whole format string, including the final nul,
3545 // has been copied.
3546 return StrLen;
3547
3548 // Otherwise, when truncating the string append a terminating nul.
3549 Type *Int8Ty = B.getInt8Ty();
3550 Value *NulOff = B.getIntN(IntBits, NCopy);
3551 Value *DstEnd = B.CreateInBoundsGEP(Int8Ty, DstArg, NulOff, "endptr");
3552 B.CreateStore(ConstantInt::get(Int8Ty, 0), DstEnd);
3553 return StrLen;
3554}
3555
3556Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3557 IRBuilderBase &B) {
3558 // Check for size
3559 ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
3560 if (!Size)
3561 return nullptr;
3562
3563 uint64_t N = Size->getZExtValue();
3564 uint64_t IntMax = maxIntN(TLI->getIntSize());
3565 if (N > IntMax)
3566 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3567 // to set errno to EOVERFLOW in this case.
3568 return nullptr;
3569
3570 Value *DstArg = CI->getArgOperand(0);
3571 Value *FmtArg = CI->getArgOperand(2);
3572
3573 // Check for a fixed format string.
3574 StringRef FormatStr;
3575 if (!getConstantStringInfo(FmtArg, FormatStr))
3576 return nullptr;
3577
3578 // If we just have a format string (nothing else crazy) transform it.
3579 if (CI->arg_size() == 3) {
3580 if (FormatStr.contains('%'))
3581 // Bail if the format string contains a directive and there are
3582 // no arguments. We could handle "%%" in the future.
3583 return nullptr;
3584
3585 return emitSnPrintfMemCpy(CI, FmtArg, FormatStr, N, B);
3586 }
3587
3588 // The remaining optimizations require the format string to be "%s" or "%c"
3589 // and have an extra operand.
3590 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3591 return nullptr;
3592
3593 // Decode the second character of the format string.
3594 if (FormatStr[1] == 'c') {
3595 if (N <= 1) {
3596 // Use an arbitary string of length 1 to transform the call into
3597 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3598 // to one.
3599 StringRef CharStr("*");
3600 return emitSnPrintfMemCpy(CI, nullptr, CharStr, N, B);
3601 }
3602
3603 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3604 if (!CI->getArgOperand(3)->getType()->isIntegerTy())
3605 return nullptr;
3606 Value *V = B.CreateTrunc(CI->getArgOperand(3), B.getInt8Ty(), "char");
3607 Value *Ptr = DstArg;
3608 B.CreateStore(V, Ptr);
3609 Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Ptr, B.getInt32(1), "nul");
3610 B.CreateStore(B.getInt8(0), Ptr);
3611 return ConstantInt::get(CI->getType(), 1);
3612 }
3613
3614 if (FormatStr[1] != 's')
3615 return nullptr;
3616
3617 Value *StrArg = CI->getArgOperand(3);
3618 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3619 StringRef Str;
3620 if (!getConstantStringInfo(StrArg, Str))
3621 return nullptr;
3622
3623 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3624}
3625
3626Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3627 if (Value *V = optimizeSnPrintFString(CI, B)) {
3628 return V;
3629 }
3630
3631 if (isKnownNonZero(CI->getOperand(1), DL))
3633 return nullptr;
3634}
3635
3636Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3637 IRBuilderBase &B) {
3638 optimizeErrorReporting(CI, B, 0);
3639
3640 // All the optimizations depend on the format string.
3641 StringRef FormatStr;
3642 if (!getConstantStringInfo(CI->getArgOperand(1), FormatStr))
3643 return nullptr;
3644
3645 // Do not do any of the following transformations if the fprintf return
3646 // value is used, in general the fprintf return value is not compatible
3647 // with fwrite(), fputc() or fputs().
3648 if (!CI->use_empty())
3649 return nullptr;
3650
3651 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3652 if (CI->arg_size() == 2) {
3653 // Could handle %% -> % if we cared.
3654 if (FormatStr.contains('%'))
3655 return nullptr; // We found a format specifier.
3656
3657 return copyFlags(
3658 *CI, emitFWrite(CI->getArgOperand(1),
3659 TLI->getAsSizeT(FormatStr.size(), *CI->getModule()),
3660 CI->getArgOperand(0), B, DL, TLI));
3661 }
3662
3663 // The remaining optimizations require the format string to be "%s" or "%c"
3664 // and have an extra operand.
3665 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3666 return nullptr;
3667
3668 // Decode the second character of the format string.
3669 if (FormatStr[1] == 'c') {
3670 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3671 if (!CI->getArgOperand(2)->getType()->isIntegerTy())
3672 return nullptr;
3673 Type *IntTy = B.getIntNTy(TLI->getIntSize());
3674 Value *V = B.CreateIntCast(CI->getArgOperand(2), IntTy, /*isSigned*/ true,
3675 "chari");
3676 return copyFlags(*CI, emitFPutC(V, CI->getArgOperand(0), B, TLI));
3677 }
3678
3679 if (FormatStr[1] == 's') {
3680 // fprintf(F, "%s", str) --> fputs(str, F)
3681 if (!CI->getArgOperand(2)->getType()->isPointerTy())
3682 return nullptr;
3683 return copyFlags(
3684 *CI, emitFPutS(CI->getArgOperand(2), CI->getArgOperand(0), B, TLI));
3685 }
3686 return nullptr;
3687}
3688
3689Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3690 Module *M = CI->getModule();
3692 FunctionType *FT = Callee->getFunctionType();
3693 if (Value *V = optimizeFPrintFString(CI, B)) {
3694 return V;
3695 }
3696
3697 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3698 // floating point arguments.
3699 if (isLibFuncEmittable(M, TLI, LibFunc_fiprintf) &&
3701 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_fiprintf,
3702 FT, Callee->getAttributes());
3703 CallInst *New = cast<CallInst>(CI->clone());
3704 New->setCalledFunction(FIPrintFFn);
3705 B.Insert(New);
3706 return New;
3707 }
3708
3709 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3710 // 128-bit floating point arguments.
3711 if (isLibFuncEmittable(M, TLI, LibFunc_small_fprintf) &&
3712 !callHasFP128Argument(CI)) {
3713 auto SmallFPrintFFn =
3714 getOrInsertLibFunc(M, *TLI, LibFunc_small_fprintf, FT,
3715 Callee->getAttributes());
3716 CallInst *New = cast<CallInst>(CI->clone());
3717 New->setCalledFunction(SmallFPrintFFn);
3718 B.Insert(New);
3719 return New;
3720 }
3721
3722 return nullptr;
3723}
3724
3725Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3726 optimizeErrorReporting(CI, B, 3);
3727
3728 // Get the element size and count.
3729 ConstantInt *SizeC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
3730 ConstantInt *CountC = dyn_cast<ConstantInt>(CI->getArgOperand(2));
3731 if (SizeC && CountC) {
3732 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3733
3734 // If this is writing zero records, remove the call (it's a noop).
3735 if (Bytes == 0)
3736 return ConstantInt::get(CI->getType(), 0);
3737
3738 // If this is writing one byte, turn it into fputc.
3739 // This optimisation is only valid, if the return value is unused.
3740 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3741 Value *Char = B.CreateLoad(B.getInt8Ty(), CI->getArgOperand(0), "char");
3742 Type *IntTy = B.getIntNTy(TLI->getIntSize());
3743 Value *Cast = B.CreateIntCast(Char, IntTy, /*isSigned*/ true, "chari");
3744 Value *NewCI = emitFPutC(Cast, CI->getArgOperand(3), B, TLI);
3745 return NewCI ? ConstantInt::get(CI->getType(), 1) : nullptr;
3746 }
3747 }
3748
3749 return nullptr;
3750}
3751
3752Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
3753 optimizeErrorReporting(CI, B, 1);
3754
3755 // Don't rewrite fputs to fwrite when optimising for size because fwrite
3756 // requires more arguments and thus extra MOVs are required.
3757 if (llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
3759 return nullptr;
3760
3761 // We can't optimize if return value is used.
3762 if (!CI->use_empty())
3763 return nullptr;
3764
3765 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
3767 if (!Len)
3768 return nullptr;
3769
3770 // Known to have no uses (see above).
3771 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
3772 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
3773 return copyFlags(
3774 *CI,
3776 ConstantInt::get(SizeTTy, Len - 1),
3777 CI->getArgOperand(1), B, DL, TLI));
3778}
3779
3780Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
3782 if (!CI->use_empty())
3783 return nullptr;
3784
3785 // Check for a constant string.
3786 // puts("") -> putchar('\n')
3787 StringRef Str;
3788 if (getConstantStringInfo(CI->getArgOperand(0), Str) && Str.empty()) {
3789 // putchar takes an argument of the same type as puts returns, i.e.,
3790 // int, which need not be 32 bits wide.
3791 Type *IntTy = CI->getType();
3792 return copyFlags(*CI, emitPutChar(ConstantInt::get(IntTy, '\n'), B, TLI));
3793 }
3794
3795 return nullptr;
3796}
3797
3798Value *LibCallSimplifier::optimizeExit(CallInst *CI) {
3799
3800 // Mark 'exit' as cold if its not exit(0) (success).
3801 const APInt *C;
3802 if (!CI->hasFnAttr(Attribute::Cold) &&
3803 match(CI->getArgOperand(0), m_APInt(C)) && !C->isZero()) {
3804 CI->addFnAttr(Attribute::Cold);
3805 }
3806 return nullptr;
3807}
3808
3809Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
3810 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
3811 return copyFlags(*CI, B.CreateMemMove(CI->getArgOperand(1), Align(1),
3812 CI->getArgOperand(0), Align(1),
3813 CI->getArgOperand(2)));
3814}
3815
3816bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
3817 SmallString<20> FloatFuncName = FuncName;
3818 FloatFuncName += 'f';
3819 return isLibFuncEmittable(M, TLI, FloatFuncName);
3820}
3821
3822Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
3823 IRBuilderBase &Builder) {
3824 Module *M = CI->getModule();
3825 LibFunc Func;
3827
3828 // Check for string/memory library functions.
3829 if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
3830 // Make sure we never change the calling convention.
3831 assert(
3832 (ignoreCallingConv(Func) ||
3834 "Optimizing string/memory libcall would change the calling convention");
3835 switch (Func) {
3836 case LibFunc_strcat:
3837 return optimizeStrCat(CI, Builder);
3838 case LibFunc_strncat:
3839 return optimizeStrNCat(CI, Builder);
3840 case LibFunc_strchr:
3841 return optimizeStrChr(CI, Builder);
3842 case LibFunc_strrchr:
3843 return optimizeStrRChr(CI, Builder);
3844 case LibFunc_strcmp:
3845 return optimizeStrCmp(CI, Builder);
3846 case LibFunc_strncmp:
3847 return optimizeStrNCmp(CI, Builder);
3848 case LibFunc_strcpy:
3849 return optimizeStrCpy(CI, Builder);
3850 case LibFunc_stpcpy:
3851 return optimizeStpCpy(CI, Builder);
3852 case LibFunc_strlcpy:
3853 return optimizeStrLCpy(CI, Builder);
3854 case LibFunc_stpncpy:
3855 return optimizeStringNCpy(CI, /*RetEnd=*/true, Builder);
3856 case LibFunc_strncpy:
3857 return optimizeStringNCpy(CI, /*RetEnd=*/false, Builder);
3858 case LibFunc_strlen:
3859 return optimizeStrLen(CI, Builder);
3860 case LibFunc_strnlen:
3861 return optimizeStrNLen(CI, Builder);
3862 case LibFunc_strpbrk:
3863 return optimizeStrPBrk(CI, Builder);
3864 case LibFunc_strndup:
3865 return optimizeStrNDup(CI, Builder);
3866 case LibFunc_strtol:
3867 case LibFunc_strtod:
3868 case LibFunc_strtof:
3869 case LibFunc_strtoul:
3870 case LibFunc_strtoll:
3871 case LibFunc_strtold:
3872 case LibFunc_strtoull:
3873 return optimizeStrTo(CI, Builder);
3874 case LibFunc_strspn:
3875 return optimizeStrSpn(CI, Builder);
3876 case LibFunc_strcspn:
3877 return optimizeStrCSpn(CI, Builder);
3878 case LibFunc_strstr:
3879 return optimizeStrStr(CI, Builder);
3880 case LibFunc_memchr:
3881 return optimizeMemChr(CI, Builder);
3882 case LibFunc_memrchr:
3883 return optimizeMemRChr(CI, Builder);
3884 case LibFunc_bcmp:
3885 return optimizeBCmp(CI, Builder);
3886 case LibFunc_memcmp:
3887 return optimizeMemCmp(CI, Builder);
3888 case LibFunc_memcpy:
3889 return optimizeMemCpy(CI, Builder);
3890 case LibFunc_memccpy:
3891 return optimizeMemCCpy(CI, Builder);
3892 case LibFunc_mempcpy:
3893 return optimizeMemPCpy(CI, Builder);
3894 case LibFunc_memmove:
3895 return optimizeMemMove(CI, Builder);
3896 case LibFunc_memset:
3897 return optimizeMemSet(CI, Builder);
3898 case LibFunc_realloc:
3899 return optimizeRealloc(CI, Builder);
3900 case LibFunc_wcslen:
3901 return optimizeWcslen(CI, Builder);
3902 case LibFunc_bcopy:
3903 return optimizeBCopy(CI, Builder);
3904 case LibFunc_Znwm:
3905 case LibFunc_ZnwmRKSt9nothrow_t:
3906 case LibFunc_ZnwmSt11align_val_t:
3907 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
3908 case LibFunc_Znam:
3909 case LibFunc_ZnamRKSt9nothrow_t:
3910 case LibFunc_ZnamSt11align_val_t:
3911 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
3912 case LibFunc_Znwm12__hot_cold_t:
3913 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
3914 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
3915 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
3916 case LibFunc_Znam12__hot_cold_t:
3917 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
3918 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
3919 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
3920 case LibFunc_size_returning_new:
3921 case LibFunc_size_returning_new_hot_cold:
3922 case LibFunc_size_returning_new_aligned:
3923 case LibFunc_size_returning_new_aligned_hot_cold:
3924 return optimizeNew(CI, Builder, Func);
3925 default:
3926 break;
3927 }
3928 }
3929 return nullptr;
3930}
3931
3932/// Constant folding nan/nanf/nanl.
3934 StringRef CharSeq;
3935 if (!getConstantStringInfo(CI->getArgOperand(0), CharSeq))
3936 return nullptr;
3937
3938 APInt Fill;
3939 // Treat empty strings as if they were zero.
3940 if (CharSeq.empty())
3941 Fill = APInt(32, 0);
3942 else if (CharSeq.getAsInteger(0, Fill))
3943 return nullptr;
3944
3945 return ConstantFP::getQNaN(CI->getType(), /*Negative=*/false, &Fill);
3946}
3947
3948Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
3949 LibFunc Func,
3950 IRBuilderBase &Builder) {
3951 const Module *M = CI->getModule();
3952
3953 // Don't optimize calls that require strict floating point semantics.
3954 if (CI->isStrictFP())
3955 return nullptr;
3956
3957 if (Value *V = optimizeSymmetric(CI, Func, Builder))
3958 return V;
3959
3960 switch (Func) {
3961 case LibFunc_sinpif:
3962 case LibFunc_sinpi:
3963 return optimizeSinCosPi(CI, /*IsSin*/true, Builder);
3964 case LibFunc_cospif:
3965 case LibFunc_cospi:
3966 return optimizeSinCosPi(CI, /*IsSin*/false, Builder);
3967 case LibFunc_powf:
3968 case LibFunc_pow:
3969 case LibFunc_powl:
3970 return optimizePow(CI, Builder);
3971 case LibFunc_exp2l:
3972 case LibFunc_exp2:
3973 case LibFunc_exp2f:
3974 return optimizeExp2(CI, Builder);
3975 case LibFunc_fabsf:
3976 case LibFunc_fabs:
3977 case LibFunc_fabsl:
3978 return replaceUnaryCall(CI, Builder, Intrinsic::fabs);
3979 case LibFunc_sqrtf:
3980 case LibFunc_sqrt:
3981 case LibFunc_sqrtl:
3982 return optimizeSqrt(CI, Builder);
3983 case LibFunc_fmod:
3984 case LibFunc_fmodf:
3985 case LibFunc_fmodl:
3986 return optimizeFMod(CI, Builder);
3987 case LibFunc_logf:
3988 case LibFunc_log:
3989 case LibFunc_logl:
3990 case LibFunc_log10f:
3991 case LibFunc_log10:
3992 case LibFunc_log10l:
3993 case LibFunc_log1pf:
3994 case LibFunc_log1p:
3995 case LibFunc_log1pl:
3996 case LibFunc_log2f:
3997 case LibFunc_log2:
3998 case LibFunc_log2l:
3999 case LibFunc_logbf:
4000 case LibFunc_logb:
4001 case LibFunc_logbl:
4002 return optimizeLog(CI, Builder);
4003 case LibFunc_tan:
4004 case LibFunc_tanf:
4005 case LibFunc_tanl:
4006 case LibFunc_sinh:
4007 case LibFunc_sinhf:
4008 case LibFunc_sinhl:
4009 case LibFunc_asinh:
4010 case LibFunc_asinhf:
4011 case LibFunc_asinhl:
4012 case LibFunc_cosh:
4013 case LibFunc_coshf:
4014 case LibFunc_coshl:
4015 case LibFunc_atanh:
4016 case LibFunc_atanhf:
4017 case LibFunc_atanhl:
4018 return optimizeTrigInversionPairs(CI, Builder);
4019 case LibFunc_ceil:
4020 return replaceUnaryCall(CI, Builder, Intrinsic::ceil);
4021 case LibFunc_floor:
4022 return replaceUnaryCall(CI, Builder, Intrinsic::floor);
4023 case LibFunc_round:
4024 return replaceUnaryCall(CI, Builder, Intrinsic::round);
4025 case LibFunc_roundeven:
4026 return replaceUnaryCall(CI, Builder, Intrinsic::roundeven);
4027 case LibFunc_nearbyint:
4028 return replaceUnaryCall(CI, Builder, Intrinsic::nearbyint);
4029 case LibFunc_rint:
4030 return replaceUnaryCall(CI, Builder, Intrinsic::rint);
4031 case LibFunc_trunc:
4032 return replaceUnaryCall(CI, Builder, Intrinsic::trunc);
4033 case LibFunc_acos:
4034 case LibFunc_acosh:
4035 case LibFunc_asin:
4036 case LibFunc_atan:
4037 case LibFunc_cbrt:
4038 case LibFunc_exp:
4039 case LibFunc_exp10:
4040 case LibFunc_expm1:
4041 case LibFunc_cos:
4042 case LibFunc_sin:
4043 case LibFunc_tanh:
4044 if (UnsafeFPShrink && hasFloatVersion(M, CI->getCalledFunction()->getName()))
4045 return optimizeUnaryDoubleFP(CI, Builder, TLI, true);
4046 return nullptr;
4047 case LibFunc_copysign:
4048 if (hasFloatVersion(M, CI->getCalledFunction()->getName()))
4049 return optimizeBinaryDoubleFP(CI, Builder, TLI);
4050 return nullptr;
4051 case LibFunc_fdim:
4052 case LibFunc_fdimf:
4053 case LibFunc_fdiml:
4054 return optimizeFdim(CI, Builder);
4055 case LibFunc_fminf:
4056 case LibFunc_fmin:
4057 case LibFunc_fminl:
4058 case LibFunc_fmaxf:
4059 case LibFunc_fmax:
4060 case LibFunc_fmaxl:
4061 return optimizeFMinFMax(CI, Builder);
4062 case LibFunc_cabs:
4063 case LibFunc_cabsf:
4064 case LibFunc_cabsl:
4065 return optimizeCAbs(CI, Builder);
4066 case LibFunc_remquo:
4067 case LibFunc_remquof:
4068 case LibFunc_remquol:
4069 return optimizeRemquo(CI, Builder);
4070 case LibFunc_nan:
4071 case LibFunc_nanf:
4072 case LibFunc_nanl:
4073 return optimizeNaN(CI);
4074 default:
4075 return nullptr;
4076 }
4077}
4078
4080 Module *M = CI->getModule();
4081 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
4082
4083 // TODO: Split out the code below that operates on FP calls so that
4084 // we can all non-FP calls with the StrictFP attribute to be
4085 // optimized.
4086 if (CI->isNoBuiltin())
4087 return nullptr;
4088
4089 LibFunc Func;
4090 Function *Callee = CI->getCalledFunction();
4091 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4092
4094 CI->getOperandBundlesAsDefs(OpBundles);
4095
4097 Builder.setDefaultOperandBundles(OpBundles);
4098
4099 // Command-line parameter overrides instruction attribute.
4100 // This can't be moved to optimizeFloatingPointLibCall() because it may be
4101 // used by the intrinsic optimizations.
4102 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
4103 UnsafeFPShrink = EnableUnsafeFPShrink;
4104 else if (isa<FPMathOperator>(CI) && CI->isFast())
4105 UnsafeFPShrink = true;
4106
4107 // First, check for intrinsics.
4108 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) {
4109 if (!IsCallingConvC)
4110 return nullptr;
4111 // The FP intrinsics have corresponding constrained versions so we don't
4112 // need to check for the StrictFP attribute here.
4113 switch (II->getIntrinsicID()) {
4114 case Intrinsic::pow:
4115 return optimizePow(CI, Builder);
4116 case Intrinsic::exp2:
4117 return optimizeExp2(CI, Builder);
4118 case Intrinsic::log:
4119 case Intrinsic::log2:
4120 case Intrinsic::log10:
4121 return optimizeLog(CI, Builder);
4122 case Intrinsic::sqrt:
4123 return optimizeSqrt(CI, Builder);
4124 case Intrinsic::memset:
4125 return optimizeMemSet(CI, Builder);
4126 case Intrinsic::memcpy:
4127 return optimizeMemCpy(CI, Builder);
4128 case Intrinsic::memmove:
4129 return optimizeMemMove(CI, Builder);
4130 default:
4131 return nullptr;
4132 }
4133 }
4134
4135 // Also try to simplify calls to fortified library functions.
4136 if (Value *SimplifiedFortifiedCI =
4137 FortifiedSimplifier.optimizeCall(CI, Builder))
4138 return SimplifiedFortifiedCI;
4139
4140 // Then check for known library functions.
4141 if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
4142 // We never change the calling convention.
4143 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4144 return nullptr;
4145 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
4146 return V;
4147 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
4148 return V;
4149 switch (Func) {
4150 case LibFunc_ffs:
4151 case LibFunc_ffsl:
4152 case LibFunc_ffsll:
4153 return optimizeFFS(CI, Builder);
4154 case LibFunc_fls:
4155 case LibFunc_flsl:
4156 case LibFunc_flsll:
4157 return optimizeFls(CI, Builder);
4158 case LibFunc_abs:
4159 case LibFunc_labs:
4160 case LibFunc_llabs:
4161 return optimizeAbs(CI, Builder);
4162 case LibFunc_isdigit:
4163 return optimizeIsDigit(CI, Builder);
4164 case LibFunc_isascii:
4165 return optimizeIsAscii(CI, Builder);
4166 case LibFunc_toascii:
4167 return optimizeToAscii(CI, Builder);
4168 case LibFunc_atoi:
4169 case LibFunc_atol:
4170 case LibFunc_atoll:
4171 return optimizeAtoi(CI, Builder);
4172 case LibFunc_strtol:
4173 case LibFunc_strtoll:
4174 return optimizeStrToInt(CI, Builder, /*AsSigned=*/true);
4175 case LibFunc_strtoul:
4176 case LibFunc_strtoull:
4177 return optimizeStrToInt(CI, Builder, /*AsSigned=*/false);
4178 case LibFunc_printf:
4179 return optimizePrintF(CI, Builder);
4180 case LibFunc_sprintf:
4181 return optimizeSPrintF(CI, Builder);
4182 case LibFunc_snprintf:
4183 return optimizeSnPrintF(CI, Builder);
4184 case LibFunc_fprintf:
4185 return optimizeFPrintF(CI, Builder);
4186 case LibFunc_fwrite:
4187 return optimizeFWrite(CI, Builder);
4188 case LibFunc_fputs:
4189 return optimizeFPuts(CI, Builder);
4190 case LibFunc_puts:
4191 return optimizePuts(CI, Builder);
4192 case LibFunc_perror:
4193 return optimizeErrorReporting(CI, Builder);
4194 case LibFunc_vfprintf:
4195 case LibFunc_fiprintf:
4196 return optimizeErrorReporting(CI, Builder, 0);
4197 case LibFunc_exit:
4198 case LibFunc_Exit:
4199 return optimizeExit(CI);
4200 default:
4201 return nullptr;
4202 }
4203 }
4204 return nullptr;
4205}
4206
4208 const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT,
4211 function_ref<void(Instruction *, Value *)> Replacer,
4212 function_ref<void(Instruction *)> Eraser)
4213 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), DT(DT), DC(DC), AC(AC),
4214 ORE(ORE), BFI(BFI), PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
4215
4216void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
4217 // Indirect through the replacer used in this instance.
4218 Replacer(I, With);
4219}
4220
4221void LibCallSimplifier::eraseFromParent(Instruction *I) {
4222 Eraser(I);
4223}
4224
4225// TODO:
4226// Additional cases that we need to add to this file:
4227//
4228// cbrt:
4229// * cbrt(expN(X)) -> expN(x/3)
4230// * cbrt(sqrt(x)) -> pow(x,1/6)
4231// * cbrt(cbrt(x)) -> pow(x,1/9)
4232//
4233// exp, expf, expl:
4234// * exp(log(x)) -> x
4235//
4236// log, logf, logl:
4237// * log(exp(x)) -> x
4238// * log(exp(y)) -> y*log(e)
4239// * log(exp10(y)) -> y*log(10)
4240// * log(sqrt(x)) -> 0.5*log(x)
4241//
4242// pow, powf, powl:
4243// * pow(sqrt(x),y) -> pow(x,y*0.5)
4244// * pow(pow(x,y),z)-> pow(x,y*z)
4245//
4246// signbit:
4247// * signbit(cnst) -> cnst'
4248// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
4249//
4250// sqrt, sqrtf, sqrtl:
4251// * sqrt(expN(x)) -> expN(x*0.5)
4252// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
4253// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
4254//
4255
4256//===----------------------------------------------------------------------===//
4257// Fortified Library Call Optimizations
4258//===----------------------------------------------------------------------===//
4259
4260bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
4261 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
4262 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
4263 // If this function takes a flag argument, the implementation may use it to
4264 // perform extra checks. Don't fold into the non-checking variant.
4265 if (FlagOp) {
4266 ConstantInt *Flag = dyn_cast<ConstantInt>(CI->getArgOperand(*FlagOp));
4267 if (!Flag || !Flag->isZero())
4268 return false;
4269 }
4270
4271 if (SizeOp && CI->getArgOperand(ObjSizeOp) == CI->getArgOperand(*SizeOp))
4272 return true;
4273
4274 if (ConstantInt *ObjSizeCI =
4275 dyn_cast<ConstantInt>(CI->getArgOperand(ObjSizeOp))) {
4276 if (ObjSizeCI->isMinusOne())
4277 return true;
4278 // If the object size wasn't -1 (unknown), bail out if we were asked to.
4279 if (OnlyLowerUnknownSize)
4280 return false;
4281 if (StrOp) {
4283 // If the length is 0 we don't know how long it is and so we can't
4284 // remove the check.
4285 if (Len)
4286 annotateDereferenceableBytes(CI, *StrOp, Len);
4287 else
4288 return false;
4289 return ObjSizeCI->getZExtValue() >= Len;
4290 }
4291
4292 if (SizeOp) {
4293 if (ConstantInt *SizeCI =
4294 dyn_cast<ConstantInt>(CI->getArgOperand(*SizeOp)))
4295 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
4296 }
4297 }
4298 return false;
4299}
4300
4301Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
4302 IRBuilderBase &B) {
4303 if (isFortifiedCallFoldable(CI, 3, 2)) {
4304 CallInst *NewCI =
4305 B.CreateMemCpy(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
4306 Align(1), CI->getArgOperand(2));
4307 mergeAttributesAndFlags(NewCI, *CI);
4308 return CI->getArgOperand(0);
4309 }
4310 return nullptr;
4311}
4312
4313Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
4314 IRBuilderBase &B) {
4315 if (isFortifiedCallFoldable(CI, 3, 2)) {
4316 CallInst *NewCI =
4317 B.CreateMemMove(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
4318 Align(1), CI->getArgOperand(2));
4319 mergeAttributesAndFlags(NewCI, *CI);
4320 return CI->getArgOperand(0);
4321 }
4322 return nullptr;
4323}
4324
4325Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
4326 IRBuilderBase &B) {
4327 if (isFortifiedCallFoldable(CI, 3, 2)) {
4328 Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
4329 CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val,
4330 CI->getArgOperand(2), Align(1));
4331 mergeAttributesAndFlags(NewCI, *CI);
4332 return CI->getArgOperand(0);
4333 }
4334 return nullptr;
4335}
4336
4337Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
4338 IRBuilderBase &B) {
4339 const DataLayout &DL = CI->getDataLayout();
4340 if (isFortifiedCallFoldable(CI, 3, 2))
4341 if (Value *Call = emitMemPCpy(CI->getArgOperand(0), CI->getArgOperand(1),
4342 CI->getArgOperand(2), B, DL, TLI)) {
4343 return mergeAttributesAndFlags(cast<CallInst>(Call), *CI);
4344 }
4345 return nullptr;
4346}
4347
4348Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
4350 LibFunc Func) {
4351 const DataLayout &DL = CI->getDataLayout();
4352 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1),
4353 *ObjSize = CI->getArgOperand(2);
4354
4355 // __stpcpy_chk(x,x,...) -> x+strlen(x)
4356 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
4357 Value *StrLen = emitStrLen(Src, B, DL, TLI);
4358 return StrLen ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, StrLen) : nullptr;
4359 }
4360
4361 // If a) we don't have any length information, or b) we know this will
4362 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
4363 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
4364 // TODO: It might be nice to get a maximum length out of the possible
4365 // string lengths for varying.
4366 if (isFortifiedCallFoldable(CI, 2, std::nullopt, 1)) {
4367 if (Func == LibFunc_strcpy_chk)
4368 return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
4369 else
4370 return copyFlags(*CI, emitStpCpy(Dst, Src, B, TLI));
4371 }
4372
4373 if (OnlyLowerUnknownSize)
4374 return nullptr;
4375
4376 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
4378 if (Len)
4379 annotateDereferenceableBytes(CI, 1, Len);
4380 else
4381 return nullptr;
4382
4383 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
4384 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
4385 Value *LenV = ConstantInt::get(SizeTTy, Len);
4386 Value *Ret = emitMemCpyChk(Dst, Src, LenV, ObjSize, B, DL, TLI);
4387 // If the function was an __stpcpy_chk, and we were able to fold it into
4388 // a __memcpy_chk, we still need to return the correct end pointer.
4389 if (Ret && Func == LibFunc_stpcpy_chk)
4390 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst,
4391 ConstantInt::get(SizeTTy, Len - 1));
4392 return copyFlags(*CI, cast<CallInst>(Ret));
4393}
4394
4395Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4396 IRBuilderBase &B) {
4397 if (isFortifiedCallFoldable(CI, 1, std::nullopt, 0))
4398 return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B,
4399 CI->getDataLayout(), TLI));
4400 return nullptr;
4401}
4402
4403Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4405 LibFunc Func) {
4406 if (isFortifiedCallFoldable(CI, 3, 2)) {
4407 if (Func == LibFunc_strncpy_chk)
4408 return copyFlags(*CI,
4410 CI->getArgOperand(2), B, TLI));
4411 else
4412 return copyFlags(*CI,
4414 CI->getArgOperand(2), B, TLI));
4415 }
4416
4417 return nullptr;
4418}
4419
4420Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4421 IRBuilderBase &B) {
4422 if (isFortifiedCallFoldable(CI, 4, 3))
4423 return copyFlags(
4424 *CI, emitMemCCpy(CI->getArgOperand(0), CI->getArgOperand(1),
4425 CI->getArgOperand(2), CI->getArgOperand(3), B, TLI));
4426
4427 return nullptr;
4428}
4429
4430Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4431 IRBuilderBase &B) {
4432 if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2)) {
4433 SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 5));
4434 return copyFlags(*CI,
4436 CI->getArgOperand(4), VariadicArgs, B, TLI));
4437 }
4438
4439 return nullptr;
4440}
4441
4442Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4443 IRBuilderBase &B) {
4444 if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1)) {
4445 SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 4));
4446 return copyFlags(*CI,
4448 VariadicArgs, B, TLI));
4449 }
4450
4451 return nullptr;
4452}
4453
4454Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4455 IRBuilderBase &B) {
4456 if (isFortifiedCallFoldable(CI, 2))
4457 return copyFlags(
4458 *CI, emitStrCat(CI->getArgOperand(0), CI->getArgOperand(1), B, TLI));
4459
4460 return nullptr;
4461}
4462
4463Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4464 IRBuilderBase &B) {
4465 if (isFortifiedCallFoldable(CI, 3))
4466 return copyFlags(*CI,
4468 CI->getArgOperand(2), B, TLI));
4469
4470 return nullptr;
4471}
4472
4473Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4474 IRBuilderBase &B) {
4475 if (isFortifiedCallFoldable(CI, 3))
4476 return copyFlags(*CI,
4478 CI->getArgOperand(2), B, TLI));
4479
4480 return nullptr;
4481}
4482
4483Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4484 IRBuilderBase &B) {
4485 if (isFortifiedCallFoldable(CI, 3))
4486 return copyFlags(*CI,
4488 CI->getArgOperand(2), B, TLI));
4489
4490 return nullptr;
4491}
4492
4493Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4494 IRBuilderBase &B) {
4495 if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2))
4496 return copyFlags(
4497 *CI, emitVSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
4498 CI->getArgOperand(4), CI->getArgOperand(5), B, TLI));
4499
4500 return nullptr;
4501}
4502
4503Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4504 IRBuilderBase &B) {
4505 if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1))
4506 return copyFlags(*CI,
4508 CI->getArgOperand(4), B, TLI));
4509
4510 return nullptr;
4511}
4512
4514 IRBuilderBase &Builder) {
4515 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4516 // Some clang users checked for _chk libcall availability using:
4517 // __has_builtin(__builtin___memcpy_chk)
4518 // When compiling with -fno-builtin, this is always true.
4519 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4520 // end up with fortified libcalls, which isn't acceptable in a freestanding
4521 // environment which only provides their non-fortified counterparts.
4522 //
4523 // Until we change clang and/or teach external users to check for availability
4524 // differently, disregard the "nobuiltin" attribute and TLI::has.
4525 //
4526 // PR23093.
4527
4528 LibFunc Func;
4529 Function *Callee = CI->getCalledFunction();
4530 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4531
4533 CI->getOperandBundlesAsDefs(OpBundles);
4534
4536 Builder.setDefaultOperandBundles(OpBundles);
4537
4538 // First, check that this is a known library functions and that the prototype
4539 // is correct.
4540 if (!TLI->getLibFunc(*Callee, Func))
4541 return nullptr;
4542
4543 // We never change the calling convention.
4544 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4545 return nullptr;
4546
4547 switch (Func) {
4548 case LibFunc_memcpy_chk:
4549 return optimizeMemCpyChk(CI, Builder);
4550 case LibFunc_mempcpy_chk:
4551 return optimizeMemPCpyChk(CI, Builder);
4552 case LibFunc_memmove_chk:
4553 return optimizeMemMoveChk(CI, Builder);
4554 case LibFunc_memset_chk:
4555 return optimizeMemSetChk(CI, Builder);
4556 case LibFunc_stpcpy_chk:
4557 case LibFunc_strcpy_chk:
4558 return optimizeStrpCpyChk(CI, Builder, Func);
4559 case LibFunc_strlen_chk:
4560 return optimizeStrLenChk(CI, Builder);
4561 case LibFunc_stpncpy_chk:
4562 case LibFunc_strncpy_chk:
4563 return optimizeStrpNCpyChk(CI, Builder, Func);
4564 case LibFunc_memccpy_chk:
4565 return optimizeMemCCpyChk(CI, Builder);
4566 case LibFunc_snprintf_chk:
4567 return optimizeSNPrintfChk(CI, Builder);
4568 case LibFunc_sprintf_chk:
4569 return optimizeSPrintfChk(CI, Builder);
4570 case LibFunc_strcat_chk:
4571 return optimizeStrCatChk(CI, Builder);
4572 case LibFunc_strlcat_chk:
4573 return optimizeStrLCat(CI, Builder);
4574 case LibFunc_strncat_chk:
4575 return optimizeStrNCatChk(CI, Builder);
4576 case LibFunc_strlcpy_chk:
4577 return optimizeStrLCpyChk(CI, Builder);
4578 case LibFunc_vsnprintf_chk:
4579 return optimizeVSNPrintfChk(CI, Builder);
4580 case LibFunc_vsprintf_chk:
4581 return optimizeVSPrintfChk(CI, Builder);
4582 default:
4583 break;
4584 }
4585 return nullptr;
4586}
4587
4589 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4590 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
static const LLT S1
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
return RetTy
std::string Name
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Hexagon Common GEP
Module.h This file contains the declarations for the Module class.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static bool isBinary(MachineInstr &MI)
const SmallVectorImpl< MachineOperand > & Cond
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isOnlyUsedInEqualityComparison(Value *V, Value *With)
Return true if it is only used in equality comparisons with With.
static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef< unsigned > ArgNos, Value *Size, const DataLayout &DL)
static cl::opt< unsigned, false, HotColdHintParser > ColdNewHintValue("cold-new-hint-value", cl::Hidden, cl::init(1), cl::desc("Value to pass to hot/cold operator new for cold allocation"))
static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg, bool UseFloat, Value *&Sin, Value *&Cos, Value *&SinCos, const TargetLibraryInfo *TLI)
static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len, const DataLayout &DL)
static Value * mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old)
static cl::opt< bool > OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false), cl::desc("Enable hot/cold operator new library calls"))
static Value * optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float for binary functions.
static bool ignoreCallingConv(LibFunc Func)
static cl::opt< bool > OptimizeExistingHotColdNew("optimize-existing-hot-cold-new", cl::Hidden, cl::init(false), cl::desc("Enable optimization of existing hot/cold operator new library calls"))
static void annotateDereferenceableBytes(CallInst *CI, ArrayRef< unsigned > ArgNos, uint64_t DereferenceableBytes)
static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg)
static Value * optimizeDoubleFP(CallInst *CI, IRBuilderBase &B, bool isBinary, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float functions.
static Value * optimizeSymmetricCall(CallInst *CI, bool IsEven, IRBuilderBase &B)
static Value * getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno, Module *M, IRBuilderBase &B, const TargetLibraryInfo *TLI)
static Value * valueHasFloatPrecision(Value *Val)
Return a variant of Val with float type.
static Value * optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS, uint64_t Len, IRBuilderBase &B, const DataLayout &DL)
static Value * createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M, IRBuilderBase &B)
static Value * convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr, uint64_t Base, bool AsSigned, IRBuilderBase &B)
static Value * memChrToCharCompare(CallInst *CI, Value *NBytes, IRBuilderBase &B, const DataLayout &DL)
static Value * copyFlags(const CallInst &Old, Value *New)
static StringRef substr(StringRef Str, uint64_t Len)
static cl::opt< unsigned, false, HotColdHintParser > HotNewHintValue("hot-new-hint-value", cl::Hidden, cl::init(254), cl::desc("Value to pass to hot/cold operator new for hot allocation"))
static bool isTrigLibCall(CallInst *CI)
static Value * optimizeNaN(CallInst *CI)
Constant folding nan/nanf/nanl.
static bool isOnlyUsedInComparisonWithZero(Value *V)
static Value * replaceUnaryCall(CallInst *CI, IRBuilderBase &B, Intrinsic::ID IID)
static bool callHasFloatingPointArgument(const CallInst *CI)
static Value * optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float for unary functions.
static bool callHasFP128Argument(const CallInst *CI)
static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI, ArrayRef< unsigned > ArgNos)
static Value * optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS, Value *Size, bool StrNCmp, IRBuilderBase &B, const DataLayout &DL)
static Value * getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth)
static cl::opt< bool > EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden, cl::init(false), cl::desc("Enable unsafe double to float " "shrinking for math lib calls"))
static cl::opt< unsigned, false, HotColdHintParser > NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(128), cl::desc("Value to pass to hot/cold operator new for " "notcold (warm) allocation"))
This file defines the SmallString class.
This file contains some functions that are useful when dealing with strings.
Value * RHS
Value * LHS
opStatus divide(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1205
bool isFiniteNonZero() const
Definition: APFloat.h:1450
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5465
opStatus subtract(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1187
bool isNegative() const
Definition: APFloat.h:1440
double convertToDouble() const
Converts this APFloat to host double value.
Definition: APFloat.cpp:5527
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: APFloat.h:1423
opStatus add(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1178
const fltSemantics & getSemantics() const
Definition: APFloat.h:1448
float convertToFloat() const
Converts this APFloat to host float value.
Definition: APFloat.cpp:5555
opStatus remainder(const APFloat &RHS)
Definition: APFloat.h:1214
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1321
bool isInteger() const
Definition: APFloat.h:1457
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition: APFloat.h:1076
Class for arbitrary precision integers.
Definition: APInt.h:78
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1150
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
A cache of @llvm.assume calls within a function.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:888
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
AttributeList addParamAttributes(LLVMContext &C, unsigned ArgNo, const AttrBuilder &B) const
Add an argument attribute to the list.
Definition: Attributes.h:645
MaybeAlign getAlignment() const
static Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:244
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:392
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:448
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Definition: InstrTypes.h:1482
void removeParamAttrs(unsigned ArgNo, const AttributeMask &AttrsToRemove)
Removes the attributes from the given argument.
Definition: InstrTypes.h:1561
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Definition: InstrTypes.h:1873
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Definition: InstrTypes.h:1549
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1349
bool doesNotAccessMemory(unsigned OpNo) const
Definition: InstrTypes.h:1718
void removeRetAttrs(const AttributeMask &AttrsToRemove)
Removes the attributes from the return value.
Definition: InstrTypes.h:1544
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
Definition: InstrTypes.h:1459
bool isStrictFP() const
Determine if the call requires strict floating point semantics.
Definition: InstrTypes.h:1879
AttributeSet getParamAttributes(unsigned ArgNo) const
Return the param attributes for this call.
Definition: InstrTypes.h:1436
uint64_t getParamDereferenceableBytes(unsigned i) const
Extract the number of dereferenceable bytes for a call or parameter (0=unknown).
Definition: InstrTypes.h:1817
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Definition: InstrTypes.h:1746
AttributeSet getRetAttributes() const
Return the return attributes for this call.
Definition: InstrTypes.h:1431
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition: InstrTypes.h:1428
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: InstrTypes.h:1923
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1294
uint64_t getParamDereferenceableOrNullBytes(unsigned i) const
Extract the number of dereferenceable_or_null bytes for a parameter (0=unknown).
Definition: InstrTypes.h:1835
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1285
unsigned arg_size() const
Definition: InstrTypes.h:1292
AttributeList getAttributes() const
Return the attributes for this call.
Definition: InstrTypes.h:1425
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1502
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
bool isNoTailCall() const
TailCallKind getTailCallKind() const
bool isMustTailCall() const
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:696
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:698
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:699
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:763
uint64_t getElementAsInteger(unsigned i) const
If this is a sequential container of integers (of any size), return the specified element in the low ...
Definition: Constants.cpp:3114
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:271
static Constant * getInfinity(Type *Ty, bool Negative=false)
Definition: Constants.cpp:1103
static Constant * getQNaN(Type *Ty, bool Negative=false, APInt *Payload=nullptr)
Definition: Constants.cpp:1035
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:214
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:208
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:163
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:157
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:373
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
bool fitsInLegalInteger(unsigned Width) const
Returns true if the specified type fits in a native integer type supported by the CPU.
Definition: DataLayout.h:312
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
This class represents an extension of floating point types.
This class represents a truncation of floating point types.
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
void setNoSignedZeros(bool B=true)
Definition: FMF.h:85
static FastMathFlags getFast()
Definition: FMF.h:51
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:791
FortifiedLibCallSimplifier(const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize=false)
Value * optimizeCall(CallInst *CI, IRBuilderBase &B)
Take the given call instruction and return a more optimal value to replace the instruction with or 0 ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:170
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:251
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:353
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:256
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:731
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:296
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
This instruction compares its operands according to the predicate given to the constructor.
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:108
void setDefaultOperandBundles(ArrayRef< OperandBundleDef > OpBundles)
Definition: IRBuilder.h:379
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
bool hasNoNaNs() const LLVM_READONLY
Determine whether the no-NaNs flag is set.
bool hasNoInfs() const LLVM_READONLY
Determine whether the no-infs flag is set.
bool hasNoSignedZeros() const LLVM_READONLY
Determine whether the no-signed-zeros flag is set.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:68
bool isFast() const LLVM_READONLY
Determine whether all fast-math-flags are set.
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:72
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
bool hasApproxFunc() const LLVM_READONLY
Determine whether the approximate-math-functions flag is set.
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition: Instruction.cpp:76
Class to represent integer types.
Definition: DerivedTypes.h:42
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:311
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:48
LibCallSimplifier(const DataLayout &DL, const TargetLibraryInfo *TLI, DominatorTree *DT, DomConditionCache *DC, AssumptionCache *AC, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, function_ref< void(Instruction *, Value *)> Replacer=&replaceAllUsesWithDefault, function_ref< void(Instruction *)> Eraser=&eraseFromParentDefault)
Value * optimizeCall(CallInst *CI, IRBuilderBase &B)
optimizeCall - Take the given call instruction and return a more optimal value to replace the instruc...
An instruction for reading from memory.
Definition: Instructions.h:176
Value * getPointerOperand()
Definition: Instructions.h:255
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition: Module.h:298
The optimization diagnostic interface.
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for applied optimization remarks.
Analysis providing profile information.
This class represents the LLVM 'select' instruction.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
bool empty() const
Definition: SmallVector.h:81
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:470
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:265
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:147
char back() const
back - Get the last character in the string.
Definition: StringRef.h:159
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:150
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition: StringRef.h:424
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition: StringRef.h:297
static constexpr size_t npos
Definition: StringRef.h:53
int compare(StringRef RHS) const
compare - Compare two strings; the result is negative, zero, or positive if this string is lexicograp...
Definition: StringRef.h:183
StringRef drop_back(size_t N=1) const
Return a StringRef equal to 'this' but with the last N elements dropped.
Definition: StringRef.h:616
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:44
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:69
R Default(T Value)
Definition: StringSwitch.h:182
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:406
static bool isCallingConvCCompatible(CallBase *CI)
Returns true if call site / callee has cdecl-compatible calling conventions.
Provides information about what library functions are available for the current target.
unsigned getWCharSize(const Module &M) const
Returns the size of the wchar_t type in bytes or 0 if the size is unknown.
ConstantInt * getAsSizeT(uint64_t V, const Module &M) const
Returns a constant materialized as a size_t type.
unsigned getSizeTSize(const Module &M) const
Returns the size of the size_t type in bits.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
StringRef getName(LibFunc F) const
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
const fltSemantics & getFltSemantics() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:270
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:264
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition: Type.h:153
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:258
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition: Type.h:156
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:237
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:355
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
op_range operands()
Definition: User.h:288
void setOperand(unsigned i, Value *Val)
Definition: User.h:233
Value * getOperand(unsigned i) const
Definition: User.h:228
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
bool use_empty() const
Definition: Value.h:344
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define UINT64_MAX
Definition: DataTypes.h:77
AttributeMask typeIncompatible(Type *Ty, AttributeSet AS, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
BinaryOp_match< LHS, RHS, Instruction::FMul > m_FMul(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
Definition: PatternMatch.h:764
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
specific_fpval m_SpecificFP(double V)
Match a specific floating point value or vector with all elements equal to the value.
Definition: PatternMatch.h:928
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:903
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
specific_fpval m_FPOne()
Match a float 1.0 or vector with all elements equal to 1.0.
Definition: PatternMatch.h:931
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
Definition: PatternMatch.h:299
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
apfloat_match m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
Definition: PatternMatch.h:316
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
m_Intrinsic_Ty< Opnd0, Opnd1 >::Ty m_CopySign(const Opnd0 &Op0, const Opnd1 &Op1)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
constexpr double e
Definition: MathExtras.h:47
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:480
@ Length
Definition: DWP.cpp:480
int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
Definition: MathExtras.h:244
Value * emitUnaryFloatFnCall(Value *Op, const TargetLibraryInfo *TLI, StringRef Name, IRBuilderBase &B, const AttributeList &Attrs)
Emit a call to the unary function named 'Name' (e.g.
Value * emitStrChr(Value *Ptr, char C, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strchr function to the builder, for the specified pointer and character.
Value * emitPutChar(Value *Char, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the putchar function. This assumes that Char is an 'int'.
Value * emitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the __memcpy_chk function to the builder.
Value * emitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strncpy function to the builder, for the specified pointer arguments and length.
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
Value * emitHotColdNewAlignedNoThrow(Value *Num, Value *Align, Value *NoThrow, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, uint8_t HotCold)
bool isKnownNeverInfinity(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not an infinity or if the floating-point vector val...
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1529
bool getConstantStringInfo(const Value *V, StringRef &Str, bool TrimAtNul=true)
This function computes the length of a null-terminated C string pointed to by V.
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
Definition: Loads.cpp:215
Value * emitSPrintf(Value *Dest, Value *Fmt, ArrayRef< Value * > VariadicArgs, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the sprintf function.
bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
Value * emitMemRChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the memrchr function, analogously to emitMemChr.
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition: APFloat.h:1599
Value * emitStrLCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strlcat function.
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:657
bool hasFloatFn(const Module *M, const TargetLibraryInfo *TLI, Type *Ty, LibFunc DoubleFn, LibFunc FloatFn, LibFunc LongDoubleFn)
Check whether the overloaded floating point function corresponding to Ty is available.
Value * emitStrNCat(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strncat function.
bool isLibFuncEmittable(const Module *M, const TargetLibraryInfo *TLI, LibFunc TheLibFunc)
Check whether the library function is available on target and also that it in the current Module is a...
Value * emitVSNPrintf(Value *Dest, Value *Size, Value *Fmt, Value *VAList, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the vsnprintf function.
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:242
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1746
Value * emitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the strncmp function to the builder.
Value * emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the memcmp function.
Value * emitBinaryFloatFnCall(Value *Op1, Value *Op2, const TargetLibraryInfo *TLI, StringRef Name, IRBuilderBase &B, const AttributeList &Attrs)
Emit a call to the binary function named 'Name' (e.g.
Value * emitFPutS(Value *Str, Value *File, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the fputs function.
Value * emitStrDup(Value *Ptr, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strdup function to the builder, for the specified pointer.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1664
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
Definition: Function.cpp:1187
Value * emitBCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the bcmp function.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition: MathExtras.h:684
uint64_t GetStringLength(const Value *V, unsigned CharSize=8)
If we can compute the length of the string pointed to by the specified pointer, return 'len+1'.
FunctionCallee getOrInsertLibFunc(Module *M, const TargetLibraryInfo &TLI, LibFunc TheLibFunc, FunctionType *T, AttributeList AttributeList)
Calls getOrInsertFunction() and then makes sure to add mandatory argument attributes.
Value * emitStrLen(Value *Ptr, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the strlen function to the builder, for the specified pointer.
Value * emitFPutC(Value *Char, Value *File, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the fputc function.
Value * emitStpNCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the stpncpy function to the builder, for the specified pointer arguments and length.
Value * emitStrCat(Value *Dest, Value *Src, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strcat function.
Value * emitVSPrintf(Value *Dest, Value *Fmt, Value *VAList, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the vsprintf function.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
Value * emitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the fwrite function.
Value * emitSNPrintf(Value *Dest, Value *Size, Value *Fmt, ArrayRef< Value * > Args, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the snprintf function.
@ Mod
The access may modify the value stored in memory.
Value * emitStpCpy(Value *Dst, Value *Src, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the stpcpy function to the builder, for the specified pointer arguments.
@ FMul
Product of floats.
@ And
Bitwise or logical AND of integers.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
DWARFExpression::Operation Op
@ NearestTiesToEven
roundTiesToEven.
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:217
Value * emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, uint8_t HotCold)
Value * emitMalloc(Value *Num, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the malloc function.
Value * emitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the memchr function.
Value * emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, uint8_t HotCold)
Value * emitPutS(Value *Str, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the puts function. This assumes that Str is some pointer.
Value * emitMemCCpy(Value *Ptr1, Value *Ptr2, Value *Val, Value *Len, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the memccpy function.
Value * emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, uint8_t HotCold)
Value * emitHotColdNew(Value *Num, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, uint8_t HotCold)
Emit a call to the hot/cold operator new function.
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
bool isGEPBasedOnPointerToString(const GEPOperator *GEP, unsigned CharSize=8)
Returns true if the GEP is based on a pointer to a string (array of.
Value * emitStrLCpy(Value *Dest, Value *Src, Value *Size, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strlcpy function.
Value * emitHotColdSizeReturningNewAligned(Value *Num, Value *Align, IRBuilderBase &B, const TargetLibraryInfo *TLI, LibFunc NewFunc, uint8_t HotCold)
Value * emitStrCpy(Value *Dst, Value *Src, IRBuilderBase &B, const TargetLibraryInfo *TLI)
Emit a call to the strcpy function to the builder, for the specified pointer arguments.
KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, unsigned Depth, const SimplifyQuery &SQ)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
Value * emitMemPCpy(Value *Dst, Value *Src, Value *Len, IRBuilderBase &B, const DataLayout &DL, const TargetLibraryInfo *TLI)
Emit a call to the mempcpy function.
uint64_t maxUIntN(uint64_t N)
Gets the maximum value for a N-bit unsigned integer.
Definition: MathExtras.h:219
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:382
#define N
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:265
static constexpr roundingMode rmTowardNegative
Definition: APFloat.h:300
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:297
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:301
opStatus
IEEE-754R 7: Default exception handling.
Definition: APFloat.h:313
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Holds functions to get, set or test bitfields.
Definition: Bitfields.h:212
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
const ConstantDataArray * Array
ConstantDataArray pointer.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:100
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:137
bool isKnownNeverInfinity() const
Return true if it's known this can never be an infinity.
static constexpr FPClassTest OrderedLessThanZeroMask
bool isKnownNeverLogicalZero(const Function &F, Type *Ty) const
Return true if it's know this can never be interpreted as a zero.
bool cannotBeOrderedLessThanZero() const
Return true if we can prove that the analyzed floating-point value is either NaN or never less than -...
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
Definition: regcomp.c:192