LLVM 18.0.0git
SimplifyLibCalls.cpp
Go to the documentation of this file.
1//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the library calls simplifier. It does not implement
10// any pass, but can't be used by other passes to do simplifications.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/APSInt.h"
19#include "llvm/Analysis/Loads.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/Module.h"
37
38#include <cmath>
39
40using namespace llvm;
41using namespace PatternMatch;
42
43static cl::opt<bool>
44 EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
45 cl::init(false),
46 cl::desc("Enable unsafe double to float "
47 "shrinking for math lib calls"));
48
49// Enable conversion of operator new calls with a MemProf hot or cold hint
50// to an operator new call that takes a hot/cold hint. Off by default since
51// not all allocators currently support this extension.
52static cl::opt<bool>
53 OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false),
54 cl::desc("Enable hot/cold operator new library calls"));
55
56namespace {
57
58// Specialized parser to ensure the hint is an 8 bit value (we can't specify
59// uint8_t to opt<> as that is interpreted to mean that we are passing a char
60// option with a specific set of values.
61struct HotColdHintParser : public cl::parser<unsigned> {
62 HotColdHintParser(cl::Option &O) : cl::parser<unsigned>(O) {}
63
64 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
65 if (Arg.getAsInteger(0, Value))
66 return O.error("'" + Arg + "' value invalid for uint argument!");
67
68 if (Value > 255)
69 return O.error("'" + Arg + "' value must be in the range [0, 255]!");
70
71 return false;
72 }
73};
74
75} // end anonymous namespace
76
77// Hot/cold operator new takes an 8 bit hotness hint, where 0 is the coldest
78// and 255 is the hottest. Default to 1 value away from the coldest and hottest
79// hints, so that the compiler hinted allocations are slightly less strong than
80// manually inserted hints at the two extremes.
82 "cold-new-hint-value", cl::Hidden, cl::init(1),
83 cl::desc("Value to pass to hot/cold operator new for cold allocation"));
85 "hot-new-hint-value", cl::Hidden, cl::init(254),
86 cl::desc("Value to pass to hot/cold operator new for hot allocation"));
87
88//===----------------------------------------------------------------------===//
89// Helper Functions
90//===----------------------------------------------------------------------===//
91
92static bool ignoreCallingConv(LibFunc Func) {
93 return Func == LibFunc_abs || Func == LibFunc_labs ||
94 Func == LibFunc_llabs || Func == LibFunc_strlen;
95}
96
97/// Return true if it is only used in equality comparisons with With.
99 for (User *U : V->users()) {
100 if (ICmpInst *IC = dyn_cast<ICmpInst>(U))
101 if (IC->isEquality() && IC->getOperand(1) == With)
102 continue;
103 // Unknown instruction.
104 return false;
105 }
106 return true;
107}
108
110 return any_of(CI->operands(), [](const Use &OI) {
111 return OI->getType()->isFloatingPointTy();
112 });
113}
114
115static bool callHasFP128Argument(const CallInst *CI) {
116 return any_of(CI->operands(), [](const Use &OI) {
117 return OI->getType()->isFP128Ty();
118 });
119}
120
121// Convert the entire string Str representing an integer in Base, up to
122// the terminating nul if present, to a constant according to the rules
123// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success
124// return the result, otherwise null.
125// The function assumes the string is encoded in ASCII and carefully
126// avoids converting sequences (including "") that the corresponding
127// library call might fail and set errno for.
128static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr,
129 uint64_t Base, bool AsSigned, IRBuilderBase &B) {
130 if (Base < 2 || Base > 36)
131 if (Base != 0)
132 // Fail for an invalid base (required by POSIX).
133 return nullptr;
134
135 // Current offset into the original string to reflect in EndPtr.
136 size_t Offset = 0;
137 // Strip leading whitespace.
138 for ( ; Offset != Str.size(); ++Offset)
139 if (!isSpace((unsigned char)Str[Offset])) {
140 Str = Str.substr(Offset);
141 break;
142 }
143
144 if (Str.empty())
145 // Fail for empty subject sequences (POSIX allows but doesn't require
146 // strtol[l]/strtoul[l] to fail with EINVAL).
147 return nullptr;
148
149 // Strip but remember the sign.
150 bool Negate = Str[0] == '-';
151 if (Str[0] == '-' || Str[0] == '+') {
152 Str = Str.drop_front();
153 if (Str.empty())
154 // Fail for a sign with nothing after it.
155 return nullptr;
156 ++Offset;
157 }
158
159 // Set Max to the absolute value of the minimum (for signed), or
160 // to the maximum (for unsigned) value representable in the type.
161 Type *RetTy = CI->getType();
162 unsigned NBits = RetTy->getPrimitiveSizeInBits();
163 uint64_t Max = AsSigned && Negate ? 1 : 0;
164 Max += AsSigned ? maxIntN(NBits) : maxUIntN(NBits);
165
166 // Autodetect Base if it's zero and consume the "0x" prefix.
167 if (Str.size() > 1) {
168 if (Str[0] == '0') {
169 if (toUpper((unsigned char)Str[1]) == 'X') {
170 if (Str.size() == 2 || (Base && Base != 16))
171 // Fail if Base doesn't allow the "0x" prefix or for the prefix
172 // alone that implementations like BSD set errno to EINVAL for.
173 return nullptr;
174
175 Str = Str.drop_front(2);
176 Offset += 2;
177 Base = 16;
178 }
179 else if (Base == 0)
180 Base = 8;
181 } else if (Base == 0)
182 Base = 10;
183 }
184 else if (Base == 0)
185 Base = 10;
186
187 // Convert the rest of the subject sequence, not including the sign,
188 // to its uint64_t representation (this assumes the source character
189 // set is ASCII).
190 uint64_t Result = 0;
191 for (unsigned i = 0; i != Str.size(); ++i) {
192 unsigned char DigVal = Str[i];
193 if (isDigit(DigVal))
194 DigVal = DigVal - '0';
195 else {
196 DigVal = toUpper(DigVal);
197 if (isAlpha(DigVal))
198 DigVal = DigVal - 'A' + 10;
199 else
200 return nullptr;
201 }
202
203 if (DigVal >= Base)
204 // Fail if the digit is not valid in the Base.
205 return nullptr;
206
207 // Add the digit and fail if the result is not representable in
208 // the (unsigned form of the) destination type.
209 bool VFlow;
210 Result = SaturatingMultiplyAdd(Result, Base, (uint64_t)DigVal, &VFlow);
211 if (VFlow || Result > Max)
212 return nullptr;
213 }
214
215 if (EndPtr) {
216 // Store the pointer to the end.
217 Value *Off = B.getInt64(Offset + Str.size());
218 Value *StrBeg = CI->getArgOperand(0);
219 Value *StrEnd = B.CreateInBoundsGEP(B.getInt8Ty(), StrBeg, Off, "endptr");
220 B.CreateStore(StrEnd, EndPtr);
221 }
222
223 if (Negate)
224 // Unsigned negation doesn't overflow.
225 Result = -Result;
226
227 return ConstantInt::get(RetTy, Result);
228}
229
231 for (User *U : V->users()) {
232 if (ICmpInst *IC = dyn_cast<ICmpInst>(U))
233 if (Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
234 if (C->isNullValue())
235 continue;
236 // Unknown instruction.
237 return false;
238 }
239 return true;
240}
241
242static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
243 const DataLayout &DL) {
245 return false;
246
247 if (!isDereferenceableAndAlignedPointer(Str, Align(1), APInt(64, Len), DL))
248 return false;
249
250 if (CI->getFunction()->hasFnAttribute(Attribute::SanitizeMemory))
251 return false;
252
253 return true;
254}
255
257 ArrayRef<unsigned> ArgNos,
258 uint64_t DereferenceableBytes) {
259 const Function *F = CI->getCaller();
260 if (!F)
261 return;
262 for (unsigned ArgNo : ArgNos) {
263 uint64_t DerefBytes = DereferenceableBytes;
264 unsigned AS = CI->getArgOperand(ArgNo)->getType()->getPointerAddressSpace();
265 if (!llvm::NullPointerIsDefined(F, AS) ||
266 CI->paramHasAttr(ArgNo, Attribute::NonNull))
267 DerefBytes = std::max(CI->getParamDereferenceableOrNullBytes(ArgNo),
268 DereferenceableBytes);
269
270 if (CI->getParamDereferenceableBytes(ArgNo) < DerefBytes) {
271 CI->removeParamAttr(ArgNo, Attribute::Dereferenceable);
272 if (!llvm::NullPointerIsDefined(F, AS) ||
273 CI->paramHasAttr(ArgNo, Attribute::NonNull))
274 CI->removeParamAttr(ArgNo, Attribute::DereferenceableOrNull);
276 CI->getContext(), DerefBytes));
277 }
278 }
279}
280
282 ArrayRef<unsigned> ArgNos) {
283 Function *F = CI->getCaller();
284 if (!F)
285 return;
286
287 for (unsigned ArgNo : ArgNos) {
288 if (!CI->paramHasAttr(ArgNo, Attribute::NoUndef))
289 CI->addParamAttr(ArgNo, Attribute::NoUndef);
290
291 if (!CI->paramHasAttr(ArgNo, Attribute::NonNull)) {
292 unsigned AS =
295 continue;
296 CI->addParamAttr(ArgNo, Attribute::NonNull);
297 }
298
299 annotateDereferenceableBytes(CI, ArgNo, 1);
300 }
301}
302
304 Value *Size, const DataLayout &DL) {
305 if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size)) {
307 annotateDereferenceableBytes(CI, ArgNos, LenC->getZExtValue());
308 } else if (isKnownNonZero(Size, DL)) {
310 const APInt *X, *Y;
311 uint64_t DerefMin = 1;
312 if (match(Size, m_Select(m_Value(), m_APInt(X), m_APInt(Y)))) {
313 DerefMin = std::min(X->getZExtValue(), Y->getZExtValue());
314 annotateDereferenceableBytes(CI, ArgNos, DerefMin);
315 }
316 }
317}
318
319// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
320// easier chaining. Calls to emit* and B.createCall should probably be wrapped
321// in this function when New is created to replace Old. Callers should take
322// care to check Old.isMustTailCall() if they aren't replacing Old directly
323// with New.
324static Value *copyFlags(const CallInst &Old, Value *New) {
325 assert(!Old.isMustTailCall() && "do not copy musttail call flags");
326 assert(!Old.isNoTailCall() && "do not copy notail call flags");
327 if (auto *NewCI = dyn_cast_or_null<CallInst>(New))
328 NewCI->setTailCallKind(Old.getTailCallKind());
329 return New;
330}
331
332static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
334 NewCI->getContext(), {NewCI->getAttributes(), Old.getAttributes()}));
336 return copyFlags(Old, NewCI);
337}
338
339// Helper to avoid truncating the length if size_t is 32-bits.
341 return Len >= Str.size() ? Str : Str.substr(0, Len);
342}
343
344//===----------------------------------------------------------------------===//
345// String and Memory Library Call Optimizations
346//===----------------------------------------------------------------------===//
347
348Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
349 // Extract some information from the instruction
350 Value *Dst = CI->getArgOperand(0);
351 Value *Src = CI->getArgOperand(1);
353
354 // See if we can get the length of the input string.
356 if (Len)
358 else
359 return nullptr;
360 --Len; // Unbias length.
361
362 // Handle the simple, do-nothing case: strcat(x, "") -> x
363 if (Len == 0)
364 return Dst;
365
366 return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, Len, B));
367}
368
369Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
370 IRBuilderBase &B) {
371 // We need to find the end of the destination string. That's where the
372 // memory is to be moved to. We just generate a call to strlen.
373 Value *DstLen = emitStrLen(Dst, B, DL, TLI);
374 if (!DstLen)
375 return nullptr;
376
377 // Now that we have the destination's length, we must index into the
378 // destination's pointer to get the actual memcpy destination (end of
379 // the string .. we're concatenating).
380 Value *CpyDst = B.CreateInBoundsGEP(B.getInt8Ty(), Dst, DstLen, "endptr");
381
382 // We have enough information to now generate the memcpy call to do the
383 // concatenation for us. Make a memcpy to copy the nul byte with align = 1.
384 B.CreateMemCpy(
385 CpyDst, Align(1), Src, Align(1),
386 ConstantInt::get(DL.getIntPtrType(Src->getContext()), Len + 1));
387 return Dst;
388}
389
390Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
391 // Extract some information from the instruction.
392 Value *Dst = CI->getArgOperand(0);
393 Value *Src = CI->getArgOperand(1);
394 Value *Size = CI->getArgOperand(2);
397 if (isKnownNonZero(Size, DL))
399
400 // We don't do anything if length is not constant.
401 ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size);
402 if (LengthArg) {
403 Len = LengthArg->getZExtValue();
404 // strncat(x, c, 0) -> x
405 if (!Len)
406 return Dst;
407 } else {
408 return nullptr;
409 }
410
411 // See if we can get the length of the input string.
412 uint64_t SrcLen = GetStringLength(Src);
413 if (SrcLen) {
414 annotateDereferenceableBytes(CI, 1, SrcLen);
415 --SrcLen; // Unbias length.
416 } else {
417 return nullptr;
418 }
419
420 // strncat(x, "", c) -> x
421 if (SrcLen == 0)
422 return Dst;
423
424 // We don't optimize this case.
425 if (Len < SrcLen)
426 return nullptr;
427
428 // strncat(x, s, c) -> strcat(x, s)
429 // s is constant so the strcat can be optimized further.
430 return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, SrcLen, B));
431}
432
433// Helper to transform memchr(S, C, N) == S to N && *S == C and, when
434// NBytes is null, strchr(S, C) to *S == C. A precondition of the function
435// is that either S is dereferenceable or the value of N is nonzero.
437 IRBuilderBase &B, const DataLayout &DL)
438{
439 Value *Src = CI->getArgOperand(0);
440 Value *CharVal = CI->getArgOperand(1);
441
442 // Fold memchr(A, C, N) == A to N && *A == C.
443 Type *CharTy = B.getInt8Ty();
444 Value *Char0 = B.CreateLoad(CharTy, Src);
445 CharVal = B.CreateTrunc(CharVal, CharTy);
446 Value *Cmp = B.CreateICmpEQ(Char0, CharVal, "char0cmp");
447
448 if (NBytes) {
449 Value *Zero = ConstantInt::get(NBytes->getType(), 0);
450 Value *And = B.CreateICmpNE(NBytes, Zero);
451 Cmp = B.CreateLogicalAnd(And, Cmp);
452 }
453
454 Value *NullPtr = Constant::getNullValue(CI->getType());
455 return B.CreateSelect(Cmp, Src, NullPtr);
456}
457
458Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
459 Value *SrcStr = CI->getArgOperand(0);
460 Value *CharVal = CI->getArgOperand(1);
462
463 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
464 return memChrToCharCompare(CI, nullptr, B, DL);
465
466 // If the second operand is non-constant, see if we can compute the length
467 // of the input string and turn this into memchr.
468 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
469 if (!CharC) {
470 uint64_t Len = GetStringLength(SrcStr);
471 if (Len)
473 else
474 return nullptr;
475
477 FunctionType *FT = Callee->getFunctionType();
478 unsigned IntBits = TLI->getIntSize();
479 if (!FT->getParamType(1)->isIntegerTy(IntBits)) // memchr needs 'int'.
480 return nullptr;
481
482 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
483 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
484 return copyFlags(*CI,
485 emitMemChr(SrcStr, CharVal, // include nul.
486 ConstantInt::get(SizeTTy, Len), B,
487 DL, TLI));
488 }
489
490 if (CharC->isZero()) {
491 Value *NullPtr = Constant::getNullValue(CI->getType());
492 if (isOnlyUsedInEqualityComparison(CI, NullPtr))
493 // Pre-empt the transformation to strlen below and fold
494 // strchr(A, '\0') == null to false.
495 return B.CreateIntToPtr(B.getTrue(), CI->getType());
496 }
497
498 // Otherwise, the character is a constant, see if the first argument is
499 // a string literal. If so, we can constant fold.
500 StringRef Str;
501 if (!getConstantStringInfo(SrcStr, Str)) {
502 if (CharC->isZero()) // strchr(p, 0) -> p + strlen(p)
503 if (Value *StrLen = emitStrLen(SrcStr, B, DL, TLI))
504 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, StrLen, "strchr");
505 return nullptr;
506 }
507
508 // Compute the offset, make sure to handle the case when we're searching for
509 // zero (a weird way to spell strlen).
510 size_t I = (0xFF & CharC->getSExtValue()) == 0
511 ? Str.size()
512 : Str.find(CharC->getSExtValue());
513 if (I == StringRef::npos) // Didn't find the char. strchr returns null.
514 return Constant::getNullValue(CI->getType());
515
516 // strchr(s+n,c) -> gep(s+n+i,c)
517 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(I), "strchr");
518}
519
520Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
521 Value *SrcStr = CI->getArgOperand(0);
522 Value *CharVal = CI->getArgOperand(1);
523 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
525
526 StringRef Str;
527 if (!getConstantStringInfo(SrcStr, Str)) {
528 // strrchr(s, 0) -> strchr(s, 0)
529 if (CharC && CharC->isZero())
530 return copyFlags(*CI, emitStrChr(SrcStr, '\0', B, TLI));
531 return nullptr;
532 }
533
534 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
535 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
536
537 // Try to expand strrchr to the memrchr nonstandard extension if it's
538 // available, or simply fail otherwise.
539 uint64_t NBytes = Str.size() + 1; // Include the terminating nul.
540 Value *Size = ConstantInt::get(SizeTTy, NBytes);
541 return copyFlags(*CI, emitMemRChr(SrcStr, CharVal, Size, B, DL, TLI));
542}
543
544Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
545 Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
546 if (Str1P == Str2P) // strcmp(x,x) -> 0
547 return ConstantInt::get(CI->getType(), 0);
548
549 StringRef Str1, Str2;
550 bool HasStr1 = getConstantStringInfo(Str1P, Str1);
551 bool HasStr2 = getConstantStringInfo(Str2P, Str2);
552
553 // strcmp(x, y) -> cnst (if both x and y are constant strings)
554 if (HasStr1 && HasStr2)
555 return ConstantInt::get(CI->getType(),
556 std::clamp(Str1.compare(Str2), -1, 1));
557
558 if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
559 return B.CreateNeg(B.CreateZExt(
560 B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
561
562 if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
563 return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
564 CI->getType());
565
566 // strcmp(P, "x") -> memcmp(P, "x", 2)
567 uint64_t Len1 = GetStringLength(Str1P);
568 if (Len1)
569 annotateDereferenceableBytes(CI, 0, Len1);
570 uint64_t Len2 = GetStringLength(Str2P);
571 if (Len2)
572 annotateDereferenceableBytes(CI, 1, Len2);
573
574 if (Len1 && Len2) {
575 return copyFlags(
576 *CI, emitMemCmp(Str1P, Str2P,
578 std::min(Len1, Len2)),
579 B, DL, TLI));
580 }
581
582 // strcmp to memcmp
583 if (!HasStr1 && HasStr2) {
584 if (canTransformToMemCmp(CI, Str1P, Len2, DL))
585 return copyFlags(
586 *CI,
587 emitMemCmp(Str1P, Str2P,
589 B, DL, TLI));
590 } else if (HasStr1 && !HasStr2) {
591 if (canTransformToMemCmp(CI, Str2P, Len1, DL))
592 return copyFlags(
593 *CI,
594 emitMemCmp(Str1P, Str2P,
596 B, DL, TLI));
597 }
598
600 return nullptr;
601}
602
603// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
604// arrays LHS and RHS and nonconstant Size.
605static Value *optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS,
606 Value *Size, bool StrNCmp,
607 IRBuilderBase &B, const DataLayout &DL);
608
609Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
610 Value *Str1P = CI->getArgOperand(0);
611 Value *Str2P = CI->getArgOperand(1);
612 Value *Size = CI->getArgOperand(2);
613 if (Str1P == Str2P) // strncmp(x,x,n) -> 0
614 return ConstantInt::get(CI->getType(), 0);
615
616 if (isKnownNonZero(Size, DL))
618 // Get the length argument if it is constant.
620 if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size))
621 Length = LengthArg->getZExtValue();
622 else
623 return optimizeMemCmpVarSize(CI, Str1P, Str2P, Size, true, B, DL);
624
625 if (Length == 0) // strncmp(x,y,0) -> 0
626 return ConstantInt::get(CI->getType(), 0);
627
628 if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
629 return copyFlags(*CI, emitMemCmp(Str1P, Str2P, Size, B, DL, TLI));
630
631 StringRef Str1, Str2;
632 bool HasStr1 = getConstantStringInfo(Str1P, Str1);
633 bool HasStr2 = getConstantStringInfo(Str2P, Str2);
634
635 // strncmp(x, y) -> cnst (if both x and y are constant strings)
636 if (HasStr1 && HasStr2) {
637 // Avoid truncating the 64-bit Length to 32 bits in ILP32.
638 StringRef SubStr1 = substr(Str1, Length);
639 StringRef SubStr2 = substr(Str2, Length);
640 return ConstantInt::get(CI->getType(),
641 std::clamp(SubStr1.compare(SubStr2), -1, 1));
642 }
643
644 if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
645 return B.CreateNeg(B.CreateZExt(
646 B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
647
648 if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
649 return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
650 CI->getType());
651
652 uint64_t Len1 = GetStringLength(Str1P);
653 if (Len1)
654 annotateDereferenceableBytes(CI, 0, Len1);
655 uint64_t Len2 = GetStringLength(Str2P);
656 if (Len2)
657 annotateDereferenceableBytes(CI, 1, Len2);
658
659 // strncmp to memcmp
660 if (!HasStr1 && HasStr2) {
661 Len2 = std::min(Len2, Length);
662 if (canTransformToMemCmp(CI, Str1P, Len2, DL))
663 return copyFlags(
664 *CI,
665 emitMemCmp(Str1P, Str2P,
667 B, DL, TLI));
668 } else if (HasStr1 && !HasStr2) {
669 Len1 = std::min(Len1, Length);
670 if (canTransformToMemCmp(CI, Str2P, Len1, DL))
671 return copyFlags(
672 *CI,
673 emitMemCmp(Str1P, Str2P,
675 B, DL, TLI));
676 }
677
678 return nullptr;
679}
680
681Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
682 Value *Src = CI->getArgOperand(0);
683 ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
684 uint64_t SrcLen = GetStringLength(Src);
685 if (SrcLen && Size) {
686 annotateDereferenceableBytes(CI, 0, SrcLen);
687 if (SrcLen <= Size->getZExtValue() + 1)
688 return copyFlags(*CI, emitStrDup(Src, B, TLI));
689 }
690
691 return nullptr;
692}
693
694Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
695 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
696 if (Dst == Src) // strcpy(x,x) -> x
697 return Src;
698
700 // See if we can get the length of the input string.
702 if (Len)
704 else
705 return nullptr;
706
707 // We have enough information to now generate the memcpy call to do the
708 // copy for us. Make a memcpy to copy the nul byte with align = 1.
709 CallInst *NewCI =
710 B.CreateMemCpy(Dst, Align(1), Src, Align(1),
712 mergeAttributesAndFlags(NewCI, *CI);
713 return Dst;
714}
715
716Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
718 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
719
720 // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
721 if (CI->use_empty())
722 return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
723
724 if (Dst == Src) { // stpcpy(x,x) -> x+strlen(x)
725 Value *StrLen = emitStrLen(Src, B, DL, TLI);
726 return StrLen ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, StrLen) : nullptr;
727 }
728
729 // See if we can get the length of the input string.
731 if (Len)
733 else
734 return nullptr;
735
736 Type *PT = Callee->getFunctionType()->getParamType(0);
737 Value *LenV = ConstantInt::get(DL.getIntPtrType(PT), Len);
738 Value *DstEnd = B.CreateInBoundsGEP(
739 B.getInt8Ty(), Dst, ConstantInt::get(DL.getIntPtrType(PT), Len - 1));
740
741 // We have enough information to now generate the memcpy call to do the
742 // copy for us. Make a memcpy to copy the nul byte with align = 1.
743 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1), LenV);
744 mergeAttributesAndFlags(NewCI, *CI);
745 return DstEnd;
746}
747
748// Optimize a call to size_t strlcpy(char*, const char*, size_t).
749
750Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
751 Value *Size = CI->getArgOperand(2);
752 if (isKnownNonZero(Size, DL))
753 // Like snprintf, the function stores into the destination only when
754 // the size argument is nonzero.
756 // The function reads the source argument regardless of Size (it returns
757 // its length).
759
760 uint64_t NBytes;
761 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Size))
762 NBytes = SizeC->getZExtValue();
763 else
764 return nullptr;
765
766 Value *Dst = CI->getArgOperand(0);
767 Value *Src = CI->getArgOperand(1);
768 if (NBytes <= 1) {
769 if (NBytes == 1)
770 // For a call to strlcpy(D, S, 1) first store a nul in *D.
771 B.CreateStore(B.getInt8(0), Dst);
772
773 // Transform strlcpy(D, S, 0) to a call to strlen(S).
774 return copyFlags(*CI, emitStrLen(Src, B, DL, TLI));
775 }
776
777 // Try to determine the length of the source, substituting its size
778 // when it's not nul-terminated (as it's required to be) to avoid
779 // reading past its end.
780 StringRef Str;
781 if (!getConstantStringInfo(Src, Str, /*TrimAtNul=*/false))
782 return nullptr;
783
784 uint64_t SrcLen = Str.find('\0');
785 // Set if the terminating nul should be copied by the call to memcpy
786 // below.
787 bool NulTerm = SrcLen < NBytes;
788
789 if (NulTerm)
790 // Overwrite NBytes with the number of bytes to copy, including
791 // the terminating nul.
792 NBytes = SrcLen + 1;
793 else {
794 // Set the length of the source for the function to return to its
795 // size, and cap NBytes at the same.
796 SrcLen = std::min(SrcLen, uint64_t(Str.size()));
797 NBytes = std::min(NBytes - 1, SrcLen);
798 }
799
800 if (SrcLen == 0) {
801 // Transform strlcpy(D, "", N) to (*D = '\0, 0).
802 B.CreateStore(B.getInt8(0), Dst);
803 return ConstantInt::get(CI->getType(), 0);
804 }
805
807 Type *PT = Callee->getFunctionType()->getParamType(0);
808 // Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
809 // bound on strlen(S) + 1 and N, optionally followed by a nul store to
810 // D[N' - 1] if necessary.
811 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
812 ConstantInt::get(DL.getIntPtrType(PT), NBytes));
813 mergeAttributesAndFlags(NewCI, *CI);
814
815 if (!NulTerm) {
816 Value *EndOff = ConstantInt::get(CI->getType(), NBytes);
817 Value *EndPtr = B.CreateInBoundsGEP(B.getInt8Ty(), Dst, EndOff);
818 B.CreateStore(B.getInt8(0), EndPtr);
819 }
820
821 // Like snprintf, strlcpy returns the number of nonzero bytes that would
822 // have been copied if the bound had been sufficiently big (which in this
823 // case is strlen(Src)).
824 return ConstantInt::get(CI->getType(), SrcLen);
825}
826
827// Optimize a call CI to either stpncpy when RetEnd is true, or to strncpy
828// otherwise.
829Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
830 IRBuilderBase &B) {
832 Value *Dst = CI->getArgOperand(0);
833 Value *Src = CI->getArgOperand(1);
834 Value *Size = CI->getArgOperand(2);
835
836 if (isKnownNonZero(Size, DL)) {
837 // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
838 // only when N is nonzero.
841 }
842
843 // If the "bound" argument is known set N to it. Otherwise set it to
844 // UINT64_MAX and handle it later.
846 if (ConstantInt *SizeC = dyn_cast<ConstantInt>(Size))
847 N = SizeC->getZExtValue();
848
849 if (N == 0)
850 // Fold st{p,r}ncpy(D, S, 0) to D.
851 return Dst;
852
853 if (N == 1) {
854 Type *CharTy = B.getInt8Ty();
855 Value *CharVal = B.CreateLoad(CharTy, Src, "stxncpy.char0");
856 B.CreateStore(CharVal, Dst);
857 if (!RetEnd)
858 // Transform strncpy(D, S, 1) to return (*D = *S), D.
859 return Dst;
860
861 // Transform stpncpy(D, S, 1) to return (*D = *S) ? D + 1 : D.
862 Value *ZeroChar = ConstantInt::get(CharTy, 0);
863 Value *Cmp = B.CreateICmpEQ(CharVal, ZeroChar, "stpncpy.char0cmp");
864
865 Value *Off1 = B.getInt32(1);
866 Value *EndPtr = B.CreateInBoundsGEP(CharTy, Dst, Off1, "stpncpy.end");
867 return B.CreateSelect(Cmp, Dst, EndPtr, "stpncpy.sel");
868 }
869
870 // If the length of the input string is known set SrcLen to it.
871 uint64_t SrcLen = GetStringLength(Src);
872 if (SrcLen)
873 annotateDereferenceableBytes(CI, 1, SrcLen);
874 else
875 return nullptr;
876
877 --SrcLen; // Unbias length.
878
879 if (SrcLen == 0) {
880 // Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
881 Align MemSetAlign =
883 CallInst *NewCI = B.CreateMemSet(Dst, B.getInt8('\0'), Size, MemSetAlign);
884 AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(0));
886 CI->getContext(), 0, ArgAttrs));
887 copyFlags(*CI, NewCI);
888 return Dst;
889 }
890
891 if (N > SrcLen + 1) {
892 if (N > 128)
893 // Bail if N is large or unknown.
894 return nullptr;
895
896 // st{p,r}ncpy(D, "a", N) -> memcpy(D, "a\0\0\0", N) for N <= 128.
897 StringRef Str;
898 if (!getConstantStringInfo(Src, Str))
899 return nullptr;
900 std::string SrcStr = Str.str();
901 // Create a bigger, nul-padded array with the same length, SrcLen,
902 // as the original string.
903 SrcStr.resize(N, '\0');
904 Src = B.CreateGlobalString(SrcStr, "str");
905 }
906
907 Type *PT = Callee->getFunctionType()->getParamType(0);
908 // st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
909 // S and N are constant.
910 CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
912 mergeAttributesAndFlags(NewCI, *CI);
913 if (!RetEnd)
914 return Dst;
915
916 // stpncpy(D, S, N) returns the address of the first null in D if it writes
917 // one, otherwise D + N.
918 Value *Off = B.getInt64(std::min(SrcLen, N));
919 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, Off, "endptr");
920}
921
922Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
923 unsigned CharSize,
924 Value *Bound) {
925 Value *Src = CI->getArgOperand(0);
926 Type *CharTy = B.getIntNTy(CharSize);
927
929 (!Bound || isKnownNonZero(Bound, DL))) {
930 // Fold strlen:
931 // strlen(x) != 0 --> *x != 0
932 // strlen(x) == 0 --> *x == 0
933 // and likewise strnlen with constant N > 0:
934 // strnlen(x, N) != 0 --> *x != 0
935 // strnlen(x, N) == 0 --> *x == 0
936 return B.CreateZExt(B.CreateLoad(CharTy, Src, "char0"),
937 CI->getType());
938 }
939
940 if (Bound) {
941 if (ConstantInt *BoundCst = dyn_cast<ConstantInt>(Bound)) {
942 if (BoundCst->isZero())
943 // Fold strnlen(s, 0) -> 0 for any s, constant or otherwise.
944 return ConstantInt::get(CI->getType(), 0);
945
946 if (BoundCst->isOne()) {
947 // Fold strnlen(s, 1) -> *s ? 1 : 0 for any s.
948 Value *CharVal = B.CreateLoad(CharTy, Src, "strnlen.char0");
949 Value *ZeroChar = ConstantInt::get(CharTy, 0);
950 Value *Cmp = B.CreateICmpNE(CharVal, ZeroChar, "strnlen.char0cmp");
951 return B.CreateZExt(Cmp, CI->getType());
952 }
953 }
954 }
955
956 if (uint64_t Len = GetStringLength(Src, CharSize)) {
957 Value *LenC = ConstantInt::get(CI->getType(), Len - 1);
958 // Fold strlen("xyz") -> 3 and strnlen("xyz", 2) -> 2
959 // and strnlen("xyz", Bound) -> min(3, Bound) for nonconstant Bound.
960 if (Bound)
961 return B.CreateBinaryIntrinsic(Intrinsic::umin, LenC, Bound);
962 return LenC;
963 }
964
965 if (Bound)
966 // Punt for strnlen for now.
967 return nullptr;
968
969 // If s is a constant pointer pointing to a string literal, we can fold
970 // strlen(s + x) to strlen(s) - x, when x is known to be in the range
971 // [0, strlen(s)] or the string has a single null terminator '\0' at the end.
972 // We only try to simplify strlen when the pointer s points to an array
973 // of CharSize elements. Otherwise, we would need to scale the offset x before
974 // doing the subtraction. This will make the optimization more complex, and
975 // it's not very useful because calling strlen for a pointer of other types is
976 // very uncommon.
977 if (GEPOperator *GEP = dyn_cast<GEPOperator>(Src)) {
978 // TODO: Handle subobjects.
979 if (!isGEPBasedOnPointerToString(GEP, CharSize))
980 return nullptr;
981
983 if (getConstantDataArrayInfo(GEP->getOperand(0), Slice, CharSize)) {
984 uint64_t NullTermIdx;
985 if (Slice.Array == nullptr) {
986 NullTermIdx = 0;
987 } else {
988 NullTermIdx = ~((uint64_t)0);
989 for (uint64_t I = 0, E = Slice.Length; I < E; ++I) {
990 if (Slice.Array->getElementAsInteger(I + Slice.Offset) == 0) {
991 NullTermIdx = I;
992 break;
993 }
994 }
995 // If the string does not have '\0', leave it to strlen to compute
996 // its length.
997 if (NullTermIdx == ~((uint64_t)0))
998 return nullptr;
999 }
1000
1001 Value *Offset = GEP->getOperand(2);
1002 KnownBits Known = computeKnownBits(Offset, DL, 0, nullptr, CI, nullptr);
1003 uint64_t ArrSize =
1004 cast<ArrayType>(GEP->getSourceElementType())->getNumElements();
1005
1006 // If Offset is not provably in the range [0, NullTermIdx], we can still
1007 // optimize if we can prove that the program has undefined behavior when
1008 // Offset is outside that range. That is the case when GEP->getOperand(0)
1009 // is a pointer to an object whose memory extent is NullTermIdx+1.
1010 if ((Known.isNonNegative() && Known.getMaxValue().ule(NullTermIdx)) ||
1011 (isa<GlobalVariable>(GEP->getOperand(0)) &&
1012 NullTermIdx == ArrSize - 1)) {
1013 Offset = B.CreateSExtOrTrunc(Offset, CI->getType());
1014 return B.CreateSub(ConstantInt::get(CI->getType(), NullTermIdx),
1015 Offset);
1016 }
1017 }
1018 }
1019
1020 // strlen(x?"foo":"bars") --> x ? 3 : 4
1021 if (SelectInst *SI = dyn_cast<SelectInst>(Src)) {
1022 uint64_t LenTrue = GetStringLength(SI->getTrueValue(), CharSize);
1023 uint64_t LenFalse = GetStringLength(SI->getFalseValue(), CharSize);
1024 if (LenTrue && LenFalse) {
1025 ORE.emit([&]() {
1026 return OptimizationRemark("instcombine", "simplify-libcalls", CI)
1027 << "folded strlen(select) to select of constants";
1028 });
1029 return B.CreateSelect(SI->getCondition(),
1030 ConstantInt::get(CI->getType(), LenTrue - 1),
1031 ConstantInt::get(CI->getType(), LenFalse - 1));
1032 }
1033 }
1034
1035 return nullptr;
1036}
1037
1038Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilderBase &B) {
1039 if (Value *V = optimizeStringLength(CI, B, 8))
1040 return V;
1042 return nullptr;
1043}
1044
1045Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
1046 Value *Bound = CI->getArgOperand(1);
1047 if (Value *V = optimizeStringLength(CI, B, 8, Bound))
1048 return V;
1049
1050 if (isKnownNonZero(Bound, DL))
1052 return nullptr;
1053}
1054
1055Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilderBase &B) {
1056 Module &M = *CI->getModule();
1057 unsigned WCharSize = TLI->getWCharSize(M) * 8;
1058 // We cannot perform this optimization without wchar_size metadata.
1059 if (WCharSize == 0)
1060 return nullptr;
1061
1062 return optimizeStringLength(CI, B, WCharSize);
1063}
1064
1065Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
1066 StringRef S1, S2;
1067 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1068 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1069
1070 // strpbrk(s, "") -> nullptr
1071 // strpbrk("", s) -> nullptr
1072 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1073 return Constant::getNullValue(CI->getType());
1074
1075 // Constant folding.
1076 if (HasS1 && HasS2) {
1077 size_t I = S1.find_first_of(S2);
1078 if (I == StringRef::npos) // No match.
1079 return Constant::getNullValue(CI->getType());
1080
1081 return B.CreateInBoundsGEP(B.getInt8Ty(), CI->getArgOperand(0),
1082 B.getInt64(I), "strpbrk");
1083 }
1084
1085 // strpbrk(s, "a") -> strchr(s, 'a')
1086 if (HasS2 && S2.size() == 1)
1087 return copyFlags(*CI, emitStrChr(CI->getArgOperand(0), S2[0], B, TLI));
1088
1089 return nullptr;
1090}
1091
1092Value *LibCallSimplifier::optimizeStrTo(CallInst *CI, IRBuilderBase &B) {
1093 Value *EndPtr = CI->getArgOperand(1);
1094 if (isa<ConstantPointerNull>(EndPtr)) {
1095 // With a null EndPtr, this function won't capture the main argument.
1096 // It would be readonly too, except that it still may write to errno.
1097 CI->addParamAttr(0, Attribute::NoCapture);
1098 }
1099
1100 return nullptr;
1101}
1102
1103Value *LibCallSimplifier::optimizeStrSpn(CallInst *CI, IRBuilderBase &B) {
1104 StringRef S1, S2;
1105 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1106 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1107
1108 // strspn(s, "") -> 0
1109 // strspn("", s) -> 0
1110 if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
1111 return Constant::getNullValue(CI->getType());
1112
1113 // Constant folding.
1114 if (HasS1 && HasS2) {
1115 size_t Pos = S1.find_first_not_of(S2);
1116 if (Pos == StringRef::npos)
1117 Pos = S1.size();
1118 return ConstantInt::get(CI->getType(), Pos);
1119 }
1120
1121 return nullptr;
1122}
1123
1124Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
1125 StringRef S1, S2;
1126 bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
1127 bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
1128
1129 // strcspn("", s) -> 0
1130 if (HasS1 && S1.empty())
1131 return Constant::getNullValue(CI->getType());
1132
1133 // Constant folding.
1134 if (HasS1 && HasS2) {
1135 size_t Pos = S1.find_first_of(S2);
1136 if (Pos == StringRef::npos)
1137 Pos = S1.size();
1138 return ConstantInt::get(CI->getType(), Pos);
1139 }
1140
1141 // strcspn(s, "") -> strlen(s)
1142 if (HasS2 && S2.empty())
1143 return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B, DL, TLI));
1144
1145 return nullptr;
1146}
1147
1148Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilderBase &B) {
1149 // fold strstr(x, x) -> x.
1150 if (CI->getArgOperand(0) == CI->getArgOperand(1))
1151 return B.CreateBitCast(CI->getArgOperand(0), CI->getType());
1152
1153 // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
1155 Value *StrLen = emitStrLen(CI->getArgOperand(1), B, DL, TLI);
1156 if (!StrLen)
1157 return nullptr;
1158 Value *StrNCmp = emitStrNCmp(CI->getArgOperand(0), CI->getArgOperand(1),
1159 StrLen, B, DL, TLI);
1160 if (!StrNCmp)
1161 return nullptr;
1162 for (User *U : llvm::make_early_inc_range(CI->users())) {
1163 ICmpInst *Old = cast<ICmpInst>(U);
1164 Value *Cmp =
1165 B.CreateICmp(Old->getPredicate(), StrNCmp,
1166 ConstantInt::getNullValue(StrNCmp->getType()), "cmp");
1167 replaceAllUsesWith(Old, Cmp);
1168 }
1169 return CI;
1170 }
1171
1172 // See if either input string is a constant string.
1173 StringRef SearchStr, ToFindStr;
1174 bool HasStr1 = getConstantStringInfo(CI->getArgOperand(0), SearchStr);
1175 bool HasStr2 = getConstantStringInfo(CI->getArgOperand(1), ToFindStr);
1176
1177 // fold strstr(x, "") -> x.
1178 if (HasStr2 && ToFindStr.empty())
1179 return B.CreateBitCast(CI->getArgOperand(0), CI->getType());
1180
1181 // If both strings are known, constant fold it.
1182 if (HasStr1 && HasStr2) {
1183 size_t Offset = SearchStr.find(ToFindStr);
1184
1185 if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
1186 return Constant::getNullValue(CI->getType());
1187
1188 // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
1189 Value *Result = CI->getArgOperand(0);
1190 Result =
1191 B.CreateConstInBoundsGEP1_64(B.getInt8Ty(), Result, Offset, "strstr");
1192 return B.CreateBitCast(Result, CI->getType());
1193 }
1194
1195 // fold strstr(x, "y") -> strchr(x, 'y').
1196 if (HasStr2 && ToFindStr.size() == 1) {
1197 Value *StrChr = emitStrChr(CI->getArgOperand(0), ToFindStr[0], B, TLI);
1198 return StrChr ? B.CreateBitCast(StrChr, CI->getType()) : nullptr;
1199 }
1200
1202 return nullptr;
1203}
1204
1205Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilderBase &B) {
1206 Value *SrcStr = CI->getArgOperand(0);
1207 Value *Size = CI->getArgOperand(2);
1209 Value *CharVal = CI->getArgOperand(1);
1210 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1211 Value *NullPtr = Constant::getNullValue(CI->getType());
1212
1213 if (LenC) {
1214 if (LenC->isZero())
1215 // Fold memrchr(x, y, 0) --> null.
1216 return NullPtr;
1217
1218 if (LenC->isOne()) {
1219 // Fold memrchr(x, y, 1) --> *x == y ? x : null for any x and y,
1220 // constant or otherwise.
1221 Value *Val = B.CreateLoad(B.getInt8Ty(), SrcStr, "memrchr.char0");
1222 // Slice off the character's high end bits.
1223 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1224 Value *Cmp = B.CreateICmpEQ(Val, CharVal, "memrchr.char0cmp");
1225 return B.CreateSelect(Cmp, SrcStr, NullPtr, "memrchr.sel");
1226 }
1227 }
1228
1229 StringRef Str;
1230 if (!getConstantStringInfo(SrcStr, Str, /*TrimAtNul=*/false))
1231 return nullptr;
1232
1233 if (Str.size() == 0)
1234 // If the array is empty fold memrchr(A, C, N) to null for any value
1235 // of C and N on the basis that the only valid value of N is zero
1236 // (otherwise the call is undefined).
1237 return NullPtr;
1238
1239 uint64_t EndOff = UINT64_MAX;
1240 if (LenC) {
1241 EndOff = LenC->getZExtValue();
1242 if (Str.size() < EndOff)
1243 // Punt out-of-bounds accesses to sanitizers and/or libc.
1244 return nullptr;
1245 }
1246
1247 if (ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal)) {
1248 // Fold memrchr(S, C, N) for a constant C.
1249 size_t Pos = Str.rfind(CharC->getZExtValue(), EndOff);
1250 if (Pos == StringRef::npos)
1251 // When the character is not in the source array fold the result
1252 // to null regardless of Size.
1253 return NullPtr;
1254
1255 if (LenC)
1256 // Fold memrchr(s, c, N) --> s + Pos for constant N > Pos.
1257 return B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(Pos));
1258
1259 if (Str.find(Str[Pos]) == Pos) {
1260 // When there is just a single occurrence of C in S, i.e., the one
1261 // in Str[Pos], fold
1262 // memrchr(s, c, N) --> N <= Pos ? null : s + Pos
1263 // for nonconstant N.
1264 Value *Cmp = B.CreateICmpULE(Size, ConstantInt::get(Size->getType(), Pos),
1265 "memrchr.cmp");
1266 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr,
1267 B.getInt64(Pos), "memrchr.ptr_plus");
1268 return B.CreateSelect(Cmp, NullPtr, SrcPlus, "memrchr.sel");
1269 }
1270 }
1271
1272 // Truncate the string to search at most EndOff characters.
1273 Str = Str.substr(0, EndOff);
1274 if (Str.find_first_not_of(Str[0]) != StringRef::npos)
1275 return nullptr;
1276
1277 // If the source array consists of all equal characters, then for any
1278 // C and N (whether in bounds or not), fold memrchr(S, C, N) to
1279 // N != 0 && *S == C ? S + N - 1 : null
1280 Type *SizeTy = Size->getType();
1281 Type *Int8Ty = B.getInt8Ty();
1282 Value *NNeZ = B.CreateICmpNE(Size, ConstantInt::get(SizeTy, 0));
1283 // Slice off the sought character's high end bits.
1284 CharVal = B.CreateTrunc(CharVal, Int8Ty);
1285 Value *CEqS0 = B.CreateICmpEQ(ConstantInt::get(Int8Ty, Str[0]), CharVal);
1286 Value *And = B.CreateLogicalAnd(NNeZ, CEqS0);
1287 Value *SizeM1 = B.CreateSub(Size, ConstantInt::get(SizeTy, 1));
1288 Value *SrcPlus =
1289 B.CreateInBoundsGEP(Int8Ty, SrcStr, SizeM1, "memrchr.ptr_plus");
1290 return B.CreateSelect(And, SrcPlus, NullPtr, "memrchr.sel");
1291}
1292
1293Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
1294 Value *SrcStr = CI->getArgOperand(0);
1295 Value *Size = CI->getArgOperand(2);
1296
1297 if (isKnownNonZero(Size, DL)) {
1299 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
1300 return memChrToCharCompare(CI, Size, B, DL);
1301 }
1302
1303 Value *CharVal = CI->getArgOperand(1);
1304 ConstantInt *CharC = dyn_cast<ConstantInt>(CharVal);
1305 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1306 Value *NullPtr = Constant::getNullValue(CI->getType());
1307
1308 // memchr(x, y, 0) -> null
1309 if (LenC) {
1310 if (LenC->isZero())
1311 return NullPtr;
1312
1313 if (LenC->isOne()) {
1314 // Fold memchr(x, y, 1) --> *x == y ? x : null for any x and y,
1315 // constant or otherwise.
1316 Value *Val = B.CreateLoad(B.getInt8Ty(), SrcStr, "memchr.char0");
1317 // Slice off the character's high end bits.
1318 CharVal = B.CreateTrunc(CharVal, B.getInt8Ty());
1319 Value *Cmp = B.CreateICmpEQ(Val, CharVal, "memchr.char0cmp");
1320 return B.CreateSelect(Cmp, SrcStr, NullPtr, "memchr.sel");
1321 }
1322 }
1323
1324 StringRef Str;
1325 if (!getConstantStringInfo(SrcStr, Str, /*TrimAtNul=*/false))
1326 return nullptr;
1327
1328 if (CharC) {
1329 size_t Pos = Str.find(CharC->getZExtValue());
1330 if (Pos == StringRef::npos)
1331 // When the character is not in the source array fold the result
1332 // to null regardless of Size.
1333 return NullPtr;
1334
1335 // Fold memchr(s, c, n) -> n <= Pos ? null : s + Pos
1336 // When the constant Size is less than or equal to the character
1337 // position also fold the result to null.
1338 Value *Cmp = B.CreateICmpULE(Size, ConstantInt::get(Size->getType(), Pos),
1339 "memchr.cmp");
1340 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, B.getInt64(Pos),
1341 "memchr.ptr");
1342 return B.CreateSelect(Cmp, NullPtr, SrcPlus);
1343 }
1344
1345 if (Str.size() == 0)
1346 // If the array is empty fold memchr(A, C, N) to null for any value
1347 // of C and N on the basis that the only valid value of N is zero
1348 // (otherwise the call is undefined).
1349 return NullPtr;
1350
1351 if (LenC)
1352 Str = substr(Str, LenC->getZExtValue());
1353
1354 size_t Pos = Str.find_first_not_of(Str[0]);
1355 if (Pos == StringRef::npos
1356 || Str.find_first_not_of(Str[Pos], Pos) == StringRef::npos) {
1357 // If the source array consists of at most two consecutive sequences
1358 // of the same characters, then for any C and N (whether in bounds or
1359 // not), fold memchr(S, C, N) to
1360 // N != 0 && *S == C ? S : null
1361 // or for the two sequences to:
1362 // N != 0 && *S == C ? S : (N > Pos && S[Pos] == C ? S + Pos : null)
1363 // ^Sel2 ^Sel1 are denoted above.
1364 // The latter makes it also possible to fold strchr() calls with strings
1365 // of the same characters.
1366 Type *SizeTy = Size->getType();
1367 Type *Int8Ty = B.getInt8Ty();
1368
1369 // Slice off the sought character's high end bits.
1370 CharVal = B.CreateTrunc(CharVal, Int8Ty);
1371
1372 Value *Sel1 = NullPtr;
1373 if (Pos != StringRef::npos) {
1374 // Handle two consecutive sequences of the same characters.
1375 Value *PosVal = ConstantInt::get(SizeTy, Pos);
1376 Value *StrPos = ConstantInt::get(Int8Ty, Str[Pos]);
1377 Value *CEqSPos = B.CreateICmpEQ(CharVal, StrPos);
1378 Value *NGtPos = B.CreateICmp(ICmpInst::ICMP_UGT, Size, PosVal);
1379 Value *And = B.CreateAnd(CEqSPos, NGtPos);
1380 Value *SrcPlus = B.CreateInBoundsGEP(B.getInt8Ty(), SrcStr, PosVal);
1381 Sel1 = B.CreateSelect(And, SrcPlus, NullPtr, "memchr.sel1");
1382 }
1383
1384 Value *Str0 = ConstantInt::get(Int8Ty, Str[0]);
1385 Value *CEqS0 = B.CreateICmpEQ(Str0, CharVal);
1386 Value *NNeZ = B.CreateICmpNE(Size, ConstantInt::get(SizeTy, 0));
1387 Value *And = B.CreateAnd(NNeZ, CEqS0);
1388 return B.CreateSelect(And, SrcStr, Sel1, "memchr.sel2");
1389 }
1390
1391 if (!LenC) {
1392 if (isOnlyUsedInEqualityComparison(CI, SrcStr))
1393 // S is dereferenceable so it's safe to load from it and fold
1394 // memchr(S, C, N) == S to N && *S == C for any C and N.
1395 // TODO: This is safe even for nonconstant S.
1396 return memChrToCharCompare(CI, Size, B, DL);
1397
1398 // From now on we need a constant length and constant array.
1399 return nullptr;
1400 }
1401
1402 bool OptForSize = CI->getFunction()->hasOptSize() ||
1403 llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
1405
1406 // If the char is variable but the input str and length are not we can turn
1407 // this memchr call into a simple bit field test. Of course this only works
1408 // when the return value is only checked against null.
1409 //
1410 // It would be really nice to reuse switch lowering here but we can't change
1411 // the CFG at this point.
1412 //
1413 // memchr("\r\n", C, 2) != nullptr -> (1 << C & ((1 << '\r') | (1 << '\n')))
1414 // != 0
1415 // after bounds check.
1416 if (OptForSize || Str.empty() || !isOnlyUsedInZeroEqualityComparison(CI))
1417 return nullptr;
1418
1419 unsigned char Max =
1420 *std::max_element(reinterpret_cast<const unsigned char *>(Str.begin()),
1421 reinterpret_cast<const unsigned char *>(Str.end()));
1422
1423 // Make sure the bit field we're about to create fits in a register on the
1424 // target.
1425 // FIXME: On a 64 bit architecture this prevents us from using the
1426 // interesting range of alpha ascii chars. We could do better by emitting
1427 // two bitfields or shifting the range by 64 if no lower chars are used.
1428 if (!DL.fitsInLegalInteger(Max + 1)) {
1429 // Build chain of ORs
1430 // Transform:
1431 // memchr("abcd", C, 4) != nullptr
1432 // to:
1433 // (C == 'a' || C == 'b' || C == 'c' || C == 'd') != 0
1434 std::string SortedStr = Str.str();
1435 llvm::sort(SortedStr);
1436 // Compute the number of of non-contiguous ranges.
1437 unsigned NonContRanges = 1;
1438 for (size_t i = 1; i < SortedStr.size(); ++i) {
1439 if (SortedStr[i] > SortedStr[i - 1] + 1) {
1440 NonContRanges++;
1441 }
1442 }
1443
1444 // Restrict this optimization to profitable cases with one or two range
1445 // checks.
1446 if (NonContRanges > 2)
1447 return nullptr;
1448
1449 SmallVector<Value *> CharCompares;
1450 for (unsigned char C : SortedStr)
1451 CharCompares.push_back(
1452 B.CreateICmpEQ(CharVal, ConstantInt::get(CharVal->getType(), C)));
1453
1454 return B.CreateIntToPtr(B.CreateOr(CharCompares), CI->getType());
1455 }
1456
1457 // For the bit field use a power-of-2 type with at least 8 bits to avoid
1458 // creating unnecessary illegal types.
1459 unsigned char Width = NextPowerOf2(std::max((unsigned char)7, Max));
1460
1461 // Now build the bit field.
1462 APInt Bitfield(Width, 0);
1463 for (char C : Str)
1464 Bitfield.setBit((unsigned char)C);
1465 Value *BitfieldC = B.getInt(Bitfield);
1466
1467 // Adjust width of "C" to the bitfield width, then mask off the high bits.
1468 Value *C = B.CreateZExtOrTrunc(CharVal, BitfieldC->getType());
1469 C = B.CreateAnd(C, B.getIntN(Width, 0xFF));
1470
1471 // First check that the bit field access is within bounds.
1472 Value *Bounds = B.CreateICmp(ICmpInst::ICMP_ULT, C, B.getIntN(Width, Width),
1473 "memchr.bounds");
1474
1475 // Create code that checks if the given bit is set in the field.
1476 Value *Shl = B.CreateShl(B.getIntN(Width, 1ULL), C);
1477 Value *Bits = B.CreateIsNotNull(B.CreateAnd(Shl, BitfieldC), "memchr.bits");
1478
1479 // Finally merge both checks and cast to pointer type. The inttoptr
1480 // implicitly zexts the i1 to intptr type.
1481 return B.CreateIntToPtr(B.CreateLogicalAnd(Bounds, Bits, "memchr"),
1482 CI->getType());
1483}
1484
1485// Optimize a memcmp or, when StrNCmp is true, strncmp call CI with constant
1486// arrays LHS and RHS and nonconstant Size.
1488 Value *Size, bool StrNCmp,
1489 IRBuilderBase &B, const DataLayout &DL) {
1490 if (LHS == RHS) // memcmp(s,s,x) -> 0
1491 return Constant::getNullValue(CI->getType());
1492
1493 StringRef LStr, RStr;
1494 if (!getConstantStringInfo(LHS, LStr, /*TrimAtNul=*/false) ||
1495 !getConstantStringInfo(RHS, RStr, /*TrimAtNul=*/false))
1496 return nullptr;
1497
1498 // If the contents of both constant arrays are known, fold a call to
1499 // memcmp(A, B, N) to
1500 // N <= Pos ? 0 : (A < B ? -1 : B < A ? +1 : 0)
1501 // where Pos is the first mismatch between A and B, determined below.
1502
1503 uint64_t Pos = 0;
1504 Value *Zero = ConstantInt::get(CI->getType(), 0);
1505 for (uint64_t MinSize = std::min(LStr.size(), RStr.size()); ; ++Pos) {
1506 if (Pos == MinSize ||
1507 (StrNCmp && (LStr[Pos] == '\0' && RStr[Pos] == '\0'))) {
1508 // One array is a leading part of the other of equal or greater
1509 // size, or for strncmp, the arrays are equal strings.
1510 // Fold the result to zero. Size is assumed to be in bounds, since
1511 // otherwise the call would be undefined.
1512 return Zero;
1513 }
1514
1515 if (LStr[Pos] != RStr[Pos])
1516 break;
1517 }
1518
1519 // Normalize the result.
1520 typedef unsigned char UChar;
1521 int IRes = UChar(LStr[Pos]) < UChar(RStr[Pos]) ? -1 : 1;
1522 Value *MaxSize = ConstantInt::get(Size->getType(), Pos);
1523 Value *Cmp = B.CreateICmp(ICmpInst::ICMP_ULE, Size, MaxSize);
1524 Value *Res = ConstantInt::get(CI->getType(), IRes);
1525 return B.CreateSelect(Cmp, Zero, Res);
1526}
1527
1528// Optimize a memcmp call CI with constant size Len.
1530 uint64_t Len, IRBuilderBase &B,
1531 const DataLayout &DL) {
1532 if (Len == 0) // memcmp(s1,s2,0) -> 0
1533 return Constant::getNullValue(CI->getType());
1534
1535 // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
1536 if (Len == 1) {
1537 Value *LHSV = B.CreateZExt(B.CreateLoad(B.getInt8Ty(), LHS, "lhsc"),
1538 CI->getType(), "lhsv");
1539 Value *RHSV = B.CreateZExt(B.CreateLoad(B.getInt8Ty(), RHS, "rhsc"),
1540 CI->getType(), "rhsv");
1541 return B.CreateSub(LHSV, RHSV, "chardiff");
1542 }
1543
1544 // memcmp(S1,S2,N/8)==0 -> (*(intN_t*)S1 != *(intN_t*)S2)==0
1545 // TODO: The case where both inputs are constants does not need to be limited
1546 // to legal integers or equality comparison. See block below this.
1547 if (DL.isLegalInteger(Len * 8) && isOnlyUsedInZeroEqualityComparison(CI)) {
1548 IntegerType *IntType = IntegerType::get(CI->getContext(), Len * 8);
1549 Align PrefAlignment = DL.getPrefTypeAlign(IntType);
1550
1551 // First, see if we can fold either argument to a constant.
1552 Value *LHSV = nullptr;
1553 if (auto *LHSC = dyn_cast<Constant>(LHS))
1554 LHSV = ConstantFoldLoadFromConstPtr(LHSC, IntType, DL);
1555
1556 Value *RHSV = nullptr;
1557 if (auto *RHSC = dyn_cast<Constant>(RHS))
1558 RHSV = ConstantFoldLoadFromConstPtr(RHSC, IntType, DL);
1559
1560 // Don't generate unaligned loads. If either source is constant data,
1561 // alignment doesn't matter for that source because there is no load.
1562 if ((LHSV || getKnownAlignment(LHS, DL, CI) >= PrefAlignment) &&
1563 (RHSV || getKnownAlignment(RHS, DL, CI) >= PrefAlignment)) {
1564 if (!LHSV)
1565 LHSV = B.CreateLoad(IntType, LHS, "lhsv");
1566 if (!RHSV)
1567 RHSV = B.CreateLoad(IntType, RHS, "rhsv");
1568 return B.CreateZExt(B.CreateICmpNE(LHSV, RHSV), CI->getType(), "memcmp");
1569 }
1570 }
1571
1572 return nullptr;
1573}
1574
1575// Most simplifications for memcmp also apply to bcmp.
1576Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
1577 IRBuilderBase &B) {
1578 Value *LHS = CI->getArgOperand(0), *RHS = CI->getArgOperand(1);
1579 Value *Size = CI->getArgOperand(2);
1580
1581 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1582
1583 if (Value *Res = optimizeMemCmpVarSize(CI, LHS, RHS, Size, false, B, DL))
1584 return Res;
1585
1586 // Handle constant Size.
1587 ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
1588 if (!LenC)
1589 return nullptr;
1590
1591 return optimizeMemCmpConstantSize(CI, LHS, RHS, LenC->getZExtValue(), B, DL);
1592}
1593
1594Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
1595 Module *M = CI->getModule();
1596 if (Value *V = optimizeMemCmpBCmpCommon(CI, B))
1597 return V;
1598
1599 // memcmp(x, y, Len) == 0 -> bcmp(x, y, Len) == 0
1600 // bcmp can be more efficient than memcmp because it only has to know that
1601 // there is a difference, not how different one is to the other.
1602 if (isLibFuncEmittable(M, TLI, LibFunc_bcmp) &&
1604 Value *LHS = CI->getArgOperand(0);
1605 Value *RHS = CI->getArgOperand(1);
1606 Value *Size = CI->getArgOperand(2);
1607 return copyFlags(*CI, emitBCmp(LHS, RHS, Size, B, DL, TLI));
1608 }
1609
1610 return nullptr;
1611}
1612
1613Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilderBase &B) {
1614 return optimizeMemCmpBCmpCommon(CI, B);
1615}
1616
1617Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
1618 Value *Size = CI->getArgOperand(2);
1619 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1620 if (isa<IntrinsicInst>(CI))
1621 return nullptr;
1622
1623 // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
1624 CallInst *NewCI = B.CreateMemCpy(CI->getArgOperand(0), Align(1),
1625 CI->getArgOperand(1), Align(1), Size);
1626 mergeAttributesAndFlags(NewCI, *CI);
1627 return CI->getArgOperand(0);
1628}
1629
1630Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
1631 Value *Dst = CI->getArgOperand(0);
1632 Value *Src = CI->getArgOperand(1);
1633 ConstantInt *StopChar = dyn_cast<ConstantInt>(CI->getArgOperand(2));
1634 ConstantInt *N = dyn_cast<ConstantInt>(CI->getArgOperand(3));
1635 StringRef SrcStr;
1636 if (CI->use_empty() && Dst == Src)
1637 return Dst;
1638 // memccpy(d, s, c, 0) -> nullptr
1639 if (N) {
1640 if (N->isNullValue())
1641 return Constant::getNullValue(CI->getType());
1642 if (!getConstantStringInfo(Src, SrcStr, /*TrimAtNul=*/false) ||
1643 // TODO: Handle zeroinitializer.
1644 !StopChar)
1645 return nullptr;
1646 } else {
1647 return nullptr;
1648 }
1649
1650 // Wrap arg 'c' of type int to char
1651 size_t Pos = SrcStr.find(StopChar->getSExtValue() & 0xFF);
1652 if (Pos == StringRef::npos) {
1653 if (N->getZExtValue() <= SrcStr.size()) {
1654 copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1),
1655 CI->getArgOperand(3)));
1656 return Constant::getNullValue(CI->getType());
1657 }
1658 return nullptr;
1659 }
1660
1661 Value *NewN =
1662 ConstantInt::get(N->getType(), std::min(uint64_t(Pos + 1), N->getZExtValue()));
1663 // memccpy -> llvm.memcpy
1664 copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1), NewN));
1665 return Pos + 1 <= N->getZExtValue()
1666 ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, NewN)
1668}
1669
1670Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
1671 Value *Dst = CI->getArgOperand(0);
1672 Value *N = CI->getArgOperand(2);
1673 // mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
1674 CallInst *NewCI =
1675 B.CreateMemCpy(Dst, Align(1), CI->getArgOperand(1), Align(1), N);
1676 // Propagate attributes, but memcpy has no return value, so make sure that
1677 // any return attributes are compliant.
1678 // TODO: Attach return value attributes to the 1st operand to preserve them?
1679 mergeAttributesAndFlags(NewCI, *CI);
1680 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, N);
1681}
1682
1683Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
1684 Value *Size = CI->getArgOperand(2);
1685 annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
1686 if (isa<IntrinsicInst>(CI))
1687 return nullptr;
1688
1689 // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
1690 CallInst *NewCI = B.CreateMemMove(CI->getArgOperand(0), Align(1),
1691 CI->getArgOperand(1), Align(1), Size);
1692 mergeAttributesAndFlags(NewCI, *CI);
1693 return CI->getArgOperand(0);
1694}
1695
1696Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
1697 Value *Size = CI->getArgOperand(2);
1699 if (isa<IntrinsicInst>(CI))
1700 return nullptr;
1701
1702 // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
1703 Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
1704 CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val, Size, Align(1));
1705 mergeAttributesAndFlags(NewCI, *CI);
1706 return CI->getArgOperand(0);
1707}
1708
1709Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
1710 if (isa<ConstantPointerNull>(CI->getArgOperand(0)))
1711 return copyFlags(*CI, emitMalloc(CI->getArgOperand(1), B, DL, TLI));
1712
1713 return nullptr;
1714}
1715
1716// When enabled, replace operator new() calls marked with a hot or cold memprof
1717// attribute with an operator new() call that takes a __hot_cold_t parameter.
1718// Currently this is supported by the open source version of tcmalloc, see:
1719// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
1720Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
1721 LibFunc &Func) {
1722 if (!OptimizeHotColdNew)
1723 return nullptr;
1724
1725 uint8_t HotCold;
1726 if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "cold")
1727 HotCold = ColdNewHintValue;
1728 else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "hot")
1729 HotCold = HotNewHintValue;
1730 else
1731 return nullptr;
1732
1733 switch (Func) {
1734 case LibFunc_Znwm:
1735 return emitHotColdNew(CI->getArgOperand(0), B, TLI,
1736 LibFunc_Znwm12__hot_cold_t, HotCold);
1737 case LibFunc_Znam:
1738 return emitHotColdNew(CI->getArgOperand(0), B, TLI,
1739 LibFunc_Znam12__hot_cold_t, HotCold);
1740 case LibFunc_ZnwmRKSt9nothrow_t:
1741 return emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
1742 TLI, LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
1743 HotCold);
1744 case LibFunc_ZnamRKSt9nothrow_t:
1745 return emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
1746 TLI, LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
1747 HotCold);
1748 case LibFunc_ZnwmSt11align_val_t:
1749 return emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
1750 TLI, LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
1751 HotCold);
1752 case LibFunc_ZnamSt11align_val_t:
1753 return emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
1754 TLI, LibFunc_ZnamSt11align_val_t12__hot_cold_t,
1755 HotCold);
1756 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
1758 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1759 TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
1760 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
1762 CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
1763 TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
1764 default:
1765 return nullptr;
1766 }
1767}
1768
1769//===----------------------------------------------------------------------===//
1770// Math Library Optimizations
1771//===----------------------------------------------------------------------===//
1772
1773// Replace a libcall \p CI with a call to intrinsic \p IID
1775 Intrinsic::ID IID) {
1776 // Propagate fast-math flags from the existing call to the new call.
1778 B.setFastMathFlags(CI->getFastMathFlags());
1779
1780 Module *M = CI->getModule();
1781 Value *V = CI->getArgOperand(0);
1782 Function *F = Intrinsic::getDeclaration(M, IID, CI->getType());
1783 CallInst *NewCall = B.CreateCall(F, V);
1784 NewCall->takeName(CI);
1785 return copyFlags(*CI, NewCall);
1786}
1787
1788/// Return a variant of Val with float type.
1789/// Currently this works in two cases: If Val is an FPExtension of a float
1790/// value to something bigger, simply return the operand.
1791/// If Val is a ConstantFP but can be converted to a float ConstantFP without
1792/// loss of precision do so.
1794 if (FPExtInst *Cast = dyn_cast<FPExtInst>(Val)) {
1795 Value *Op = Cast->getOperand(0);
1796 if (Op->getType()->isFloatTy())
1797 return Op;
1798 }
1799 if (ConstantFP *Const = dyn_cast<ConstantFP>(Val)) {
1800 APFloat F = Const->getValueAPF();
1801 bool losesInfo;
1803 &losesInfo);
1804 if (!losesInfo)
1805 return ConstantFP::get(Const->getContext(), F);
1806 }
1807 return nullptr;
1808}
1809
1810/// Shrink double -> float functions.
1812 bool isBinary, const TargetLibraryInfo *TLI,
1813 bool isPrecise = false) {
1814 Function *CalleeFn = CI->getCalledFunction();
1815 if (!CI->getType()->isDoubleTy() || !CalleeFn)
1816 return nullptr;
1817
1818 // If not all the uses of the function are converted to float, then bail out.
1819 // This matters if the precision of the result is more important than the
1820 // precision of the arguments.
1821 if (isPrecise)
1822 for (User *U : CI->users()) {
1823 FPTruncInst *Cast = dyn_cast<FPTruncInst>(U);
1824 if (!Cast || !Cast->getType()->isFloatTy())
1825 return nullptr;
1826 }
1827
1828 // If this is something like 'g((double) float)', convert to 'gf(float)'.
1829 Value *V[2];
1831 V[1] = isBinary ? valueHasFloatPrecision(CI->getArgOperand(1)) : nullptr;
1832 if (!V[0] || (isBinary && !V[1]))
1833 return nullptr;
1834
1835 // If call isn't an intrinsic, check that it isn't within a function with the
1836 // same name as the float version of this call, otherwise the result is an
1837 // infinite loop. For example, from MinGW-w64:
1838 //
1839 // float expf(float val) { return (float) exp((double) val); }
1840 StringRef CalleeName = CalleeFn->getName();
1841 bool IsIntrinsic = CalleeFn->isIntrinsic();
1842 if (!IsIntrinsic) {
1843 StringRef CallerName = CI->getFunction()->getName();
1844 if (!CallerName.empty() && CallerName.back() == 'f' &&
1845 CallerName.size() == (CalleeName.size() + 1) &&
1846 CallerName.startswith(CalleeName))
1847 return nullptr;
1848 }
1849
1850 // Propagate the math semantics from the current function to the new function.
1852 B.setFastMathFlags(CI->getFastMathFlags());
1853
1854 // g((double) float) -> (double) gf(float)
1855 Value *R;
1856 if (IsIntrinsic) {
1857 Module *M = CI->getModule();
1858 Intrinsic::ID IID = CalleeFn->getIntrinsicID();
1859 Function *Fn = Intrinsic::getDeclaration(M, IID, B.getFloatTy());
1860 R = isBinary ? B.CreateCall(Fn, V) : B.CreateCall(Fn, V[0]);
1861 } else {
1862 AttributeList CalleeAttrs = CalleeFn->getAttributes();
1863 R = isBinary ? emitBinaryFloatFnCall(V[0], V[1], TLI, CalleeName, B,
1864 CalleeAttrs)
1865 : emitUnaryFloatFnCall(V[0], TLI, CalleeName, B, CalleeAttrs);
1866 }
1867 return B.CreateFPExt(R, B.getDoubleTy());
1868}
1869
1870/// Shrink double -> float for unary functions.
1872 const TargetLibraryInfo *TLI,
1873 bool isPrecise = false) {
1874 return optimizeDoubleFP(CI, B, false, TLI, isPrecise);
1875}
1876
1877/// Shrink double -> float for binary functions.
1879 const TargetLibraryInfo *TLI,
1880 bool isPrecise = false) {
1881 return optimizeDoubleFP(CI, B, true, TLI, isPrecise);
1882}
1883
1884// cabs(z) -> sqrt((creal(z)*creal(z)) + (cimag(z)*cimag(z)))
1885Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
1886 if (!CI->isFast())
1887 return nullptr;
1888
1889 // Propagate fast-math flags from the existing call to new instructions.
1891 B.setFastMathFlags(CI->getFastMathFlags());
1892
1893 Value *Real, *Imag;
1894 if (CI->arg_size() == 1) {
1895 Value *Op = CI->getArgOperand(0);
1896 assert(Op->getType()->isArrayTy() && "Unexpected signature for cabs!");
1897 Real = B.CreateExtractValue(Op, 0, "real");
1898 Imag = B.CreateExtractValue(Op, 1, "imag");
1899 } else {
1900 assert(CI->arg_size() == 2 && "Unexpected signature for cabs!");
1901 Real = CI->getArgOperand(0);
1902 Imag = CI->getArgOperand(1);
1903 }
1904
1905 Value *RealReal = B.CreateFMul(Real, Real);
1906 Value *ImagImag = B.CreateFMul(Imag, Imag);
1907
1908 Function *FSqrt = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::sqrt,
1909 CI->getType());
1910 return copyFlags(
1911 *CI, B.CreateCall(FSqrt, B.CreateFAdd(RealReal, ImagImag), "cabs"));
1912}
1913
1915 IRBuilderBase &B) {
1916 if (!isa<FPMathOperator>(Call))
1917 return nullptr;
1918
1920 B.setFastMathFlags(Call->getFastMathFlags());
1921
1922 // TODO: Can this be shared to also handle LLVM intrinsics?
1923 Value *X;
1924 switch (Func) {
1925 case LibFunc_sin:
1926 case LibFunc_sinf:
1927 case LibFunc_sinl:
1928 case LibFunc_tan:
1929 case LibFunc_tanf:
1930 case LibFunc_tanl:
1931 // sin(-X) --> -sin(X)
1932 // tan(-X) --> -tan(X)
1933 if (match(Call->getArgOperand(0), m_OneUse(m_FNeg(m_Value(X)))))
1934 return B.CreateFNeg(
1935 copyFlags(*Call, B.CreateCall(Call->getCalledFunction(), X)));
1936 break;
1937 case LibFunc_cos:
1938 case LibFunc_cosf:
1939 case LibFunc_cosl:
1940 // cos(-X) --> cos(X)
1941 if (match(Call->getArgOperand(0), m_FNeg(m_Value(X))))
1942 return copyFlags(*Call,
1943 B.CreateCall(Call->getCalledFunction(), X, "cos"));
1944 break;
1945 default:
1946 break;
1947 }
1948 return nullptr;
1949}
1950
1951// Return a properly extended integer (DstWidth bits wide) if the operation is
1952// an itofp.
1953static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
1954 if (isa<SIToFPInst>(I2F) || isa<UIToFPInst>(I2F)) {
1955 Value *Op = cast<Instruction>(I2F)->getOperand(0);
1956 // Make sure that the exponent fits inside an "int" of size DstWidth,
1957 // thus avoiding any range issues that FP has not.
1958 unsigned BitWidth = Op->getType()->getPrimitiveSizeInBits();
1959 if (BitWidth < DstWidth ||
1960 (BitWidth == DstWidth && isa<SIToFPInst>(I2F)))
1961 return isa<SIToFPInst>(I2F) ? B.CreateSExt(Op, B.getIntNTy(DstWidth))
1962 : B.CreateZExt(Op, B.getIntNTy(DstWidth));
1963 }
1964
1965 return nullptr;
1966}
1967
1968/// Use exp{,2}(x * y) for pow(exp{,2}(x), y);
1969/// ldexp(1.0, x) for pow(2.0, itofp(x)); exp2(n * x) for pow(2.0 ** n, x);
1970/// exp10(x) for pow(10.0, x); exp2(log2(n) * x) for pow(n, x).
1971Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
1972 Module *M = Pow->getModule();
1973 Value *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1);
1974 Module *Mod = Pow->getModule();
1975 Type *Ty = Pow->getType();
1976 bool Ignored;
1977
1978 // Evaluate special cases related to a nested function as the base.
1979
1980 // pow(exp(x), y) -> exp(x * y)
1981 // pow(exp2(x), y) -> exp2(x * y)
1982 // If exp{,2}() is used only once, it is better to fold two transcendental
1983 // math functions into one. If used again, exp{,2}() would still have to be
1984 // called with the original argument, then keep both original transcendental
1985 // functions. However, this transformation is only safe with fully relaxed
1986 // math semantics, since, besides rounding differences, it changes overflow
1987 // and underflow behavior quite dramatically. For example:
1988 // pow(exp(1000), 0.001) = pow(inf, 0.001) = inf
1989 // Whereas:
1990 // exp(1000 * 0.001) = exp(1)
1991 // TODO: Loosen the requirement for fully relaxed math semantics.
1992 // TODO: Handle exp10() when more targets have it available.
1993 CallInst *BaseFn = dyn_cast<CallInst>(Base);
1994 if (BaseFn && BaseFn->hasOneUse() && BaseFn->isFast() && Pow->isFast()) {
1995 LibFunc LibFn;
1996
1997 Function *CalleeFn = BaseFn->getCalledFunction();
1998 if (CalleeFn && TLI->getLibFunc(CalleeFn->getName(), LibFn) &&
1999 isLibFuncEmittable(M, TLI, LibFn)) {
2000 StringRef ExpName;
2002 Value *ExpFn;
2003 LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
2004
2005 switch (LibFn) {
2006 default:
2007 return nullptr;
2008 case LibFunc_expf:
2009 case LibFunc_exp:
2010 case LibFunc_expl:
2011 ExpName = TLI->getName(LibFunc_exp);
2012 ID = Intrinsic::exp;
2013 LibFnFloat = LibFunc_expf;
2014 LibFnDouble = LibFunc_exp;
2015 LibFnLongDouble = LibFunc_expl;
2016 break;
2017 case LibFunc_exp2f:
2018 case LibFunc_exp2:
2019 case LibFunc_exp2l:
2020 ExpName = TLI->getName(LibFunc_exp2);
2021 ID = Intrinsic::exp2;
2022 LibFnFloat = LibFunc_exp2f;
2023 LibFnDouble = LibFunc_exp2;
2024 LibFnLongDouble = LibFunc_exp2l;
2025 break;
2026 }
2027
2028 // Create new exp{,2}() with the product as its argument.
2029 Value *FMul = B.CreateFMul(BaseFn->getArgOperand(0), Expo, "mul");
2030 ExpFn = BaseFn->doesNotAccessMemory()
2031 ? B.CreateCall(Intrinsic::getDeclaration(Mod, ID, Ty),
2032 FMul, ExpName)
2033 : emitUnaryFloatFnCall(FMul, TLI, LibFnDouble, LibFnFloat,
2034 LibFnLongDouble, B,
2035 BaseFn->getAttributes());
2036
2037 // Since the new exp{,2}() is different from the original one, dead code
2038 // elimination cannot be trusted to remove it, since it may have side
2039 // effects (e.g., errno). When the only consumer for the original
2040 // exp{,2}() is pow(), then it has to be explicitly erased.
2041 substituteInParent(BaseFn, ExpFn);
2042 return ExpFn;
2043 }
2044 }
2045
2046 // Evaluate special cases related to a constant base.
2047
2048 const APFloat *BaseF;
2049 if (!match(Pow->getArgOperand(0), m_APFloat(BaseF)))
2050 return nullptr;
2051
2052 AttributeList NoAttrs; // Attributes are only meaningful on the original call
2053
2054 // pow(2.0, itofp(x)) -> ldexp(1.0, x)
2055 // TODO: This does not work for vectors because there is no ldexp intrinsic.
2056 if (!Ty->isVectorTy() && match(Base, m_SpecificFP(2.0)) &&
2057 (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo)) &&
2058 hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl)) {
2059 if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize()))
2060 return copyFlags(*Pow,
2061 emitBinaryFloatFnCall(ConstantFP::get(Ty, 1.0), ExpoI,
2062 TLI, LibFunc_ldexp, LibFunc_ldexpf,
2063 LibFunc_ldexpl, B, NoAttrs));
2064 }
2065
2066 // pow(2.0 ** n, x) -> exp2(n * x)
2067 if (hasFloatFn(M, TLI, Ty, LibFunc_exp2, LibFunc_exp2f, LibFunc_exp2l)) {
2068 APFloat BaseR = APFloat(1.0);
2069 BaseR.convert(BaseF->getSemantics(), APFloat::rmTowardZero, &Ignored);
2070 BaseR = BaseR / *BaseF;
2071 bool IsInteger = BaseF->isInteger(), IsReciprocal = BaseR.isInteger();
2072 const APFloat *NF = IsReciprocal ? &BaseR : BaseF;
2073 APSInt NI(64, false);
2074 if ((IsInteger || IsReciprocal) &&
2075 NF->convertToInteger(NI, APFloat::rmTowardZero, &Ignored) ==
2076 APFloat::opOK &&
2077 NI > 1 && NI.isPowerOf2()) {
2078 double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
2079 Value *FMul = B.CreateFMul(Expo, ConstantFP::get(Ty, N), "mul");
2080 if (Pow->doesNotAccessMemory())
2081 return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
2082 Mod, Intrinsic::exp2, Ty),
2083 FMul, "exp2"));
2084 else
2085 return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
2086 LibFunc_exp2f,
2087 LibFunc_exp2l, B, NoAttrs));
2088 }
2089 }
2090
2091 // pow(10.0, x) -> exp10(x)
2092 // TODO: There is no exp10() intrinsic yet, but some day there shall be one.
2093 if (match(Base, m_SpecificFP(10.0)) &&
2094 hasFloatFn(M, TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l))
2095 return copyFlags(*Pow, emitUnaryFloatFnCall(Expo, TLI, LibFunc_exp10,
2096 LibFunc_exp10f, LibFunc_exp10l,
2097 B, NoAttrs));
2098
2099 // pow(x, y) -> exp2(log2(x) * y)
2100 if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
2101 !BaseF->isNegative()) {
2102 // pow(1, inf) is defined to be 1 but exp2(log2(1) * inf) evaluates to NaN.
2103 // Luckily optimizePow has already handled the x == 1 case.
2104 assert(!match(Base, m_FPOne()) &&
2105 "pow(1.0, y) should have been simplified earlier!");
2106
2107 Value *Log = nullptr;
2108 if (Ty->isFloatTy())
2109 Log = ConstantFP::get(Ty, std::log2(BaseF->convertToFloat()));
2110 else if (Ty->isDoubleTy())
2111 Log = ConstantFP::get(Ty, std::log2(BaseF->convertToDouble()));
2112
2113 if (Log) {
2114 Value *FMul = B.CreateFMul(Log, Expo, "mul");
2115 if (Pow->doesNotAccessMemory())
2116 return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
2117 Mod, Intrinsic::exp2, Ty),
2118 FMul, "exp2"));
2119 else if (hasFloatFn(M, TLI, Ty, LibFunc_exp2, LibFunc_exp2f,
2120 LibFunc_exp2l))
2121 return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
2122 LibFunc_exp2f,
2123 LibFunc_exp2l, B, NoAttrs));
2124 }
2125 }
2126
2127 return nullptr;
2128}
2129
2130static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
2131 Module *M, IRBuilderBase &B,
2132 const TargetLibraryInfo *TLI) {
2133 // If errno is never set, then use the intrinsic for sqrt().
2134 if (NoErrno) {
2135 Function *SqrtFn =
2136 Intrinsic::getDeclaration(M, Intrinsic::sqrt, V->getType());
2137 return B.CreateCall(SqrtFn, V, "sqrt");
2138 }
2139
2140 // Otherwise, use the libcall for sqrt().
2141 if (hasFloatFn(M, TLI, V->getType(), LibFunc_sqrt, LibFunc_sqrtf,
2142 LibFunc_sqrtl))
2143 // TODO: We also should check that the target can in fact lower the sqrt()
2144 // libcall. We currently have no way to ask this question, so we ask if
2145 // the target has a sqrt() libcall, which is not exactly the same.
2146 return emitUnaryFloatFnCall(V, TLI, LibFunc_sqrt, LibFunc_sqrtf,
2147 LibFunc_sqrtl, B, Attrs);
2148
2149 return nullptr;
2150}
2151
2152/// Use square root in place of pow(x, +/-0.5).
2153Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
2154 Value *Sqrt, *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1);
2155 Module *Mod = Pow->getModule();
2156 Type *Ty = Pow->getType();
2157
2158 const APFloat *ExpoF;
2159 if (!match(Expo, m_APFloat(ExpoF)) ||
2160 (!ExpoF->isExactlyValue(0.5) && !ExpoF->isExactlyValue(-0.5)))
2161 return nullptr;
2162
2163 // Converting pow(X, -0.5) to 1/sqrt(X) may introduce an extra rounding step,
2164 // so that requires fast-math-flags (afn or reassoc).
2165 if (ExpoF->isNegative() && (!Pow->hasApproxFunc() && !Pow->hasAllowReassoc()))
2166 return nullptr;
2167
2168 // If we have a pow() library call (accesses memory) and we can't guarantee
2169 // that the base is not an infinity, give up:
2170 // pow(-Inf, 0.5) is optionally required to have a result of +Inf (not setting
2171 // errno), but sqrt(-Inf) is required by various standards to set errno.
2172 if (!Pow->doesNotAccessMemory() && !Pow->hasNoInfs() &&
2173 !isKnownNeverInfinity(Base, DL, TLI, 0, AC, Pow))
2174 return nullptr;
2175
2177 TLI);
2178 if (!Sqrt)
2179 return nullptr;
2180
2181 // Handle signed zero base by expanding to fabs(sqrt(x)).
2182 if (!Pow->hasNoSignedZeros()) {
2183 Function *FAbsFn = Intrinsic::getDeclaration(Mod, Intrinsic::fabs, Ty);
2184 Sqrt = B.CreateCall(FAbsFn, Sqrt, "abs");
2185 }
2186
2187 Sqrt = copyFlags(*Pow, Sqrt);
2188
2189 // Handle non finite base by expanding to
2190 // (x == -infinity ? +infinity : sqrt(x)).
2191 if (!Pow->hasNoInfs()) {
2192 Value *PosInf = ConstantFP::getInfinity(Ty),
2193 *NegInf = ConstantFP::getInfinity(Ty, true);
2194 Value *FCmp = B.CreateFCmpOEQ(Base, NegInf, "isinf");
2195 Sqrt = B.CreateSelect(FCmp, PosInf, Sqrt);
2196 }
2197
2198 // If the exponent is negative, then get the reciprocal.
2199 if (ExpoF->isNegative())
2200 Sqrt = B.CreateFDiv(ConstantFP::get(Ty, 1.0), Sqrt, "reciprocal");
2201
2202 return Sqrt;
2203}
2204
2206 IRBuilderBase &B) {
2207 Value *Args[] = {Base, Expo};
2208 Type *Types[] = {Base->getType(), Expo->getType()};
2209 Function *F = Intrinsic::getDeclaration(M, Intrinsic::powi, Types);
2210 return B.CreateCall(F, Args);
2211}
2212
2213Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
2214 Value *Base = Pow->getArgOperand(0);
2215 Value *Expo = Pow->getArgOperand(1);
2217 StringRef Name = Callee->getName();
2218 Type *Ty = Pow->getType();
2219 Module *M = Pow->getModule();
2220 bool AllowApprox = Pow->hasApproxFunc();
2221 bool Ignored;
2222
2223 // Propagate the math semantics from the call to any created instructions.
2225 B.setFastMathFlags(Pow->getFastMathFlags());
2226 // Evaluate special cases related to the base.
2227
2228 // pow(1.0, x) -> 1.0
2229 if (match(Base, m_FPOne()))
2230 return Base;
2231
2232 if (Value *Exp = replacePowWithExp(Pow, B))
2233 return Exp;
2234
2235 // Evaluate special cases related to the exponent.
2236
2237 // pow(x, -1.0) -> 1.0 / x
2238 if (match(Expo, m_SpecificFP(-1.0)))
2239 return B.CreateFDiv(ConstantFP::get(Ty, 1.0), Base, "reciprocal");
2240
2241 // pow(x, +/-0.0) -> 1.0
2242 if (match(Expo, m_AnyZeroFP()))
2243 return ConstantFP::get(Ty, 1.0);
2244
2245 // pow(x, 1.0) -> x
2246 if (match(Expo, m_FPOne()))
2247 return Base;
2248
2249 // pow(x, 2.0) -> x * x
2250 if (match(Expo, m_SpecificFP(2.0)))
2251 return B.CreateFMul(Base, Base, "square");
2252
2253 if (Value *Sqrt = replacePowWithSqrt(Pow, B))
2254 return Sqrt;
2255
2256 // If we can approximate pow:
2257 // pow(x, n) -> powi(x, n) * sqrt(x) if n has exactly a 0.5 fraction
2258 // pow(x, n) -> powi(x, n) if n is a constant signed integer value
2259 const APFloat *ExpoF;
2260 if (AllowApprox && match(Expo, m_APFloat(ExpoF)) &&
2261 !ExpoF->isExactlyValue(0.5) && !ExpoF->isExactlyValue(-0.5)) {
2262 APFloat ExpoA(abs(*ExpoF));
2263 APFloat ExpoI(*ExpoF);
2264 Value *Sqrt = nullptr;
2265 if (!ExpoA.isInteger()) {
2266 APFloat Expo2 = ExpoA;
2267 // To check if ExpoA is an integer + 0.5, we add it to itself. If there
2268 // is no floating point exception and the result is an integer, then
2269 // ExpoA == integer + 0.5
2270 if (Expo2.add(ExpoA, APFloat::rmNearestTiesToEven) != APFloat::opOK)
2271 return nullptr;
2272
2273 if (!Expo2.isInteger())
2274 return nullptr;
2275
2276 if (ExpoI.roundToIntegral(APFloat::rmTowardNegative) !=
2278 return nullptr;
2279 if (!ExpoI.isInteger())
2280 return nullptr;
2281 ExpoF = &ExpoI;
2282
2284 B, TLI);
2285 if (!Sqrt)
2286 return nullptr;
2287 }
2288
2289 // 0.5 fraction is now optionally handled.
2290 // Do pow -> powi for remaining integer exponent
2291 APSInt IntExpo(TLI->getIntSize(), /*isUnsigned=*/false);
2292 if (ExpoF->isInteger() &&
2293 ExpoF->convertToInteger(IntExpo, APFloat::rmTowardZero, &Ignored) ==
2294 APFloat::opOK) {
2295 Value *PowI = copyFlags(
2296 *Pow,
2298 Base, ConstantInt::get(B.getIntNTy(TLI->getIntSize()), IntExpo),
2299 M, B));
2300
2301 if (PowI && Sqrt)
2302 return B.CreateFMul(PowI, Sqrt);
2303
2304 return PowI;
2305 }
2306 }
2307
2308 // powf(x, itofp(y)) -> powi(x, y)
2309 if (AllowApprox && (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo))) {
2310 if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize()))
2311 return copyFlags(*Pow, createPowWithIntegerExponent(Base, ExpoI, M, B));
2312 }
2313
2314 // Shrink pow() to powf() if the arguments are single precision,
2315 // unless the result is expected to be double precision.
2316 if (UnsafeFPShrink && Name == TLI->getName(LibFunc_pow) &&
2317 hasFloatVersion(M, Name)) {
2318 if (Value *Shrunk = optimizeBinaryDoubleFP(Pow, B, TLI, true))
2319 return Shrunk;
2320 }
2321
2322 return nullptr;
2323}
2324
2325Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
2326 Module *M = CI->getModule();
2328 StringRef Name = Callee->getName();
2329 Value *Ret = nullptr;
2330 if (UnsafeFPShrink && Name == TLI->getName(LibFunc_exp2) &&
2331 hasFloatVersion(M, Name))
2332 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2333
2334 // Bail out for vectors because the code below only expects scalars.
2335 // TODO: This could be allowed if we had a ldexp intrinsic (D14327).
2336 Type *Ty = CI->getType();
2337 if (Ty->isVectorTy())
2338 return Ret;
2339
2340 // exp2(sitofp(x)) -> ldexp(1.0, sext(x)) if sizeof(x) <= IntSize
2341 // exp2(uitofp(x)) -> ldexp(1.0, zext(x)) if sizeof(x) < IntSize
2342 Value *Op = CI->getArgOperand(0);
2343 if ((isa<SIToFPInst>(Op) || isa<UIToFPInst>(Op)) &&
2344 hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl)) {
2345 if (Value *Exp = getIntToFPVal(Op, B, TLI->getIntSize())) {
2347 B.setFastMathFlags(CI->getFastMathFlags());
2348 return copyFlags(
2349 *CI, emitBinaryFloatFnCall(ConstantFP::get(Ty, 1.0), Exp, TLI,
2350 LibFunc_ldexp, LibFunc_ldexpf,
2351 LibFunc_ldexpl, B, AttributeList()));
2352 }
2353 }
2354
2355 return Ret;
2356}
2357
2358Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B) {
2359 Module *M = CI->getModule();
2360
2361 // If we can shrink the call to a float function rather than a double
2362 // function, do that first.
2364 StringRef Name = Callee->getName();
2365 if ((Name == "fmin" || Name == "fmax") && hasFloatVersion(M, Name))
2366 if (Value *Ret = optimizeBinaryDoubleFP(CI, B, TLI))
2367 return Ret;
2368
2369 // The LLVM intrinsics minnum/maxnum correspond to fmin/fmax. Canonicalize to
2370 // the intrinsics for improved optimization (for example, vectorization).
2371 // No-signed-zeros is implied by the definitions of fmax/fmin themselves.
2372 // From the C standard draft WG14/N1256:
2373 // "Ideally, fmax would be sensitive to the sign of zero, for example
2374 // fmax(-0.0, +0.0) would return +0; however, implementation in software
2375 // might be impractical."
2377 FastMathFlags FMF = CI->getFastMathFlags();
2378 FMF.setNoSignedZeros();
2379 B.setFastMathFlags(FMF);
2380
2381 Intrinsic::ID IID = Callee->getName().startswith("fmin") ? Intrinsic::minnum
2382 : Intrinsic::maxnum;
2384 return copyFlags(
2385 *CI, B.CreateCall(F, {CI->getArgOperand(0), CI->getArgOperand(1)}));
2386}
2387
2388Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
2389 Function *LogFn = Log->getCalledFunction();
2390 StringRef LogNm = LogFn->getName();
2391 Intrinsic::ID LogID = LogFn->getIntrinsicID();
2392 Module *Mod = Log->getModule();
2393 Type *Ty = Log->getType();
2394 Value *Ret = nullptr;
2395
2396 if (UnsafeFPShrink && hasFloatVersion(Mod, LogNm))
2397 Ret = optimizeUnaryDoubleFP(Log, B, TLI, true);
2398
2399 // The earlier call must also be 'fast' in order to do these transforms.
2400 CallInst *Arg = dyn_cast<CallInst>(Log->getArgOperand(0));
2401 if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
2402 return Ret;
2403
2404 LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
2405
2406 // This is only applicable to log(), log2(), log10().
2407 if (TLI->getLibFunc(LogNm, LogLb))
2408 switch (LogLb) {
2409 case LibFunc_logf:
2410 LogID = Intrinsic::log;
2411 ExpLb = LibFunc_expf;
2412 Exp2Lb = LibFunc_exp2f;
2413 Exp10Lb = LibFunc_exp10f;
2414 PowLb = LibFunc_powf;
2415 break;
2416 case LibFunc_log:
2417 LogID = Intrinsic::log;
2418 ExpLb = LibFunc_exp;
2419 Exp2Lb = LibFunc_exp2;
2420 Exp10Lb = LibFunc_exp10;
2421 PowLb = LibFunc_pow;
2422 break;
2423 case LibFunc_logl:
2424 LogID = Intrinsic::log;
2425 ExpLb = LibFunc_expl;
2426 Exp2Lb = LibFunc_exp2l;
2427 Exp10Lb = LibFunc_exp10l;
2428 PowLb = LibFunc_powl;
2429 break;
2430 case LibFunc_log2f:
2431 LogID = Intrinsic::log2;
2432 ExpLb = LibFunc_expf;
2433 Exp2Lb = LibFunc_exp2f;
2434 Exp10Lb = LibFunc_exp10f;
2435 PowLb = LibFunc_powf;
2436 break;
2437 case LibFunc_log2:
2438 LogID = Intrinsic::log2;
2439 ExpLb = LibFunc_exp;
2440 Exp2Lb = LibFunc_exp2;
2441 Exp10Lb = LibFunc_exp10;
2442 PowLb = LibFunc_pow;
2443 break;
2444 case LibFunc_log2l:
2445 LogID = Intrinsic::log2;
2446 ExpLb = LibFunc_expl;
2447 Exp2Lb = LibFunc_exp2l;
2448 Exp10Lb = LibFunc_exp10l;
2449 PowLb = LibFunc_powl;
2450 break;
2451 case LibFunc_log10f:
2452 LogID = Intrinsic::log10;
2453 ExpLb = LibFunc_expf;
2454 Exp2Lb = LibFunc_exp2f;
2455 Exp10Lb = LibFunc_exp10f;
2456 PowLb = LibFunc_powf;
2457 break;
2458 case LibFunc_log10:
2459 LogID = Intrinsic::log10;
2460 ExpLb = LibFunc_exp;
2461 Exp2Lb = LibFunc_exp2;
2462 Exp10Lb = LibFunc_exp10;
2463 PowLb = LibFunc_pow;
2464 break;
2465 case LibFunc_log10l:
2466 LogID = Intrinsic::log10;
2467 ExpLb = LibFunc_expl;
2468 Exp2Lb = LibFunc_exp2l;
2469 Exp10Lb = LibFunc_exp10l;
2470 PowLb = LibFunc_powl;
2471 break;
2472 default:
2473 return Ret;
2474 }
2475 else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
2476 LogID == Intrinsic::log10) {
2477 if (Ty->getScalarType()->isFloatTy()) {
2478 ExpLb = LibFunc_expf;
2479 Exp2Lb = LibFunc_exp2f;
2480 Exp10Lb = LibFunc_exp10f;
2481 PowLb = LibFunc_powf;
2482 } else if (Ty->getScalarType()->isDoubleTy()) {
2483 ExpLb = LibFunc_exp;
2484 Exp2Lb = LibFunc_exp2;
2485 Exp10Lb = LibFunc_exp10;
2486 PowLb = LibFunc_pow;
2487 } else
2488 return Ret;
2489 } else
2490 return Ret;
2491
2493 B.setFastMathFlags(FastMathFlags::getFast());
2494
2495 Intrinsic::ID ArgID = Arg->getIntrinsicID();
2496 LibFunc ArgLb = NotLibFunc;
2497 TLI->getLibFunc(*Arg, ArgLb);
2498
2499 // log(pow(x,y)) -> y*log(x)
2500 AttributeList NoAttrs;
2501 if (ArgLb == PowLb || ArgID == Intrinsic::pow) {
2502 Value *LogX =
2503 Log->doesNotAccessMemory()
2504 ? B.CreateCall(Intrinsic::getDeclaration(Mod, LogID, Ty),
2505 Arg->getOperand(0), "log")
2506 : emitUnaryFloatFnCall(Arg->getOperand(0), TLI, LogNm, B, NoAttrs);
2507 Value *MulY = B.CreateFMul(Arg->getArgOperand(1), LogX, "mul");
2508 // Since pow() may have side effects, e.g. errno,
2509 // dead code elimination may not be trusted to remove it.
2510 substituteInParent(Arg, MulY);
2511 return MulY;
2512 }
2513
2514 // log(exp{,2,10}(y)) -> y*log({e,2,10})
2515 // TODO: There is no exp10() intrinsic yet.
2516 if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
2517 ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
2518 Constant *Eul;
2519 if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
2520 // FIXME: Add more precise value of e for long double.
2521 Eul = ConstantFP::get(Log->getType(), numbers::e);
2522 else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
2523 Eul = ConstantFP::get(Log->getType(), 2.0);
2524 else
2525 Eul = ConstantFP::get(Log->getType(), 10.0);
2526 Value *LogE = Log->doesNotAccessMemory()
2527 ? B.CreateCall(Intrinsic::getDeclaration(Mod, LogID, Ty),
2528 Eul, "log")
2529 : emitUnaryFloatFnCall(Eul, TLI, LogNm, B, NoAttrs);
2530 Value *MulY = B.CreateFMul(Arg->getArgOperand(0), LogE, "mul");
2531 // Since exp() may have side effects, e.g. errno,
2532 // dead code elimination may not be trusted to remove it.
2533 substituteInParent(Arg, MulY);
2534 return MulY;
2535 }
2536
2537 return Ret;
2538}
2539
2540Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
2541 Module *M = CI->getModule();
2543 Value *Ret = nullptr;
2544 // TODO: Once we have a way (other than checking for the existince of the
2545 // libcall) to tell whether our target can lower @llvm.sqrt, relax the
2546 // condition below.
2547 if (isLibFuncEmittable(M, TLI, LibFunc_sqrtf) &&
2548 (Callee->getName() == "sqrt" ||
2549 Callee->getIntrinsicID() == Intrinsic::sqrt))
2550 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2551
2552 if (!CI->isFast())
2553 return Ret;
2554
2555 Instruction *I = dyn_cast<Instruction>(CI->getArgOperand(0));
2556 if (!I || I->getOpcode() != Instruction::FMul || !I->isFast())
2557 return Ret;
2558
2559 // We're looking for a repeated factor in a multiplication tree,
2560 // so we can do this fold: sqrt(x * x) -> fabs(x);
2561 // or this fold: sqrt((x * x) * y) -> fabs(x) * sqrt(y).
2562 Value *Op0 = I->getOperand(0);
2563 Value *Op1 = I->getOperand(1);
2564 Value *RepeatOp = nullptr;
2565 Value *OtherOp = nullptr;
2566 if (Op0 == Op1) {
2567 // Simple match: the operands of the multiply are identical.
2568 RepeatOp = Op0;
2569 } else {
2570 // Look for a more complicated pattern: one of the operands is itself
2571 // a multiply, so search for a common factor in that multiply.
2572 // Note: We don't bother looking any deeper than this first level or for
2573 // variations of this pattern because instcombine's visitFMUL and/or the
2574 // reassociation pass should give us this form.
2575 Value *OtherMul0, *OtherMul1;
2576 if (match(Op0, m_FMul(m_Value(OtherMul0), m_Value(OtherMul1)))) {
2577 // Pattern: sqrt((x * y) * z)
2578 if (OtherMul0 == OtherMul1 && cast<Instruction>(Op0)->isFast()) {
2579 // Matched: sqrt((x * x) * z)
2580 RepeatOp = OtherMul0;
2581 OtherOp = Op1;
2582 }
2583 }
2584 }
2585 if (!RepeatOp)
2586 return Ret;
2587
2588 // Fast math flags for any created instructions should match the sqrt
2589 // and multiply.
2591 B.setFastMathFlags(I->getFastMathFlags());
2592
2593 // If we found a repeated factor, hoist it out of the square root and
2594 // replace it with the fabs of that factor.
2595 Type *ArgType = I->getType();
2596 Function *Fabs = Intrinsic::getDeclaration(M, Intrinsic::fabs, ArgType);
2597 Value *FabsCall = B.CreateCall(Fabs, RepeatOp, "fabs");
2598 if (OtherOp) {
2599 // If we found a non-repeated factor, we still need to get its square
2600 // root. We then multiply that by the value that was simplified out
2601 // of the square root calculation.
2602 Function *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::sqrt, ArgType);
2603 Value *SqrtCall = B.CreateCall(Sqrt, OtherOp, "sqrt");
2604 return copyFlags(*CI, B.CreateFMul(FabsCall, SqrtCall));
2605 }
2606 return copyFlags(*CI, FabsCall);
2607}
2608
2609// TODO: Generalize to handle any trig function and its inverse.
2610Value *LibCallSimplifier::optimizeTan(CallInst *CI, IRBuilderBase &B) {
2611 Module *M = CI->getModule();
2613 Value *Ret = nullptr;
2614 StringRef Name = Callee->getName();
2615 if (UnsafeFPShrink && Name == "tan" && hasFloatVersion(M, Name))
2616 Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
2617
2618 Value *Op1 = CI->getArgOperand(0);
2619 auto *OpC = dyn_cast<CallInst>(Op1);
2620 if (!OpC)
2621 return Ret;
2622
2623 // Both calls must be 'fast' in order to remove them.
2624 if (!CI->isFast() || !OpC->isFast())
2625 return Ret;
2626
2627 // tan(atan(x)) -> x
2628 // tanf(atanf(x)) -> x
2629 // tanl(atanl(x)) -> x
2630 LibFunc Func;
2631 Function *F = OpC->getCalledFunction();
2632 if (F && TLI->getLibFunc(F->getName(), Func) &&
2633 isLibFuncEmittable(M, TLI, Func) &&
2634 ((Func == LibFunc_atan && Callee->getName() == "tan") ||
2635 (Func == LibFunc_atanf && Callee->getName() == "tanf") ||
2636 (Func == LibFunc_atanl && Callee->getName() == "tanl")))
2637 Ret = OpC->getArgOperand(0);
2638 return Ret;
2639}
2640
2641static bool isTrigLibCall(CallInst *CI) {
2642 // We can only hope to do anything useful if we can ignore things like errno
2643 // and floating-point exceptions.
2644 // We already checked the prototype.
2645 return CI->doesNotThrow() && CI->doesNotAccessMemory();
2646}
2647
2648static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg,
2649 bool UseFloat, Value *&Sin, Value *&Cos,
2650 Value *&SinCos, const TargetLibraryInfo *TLI) {
2651 Module *M = OrigCallee->getParent();
2652 Type *ArgTy = Arg->getType();
2653 Type *ResTy;
2655
2656 Triple T(OrigCallee->getParent()->getTargetTriple());
2657 if (UseFloat) {
2658 Name = "__sincospif_stret";
2659
2660 assert(T.getArch() != Triple::x86 && "x86 messy and unsupported for now");
2661 // x86_64 can't use {float, float} since that would be returned in both
2662 // xmm0 and xmm1, which isn't what a real struct would do.
2663 ResTy = T.getArch() == Triple::x86_64
2664 ? static_cast<Type *>(FixedVectorType::get(ArgTy, 2))
2665 : static_cast<Type *>(StructType::get(ArgTy, ArgTy));
2666 } else {
2667 Name = "__sincospi_stret";
2668 ResTy = StructType::get(ArgTy, ArgTy);
2669 }
2670
2671 if (!isLibFuncEmittable(M, TLI, Name))
2672 return false;
2673 LibFunc TheLibFunc;
2674 TLI->getLibFunc(Name, TheLibFunc);
2676 M, *TLI, TheLibFunc, OrigCallee->getAttributes(), ResTy, ArgTy);
2677
2678 if (Instruction *ArgInst = dyn_cast<Instruction>(Arg)) {
2679 // If the argument is an instruction, it must dominate all uses so put our
2680 // sincos call there.
2681 B.SetInsertPoint(ArgInst->getParent(), ++ArgInst->getIterator());
2682 } else {
2683 // Otherwise (e.g. for a constant) the beginning of the function is as
2684 // good a place as any.
2685 BasicBlock &EntryBB = B.GetInsertBlock()->getParent()->getEntryBlock();
2686 B.SetInsertPoint(&EntryBB, EntryBB.begin());
2687 }
2688
2689 SinCos = B.CreateCall(Callee, Arg, "sincospi");
2690
2691 if (SinCos->getType()->isStructTy()) {
2692 Sin = B.CreateExtractValue(SinCos, 0, "sinpi");
2693 Cos = B.CreateExtractValue(SinCos, 1, "cospi");
2694 } else {
2695 Sin = B.CreateExtractElement(SinCos, ConstantInt::get(B.getInt32Ty(), 0),
2696 "sinpi");
2697 Cos = B.CreateExtractElement(SinCos, ConstantInt::get(B.getInt32Ty(), 1),
2698 "cospi");
2699 }
2700
2701 return true;
2702}
2703
2704Value *LibCallSimplifier::optimizeSinCosPi(CallInst *CI, bool IsSin, IRBuilderBase &B) {
2705 // Make sure the prototype is as expected, otherwise the rest of the
2706 // function is probably invalid and likely to abort.
2707 if (!isTrigLibCall(CI))
2708 return nullptr;
2709
2710 Value *Arg = CI->getArgOperand(0);
2713 SmallVector<CallInst *, 1> SinCosCalls;
2714
2715 bool IsFloat = Arg->getType()->isFloatTy();
2716
2717 // Look for all compatible sinpi, cospi and sincospi calls with the same
2718 // argument. If there are enough (in some sense) we can make the
2719 // substitution.
2720 Function *F = CI->getFunction();
2721 for (User *U : Arg->users())
2722 classifyArgUse(U, F, IsFloat, SinCalls, CosCalls, SinCosCalls);
2723
2724 // It's only worthwhile if both sinpi and cospi are actually used.
2725 if (SinCalls.empty() || CosCalls.empty())
2726 return nullptr;
2727
2728 Value *Sin, *Cos, *SinCos;
2729 if (!insertSinCosCall(B, CI->getCalledFunction(), Arg, IsFloat, Sin, Cos,
2730 SinCos, TLI))
2731 return nullptr;
2732
2733 auto replaceTrigInsts = [this](SmallVectorImpl<CallInst *> &Calls,
2734 Value *Res) {
2735 for (CallInst *C : Calls)
2736 replaceAllUsesWith(C, Res);
2737 };
2738
2739 replaceTrigInsts(SinCalls, Sin);
2740 replaceTrigInsts(CosCalls, Cos);
2741 replaceTrigInsts(SinCosCalls, SinCos);
2742
2743 return IsSin ? Sin : Cos;
2744}
2745
2746void LibCallSimplifier::classifyArgUse(
2747 Value *Val, Function *F, bool IsFloat,
2750 SmallVectorImpl<CallInst *> &SinCosCalls) {
2751 auto *CI = dyn_cast<CallInst>(Val);
2752 if (!CI || CI->use_empty())
2753 return;
2754
2755 // Don't consider calls in other functions.
2756 if (CI->getFunction() != F)
2757 return;
2758
2759 Module *M = CI->getModule();
2761 LibFunc Func;
2762 if (!Callee || !TLI->getLibFunc(*Callee, Func) ||
2763 !isLibFuncEmittable(M, TLI, Func) ||
2764 !isTrigLibCall(CI))
2765 return;
2766
2767 if (IsFloat) {
2768 if (Func == LibFunc_sinpif)
2769 SinCalls.push_back(CI);
2770 else if (Func == LibFunc_cospif)
2771 CosCalls.push_back(CI);
2772 else if (Func == LibFunc_sincospif_stret)
2773 SinCosCalls.push_back(CI);
2774 } else {
2775 if (Func == LibFunc_sinpi)
2776 SinCalls.push_back(CI);
2777 else if (Func == LibFunc_cospi)
2778 CosCalls.push_back(CI);
2779 else if (Func == LibFunc_sincospi_stret)
2780 SinCosCalls.push_back(CI);
2781 }
2782}
2783
2784//===----------------------------------------------------------------------===//
2785// Integer Library Call Optimizations
2786//===----------------------------------------------------------------------===//
2787
2788Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
2789 // All variants of ffs return int which need not be 32 bits wide.
2790 // ffs{,l,ll}(x) -> x != 0 ? (int)llvm.cttz(x)+1 : 0
2791 Type *RetType = CI->getType();
2792 Value *Op = CI->getArgOperand(0);
2793 Type *ArgType = Op->getType();
2795 Intrinsic::cttz, ArgType);
2796 Value *V = B.CreateCall(F, {Op, B.getTrue()}, "cttz");
2797 V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1));
2798 V = B.CreateIntCast(V, RetType, false);
2799
2800 Value *Cond = B.CreateICmpNE(Op, Constant::getNullValue(ArgType));
2801 return B.CreateSelect(Cond, V, ConstantInt::get(RetType, 0));
2802}
2803
2804Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
2805 // All variants of fls return int which need not be 32 bits wide.
2806 // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
2807 Value *Op = CI->getArgOperand(0);
2808 Type *ArgType = Op->getType();
2810 Intrinsic::ctlz, ArgType);
2811 Value *V = B.CreateCall(F, {Op, B.getFalse()}, "ctlz");
2812 V = B.CreateSub(ConstantInt::get(V->getType(), ArgType->getIntegerBitWidth()),
2813 V);
2814 return B.CreateIntCast(V, CI->getType(), false);
2815}
2816
2817Value *LibCallSimplifier::optimizeAbs(CallInst *CI, IRBuilderBase &B) {
2818 // abs(x) -> x <s 0 ? -x : x
2819 // The negation has 'nsw' because abs of INT_MIN is undefined.
2820 Value *X = CI->getArgOperand(0);
2821 Value *IsNeg = B.CreateIsNeg(X);
2822 Value *NegX = B.CreateNSWNeg(X, "neg");
2823 return B.CreateSelect(IsNeg, NegX, X);
2824}
2825
2826Value *LibCallSimplifier::optimizeIsDigit(CallInst *CI, IRBuilderBase &B) {
2827 // isdigit(c) -> (c-'0') <u 10
2828 Value *Op = CI->getArgOperand(0);
2829 Type *ArgType = Op->getType();
2830 Op = B.CreateSub(Op, ConstantInt::get(ArgType, '0'), "isdigittmp");
2831 Op = B.CreateICmpULT(Op, ConstantInt::get(ArgType, 10), "isdigit");
2832 return B.CreateZExt(Op, CI->getType());
2833}
2834
2835Value *LibCallSimplifier::optimizeIsAscii(CallInst *CI, IRBuilderBase &B) {
2836 // isascii(c) -> c <u 128
2837 Value *Op = CI->getArgOperand(0);
2838 Type *ArgType = Op->getType();
2839 Op = B.CreateICmpULT(Op, ConstantInt::get(ArgType, 128), "isascii");
2840 return B.CreateZExt(Op, CI->getType());
2841}
2842
2843Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) {
2844 // toascii(c) -> c & 0x7f
2845 return B.CreateAnd(CI->getArgOperand(0),
2846 ConstantInt::get(CI->getType(), 0x7F));
2847}
2848
2849// Fold calls to atoi, atol, and atoll.
2850Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) {
2851 CI->addParamAttr(0, Attribute::NoCapture);
2852
2853 StringRef Str;
2854 if (!getConstantStringInfo(CI->getArgOperand(0), Str))
2855 return nullptr;
2856
2857 return convertStrToInt(CI, Str, nullptr, 10, /*AsSigned=*/true, B);
2858}
2859
2860// Fold calls to strtol, strtoll, strtoul, and strtoull.
2861Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
2862 bool AsSigned) {
2863 Value *EndPtr = CI->getArgOperand(1);
2864 if (isa<ConstantPointerNull>(EndPtr)) {
2865 // With a null EndPtr, this function won't capture the main argument.
2866 // It would be readonly too, except that it still may write to errno.
2867 CI->addParamAttr(0, Attribute::NoCapture);
2868 EndPtr = nullptr;
2869 } else if (!isKnownNonZero(EndPtr, DL))
2870 return nullptr;
2871
2872 StringRef Str;
2873 if (!getConstantStringInfo(CI->getArgOperand(0), Str))
2874 return nullptr;
2875
2876 if (ConstantInt *CInt = dyn_cast<ConstantInt>(CI->getArgOperand(2))) {
2877 return convertStrToInt(CI, Str, EndPtr, CInt->getSExtValue(), AsSigned, B);
2878 }
2879
2880 return nullptr;
2881}
2882
2883//===----------------------------------------------------------------------===//
2884// Formatting and IO Library Call Optimizations
2885//===----------------------------------------------------------------------===//
2886
2887static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg);
2888
2889Value *LibCallSimplifier::optimizeErrorReporting(CallInst *CI, IRBuilderBase &B,
2890 int StreamArg) {
2892 // Error reporting calls should be cold, mark them as such.
2893 // This applies even to non-builtin calls: it is only a hint and applies to
2894 // functions that the frontend might not understand as builtins.
2895
2896 // This heuristic was suggested in:
2897 // Improving Static Branch Prediction in a Compiler
2898 // Brian L. Deitrich, Ben-Chung Cheng, Wen-mei W. Hwu
2899 // Proceedings of PACT'98, Oct. 1998, IEEE
2900 if (!CI->hasFnAttr(Attribute::Cold) &&
2901 isReportingError(Callee, CI, StreamArg)) {
2902 CI->addFnAttr(Attribute::Cold);
2903 }
2904
2905 return nullptr;
2906}
2907
2908static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg) {
2909 if (!Callee || !Callee->isDeclaration())
2910 return false;
2911
2912 if (StreamArg < 0)
2913 return true;
2914
2915 // These functions might be considered cold, but only if their stream
2916 // argument is stderr.
2917
2918 if (StreamArg >= (int)CI->arg_size())
2919 return false;
2920 LoadInst *LI = dyn_cast<LoadInst>(CI->getArgOperand(StreamArg));
2921 if (!LI)
2922 return false;
2923 GlobalVariable *GV = dyn_cast<GlobalVariable>(LI->getPointerOperand());
2924 if (!GV || !GV->isDeclaration())
2925 return false;
2926 return GV->getName() == "stderr";
2927}
2928
2929Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
2930 // Check for a fixed format string.
2931 StringRef FormatStr;
2932 if (!getConstantStringInfo(CI->getArgOperand(0), FormatStr))
2933 return nullptr;
2934
2935 // Empty format string -> noop.
2936 if (FormatStr.empty()) // Tolerate printf's declared void.
2937 return CI->use_empty() ? (Value *)CI : ConstantInt::get(CI->getType(), 0);
2938
2939 // Do not do any of the following transformations if the printf return value
2940 // is used, in general the printf return value is not compatible with either
2941 // putchar() or puts().
2942 if (!CI->use_empty())
2943 return nullptr;
2944
2945 Type *IntTy = CI->getType();
2946 // printf("x") -> putchar('x'), even for "%" and "%%".
2947 if (FormatStr.size() == 1 || FormatStr == "%%") {
2948 // Convert the character to unsigned char before passing it to putchar
2949 // to avoid host-specific sign extension in the IR. Putchar converts
2950 // it to unsigned char regardless.
2951 Value *IntChar = ConstantInt::get(IntTy, (unsigned char)FormatStr[0]);
2952 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
2953 }
2954
2955 // Try to remove call or emit putchar/puts.
2956 if (FormatStr == "%s" && CI->arg_size() > 1) {
2957 StringRef OperandStr;
2958 if (!getConstantStringInfo(CI->getOperand(1), OperandStr))
2959 return nullptr;
2960 // printf("%s", "") --> NOP
2961 if (OperandStr.empty())
2962 return (Value *)CI;
2963 // printf("%s", "a") --> putchar('a')
2964 if (OperandStr.size() == 1) {
2965 // Convert the character to unsigned char before passing it to putchar
2966 // to avoid host-specific sign extension in the IR. Putchar converts
2967 // it to unsigned char regardless.
2968 Value *IntChar = ConstantInt::get(IntTy, (unsigned char)OperandStr[0]);
2969 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
2970 }
2971 // printf("%s", str"\n") --> puts(str)
2972 if (OperandStr.back() == '\n') {
2973 OperandStr = OperandStr.drop_back();
2974 Value *GV = B.CreateGlobalString(OperandStr, "str");
2975 return copyFlags(*CI, emitPutS(GV, B, TLI));
2976 }
2977 return nullptr;
2978 }
2979
2980 // printf("foo\n") --> puts("foo")
2981 if (FormatStr.back() == '\n' &&
2982 !FormatStr.contains('%')) { // No format characters.
2983 // Create a string literal with no \n on it. We expect the constant merge
2984 // pass to be run after this pass, to merge duplicate strings.
2985 FormatStr = FormatStr.drop_back();
2986 Value *GV = B.CreateGlobalString(FormatStr, "str");
2987 return copyFlags(*CI, emitPutS(GV, B, TLI));
2988 }
2989
2990 // Optimize specific format strings.
2991 // printf("%c", chr) --> putchar(chr)
2992 if (FormatStr == "%c" && CI->arg_size() > 1 &&
2993 CI->getArgOperand(1)->getType()->isIntegerTy()) {
2994 // Convert the argument to the type expected by putchar, i.e., int, which
2995 // need not be 32 bits wide but which is the same as printf's return type.
2996 Value *IntChar = B.CreateIntCast(CI->getArgOperand(1), IntTy, false);
2997 return copyFlags(*CI, emitPutChar(IntChar, B, TLI));
2998 }
2999
3000 // printf("%s\n", str) --> puts(str)
3001 if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
3002 CI->getArgOperand(1)->getType()->isPointerTy())
3003 return copyFlags(*CI, emitPutS(CI->getArgOperand(1), B, TLI));
3004 return nullptr;
3005}
3006
3007Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilderBase &B) {
3008
3009 Module *M = CI->getModule();
3011 FunctionType *FT = Callee->getFunctionType();
3012 if (Value *V = optimizePrintFString(CI, B)) {
3013 return V;
3014 }
3015
3017
3018 // printf(format, ...) -> iprintf(format, ...) if no floating point
3019 // arguments.
3020 if (isLibFuncEmittable(M, TLI, LibFunc_iprintf) &&
3022 FunctionCallee IPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_iprintf, FT,
3023 Callee->getAttributes());
3024 CallInst *New = cast<CallInst>(CI->clone());
3025 New->setCalledFunction(IPrintFFn);
3026 B.Insert(New);
3027 return New;
3028 }
3029
3030 // printf(format, ...) -> __small_printf(format, ...) if no 128-bit floating point
3031 // arguments.
3032 if (isLibFuncEmittable(M, TLI, LibFunc_small_printf) &&
3033 !callHasFP128Argument(CI)) {
3034 auto SmallPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_small_printf, FT,
3035 Callee->getAttributes());
3036 CallInst *New = cast<CallInst>(CI->clone());
3037 New->setCalledFunction(SmallPrintFFn);
3038 B.Insert(New);
3039 return New;
3040 }
3041
3042 return nullptr;
3043}
3044
3045Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
3046 IRBuilderBase &B) {
3047 // Check for a fixed format string.
3048 StringRef FormatStr;
3049 if (!getConstantStringInfo(CI->getArgOperand(1), FormatStr))
3050 return nullptr;
3051
3052 // If we just have a format string (nothing else crazy) transform it.
3053 Value *Dest = CI->getArgOperand(0);
3054 if (CI->arg_size() == 2) {
3055 // Make sure there's no % in the constant array. We could try to handle
3056 // %% -> % in the future if we cared.
3057 if (FormatStr.contains('%'))
3058 return nullptr; // we found a format specifier, bail out.
3059
3060 // sprintf(str, fmt) -> llvm.memcpy(align 1 str, align 1 fmt, strlen(fmt)+1)
3061 B.CreateMemCpy(
3062 Dest, Align(1), CI->getArgOperand(1), Align(1),
3064 FormatStr.size() + 1)); // Copy the null byte.
3065 return ConstantInt::get(CI->getType(), FormatStr.size());
3066 }
3067
3068 // The remaining optimizations require the format string to be "%s" or "%c"
3069 // and have an extra operand.
3070 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3071 return nullptr;
3072
3073 // Decode the second character of the format string.
3074 if (FormatStr[1] == 'c') {
3075 // sprintf(dst, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3076 if (!CI->getArgOperand(2)->getType()->isIntegerTy())
3077 return nullptr;
3078 Value *V = B.CreateTrunc(CI->getArgOperand(2), B.getInt8Ty(), "char");
3079 Value *Ptr = Dest;
3080 B.CreateStore(V, Ptr);
3081 Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Ptr, B.getInt32(1), "nul");
3082 B.CreateStore(B.getInt8(0), Ptr);
3083
3084 return ConstantInt::get(CI->getType(), 1);
3085 }
3086
3087 if (FormatStr[1] == 's') {
3088 // sprintf(dest, "%s", str) -> llvm.memcpy(align 1 dest, align 1 str,
3089 // strlen(str)+1)
3090 if (!CI->getArgOperand(2)->getType()->isPointerTy())
3091 return nullptr;
3092
3093 if (CI->use_empty())
3094 // sprintf(dest, "%s", str) -> strcpy(dest, str)
3095 return copyFlags(*CI, emitStrCpy(Dest, CI->getArgOperand(2), B, TLI));
3096
3097 uint64_t SrcLen = GetStringLength(CI->getArgOperand(2));
3098 if (SrcLen) {
3099 B.CreateMemCpy(
3100 Dest, Align(1), CI->getArgOperand(2), Align(1),
3101 ConstantInt::get(DL.getIntPtrType(CI->getContext()), SrcLen));
3102 // Returns total number of characters written without null-character.
3103 return ConstantInt::get(CI->getType(), SrcLen - 1);
3104 } else if (Value *V = emitStpCpy(Dest, CI->getArgOperand(2), B, TLI)) {
3105 // sprintf(dest, "%s", str) -> stpcpy(dest, str) - dest
3106 Value *PtrDiff = B.CreatePtrDiff(B.getInt8Ty(), V, Dest);
3107 return B.CreateIntCast(PtrDiff, CI->getType(), false);
3108 }
3109
3110 bool OptForSize = CI->getFunction()->hasOptSize() ||
3111 llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
3113 if (OptForSize)
3114 return nullptr;
3115
3116 Value *Len = emitStrLen(CI->getArgOperand(2), B, DL, TLI);
3117 if (!Len)
3118 return nullptr;
3119 Value *IncLen =
3120 B.CreateAdd(Len, ConstantInt::get(Len->getType(), 1), "leninc");
3121 B.CreateMemCpy(Dest, Align(1), CI->getArgOperand(2), Align(1), IncLen);
3122
3123 // The sprintf result is the unincremented number of bytes in the string.
3124 return B.CreateIntCast(Len, CI->getType(), false);
3125 }
3126 return nullptr;
3127}
3128
3129Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilderBase &B) {
3130 Module *M = CI->getModule();
3132 FunctionType *FT = Callee->getFunctionType();
3133 if (Value *V = optimizeSPrintFString(CI, B)) {
3134 return V;
3135 }
3136
3138
3139 // sprintf(str, format, ...) -> siprintf(str, format, ...) if no floating
3140 // point arguments.
3141 if (isLibFuncEmittable(M, TLI, LibFunc_siprintf) &&
3143 FunctionCallee SIPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_siprintf,
3144 FT, Callee->getAttributes());
3145 CallInst *New = cast<CallInst>(CI->clone());
3146 New->setCalledFunction(SIPrintFFn);
3147 B.Insert(New);
3148 return New;
3149 }
3150
3151 // sprintf(str, format, ...) -> __small_sprintf(str, format, ...) if no 128-bit
3152 // floating point arguments.
3153 if (isLibFuncEmittable(M, TLI, LibFunc_small_sprintf) &&
3154 !callHasFP128Argument(CI)) {
3155 auto SmallSPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_small_sprintf, FT,
3156 Callee->getAttributes());
3157 CallInst *New = cast<CallInst>(CI->clone());
3158 New->setCalledFunction(SmallSPrintFFn);
3159 B.Insert(New);
3160 return New;
3161 }
3162
3163 return nullptr;
3164}
3165
3166// Transform an snprintf call CI with the bound N to format the string Str
3167// either to a call to memcpy, or to single character a store, or to nothing,
3168// and fold the result to a constant. A nonnull StrArg refers to the string
3169// argument being formatted. Otherwise the call is one with N < 2 and
3170// the "%c" directive to format a single character.
3171Value *LibCallSimplifier::emitSnPrintfMemCpy(CallInst *CI, Value *StrArg,
3172 StringRef Str, uint64_t N,
3173 IRBuilderBase &B) {
3174 assert(StrArg || (N < 2 && Str.size() == 1));
3175
3176 unsigned IntBits = TLI->getIntSize();
3177 uint64_t IntMax = maxIntN(IntBits);
3178 if (Str.size() > IntMax)
3179 // Bail if the string is longer than INT_MAX. POSIX requires
3180 // implementations to set errno to EOVERFLOW in this case, in
3181 // addition to when N is larger than that (checked by the caller).
3182 return nullptr;
3183
3184 Value *StrLen = ConstantInt::get(CI->getType(), Str.size());
3185 if (N == 0)
3186 return StrLen;
3187
3188 // Set to the number of bytes to copy fron StrArg which is also
3189 // the offset of the terinating nul.
3190 uint64_t NCopy;
3191 if (N > Str.size())
3192 // Copy the full string, including the terminating nul (which must
3193 // be present regardless of the bound).
3194 NCopy = Str.size() + 1;
3195 else
3196 NCopy = N - 1;
3197
3198 Value *DstArg = CI->getArgOperand(0);
3199 if (NCopy && StrArg)
3200 // Transform the call to lvm.memcpy(dst, fmt, N).
3201 copyFlags(
3202 *CI,
3203 B.CreateMemCpy(
3204 DstArg, Align(1), StrArg, Align(1),
3205 ConstantInt::get(DL.getIntPtrType(CI->getContext()), NCopy)));
3206
3207 if (N > Str.size())
3208 // Return early when the whole format string, including the final nul,
3209 // has been copied.
3210 return StrLen;
3211
3212 // Otherwise, when truncating the string append a terminating nul.
3213 Type *Int8Ty = B.getInt8Ty();
3214 Value *NulOff = B.getIntN(IntBits, NCopy);
3215 Value *DstEnd = B.CreateInBoundsGEP(Int8Ty, DstArg, NulOff, "endptr");
3216 B.CreateStore(ConstantInt::get(Int8Ty, 0), DstEnd);
3217 return StrLen;
3218}
3219
3220Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
3221 IRBuilderBase &B) {
3222 // Check for size
3223 ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
3224 if (!Size)
3225 return nullptr;
3226
3227 uint64_t N = Size->getZExtValue();
3228 uint64_t IntMax = maxIntN(TLI->getIntSize());
3229 if (N > IntMax)
3230 // Bail if the bound exceeds INT_MAX. POSIX requires implementations
3231 // to set errno to EOVERFLOW in this case.
3232 return nullptr;
3233
3234 Value *DstArg = CI->getArgOperand(0);
3235 Value *FmtArg = CI->getArgOperand(2);
3236
3237 // Check for a fixed format string.
3238 StringRef FormatStr;
3239 if (!getConstantStringInfo(FmtArg, FormatStr))
3240 return nullptr;
3241
3242 // If we just have a format string (nothing else crazy) transform it.
3243 if (CI->arg_size() == 3) {
3244 if (FormatStr.contains('%'))
3245 // Bail if the format string contains a directive and there are
3246 // no arguments. We could handle "%%" in the future.
3247 return nullptr;
3248
3249 return emitSnPrintfMemCpy(CI, FmtArg, FormatStr, N, B);
3250 }
3251
3252 // The remaining optimizations require the format string to be "%s" or "%c"
3253 // and have an extra operand.
3254 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() != 4)
3255 return nullptr;
3256
3257 // Decode the second character of the format string.
3258 if (FormatStr[1] == 'c') {
3259 if (N <= 1) {
3260 // Use an arbitary string of length 1 to transform the call into
3261 // either a nul store (N == 1) or a no-op (N == 0) and fold it
3262 // to one.
3263 StringRef CharStr("*");
3264 return emitSnPrintfMemCpy(CI, nullptr, CharStr, N, B);
3265 }
3266
3267 // snprintf(dst, size, "%c", chr) --> *(i8*)dst = chr; *((i8*)dst+1) = 0
3268 if (!CI->getArgOperand(3)->getType()->isIntegerTy())
3269 return nullptr;
3270 Value *V = B.CreateTrunc(CI->getArgOperand(3), B.getInt8Ty(), "char");
3271 Value *Ptr = DstArg;
3272 B.CreateStore(V, Ptr);
3273 Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Ptr, B.getInt32(1), "nul");
3274 B.CreateStore(B.getInt8(0), Ptr);
3275 return ConstantInt::get(CI->getType(), 1);
3276 }
3277
3278 if (FormatStr[1] != 's')
3279 return nullptr;
3280
3281 Value *StrArg = CI->getArgOperand(3);
3282 // snprintf(dest, size, "%s", str) to llvm.memcpy(dest, str, len+1, 1)
3283 StringRef Str;
3284 if (!getConstantStringInfo(StrArg, Str))
3285 return nullptr;
3286
3287 return emitSnPrintfMemCpy(CI, StrArg, Str, N, B);
3288}
3289
3290Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
3291 if (Value *V = optimizeSnPrintFString(CI, B)) {
3292 return V;
3293 }
3294
3295 if (isKnownNonZero(CI->getOperand(1), DL))
3297 return nullptr;
3298}
3299
3300Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
3301 IRBuilderBase &B) {
3302 optimizeErrorReporting(CI, B, 0);
3303
3304 // All the optimizations depend on the format string.
3305 StringRef FormatStr;
3306 if (!getConstantStringInfo(CI->getArgOperand(1), FormatStr))
3307 return nullptr;
3308
3309 // Do not do any of the following transformations if the fprintf return
3310 // value is used, in general the fprintf return value is not compatible
3311 // with fwrite(), fputc() or fputs().
3312 if (!CI->use_empty())
3313 return nullptr;
3314
3315 // fprintf(F, "foo") --> fwrite("foo", 3, 1, F)
3316 if (CI->arg_size() == 2) {
3317 // Could handle %% -> % if we cared.
3318 if (FormatStr.contains('%'))
3319 return nullptr; // We found a format specifier.
3320
3321 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
3322 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
3323 return copyFlags(
3324 *CI, emitFWrite(CI->getArgOperand(1),
3325 ConstantInt::get(SizeTTy, FormatStr.size()),
3326 CI->getArgOperand(0), B, DL, TLI));
3327 }
3328
3329 // The remaining optimizations require the format string to be "%s" or "%c"
3330 // and have an extra operand.
3331 if (FormatStr.size() != 2 || FormatStr[0] != '%' || CI->arg_size() < 3)
3332 return nullptr;
3333
3334 // Decode the second character of the format string.
3335 if (FormatStr[1] == 'c') {
3336 // fprintf(F, "%c", chr) --> fputc((int)chr, F)
3337 if (!CI->getArgOperand(2)->getType()->isIntegerTy())
3338 return nullptr;
3339 Type *IntTy = B.getIntNTy(TLI->getIntSize());
3340 Value *V = B.CreateIntCast(CI->getArgOperand(2), IntTy, /*isSigned*/ true,
3341 "chari");
3342 return copyFlags(*CI, emitFPutC(V, CI->getArgOperand(0), B, TLI));
3343 }
3344
3345 if (FormatStr[1] == 's') {
3346 // fprintf(F, "%s", str) --> fputs(str, F)
3347 if (!CI->getArgOperand(2)->getType()->isPointerTy())
3348 return nullptr;
3349 return copyFlags(
3350 *CI, emitFPutS(CI->getArgOperand(2), CI->getArgOperand(0), B, TLI));
3351 }
3352 return nullptr;
3353}
3354
3355Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilderBase &B) {
3356 Module *M = CI->getModule();
3358 FunctionType *FT = Callee->getFunctionType();
3359 if (Value *V = optimizeFPrintFString(CI, B)) {
3360 return V;
3361 }
3362
3363 // fprintf(stream, format, ...) -> fiprintf(stream, format, ...) if no
3364 // floating point arguments.
3365 if (isLibFuncEmittable(M, TLI, LibFunc_fiprintf) &&
3367 FunctionCallee FIPrintFFn = getOrInsertLibFunc(M, *TLI, LibFunc_fiprintf,
3368 FT, Callee->getAttributes());
3369 CallInst *New = cast<CallInst>(CI->clone());
3370 New->setCalledFunction(FIPrintFFn);
3371 B.Insert(New);
3372 return New;
3373 }
3374
3375 // fprintf(stream, format, ...) -> __small_fprintf(stream, format, ...) if no
3376 // 128-bit floating point arguments.
3377 if (isLibFuncEmittable(M, TLI, LibFunc_small_fprintf) &&
3378 !callHasFP128Argument(CI)) {
3379 auto SmallFPrintFFn =
3380 getOrInsertLibFunc(M, *TLI, LibFunc_small_fprintf, FT,
3381 Callee->getAttributes());
3382 CallInst *New = cast<CallInst>(CI->clone());
3383 New->setCalledFunction(SmallFPrintFFn);
3384 B.Insert(New);
3385 return New;
3386 }
3387
3388 return nullptr;
3389}
3390
3391Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilderBase &B) {
3392 optimizeErrorReporting(CI, B, 3);
3393
3394 // Get the element size and count.
3395 ConstantInt *SizeC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
3396 ConstantInt *CountC = dyn_cast<ConstantInt>(CI->getArgOperand(2));
3397 if (SizeC && CountC) {
3398 uint64_t Bytes = SizeC->getZExtValue() * CountC->getZExtValue();
3399
3400 // If this is writing zero records, remove the call (it's a noop).
3401 if (Bytes == 0)
3402 return ConstantInt::get(CI->getType(), 0);
3403
3404 // If this is writing one byte, turn it into fputc.
3405 // This optimisation is only valid, if the return value is unused.
3406 if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
3407 Value *Char = B.CreateLoad(B.getInt8Ty(), CI->getArgOperand(0), "char");
3408 Type *IntTy = B.getIntNTy(TLI->getIntSize());
3409 Value *Cast = B.CreateIntCast(Char, IntTy, /*isSigned*/ true, "chari");
3410 Value *NewCI = emitFPutC(Cast, CI->getArgOperand(3), B, TLI);
3411 return NewCI ? ConstantInt::get(CI->getType(), 1) : nullptr;
3412 }
3413 }
3414
3415 return nullptr;
3416}
3417
3418Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
3419 optimizeErrorReporting(CI, B, 1);
3420
3421 // Don't rewrite fputs to fwrite when optimising for size because fwrite
3422 // requires more arguments and thus extra MOVs are required.
3423 bool OptForSize = CI->getFunction()->hasOptSize() ||
3424 llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI,
3426 if (OptForSize)
3427 return nullptr;
3428
3429 // We can't optimize if return value is used.
3430 if (!CI->use_empty())
3431 return nullptr;
3432
3433 // fputs(s,F) --> fwrite(s,strlen(s),1,F)
3435 if (!Len)
3436 return nullptr;
3437
3438 // Known to have no uses (see above).
3439 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
3440 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
3441 return copyFlags(
3442 *CI,
3444 ConstantInt::get(SizeTTy, Len - 1),
3445 CI->getArgOperand(1), B, DL, TLI));
3446}
3447
3448Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
3450 if (!CI->use_empty())
3451 return nullptr;
3452
3453 // Check for a constant string.
3454 // puts("") -> putchar('\n')
3455 StringRef Str;
3456 if (getConstantStringInfo(CI->getArgOperand(0), Str) && Str.empty()) {
3457 // putchar takes an argument of the same type as puts returns, i.e.,
3458 // int, which need not be 32 bits wide.
3459 Type *IntTy = CI->getType();
3460 return copyFlags(*CI, emitPutChar(ConstantInt::get(IntTy, '\n'), B, TLI));
3461 }
3462
3463 return nullptr;
3464}
3465
3466Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
3467 // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
3468 return copyFlags(*CI, B.CreateMemMove(CI->getArgOperand(1), Align(1),
3469 CI->getArgOperand(0), Align(1),
3470 CI->getArgOperand(2)));
3471}
3472
3473bool LibCallSimplifier::hasFloatVersion(const Module *M, StringRef FuncName) {
3474 SmallString<20> FloatFuncName = FuncName;
3475 FloatFuncName += 'f';
3476 return isLibFuncEmittable(M, TLI, FloatFuncName);
3477}
3478
3479Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
3480 IRBuilderBase &Builder) {
3481 Module *M = CI->getModule();
3482 LibFunc Func;
3484 // Check for string/memory library functions.
3485 if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
3486 // Make sure we never change the calling convention.
3487 assert(
3488 (ignoreCallingConv(Func) ||
3490 "Optimizing string/memory libcall would change the calling convention");
3491 switch (Func) {
3492 case LibFunc_strcat:
3493 return optimizeStrCat(CI, Builder);
3494 case LibFunc_strncat:
3495 return optimizeStrNCat(CI, Builder);
3496 case LibFunc_strchr:
3497 return optimizeStrChr(CI, Builder);
3498 case LibFunc_strrchr:
3499 return optimizeStrRChr(CI, Builder);
3500 case LibFunc_strcmp:
3501 return optimizeStrCmp(CI, Builder);
3502 case LibFunc_strncmp:
3503 return optimizeStrNCmp(CI, Builder);
3504 case LibFunc_strcpy:
3505 return optimizeStrCpy(CI, Builder);
3506 case LibFunc_stpcpy:
3507 return optimizeStpCpy(CI, Builder);
3508 case LibFunc_strlcpy:
3509 return optimizeStrLCpy(CI, Builder);
3510 case LibFunc_stpncpy:
3511 return optimizeStringNCpy(CI, /*RetEnd=*/true, Builder);
3512 case LibFunc_strncpy:
3513 return optimizeStringNCpy(CI, /*RetEnd=*/false, Builder);
3514 case LibFunc_strlen:
3515 return optimizeStrLen(CI, Builder);
3516 case LibFunc_strnlen:
3517 return optimizeStrNLen(CI, Builder);
3518 case LibFunc_strpbrk:
3519 return optimizeStrPBrk(CI, Builder);
3520 case LibFunc_strndup:
3521 return optimizeStrNDup(CI, Builder);
3522 case LibFunc_strtol:
3523 case LibFunc_strtod:
3524 case LibFunc_strtof:
3525 case LibFunc_strtoul:
3526 case LibFunc_strtoll:
3527 case LibFunc_strtold:
3528 case LibFunc_strtoull:
3529 return optimizeStrTo(CI, Builder);
3530 case LibFunc_strspn:
3531 return optimizeStrSpn(CI, Builder);
3532 case LibFunc_strcspn:
3533 return optimizeStrCSpn(CI, Builder);
3534 case LibFunc_strstr:
3535 return optimizeStrStr(CI, Builder);
3536 case LibFunc_memchr:
3537 return optimizeMemChr(CI, Builder);
3538 case LibFunc_memrchr:
3539 return optimizeMemRChr(CI, Builder);
3540 case LibFunc_bcmp:
3541 return optimizeBCmp(CI, Builder);
3542 case LibFunc_memcmp:
3543 return optimizeMemCmp(CI, Builder);
3544 case LibFunc_memcpy:
3545 return optimizeMemCpy(CI, Builder);
3546 case LibFunc_memccpy:
3547 return optimizeMemCCpy(CI, Builder);
3548 case LibFunc_mempcpy:
3549 return optimizeMemPCpy(CI, Builder);
3550 case LibFunc_memmove:
3551 return optimizeMemMove(CI, Builder);
3552 case LibFunc_memset:
3553 return optimizeMemSet(CI, Builder);
3554 case LibFunc_realloc:
3555 return optimizeRealloc(CI, Builder);
3556 case LibFunc_wcslen:
3557 return optimizeWcslen(CI, Builder);
3558 case LibFunc_bcopy:
3559 return optimizeBCopy(CI, Builder);
3560 case LibFunc_Znwm:
3561 case LibFunc_ZnwmRKSt9nothrow_t:
3562 case LibFunc_ZnwmSt11align_val_t:
3563 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
3564 case LibFunc_Znam:
3565 case LibFunc_ZnamRKSt9nothrow_t:
3566 case LibFunc_ZnamSt11align_val_t:
3567 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
3568 return optimizeNew(CI, Builder, Func);
3569 default:
3570 break;
3571 }
3572 }
3573 return nullptr;
3574}
3575
3576Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
3577 LibFunc Func,
3578 IRBuilderBase &Builder) {
3579 const Module *M = CI->getModule();
3580
3581 // Don't optimize calls that require strict floating point semantics.
3582 if (CI->isStrictFP())
3583 return nullptr;
3584
3585 if (Value *V = optimizeTrigReflections(CI, Func, Builder))
3586 return V;
3587
3588 switch (Func) {
3589 case LibFunc_sinpif:
3590 case LibFunc_sinpi:
3591 return optimizeSinCosPi(CI, /*IsSin*/true, Builder);
3592 case LibFunc_cospif:
3593 case LibFunc_cospi:
3594 return optimizeSinCosPi(CI, /*IsSin*/false, Builder);
3595 case LibFunc_powf:
3596 case LibFunc_pow:
3597 case LibFunc_powl:
3598 return optimizePow(CI, Builder);
3599 case LibFunc_exp2l:
3600 case LibFunc_exp2:
3601 case LibFunc_exp2f:
3602 return optimizeExp2(CI, Builder);
3603 case LibFunc_fabsf:
3604 case LibFunc_fabs:
3605 case LibFunc_fabsl:
3606 return replaceUnaryCall(CI, Builder, Intrinsic::fabs);
3607 case LibFunc_sqrtf:
3608 case LibFunc_sqrt:
3609 case LibFunc_sqrtl:
3610 return optimizeSqrt(CI, Builder);
3611 case LibFunc_logf:
3612 case LibFunc_log:
3613 case LibFunc_logl:
3614 case LibFunc_log10f:
3615 case LibFunc_log10:
3616 case LibFunc_log10l:
3617 case LibFunc_log1pf:
3618 case LibFunc_log1p:
3619 case LibFunc_log1pl:
3620 case LibFunc_log2f:
3621 case LibFunc_log2:
3622 case LibFunc_log2l:
3623 case LibFunc_logbf:
3624 case LibFunc_logb:
3625 case LibFunc_logbl:
3626 return optimizeLog(CI, Builder);
3627 case LibFunc_tan:
3628 case LibFunc_tanf:
3629 case LibFunc_tanl:
3630 return optimizeTan(CI, Builder);
3631 case LibFunc_ceil:
3632 return replaceUnaryCall(CI, Builder, Intrinsic::ceil);
3633 case LibFunc_floor:
3634 return replaceUnaryCall(CI, Builder, Intrinsic::floor);
3635 case LibFunc_round:
3636 return replaceUnaryCall(CI, Builder, Intrinsic::round);
3637 case LibFunc_roundeven:
3638 return replaceUnaryCall(CI, Builder, Intrinsic::roundeven);
3639 case LibFunc_nearbyint:
3640 return replaceUnaryCall(CI, Builder, Intrinsic::nearbyint);
3641 case LibFunc_rint:
3642 return replaceUnaryCall(CI, Builder, Intrinsic::rint);
3643 case LibFunc_trunc:
3644 return replaceUnaryCall(CI, Builder, Intrinsic::trunc);
3645 case LibFunc_acos:
3646 case LibFunc_acosh:
3647 case LibFunc_asin:
3648 case LibFunc_asinh:
3649 case LibFunc_atan:
3650 case LibFunc_atanh:
3651 case LibFunc_cbrt:
3652 case LibFunc_cosh:
3653 case LibFunc_exp:
3654 case LibFunc_exp10:
3655 case LibFunc_expm1:
3656 case LibFunc_cos:
3657 case LibFunc_sin:
3658 case LibFunc_sinh:
3659 case LibFunc_tanh:
3660 if (UnsafeFPShrink && hasFloatVersion(M, CI->getCalledFunction()->getName()))
3661 return optimizeUnaryDoubleFP(CI, Builder, TLI, true);
3662 return nullptr;
3663 case LibFunc_copysign:
3664 if (hasFloatVersion(M, CI->getCalledFunction()->getName()))
3665 return optimizeBinaryDoubleFP(CI, Builder, TLI);
3666 return nullptr;
3667 case LibFunc_fminf:
3668 case LibFunc_fmin:
3669 case LibFunc_fminl:
3670 case LibFunc_fmaxf:
3671 case LibFunc_fmax:
3672 case LibFunc_fmaxl:
3673 return optimizeFMinFMax(CI, Builder);
3674 case LibFunc_cabs:
3675 case LibFunc_cabsf:
3676 case LibFunc_cabsl:
3677 return optimizeCAbs(CI, Builder);
3678 default:
3679 return nullptr;
3680 }
3681}
3682
3684 Module *M = CI->getModule();
3685 assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
3686
3687 // TODO: Split out the code below that operates on FP calls so that
3688 // we can all non-FP calls with the StrictFP attribute to be
3689 // optimized.
3690 if (CI->isNoBuiltin())
3691 return nullptr;
3692
3693 LibFunc Func;
3694 Function *Callee = CI->getCalledFunction();
3695 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
3696
3698 CI->getOperandBundlesAsDefs(OpBundles);
3699
3701 Builder.setDefaultOperandBundles(OpBundles);
3702
3703 // Command-line parameter overrides instruction attribute.
3704 // This can't be moved to optimizeFloatingPointLibCall() because it may be
3705 // used by the intrinsic optimizations.
3706 if (EnableUnsafeFPShrink.getNumOccurrences() > 0)
3707 UnsafeFPShrink = EnableUnsafeFPShrink;
3708 else if (isa<FPMathOperator>(CI) && CI->isFast())
3709 UnsafeFPShrink = true;
3710
3711 // First, check for intrinsics.
3712 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) {
3713 if (!IsCallingConvC)
3714 return nullptr;
3715 // The FP intrinsics have corresponding constrained versions so we don't
3716 // need to check for the StrictFP attribute here.
3717 switch (II->getIntrinsicID()) {
3718 case Intrinsic::pow:
3719 return optimizePow(CI, Builder);
3720 case Intrinsic::exp2:
3721 return optimizeExp2(CI, Builder);
3722 case Intrinsic::log:
3723 case Intrinsic::log2:
3724 case Intrinsic::log10:
3725 return optimizeLog(CI, Builder);
3726 case Intrinsic::sqrt:
3727 return optimizeSqrt(CI, Builder);
3728 case Intrinsic::memset:
3729 return optimizeMemSet(CI, Builder);
3730 case Intrinsic::memcpy:
3731 return optimizeMemCpy(CI, Builder);
3732 case Intrinsic::memmove:
3733 return optimizeMemMove(CI, Builder);
3734 default:
3735 return nullptr;
3736 }
3737 }
3738
3739 // Also try to simplify calls to fortified library functions.
3740 if (Value *SimplifiedFortifiedCI =
3741 FortifiedSimplifier.optimizeCall(CI, Builder)) {
3742 // Try to further simplify the result.
3743 CallInst *SimplifiedCI = dyn_cast<CallInst>(SimplifiedFortifiedCI);
3744 if (SimplifiedCI && SimplifiedCI->getCalledFunction()) {
3745 // Ensure that SimplifiedCI's uses are complete, since some calls have
3746 // their uses analyzed.
3747 replaceAllUsesWith(CI, SimplifiedCI);
3748
3749 // Set insertion point to SimplifiedCI to guarantee we reach all uses
3750 // we might replace later on.
3752 Builder.SetInsertPoint(SimplifiedCI);
3753 if (Value *V = optimizeStringMemoryLibCall(SimplifiedCI, Builder)) {
3754 // If we were able to further simplify, remove the now redundant call.
3755 substituteInParent(SimplifiedCI, V);
3756 return V;
3757 }
3758 }
3759 return SimplifiedFortifiedCI;
3760 }
3761
3762 // Then check for known library functions.
3763 if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
3764 // We never change the calling convention.
3765 if (!ignoreCallingConv(Func) && !IsCallingConvC)
3766 return nullptr;
3767 if (Value *V = optimizeStringMemoryLibCall(CI, Builder))
3768 return V;
3769 if (Value *V = optimizeFloatingPointLibCall(CI, Func, Builder))
3770 return V;
3771 switch (Func) {
3772 case LibFunc_ffs:
3773 case LibFunc_ffsl:
3774 case LibFunc_ffsll:
3775 return optimizeFFS(CI, Builder);
3776 case LibFunc_fls:
3777 case LibFunc_flsl:
3778 case LibFunc_flsll:
3779 return optimizeFls(CI, Builder);
3780 case LibFunc_abs:
3781 case LibFunc_labs:
3782 case LibFunc_llabs:
3783 return optimizeAbs(CI, Builder);
3784 case LibFunc_isdigit:
3785 return optimizeIsDigit(CI, Builder);
3786 case LibFunc_isascii:
3787 return optimizeIsAscii(CI, Builder);
3788 case LibFunc_toascii:
3789 return optimizeToAscii(CI, Builder);
3790 case LibFunc_atoi:
3791 case LibFunc_atol:
3792 case LibFunc_atoll:
3793 return optimizeAtoi(CI, Builder);
3794 case LibFunc_strtol:
3795 case LibFunc_strtoll:
3796 return optimizeStrToInt(CI, Builder, /*AsSigned=*/true);
3797 case LibFunc_strtoul:
3798 case LibFunc_strtoull:
3799 return optimizeStrToInt(CI, Builder, /*AsSigned=*/false);
3800 case LibFunc_printf:
3801 return optimizePrintF(CI, Builder);
3802 case LibFunc_sprintf:
3803 return optimizeSPrintF(CI, Builder);
3804 case LibFunc_snprintf:
3805 return optimizeSnPrintF(CI, Builder);
3806 case LibFunc_fprintf:
3807 return optimizeFPrintF(CI, Builder);
3808 case LibFunc_fwrite:
3809 return optimizeFWrite(CI, Builder);
3810 case LibFunc_fputs:
3811 return optimizeFPuts(CI, Builder);
3812 case LibFunc_puts:
3813 return optimizePuts(CI, Builder);
3814 case LibFunc_perror:
3815 return optimizeErrorReporting(CI, Builder);
3816 case LibFunc_vfprintf:
3817 case LibFunc_fiprintf:
3818 return optimizeErrorReporting(CI, Builder, 0);
3819 default:
3820 return nullptr;
3821 }
3822 }
3823 return nullptr;
3824}
3825
3827 const DataLayout &DL, const TargetLibraryInfo *TLI, AssumptionCache *AC,
3829 ProfileSummaryInfo *PSI,
3830 function_ref<void(Instruction *, Value *)> Replacer,
3831 function_ref<void(Instruction *)> Eraser)
3832 : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), AC(AC), ORE(ORE), BFI(BFI),
3833 PSI(PSI), Replacer(Replacer), Eraser(Eraser) {}
3834
3835void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) {
3836 // Indirect through the replacer used in this instance.
3837 Replacer(I, With);
3838}
3839
3840void LibCallSimplifier::eraseFromParent(Instruction *I) {
3841 Eraser(I);
3842}
3843
3844// TODO:
3845// Additional cases that we need to add to this file:
3846//
3847// cbrt:
3848// * cbrt(expN(X)) -> expN(x/3)
3849// * cbrt(sqrt(x)) -> pow(x,1/6)
3850// * cbrt(cbrt(x)) -> pow(x,1/9)
3851//
3852// exp, expf, expl:
3853// * exp(log(x)) -> x
3854//
3855// log, logf, logl:
3856// * log(exp(x)) -> x
3857// * log(exp(y)) -> y*log(e)
3858// * log(exp10(y)) -> y*log(10)
3859// * log(sqrt(x)) -> 0.5*log(x)
3860//
3861// pow, powf, powl:
3862// * pow(sqrt(x),y) -> pow(x,y*0.5)
3863// * pow(pow(x,y),z)-> pow(x,y*z)
3864//
3865// signbit:
3866// * signbit(cnst) -> cnst'
3867// * signbit(nncst) -> 0 (if pstv is a non-negative constant)
3868//
3869// sqrt, sqrtf, sqrtl:
3870// * sqrt(expN(x)) -> expN(x*0.5)
3871// * sqrt(Nroot(x)) -> pow(x,1/(2*N))
3872// * sqrt(pow(x,y)) -> pow(|x|,y*0.5)
3873//
3874
3875//===----------------------------------------------------------------------===//
3876// Fortified Library Call Optimizations
3877//===----------------------------------------------------------------------===//
3878
3879bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
3880 CallInst *CI, unsigned ObjSizeOp, std::optional<unsigned> SizeOp,
3881 std::optional<unsigned> StrOp, std::optional<unsigned> FlagOp) {
3882 // If this function takes a flag argument, the implementation may use it to
3883 // perform extra checks. Don't fold into the non-checking variant.
3884 if (FlagOp) {
3885 ConstantInt *Flag = dyn_cast<ConstantInt>(CI->getArgOperand(*FlagOp));
3886 if (!Flag || !Flag->isZero())
3887 return false;
3888 }
3889
3890 if (SizeOp && CI->getArgOperand(ObjSizeOp) == CI->getArgOperand(*SizeOp))
3891 return true;
3892
3893 if (ConstantInt *ObjSizeCI =
3894 dyn_cast<ConstantInt>(CI->getArgOperand(ObjSizeOp))) {
3895 if (ObjSizeCI->isMinusOne())
3896 return true;
3897 // If the object size wasn't -1 (unknown), bail out if we were asked to.
3898 if (OnlyLowerUnknownSize)
3899 return false;
3900 if (StrOp) {
3902 // If the length is 0 we don't know how long it is and so we can't
3903 // remove the check.
3904 if (Len)
3905 annotateDereferenceableBytes(CI, *StrOp, Len);
3906 else
3907 return false;
3908 return ObjSizeCI->getZExtValue() >= Len;
3909 }
3910
3911 if (SizeOp) {
3912 if (ConstantInt *SizeCI =
3913 dyn_cast<ConstantInt>(CI->getArgOperand(*SizeOp)))
3914 return ObjSizeCI->getZExtValue() >= SizeCI->getZExtValue();
3915 }
3916 }
3917 return false;
3918}
3919
3920Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
3921 IRBuilderBase &B) {
3922 if (isFortifiedCallFoldable(CI, 3, 2)) {
3923 CallInst *NewCI =
3924 B.CreateMemCpy(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
3925 Align(1), CI->getArgOperand(2));
3926 mergeAttributesAndFlags(NewCI, *CI);
3927 return CI->getArgOperand(0);
3928 }
3929 return nullptr;
3930}
3931
3932Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
3933 IRBuilderBase &B) {
3934 if (isFortifiedCallFoldable(CI, 3, 2)) {
3935 CallInst *NewCI =
3936 B.CreateMemMove(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
3937 Align(1), CI->getArgOperand(2));
3938 mergeAttributesAndFlags(NewCI, *CI);
3939 return CI->getArgOperand(0);
3940 }
3941 return nullptr;
3942}
3943
3944Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
3945 IRBuilderBase &B) {
3946 if (isFortifiedCallFoldable(CI, 3, 2)) {
3947 Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
3948 CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val,
3949 CI->getArgOperand(2), Align(1));
3950 mergeAttributesAndFlags(NewCI, *CI);
3951 return CI->getArgOperand(0);
3952 }
3953 return nullptr;
3954}
3955
3956Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
3957 IRBuilderBase &B) {
3958 const DataLayout &DL = CI->getModule()->getDataLayout();
3959 if (isFortifiedCallFoldable(CI, 3, 2))
3960 if (Value *Call = emitMemPCpy(CI->getArgOperand(0), CI->getArgOperand(1),
3961 CI->getArgOperand(2), B, DL, TLI)) {
3962 return mergeAttributesAndFlags(cast<CallInst>(Call), *CI);
3963 }
3964 return nullptr;
3965}
3966
3967Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
3969 LibFunc Func) {
3970 const DataLayout &DL = CI->getModule()->getDataLayout();
3971 Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1),
3972 *ObjSize = CI->getArgOperand(2);
3973
3974 // __stpcpy_chk(x,x,...) -> x+strlen(x)
3975 if (Func == LibFunc_stpcpy_chk && !OnlyLowerUnknownSize && Dst == Src) {
3976 Value *StrLen = emitStrLen(Src, B, DL, TLI);
3977 return StrLen ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, StrLen) : nullptr;
3978 }
3979
3980 // If a) we don't have any length information, or b) we know this will
3981 // fit then just lower to a plain st[rp]cpy. Otherwise we'll keep our
3982 // st[rp]cpy_chk call which may fail at runtime if the size is too long.
3983 // TODO: It might be nice to get a maximum length out of the possible
3984 // string lengths for varying.
3985 if (isFortifiedCallFoldable(CI, 2, std::nullopt, 1)) {
3986 if (Func == LibFunc_strcpy_chk)
3987 return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
3988 else
3989 return copyFlags(*CI, emitStpCpy(Dst, Src, B, TLI));
3990 }
3991
3992 if (OnlyLowerUnknownSize)
3993 return nullptr;
3994
3995 // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
3997 if (Len)
3998 annotateDereferenceableBytes(CI, 1, Len);
3999 else
4000 return nullptr;
4001
4002 unsigned SizeTBits = TLI->getSizeTSize(*CI->getModule());
4003 Type *SizeTTy = IntegerType::get(CI->getContext(), SizeTBits);
4004 Value *LenV = ConstantInt::get(SizeTTy, Len);
4005 Value *Ret = emitMemCpyChk(Dst, Src, LenV, ObjSize, B, DL, TLI);
4006 // If the function was an __stpcpy_chk, and we were able to fold it into
4007 // a __memcpy_chk, we still need to return the correct end pointer.
4008 if (Ret && Func == LibFunc_stpcpy_chk)
4009 return B.CreateInBoundsGEP(B.getInt8Ty(), Dst,
4010 ConstantInt::get(SizeTTy, Len - 1));
4011 return copyFlags(*CI, cast<CallInst>(Ret));
4012}
4013
4014Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
4015 IRBuilderBase &B) {
4016 if (isFortifiedCallFoldable(CI, 1, std::nullopt, 0))
4017 return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B,
4018 CI->getModule()->getDataLayout(), TLI));
4019 return nullptr;
4020}
4021
4022Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
4024 LibFunc Func) {
4025 if (isFortifiedCallFoldable(CI, 3, 2)) {
4026 if (Func == LibFunc_strncpy_chk)
4027 return copyFlags(*CI,
4029 CI->getArgOperand(2), B, TLI));
4030 else
4031 return copyFlags(*CI,
4033 CI->getArgOperand(2), B, TLI));
4034 }
4035
4036 return nullptr;
4037}
4038
4039Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
4040 IRBuilderBase &B) {
4041 if (isFortifiedCallFoldable(CI, 4, 3))
4042 return copyFlags(
4043 *CI, emitMemCCpy(CI->getArgOperand(0), CI->getArgOperand(1),
4044 CI->getArgOperand(2), CI->getArgOperand(3), B, TLI));
4045
4046 return nullptr;
4047}
4048
4049Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
4050 IRBuilderBase &B) {
4051 if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2)) {
4052 SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 5));
4053 return copyFlags(*CI,
4055 CI->getArgOperand(4), VariadicArgs, B, TLI));
4056 }
4057
4058 return nullptr;
4059}
4060
4061Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
4062 IRBuilderBase &B) {
4063 if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1)) {
4064 SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 4));
4065 return copyFlags(*CI,
4067 VariadicArgs, B, TLI));
4068 }
4069
4070 return nullptr;
4071}
4072
4073Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
4074 IRBuilderBase &B) {
4075 if (isFortifiedCallFoldable(CI, 2))
4076 return copyFlags(
4077 *CI, emitStrCat(CI->getArgOperand(0), CI->getArgOperand(1), B, TLI));
4078
4079 return nullptr;
4080}
4081
4082Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
4083 IRBuilderBase &B) {
4084 if (isFortifiedCallFoldable(CI, 3))
4085 return copyFlags(*CI,
4087 CI->getArgOperand(2), B, TLI));
4088
4089 return nullptr;
4090}
4091
4092Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
4093 IRBuilderBase &B) {
4094 if (isFortifiedCallFoldable(CI, 3))
4095 return copyFlags(*CI,
4097 CI->getArgOperand(2), B, TLI));
4098
4099 return nullptr;
4100}
4101
4102Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
4103 IRBuilderBase &B) {
4104 if (isFortifiedCallFoldable(CI, 3))
4105 return copyFlags(*CI,
4107 CI->getArgOperand(2), B, TLI));
4108
4109 return nullptr;
4110}
4111
4112Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
4113 IRBuilderBase &B) {
4114 if (isFortifiedCallFoldable(CI, 3, 1, std::nullopt, 2))
4115 return copyFlags(
4116 *CI, emitVSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
4117 CI->getArgOperand(4), CI->getArgOperand(5), B, TLI));
4118
4119 return nullptr;
4120}
4121
4122Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
4123 IRBuilderBase &B) {
4124 if (isFortifiedCallFoldable(CI, 2, std::nullopt, std::nullopt, 1))
4125 return copyFlags(*CI,
4127 CI->getArgOperand(4), B, TLI));
4128
4129 return nullptr;
4130}
4131
4133 IRBuilderBase &Builder) {
4134 // FIXME: We shouldn't be changing "nobuiltin" or TLI unavailable calls here.
4135 // Some clang users checked for _chk libcall availability using:
4136 // __has_builtin(__builtin___memcpy_chk)
4137 // When compiling with -fno-builtin, this is always true.
4138 // When passing -ffreestanding/-mkernel, which both imply -fno-builtin, we
4139 // end up with fortified libcalls, which isn't acceptable in a freestanding
4140 // environment which only provides their non-fortified counterparts.
4141 //
4142 // Until we change clang and/or teach external users to check for availability
4143 // differently, disregard the "nobuiltin" attribute and TLI::has.
4144 //
4145 // PR23093.
4146
4147 LibFunc Func;
4148 Function *Callee = CI->getCalledFunction();
4149 bool IsCallingConvC = TargetLibraryInfoImpl::isCallingConvCCompatible(CI);
4150
4152 CI->getOperandBundlesAsDefs(OpBundles);
4153
4155 Builder.setDefaultOperandBundles(OpBundles);
4156
4157 // First, check that this is a known library functions and that the prototype
4158 // is correct.
4159 if (!TLI->getLibFunc(*Callee, Func))
4160 return nullptr;
4161
4162 // We never change the calling convention.
4163 if (!ignoreCallingConv(Func) && !IsCallingConvC)
4164 return nullptr;
4165
4166 switch (Func) {
4167 case LibFunc_memcpy_chk:
4168 return optimizeMemCpyChk(CI, Builder);
4169 case LibFunc_mempcpy_chk:
4170 return optimizeMemPCpyChk(CI, Builder);
4171 case LibFunc_memmove_chk:
4172 return optimizeMemMoveChk(CI, Builder);
4173 case LibFunc_memset_chk:
4174 return optimizeMemSetChk(CI, Builder);
4175 case LibFunc_stpcpy_chk:
4176 case LibFunc_strcpy_chk:
4177 return optimizeStrpCpyChk(CI, Builder, Func);
4178 case LibFunc_strlen_chk:
4179 return optimizeStrLenChk(CI, Builder);
4180 case LibFunc_stpncpy_chk:
4181 case LibFunc_strncpy_chk:
4182 return optimizeStrpNCpyChk(CI, Builder, Func);
4183 case LibFunc_memccpy_chk:
4184 return optimizeMemCCpyChk(CI, Builder);
4185 case LibFunc_snprintf_chk:
4186 return optimizeSNPrintfChk(CI, Builder);
4187 case LibFunc_sprintf_chk:
4188 return optimizeSPrintfChk(CI, Builder);
4189 case LibFunc_strcat_chk:
4190 return optimizeStrCatChk(CI, Builder);
4191 case LibFunc_strlcat_chk:
4192 return optimizeStrLCat(CI, Builder);
4193 case LibFunc_strncat_chk:
4194 return optimizeStrNCatChk(CI, Builder);
4195 case LibFunc_strlcpy_chk:
4196 return optimizeStrLCpyChk(CI, Builder);
4197 case LibFunc_vsnprintf_chk:
4198 return optimizeVSNPrintfChk(CI, Builder);
4199 case LibFunc_vsprintf_chk:
4200 return optimizeVSPrintfChk(CI, Builder);
4201 default:
4202 break;
4203 }
4204 return nullptr;
4205}
4206
4208 const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize)
4209 : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
assume Assume Builder
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
return RetTy
std::string Name
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Hexagon Common GEP
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static bool isBinary(MachineInstr &MI)
const SmallVectorImpl< MachineOperand > & Cond
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isOnlyUsedInEqualityComparison(Value *V, Value *With)
Return true if it is only used in equality comparisons with With.
static Value * optimizeTrigReflections(CallInst *Call, LibFunc Func, IRBuilderBase &B)
static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef< unsigned > ArgNos, Value *Size, const DataLayout &DL)
static cl::opt< unsigned, false, HotColdHintParser > ColdNewHintValue("cold-new-hint-value", cl::Hidden, cl::init(1), cl::desc("Value to pass to hot/cold operator new for cold allocation"))
static bool insertSinCosCall(IRBuilderBase &B, Function *OrigCallee, Value *Arg, bool UseFloat, Value *&Sin, Value *&Cos, Value *&SinCos, const TargetLibraryInfo *TLI)
static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len, const DataLayout &DL)
static Value * mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old)
static cl::opt< bool > OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false), cl::desc("Enable hot/cold operator new library calls"))
static Value * optimizeBinaryDoubleFP(CallInst *CI, IRBuilderBase &B, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float for binary functions.
static bool ignoreCallingConv(LibFunc Func)
static void annotateDereferenceableBytes(CallInst *CI, ArrayRef< unsigned > ArgNos, uint64_t DereferenceableBytes)
static bool isReportingError(Function *Callee, CallInst *CI, int StreamArg)
static Value * optimizeDoubleFP(CallInst *CI, IRBuilderBase &B, bool isBinary, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float functions.
static Value * getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno, Module *M, IRBuilderBase &B, const TargetLibraryInfo *TLI)
static Value * valueHasFloatPrecision(Value *Val)
Return a variant of Val with float type.
static Value * optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS, uint64_t Len, IRBuilderBase &B, const DataLayout &DL)
static Value * createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M, IRBuilderBase &B)
static Value * convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr, uint64_t Base, bool AsSigned, IRBuilderBase &B)
static Value * memChrToCharCompare(CallInst *CI, Value *NBytes, IRBuilderBase &B, const DataLayout &DL)
static Value * copyFlags(const CallInst &Old, Value *New)
static StringRef substr(StringRef Str, uint64_t Len)
static cl::opt< unsigned, false, HotColdHintParser > HotNewHintValue("hot-new-hint-value", cl::Hidden, cl::init(254), cl::desc("Value to pass to hot/cold operator new for hot allocation"))
static bool isTrigLibCall(CallInst *CI)
static bool isOnlyUsedInComparisonWithZero(Value *V)
static Value * replaceUnaryCall(CallInst *CI, IRBuilderBase &B, Intrinsic::ID IID)
static bool callHasFloatingPointArgument(const CallInst *CI)
static Value * optimizeUnaryDoubleFP(CallInst *CI, IRBuilderBase &B, const TargetLibraryInfo *TLI, bool isPrecise=false)
Shrink double -> float for unary functions.
static bool callHasFP128Argument(const CallInst *CI)
static void annotateNonNullNoUndefBasedOnAccess(CallInst *CI, ArrayRef< unsigned > ArgNos)
static Value * optimizeMemCmpVarSize(CallInst *CI, Value *LHS, Value *RHS, Value *Size, bool StrNCmp, IRBuilderBase &B, const DataLayout &DL)
static Value * getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth)
static cl::opt< bool > EnableUnsafeFPShrink("enable-double-float-shrink", cl::Hidden, cl::init(false), cl::desc("Enable unsafe double to float " "shrinking for math lib calls"))
This file defines the SmallString class.
This file contains some functions that are useful when dealing with strings.
Value * RHS
Value * LHS
bool isFiniteNonZero() const
Definition: APFloat.h:1303
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5196
bool isNegative() const
Definition: APFloat.h:1293
double convertToDouble() const
Converts this APFloat to host double value.
Definition: APFloat.cpp:5255
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: APFloat.h:1276
opStatus add(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1040
const fltSemantics & getSemantics() const
Definition: APFloat.h:1301
float convertToFloat() const
Converts this APFloat to host float value.
Definition: APFloat.cpp:5268
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1183
bool isInteger() const
Definition: APFloat.h:1310
Class for arbitrary precision integers.
Definition: APInt.h:76
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1122
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
A cache of @llvm.assume calls within a function.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:826
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
AttributeList addParamAttributes(LLVMContext &C, unsigned ArgNo, const AttrBuilder &B) const
Add an argument attribute to the list.
Definition: Attributes.h:594
MaybeAlign getAlignment() const
Definition: Attributes.cpp:795
static Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:178
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:318
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:335
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Definition: InstrTypes.h:1521
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Definition: InstrTypes.h:1876
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Definition: InstrTypes.h:1588
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1412
bool doesNotAccessMemory(unsigned OpNo) const
Definition: InstrTypes.h:1734
void removeRetAttrs(const AttributeMask &AttrsToRemove)
Removes the attributes from the return value.
Definition: InstrTypes.h:1583
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
Definition: InstrTypes.h:1498
bool isStrictFP() const
Determine if the call requires strict floating point semantics.
Definition: InstrTypes.h:1882
uint64_t getParamDereferenceableBytes(unsigned i) const
Extract the number of dereferenceable bytes for a call or parameter (0=unknown).
Definition: InstrTypes.h:1824
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1493
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: InstrTypes.h:1926
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1357
uint64_t getParamDereferenceableOrNullBytes(unsigned i) const
Extract the number of dereferenceable_or_null bytes for a parameter (0=unknown).
Definition: InstrTypes.h:1842
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1348
unsigned arg_size() const
Definition: InstrTypes.h:1355
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1489
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1541
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
bool isNoTailCall() const
TailCallKind getTailCallKind() const
bool isMustTailCall() const
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:734
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:736
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:737
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:801
uint64_t getElementAsInteger(unsigned i) const
If this is a sequential container of integers (of any size), return the specified element in the low ...
Definition: Constants.cpp:3014
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:260
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:927
static Constant * getInfinity(Type *Ty, bool Negative=false)
Definition: Constants.cpp:1027
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:203
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:197
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:151
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:145
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:356
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:876
bool fitsInLegalInteger(unsigned Width) const
Returns true if the specified type fits in a native integer type supported by the CPU.
Definition: DataLayout.h:359
This class represents an extension of floating point types.
This class represents a truncation of floating point types.
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
void setNoSignedZeros(bool B=true)
Definition: FMF.h:85
static FastMathFlags getFast()
Definition: FMF.h:51
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:693
FortifiedLibCallSimplifier(const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize=false)
Value * optimizeCall(CallInst *CI, IRBuilderBase &B)
Take the given call instruction and return a more optimal value to replace the instruction with or 0 ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition: Function.h:647
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:206
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:315
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:211
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:645
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:273
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
This instruction compares its operands according to the predicate given to the constructor.
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
bool hasNoNaNs() const LLVM_READONLY
Determine whether the no-NaNs flag is set.
bool hasNoInfs() const LLVM_READONLY
Determine whether the no-infs flag is set.
bool hasNoSignedZeros() const LLVM_READONLY
Determine whether the no-signed-zeros flag is set.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:71
const BasicBlock * getParent() const
Definition: Instruction.h:90
bool isFast() const LLVM_READONLY
Determine whether all fast-math-flags are set.
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:75
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
bool hasApproxFunc() const LLVM_READONLY
Determine whether the approximate-math-functions flag is set.
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Class to represent integer types.
Definition: DerivedTypes.h:40
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:279
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
Value * optimizeCall(CallInst *CI, IRBuilderBase &B)
optimizeCall - Take the given call instruction and return a more optimal value to replace the instruc...
LibCallSimplifier(const DataLayout &DL, const TargetLibraryInfo *TLI, AssumptionCache *AC, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, function_ref< void(Instruction *, Value *)> Replacer=&replaceAllUsesWithDefault, function_ref< void(Instruction *)> Eraser=&eraseFromParentDefault)
An instruction for reading from memory.
Definition: Instructions.h:177
Value * getPointerOperand()
Definition: Instructions.h:264
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition: Module.h:258
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:254
The optimization diagnostic interface.
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for applied optimization remarks.
Analysis providing profile information.
This class represents the LLVM 'select' instruction.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
bool empty() const
Definition: SmallVector.h:94
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:474
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
char back() const
back - Get the last character in the string.
Definition: StringRef.h:146
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition: StringRef.h:428
bool startswith(StringRef Prefix) const
Definition: StringRef.h:261
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:381
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition: StringRef.h:301
static constexpr size_t npos
Definition: StringRef.h:52
int compare(StringRef RHS) const
compare - Compare two strings; the result is negative, zero, or positive if this string is lexicograp...
Definition: StringRef.h:177
StringRef drop_back(size_t N=1) const
Return a StringRef equal to 'this' but with the last N elements dropped.
Definition: StringRef.h:620
size_t find_first_not_of(char C, size_t From=0) const
Find the first character in the string that is not C or npos if not found.
Definition: StringRef.cpp:251
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:374
static bool isCallingConvCCompatible(CallBase *CI)
Returns true if call site / callee has cdecl-compatible calling conventions.
Provides information about what library functions are available for the current target.
unsigned getWCharSize(const Module &M) const
Returns the size of the wchar_t type in bytes or 0 if the size is unknown.
unsigned getSizeTSize(const Module &M) const
Returns the size of the size_t type in bits.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
StringRef getName(LibFunc F) const
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition: Type.h:154
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:249
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition: Type.h:157
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
op_range operands()
Definition: