LLVM 22.0.0git
InstCombineCalls.cpp
Go to the documentation of this file.
1//===- InstCombineCalls.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitCall, visitInvoke, and visitCallBr functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/Statistic.h"
26#include "llvm/Analysis/Loads.h"
31#include "llvm/IR/Attributes.h"
32#include "llvm/IR/BasicBlock.h"
33#include "llvm/IR/Constant.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DebugInfo.h"
38#include "llvm/IR/Function.h"
40#include "llvm/IR/InlineAsm.h"
41#include "llvm/IR/InstrTypes.h"
42#include "llvm/IR/Instruction.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsAArch64.h"
47#include "llvm/IR/IntrinsicsAMDGPU.h"
48#include "llvm/IR/IntrinsicsARM.h"
49#include "llvm/IR/IntrinsicsHexagon.h"
50#include "llvm/IR/LLVMContext.h"
51#include "llvm/IR/Metadata.h"
53#include "llvm/IR/Statepoint.h"
54#include "llvm/IR/Type.h"
55#include "llvm/IR/User.h"
56#include "llvm/IR/Value.h"
57#include "llvm/IR/ValueHandle.h"
62#include "llvm/Support/Debug.h"
73#include <algorithm>
74#include <cassert>
75#include <cstdint>
76#include <optional>
77#include <utility>
78#include <vector>
79
80#define DEBUG_TYPE "instcombine"
82
83using namespace llvm;
84using namespace PatternMatch;
85
86STATISTIC(NumSimplified, "Number of library calls simplified");
87
89 "instcombine-guard-widening-window",
90 cl::init(3),
91 cl::desc("How wide an instruction window to bypass looking for "
92 "another guard"));
93
94/// Return the specified type promoted as it would be to pass though a va_arg
95/// area.
97 if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
98 if (ITy->getBitWidth() < 32)
99 return Type::getInt32Ty(Ty->getContext());
100 }
101 return Ty;
102}
103
104/// Recognize a memcpy/memmove from a trivially otherwise unused alloca.
105/// TODO: This should probably be integrated with visitAllocSites, but that
106/// requires a deeper change to allow either unread or unwritten objects.
108 auto *Src = MI->getRawSource();
109 while (isa<GetElementPtrInst>(Src)) {
110 if (!Src->hasOneUse())
111 return false;
112 Src = cast<Instruction>(Src)->getOperand(0);
113 }
114 return isa<AllocaInst>(Src) && Src->hasOneUse();
115}
116
118 Align DstAlign = getKnownAlignment(MI->getRawDest(), DL, MI, &AC, &DT);
119 MaybeAlign CopyDstAlign = MI->getDestAlign();
120 if (!CopyDstAlign || *CopyDstAlign < DstAlign) {
121 MI->setDestAlignment(DstAlign);
122 return MI;
123 }
124
125 Align SrcAlign = getKnownAlignment(MI->getRawSource(), DL, MI, &AC, &DT);
126 MaybeAlign CopySrcAlign = MI->getSourceAlign();
127 if (!CopySrcAlign || *CopySrcAlign < SrcAlign) {
128 MI->setSourceAlignment(SrcAlign);
129 return MI;
130 }
131
132 // If we have a store to a location which is known constant, we can conclude
133 // that the store must be storing the constant value (else the memory
134 // wouldn't be constant), and this must be a noop.
135 if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {
136 // Set the size of the copy to 0, it will be deleted on the next iteration.
137 MI->setLength((uint64_t)0);
138 return MI;
139 }
140
141 // If the source is provably undef, the memcpy/memmove doesn't do anything
142 // (unless the transfer is volatile).
143 if (hasUndefSource(MI) && !MI->isVolatile()) {
144 // Set the size of the copy to 0, it will be deleted on the next iteration.
145 MI->setLength((uint64_t)0);
146 return MI;
147 }
148
149 // If MemCpyInst length is 1/2/4/8 bytes then replace memcpy with
150 // load/store.
151 ConstantInt *MemOpLength = dyn_cast<ConstantInt>(MI->getLength());
152 if (!MemOpLength) return nullptr;
153
154 // Source and destination pointer types are always "i8*" for intrinsic. See
155 // if the size is something we can handle with a single primitive load/store.
156 // A single load+store correctly handles overlapping memory in the memmove
157 // case.
158 uint64_t Size = MemOpLength->getLimitedValue();
159 assert(Size && "0-sized memory transferring should be removed already.");
160
161 if (Size > 8 || (Size&(Size-1)))
162 return nullptr; // If not 1/2/4/8 bytes, exit.
163
164 // If it is an atomic and alignment is less than the size then we will
165 // introduce the unaligned memory access which will be later transformed
166 // into libcall in CodeGen. This is not evident performance gain so disable
167 // it now.
168 if (MI->isAtomic())
169 if (*CopyDstAlign < Size || *CopySrcAlign < Size)
170 return nullptr;
171
172 // Use an integer load+store unless we can find something better.
173 IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
174
175 // If the memcpy has metadata describing the members, see if we can get the
176 // TBAA, scope and noalias tags describing our copy.
177 AAMDNodes AACopyMD = MI->getAAMetadata().adjustForAccess(Size);
178
179 Value *Src = MI->getArgOperand(1);
180 Value *Dest = MI->getArgOperand(0);
181 LoadInst *L = Builder.CreateLoad(IntType, Src);
182 // Alignment from the mem intrinsic will be better, so use it.
183 L->setAlignment(*CopySrcAlign);
184 L->setAAMetadata(AACopyMD);
185 MDNode *LoopMemParallelMD =
186 MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
187 if (LoopMemParallelMD)
188 L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
189 MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);
190 if (AccessGroupMD)
191 L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
192
193 StoreInst *S = Builder.CreateStore(L, Dest);
194 // Alignment from the mem intrinsic will be better, so use it.
195 S->setAlignment(*CopyDstAlign);
196 S->setAAMetadata(AACopyMD);
197 if (LoopMemParallelMD)
198 S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
199 if (AccessGroupMD)
200 S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
201 S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
202
203 if (auto *MT = dyn_cast<MemTransferInst>(MI)) {
204 // non-atomics can be volatile
205 L->setVolatile(MT->isVolatile());
206 S->setVolatile(MT->isVolatile());
207 }
208 if (MI->isAtomic()) {
209 // atomics have to be unordered
210 L->setOrdering(AtomicOrdering::Unordered);
212 }
213
214 // Set the size of the copy to 0, it will be deleted on the next iteration.
215 MI->setLength((uint64_t)0);
216 return MI;
217}
218
220 const Align KnownAlignment =
221 getKnownAlignment(MI->getDest(), DL, MI, &AC, &DT);
222 MaybeAlign MemSetAlign = MI->getDestAlign();
223 if (!MemSetAlign || *MemSetAlign < KnownAlignment) {
224 MI->setDestAlignment(KnownAlignment);
225 return MI;
226 }
227
228 // If we have a store to a location which is known constant, we can conclude
229 // that the store must be storing the constant value (else the memory
230 // wouldn't be constant), and this must be a noop.
231 if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {
232 // Set the size of the copy to 0, it will be deleted on the next iteration.
233 MI->setLength((uint64_t)0);
234 return MI;
235 }
236
237 // Remove memset with an undef value.
238 // FIXME: This is technically incorrect because it might overwrite a poison
239 // value. Change to PoisonValue once #52930 is resolved.
240 if (isa<UndefValue>(MI->getValue())) {
241 // Set the size of the copy to 0, it will be deleted on the next iteration.
242 MI->setLength((uint64_t)0);
243 return MI;
244 }
245
246 // Extract the length and alignment and fill if they are constant.
247 ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());
248 ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue());
249 if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8))
250 return nullptr;
251 const uint64_t Len = LenC->getLimitedValue();
252 assert(Len && "0-sized memory setting should be removed already.");
253 const Align Alignment = MI->getDestAlign().valueOrOne();
254
255 // If it is an atomic and alignment is less than the size then we will
256 // introduce the unaligned memory access which will be later transformed
257 // into libcall in CodeGen. This is not evident performance gain so disable
258 // it now.
259 if (MI->isAtomic() && Alignment < Len)
260 return nullptr;
261
262 // memset(s,c,n) -> store s, c (for n=1,2,4,8)
263 if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
264 Value *Dest = MI->getDest();
265
266 // Extract the fill value and store.
267 Constant *FillVal = ConstantInt::get(
268 MI->getContext(), APInt::getSplat(Len * 8, FillC->getValue()));
269 StoreInst *S = Builder.CreateStore(FillVal, Dest, MI->isVolatile());
270 S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
271 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(S)) {
272 if (llvm::is_contained(DbgAssign->location_ops(), FillC))
273 DbgAssign->replaceVariableLocationOp(FillC, FillVal);
274 }
275
276 S->setAlignment(Alignment);
277 if (MI->isAtomic())
279
280 // Set the size of the copy to 0, it will be deleted on the next iteration.
281 MI->setLength((uint64_t)0);
282 return MI;
283 }
284
285 return nullptr;
286}
287
288// TODO, Obvious Missing Transforms:
289// * Narrow width by halfs excluding zero/undef lanes
290Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
291 Value *LoadPtr = II.getArgOperand(0);
292 const Align Alignment =
293 cast<ConstantInt>(II.getArgOperand(1))->getAlignValue();
294
295 // If the mask is all ones or undefs, this is a plain vector load of the 1st
296 // argument.
297 if (maskIsAllOneOrUndef(II.getArgOperand(2))) {
298 LoadInst *L = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
299 "unmaskedload");
300 L->copyMetadata(II);
301 return L;
302 }
303
304 // If we can unconditionally load from this address, replace with a
305 // load/select idiom. TODO: use DT for context sensitive query
306 if (isDereferenceablePointer(LoadPtr, II.getType(),
307 II.getDataLayout(), &II, &AC)) {
308 LoadInst *LI = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
309 "unmaskedload");
310 LI->copyMetadata(II);
311 return Builder.CreateSelect(II.getArgOperand(2), LI, II.getArgOperand(3));
312 }
313
314 return nullptr;
315}
316
317// TODO, Obvious Missing Transforms:
318// * Single constant active lane -> store
319// * Narrow width by halfs excluding zero/undef lanes
320Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) {
321 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
322 if (!ConstMask)
323 return nullptr;
324
325 // If the mask is all zeros, this instruction does nothing.
326 if (ConstMask->isNullValue())
328
329 // If the mask is all ones, this is a plain vector store of the 1st argument.
330 if (ConstMask->isAllOnesValue()) {
331 Value *StorePtr = II.getArgOperand(1);
332 Align Alignment = cast<ConstantInt>(II.getArgOperand(2))->getAlignValue();
333 StoreInst *S =
334 new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);
335 S->copyMetadata(II);
336 return S;
337 }
338
339 if (isa<ScalableVectorType>(ConstMask->getType()))
340 return nullptr;
341
342 // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
343 APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
344 APInt PoisonElts(DemandedElts.getBitWidth(), 0);
345 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts,
346 PoisonElts))
347 return replaceOperand(II, 0, V);
348
349 return nullptr;
350}
351
352// TODO, Obvious Missing Transforms:
353// * Single constant active lane load -> load
354// * Dereferenceable address & few lanes -> scalarize speculative load/selects
355// * Adjacent vector addresses -> masked.load
356// * Narrow width by halfs excluding zero/undef lanes
357// * Vector incrementing address -> vector masked load
358Instruction *InstCombinerImpl::simplifyMaskedGather(IntrinsicInst &II) {
359 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(2));
360 if (!ConstMask)
361 return nullptr;
362
363 // Vector splat address w/known mask -> scalar load
364 // Fold the gather to load the source vector first lane
365 // because it is reloading the same value each time
366 if (ConstMask->isAllOnesValue())
367 if (auto *SplatPtr = getSplatValue(II.getArgOperand(0))) {
368 auto *VecTy = cast<VectorType>(II.getType());
369 const Align Alignment =
370 cast<ConstantInt>(II.getArgOperand(1))->getAlignValue();
371 LoadInst *L = Builder.CreateAlignedLoad(VecTy->getElementType(), SplatPtr,
372 Alignment, "load.scalar");
373 Value *Shuf =
374 Builder.CreateVectorSplat(VecTy->getElementCount(), L, "broadcast");
376 }
377
378 return nullptr;
379}
380
381// TODO, Obvious Missing Transforms:
382// * Single constant active lane -> store
383// * Adjacent vector addresses -> masked.store
384// * Narrow store width by halfs excluding zero/undef lanes
385// * Vector incrementing address -> vector masked store
386Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
387 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
388 if (!ConstMask)
389 return nullptr;
390
391 // If the mask is all zeros, a scatter does nothing.
392 if (ConstMask->isNullValue())
394
395 // Vector splat address -> scalar store
396 if (auto *SplatPtr = getSplatValue(II.getArgOperand(1))) {
397 // scatter(splat(value), splat(ptr), non-zero-mask) -> store value, ptr
398 if (auto *SplatValue = getSplatValue(II.getArgOperand(0))) {
399 if (maskContainsAllOneOrUndef(ConstMask)) {
400 Align Alignment =
401 cast<ConstantInt>(II.getArgOperand(2))->getAlignValue();
402 StoreInst *S = new StoreInst(SplatValue, SplatPtr, /*IsVolatile=*/false,
403 Alignment);
404 S->copyMetadata(II);
405 return S;
406 }
407 }
408 // scatter(vector, splat(ptr), splat(true)) -> store extract(vector,
409 // lastlane), ptr
410 if (ConstMask->isAllOnesValue()) {
411 Align Alignment = cast<ConstantInt>(II.getArgOperand(2))->getAlignValue();
412 VectorType *WideLoadTy = cast<VectorType>(II.getArgOperand(1)->getType());
413 ElementCount VF = WideLoadTy->getElementCount();
414 Value *RunTimeVF = Builder.CreateElementCount(Builder.getInt32Ty(), VF);
415 Value *LastLane = Builder.CreateSub(RunTimeVF, Builder.getInt32(1));
416 Value *Extract =
417 Builder.CreateExtractElement(II.getArgOperand(0), LastLane);
418 StoreInst *S =
419 new StoreInst(Extract, SplatPtr, /*IsVolatile=*/false, Alignment);
420 S->copyMetadata(II);
421 return S;
422 }
423 }
424 if (isa<ScalableVectorType>(ConstMask->getType()))
425 return nullptr;
426
427 // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
428 APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
429 APInt PoisonElts(DemandedElts.getBitWidth(), 0);
430 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts,
431 PoisonElts))
432 return replaceOperand(II, 0, V);
433 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts,
434 PoisonElts))
435 return replaceOperand(II, 1, V);
436
437 return nullptr;
438}
439
440/// This function transforms launder.invariant.group and strip.invariant.group
441/// like:
442/// launder(launder(%x)) -> launder(%x) (the result is not the argument)
443/// launder(strip(%x)) -> launder(%x)
444/// strip(strip(%x)) -> strip(%x) (the result is not the argument)
445/// strip(launder(%x)) -> strip(%x)
446/// This is legal because it preserves the most recent information about
447/// the presence or absence of invariant.group.
449 InstCombinerImpl &IC) {
450 auto *Arg = II.getArgOperand(0);
451 auto *StrippedArg = Arg->stripPointerCasts();
452 auto *StrippedInvariantGroupsArg = StrippedArg;
453 while (auto *Intr = dyn_cast<IntrinsicInst>(StrippedInvariantGroupsArg)) {
454 if (Intr->getIntrinsicID() != Intrinsic::launder_invariant_group &&
455 Intr->getIntrinsicID() != Intrinsic::strip_invariant_group)
456 break;
457 StrippedInvariantGroupsArg = Intr->getArgOperand(0)->stripPointerCasts();
458 }
459 if (StrippedArg == StrippedInvariantGroupsArg)
460 return nullptr; // No launders/strips to remove.
461
462 Value *Result = nullptr;
463
464 if (II.getIntrinsicID() == Intrinsic::launder_invariant_group)
465 Result = IC.Builder.CreateLaunderInvariantGroup(StrippedInvariantGroupsArg);
466 else if (II.getIntrinsicID() == Intrinsic::strip_invariant_group)
467 Result = IC.Builder.CreateStripInvariantGroup(StrippedInvariantGroupsArg);
468 else
470 "simplifyInvariantGroupIntrinsic only handles launder and strip");
471 if (Result->getType()->getPointerAddressSpace() !=
472 II.getType()->getPointerAddressSpace())
473 Result = IC.Builder.CreateAddrSpaceCast(Result, II.getType());
474
475 return cast<Instruction>(Result);
476}
477
479 assert((II.getIntrinsicID() == Intrinsic::cttz ||
480 II.getIntrinsicID() == Intrinsic::ctlz) &&
481 "Expected cttz or ctlz intrinsic");
482 bool IsTZ = II.getIntrinsicID() == Intrinsic::cttz;
483 Value *Op0 = II.getArgOperand(0);
484 Value *Op1 = II.getArgOperand(1);
485 Value *X;
486 // ctlz(bitreverse(x)) -> cttz(x)
487 // cttz(bitreverse(x)) -> ctlz(x)
488 if (match(Op0, m_BitReverse(m_Value(X)))) {
489 Intrinsic::ID ID = IsTZ ? Intrinsic::ctlz : Intrinsic::cttz;
490 Function *F =
491 Intrinsic::getOrInsertDeclaration(II.getModule(), ID, II.getType());
492 return CallInst::Create(F, {X, II.getArgOperand(1)});
493 }
494
495 if (II.getType()->isIntOrIntVectorTy(1)) {
496 // ctlz/cttz i1 Op0 --> not Op0
497 if (match(Op1, m_Zero()))
498 return BinaryOperator::CreateNot(Op0);
499 // If zero is poison, then the input can be assumed to be "true", so the
500 // instruction simplifies to "false".
501 assert(match(Op1, m_One()) && "Expected ctlz/cttz operand to be 0 or 1");
502 return IC.replaceInstUsesWith(II, ConstantInt::getNullValue(II.getType()));
503 }
504
505 // If ctlz/cttz is only used as a shift amount, set is_zero_poison to true.
506 if (II.hasOneUse() && match(Op1, m_Zero()) &&
507 match(II.user_back(), m_Shift(m_Value(), m_Specific(&II)))) {
508 II.dropUBImplyingAttrsAndMetadata();
509 return IC.replaceOperand(II, 1, IC.Builder.getTrue());
510 }
511
512 Constant *C;
513
514 if (IsTZ) {
515 // cttz(-x) -> cttz(x)
516 if (match(Op0, m_Neg(m_Value(X))))
517 return IC.replaceOperand(II, 0, X);
518
519 // cttz(-x & x) -> cttz(x)
520 if (match(Op0, m_c_And(m_Neg(m_Value(X)), m_Deferred(X))))
521 return IC.replaceOperand(II, 0, X);
522
523 // cttz(sext(x)) -> cttz(zext(x))
524 if (match(Op0, m_OneUse(m_SExt(m_Value(X))))) {
525 auto *Zext = IC.Builder.CreateZExt(X, II.getType());
526 auto *CttzZext =
527 IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, Zext, Op1);
528 return IC.replaceInstUsesWith(II, CttzZext);
529 }
530
531 // Zext doesn't change the number of trailing zeros, so narrow:
532 // cttz(zext(x)) -> zext(cttz(x)) if the 'ZeroIsPoison' parameter is 'true'.
533 if (match(Op0, m_OneUse(m_ZExt(m_Value(X)))) && match(Op1, m_One())) {
534 auto *Cttz = IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, X,
535 IC.Builder.getTrue());
536 auto *ZextCttz = IC.Builder.CreateZExt(Cttz, II.getType());
537 return IC.replaceInstUsesWith(II, ZextCttz);
538 }
539
540 // cttz(abs(x)) -> cttz(x)
541 // cttz(nabs(x)) -> cttz(x)
542 Value *Y;
544 if (SPF == SPF_ABS || SPF == SPF_NABS)
545 return IC.replaceOperand(II, 0, X);
546
548 return IC.replaceOperand(II, 0, X);
549
550 // cttz(shl(%const, %val), 1) --> add(cttz(%const, 1), %val)
551 if (match(Op0, m_Shl(m_ImmConstant(C), m_Value(X))) &&
552 match(Op1, m_One())) {
553 Value *ConstCttz =
554 IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1);
555 return BinaryOperator::CreateAdd(ConstCttz, X);
556 }
557
558 // cttz(lshr exact (%const, %val), 1) --> sub(cttz(%const, 1), %val)
559 if (match(Op0, m_Exact(m_LShr(m_ImmConstant(C), m_Value(X)))) &&
560 match(Op1, m_One())) {
561 Value *ConstCttz =
562 IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1);
563 return BinaryOperator::CreateSub(ConstCttz, X);
564 }
565
566 // cttz(add(lshr(UINT_MAX, %val), 1)) --> sub(width, %val)
567 if (match(Op0, m_Add(m_LShr(m_AllOnes(), m_Value(X)), m_One()))) {
568 Value *Width =
569 ConstantInt::get(II.getType(), II.getType()->getScalarSizeInBits());
570 return BinaryOperator::CreateSub(Width, X);
571 }
572 } else {
573 // ctlz(lshr(%const, %val), 1) --> add(ctlz(%const, 1), %val)
574 if (match(Op0, m_LShr(m_ImmConstant(C), m_Value(X))) &&
575 match(Op1, m_One())) {
576 Value *ConstCtlz =
577 IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1);
578 return BinaryOperator::CreateAdd(ConstCtlz, X);
579 }
580
581 // ctlz(shl nuw (%const, %val), 1) --> sub(ctlz(%const, 1), %val)
582 if (match(Op0, m_NUWShl(m_ImmConstant(C), m_Value(X))) &&
583 match(Op1, m_One())) {
584 Value *ConstCtlz =
585 IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1);
586 return BinaryOperator::CreateSub(ConstCtlz, X);
587 }
588 }
589
590 // cttz(Pow2) -> Log2(Pow2)
591 // ctlz(Pow2) -> BitWidth - 1 - Log2(Pow2)
592 if (auto *R = IC.tryGetLog2(Op0, match(Op1, m_One()))) {
593 if (IsTZ)
594 return IC.replaceInstUsesWith(II, R);
595 BinaryOperator *BO = BinaryOperator::CreateSub(
596 ConstantInt::get(R->getType(), R->getType()->getScalarSizeInBits() - 1),
597 R);
598 BO->setHasNoSignedWrap();
600 return BO;
601 }
602
603 KnownBits Known = IC.computeKnownBits(Op0, &II);
604
605 // Create a mask for bits above (ctlz) or below (cttz) the first known one.
606 unsigned PossibleZeros = IsTZ ? Known.countMaxTrailingZeros()
607 : Known.countMaxLeadingZeros();
608 unsigned DefiniteZeros = IsTZ ? Known.countMinTrailingZeros()
609 : Known.countMinLeadingZeros();
610
611 // If all bits above (ctlz) or below (cttz) the first known one are known
612 // zero, this value is constant.
613 // FIXME: This should be in InstSimplify because we're replacing an
614 // instruction with a constant.
615 if (PossibleZeros == DefiniteZeros) {
616 auto *C = ConstantInt::get(Op0->getType(), DefiniteZeros);
617 return IC.replaceInstUsesWith(II, C);
618 }
619
620 // If the input to cttz/ctlz is known to be non-zero,
621 // then change the 'ZeroIsPoison' parameter to 'true'
622 // because we know the zero behavior can't affect the result.
623 if (!Known.One.isZero() ||
625 if (!match(II.getArgOperand(1), m_One()))
626 return IC.replaceOperand(II, 1, IC.Builder.getTrue());
627 }
628
629 // Add range attribute since known bits can't completely reflect what we know.
630 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
631 if (BitWidth != 1 && !II.hasRetAttr(Attribute::Range) &&
632 !II.getMetadata(LLVMContext::MD_range)) {
633 ConstantRange Range(APInt(BitWidth, DefiniteZeros),
634 APInt(BitWidth, PossibleZeros + 1));
635 II.addRangeRetAttr(Range);
636 return &II;
637 }
638
639 return nullptr;
640}
641
643 assert(II.getIntrinsicID() == Intrinsic::ctpop &&
644 "Expected ctpop intrinsic");
645 Type *Ty = II.getType();
646 unsigned BitWidth = Ty->getScalarSizeInBits();
647 Value *Op0 = II.getArgOperand(0);
648 Value *X, *Y;
649
650 // ctpop(bitreverse(x)) -> ctpop(x)
651 // ctpop(bswap(x)) -> ctpop(x)
652 if (match(Op0, m_BitReverse(m_Value(X))) || match(Op0, m_BSwap(m_Value(X))))
653 return IC.replaceOperand(II, 0, X);
654
655 // ctpop(rot(x)) -> ctpop(x)
656 if ((match(Op0, m_FShl(m_Value(X), m_Value(Y), m_Value())) ||
657 match(Op0, m_FShr(m_Value(X), m_Value(Y), m_Value()))) &&
658 X == Y)
659 return IC.replaceOperand(II, 0, X);
660
661 // ctpop(x | -x) -> bitwidth - cttz(x, false)
662 if (Op0->hasOneUse() &&
663 match(Op0, m_c_Or(m_Value(X), m_Neg(m_Deferred(X))))) {
664 auto *Cttz = IC.Builder.CreateIntrinsic(Intrinsic::cttz, Ty,
665 {X, IC.Builder.getFalse()});
666 auto *Bw = ConstantInt::get(Ty, APInt(BitWidth, BitWidth));
667 return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Bw, Cttz));
668 }
669
670 // ctpop(~x & (x - 1)) -> cttz(x, false)
671 if (match(Op0,
673 Function *F =
674 Intrinsic::getOrInsertDeclaration(II.getModule(), Intrinsic::cttz, Ty);
675 return CallInst::Create(F, {X, IC.Builder.getFalse()});
676 }
677
678 // Zext doesn't change the number of set bits, so narrow:
679 // ctpop (zext X) --> zext (ctpop X)
680 if (match(Op0, m_OneUse(m_ZExt(m_Value(X))))) {
681 Value *NarrowPop = IC.Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, X);
682 return CastInst::Create(Instruction::ZExt, NarrowPop, Ty);
683 }
684
685 KnownBits Known(BitWidth);
686 IC.computeKnownBits(Op0, Known, &II);
687
688 // If all bits are zero except for exactly one fixed bit, then the result
689 // must be 0 or 1, and we can get that answer by shifting to LSB:
690 // ctpop (X & 32) --> (X & 32) >> 5
691 // TODO: Investigate removing this as its likely unnecessary given the below
692 // `isKnownToBeAPowerOfTwo` check.
693 if ((~Known.Zero).isPowerOf2())
694 return BinaryOperator::CreateLShr(
695 Op0, ConstantInt::get(Ty, (~Known.Zero).exactLogBase2()));
696
697 // More generally we can also handle non-constant power of 2 patterns such as
698 // shl/shr(Pow2, X), (X & -X), etc... by transforming:
699 // ctpop(Pow2OrZero) --> icmp ne X, 0
700 if (IC.isKnownToBeAPowerOfTwo(Op0, /* OrZero */ true))
701 return CastInst::Create(Instruction::ZExt,
704 Ty);
705
706 // Add range attribute since known bits can't completely reflect what we know.
707 if (BitWidth != 1) {
708 ConstantRange OldRange =
709 II.getRange().value_or(ConstantRange::getFull(BitWidth));
710
711 unsigned Lower = Known.countMinPopulation();
712 unsigned Upper = Known.countMaxPopulation() + 1;
713
714 if (Lower == 0 && OldRange.contains(APInt::getZero(BitWidth)) &&
716 Lower = 1;
717
719 Range = Range.intersectWith(OldRange, ConstantRange::Unsigned);
720
721 if (Range != OldRange) {
722 II.addRangeRetAttr(Range);
723 return &II;
724 }
725 }
726
727 return nullptr;
728}
729
730/// Convert a table lookup to shufflevector if the mask is constant.
731/// This could benefit tbl1 if the mask is { 7,6,5,4,3,2,1,0 }, in
732/// which case we could lower the shufflevector with rev64 instructions
733/// as it's actually a byte reverse.
735 InstCombiner::BuilderTy &Builder) {
736 // Bail out if the mask is not a constant.
737 auto *C = dyn_cast<Constant>(II.getArgOperand(1));
738 if (!C)
739 return nullptr;
740
741 auto *VecTy = cast<FixedVectorType>(II.getType());
742 unsigned NumElts = VecTy->getNumElements();
743
744 // Only perform this transformation for <8 x i8> vector types.
745 if (!VecTy->getElementType()->isIntegerTy(8) || NumElts != 8)
746 return nullptr;
747
748 int Indexes[8];
749
750 for (unsigned I = 0; I < NumElts; ++I) {
751 Constant *COp = C->getAggregateElement(I);
752
753 if (!COp || !isa<ConstantInt>(COp))
754 return nullptr;
755
756 Indexes[I] = cast<ConstantInt>(COp)->getLimitedValue();
757
758 // Make sure the mask indices are in range.
759 if ((unsigned)Indexes[I] >= NumElts)
760 return nullptr;
761 }
762
763 auto *V1 = II.getArgOperand(0);
764 auto *V2 = Constant::getNullValue(V1->getType());
765 return Builder.CreateShuffleVector(V1, V2, ArrayRef(Indexes));
766}
767
768// Returns true iff the 2 intrinsics have the same operands, limiting the
769// comparison to the first NumOperands.
770static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E,
771 unsigned NumOperands) {
772 assert(I.arg_size() >= NumOperands && "Not enough operands");
773 assert(E.arg_size() >= NumOperands && "Not enough operands");
774 for (unsigned i = 0; i < NumOperands; i++)
775 if (I.getArgOperand(i) != E.getArgOperand(i))
776 return false;
777 return true;
778}
779
780// Remove trivially empty start/end intrinsic ranges, i.e. a start
781// immediately followed by an end (ignoring debuginfo or other
782// start/end intrinsics in between). As this handles only the most trivial
783// cases, tracking the nesting level is not needed:
784//
785// call @llvm.foo.start(i1 0)
786// call @llvm.foo.start(i1 0) ; This one won't be skipped: it will be removed
787// call @llvm.foo.end(i1 0)
788// call @llvm.foo.end(i1 0) ; &I
789static bool
791 std::function<bool(const IntrinsicInst &)> IsStart) {
792 // We start from the end intrinsic and scan backwards, so that InstCombine
793 // has already processed (and potentially removed) all the instructions
794 // before the end intrinsic.
795 BasicBlock::reverse_iterator BI(EndI), BE(EndI.getParent()->rend());
796 for (; BI != BE; ++BI) {
797 if (auto *I = dyn_cast<IntrinsicInst>(&*BI)) {
798 if (I->isDebugOrPseudoInst() ||
799 I->getIntrinsicID() == EndI.getIntrinsicID())
800 continue;
801 if (IsStart(*I)) {
802 if (haveSameOperands(EndI, *I, EndI.arg_size())) {
804 IC.eraseInstFromFunction(EndI);
805 return true;
806 }
807 // Skip start intrinsics that don't pair with this end intrinsic.
808 continue;
809 }
810 }
811 break;
812 }
813
814 return false;
815}
816
818 removeTriviallyEmptyRange(I, *this, [&I](const IntrinsicInst &II) {
819 // Bail out on the case where the source va_list of a va_copy is destroyed
820 // immediately by a follow-up va_end.
821 return II.getIntrinsicID() == Intrinsic::vastart ||
822 (II.getIntrinsicID() == Intrinsic::vacopy &&
823 I.getArgOperand(0) != II.getArgOperand(1));
824 });
825 return nullptr;
826}
827
829 assert(Call.arg_size() > 1 && "Need at least 2 args to swap");
830 Value *Arg0 = Call.getArgOperand(0), *Arg1 = Call.getArgOperand(1);
831 if (isa<Constant>(Arg0) && !isa<Constant>(Arg1)) {
832 Call.setArgOperand(0, Arg1);
833 Call.setArgOperand(1, Arg0);
834 return &Call;
835 }
836 return nullptr;
837}
838
839/// Creates a result tuple for an overflow intrinsic \p II with a given
840/// \p Result and a constant \p Overflow value.
842 Constant *Overflow) {
843 Constant *V[] = {PoisonValue::get(Result->getType()), Overflow};
844 StructType *ST = cast<StructType>(II->getType());
846 return InsertValueInst::Create(Struct, Result, 0);
847}
848
850InstCombinerImpl::foldIntrinsicWithOverflowCommon(IntrinsicInst *II) {
851 WithOverflowInst *WO = cast<WithOverflowInst>(II);
852 Value *OperationResult = nullptr;
853 Constant *OverflowResult = nullptr;
854 if (OptimizeOverflowCheck(WO->getBinaryOp(), WO->isSigned(), WO->getLHS(),
855 WO->getRHS(), *WO, OperationResult, OverflowResult))
856 return createOverflowTuple(WO, OperationResult, OverflowResult);
857
858 // See whether we can optimize the overflow check with assumption information.
859 for (User *U : WO->users()) {
860 if (!match(U, m_ExtractValue<1>(m_Value())))
861 continue;
862
863 for (auto &AssumeVH : AC.assumptionsFor(U)) {
864 if (!AssumeVH)
865 continue;
866 CallInst *I = cast<CallInst>(AssumeVH);
867 if (!match(I->getArgOperand(0), m_Not(m_Specific(U))))
868 continue;
869 if (!isValidAssumeForContext(I, II, /*DT=*/nullptr,
870 /*AllowEphemerals=*/true))
871 continue;
872 Value *Result =
873 Builder.CreateBinOp(WO->getBinaryOp(), WO->getLHS(), WO->getRHS());
874 Result->takeName(WO);
875 if (auto *Inst = dyn_cast<Instruction>(Result)) {
876 if (WO->isSigned())
877 Inst->setHasNoSignedWrap();
878 else
879 Inst->setHasNoUnsignedWrap();
880 }
881 return createOverflowTuple(WO, Result,
882 ConstantInt::getFalse(U->getType()));
883 }
884 }
885
886 return nullptr;
887}
888
889static bool inputDenormalIsIEEE(const Function &F, const Type *Ty) {
890 Ty = Ty->getScalarType();
891 return F.getDenormalMode(Ty->getFltSemantics()).Input == DenormalMode::IEEE;
892}
893
894static bool inputDenormalIsDAZ(const Function &F, const Type *Ty) {
895 Ty = Ty->getScalarType();
896 return F.getDenormalMode(Ty->getFltSemantics()).inputsAreZero();
897}
898
899/// \returns the compare predicate type if the test performed by
900/// llvm.is.fpclass(x, \p Mask) is equivalent to fcmp o__ x, 0.0 with the
901/// floating-point environment assumed for \p F for type \p Ty
903 const Function &F, Type *Ty) {
904 switch (static_cast<unsigned>(Mask)) {
905 case fcZero:
906 if (inputDenormalIsIEEE(F, Ty))
907 return FCmpInst::FCMP_OEQ;
908 break;
909 case fcZero | fcSubnormal:
910 if (inputDenormalIsDAZ(F, Ty))
911 return FCmpInst::FCMP_OEQ;
912 break;
913 case fcPositive | fcNegZero:
914 if (inputDenormalIsIEEE(F, Ty))
915 return FCmpInst::FCMP_OGE;
916 break;
918 if (inputDenormalIsDAZ(F, Ty))
919 return FCmpInst::FCMP_OGE;
920 break;
922 if (inputDenormalIsIEEE(F, Ty))
923 return FCmpInst::FCMP_OGT;
924 break;
925 case fcNegative | fcPosZero:
926 if (inputDenormalIsIEEE(F, Ty))
927 return FCmpInst::FCMP_OLE;
928 break;
930 if (inputDenormalIsDAZ(F, Ty))
931 return FCmpInst::FCMP_OLE;
932 break;
934 if (inputDenormalIsIEEE(F, Ty))
935 return FCmpInst::FCMP_OLT;
936 break;
937 case fcPosNormal | fcPosInf:
938 if (inputDenormalIsDAZ(F, Ty))
939 return FCmpInst::FCMP_OGT;
940 break;
941 case fcNegNormal | fcNegInf:
942 if (inputDenormalIsDAZ(F, Ty))
943 return FCmpInst::FCMP_OLT;
944 break;
945 case ~fcZero & ~fcNan:
946 if (inputDenormalIsIEEE(F, Ty))
947 return FCmpInst::FCMP_ONE;
948 break;
949 case ~(fcZero | fcSubnormal) & ~fcNan:
950 if (inputDenormalIsDAZ(F, Ty))
951 return FCmpInst::FCMP_ONE;
952 break;
953 default:
954 break;
955 }
956
958}
959
960Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) {
961 Value *Src0 = II.getArgOperand(0);
962 Value *Src1 = II.getArgOperand(1);
963 const ConstantInt *CMask = cast<ConstantInt>(Src1);
964 FPClassTest Mask = static_cast<FPClassTest>(CMask->getZExtValue());
965 const bool IsUnordered = (Mask & fcNan) == fcNan;
966 const bool IsOrdered = (Mask & fcNan) == fcNone;
967 const FPClassTest OrderedMask = Mask & ~fcNan;
968 const FPClassTest OrderedInvertedMask = ~OrderedMask & ~fcNan;
969
970 const bool IsStrict =
971 II.getFunction()->getAttributes().hasFnAttr(Attribute::StrictFP);
972
973 Value *FNegSrc;
974 if (match(Src0, m_FNeg(m_Value(FNegSrc)))) {
975 // is.fpclass (fneg x), mask -> is.fpclass x, (fneg mask)
976
977 II.setArgOperand(1, ConstantInt::get(Src1->getType(), fneg(Mask)));
978 return replaceOperand(II, 0, FNegSrc);
979 }
980
981 Value *FAbsSrc;
982 if (match(Src0, m_FAbs(m_Value(FAbsSrc)))) {
983 II.setArgOperand(1, ConstantInt::get(Src1->getType(), inverse_fabs(Mask)));
984 return replaceOperand(II, 0, FAbsSrc);
985 }
986
987 if ((OrderedMask == fcInf || OrderedInvertedMask == fcInf) &&
988 (IsOrdered || IsUnordered) && !IsStrict) {
989 // is.fpclass(x, fcInf) -> fcmp oeq fabs(x), +inf
990 // is.fpclass(x, ~fcInf) -> fcmp one fabs(x), +inf
991 // is.fpclass(x, fcInf|fcNan) -> fcmp ueq fabs(x), +inf
992 // is.fpclass(x, ~(fcInf|fcNan)) -> fcmp une fabs(x), +inf
996 if (OrderedInvertedMask == fcInf)
997 Pred = IsUnordered ? FCmpInst::FCMP_UNE : FCmpInst::FCMP_ONE;
998
999 Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Src0);
1000 Value *CmpInf = Builder.CreateFCmp(Pred, Fabs, Inf);
1001 CmpInf->takeName(&II);
1002 return replaceInstUsesWith(II, CmpInf);
1003 }
1004
1005 if ((OrderedMask == fcPosInf || OrderedMask == fcNegInf) &&
1006 (IsOrdered || IsUnordered) && !IsStrict) {
1007 // is.fpclass(x, fcPosInf) -> fcmp oeq x, +inf
1008 // is.fpclass(x, fcNegInf) -> fcmp oeq x, -inf
1009 // is.fpclass(x, fcPosInf|fcNan) -> fcmp ueq x, +inf
1010 // is.fpclass(x, fcNegInf|fcNan) -> fcmp ueq x, -inf
1011 Constant *Inf =
1012 ConstantFP::getInfinity(Src0->getType(), OrderedMask == fcNegInf);
1013 Value *EqInf = IsUnordered ? Builder.CreateFCmpUEQ(Src0, Inf)
1014 : Builder.CreateFCmpOEQ(Src0, Inf);
1015
1016 EqInf->takeName(&II);
1017 return replaceInstUsesWith(II, EqInf);
1018 }
1019
1020 if ((OrderedInvertedMask == fcPosInf || OrderedInvertedMask == fcNegInf) &&
1021 (IsOrdered || IsUnordered) && !IsStrict) {
1022 // is.fpclass(x, ~fcPosInf) -> fcmp one x, +inf
1023 // is.fpclass(x, ~fcNegInf) -> fcmp one x, -inf
1024 // is.fpclass(x, ~fcPosInf|fcNan) -> fcmp une x, +inf
1025 // is.fpclass(x, ~fcNegInf|fcNan) -> fcmp une x, -inf
1027 OrderedInvertedMask == fcNegInf);
1028 Value *NeInf = IsUnordered ? Builder.CreateFCmpUNE(Src0, Inf)
1029 : Builder.CreateFCmpONE(Src0, Inf);
1030 NeInf->takeName(&II);
1031 return replaceInstUsesWith(II, NeInf);
1032 }
1033
1034 if (Mask == fcNan && !IsStrict) {
1035 // Equivalent of isnan. Replace with standard fcmp if we don't care about FP
1036 // exceptions.
1037 Value *IsNan =
1038 Builder.CreateFCmpUNO(Src0, ConstantFP::getZero(Src0->getType()));
1039 IsNan->takeName(&II);
1040 return replaceInstUsesWith(II, IsNan);
1041 }
1042
1043 if (Mask == (~fcNan & fcAllFlags) && !IsStrict) {
1044 // Equivalent of !isnan. Replace with standard fcmp.
1045 Value *FCmp =
1046 Builder.CreateFCmpORD(Src0, ConstantFP::getZero(Src0->getType()));
1047 FCmp->takeName(&II);
1048 return replaceInstUsesWith(II, FCmp);
1049 }
1050
1052
1053 // Try to replace with an fcmp with 0
1054 //
1055 // is.fpclass(x, fcZero) -> fcmp oeq x, 0.0
1056 // is.fpclass(x, fcZero | fcNan) -> fcmp ueq x, 0.0
1057 // is.fpclass(x, ~fcZero & ~fcNan) -> fcmp one x, 0.0
1058 // is.fpclass(x, ~fcZero) -> fcmp une x, 0.0
1059 //
1060 // is.fpclass(x, fcPosSubnormal | fcPosNormal | fcPosInf) -> fcmp ogt x, 0.0
1061 // is.fpclass(x, fcPositive | fcNegZero) -> fcmp oge x, 0.0
1062 //
1063 // is.fpclass(x, fcNegSubnormal | fcNegNormal | fcNegInf) -> fcmp olt x, 0.0
1064 // is.fpclass(x, fcNegative | fcPosZero) -> fcmp ole x, 0.0
1065 //
1066 if (!IsStrict && (IsOrdered || IsUnordered) &&
1067 (PredType = fpclassTestIsFCmp0(OrderedMask, *II.getFunction(),
1068 Src0->getType())) !=
1071 // Equivalent of == 0.
1072 Value *FCmp = Builder.CreateFCmp(
1073 IsUnordered ? FCmpInst::getUnorderedPredicate(PredType) : PredType,
1074 Src0, Zero);
1075
1076 FCmp->takeName(&II);
1077 return replaceInstUsesWith(II, FCmp);
1078 }
1079
1080 KnownFPClass Known = computeKnownFPClass(Src0, Mask, &II);
1081
1082 // Clear test bits we know must be false from the source value.
1083 // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
1084 // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other
1085 if ((Mask & Known.KnownFPClasses) != Mask) {
1086 II.setArgOperand(
1087 1, ConstantInt::get(Src1->getType(), Mask & Known.KnownFPClasses));
1088 return &II;
1089 }
1090
1091 // If none of the tests which can return false are possible, fold to true.
1092 // fp_class (nnan x), ~(qnan|snan) -> true
1093 // fp_class (ninf x), ~(ninf|pinf) -> true
1094 if (Mask == Known.KnownFPClasses)
1095 return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
1096
1097 return nullptr;
1098}
1099
1100static std::optional<bool> getKnownSign(Value *Op, const SimplifyQuery &SQ) {
1101 KnownBits Known = computeKnownBits(Op, SQ);
1102 if (Known.isNonNegative())
1103 return false;
1104 if (Known.isNegative())
1105 return true;
1106
1107 Value *X, *Y;
1108 if (match(Op, m_NSWSub(m_Value(X), m_Value(Y))))
1110
1111 return std::nullopt;
1112}
1113
1114static std::optional<bool> getKnownSignOrZero(Value *Op,
1115 const SimplifyQuery &SQ) {
1116 if (std::optional<bool> Sign = getKnownSign(Op, SQ))
1117 return Sign;
1118
1119 Value *X, *Y;
1120 if (match(Op, m_NSWSub(m_Value(X), m_Value(Y))))
1122
1123 return std::nullopt;
1124}
1125
1126/// Return true if two values \p Op0 and \p Op1 are known to have the same sign.
1127static bool signBitMustBeTheSame(Value *Op0, Value *Op1,
1128 const SimplifyQuery &SQ) {
1129 std::optional<bool> Known1 = getKnownSign(Op1, SQ);
1130 if (!Known1)
1131 return false;
1132 std::optional<bool> Known0 = getKnownSign(Op0, SQ);
1133 if (!Known0)
1134 return false;
1135 return *Known0 == *Known1;
1136}
1137
1138/// Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0. This
1139/// can trigger other combines.
1141 InstCombiner::BuilderTy &Builder) {
1142 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1143 assert((MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin ||
1144 MinMaxID == Intrinsic::umax || MinMaxID == Intrinsic::umin) &&
1145 "Expected a min or max intrinsic");
1146
1147 // TODO: Match vectors with undef elements, but undef may not propagate.
1148 Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);
1149 Value *X;
1150 const APInt *C0, *C1;
1151 if (!match(Op0, m_OneUse(m_Add(m_Value(X), m_APInt(C0)))) ||
1152 !match(Op1, m_APInt(C1)))
1153 return nullptr;
1154
1155 // Check for necessary no-wrap and overflow constraints.
1156 bool IsSigned = MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin;
1157 auto *Add = cast<BinaryOperator>(Op0);
1158 if ((IsSigned && !Add->hasNoSignedWrap()) ||
1159 (!IsSigned && !Add->hasNoUnsignedWrap()))
1160 return nullptr;
1161
1162 // If the constant difference overflows, then instsimplify should reduce the
1163 // min/max to the add or C1.
1164 bool Overflow;
1165 APInt CDiff =
1166 IsSigned ? C1->ssub_ov(*C0, Overflow) : C1->usub_ov(*C0, Overflow);
1167 assert(!Overflow && "Expected simplify of min/max");
1168
1169 // min/max (add X, C0), C1 --> add (min/max X, C1 - C0), C0
1170 // Note: the "mismatched" no-overflow setting does not propagate.
1171 Constant *NewMinMaxC = ConstantInt::get(II->getType(), CDiff);
1172 Value *NewMinMax = Builder.CreateBinaryIntrinsic(MinMaxID, X, NewMinMaxC);
1173 return IsSigned ? BinaryOperator::CreateNSWAdd(NewMinMax, Add->getOperand(1))
1174 : BinaryOperator::CreateNUWAdd(NewMinMax, Add->getOperand(1));
1175}
1176/// Match a sadd_sat or ssub_sat which is using min/max to clamp the value.
1177Instruction *InstCombinerImpl::matchSAddSubSat(IntrinsicInst &MinMax1) {
1178 Type *Ty = MinMax1.getType();
1179
1180 // We are looking for a tree of:
1181 // max(INT_MIN, min(INT_MAX, add(sext(A), sext(B))))
1182 // Where the min and max could be reversed
1183 Instruction *MinMax2;
1184 BinaryOperator *AddSub;
1185 const APInt *MinValue, *MaxValue;
1186 if (match(&MinMax1, m_SMin(m_Instruction(MinMax2), m_APInt(MaxValue)))) {
1187 if (!match(MinMax2, m_SMax(m_BinOp(AddSub), m_APInt(MinValue))))
1188 return nullptr;
1189 } else if (match(&MinMax1,
1190 m_SMax(m_Instruction(MinMax2), m_APInt(MinValue)))) {
1191 if (!match(MinMax2, m_SMin(m_BinOp(AddSub), m_APInt(MaxValue))))
1192 return nullptr;
1193 } else
1194 return nullptr;
1195
1196 // Check that the constants clamp a saturate, and that the new type would be
1197 // sensible to convert to.
1198 if (!(*MaxValue + 1).isPowerOf2() || -*MinValue != *MaxValue + 1)
1199 return nullptr;
1200 // In what bitwidth can this be treated as saturating arithmetics?
1201 unsigned NewBitWidth = (*MaxValue + 1).logBase2() + 1;
1202 // FIXME: This isn't quite right for vectors, but using the scalar type is a
1203 // good first approximation for what should be done there.
1204 if (!shouldChangeType(Ty->getScalarType()->getIntegerBitWidth(), NewBitWidth))
1205 return nullptr;
1206
1207 // Also make sure that the inner min/max and the add/sub have one use.
1208 if (!MinMax2->hasOneUse() || !AddSub->hasOneUse())
1209 return nullptr;
1210
1211 // Create the new type (which can be a vector type)
1212 Type *NewTy = Ty->getWithNewBitWidth(NewBitWidth);
1213
1214 Intrinsic::ID IntrinsicID;
1215 if (AddSub->getOpcode() == Instruction::Add)
1216 IntrinsicID = Intrinsic::sadd_sat;
1217 else if (AddSub->getOpcode() == Instruction::Sub)
1218 IntrinsicID = Intrinsic::ssub_sat;
1219 else
1220 return nullptr;
1221
1222 // The two operands of the add/sub must be nsw-truncatable to the NewTy. This
1223 // is usually achieved via a sext from a smaller type.
1224 if (ComputeMaxSignificantBits(AddSub->getOperand(0), AddSub) > NewBitWidth ||
1225 ComputeMaxSignificantBits(AddSub->getOperand(1), AddSub) > NewBitWidth)
1226 return nullptr;
1227
1228 // Finally create and return the sat intrinsic, truncated to the new type
1229 Value *AT = Builder.CreateTrunc(AddSub->getOperand(0), NewTy);
1230 Value *BT = Builder.CreateTrunc(AddSub->getOperand(1), NewTy);
1231 Value *Sat = Builder.CreateIntrinsic(IntrinsicID, NewTy, {AT, BT});
1232 return CastInst::Create(Instruction::SExt, Sat, Ty);
1233}
1234
1235
1236/// If we have a clamp pattern like max (min X, 42), 41 -- where the output
1237/// can only be one of two possible constant values -- turn that into a select
1238/// of constants.
1240 InstCombiner::BuilderTy &Builder) {
1241 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1242 Value *X;
1243 const APInt *C0, *C1;
1244 if (!match(I1, m_APInt(C1)) || !I0->hasOneUse())
1245 return nullptr;
1246
1248 switch (II->getIntrinsicID()) {
1249 case Intrinsic::smax:
1250 if (match(I0, m_SMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)
1251 Pred = ICmpInst::ICMP_SGT;
1252 break;
1253 case Intrinsic::smin:
1254 if (match(I0, m_SMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)
1255 Pred = ICmpInst::ICMP_SLT;
1256 break;
1257 case Intrinsic::umax:
1258 if (match(I0, m_UMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)
1259 Pred = ICmpInst::ICMP_UGT;
1260 break;
1261 case Intrinsic::umin:
1262 if (match(I0, m_UMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)
1263 Pred = ICmpInst::ICMP_ULT;
1264 break;
1265 default:
1266 llvm_unreachable("Expected min/max intrinsic");
1267 }
1268 if (Pred == CmpInst::BAD_ICMP_PREDICATE)
1269 return nullptr;
1270
1271 // max (min X, 42), 41 --> X > 41 ? 42 : 41
1272 // min (max X, 42), 43 --> X < 43 ? 42 : 43
1273 Value *Cmp = Builder.CreateICmp(Pred, X, I1);
1274 return SelectInst::Create(Cmp, ConstantInt::get(II->getType(), *C0), I1);
1275}
1276
1277/// If this min/max has a constant operand and an operand that is a matching
1278/// min/max with a constant operand, constant-fold the 2 constant operands.
1280 IRBuilderBase &Builder,
1281 const SimplifyQuery &SQ) {
1282 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1283 auto *LHS = dyn_cast<MinMaxIntrinsic>(II->getArgOperand(0));
1284 if (!LHS)
1285 return nullptr;
1286
1287 Constant *C0, *C1;
1288 if (!match(LHS->getArgOperand(1), m_ImmConstant(C0)) ||
1289 !match(II->getArgOperand(1), m_ImmConstant(C1)))
1290 return nullptr;
1291
1292 // max (max X, C0), C1 --> max X, (max C0, C1)
1293 // min (min X, C0), C1 --> min X, (min C0, C1)
1294 // umax (smax X, nneg C0), nneg C1 --> smax X, (umax C0, C1)
1295 // smin (umin X, nneg C0), nneg C1 --> umin X, (smin C0, C1)
1296 Intrinsic::ID InnerMinMaxID = LHS->getIntrinsicID();
1297 if (InnerMinMaxID != MinMaxID &&
1298 !(((MinMaxID == Intrinsic::umax && InnerMinMaxID == Intrinsic::smax) ||
1299 (MinMaxID == Intrinsic::smin && InnerMinMaxID == Intrinsic::umin)) &&
1300 isKnownNonNegative(C0, SQ) && isKnownNonNegative(C1, SQ)))
1301 return nullptr;
1302
1304 Value *CondC = Builder.CreateICmp(Pred, C0, C1);
1305 Value *NewC = Builder.CreateSelect(CondC, C0, C1);
1306 return Builder.CreateIntrinsic(InnerMinMaxID, II->getType(),
1307 {LHS->getArgOperand(0), NewC});
1308}
1309
1310/// If this min/max has a matching min/max operand with a constant, try to push
1311/// the constant operand into this instruction. This can enable more folds.
1312static Instruction *
1314 InstCombiner::BuilderTy &Builder) {
1315 // Match and capture a min/max operand candidate.
1316 Value *X, *Y;
1317 Constant *C;
1318 Instruction *Inner;
1320 m_Instruction(Inner),
1322 m_Value(Y))))
1323 return nullptr;
1324
1325 // The inner op must match. Check for constants to avoid infinite loops.
1326 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1327 auto *InnerMM = dyn_cast<IntrinsicInst>(Inner);
1328 if (!InnerMM || InnerMM->getIntrinsicID() != MinMaxID ||
1330 return nullptr;
1331
1332 // max (max X, C), Y --> max (max X, Y), C
1334 MinMaxID, II->getType());
1335 Value *NewInner = Builder.CreateBinaryIntrinsic(MinMaxID, X, Y);
1336 NewInner->takeName(Inner);
1337 return CallInst::Create(MinMax, {NewInner, C});
1338}
1339
1340/// Reduce a sequence of min/max intrinsics with a common operand.
1342 // Match 3 of the same min/max ops. Example: umin(umin(), umin()).
1343 auto *LHS = dyn_cast<IntrinsicInst>(II->getArgOperand(0));
1344 auto *RHS = dyn_cast<IntrinsicInst>(II->getArgOperand(1));
1345 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1346 if (!LHS || !RHS || LHS->getIntrinsicID() != MinMaxID ||
1347 RHS->getIntrinsicID() != MinMaxID ||
1348 (!LHS->hasOneUse() && !RHS->hasOneUse()))
1349 return nullptr;
1350
1351 Value *A = LHS->getArgOperand(0);
1352 Value *B = LHS->getArgOperand(1);
1353 Value *C = RHS->getArgOperand(0);
1354 Value *D = RHS->getArgOperand(1);
1355
1356 // Look for a common operand.
1357 Value *MinMaxOp = nullptr;
1358 Value *ThirdOp = nullptr;
1359 if (LHS->hasOneUse()) {
1360 // If the LHS is only used in this chain and the RHS is used outside of it,
1361 // reuse the RHS min/max because that will eliminate the LHS.
1362 if (D == A || C == A) {
1363 // min(min(a, b), min(c, a)) --> min(min(c, a), b)
1364 // min(min(a, b), min(a, d)) --> min(min(a, d), b)
1365 MinMaxOp = RHS;
1366 ThirdOp = B;
1367 } else if (D == B || C == B) {
1368 // min(min(a, b), min(c, b)) --> min(min(c, b), a)
1369 // min(min(a, b), min(b, d)) --> min(min(b, d), a)
1370 MinMaxOp = RHS;
1371 ThirdOp = A;
1372 }
1373 } else {
1374 assert(RHS->hasOneUse() && "Expected one-use operand");
1375 // Reuse the LHS. This will eliminate the RHS.
1376 if (D == A || D == B) {
1377 // min(min(a, b), min(c, a)) --> min(min(a, b), c)
1378 // min(min(a, b), min(c, b)) --> min(min(a, b), c)
1379 MinMaxOp = LHS;
1380 ThirdOp = C;
1381 } else if (C == A || C == B) {
1382 // min(min(a, b), min(b, d)) --> min(min(a, b), d)
1383 // min(min(a, b), min(c, b)) --> min(min(a, b), d)
1384 MinMaxOp = LHS;
1385 ThirdOp = D;
1386 }
1387 }
1388
1389 if (!MinMaxOp || !ThirdOp)
1390 return nullptr;
1391
1392 Module *Mod = II->getModule();
1393 Function *MinMax =
1394 Intrinsic::getOrInsertDeclaration(Mod, MinMaxID, II->getType());
1395 return CallInst::Create(MinMax, { MinMaxOp, ThirdOp });
1396}
1397
1398/// If all arguments of the intrinsic are unary shuffles with the same mask,
1399/// try to shuffle after the intrinsic.
1402 if (!isTriviallyVectorizable(II->getIntrinsicID()) ||
1403 !II->getCalledFunction()->isSpeculatable())
1404 return nullptr;
1405
1406 Value *X;
1407 Constant *C;
1408 ArrayRef<int> Mask;
1409 auto *NonConstArg = find_if_not(II->args(), [&II](Use &Arg) {
1410 return isa<Constant>(Arg.get()) ||
1411 isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
1412 Arg.getOperandNo(), nullptr);
1413 });
1414 if (!NonConstArg ||
1415 !match(NonConstArg, m_Shuffle(m_Value(X), m_Poison(), m_Mask(Mask))))
1416 return nullptr;
1417
1418 // At least 1 operand must be a shuffle with 1 use because we are creating 2
1419 // instructions.
1420 if (none_of(II->args(), match_fn(m_OneUse(m_Shuffle(m_Value(), m_Value())))))
1421 return nullptr;
1422
1423 // See if all arguments are shuffled with the same mask.
1425 Type *SrcTy = X->getType();
1426 for (Use &Arg : II->args()) {
1427 if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
1428 Arg.getOperandNo(), nullptr))
1429 NewArgs.push_back(Arg);
1430 else if (match(&Arg,
1431 m_Shuffle(m_Value(X), m_Poison(), m_SpecificMask(Mask))) &&
1432 X->getType() == SrcTy)
1433 NewArgs.push_back(X);
1434 else if (match(&Arg, m_ImmConstant(C))) {
1435 // If it's a constant, try find the constant that would be shuffled to C.
1436 if (Constant *ShuffledC =
1437 unshuffleConstant(Mask, C, cast<VectorType>(SrcTy)))
1438 NewArgs.push_back(ShuffledC);
1439 else
1440 return nullptr;
1441 } else
1442 return nullptr;
1443 }
1444
1445 // intrinsic (shuf X, M), (shuf Y, M), ... --> shuf (intrinsic X, Y, ...), M
1446 Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
1447 // Result type might be a different vector width.
1448 // TODO: Check that the result type isn't widened?
1449 VectorType *ResTy =
1450 VectorType::get(II->getType()->getScalarType(), cast<VectorType>(SrcTy));
1451 Value *NewIntrinsic =
1452 Builder.CreateIntrinsic(ResTy, II->getIntrinsicID(), NewArgs, FPI);
1453 return new ShuffleVectorInst(NewIntrinsic, Mask);
1454}
1455
1456/// If all arguments of the intrinsic are reverses, try to pull the reverse
1457/// after the intrinsic.
1459 if (!isTriviallyVectorizable(II->getIntrinsicID()))
1460 return nullptr;
1461
1462 // At least 1 operand must be a reverse with 1 use because we are creating 2
1463 // instructions.
1464 if (none_of(II->args(), [](Value *V) {
1465 return match(V, m_OneUse(m_VecReverse(m_Value())));
1466 }))
1467 return nullptr;
1468
1469 Value *X;
1470 Constant *C;
1471 SmallVector<Value *> NewArgs;
1472 for (Use &Arg : II->args()) {
1473 if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
1474 Arg.getOperandNo(), nullptr))
1475 NewArgs.push_back(Arg);
1476 else if (match(&Arg, m_VecReverse(m_Value(X))))
1477 NewArgs.push_back(X);
1478 else if (isSplatValue(Arg))
1479 NewArgs.push_back(Arg);
1480 else if (match(&Arg, m_ImmConstant(C)))
1481 NewArgs.push_back(Builder.CreateVectorReverse(C));
1482 else
1483 return nullptr;
1484 }
1485
1486 // intrinsic (reverse X), (reverse Y), ... --> reverse (intrinsic X, Y, ...)
1487 Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
1488 Instruction *NewIntrinsic = Builder.CreateIntrinsic(
1489 II->getType(), II->getIntrinsicID(), NewArgs, FPI);
1490 return Builder.CreateVectorReverse(NewIntrinsic);
1491}
1492
1493/// Fold the following cases and accepts bswap and bitreverse intrinsics:
1494/// bswap(logic_op(bswap(x), y)) --> logic_op(x, bswap(y))
1495/// bswap(logic_op(bswap(x), bswap(y))) --> logic_op(x, y) (ignores multiuse)
1496template <Intrinsic::ID IntrID>
1498 InstCombiner::BuilderTy &Builder) {
1499 static_assert(IntrID == Intrinsic::bswap || IntrID == Intrinsic::bitreverse,
1500 "This helper only supports BSWAP and BITREVERSE intrinsics");
1501
1502 Value *X, *Y;
1503 // Find bitwise logic op. Check that it is a BinaryOperator explicitly so we
1504 // don't match ConstantExpr that aren't meaningful for this transform.
1507 Value *OldReorderX, *OldReorderY;
1509
1510 // If both X and Y are bswap/bitreverse, the transform reduces the number
1511 // of instructions even if there's multiuse.
1512 // If only one operand is bswap/bitreverse, we need to ensure the operand
1513 // have only one use.
1514 if (match(X, m_Intrinsic<IntrID>(m_Value(OldReorderX))) &&
1515 match(Y, m_Intrinsic<IntrID>(m_Value(OldReorderY)))) {
1516 return BinaryOperator::Create(Op, OldReorderX, OldReorderY);
1517 }
1518
1519 if (match(X, m_OneUse(m_Intrinsic<IntrID>(m_Value(OldReorderX))))) {
1520 Value *NewReorder = Builder.CreateUnaryIntrinsic(IntrID, Y);
1521 return BinaryOperator::Create(Op, OldReorderX, NewReorder);
1522 }
1523
1524 if (match(Y, m_OneUse(m_Intrinsic<IntrID>(m_Value(OldReorderY))))) {
1525 Value *NewReorder = Builder.CreateUnaryIntrinsic(IntrID, X);
1526 return BinaryOperator::Create(Op, NewReorder, OldReorderY);
1527 }
1528 }
1529 return nullptr;
1530}
1531
1532/// Helper to match idempotent binary intrinsics, namely, intrinsics where
1533/// `f(f(x, y), y) == f(x, y)` holds.
1535 switch (IID) {
1536 case Intrinsic::smax:
1537 case Intrinsic::smin:
1538 case Intrinsic::umax:
1539 case Intrinsic::umin:
1540 case Intrinsic::maximum:
1541 case Intrinsic::minimum:
1542 case Intrinsic::maximumnum:
1543 case Intrinsic::minimumnum:
1544 case Intrinsic::maxnum:
1545 case Intrinsic::minnum:
1546 return true;
1547 default:
1548 return false;
1549 }
1550}
1551
1552/// Attempt to simplify value-accumulating recurrences of kind:
1553/// %umax.acc = phi i8 [ %umax, %backedge ], [ %a, %entry ]
1554/// %umax = call i8 @llvm.umax.i8(i8 %umax.acc, i8 %b)
1555/// And let the idempotent binary intrinsic be hoisted, when the operands are
1556/// known to be loop-invariant.
1558 IntrinsicInst *II) {
1559 PHINode *PN;
1560 Value *Init, *OtherOp;
1561
1562 // A binary intrinsic recurrence with loop-invariant operands is equivalent to
1563 // `call @llvm.binary.intrinsic(Init, OtherOp)`.
1564 auto IID = II->getIntrinsicID();
1565 if (!isIdempotentBinaryIntrinsic(IID) ||
1567 !IC.getDominatorTree().dominates(OtherOp, PN))
1568 return nullptr;
1569
1570 auto *InvariantBinaryInst =
1571 IC.Builder.CreateBinaryIntrinsic(IID, Init, OtherOp);
1572 if (isa<FPMathOperator>(InvariantBinaryInst))
1573 cast<Instruction>(InvariantBinaryInst)->copyFastMathFlags(II);
1574 return InvariantBinaryInst;
1575}
1576
1577static Value *simplifyReductionOperand(Value *Arg, bool CanReorderLanes) {
1578 if (!CanReorderLanes)
1579 return nullptr;
1580
1581 Value *V;
1582 if (match(Arg, m_VecReverse(m_Value(V))))
1583 return V;
1584
1585 ArrayRef<int> Mask;
1586 if (!isa<FixedVectorType>(Arg->getType()) ||
1587 !match(Arg, m_Shuffle(m_Value(V), m_Undef(), m_Mask(Mask))) ||
1588 !cast<ShuffleVectorInst>(Arg)->isSingleSource())
1589 return nullptr;
1590
1591 int Sz = Mask.size();
1592 SmallBitVector UsedIndices(Sz);
1593 for (int Idx : Mask) {
1594 if (Idx == PoisonMaskElem || UsedIndices.test(Idx))
1595 return nullptr;
1596 UsedIndices.set(Idx);
1597 }
1598
1599 // Can remove shuffle iff just shuffled elements, no repeats, undefs, or
1600 // other changes.
1601 return UsedIndices.all() ? V : nullptr;
1602}
1603
1604/// Fold an unsigned minimum of trailing or leading zero bits counts:
1605/// umin(cttz(CtOp, ZeroUndef), ConstOp) --> cttz(CtOp | (1 << ConstOp))
1606/// umin(ctlz(CtOp, ZeroUndef), ConstOp) --> ctlz(CtOp | (SignedMin
1607/// >> ConstOp))
1608template <Intrinsic::ID IntrID>
1609static Value *
1611 const DataLayout &DL,
1612 InstCombiner::BuilderTy &Builder) {
1613 static_assert(IntrID == Intrinsic::cttz || IntrID == Intrinsic::ctlz,
1614 "This helper only supports cttz and ctlz intrinsics");
1615
1616 Value *CtOp;
1617 Value *ZeroUndef;
1618 if (!match(I0,
1619 m_OneUse(m_Intrinsic<IntrID>(m_Value(CtOp), m_Value(ZeroUndef)))))
1620 return nullptr;
1621
1622 unsigned BitWidth = I1->getType()->getScalarSizeInBits();
1623 auto LessBitWidth = [BitWidth](auto &C) { return C.ult(BitWidth); };
1624 if (!match(I1, m_CheckedInt(LessBitWidth)))
1625 // We have a constant >= BitWidth (which can be handled by CVP)
1626 // or a non-splat vector with elements < and >= BitWidth
1627 return nullptr;
1628
1629 Type *Ty = I1->getType();
1631 IntrID == Intrinsic::cttz ? Instruction::Shl : Instruction::LShr,
1632 IntrID == Intrinsic::cttz
1633 ? ConstantInt::get(Ty, 1)
1634 : ConstantInt::get(Ty, APInt::getSignedMinValue(BitWidth)),
1635 cast<Constant>(I1), DL);
1636 return Builder.CreateBinaryIntrinsic(
1637 IntrID, Builder.CreateOr(CtOp, NewConst),
1638 ConstantInt::getTrue(ZeroUndef->getType()));
1639}
1640
1641/// Return whether "X LOp (Y ROp Z)" is always equal to
1642/// "(X LOp Y) ROp (X LOp Z)".
1644 bool HasNSW, Intrinsic::ID ROp) {
1645 switch (ROp) {
1646 case Intrinsic::umax:
1647 case Intrinsic::umin:
1648 if (HasNUW && LOp == Instruction::Add)
1649 return true;
1650 if (HasNUW && LOp == Instruction::Shl)
1651 return true;
1652 return false;
1653 case Intrinsic::smax:
1654 case Intrinsic::smin:
1655 return HasNSW && LOp == Instruction::Add;
1656 default:
1657 return false;
1658 }
1659}
1660
1661// Attempts to factorise a common term
1662// in an instruction that has the form "(A op' B) op (C op' D)
1663// where op is an intrinsic and op' is a binop
1664static Value *
1666 InstCombiner::BuilderTy &Builder) {
1667 Value *LHS = II->getOperand(0), *RHS = II->getOperand(1);
1668 Intrinsic::ID TopLevelOpcode = II->getIntrinsicID();
1669
1672
1673 if (!Op0 || !Op1)
1674 return nullptr;
1675
1676 if (Op0->getOpcode() != Op1->getOpcode())
1677 return nullptr;
1678
1679 if (!Op0->hasOneUse() || !Op1->hasOneUse())
1680 return nullptr;
1681
1682 Instruction::BinaryOps InnerOpcode =
1683 static_cast<Instruction::BinaryOps>(Op0->getOpcode());
1684 bool HasNUW = Op0->hasNoUnsignedWrap() && Op1->hasNoUnsignedWrap();
1685 bool HasNSW = Op0->hasNoSignedWrap() && Op1->hasNoSignedWrap();
1686
1687 if (!leftDistributesOverRight(InnerOpcode, HasNUW, HasNSW, TopLevelOpcode))
1688 return nullptr;
1689
1690 Value *A = Op0->getOperand(0);
1691 Value *B = Op0->getOperand(1);
1692 Value *C = Op1->getOperand(0);
1693 Value *D = Op1->getOperand(1);
1694
1695 // Attempts to swap variables such that A equals C or B equals D,
1696 // if the inner operation is commutative.
1697 if (Op0->isCommutative() && A != C && B != D) {
1698 if (A == D || B == C)
1699 std::swap(C, D);
1700 else
1701 return nullptr;
1702 }
1703
1704 BinaryOperator *NewBinop;
1705 if (A == C) {
1706 Value *NewIntrinsic = Builder.CreateBinaryIntrinsic(TopLevelOpcode, B, D);
1707 NewBinop =
1708 cast<BinaryOperator>(Builder.CreateBinOp(InnerOpcode, A, NewIntrinsic));
1709 } else if (B == D) {
1710 Value *NewIntrinsic = Builder.CreateBinaryIntrinsic(TopLevelOpcode, A, C);
1711 NewBinop =
1712 cast<BinaryOperator>(Builder.CreateBinOp(InnerOpcode, NewIntrinsic, B));
1713 } else {
1714 return nullptr;
1715 }
1716
1717 NewBinop->setHasNoUnsignedWrap(HasNUW);
1718 NewBinop->setHasNoSignedWrap(HasNSW);
1719
1720 return NewBinop;
1721}
1722
1723/// CallInst simplification. This mostly only handles folding of intrinsic
1724/// instructions. For normal calls, it allows visitCallBase to do the heavy
1725/// lifting.
1727 // Don't try to simplify calls without uses. It will not do anything useful,
1728 // but will result in the following folds being skipped.
1729 if (!CI.use_empty()) {
1730 SmallVector<Value *, 8> Args(CI.args());
1731 if (Value *V = simplifyCall(&CI, CI.getCalledOperand(), Args,
1732 SQ.getWithInstruction(&CI)))
1733 return replaceInstUsesWith(CI, V);
1734 }
1735
1736 if (Value *FreedOp = getFreedOperand(&CI, &TLI))
1737 return visitFree(CI, FreedOp);
1738
1739 // If the caller function (i.e. us, the function that contains this CallInst)
1740 // is nounwind, mark the call as nounwind, even if the callee isn't.
1741 if (CI.getFunction()->doesNotThrow() && !CI.doesNotThrow()) {
1742 CI.setDoesNotThrow();
1743 return &CI;
1744 }
1745
1747 if (!II)
1748 return visitCallBase(CI);
1749
1750 // Intrinsics cannot occur in an invoke or a callbr, so handle them here
1751 // instead of in visitCallBase.
1752 if (auto *MI = dyn_cast<AnyMemIntrinsic>(II)) {
1753 if (auto NumBytes = MI->getLengthInBytes()) {
1754 // memmove/cpy/set of zero bytes is a noop.
1755 if (NumBytes->isZero())
1756 return eraseInstFromFunction(CI);
1757
1758 // For atomic unordered mem intrinsics if len is not a positive or
1759 // not a multiple of element size then behavior is undefined.
1760 if (MI->isAtomic() &&
1761 (NumBytes->isNegative() ||
1762 (NumBytes->getZExtValue() % MI->getElementSizeInBytes() != 0))) {
1764 assert(MI->getType()->isVoidTy() &&
1765 "non void atomic unordered mem intrinsic");
1766 return eraseInstFromFunction(*MI);
1767 }
1768 }
1769
1770 // No other transformations apply to volatile transfers.
1771 if (MI->isVolatile())
1772 return nullptr;
1773
1775 // memmove(x,x,size) -> noop.
1776 if (MTI->getSource() == MTI->getDest())
1777 return eraseInstFromFunction(CI);
1778 }
1779
1780 auto IsPointerUndefined = [MI](Value *Ptr) {
1781 return isa<ConstantPointerNull>(Ptr) &&
1783 MI->getFunction(),
1784 cast<PointerType>(Ptr->getType())->getAddressSpace());
1785 };
1786 bool SrcIsUndefined = false;
1787 // If we can determine a pointer alignment that is bigger than currently
1788 // set, update the alignment.
1789 if (auto *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
1791 return I;
1792 SrcIsUndefined = IsPointerUndefined(MTI->getRawSource());
1793 } else if (auto *MSI = dyn_cast<AnyMemSetInst>(MI)) {
1794 if (Instruction *I = SimplifyAnyMemSet(MSI))
1795 return I;
1796 }
1797
1798 // If src/dest is null, this memory intrinsic must be a noop.
1799 if (SrcIsUndefined || IsPointerUndefined(MI->getRawDest())) {
1800 Builder.CreateAssumption(Builder.CreateIsNull(MI->getLength()));
1801 return eraseInstFromFunction(CI);
1802 }
1803
1804 // If we have a memmove and the source operation is a constant global,
1805 // then the source and dest pointers can't alias, so we can change this
1806 // into a call to memcpy.
1807 if (auto *MMI = dyn_cast<AnyMemMoveInst>(MI)) {
1808 if (GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource()))
1809 if (GVSrc->isConstant()) {
1810 Module *M = CI.getModule();
1811 Intrinsic::ID MemCpyID =
1812 MMI->isAtomic()
1813 ? Intrinsic::memcpy_element_unordered_atomic
1814 : Intrinsic::memcpy;
1815 Type *Tys[3] = { CI.getArgOperand(0)->getType(),
1816 CI.getArgOperand(1)->getType(),
1817 CI.getArgOperand(2)->getType() };
1819 Intrinsic::getOrInsertDeclaration(M, MemCpyID, Tys));
1820 return II;
1821 }
1822 }
1823 }
1824
1825 // For fixed width vector result intrinsics, use the generic demanded vector
1826 // support.
1827 if (auto *IIFVTy = dyn_cast<FixedVectorType>(II->getType())) {
1828 auto VWidth = IIFVTy->getNumElements();
1829 APInt PoisonElts(VWidth, 0);
1830 APInt AllOnesEltMask(APInt::getAllOnes(VWidth));
1831 if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, PoisonElts)) {
1832 if (V != II)
1833 return replaceInstUsesWith(*II, V);
1834 return II;
1835 }
1836 }
1837
1838 if (II->isCommutative()) {
1839 if (auto Pair = matchSymmetricPair(II->getOperand(0), II->getOperand(1))) {
1840 replaceOperand(*II, 0, Pair->first);
1841 replaceOperand(*II, 1, Pair->second);
1842 return II;
1843 }
1844
1845 if (CallInst *NewCall = canonicalizeConstantArg0ToArg1(CI))
1846 return NewCall;
1847 }
1848
1849 // Unused constrained FP intrinsic calls may have declared side effect, which
1850 // prevents it from being removed. In some cases however the side effect is
1851 // actually absent. To detect this case, call SimplifyConstrainedFPCall. If it
1852 // returns a replacement, the call may be removed.
1853 if (CI.use_empty() && isa<ConstrainedFPIntrinsic>(CI)) {
1854 if (simplifyConstrainedFPCall(&CI, SQ.getWithInstruction(&CI)))
1855 return eraseInstFromFunction(CI);
1856 }
1857
1858 Intrinsic::ID IID = II->getIntrinsicID();
1859 switch (IID) {
1860 case Intrinsic::objectsize: {
1861 SmallVector<Instruction *> InsertedInstructions;
1862 if (Value *V = lowerObjectSizeCall(II, DL, &TLI, AA, /*MustSucceed=*/false,
1863 &InsertedInstructions)) {
1864 for (Instruction *Inserted : InsertedInstructions)
1865 Worklist.add(Inserted);
1866 return replaceInstUsesWith(CI, V);
1867 }
1868 return nullptr;
1869 }
1870 case Intrinsic::abs: {
1871 Value *IIOperand = II->getArgOperand(0);
1872 bool IntMinIsPoison = cast<Constant>(II->getArgOperand(1))->isOneValue();
1873
1874 // abs(-x) -> abs(x)
1875 Value *X;
1876 if (match(IIOperand, m_Neg(m_Value(X)))) {
1877 if (cast<Instruction>(IIOperand)->hasNoSignedWrap() || IntMinIsPoison)
1878 replaceOperand(*II, 1, Builder.getTrue());
1879 return replaceOperand(*II, 0, X);
1880 }
1881 if (match(IIOperand, m_c_Select(m_Neg(m_Value(X)), m_Deferred(X))))
1882 return replaceOperand(*II, 0, X);
1883
1884 Value *Y;
1885 // abs(a * abs(b)) -> abs(a * b)
1886 if (match(IIOperand,
1889 bool NSW =
1890 cast<Instruction>(IIOperand)->hasNoSignedWrap() && IntMinIsPoison;
1891 auto *XY = NSW ? Builder.CreateNSWMul(X, Y) : Builder.CreateMul(X, Y);
1892 return replaceOperand(*II, 0, XY);
1893 }
1894
1895 if (std::optional<bool> Known =
1896 getKnownSignOrZero(IIOperand, SQ.getWithInstruction(II))) {
1897 // abs(x) -> x if x >= 0 (include abs(x-y) --> x - y where x >= y)
1898 // abs(x) -> x if x > 0 (include abs(x-y) --> x - y where x > y)
1899 if (!*Known)
1900 return replaceInstUsesWith(*II, IIOperand);
1901
1902 // abs(x) -> -x if x < 0
1903 // abs(x) -> -x if x < = 0 (include abs(x-y) --> y - x where x <= y)
1904 if (IntMinIsPoison)
1905 return BinaryOperator::CreateNSWNeg(IIOperand);
1906 return BinaryOperator::CreateNeg(IIOperand);
1907 }
1908
1909 // abs (sext X) --> zext (abs X*)
1910 // Clear the IsIntMin (nsw) bit on the abs to allow narrowing.
1911 if (match(IIOperand, m_OneUse(m_SExt(m_Value(X))))) {
1912 Value *NarrowAbs =
1913 Builder.CreateBinaryIntrinsic(Intrinsic::abs, X, Builder.getFalse());
1914 return CastInst::Create(Instruction::ZExt, NarrowAbs, II->getType());
1915 }
1916
1917 // Match a complicated way to check if a number is odd/even:
1918 // abs (srem X, 2) --> and X, 1
1919 const APInt *C;
1920 if (match(IIOperand, m_SRem(m_Value(X), m_APInt(C))) && *C == 2)
1921 return BinaryOperator::CreateAnd(X, ConstantInt::get(II->getType(), 1));
1922
1923 break;
1924 }
1925 case Intrinsic::umin: {
1926 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1927 // umin(x, 1) == zext(x != 0)
1928 if (match(I1, m_One())) {
1929 assert(II->getType()->getScalarSizeInBits() != 1 &&
1930 "Expected simplify of umin with max constant");
1931 Value *Zero = Constant::getNullValue(I0->getType());
1932 Value *Cmp = Builder.CreateICmpNE(I0, Zero);
1933 return CastInst::Create(Instruction::ZExt, Cmp, II->getType());
1934 }
1935 // umin(cttz(x), const) --> cttz(x | (1 << const))
1936 if (Value *FoldedCttz =
1938 I0, I1, DL, Builder))
1939 return replaceInstUsesWith(*II, FoldedCttz);
1940 // umin(ctlz(x), const) --> ctlz(x | (SignedMin >> const))
1941 if (Value *FoldedCtlz =
1943 I0, I1, DL, Builder))
1944 return replaceInstUsesWith(*II, FoldedCtlz);
1945 [[fallthrough]];
1946 }
1947 case Intrinsic::umax: {
1948 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1949 Value *X, *Y;
1950 if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_ZExt(m_Value(Y))) &&
1951 (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
1952 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
1953 return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());
1954 }
1955 Constant *C;
1956 if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_Constant(C)) &&
1957 I0->hasOneUse()) {
1958 if (Constant *NarrowC = getLosslessUnsignedTrunc(C, X->getType(), DL)) {
1959 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);
1960 return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());
1961 }
1962 }
1963 // If C is not 0:
1964 // umax(nuw_shl(x, C), x + 1) -> x == 0 ? 1 : nuw_shl(x, C)
1965 // If C is not 0 or 1:
1966 // umax(nuw_mul(x, C), x + 1) -> x == 0 ? 1 : nuw_mul(x, C)
1967 auto foldMaxMulShift = [&](Value *A, Value *B) -> Instruction * {
1968 const APInt *C;
1969 Value *X;
1970 if (!match(A, m_NUWShl(m_Value(X), m_APInt(C))) &&
1971 !(match(A, m_NUWMul(m_Value(X), m_APInt(C))) && !C->isOne()))
1972 return nullptr;
1973 if (C->isZero())
1974 return nullptr;
1975 if (!match(B, m_OneUse(m_Add(m_Specific(X), m_One()))))
1976 return nullptr;
1977
1978 Value *Cmp = Builder.CreateICmpEQ(X, ConstantInt::get(X->getType(), 0));
1979 Value *NewSelect =
1980 Builder.CreateSelect(Cmp, ConstantInt::get(X->getType(), 1), A);
1981 return replaceInstUsesWith(*II, NewSelect);
1982 };
1983
1984 if (IID == Intrinsic::umax) {
1985 if (Instruction *I = foldMaxMulShift(I0, I1))
1986 return I;
1987 if (Instruction *I = foldMaxMulShift(I1, I0))
1988 return I;
1989 }
1990
1991 // If both operands of unsigned min/max are sign-extended, it is still ok
1992 // to narrow the operation.
1993 [[fallthrough]];
1994 }
1995 case Intrinsic::smax:
1996 case Intrinsic::smin: {
1997 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1998 Value *X, *Y;
1999 if (match(I0, m_SExt(m_Value(X))) && match(I1, m_SExt(m_Value(Y))) &&
2000 (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
2001 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
2002 return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());
2003 }
2004
2005 Constant *C;
2006 if (match(I0, m_SExt(m_Value(X))) && match(I1, m_Constant(C)) &&
2007 I0->hasOneUse()) {
2008 if (Constant *NarrowC = getLosslessSignedTrunc(C, X->getType(), DL)) {
2009 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);
2010 return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());
2011 }
2012 }
2013
2014 // smax(smin(X, MinC), MaxC) -> smin(smax(X, MaxC), MinC) if MinC s>= MaxC
2015 // umax(umin(X, MinC), MaxC) -> umin(umax(X, MaxC), MinC) if MinC u>= MaxC
2016 const APInt *MinC, *MaxC;
2017 auto CreateCanonicalClampForm = [&](bool IsSigned) {
2018 auto MaxIID = IsSigned ? Intrinsic::smax : Intrinsic::umax;
2019 auto MinIID = IsSigned ? Intrinsic::smin : Intrinsic::umin;
2020 Value *NewMax = Builder.CreateBinaryIntrinsic(
2021 MaxIID, X, ConstantInt::get(X->getType(), *MaxC));
2022 return replaceInstUsesWith(
2023 *II, Builder.CreateBinaryIntrinsic(
2024 MinIID, NewMax, ConstantInt::get(X->getType(), *MinC)));
2025 };
2026 if (IID == Intrinsic::smax &&
2028 m_APInt(MinC)))) &&
2029 match(I1, m_APInt(MaxC)) && MinC->sgt(*MaxC))
2030 return CreateCanonicalClampForm(true);
2031 if (IID == Intrinsic::umax &&
2033 m_APInt(MinC)))) &&
2034 match(I1, m_APInt(MaxC)) && MinC->ugt(*MaxC))
2035 return CreateCanonicalClampForm(false);
2036
2037 // umin(i1 X, i1 Y) -> and i1 X, Y
2038 // smax(i1 X, i1 Y) -> and i1 X, Y
2039 if ((IID == Intrinsic::umin || IID == Intrinsic::smax) &&
2040 II->getType()->isIntOrIntVectorTy(1)) {
2041 return BinaryOperator::CreateAnd(I0, I1);
2042 }
2043
2044 // umax(i1 X, i1 Y) -> or i1 X, Y
2045 // smin(i1 X, i1 Y) -> or i1 X, Y
2046 if ((IID == Intrinsic::umax || IID == Intrinsic::smin) &&
2047 II->getType()->isIntOrIntVectorTy(1)) {
2048 return BinaryOperator::CreateOr(I0, I1);
2049 }
2050
2051 // smin(smax(X, -1), 1) -> scmp(X, 0)
2052 // smax(smin(X, 1), -1) -> scmp(X, 0)
2053 // At this point, smax(smin(X, 1), -1) is changed to smin(smax(X, -1)
2054 // And i1's have been changed to and/ors
2055 // So we only need to check for smin
2056 if (IID == Intrinsic::smin) {
2057 if (match(I0, m_OneUse(m_SMax(m_Value(X), m_AllOnes()))) &&
2058 match(I1, m_One())) {
2059 Value *Zero = ConstantInt::get(X->getType(), 0);
2060 return replaceInstUsesWith(
2061 CI,
2062 Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {X, Zero}));
2063 }
2064 }
2065
2066 if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
2067 // smax (neg nsw X), (neg nsw Y) --> neg nsw (smin X, Y)
2068 // smin (neg nsw X), (neg nsw Y) --> neg nsw (smax X, Y)
2069 // TODO: Canonicalize neg after min/max if I1 is constant.
2070 if (match(I0, m_NSWNeg(m_Value(X))) && match(I1, m_NSWNeg(m_Value(Y))) &&
2071 (I0->hasOneUse() || I1->hasOneUse())) {
2073 Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, Y);
2074 return BinaryOperator::CreateNSWNeg(InvMaxMin);
2075 }
2076 }
2077
2078 // (umax X, (xor X, Pow2))
2079 // -> (or X, Pow2)
2080 // (umin X, (xor X, Pow2))
2081 // -> (and X, ~Pow2)
2082 // (smax X, (xor X, Pos_Pow2))
2083 // -> (or X, Pos_Pow2)
2084 // (smin X, (xor X, Pos_Pow2))
2085 // -> (and X, ~Pos_Pow2)
2086 // (smax X, (xor X, Neg_Pow2))
2087 // -> (and X, ~Neg_Pow2)
2088 // (smin X, (xor X, Neg_Pow2))
2089 // -> (or X, Neg_Pow2)
2090 if ((match(I0, m_c_Xor(m_Specific(I1), m_Value(X))) ||
2091 match(I1, m_c_Xor(m_Specific(I0), m_Value(X)))) &&
2092 isKnownToBeAPowerOfTwo(X, /* OrZero */ true)) {
2093 bool UseOr = IID == Intrinsic::smax || IID == Intrinsic::umax;
2094 bool UseAndN = IID == Intrinsic::smin || IID == Intrinsic::umin;
2095
2096 if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
2097 auto KnownSign = getKnownSign(X, SQ.getWithInstruction(II));
2098 if (KnownSign == std::nullopt) {
2099 UseOr = false;
2100 UseAndN = false;
2101 } else if (*KnownSign /* true is Signed. */) {
2102 UseOr ^= true;
2103 UseAndN ^= true;
2104 Type *Ty = I0->getType();
2105 // Negative power of 2 must be IntMin. It's possible to be able to
2106 // prove negative / power of 2 without actually having known bits, so
2107 // just get the value by hand.
2109 Ty, APInt::getSignedMinValue(Ty->getScalarSizeInBits()));
2110 }
2111 }
2112 if (UseOr)
2113 return BinaryOperator::CreateOr(I0, X);
2114 else if (UseAndN)
2115 return BinaryOperator::CreateAnd(I0, Builder.CreateNot(X));
2116 }
2117
2118 // If we can eliminate ~A and Y is free to invert:
2119 // max ~A, Y --> ~(min A, ~Y)
2120 //
2121 // Examples:
2122 // max ~A, ~Y --> ~(min A, Y)
2123 // max ~A, C --> ~(min A, ~C)
2124 // max ~A, (max ~Y, ~Z) --> ~min( A, (min Y, Z))
2125 auto moveNotAfterMinMax = [&](Value *X, Value *Y) -> Instruction * {
2126 Value *A;
2127 if (match(X, m_OneUse(m_Not(m_Value(A)))) &&
2128 !isFreeToInvert(A, A->hasOneUse())) {
2129 if (Value *NotY = getFreelyInverted(Y, Y->hasOneUse(), &Builder)) {
2131 Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, A, NotY);
2132 return BinaryOperator::CreateNot(InvMaxMin);
2133 }
2134 }
2135 return nullptr;
2136 };
2137
2138 if (Instruction *I = moveNotAfterMinMax(I0, I1))
2139 return I;
2140 if (Instruction *I = moveNotAfterMinMax(I1, I0))
2141 return I;
2142
2144 return I;
2145
2146 // minmax (X & NegPow2C, Y & NegPow2C) --> minmax(X, Y) & NegPow2C
2147 const APInt *RHSC;
2148 if (match(I0, m_OneUse(m_And(m_Value(X), m_NegatedPower2(RHSC)))) &&
2149 match(I1, m_OneUse(m_And(m_Value(Y), m_SpecificInt(*RHSC)))))
2150 return BinaryOperator::CreateAnd(Builder.CreateBinaryIntrinsic(IID, X, Y),
2151 ConstantInt::get(II->getType(), *RHSC));
2152
2153 // smax(X, -X) --> abs(X)
2154 // smin(X, -X) --> -abs(X)
2155 // umax(X, -X) --> -abs(X)
2156 // umin(X, -X) --> abs(X)
2157 if (isKnownNegation(I0, I1)) {
2158 // We can choose either operand as the input to abs(), but if we can
2159 // eliminate the only use of a value, that's better for subsequent
2160 // transforms/analysis.
2161 if (I0->hasOneUse() && !I1->hasOneUse())
2162 std::swap(I0, I1);
2163
2164 // This is some variant of abs(). See if we can propagate 'nsw' to the abs
2165 // operation and potentially its negation.
2166 bool IntMinIsPoison = isKnownNegation(I0, I1, /* NeedNSW */ true);
2167 Value *Abs = Builder.CreateBinaryIntrinsic(
2168 Intrinsic::abs, I0,
2169 ConstantInt::getBool(II->getContext(), IntMinIsPoison));
2170
2171 // We don't have a "nabs" intrinsic, so negate if needed based on the
2172 // max/min operation.
2173 if (IID == Intrinsic::smin || IID == Intrinsic::umax)
2174 Abs = Builder.CreateNeg(Abs, "nabs", IntMinIsPoison);
2175 return replaceInstUsesWith(CI, Abs);
2176 }
2177
2179 return Sel;
2180
2181 if (Instruction *SAdd = matchSAddSubSat(*II))
2182 return SAdd;
2183
2184 if (Value *NewMinMax = reassociateMinMaxWithConstants(II, Builder, SQ))
2185 return replaceInstUsesWith(*II, NewMinMax);
2186
2188 return R;
2189
2190 if (Instruction *NewMinMax = factorizeMinMaxTree(II))
2191 return NewMinMax;
2192
2193 // Try to fold minmax with constant RHS based on range information
2194 if (match(I1, m_APIntAllowPoison(RHSC))) {
2195 ICmpInst::Predicate Pred =
2197 bool IsSigned = MinMaxIntrinsic::isSigned(IID);
2199 I0, IsSigned, SQ.getWithInstruction(II));
2200 if (!LHS_CR.isFullSet()) {
2201 if (LHS_CR.icmp(Pred, *RHSC))
2202 return replaceInstUsesWith(*II, I0);
2203 if (LHS_CR.icmp(ICmpInst::getSwappedPredicate(Pred), *RHSC))
2204 return replaceInstUsesWith(*II,
2205 ConstantInt::get(II->getType(), *RHSC));
2206 }
2207 }
2208
2210 return replaceInstUsesWith(*II, V);
2211
2212 break;
2213 }
2214 case Intrinsic::scmp: {
2215 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
2216 Value *LHS, *RHS;
2217 if (match(I0, m_NSWSub(m_Value(LHS), m_Value(RHS))) && match(I1, m_Zero()))
2218 return replaceInstUsesWith(
2219 CI,
2220 Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {LHS, RHS}));
2221 break;
2222 }
2223 case Intrinsic::bitreverse: {
2224 Value *IIOperand = II->getArgOperand(0);
2225 // bitrev (zext i1 X to ?) --> X ? SignBitC : 0
2226 Value *X;
2227 if (match(IIOperand, m_ZExt(m_Value(X))) &&
2228 X->getType()->isIntOrIntVectorTy(1)) {
2229 Type *Ty = II->getType();
2230 APInt SignBit = APInt::getSignMask(Ty->getScalarSizeInBits());
2231 return SelectInst::Create(X, ConstantInt::get(Ty, SignBit),
2233 }
2234
2235 if (Instruction *crossLogicOpFold =
2237 return crossLogicOpFold;
2238
2239 break;
2240 }
2241 case Intrinsic::bswap: {
2242 Value *IIOperand = II->getArgOperand(0);
2243
2244 // Try to canonicalize bswap-of-logical-shift-by-8-bit-multiple as
2245 // inverse-shift-of-bswap:
2246 // bswap (shl X, Y) --> lshr (bswap X), Y
2247 // bswap (lshr X, Y) --> shl (bswap X), Y
2248 Value *X, *Y;
2249 if (match(IIOperand, m_OneUse(m_LogicalShift(m_Value(X), m_Value(Y))))) {
2250 unsigned BitWidth = IIOperand->getType()->getScalarSizeInBits();
2252 Value *NewSwap = Builder.CreateUnaryIntrinsic(Intrinsic::bswap, X);
2253 BinaryOperator::BinaryOps InverseShift =
2254 cast<BinaryOperator>(IIOperand)->getOpcode() == Instruction::Shl
2255 ? Instruction::LShr
2256 : Instruction::Shl;
2257 return BinaryOperator::Create(InverseShift, NewSwap, Y);
2258 }
2259 }
2260
2261 KnownBits Known = computeKnownBits(IIOperand, II);
2262 uint64_t LZ = alignDown(Known.countMinLeadingZeros(), 8);
2263 uint64_t TZ = alignDown(Known.countMinTrailingZeros(), 8);
2264 unsigned BW = Known.getBitWidth();
2265
2266 // bswap(x) -> shift(x) if x has exactly one "active byte"
2267 if (BW - LZ - TZ == 8) {
2268 assert(LZ != TZ && "active byte cannot be in the middle");
2269 if (LZ > TZ) // -> shl(x) if the "active byte" is in the low part of x
2270 return BinaryOperator::CreateNUWShl(
2271 IIOperand, ConstantInt::get(IIOperand->getType(), LZ - TZ));
2272 // -> lshr(x) if the "active byte" is in the high part of x
2273 return BinaryOperator::CreateExactLShr(
2274 IIOperand, ConstantInt::get(IIOperand->getType(), TZ - LZ));
2275 }
2276
2277 // bswap(trunc(bswap(x))) -> trunc(lshr(x, c))
2278 if (match(IIOperand, m_Trunc(m_BSwap(m_Value(X))))) {
2279 unsigned C = X->getType()->getScalarSizeInBits() - BW;
2280 Value *CV = ConstantInt::get(X->getType(), C);
2281 Value *V = Builder.CreateLShr(X, CV);
2282 return new TruncInst(V, IIOperand->getType());
2283 }
2284
2285 if (Instruction *crossLogicOpFold =
2287 return crossLogicOpFold;
2288 }
2289
2290 // Try to fold into bitreverse if bswap is the root of the expression tree.
2291 if (Instruction *BitOp = matchBSwapOrBitReverse(*II, /*MatchBSwaps*/ false,
2292 /*MatchBitReversals*/ true))
2293 return BitOp;
2294 break;
2295 }
2296 case Intrinsic::masked_load:
2297 if (Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II))
2298 return replaceInstUsesWith(CI, SimplifiedMaskedOp);
2299 break;
2300 case Intrinsic::masked_store:
2301 return simplifyMaskedStore(*II);
2302 case Intrinsic::masked_gather:
2303 return simplifyMaskedGather(*II);
2304 case Intrinsic::masked_scatter:
2305 return simplifyMaskedScatter(*II);
2306 case Intrinsic::launder_invariant_group:
2307 case Intrinsic::strip_invariant_group:
2308 if (auto *SkippedBarrier = simplifyInvariantGroupIntrinsic(*II, *this))
2309 return replaceInstUsesWith(*II, SkippedBarrier);
2310 break;
2311 case Intrinsic::powi:
2312 if (ConstantInt *Power = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
2313 // 0 and 1 are handled in instsimplify
2314 // powi(x, -1) -> 1/x
2315 if (Power->isMinusOne())
2316 return BinaryOperator::CreateFDivFMF(ConstantFP::get(CI.getType(), 1.0),
2317 II->getArgOperand(0), II);
2318 // powi(x, 2) -> x*x
2319 if (Power->equalsInt(2))
2320 return BinaryOperator::CreateFMulFMF(II->getArgOperand(0),
2321 II->getArgOperand(0), II);
2322
2323 if (!Power->getValue()[0]) {
2324 Value *X;
2325 // If power is even:
2326 // powi(-x, p) -> powi(x, p)
2327 // powi(fabs(x), p) -> powi(x, p)
2328 // powi(copysign(x, y), p) -> powi(x, p)
2329 if (match(II->getArgOperand(0), m_FNeg(m_Value(X))) ||
2330 match(II->getArgOperand(0), m_FAbs(m_Value(X))) ||
2331 match(II->getArgOperand(0),
2333 return replaceOperand(*II, 0, X);
2334 }
2335 }
2336 break;
2337
2338 case Intrinsic::cttz:
2339 case Intrinsic::ctlz:
2340 if (auto *I = foldCttzCtlz(*II, *this))
2341 return I;
2342 break;
2343
2344 case Intrinsic::ctpop:
2345 if (auto *I = foldCtpop(*II, *this))
2346 return I;
2347 break;
2348
2349 case Intrinsic::fshl:
2350 case Intrinsic::fshr: {
2351 Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);
2352 Type *Ty = II->getType();
2353 unsigned BitWidth = Ty->getScalarSizeInBits();
2354 Constant *ShAmtC;
2355 if (match(II->getArgOperand(2), m_ImmConstant(ShAmtC))) {
2356 // Canonicalize a shift amount constant operand to modulo the bit-width.
2357 Constant *WidthC = ConstantInt::get(Ty, BitWidth);
2358 Constant *ModuloC =
2359 ConstantFoldBinaryOpOperands(Instruction::URem, ShAmtC, WidthC, DL);
2360 if (!ModuloC)
2361 return nullptr;
2362 if (ModuloC != ShAmtC)
2363 return replaceOperand(*II, 2, ModuloC);
2364
2366 ShAmtC, DL),
2367 m_One()) &&
2368 "Shift amount expected to be modulo bitwidth");
2369
2370 // Canonicalize funnel shift right by constant to funnel shift left. This
2371 // is not entirely arbitrary. For historical reasons, the backend may
2372 // recognize rotate left patterns but miss rotate right patterns.
2373 if (IID == Intrinsic::fshr) {
2374 // fshr X, Y, C --> fshl X, Y, (BitWidth - C) if C is not zero.
2375 if (!isKnownNonZero(ShAmtC, SQ.getWithInstruction(II)))
2376 return nullptr;
2377
2378 Constant *LeftShiftC = ConstantExpr::getSub(WidthC, ShAmtC);
2379 Module *Mod = II->getModule();
2380 Function *Fshl =
2381 Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::fshl, Ty);
2382 return CallInst::Create(Fshl, { Op0, Op1, LeftShiftC });
2383 }
2384 assert(IID == Intrinsic::fshl &&
2385 "All funnel shifts by simple constants should go left");
2386
2387 // fshl(X, 0, C) --> shl X, C
2388 // fshl(X, undef, C) --> shl X, C
2389 if (match(Op1, m_ZeroInt()) || match(Op1, m_Undef()))
2390 return BinaryOperator::CreateShl(Op0, ShAmtC);
2391
2392 // fshl(0, X, C) --> lshr X, (BW-C)
2393 // fshl(undef, X, C) --> lshr X, (BW-C)
2394 if (match(Op0, m_ZeroInt()) || match(Op0, m_Undef()))
2395 return BinaryOperator::CreateLShr(Op1,
2396 ConstantExpr::getSub(WidthC, ShAmtC));
2397
2398 // fshl i16 X, X, 8 --> bswap i16 X (reduce to more-specific form)
2399 if (Op0 == Op1 && BitWidth == 16 && match(ShAmtC, m_SpecificInt(8))) {
2400 Module *Mod = II->getModule();
2401 Function *Bswap =
2402 Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::bswap, Ty);
2403 return CallInst::Create(Bswap, { Op0 });
2404 }
2405 if (Instruction *BitOp =
2406 matchBSwapOrBitReverse(*II, /*MatchBSwaps*/ true,
2407 /*MatchBitReversals*/ true))
2408 return BitOp;
2409
2410 // R = fshl(X, X, C2)
2411 // fshl(R, R, C1) --> fshl(X, X, (C1 + C2) % bitsize)
2412 Value *InnerOp;
2413 const APInt *ShAmtInnerC, *ShAmtOuterC;
2414 if (match(Op0, m_FShl(m_Value(InnerOp), m_Deferred(InnerOp),
2415 m_APInt(ShAmtInnerC))) &&
2416 match(ShAmtC, m_APInt(ShAmtOuterC)) && Op0 == Op1) {
2417 APInt Sum = *ShAmtOuterC + *ShAmtInnerC;
2418 APInt Modulo = Sum.urem(APInt(Sum.getBitWidth(), BitWidth));
2419 if (Modulo.isZero())
2420 return replaceInstUsesWith(*II, InnerOp);
2421 Constant *ModuloC = ConstantInt::get(Ty, Modulo);
2423 {InnerOp, InnerOp, ModuloC});
2424 }
2425 }
2426
2427 // fshl(X, X, Neg(Y)) --> fshr(X, X, Y)
2428 // fshr(X, X, Neg(Y)) --> fshl(X, X, Y)
2429 // if BitWidth is a power-of-2
2430 Value *Y;
2431 if (Op0 == Op1 && isPowerOf2_32(BitWidth) &&
2432 match(II->getArgOperand(2), m_Neg(m_Value(Y)))) {
2433 Module *Mod = II->getModule();
2435 Mod, IID == Intrinsic::fshl ? Intrinsic::fshr : Intrinsic::fshl, Ty);
2436 return CallInst::Create(OppositeShift, {Op0, Op1, Y});
2437 }
2438
2439 // fshl(X, 0, Y) --> shl(X, and(Y, BitWidth - 1)) if bitwidth is a
2440 // power-of-2
2441 if (IID == Intrinsic::fshl && isPowerOf2_32(BitWidth) &&
2442 match(Op1, m_ZeroInt())) {
2443 Value *Op2 = II->getArgOperand(2);
2444 Value *And = Builder.CreateAnd(Op2, ConstantInt::get(Ty, BitWidth - 1));
2445 return BinaryOperator::CreateShl(Op0, And);
2446 }
2447
2448 // Left or right might be masked.
2450 return &CI;
2451
2452 // The shift amount (operand 2) of a funnel shift is modulo the bitwidth,
2453 // so only the low bits of the shift amount are demanded if the bitwidth is
2454 // a power-of-2.
2455 if (!isPowerOf2_32(BitWidth))
2456 break;
2458 KnownBits Op2Known(BitWidth);
2459 if (SimplifyDemandedBits(II, 2, Op2Demanded, Op2Known))
2460 return &CI;
2461 break;
2462 }
2463 case Intrinsic::ptrmask: {
2464 unsigned BitWidth = DL.getPointerTypeSizeInBits(II->getType());
2465 KnownBits Known(BitWidth);
2467 return II;
2468
2469 Value *InnerPtr, *InnerMask;
2470 bool Changed = false;
2471 // Combine:
2472 // (ptrmask (ptrmask p, A), B)
2473 // -> (ptrmask p, (and A, B))
2474 if (match(II->getArgOperand(0),
2476 m_Value(InnerMask))))) {
2477 assert(II->getArgOperand(1)->getType() == InnerMask->getType() &&
2478 "Mask types must match");
2479 // TODO: If InnerMask == Op1, we could copy attributes from inner
2480 // callsite -> outer callsite.
2481 Value *NewMask = Builder.CreateAnd(II->getArgOperand(1), InnerMask);
2482 replaceOperand(CI, 0, InnerPtr);
2483 replaceOperand(CI, 1, NewMask);
2484 Changed = true;
2485 }
2486
2487 // See if we can deduce non-null.
2488 if (!CI.hasRetAttr(Attribute::NonNull) &&
2489 (Known.isNonZero() ||
2490 isKnownNonZero(II, getSimplifyQuery().getWithInstruction(II)))) {
2491 CI.addRetAttr(Attribute::NonNull);
2492 Changed = true;
2493 }
2494
2495 unsigned NewAlignmentLog =
2497 std::min(BitWidth - 1, Known.countMinTrailingZeros()));
2498 // Known bits will capture if we had alignment information associated with
2499 // the pointer argument.
2500 if (NewAlignmentLog > Log2(CI.getRetAlign().valueOrOne())) {
2502 CI.getContext(), Align(uint64_t(1) << NewAlignmentLog)));
2503 Changed = true;
2504 }
2505 if (Changed)
2506 return &CI;
2507 break;
2508 }
2509 case Intrinsic::uadd_with_overflow:
2510 case Intrinsic::sadd_with_overflow: {
2511 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2512 return I;
2513
2514 // Given 2 constant operands whose sum does not overflow:
2515 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
2516 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
2517 Value *X;
2518 const APInt *C0, *C1;
2519 Value *Arg0 = II->getArgOperand(0);
2520 Value *Arg1 = II->getArgOperand(1);
2521 bool IsSigned = IID == Intrinsic::sadd_with_overflow;
2522 bool HasNWAdd = IsSigned
2523 ? match(Arg0, m_NSWAddLike(m_Value(X), m_APInt(C0)))
2524 : match(Arg0, m_NUWAddLike(m_Value(X), m_APInt(C0)));
2525 if (HasNWAdd && match(Arg1, m_APInt(C1))) {
2526 bool Overflow;
2527 APInt NewC =
2528 IsSigned ? C1->sadd_ov(*C0, Overflow) : C1->uadd_ov(*C0, Overflow);
2529 if (!Overflow)
2530 return replaceInstUsesWith(
2531 *II, Builder.CreateBinaryIntrinsic(
2532 IID, X, ConstantInt::get(Arg1->getType(), NewC)));
2533 }
2534 break;
2535 }
2536
2537 case Intrinsic::umul_with_overflow:
2538 case Intrinsic::smul_with_overflow:
2539 case Intrinsic::usub_with_overflow:
2540 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2541 return I;
2542 break;
2543
2544 case Intrinsic::ssub_with_overflow: {
2545 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2546 return I;
2547
2548 Constant *C;
2549 Value *Arg0 = II->getArgOperand(0);
2550 Value *Arg1 = II->getArgOperand(1);
2551 // Given a constant C that is not the minimum signed value
2552 // for an integer of a given bit width:
2553 //
2554 // ssubo X, C -> saddo X, -C
2555 if (match(Arg1, m_Constant(C)) && C->isNotMinSignedValue()) {
2556 Value *NegVal = ConstantExpr::getNeg(C);
2557 // Build a saddo call that is equivalent to the discovered
2558 // ssubo call.
2559 return replaceInstUsesWith(
2560 *II, Builder.CreateBinaryIntrinsic(Intrinsic::sadd_with_overflow,
2561 Arg0, NegVal));
2562 }
2563
2564 break;
2565 }
2566
2567 case Intrinsic::uadd_sat:
2568 case Intrinsic::sadd_sat:
2569 case Intrinsic::usub_sat:
2570 case Intrinsic::ssub_sat: {
2572 Type *Ty = SI->getType();
2573 Value *Arg0 = SI->getLHS();
2574 Value *Arg1 = SI->getRHS();
2575
2576 // Make use of known overflow information.
2577 OverflowResult OR = computeOverflow(SI->getBinaryOp(), SI->isSigned(),
2578 Arg0, Arg1, SI);
2579 switch (OR) {
2581 break;
2583 if (SI->isSigned())
2584 return BinaryOperator::CreateNSW(SI->getBinaryOp(), Arg0, Arg1);
2585 else
2586 return BinaryOperator::CreateNUW(SI->getBinaryOp(), Arg0, Arg1);
2588 unsigned BitWidth = Ty->getScalarSizeInBits();
2589 APInt Min = APSInt::getMinValue(BitWidth, !SI->isSigned());
2590 return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Min));
2591 }
2593 unsigned BitWidth = Ty->getScalarSizeInBits();
2594 APInt Max = APSInt::getMaxValue(BitWidth, !SI->isSigned());
2595 return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Max));
2596 }
2597 }
2598
2599 // usub_sat((sub nuw C, A), C1) -> usub_sat(usub_sat(C, C1), A)
2600 // which after that:
2601 // usub_sat((sub nuw C, A), C1) -> usub_sat(C - C1, A) if C1 u< C
2602 // usub_sat((sub nuw C, A), C1) -> 0 otherwise
2603 Constant *C, *C1;
2604 Value *A;
2605 if (IID == Intrinsic::usub_sat &&
2606 match(Arg0, m_NUWSub(m_ImmConstant(C), m_Value(A))) &&
2607 match(Arg1, m_ImmConstant(C1))) {
2608 auto *NewC = Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, C, C1);
2609 auto *NewSub =
2610 Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, NewC, A);
2611 return replaceInstUsesWith(*SI, NewSub);
2612 }
2613
2614 // ssub.sat(X, C) -> sadd.sat(X, -C) if C != MIN
2615 if (IID == Intrinsic::ssub_sat && match(Arg1, m_Constant(C)) &&
2616 C->isNotMinSignedValue()) {
2617 Value *NegVal = ConstantExpr::getNeg(C);
2618 return replaceInstUsesWith(
2619 *II, Builder.CreateBinaryIntrinsic(
2620 Intrinsic::sadd_sat, Arg0, NegVal));
2621 }
2622
2623 // sat(sat(X + Val2) + Val) -> sat(X + (Val+Val2))
2624 // sat(sat(X - Val2) - Val) -> sat(X - (Val+Val2))
2625 // if Val and Val2 have the same sign
2626 if (auto *Other = dyn_cast<IntrinsicInst>(Arg0)) {
2627 Value *X;
2628 const APInt *Val, *Val2;
2629 APInt NewVal;
2630 bool IsUnsigned =
2631 IID == Intrinsic::uadd_sat || IID == Intrinsic::usub_sat;
2632 if (Other->getIntrinsicID() == IID &&
2633 match(Arg1, m_APInt(Val)) &&
2634 match(Other->getArgOperand(0), m_Value(X)) &&
2635 match(Other->getArgOperand(1), m_APInt(Val2))) {
2636 if (IsUnsigned)
2637 NewVal = Val->uadd_sat(*Val2);
2638 else if (Val->isNonNegative() == Val2->isNonNegative()) {
2639 bool Overflow;
2640 NewVal = Val->sadd_ov(*Val2, Overflow);
2641 if (Overflow) {
2642 // Both adds together may add more than SignedMaxValue
2643 // without saturating the final result.
2644 break;
2645 }
2646 } else {
2647 // Cannot fold saturated addition with different signs.
2648 break;
2649 }
2650
2651 return replaceInstUsesWith(
2652 *II, Builder.CreateBinaryIntrinsic(
2653 IID, X, ConstantInt::get(II->getType(), NewVal)));
2654 }
2655 }
2656 break;
2657 }
2658
2659 case Intrinsic::minnum:
2660 case Intrinsic::maxnum:
2661 case Intrinsic::minimum:
2662 case Intrinsic::maximum: {
2663 Value *Arg0 = II->getArgOperand(0);
2664 Value *Arg1 = II->getArgOperand(1);
2665 Value *X, *Y;
2666 if (match(Arg0, m_FNeg(m_Value(X))) && match(Arg1, m_FNeg(m_Value(Y))) &&
2667 (Arg0->hasOneUse() || Arg1->hasOneUse())) {
2668 // If both operands are negated, invert the call and negate the result:
2669 // min(-X, -Y) --> -(max(X, Y))
2670 // max(-X, -Y) --> -(min(X, Y))
2671 Intrinsic::ID NewIID;
2672 switch (IID) {
2673 case Intrinsic::maxnum:
2674 NewIID = Intrinsic::minnum;
2675 break;
2676 case Intrinsic::minnum:
2677 NewIID = Intrinsic::maxnum;
2678 break;
2679 case Intrinsic::maximum:
2680 NewIID = Intrinsic::minimum;
2681 break;
2682 case Intrinsic::minimum:
2683 NewIID = Intrinsic::maximum;
2684 break;
2685 default:
2686 llvm_unreachable("unexpected intrinsic ID");
2687 }
2688 Value *NewCall = Builder.CreateBinaryIntrinsic(NewIID, X, Y, II);
2689 Instruction *FNeg = UnaryOperator::CreateFNeg(NewCall);
2690 FNeg->copyIRFlags(II);
2691 return FNeg;
2692 }
2693
2694 // m(m(X, C2), C1) -> m(X, C)
2695 const APFloat *C1, *C2;
2696 if (auto *M = dyn_cast<IntrinsicInst>(Arg0)) {
2697 if (M->getIntrinsicID() == IID && match(Arg1, m_APFloat(C1)) &&
2698 ((match(M->getArgOperand(0), m_Value(X)) &&
2699 match(M->getArgOperand(1), m_APFloat(C2))) ||
2700 (match(M->getArgOperand(1), m_Value(X)) &&
2701 match(M->getArgOperand(0), m_APFloat(C2))))) {
2702 APFloat Res(0.0);
2703 switch (IID) {
2704 case Intrinsic::maxnum:
2705 Res = maxnum(*C1, *C2);
2706 break;
2707 case Intrinsic::minnum:
2708 Res = minnum(*C1, *C2);
2709 break;
2710 case Intrinsic::maximum:
2711 Res = maximum(*C1, *C2);
2712 break;
2713 case Intrinsic::minimum:
2714 Res = minimum(*C1, *C2);
2715 break;
2716 default:
2717 llvm_unreachable("unexpected intrinsic ID");
2718 }
2719 // TODO: Conservatively intersecting FMF. If Res == C2, the transform
2720 // was a simplification (so Arg0 and its original flags could
2721 // propagate?)
2722 Value *V = Builder.CreateBinaryIntrinsic(
2723 IID, X, ConstantFP::get(Arg0->getType(), Res),
2725 return replaceInstUsesWith(*II, V);
2726 }
2727 }
2728
2729 // m((fpext X), (fpext Y)) -> fpext (m(X, Y))
2730 if (match(Arg0, m_OneUse(m_FPExt(m_Value(X)))) &&
2731 match(Arg1, m_OneUse(m_FPExt(m_Value(Y)))) &&
2732 X->getType() == Y->getType()) {
2733 Value *NewCall =
2734 Builder.CreateBinaryIntrinsic(IID, X, Y, II, II->getName());
2735 return new FPExtInst(NewCall, II->getType());
2736 }
2737
2738 // max X, -X --> fabs X
2739 // min X, -X --> -(fabs X)
2740 // TODO: Remove one-use limitation? That is obviously better for max,
2741 // hence why we don't check for one-use for that. However,
2742 // it would be an extra instruction for min (fnabs), but
2743 // that is still likely better for analysis and codegen.
2744 auto IsMinMaxOrXNegX = [IID, &X](Value *Op0, Value *Op1) {
2745 if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_Specific(X)))
2746 return Op0->hasOneUse() ||
2747 (IID != Intrinsic::minimum && IID != Intrinsic::minnum);
2748 return false;
2749 };
2750
2751 if (IsMinMaxOrXNegX(Arg0, Arg1) || IsMinMaxOrXNegX(Arg1, Arg0)) {
2752 Value *R = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II);
2753 if (IID == Intrinsic::minimum || IID == Intrinsic::minnum)
2754 R = Builder.CreateFNegFMF(R, II);
2755 return replaceInstUsesWith(*II, R);
2756 }
2757
2758 break;
2759 }
2760 case Intrinsic::matrix_multiply: {
2761 // Optimize negation in matrix multiplication.
2762
2763 // -A * -B -> A * B
2764 Value *A, *B;
2765 if (match(II->getArgOperand(0), m_FNeg(m_Value(A))) &&
2766 match(II->getArgOperand(1), m_FNeg(m_Value(B)))) {
2767 replaceOperand(*II, 0, A);
2768 replaceOperand(*II, 1, B);
2769 return II;
2770 }
2771
2772 Value *Op0 = II->getOperand(0);
2773 Value *Op1 = II->getOperand(1);
2774 Value *OpNotNeg, *NegatedOp;
2775 unsigned NegatedOpArg, OtherOpArg;
2776 if (match(Op0, m_FNeg(m_Value(OpNotNeg)))) {
2777 NegatedOp = Op0;
2778 NegatedOpArg = 0;
2779 OtherOpArg = 1;
2780 } else if (match(Op1, m_FNeg(m_Value(OpNotNeg)))) {
2781 NegatedOp = Op1;
2782 NegatedOpArg = 1;
2783 OtherOpArg = 0;
2784 } else
2785 // Multiplication doesn't have a negated operand.
2786 break;
2787
2788 // Only optimize if the negated operand has only one use.
2789 if (!NegatedOp->hasOneUse())
2790 break;
2791
2792 Value *OtherOp = II->getOperand(OtherOpArg);
2793 VectorType *RetTy = cast<VectorType>(II->getType());
2794 VectorType *NegatedOpTy = cast<VectorType>(NegatedOp->getType());
2795 VectorType *OtherOpTy = cast<VectorType>(OtherOp->getType());
2796 ElementCount NegatedCount = NegatedOpTy->getElementCount();
2797 ElementCount OtherCount = OtherOpTy->getElementCount();
2798 ElementCount RetCount = RetTy->getElementCount();
2799 // (-A) * B -> A * (-B), if it is cheaper to negate B and vice versa.
2800 if (ElementCount::isKnownGT(NegatedCount, OtherCount) &&
2801 ElementCount::isKnownLT(OtherCount, RetCount)) {
2802 Value *InverseOtherOp = Builder.CreateFNeg(OtherOp);
2803 replaceOperand(*II, NegatedOpArg, OpNotNeg);
2804 replaceOperand(*II, OtherOpArg, InverseOtherOp);
2805 return II;
2806 }
2807 // (-A) * B -> -(A * B), if it is cheaper to negate the result
2808 if (ElementCount::isKnownGT(NegatedCount, RetCount)) {
2809 SmallVector<Value *, 5> NewArgs(II->args());
2810 NewArgs[NegatedOpArg] = OpNotNeg;
2811 Instruction *NewMul =
2812 Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);
2813 return replaceInstUsesWith(*II, Builder.CreateFNegFMF(NewMul, II));
2814 }
2815 break;
2816 }
2817 case Intrinsic::fmuladd: {
2818 // Try to simplify the underlying FMul.
2819 if (Value *V =
2820 simplifyFMulInst(II->getArgOperand(0), II->getArgOperand(1),
2821 II->getFastMathFlags(), SQ.getWithInstruction(II)))
2822 return BinaryOperator::CreateFAddFMF(V, II->getArgOperand(2),
2823 II->getFastMathFlags());
2824
2825 [[fallthrough]];
2826 }
2827 case Intrinsic::fma: {
2828 // fma fneg(x), fneg(y), z -> fma x, y, z
2829 Value *Src0 = II->getArgOperand(0);
2830 Value *Src1 = II->getArgOperand(1);
2831 Value *Src2 = II->getArgOperand(2);
2832 Value *X, *Y;
2833 if (match(Src0, m_FNeg(m_Value(X))) && match(Src1, m_FNeg(m_Value(Y)))) {
2834 replaceOperand(*II, 0, X);
2835 replaceOperand(*II, 1, Y);
2836 return II;
2837 }
2838
2839 // fma fabs(x), fabs(x), z -> fma x, x, z
2840 if (match(Src0, m_FAbs(m_Value(X))) &&
2841 match(Src1, m_FAbs(m_Specific(X)))) {
2842 replaceOperand(*II, 0, X);
2843 replaceOperand(*II, 1, X);
2844 return II;
2845 }
2846
2847 // Try to simplify the underlying FMul. We can only apply simplifications
2848 // that do not require rounding.
2849 if (Value *V = simplifyFMAFMul(Src0, Src1, II->getFastMathFlags(),
2850 SQ.getWithInstruction(II)))
2851 return BinaryOperator::CreateFAddFMF(V, Src2, II->getFastMathFlags());
2852
2853 // fma x, y, 0 -> fmul x, y
2854 // This is always valid for -0.0, but requires nsz for +0.0 as
2855 // -0.0 + 0.0 = 0.0, which would not be the same as the fmul on its own.
2856 if (match(Src2, m_NegZeroFP()) ||
2857 (match(Src2, m_PosZeroFP()) && II->getFastMathFlags().noSignedZeros()))
2858 return BinaryOperator::CreateFMulFMF(Src0, Src1, II);
2859
2860 // fma x, -1.0, y -> fsub y, x
2861 if (match(Src1, m_SpecificFP(-1.0)))
2862 return BinaryOperator::CreateFSubFMF(Src2, Src0, II);
2863
2864 break;
2865 }
2866 case Intrinsic::copysign: {
2867 Value *Mag = II->getArgOperand(0), *Sign = II->getArgOperand(1);
2868 if (std::optional<bool> KnownSignBit = computeKnownFPSignBit(
2869 Sign, getSimplifyQuery().getWithInstruction(II))) {
2870 if (*KnownSignBit) {
2871 // If we know that the sign argument is negative, reduce to FNABS:
2872 // copysign Mag, -Sign --> fneg (fabs Mag)
2873 Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);
2874 return replaceInstUsesWith(*II, Builder.CreateFNegFMF(Fabs, II));
2875 }
2876
2877 // If we know that the sign argument is positive, reduce to FABS:
2878 // copysign Mag, +Sign --> fabs Mag
2879 Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);
2880 return replaceInstUsesWith(*II, Fabs);
2881 }
2882
2883 // Propagate sign argument through nested calls:
2884 // copysign Mag, (copysign ?, X) --> copysign Mag, X
2885 Value *X;
2887 Value *CopySign =
2888 Builder.CreateCopySign(Mag, X, FMFSource::intersect(II, Sign));
2889 return replaceInstUsesWith(*II, CopySign);
2890 }
2891
2892 // Clear sign-bit of constant magnitude:
2893 // copysign -MagC, X --> copysign MagC, X
2894 // TODO: Support constant folding for fabs
2895 const APFloat *MagC;
2896 if (match(Mag, m_APFloat(MagC)) && MagC->isNegative()) {
2897 APFloat PosMagC = *MagC;
2898 PosMagC.clearSign();
2899 return replaceOperand(*II, 0, ConstantFP::get(Mag->getType(), PosMagC));
2900 }
2901
2902 // Peek through changes of magnitude's sign-bit. This call rewrites those:
2903 // copysign (fabs X), Sign --> copysign X, Sign
2904 // copysign (fneg X), Sign --> copysign X, Sign
2905 if (match(Mag, m_FAbs(m_Value(X))) || match(Mag, m_FNeg(m_Value(X))))
2906 return replaceOperand(*II, 0, X);
2907
2908 break;
2909 }
2910 case Intrinsic::fabs: {
2911 Value *Cond, *TVal, *FVal;
2912 Value *Arg = II->getArgOperand(0);
2913 Value *X;
2914 // fabs (-X) --> fabs (X)
2915 if (match(Arg, m_FNeg(m_Value(X)))) {
2916 CallInst *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II);
2917 return replaceInstUsesWith(CI, Fabs);
2918 }
2919
2920 if (match(Arg, m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))) {
2921 // fabs (select Cond, TrueC, FalseC) --> select Cond, AbsT, AbsF
2922 if (Arg->hasOneUse() ? (isa<Constant>(TVal) || isa<Constant>(FVal))
2923 : (isa<Constant>(TVal) && isa<Constant>(FVal))) {
2924 CallInst *AbsT = Builder.CreateCall(II->getCalledFunction(), {TVal});
2925 CallInst *AbsF = Builder.CreateCall(II->getCalledFunction(), {FVal});
2926 SelectInst *SI = SelectInst::Create(Cond, AbsT, AbsF);
2927 FastMathFlags FMF1 = II->getFastMathFlags();
2928 FastMathFlags FMF2 = cast<SelectInst>(Arg)->getFastMathFlags();
2929 FMF2.setNoSignedZeros(false);
2930 SI->setFastMathFlags(FMF1 | FMF2);
2931 return SI;
2932 }
2933 // fabs (select Cond, -FVal, FVal) --> fabs FVal
2934 if (match(TVal, m_FNeg(m_Specific(FVal))))
2935 return replaceOperand(*II, 0, FVal);
2936 // fabs (select Cond, TVal, -TVal) --> fabs TVal
2937 if (match(FVal, m_FNeg(m_Specific(TVal))))
2938 return replaceOperand(*II, 0, TVal);
2939 }
2940
2941 Value *Magnitude, *Sign;
2942 if (match(II->getArgOperand(0),
2943 m_CopySign(m_Value(Magnitude), m_Value(Sign)))) {
2944 // fabs (copysign x, y) -> (fabs x)
2945 CallInst *AbsSign =
2946 Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Magnitude, II);
2947 return replaceInstUsesWith(*II, AbsSign);
2948 }
2949
2950 [[fallthrough]];
2951 }
2952 case Intrinsic::ceil:
2953 case Intrinsic::floor:
2954 case Intrinsic::round:
2955 case Intrinsic::roundeven:
2956 case Intrinsic::nearbyint:
2957 case Intrinsic::rint:
2958 case Intrinsic::trunc: {
2959 Value *ExtSrc;
2960 if (match(II->getArgOperand(0), m_OneUse(m_FPExt(m_Value(ExtSrc))))) {
2961 // Narrow the call: intrinsic (fpext x) -> fpext (intrinsic x)
2962 Value *NarrowII = Builder.CreateUnaryIntrinsic(IID, ExtSrc, II);
2963 return new FPExtInst(NarrowII, II->getType());
2964 }
2965 break;
2966 }
2967 case Intrinsic::cos:
2968 case Intrinsic::amdgcn_cos: {
2969 Value *X, *Sign;
2970 Value *Src = II->getArgOperand(0);
2971 if (match(Src, m_FNeg(m_Value(X))) || match(Src, m_FAbs(m_Value(X))) ||
2972 match(Src, m_CopySign(m_Value(X), m_Value(Sign)))) {
2973 // cos(-x) --> cos(x)
2974 // cos(fabs(x)) --> cos(x)
2975 // cos(copysign(x, y)) --> cos(x)
2976 return replaceOperand(*II, 0, X);
2977 }
2978 break;
2979 }
2980 case Intrinsic::sin:
2981 case Intrinsic::amdgcn_sin: {
2982 Value *X;
2983 if (match(II->getArgOperand(0), m_OneUse(m_FNeg(m_Value(X))))) {
2984 // sin(-x) --> -sin(x)
2985 Value *NewSin = Builder.CreateUnaryIntrinsic(IID, X, II);
2986 return UnaryOperator::CreateFNegFMF(NewSin, II);
2987 }
2988 break;
2989 }
2990 case Intrinsic::ldexp: {
2991 // ldexp(ldexp(x, a), b) -> ldexp(x, a + b)
2992 //
2993 // The danger is if the first ldexp would overflow to infinity or underflow
2994 // to zero, but the combined exponent avoids it. We ignore this with
2995 // reassoc.
2996 //
2997 // It's also safe to fold if we know both exponents are >= 0 or <= 0 since
2998 // it would just double down on the overflow/underflow which would occur
2999 // anyway.
3000 //
3001 // TODO: Could do better if we had range tracking for the input value
3002 // exponent. Also could broaden sign check to cover == 0 case.
3003 Value *Src = II->getArgOperand(0);
3004 Value *Exp = II->getArgOperand(1);
3005 Value *InnerSrc;
3006 Value *InnerExp;
3008 m_Value(InnerSrc), m_Value(InnerExp)))) &&
3009 Exp->getType() == InnerExp->getType()) {
3010 FastMathFlags FMF = II->getFastMathFlags();
3011 FastMathFlags InnerFlags = cast<FPMathOperator>(Src)->getFastMathFlags();
3012
3013 if ((FMF.allowReassoc() && InnerFlags.allowReassoc()) ||
3014 signBitMustBeTheSame(Exp, InnerExp, SQ.getWithInstruction(II))) {
3015 // TODO: Add nsw/nuw probably safe if integer type exceeds exponent
3016 // width.
3017 Value *NewExp = Builder.CreateAdd(InnerExp, Exp);
3018 II->setArgOperand(1, NewExp);
3019 II->setFastMathFlags(InnerFlags); // Or the inner flags.
3020 return replaceOperand(*II, 0, InnerSrc);
3021 }
3022 }
3023
3024 // ldexp(x, zext(i1 y)) -> fmul x, (select y, 2.0, 1.0)
3025 // ldexp(x, sext(i1 y)) -> fmul x, (select y, 0.5, 1.0)
3026 Value *ExtSrc;
3027 if (match(Exp, m_ZExt(m_Value(ExtSrc))) &&
3028 ExtSrc->getType()->getScalarSizeInBits() == 1) {
3029 Value *Select =
3030 Builder.CreateSelect(ExtSrc, ConstantFP::get(II->getType(), 2.0),
3031 ConstantFP::get(II->getType(), 1.0));
3033 }
3034 if (match(Exp, m_SExt(m_Value(ExtSrc))) &&
3035 ExtSrc->getType()->getScalarSizeInBits() == 1) {
3036 Value *Select =
3037 Builder.CreateSelect(ExtSrc, ConstantFP::get(II->getType(), 0.5),
3038 ConstantFP::get(II->getType(), 1.0));
3040 }
3041
3042 // ldexp(x, c ? exp : 0) -> c ? ldexp(x, exp) : x
3043 // ldexp(x, c ? 0 : exp) -> c ? x : ldexp(x, exp)
3044 ///
3045 // TODO: If we cared, should insert a canonicalize for x
3046 Value *SelectCond, *SelectLHS, *SelectRHS;
3047 if (match(II->getArgOperand(1),
3048 m_OneUse(m_Select(m_Value(SelectCond), m_Value(SelectLHS),
3049 m_Value(SelectRHS))))) {
3050 Value *NewLdexp = nullptr;
3051 Value *Select = nullptr;
3052 if (match(SelectRHS, m_ZeroInt())) {
3053 NewLdexp = Builder.CreateLdexp(Src, SelectLHS, II);
3054 Select = Builder.CreateSelect(SelectCond, NewLdexp, Src);
3055 } else if (match(SelectLHS, m_ZeroInt())) {
3056 NewLdexp = Builder.CreateLdexp(Src, SelectRHS, II);
3057 Select = Builder.CreateSelect(SelectCond, Src, NewLdexp);
3058 }
3059
3060 if (NewLdexp) {
3061 Select->takeName(II);
3062 return replaceInstUsesWith(*II, Select);
3063 }
3064 }
3065
3066 break;
3067 }
3068 case Intrinsic::ptrauth_auth:
3069 case Intrinsic::ptrauth_resign: {
3070 // (sign|resign) + (auth|resign) can be folded by omitting the middle
3071 // sign+auth component if the key and discriminator match.
3072 bool NeedSign = II->getIntrinsicID() == Intrinsic::ptrauth_resign;
3073 Value *Ptr = II->getArgOperand(0);
3074 Value *Key = II->getArgOperand(1);
3075 Value *Disc = II->getArgOperand(2);
3076
3077 // AuthKey will be the key we need to end up authenticating against in
3078 // whatever we replace this sequence with.
3079 Value *AuthKey = nullptr, *AuthDisc = nullptr, *BasePtr;
3080 if (const auto *CI = dyn_cast<CallBase>(Ptr)) {
3081 BasePtr = CI->getArgOperand(0);
3082 if (CI->getIntrinsicID() == Intrinsic::ptrauth_sign) {
3083 if (CI->getArgOperand(1) != Key || CI->getArgOperand(2) != Disc)
3084 break;
3085 } else if (CI->getIntrinsicID() == Intrinsic::ptrauth_resign) {
3086 if (CI->getArgOperand(3) != Key || CI->getArgOperand(4) != Disc)
3087 break;
3088 AuthKey = CI->getArgOperand(1);
3089 AuthDisc = CI->getArgOperand(2);
3090 } else
3091 break;
3092 } else if (const auto *PtrToInt = dyn_cast<PtrToIntOperator>(Ptr)) {
3093 // ptrauth constants are equivalent to a call to @llvm.ptrauth.sign for
3094 // our purposes, so check for that too.
3095 const auto *CPA = dyn_cast<ConstantPtrAuth>(PtrToInt->getOperand(0));
3096 if (!CPA || !CPA->isKnownCompatibleWith(Key, Disc, DL))
3097 break;
3098
3099 // resign(ptrauth(p,ks,ds),ks,ds,kr,dr) -> ptrauth(p,kr,dr)
3100 if (NeedSign && isa<ConstantInt>(II->getArgOperand(4))) {
3101 auto *SignKey = cast<ConstantInt>(II->getArgOperand(3));
3102 auto *SignDisc = cast<ConstantInt>(II->getArgOperand(4));
3103 auto *SignAddrDisc = ConstantPointerNull::get(Builder.getPtrTy());
3104 auto *NewCPA = ConstantPtrAuth::get(CPA->getPointer(), SignKey,
3105 SignDisc, SignAddrDisc);
3107 *II, ConstantExpr::getPointerCast(NewCPA, II->getType()));
3108 return eraseInstFromFunction(*II);
3109 }
3110
3111 // auth(ptrauth(p,k,d),k,d) -> p
3112 BasePtr = Builder.CreatePtrToInt(CPA->getPointer(), II->getType());
3113 } else
3114 break;
3115
3116 unsigned NewIntrin;
3117 if (AuthKey && NeedSign) {
3118 // resign(0,1) + resign(1,2) = resign(0, 2)
3119 NewIntrin = Intrinsic::ptrauth_resign;
3120 } else if (AuthKey) {
3121 // resign(0,1) + auth(1) = auth(0)
3122 NewIntrin = Intrinsic::ptrauth_auth;
3123 } else if (NeedSign) {
3124 // sign(0) + resign(0, 1) = sign(1)
3125 NewIntrin = Intrinsic::ptrauth_sign;
3126 } else {
3127 // sign(0) + auth(0) = nop
3128 replaceInstUsesWith(*II, BasePtr);
3129 return eraseInstFromFunction(*II);
3130 }
3131
3132 SmallVector<Value *, 4> CallArgs;
3133 CallArgs.push_back(BasePtr);
3134 if (AuthKey) {
3135 CallArgs.push_back(AuthKey);
3136 CallArgs.push_back(AuthDisc);
3137 }
3138
3139 if (NeedSign) {
3140 CallArgs.push_back(II->getArgOperand(3));
3141 CallArgs.push_back(II->getArgOperand(4));
3142 }
3143
3144 Function *NewFn =
3145 Intrinsic::getOrInsertDeclaration(II->getModule(), NewIntrin);
3146 return CallInst::Create(NewFn, CallArgs);
3147 }
3148 case Intrinsic::arm_neon_vtbl1:
3149 case Intrinsic::aarch64_neon_tbl1:
3150 if (Value *V = simplifyNeonTbl1(*II, Builder))
3151 return replaceInstUsesWith(*II, V);
3152 break;
3153
3154 case Intrinsic::arm_neon_vmulls:
3155 case Intrinsic::arm_neon_vmullu:
3156 case Intrinsic::aarch64_neon_smull:
3157 case Intrinsic::aarch64_neon_umull: {
3158 Value *Arg0 = II->getArgOperand(0);
3159 Value *Arg1 = II->getArgOperand(1);
3160
3161 // Handle mul by zero first:
3163 return replaceInstUsesWith(CI, ConstantAggregateZero::get(II->getType()));
3164 }
3165
3166 // Check for constant LHS & RHS - in this case we just simplify.
3167 bool Zext = (IID == Intrinsic::arm_neon_vmullu ||
3168 IID == Intrinsic::aarch64_neon_umull);
3169 VectorType *NewVT = cast<VectorType>(II->getType());
3170 if (Constant *CV0 = dyn_cast<Constant>(Arg0)) {
3171 if (Constant *CV1 = dyn_cast<Constant>(Arg1)) {
3172 Value *V0 = Builder.CreateIntCast(CV0, NewVT, /*isSigned=*/!Zext);
3173 Value *V1 = Builder.CreateIntCast(CV1, NewVT, /*isSigned=*/!Zext);
3174 return replaceInstUsesWith(CI, Builder.CreateMul(V0, V1));
3175 }
3176
3177 // Couldn't simplify - canonicalize constant to the RHS.
3178 std::swap(Arg0, Arg1);
3179 }
3180
3181 // Handle mul by one:
3182 if (Constant *CV1 = dyn_cast<Constant>(Arg1))
3183 if (ConstantInt *Splat =
3184 dyn_cast_or_null<ConstantInt>(CV1->getSplatValue()))
3185 if (Splat->isOne())
3186 return CastInst::CreateIntegerCast(Arg0, II->getType(),
3187 /*isSigned=*/!Zext);
3188
3189 break;
3190 }
3191 case Intrinsic::arm_neon_aesd:
3192 case Intrinsic::arm_neon_aese:
3193 case Intrinsic::aarch64_crypto_aesd:
3194 case Intrinsic::aarch64_crypto_aese:
3195 case Intrinsic::aarch64_sve_aesd:
3196 case Intrinsic::aarch64_sve_aese: {
3197 Value *DataArg = II->getArgOperand(0);
3198 Value *KeyArg = II->getArgOperand(1);
3199
3200 // Accept zero on either operand.
3201 if (!match(KeyArg, m_ZeroInt()))
3202 std::swap(KeyArg, DataArg);
3203
3204 // Try to use the builtin XOR in AESE and AESD to eliminate a prior XOR
3205 Value *Data, *Key;
3206 if (match(KeyArg, m_ZeroInt()) &&
3207 match(DataArg, m_Xor(m_Value(Data), m_Value(Key)))) {
3208 replaceOperand(*II, 0, Data);
3209 replaceOperand(*II, 1, Key);
3210 return II;
3211 }
3212 break;
3213 }
3214 case Intrinsic::hexagon_V6_vandvrt:
3215 case Intrinsic::hexagon_V6_vandvrt_128B: {
3216 // Simplify Q -> V -> Q conversion.
3217 if (auto Op0 = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
3218 Intrinsic::ID ID0 = Op0->getIntrinsicID();
3219 if (ID0 != Intrinsic::hexagon_V6_vandqrt &&
3220 ID0 != Intrinsic::hexagon_V6_vandqrt_128B)
3221 break;
3222 Value *Bytes = Op0->getArgOperand(1), *Mask = II->getArgOperand(1);
3223 uint64_t Bytes1 = computeKnownBits(Bytes, Op0).One.getZExtValue();
3224 uint64_t Mask1 = computeKnownBits(Mask, II).One.getZExtValue();
3225 // Check if every byte has common bits in Bytes and Mask.
3226 uint64_t C = Bytes1 & Mask1;
3227 if ((C & 0xFF) && (C & 0xFF00) && (C & 0xFF0000) && (C & 0xFF000000))
3228 return replaceInstUsesWith(*II, Op0->getArgOperand(0));
3229 }
3230 break;
3231 }
3232 case Intrinsic::stackrestore: {
3233 enum class ClassifyResult {
3234 None,
3235 Alloca,
3236 StackRestore,
3237 CallWithSideEffects,
3238 };
3239 auto Classify = [](const Instruction *I) {
3240 if (isa<AllocaInst>(I))
3241 return ClassifyResult::Alloca;
3242
3243 if (auto *CI = dyn_cast<CallInst>(I)) {
3244 if (auto *II = dyn_cast<IntrinsicInst>(CI)) {
3245 if (II->getIntrinsicID() == Intrinsic::stackrestore)
3246 return ClassifyResult::StackRestore;
3247
3248 if (II->mayHaveSideEffects())
3249 return ClassifyResult::CallWithSideEffects;
3250 } else {
3251 // Consider all non-intrinsic calls to be side effects
3252 return ClassifyResult::CallWithSideEffects;
3253 }
3254 }
3255
3256 return ClassifyResult::None;
3257 };
3258
3259 // If the stacksave and the stackrestore are in the same BB, and there is
3260 // no intervening call, alloca, or stackrestore of a different stacksave,
3261 // remove the restore. This can happen when variable allocas are DCE'd.
3262 if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
3263 if (SS->getIntrinsicID() == Intrinsic::stacksave &&
3264 SS->getParent() == II->getParent()) {
3265 BasicBlock::iterator BI(SS);
3266 bool CannotRemove = false;
3267 for (++BI; &*BI != II; ++BI) {
3268 switch (Classify(&*BI)) {
3269 case ClassifyResult::None:
3270 // So far so good, look at next instructions.
3271 break;
3272
3273 case ClassifyResult::StackRestore:
3274 // If we found an intervening stackrestore for a different
3275 // stacksave, we can't remove the stackrestore. Otherwise, continue.
3276 if (cast<IntrinsicInst>(*BI).getArgOperand(0) != SS)
3277 CannotRemove = true;
3278 break;
3279
3280 case ClassifyResult::Alloca:
3281 case ClassifyResult::CallWithSideEffects:
3282 // If we found an alloca, a non-intrinsic call, or an intrinsic
3283 // call with side effects, we can't remove the stackrestore.
3284 CannotRemove = true;
3285 break;
3286 }
3287 if (CannotRemove)
3288 break;
3289 }
3290
3291 if (!CannotRemove)
3292 return eraseInstFromFunction(CI);
3293 }
3294 }
3295
3296 // Scan down this block to see if there is another stack restore in the
3297 // same block without an intervening call/alloca.
3299 Instruction *TI = II->getParent()->getTerminator();
3300 bool CannotRemove = false;
3301 for (++BI; &*BI != TI; ++BI) {
3302 switch (Classify(&*BI)) {
3303 case ClassifyResult::None:
3304 // So far so good, look at next instructions.
3305 break;
3306
3307 case ClassifyResult::StackRestore:
3308 // If there is a stackrestore below this one, remove this one.
3309 return eraseInstFromFunction(CI);
3310
3311 case ClassifyResult::Alloca:
3312 case ClassifyResult::CallWithSideEffects:
3313 // If we found an alloca, a non-intrinsic call, or an intrinsic call
3314 // with side effects (such as llvm.stacksave and llvm.read_register),
3315 // we can't remove the stack restore.
3316 CannotRemove = true;
3317 break;
3318 }
3319 if (CannotRemove)
3320 break;
3321 }
3322
3323 // If the stack restore is in a return, resume, or unwind block and if there
3324 // are no allocas or calls between the restore and the return, nuke the
3325 // restore.
3326 if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI)))
3327 return eraseInstFromFunction(CI);
3328 break;
3329 }
3330 case Intrinsic::lifetime_end:
3331 // Asan needs to poison memory to detect invalid access which is possible
3332 // even for empty lifetime range.
3333 if (II->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
3334 II->getFunction()->hasFnAttribute(Attribute::SanitizeMemory) ||
3335 II->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
3336 break;
3337
3338 if (removeTriviallyEmptyRange(*II, *this, [](const IntrinsicInst &I) {
3339 return I.getIntrinsicID() == Intrinsic::lifetime_start;
3340 }))
3341 return nullptr;
3342 break;
3343 case Intrinsic::assume: {
3344 Value *IIOperand = II->getArgOperand(0);
3346 II->getOperandBundlesAsDefs(OpBundles);
3347
3348 /// This will remove the boolean Condition from the assume given as
3349 /// argument and remove the assume if it becomes useless.
3350 /// always returns nullptr for use as a return values.
3351 auto RemoveConditionFromAssume = [&](Instruction *Assume) -> Instruction * {
3352 assert(isa<AssumeInst>(Assume));
3354 return eraseInstFromFunction(CI);
3355 replaceUse(II->getOperandUse(0), ConstantInt::getTrue(II->getContext()));
3356 return nullptr;
3357 };
3358 // Remove an assume if it is followed by an identical assume.
3359 // TODO: Do we need this? Unless there are conflicting assumptions, the
3360 // computeKnownBits(IIOperand) below here eliminates redundant assumes.
3361 Instruction *Next = II->getNextNode();
3363 return RemoveConditionFromAssume(Next);
3364
3365 // Canonicalize assume(a && b) -> assume(a); assume(b);
3366 // Note: New assumption intrinsics created here are registered by
3367 // the InstCombineIRInserter object.
3368 FunctionType *AssumeIntrinsicTy = II->getFunctionType();
3369 Value *AssumeIntrinsic = II->getCalledOperand();
3370 Value *A, *B;
3371 if (match(IIOperand, m_LogicalAnd(m_Value(A), m_Value(B)))) {
3372 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, A, OpBundles,
3373 II->getName());
3374 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, B, II->getName());
3375 return eraseInstFromFunction(*II);
3376 }
3377 // assume(!(a || b)) -> assume(!a); assume(!b);
3378 if (match(IIOperand, m_Not(m_LogicalOr(m_Value(A), m_Value(B))))) {
3379 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
3380 Builder.CreateNot(A), OpBundles, II->getName());
3381 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
3382 Builder.CreateNot(B), II->getName());
3383 return eraseInstFromFunction(*II);
3384 }
3385
3386 // assume( (load addr) != null ) -> add 'nonnull' metadata to load
3387 // (if assume is valid at the load)
3388 Instruction *LHS;
3390 m_Zero())) &&
3391 LHS->getOpcode() == Instruction::Load &&
3392 LHS->getType()->isPointerTy() &&
3393 isValidAssumeForContext(II, LHS, &DT)) {
3394 MDNode *MD = MDNode::get(II->getContext(), {});
3395 LHS->setMetadata(LLVMContext::MD_nonnull, MD);
3396 LHS->setMetadata(LLVMContext::MD_noundef, MD);
3397 return RemoveConditionFromAssume(II);
3398
3399 // TODO: apply nonnull return attributes to calls and invokes
3400 // TODO: apply range metadata for range check patterns?
3401 }
3402
3403 for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
3404 OperandBundleUse OBU = II->getOperandBundleAt(Idx);
3405
3406 // Separate storage assumptions apply to the underlying allocations, not
3407 // any particular pointer within them. When evaluating the hints for AA
3408 // purposes we getUnderlyingObject them; by precomputing the answers here
3409 // we can avoid having to do so repeatedly there.
3410 if (OBU.getTagName() == "separate_storage") {
3411 assert(OBU.Inputs.size() == 2);
3412 auto MaybeSimplifyHint = [&](const Use &U) {
3413 Value *Hint = U.get();
3414 // Not having a limit is safe because InstCombine removes unreachable
3415 // code.
3416 Value *UnderlyingObject = getUnderlyingObject(Hint, /*MaxLookup*/ 0);
3417 if (Hint != UnderlyingObject)
3418 replaceUse(const_cast<Use &>(U), UnderlyingObject);
3419 };
3420 MaybeSimplifyHint(OBU.Inputs[0]);
3421 MaybeSimplifyHint(OBU.Inputs[1]);
3422 }
3423
3424 // Try to remove redundant alignment assumptions.
3425 if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) {
3427 *cast<AssumeInst>(II), II->arg_size() + Idx);
3428 if (!RK || RK.AttrKind != Attribute::Alignment ||
3430 continue;
3431
3432 // Remove align 1 bundles; they don't add any useful information.
3433 if (RK.ArgValue == 1)
3435
3436 // Don't try to remove align assumptions for pointers derived from
3437 // arguments. We might lose information if the function gets inline and
3438 // the align argument attribute disappears.
3440 if (!UO || isa<Argument>(UO))
3441 continue;
3442
3443 // Compute known bits for the pointer, passing nullptr as context to
3444 // avoid computeKnownBits using the assumption we are about to remove
3445 // for reasoning.
3446 KnownBits Known = computeKnownBits(RK.WasOn, /*CtxI=*/nullptr);
3447 unsigned TZ = std::min(Known.countMinTrailingZeros(),
3449 if ((1ULL << TZ) < RK.ArgValue)
3450 continue;
3452 }
3453 }
3454
3455 // Convert nonnull assume like:
3456 // %A = icmp ne i32* %PTR, null
3457 // call void @llvm.assume(i1 %A)
3458 // into
3459 // call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ]
3461 match(IIOperand,
3463 A->getType()->isPointerTy()) {
3464 if (auto *Replacement = buildAssumeFromKnowledge(
3465 {RetainedKnowledge{Attribute::NonNull, 0, A}}, Next, &AC, &DT)) {
3466
3467 Replacement->insertBefore(Next->getIterator());
3468 AC.registerAssumption(Replacement);
3469 return RemoveConditionFromAssume(II);
3470 }
3471 }
3472
3473 // Convert alignment assume like:
3474 // %B = ptrtoint i32* %A to i64
3475 // %C = and i64 %B, Constant
3476 // %D = icmp eq i64 %C, 0
3477 // call void @llvm.assume(i1 %D)
3478 // into
3479 // call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 Constant + 1)]
3480 uint64_t AlignMask = 1;
3482 (match(IIOperand, m_Not(m_Trunc(m_Value(A)))) ||
3483 match(IIOperand,
3485 m_And(m_Value(A), m_ConstantInt(AlignMask)),
3486 m_Zero())))) {
3487 if (isPowerOf2_64(AlignMask + 1)) {
3488 uint64_t Offset = 0;
3490 if (match(A, m_PtrToInt(m_Value(A)))) {
3491 /// Note: this doesn't preserve the offset information but merges
3492 /// offset and alignment.
3493 /// TODO: we can generate a GEP instead of merging the alignment with
3494 /// the offset.
3495 RetainedKnowledge RK{Attribute::Alignment,
3496 (unsigned)MinAlign(Offset, AlignMask + 1), A};
3497 if (auto *Replacement =
3499
3500 Replacement->insertAfter(II->getIterator());
3501 AC.registerAssumption(Replacement);
3502 }
3503 return RemoveConditionFromAssume(II);
3504 }
3505 }
3506 }
3507
3508 /// Canonicalize Knowledge in operand bundles.
3509 if (EnableKnowledgeRetention && II->hasOperandBundles()) {
3510 for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
3511 auto &BOI = II->bundle_op_info_begin()[Idx];
3514 if (BOI.End - BOI.Begin > 2)
3515 continue; // Prevent reducing knowledge in an align with offset since
3516 // extracting a RetainedKnowledge from them looses offset
3517 // information
3518 RetainedKnowledge CanonRK =
3521 &getDominatorTree());
3522 if (CanonRK == RK)
3523 continue;
3524 if (!CanonRK) {
3525 if (BOI.End - BOI.Begin > 0) {
3526 Worklist.pushValue(II->op_begin()[BOI.Begin]);
3527 Value::dropDroppableUse(II->op_begin()[BOI.Begin]);
3528 }
3529 continue;
3530 }
3531 assert(RK.AttrKind == CanonRK.AttrKind);
3532 if (BOI.End - BOI.Begin > 0)
3533 II->op_begin()[BOI.Begin].set(CanonRK.WasOn);
3534 if (BOI.End - BOI.Begin > 1)
3535 II->op_begin()[BOI.Begin + 1].set(ConstantInt::get(
3536 Type::getInt64Ty(II->getContext()), CanonRK.ArgValue));
3537 if (RK.WasOn)
3538 Worklist.pushValue(RK.WasOn);
3539 return II;
3540 }
3541 }
3542
3543 // If there is a dominating assume with the same condition as this one,
3544 // then this one is redundant, and should be removed.
3545 KnownBits Known(1);
3546 computeKnownBits(IIOperand, Known, II);
3548 return eraseInstFromFunction(*II);
3549
3550 // assume(false) is unreachable.
3551 if (match(IIOperand, m_CombineOr(m_Zero(), m_Undef()))) {
3553 return eraseInstFromFunction(*II);
3554 }
3555
3556 // Update the cache of affected values for this assumption (we might be
3557 // here because we just simplified the condition).
3558 AC.updateAffectedValues(cast<AssumeInst>(II));
3559 break;
3560 }
3561 case Intrinsic::experimental_guard: {
3562 // Is this guard followed by another guard? We scan forward over a small
3563 // fixed window of instructions to handle common cases with conditions
3564 // computed between guards.
3565 Instruction *NextInst = II->getNextNode();
3566 for (unsigned i = 0; i < GuardWideningWindow; i++) {
3567 // Note: Using context-free form to avoid compile time blow up
3568 if (!isSafeToSpeculativelyExecute(NextInst))
3569 break;
3570 NextInst = NextInst->getNextNode();
3571 }
3572 Value *NextCond = nullptr;
3573 if (match(NextInst,
3575 Value *CurrCond = II->getArgOperand(0);
3576
3577 // Remove a guard that it is immediately preceded by an identical guard.
3578 // Otherwise canonicalize guard(a); guard(b) -> guard(a & b).
3579 if (CurrCond != NextCond) {
3580 Instruction *MoveI = II->getNextNode();
3581 while (MoveI != NextInst) {
3582 auto *Temp = MoveI;
3583 MoveI = MoveI->getNextNode();
3584 Temp->moveBefore(II->getIterator());
3585 }
3586 replaceOperand(*II, 0, Builder.CreateAnd(CurrCond, NextCond));
3587 }
3588 eraseInstFromFunction(*NextInst);
3589 return II;
3590 }
3591 break;
3592 }
3593 case Intrinsic::vector_insert: {
3594 Value *Vec = II->getArgOperand(0);
3595 Value *SubVec = II->getArgOperand(1);
3596 Value *Idx = II->getArgOperand(2);
3597 auto *DstTy = dyn_cast<FixedVectorType>(II->getType());
3598 auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
3599 auto *SubVecTy = dyn_cast<FixedVectorType>(SubVec->getType());
3600
3601 // Only canonicalize if the destination vector, Vec, and SubVec are all
3602 // fixed vectors.
3603 if (DstTy && VecTy && SubVecTy) {
3604 unsigned DstNumElts = DstTy->getNumElements();
3605 unsigned VecNumElts = VecTy->getNumElements();
3606 unsigned SubVecNumElts = SubVecTy->getNumElements();
3607 unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
3608
3609 // An insert that entirely overwrites Vec with SubVec is a nop.
3610 if (VecNumElts == SubVecNumElts)
3611 return replaceInstUsesWith(CI, SubVec);
3612
3613 // Widen SubVec into a vector of the same width as Vec, since
3614 // shufflevector requires the two input vectors to be the same width.
3615 // Elements beyond the bounds of SubVec within the widened vector are
3616 // undefined.
3617 SmallVector<int, 8> WidenMask;
3618 unsigned i;
3619 for (i = 0; i != SubVecNumElts; ++i)
3620 WidenMask.push_back(i);
3621 for (; i != VecNumElts; ++i)
3622 WidenMask.push_back(PoisonMaskElem);
3623
3624 Value *WidenShuffle = Builder.CreateShuffleVector(SubVec, WidenMask);
3625
3627 for (unsigned i = 0; i != IdxN; ++i)
3628 Mask.push_back(i);
3629 for (unsigned i = DstNumElts; i != DstNumElts + SubVecNumElts; ++i)
3630 Mask.push_back(i);
3631 for (unsigned i = IdxN + SubVecNumElts; i != DstNumElts; ++i)
3632 Mask.push_back(i);
3633
3634 Value *Shuffle = Builder.CreateShuffleVector(Vec, WidenShuffle, Mask);
3635 return replaceInstUsesWith(CI, Shuffle);
3636 }
3637 break;
3638 }
3639 case Intrinsic::vector_extract: {
3640 Value *Vec = II->getArgOperand(0);
3641 Value *Idx = II->getArgOperand(1);
3642
3643 Type *ReturnType = II->getType();
3644 // (extract_vector (insert_vector InsertTuple, InsertValue, InsertIdx),
3645 // ExtractIdx)
3646 unsigned ExtractIdx = cast<ConstantInt>(Idx)->getZExtValue();
3647 Value *InsertTuple, *InsertIdx, *InsertValue;
3649 m_Value(InsertValue),
3650 m_Value(InsertIdx))) &&
3651 InsertValue->getType() == ReturnType) {
3652 unsigned Index = cast<ConstantInt>(InsertIdx)->getZExtValue();
3653 // Case where we get the same index right after setting it.
3654 // extract.vector(insert.vector(InsertTuple, InsertValue, Idx), Idx) -->
3655 // InsertValue
3656 if (ExtractIdx == Index)
3657 return replaceInstUsesWith(CI, InsertValue);
3658 // If we are getting a different index than what was set in the
3659 // insert.vector intrinsic. We can just set the input tuple to the one up
3660 // in the chain. extract.vector(insert.vector(InsertTuple, InsertValue,
3661 // InsertIndex), ExtractIndex)
3662 // --> extract.vector(InsertTuple, ExtractIndex)
3663 else
3664 return replaceOperand(CI, 0, InsertTuple);
3665 }
3666
3667 auto *DstTy = dyn_cast<VectorType>(ReturnType);
3668 auto *VecTy = dyn_cast<VectorType>(Vec->getType());
3669
3670 if (DstTy && VecTy) {
3671 auto DstEltCnt = DstTy->getElementCount();
3672 auto VecEltCnt = VecTy->getElementCount();
3673 unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
3674
3675 // Extracting the entirety of Vec is a nop.
3676 if (DstEltCnt == VecTy->getElementCount()) {
3677 replaceInstUsesWith(CI, Vec);
3678 return eraseInstFromFunction(CI);
3679 }
3680
3681 // Only canonicalize to shufflevector if the destination vector and
3682 // Vec are fixed vectors.
3683 if (VecEltCnt.isScalable() || DstEltCnt.isScalable())
3684 break;
3685
3687 for (unsigned i = 0; i != DstEltCnt.getKnownMinValue(); ++i)
3688 Mask.push_back(IdxN + i);
3689
3690 Value *Shuffle = Builder.CreateShuffleVector(Vec, Mask);
3691 return replaceInstUsesWith(CI, Shuffle);
3692 }
3693 break;
3694 }
3695 case Intrinsic::experimental_vp_reverse: {
3696 Value *X;
3697 Value *Vec = II->getArgOperand(0);
3698 Value *Mask = II->getArgOperand(1);
3699 if (!match(Mask, m_AllOnes()))
3700 break;
3701 Value *EVL = II->getArgOperand(2);
3702 // TODO: Canonicalize experimental.vp.reverse after unop/binops?
3703 // rev(unop rev(X)) --> unop X
3704 if (match(Vec,
3706 m_Value(X), m_AllOnes(), m_Specific(EVL)))))) {
3707 auto *OldUnOp = cast<UnaryOperator>(Vec);
3709 OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(),
3710 II->getIterator());
3711 return replaceInstUsesWith(CI, NewUnOp);
3712 }
3713 break;
3714 }
3715 case Intrinsic::vector_reduce_or:
3716 case Intrinsic::vector_reduce_and: {
3717 // Canonicalize logical or/and reductions:
3718 // Or reduction for i1 is represented as:
3719 // %val = bitcast <ReduxWidth x i1> to iReduxWidth
3720 // %res = cmp ne iReduxWidth %val, 0
3721 // And reduction for i1 is represented as:
3722 // %val = bitcast <ReduxWidth x i1> to iReduxWidth
3723 // %res = cmp eq iReduxWidth %val, 11111
3724 Value *Arg = II->getArgOperand(0);
3725 Value *Vect;
3726
3727 if (Value *NewOp =
3728 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3729 replaceUse(II->getOperandUse(0), NewOp);
3730 return II;
3731 }
3732
3733 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3734 if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
3735 if (FTy->getElementType() == Builder.getInt1Ty()) {
3736 Value *Res = Builder.CreateBitCast(
3737 Vect, Builder.getIntNTy(FTy->getNumElements()));
3738 if (IID == Intrinsic::vector_reduce_and) {
3739 Res = Builder.CreateICmpEQ(
3741 } else {
3742 assert(IID == Intrinsic::vector_reduce_or &&
3743 "Expected or reduction.");
3744 Res = Builder.CreateIsNotNull(Res);
3745 }
3746 if (Arg != Vect)
3747 Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
3748 II->getType());
3749 return replaceInstUsesWith(CI, Res);
3750 }
3751 }
3752 [[fallthrough]];
3753 }
3754 case Intrinsic::vector_reduce_add: {
3755 if (IID == Intrinsic::vector_reduce_add) {
3756 // Convert vector_reduce_add(ZExt(<n x i1>)) to
3757 // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
3758 // Convert vector_reduce_add(SExt(<n x i1>)) to
3759 // -ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
3760 // Convert vector_reduce_add(<n x i1>) to
3761 // Trunc(ctpop(bitcast <n x i1> to in)).
3762 Value *Arg = II->getArgOperand(0);
3763 Value *Vect;
3764
3765 if (Value *NewOp =
3766 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3767 replaceUse(II->getOperandUse(0), NewOp);
3768 return II;
3769 }
3770
3771 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3772 if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
3773 if (FTy->getElementType() == Builder.getInt1Ty()) {
3774 Value *V = Builder.CreateBitCast(
3775 Vect, Builder.getIntNTy(FTy->getNumElements()));
3776 Value *Res = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, V);
3777 if (Res->getType() != II->getType())
3778 Res = Builder.CreateZExtOrTrunc(Res, II->getType());
3779 if (Arg != Vect &&
3780 cast<Instruction>(Arg)->getOpcode() == Instruction::SExt)
3781 Res = Builder.CreateNeg(Res);
3782 return replaceInstUsesWith(CI, Res);
3783 }
3784 }
3785
3786 // vector.reduce.add.vNiM(splat(%x)) -> mul(%x, N)
3787 if (Value *Splat = getSplatValue(Arg)) {
3788 ElementCount VecToReduceCount =
3789 cast<VectorType>(Arg->getType())->getElementCount();
3790 if (VecToReduceCount.isFixed()) {
3791 unsigned VectorSize = VecToReduceCount.getFixedValue();
3792 return BinaryOperator::CreateMul(
3793 Splat, ConstantInt::get(Splat->getType(), VectorSize));
3794 }
3795 }
3796 }
3797 [[fallthrough]];
3798 }
3799 case Intrinsic::vector_reduce_xor: {
3800 if (IID == Intrinsic::vector_reduce_xor) {
3801 // Exclusive disjunction reduction over the vector with
3802 // (potentially-extended) i1 element type is actually a
3803 // (potentially-extended) arithmetic `add` reduction over the original
3804 // non-extended value:
3805 // vector_reduce_xor(?ext(<n x i1>))
3806 // -->
3807 // ?ext(vector_reduce_add(<n x i1>))
3808 Value *Arg = II->getArgOperand(0);
3809 Value *Vect;
3810
3811 if (Value *NewOp =
3812 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3813 replaceUse(II->getOperandUse(0), NewOp);
3814 return II;
3815 }
3816
3817 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3818 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3819 if (VTy->getElementType() == Builder.getInt1Ty()) {
3820 Value *Res = Builder.CreateAddReduce(Vect);
3821 if (Arg != Vect)
3822 Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
3823 II->getType());
3824 return replaceInstUsesWith(CI, Res);
3825 }
3826 }
3827 }
3828 [[fallthrough]];
3829 }
3830 case Intrinsic::vector_reduce_mul: {
3831 if (IID == Intrinsic::vector_reduce_mul) {
3832 // Multiplicative reduction over the vector with (potentially-extended)
3833 // i1 element type is actually a (potentially zero-extended)
3834 // logical `and` reduction over the original non-extended value:
3835 // vector_reduce_mul(?ext(<n x i1>))
3836 // -->
3837 // zext(vector_reduce_and(<n x i1>))
3838 Value *Arg = II->getArgOperand(0);
3839 Value *Vect;
3840
3841 if (Value *NewOp =
3842 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3843 replaceUse(II->getOperandUse(0), NewOp);
3844 return II;
3845 }
3846
3847 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3848 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3849 if (VTy->getElementType() == Builder.getInt1Ty()) {
3850 Value *Res = Builder.CreateAndReduce(Vect);
3851 if (Res->getType() != II->getType())
3852 Res = Builder.CreateZExt(Res, II->getType());
3853 return replaceInstUsesWith(CI, Res);
3854 }
3855 }
3856 }
3857 [[fallthrough]];
3858 }
3859 case Intrinsic::vector_reduce_umin:
3860 case Intrinsic::vector_reduce_umax: {
3861 if (IID == Intrinsic::vector_reduce_umin ||
3862 IID == Intrinsic::vector_reduce_umax) {
3863 // UMin/UMax reduction over the vector with (potentially-extended)
3864 // i1 element type is actually a (potentially-extended)
3865 // logical `and`/`or` reduction over the original non-extended value:
3866 // vector_reduce_u{min,max}(?ext(<n x i1>))
3867 // -->
3868 // ?ext(vector_reduce_{and,or}(<n x i1>))
3869 Value *Arg = II->getArgOperand(0);
3870 Value *Vect;
3871
3872 if (Value *NewOp =
3873 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3874 replaceUse(II->getOperandUse(0), NewOp);
3875 return II;
3876 }
3877
3878 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3879 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3880 if (VTy->getElementType() == Builder.getInt1Ty()) {
3881 Value *Res = IID == Intrinsic::vector_reduce_umin
3882 ? Builder.CreateAndReduce(Vect)
3883 : Builder.CreateOrReduce(Vect);
3884 if (Arg != Vect)
3885 Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
3886 II->getType());
3887 return replaceInstUsesWith(CI, Res);
3888 }
3889 }
3890 }
3891 [[fallthrough]];
3892 }
3893 case Intrinsic::vector_reduce_smin:
3894 case Intrinsic::vector_reduce_smax: {
3895 if (IID == Intrinsic::vector_reduce_smin ||
3896 IID == Intrinsic::vector_reduce_smax) {
3897 // SMin/SMax reduction over the vector with (potentially-extended)
3898 // i1 element type is actually a (potentially-extended)
3899 // logical `and`/`or` reduction over the original non-extended value:
3900 // vector_reduce_s{min,max}(<n x i1>)
3901 // -->
3902 // vector_reduce_{or,and}(<n x i1>)
3903 // and
3904 // vector_reduce_s{min,max}(sext(<n x i1>))
3905 // -->
3906 // sext(vector_reduce_{or,and}(<n x i1>))
3907 // and
3908 // vector_reduce_s{min,max}(zext(<n x i1>))
3909 // -->
3910 // zext(vector_reduce_{and,or}(<n x i1>))
3911 Value *Arg = II->getArgOperand(0);
3912 Value *Vect;
3913
3914 if (Value *NewOp =
3915 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3916 replaceUse(II->getOperandUse(0), NewOp);
3917 return II;
3918 }
3919
3920 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3921 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3922 if (VTy->getElementType() == Builder.getInt1Ty()) {
3923 Instruction::CastOps ExtOpc = Instruction::CastOps::CastOpsEnd;
3924 if (Arg != Vect)
3925 ExtOpc = cast<CastInst>(Arg)->getOpcode();
3926 Value *Res = ((IID == Intrinsic::vector_reduce_smin) ==
3927 (ExtOpc == Instruction::CastOps::ZExt))
3928 ? Builder.CreateAndReduce(Vect)
3929 : Builder.CreateOrReduce(Vect);
3930 if (Arg != Vect)
3931 Res = Builder.CreateCast(ExtOpc, Res, II->getType());
3932 return replaceInstUsesWith(CI, Res);
3933 }
3934 }
3935 }
3936 [[fallthrough]];
3937 }
3938 case Intrinsic::vector_reduce_fmax:
3939 case Intrinsic::vector_reduce_fmin:
3940 case Intrinsic::vector_reduce_fadd:
3941 case Intrinsic::vector_reduce_fmul: {
3942 bool CanReorderLanes = (IID != Intrinsic::vector_reduce_fadd &&
3943 IID != Intrinsic::vector_reduce_fmul) ||
3944 II->hasAllowReassoc();
3945 const unsigned ArgIdx = (IID == Intrinsic::vector_reduce_fadd ||
3946 IID == Intrinsic::vector_reduce_fmul)
3947 ? 1
3948 : 0;
3949 Value *Arg = II->getArgOperand(ArgIdx);
3950 if (Value *NewOp = simplifyReductionOperand(Arg, CanReorderLanes)) {
3951 replaceUse(II->getOperandUse(ArgIdx), NewOp);
3952 return nullptr;
3953 }
3954 break;
3955 }
3956 case Intrinsic::is_fpclass: {
3957 if (Instruction *I = foldIntrinsicIsFPClass(*II))
3958 return I;
3959 break;
3960 }
3961 case Intrinsic::threadlocal_address: {
3962 Align MinAlign = getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);
3963 MaybeAlign Align = II->getRetAlign();
3964 if (MinAlign > Align.valueOrOne()) {
3965 II->addRetAttr(Attribute::getWithAlignment(II->getContext(), MinAlign));
3966 return II;
3967 }
3968 break;
3969 }
3970 case Intrinsic::frexp: {
3971 Value *X;
3972 // The first result is idempotent with the added complication of the struct
3973 // return, and the second result is zero because the value is already
3974 // normalized.
3975 if (match(II->getArgOperand(0), m_ExtractValue<0>(m_Value(X)))) {
3977 X = Builder.CreateInsertValue(
3978 X, Constant::getNullValue(II->getType()->getStructElementType(1)),
3979 1);
3980 return replaceInstUsesWith(*II, X);
3981 }
3982 }
3983 break;
3984 }
3985 case Intrinsic::get_active_lane_mask: {
3986 const APInt *Op0, *Op1;
3987 if (match(II->getOperand(0), m_StrictlyPositive(Op0)) &&
3988 match(II->getOperand(1), m_APInt(Op1))) {
3989 Type *OpTy = II->getOperand(0)->getType();
3990 return replaceInstUsesWith(
3991 *II, Builder.CreateIntrinsic(
3992 II->getType(), Intrinsic::get_active_lane_mask,
3993 {Constant::getNullValue(OpTy),
3994 ConstantInt::get(OpTy, Op1->usub_sat(*Op0))}));
3995 }
3996 break;
3997 }
3998 default: {
3999 // Handle target specific intrinsics
4000 std::optional<Instruction *> V = targetInstCombineIntrinsic(*II);
4001 if (V)
4002 return *V;
4003 break;
4004 }
4005 }
4006
4007 // Try to fold intrinsic into select/phi operands. This is legal if:
4008 // * The intrinsic is speculatable.
4009 // * The select condition is not a vector, or the intrinsic does not
4010 // perform cross-lane operations.
4013 for (Value *Op : II->args()) {
4014 if (auto *Sel = dyn_cast<SelectInst>(Op))
4015 if (Instruction *R = FoldOpIntoSelect(*II, Sel))
4016 return R;
4017 if (auto *Phi = dyn_cast<PHINode>(Op))
4018 if (Instruction *R = foldOpIntoPhi(*II, Phi))
4019 return R;
4020 }
4021
4023 return Shuf;
4024
4026 return replaceInstUsesWith(*II, Reverse);
4027
4029 return replaceInstUsesWith(*II, Res);
4030
4031 // Some intrinsics (like experimental_gc_statepoint) can be used in invoke
4032 // context, so it is handled in visitCallBase and we should trigger it.
4033 return visitCallBase(*II);
4034}
4035
4036// Fence instruction simplification
4038 auto *NFI = dyn_cast<FenceInst>(FI.getNextNode());
4039 // This check is solely here to handle arbitrary target-dependent syncscopes.
4040 // TODO: Can remove if does not matter in practice.
4041 if (NFI && FI.isIdenticalTo(NFI))
4042 return eraseInstFromFunction(FI);
4043
4044 // Returns true if FI1 is identical or stronger fence than FI2.
4045 auto isIdenticalOrStrongerFence = [](FenceInst *FI1, FenceInst *FI2) {
4046 auto FI1SyncScope = FI1->getSyncScopeID();
4047 // Consider same scope, where scope is global or single-thread.
4048 if (FI1SyncScope != FI2->getSyncScopeID() ||
4049 (FI1SyncScope != SyncScope::System &&
4050 FI1SyncScope != SyncScope::SingleThread))
4051 return false;
4052
4053 return isAtLeastOrStrongerThan(FI1->getOrdering(), FI2->getOrdering());
4054 };
4055 if (NFI && isIdenticalOrStrongerFence(NFI, &FI))
4056 return eraseInstFromFunction(FI);
4057
4058 if (auto *PFI = dyn_cast_or_null<FenceInst>(FI.getPrevNode()))
4059 if (isIdenticalOrStrongerFence(PFI, &FI))
4060 return eraseInstFromFunction(FI);
4061 return nullptr;
4062}
4063
4064// InvokeInst simplification
4066 return visitCallBase(II);
4067}
4068
4069// CallBrInst simplification
4071 return visitCallBase(CBI);
4072}
4073
4074Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
4075 if (!CI->getCalledFunction()) return nullptr;
4076
4077 // Skip optimizing notail and musttail calls so
4078 // LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.
4079 // LibCallSimplifier::optimizeCall should try to preserve tail calls though.
4080 if (CI->isMustTailCall() || CI->isNoTailCall())
4081 return nullptr;
4082
4083 auto InstCombineRAUW = [this](Instruction *From, Value *With) {
4084 replaceInstUsesWith(*From, With);
4085 };
4086 auto InstCombineErase = [this](Instruction *I) {
4088 };
4089 LibCallSimplifier Simplifier(DL, &TLI, &DT, &DC, &AC, ORE, BFI, PSI,
4090 InstCombineRAUW, InstCombineErase);
4091 if (Value *With = Simplifier.optimizeCall(CI, Builder)) {
4092 ++NumSimplified;
4093 return CI->use_empty() ? CI : replaceInstUsesWith(*CI, With);
4094 }
4095
4096 return nullptr;
4097}
4098
4100 // Strip off at most one level of pointer casts, looking for an alloca. This
4101 // is good enough in practice and simpler than handling any number of casts.
4102 Value *Underlying = TrampMem->stripPointerCasts();
4103 if (Underlying != TrampMem &&
4104 (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem))
4105 return nullptr;
4106 if (!isa<AllocaInst>(Underlying))
4107 return nullptr;
4108
4109 IntrinsicInst *InitTrampoline = nullptr;
4110 for (User *U : TrampMem->users()) {
4112 if (!II)
4113 return nullptr;
4114 if (II->getIntrinsicID() == Intrinsic::init_trampoline) {
4115 if (InitTrampoline)
4116 // More than one init_trampoline writes to this value. Give up.
4117 return nullptr;
4118 InitTrampoline = II;
4119 continue;
4120 }
4121 if (II->getIntrinsicID() == Intrinsic::adjust_trampoline)
4122 // Allow any number of calls to adjust.trampoline.
4123 continue;
4124 return nullptr;
4125 }
4126
4127 // No call to init.trampoline found.
4128 if (!InitTrampoline)
4129 return nullptr;
4130
4131 // Check that the alloca is being used in the expected way.
4132 if (InitTrampoline->getOperand(0) != TrampMem)
4133 return nullptr;
4134
4135 return InitTrampoline;
4136}
4137
4139 Value *TrampMem) {
4140 // Visit all the previous instructions in the basic block, and try to find a
4141 // init.trampoline which has a direct path to the adjust.trampoline.
4142 for (BasicBlock::iterator I = AdjustTramp->getIterator(),
4143 E = AdjustTramp->getParent()->begin();
4144 I != E;) {
4145 Instruction *Inst = &*--I;
4147 if (II->getIntrinsicID() == Intrinsic::init_trampoline &&
4148 II->getOperand(0) == TrampMem)
4149 return II;
4150 if (Inst->mayWriteToMemory())
4151 return nullptr;
4152 }
4153 return nullptr;
4154}
4155
4156// Given a call to llvm.adjust.trampoline, find and return the corresponding
4157// call to llvm.init.trampoline if the call to the trampoline can be optimized
4158// to a direct call to a function. Otherwise return NULL.
4160 Callee = Callee->stripPointerCasts();
4161 IntrinsicInst *AdjustTramp = dyn_cast<IntrinsicInst>(Callee);
4162 if (!AdjustTramp ||
4163 AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline)
4164 return nullptr;
4165
4166 Value *TrampMem = AdjustTramp->getOperand(0);
4167
4169 return IT;
4170 if (IntrinsicInst *IT = findInitTrampolineFromBB(AdjustTramp, TrampMem))
4171 return IT;
4172 return nullptr;
4173}
4174
4175Instruction *InstCombinerImpl::foldPtrAuthIntrinsicCallee(CallBase &Call) {
4176 const Value *Callee = Call.getCalledOperand();
4177 const auto *IPC = dyn_cast<IntToPtrInst>(Callee);
4178 if (!IPC || !IPC->isNoopCast(DL))
4179 return nullptr;
4180
4181 const auto *II = dyn_cast<IntrinsicInst>(IPC->getOperand(0));
4182 if (!II)
4183 return nullptr;
4184
4185 Intrinsic::ID IIID = II->getIntrinsicID();
4186 if (IIID != Intrinsic::ptrauth_resign && IIID != Intrinsic::ptrauth_sign)
4187 return nullptr;
4188
4189 // Isolate the ptrauth bundle from the others.
4190 std::optional<OperandBundleUse> PtrAuthBundleOrNone;
4192 for (unsigned BI = 0, BE = Call.getNumOperandBundles(); BI != BE; ++BI) {
4193 OperandBundleUse Bundle = Call.getOperandBundleAt(BI);
4194 if (Bundle.getTagID() == LLVMContext::OB_ptrauth)
4195 PtrAuthBundleOrNone = Bundle;
4196 else
4197 NewBundles.emplace_back(Bundle);
4198 }
4199
4200 if (!PtrAuthBundleOrNone)
4201 return nullptr;
4202
4203 Value *NewCallee = nullptr;
4204 switch (IIID) {
4205 // call(ptrauth.resign(p)), ["ptrauth"()] -> call p, ["ptrauth"()]
4206 // assuming the call bundle and the sign operands match.
4207 case Intrinsic::ptrauth_resign: {
4208 // Resign result key should match bundle.
4209 if (II->getOperand(3) != PtrAuthBundleOrNone->Inputs[0])
4210 return nullptr;
4211 // Resign result discriminator should match bundle.
4212 if (II->getOperand(4) != PtrAuthBundleOrNone->Inputs[1])
4213 return nullptr;
4214
4215 // Resign input (auth) key should also match: we can't change the key on
4216 // the new call we're generating, because we don't know what keys are valid.
4217 if (II->getOperand(1) != PtrAuthBundleOrNone->Inputs[0])
4218 return nullptr;
4219
4220 Value *NewBundleOps[] = {II->getOperand(1), II->getOperand(2)};
4221 NewBundles.emplace_back("ptrauth", NewBundleOps);
4222 NewCallee = II->getOperand(0);
4223 break;
4224 }
4225
4226 // call(ptrauth.sign(p)), ["ptrauth"()] -> call p
4227 // assuming the call bundle and the sign operands match.
4228 // Non-ptrauth indirect calls are undesirable, but so is ptrauth.sign.
4229 case Intrinsic::ptrauth_sign: {
4230 // Sign key should match bundle.
4231 if (II->getOperand(1) != PtrAuthBundleOrNone->Inputs[0])
4232 return nullptr;
4233 // Sign discriminator should match bundle.
4234 if (II->getOperand(2) != PtrAuthBundleOrNone->Inputs[1])
4235 return nullptr;
4236 NewCallee = II->getOperand(0);
4237 break;
4238 }
4239 default:
4240 llvm_unreachable("unexpected intrinsic ID");
4241 }
4242
4243 if (!NewCallee)
4244 return nullptr;
4245
4246 NewCallee = Builder.CreateBitOrPointerCast(NewCallee, Callee->getType());
4247 CallBase *NewCall = CallBase::Create(&Call, NewBundles);
4248 NewCall->setCalledOperand(NewCallee);
4249 return NewCall;
4250}
4251
4252Instruction *InstCombinerImpl::foldPtrAuthConstantCallee(CallBase &Call) {
4254 if (!CPA)
4255 return nullptr;
4256
4257 auto *CalleeF = dyn_cast<Function>(CPA->getPointer());
4258 // If the ptrauth constant isn't based on a function pointer, bail out.
4259 if (!CalleeF)
4260 return nullptr;
4261
4262 // Inspect the call ptrauth bundle to check it matches the ptrauth constant.
4264 if (!PAB)
4265 return nullptr;
4266
4267 auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
4268 Value *Discriminator = PAB->Inputs[1];
4269
4270 // If the bundle doesn't match, this is probably going to fail to auth.
4271 if (!CPA->isKnownCompatibleWith(Key, Discriminator, DL))
4272 return nullptr;
4273
4274 // If the bundle matches the constant, proceed in making this a direct call.
4276 NewCall->setCalledOperand(CalleeF);
4277 return NewCall;
4278}
4279
4280bool InstCombinerImpl::annotateAnyAllocSite(CallBase &Call,
4281 const TargetLibraryInfo *TLI) {
4282 // Note: We only handle cases which can't be driven from generic attributes
4283 // here. So, for example, nonnull and noalias (which are common properties
4284 // of some allocation functions) are expected to be handled via annotation
4285 // of the respective allocator declaration with generic attributes.
4286 bool Changed = false;
4287
4288 if (!Call.getType()->isPointerTy())
4289 return Changed;
4290
4291 std::optional<APInt> Size = getAllocSize(&Call, TLI);
4292 if (Size && *Size != 0) {
4293 // TODO: We really should just emit deref_or_null here and then
4294 // let the generic inference code combine that with nonnull.
4295 if (Call.hasRetAttr(Attribute::NonNull)) {
4296 Changed = !Call.hasRetAttr(Attribute::Dereferenceable);
4298 Call.getContext(), Size->getLimitedValue()));
4299 } else {
4300 Changed = !Call.hasRetAttr(Attribute::DereferenceableOrNull);
4302 Call.getContext(), Size->getLimitedValue()));
4303 }
4304 }
4305
4306 // Add alignment attribute if alignment is a power of two constant.
4307 Value *Alignment = getAllocAlignment(&Call, TLI);
4308 if (!Alignment)
4309 return Changed;
4310
4311 ConstantInt *AlignOpC = dyn_cast<ConstantInt>(Alignment);
4312 if (AlignOpC && AlignOpC->getValue().ult(llvm::Value::MaximumAlignment)) {
4313 uint64_t AlignmentVal = AlignOpC->getZExtValue();
4314 if (llvm::isPowerOf2_64(AlignmentVal)) {
4315 Align ExistingAlign = Call.getRetAlign().valueOrOne();
4316 Align NewAlign = Align(AlignmentVal);
4317 if (NewAlign > ExistingAlign) {
4320 Changed = true;
4321 }
4322 }
4323 }
4324 return Changed;
4325}
4326
4327/// Improvements for call, callbr and invoke instructions.
4328Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
4329 bool Changed = annotateAnyAllocSite(Call, &TLI);
4330
4331 // Mark any parameters that are known to be non-null with the nonnull
4332 // attribute. This is helpful for inlining calls to functions with null
4333 // checks on their arguments.
4334 SmallVector<unsigned, 4> ArgNos;
4335 unsigned ArgNo = 0;
4336
4337 for (Value *V : Call.args()) {
4338 if (V->getType()->isPointerTy()) {
4339 // Simplify the nonnull operand if the parameter is known to be nonnull.
4340 // Otherwise, try to infer nonnull for it.
4341 bool HasDereferenceable = Call.getParamDereferenceableBytes(ArgNo) > 0;
4342 if (Call.paramHasAttr(ArgNo, Attribute::NonNull) ||
4343 (HasDereferenceable &&
4345 V->getType()->getPointerAddressSpace()))) {
4346 if (Value *Res = simplifyNonNullOperand(V, HasDereferenceable)) {
4347 replaceOperand(Call, ArgNo, Res);
4348 Changed = true;
4349 }
4350 } else if (isKnownNonZero(V,
4351 getSimplifyQuery().getWithInstruction(&Call))) {
4352 ArgNos.push_back(ArgNo);
4353 }
4354 }
4355 ArgNo++;
4356 }
4357
4358 assert(ArgNo == Call.arg_size() && "Call arguments not processed correctly.");
4359
4360 if (!ArgNos.empty()) {
4361 AttributeList AS = Call.getAttributes();
4362 LLVMContext &Ctx = Call.getContext();
4363 AS = AS.addParamAttribute(Ctx, ArgNos,
4364 Attribute::get(Ctx, Attribute::NonNull));
4365 Call.setAttributes(AS);
4366 Changed = true;
4367 }
4368
4369 // If the callee is a pointer to a function, attempt to move any casts to the
4370 // arguments of the call/callbr/invoke.
4372 Function *CalleeF = dyn_cast<Function>(Callee);
4373 if ((!CalleeF || CalleeF->getFunctionType() != Call.getFunctionType()) &&
4374 transformConstExprCastCall(Call))
4375 return nullptr;
4376
4377 if (CalleeF) {
4378 // Remove the convergent attr on calls when the callee is not convergent.
4379 if (Call.isConvergent() && !CalleeF->isConvergent() &&
4380 !CalleeF->isIntrinsic()) {
4381 LLVM_DEBUG(dbgs() << "Removing convergent attr from instr " << Call
4382 << "\n");
4384 return &Call;
4385 }
4386
4387 // If the call and callee calling conventions don't match, and neither one
4388 // of the calling conventions is compatible with C calling convention
4389 // this call must be unreachable, as the call is undefined.
4390 if ((CalleeF->getCallingConv() != Call.getCallingConv() &&
4391 !(CalleeF->getCallingConv() == llvm::CallingConv::C &&
4395 // Only do this for calls to a function with a body. A prototype may
4396 // not actually end up matching the implementation's calling conv for a
4397 // variety of reasons (e.g. it may be written in assembly).
4398 !CalleeF->isDeclaration()) {
4399 Instruction *OldCall = &Call;
4401 // If OldCall does not return void then replaceInstUsesWith poison.
4402 // This allows ValueHandlers and custom metadata to adjust itself.
4403 if (!OldCall->getType()->isVoidTy())
4404 replaceInstUsesWith(*OldCall, PoisonValue::get(OldCall->getType()));
4405 if (isa<CallInst>(OldCall))
4406 return eraseInstFromFunction(*OldCall);
4407
4408 // We cannot remove an invoke or a callbr, because it would change thexi
4409 // CFG, just change the callee to a null pointer.
4410 cast<CallBase>(OldCall)->setCalledFunction(
4411 CalleeF->getFunctionType(),
4412 Constant::getNullValue(CalleeF->getType()));
4413 return nullptr;
4414 }
4415 }
4416
4417 // Calling a null function pointer is undefined if a null address isn't
4418 // dereferenceable.
4419 if ((isa<ConstantPointerNull>(Callee) &&
4421 isa<UndefValue>(Callee)) {
4422 // If Call does not return void then replaceInstUsesWith poison.
4423 // This allows ValueHandlers and custom metadata to adjust itself.
4424 if (!Call.getType()->isVoidTy())
4426
4427 if (Call.isTerminator()) {
4428 // Can't remove an invoke or callbr because we cannot change the CFG.
4429 return nullptr;
4430 }
4431
4432 // This instruction is not reachable, just remove it.
4435 }
4436
4437 if (IntrinsicInst *II = findInitTrampoline(Callee))
4438 return transformCallThroughTrampoline(Call, *II);
4439
4440 // Combine calls involving pointer authentication intrinsics.
4441 if (Instruction *NewCall = foldPtrAuthIntrinsicCallee(Call))
4442 return NewCall;
4443
4444 // Combine calls to ptrauth constants.
4445 if (Instruction *NewCall = foldPtrAuthConstantCallee(Call))
4446 return NewCall;
4447
4448 if (isa<InlineAsm>(Callee) && !Call.doesNotThrow()) {
4449 InlineAsm *IA = cast<InlineAsm>(Callee);
4450 if (!IA->canThrow()) {
4451 // Normal inline asm calls cannot throw - mark them
4452 // 'nounwind'.
4454 Changed = true;
4455 }
4456 }
4457
4458 // Try to optimize the call if possible, we require DataLayout for most of
4459 // this. None of these calls are seen as possibly dead so go ahead and
4460 // delete the instruction now.
4461 if (CallInst *CI = dyn_cast<CallInst>(&Call)) {
4462 Instruction *I = tryOptimizeCall(CI);
4463 // If we changed something return the result, etc. Otherwise let
4464 // the fallthrough check.
4465 if (I) return eraseInstFromFunction(*I);
4466 }
4467
4468 if (!Call.use_empty() && !Call.isMustTailCall())
4469 if (Value *ReturnedArg = Call.getReturnedArgOperand()) {
4470 Type *CallTy = Call.getType();
4471 Type *RetArgTy = ReturnedArg->getType();
4472 if (RetArgTy->canLosslesslyBitCastTo(CallTy))
4473 return replaceInstUsesWith(
4474 Call, Builder.CreateBitOrPointerCast(ReturnedArg, CallTy));
4475 }
4476
4477 // Drop unnecessary callee_type metadata from calls that were converted
4478 // into direct calls.
4479 if (Call.getMetadata(LLVMContext::MD_callee_type) && !Call.isIndirectCall()) {
4480 Call.setMetadata(LLVMContext::MD_callee_type, nullptr);
4481 Changed = true;
4482 }
4483
4484 // Drop unnecessary kcfi operand bundles from calls that were converted
4485 // into direct calls.
4487 if (Bundle && !Call.isIndirectCall()) {
4488 DEBUG_WITH_TYPE(DEBUG_TYPE "-kcfi", {
4489 if (CalleeF) {
4490 ConstantInt *FunctionType = nullptr;
4491 ConstantInt *ExpectedType = cast<ConstantInt>(Bundle->Inputs[0]);
4492
4493 if (MDNode *MD = CalleeF->getMetadata(LLVMContext::MD_kcfi_type))
4494 FunctionType = mdconst::extract<ConstantInt>(MD->getOperand(0));
4495
4496 if (FunctionType &&
4497 FunctionType->getZExtValue() != ExpectedType->getZExtValue())
4498 dbgs() << Call.getModule()->getName()
4499 << ": warning: kcfi: " << Call.getCaller()->getName()
4500 << ": call to " << CalleeF->getName()
4501 << " using a mismatching function pointer type\n";
4502 }
4503 });
4504
4506 }
4507
4508 if (isRemovableAlloc(&Call, &TLI))
4509 return visitAllocSite(Call);
4510
4511 // Handle intrinsics which can be used in both call and invoke context.
4512 switch (Call.getIntrinsicID()) {
4513 case Intrinsic::experimental_gc_statepoint: {
4514 GCStatepointInst &GCSP = *cast<GCStatepointInst>(&Call);
4515 SmallPtrSet<Value *, 32> LiveGcValues;
4516 for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {
4517 GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);
4518
4519 // Remove the relocation if unused.
4520 if (GCR.use_empty()) {
4522 continue;
4523 }
4524
4525 Value *DerivedPtr = GCR.getDerivedPtr();
4526 Value *BasePtr = GCR.getBasePtr();
4527
4528 // Undef is undef, even after relocation.
4529 if (isa<UndefValue>(DerivedPtr) || isa<UndefValue>(BasePtr)) {
4532 continue;
4533 }
4534
4535 if (auto *PT = dyn_cast<PointerType>(GCR.getType())) {
4536 // The relocation of null will be null for most any collector.
4537 // TODO: provide a hook for this in GCStrategy. There might be some
4538 // weird collector this property does not hold for.
4539 if (isa<ConstantPointerNull>(DerivedPtr)) {
4540 // Use null-pointer of gc_relocate's type to replace it.
4543 continue;
4544 }
4545
4546 // isKnownNonNull -> nonnull attribute
4547 if (!GCR.hasRetAttr(Attribute::NonNull) &&
4548 isKnownNonZero(DerivedPtr,
4549 getSimplifyQuery().getWithInstruction(&Call))) {
4550 GCR.addRetAttr(Attribute::NonNull);
4551 // We discovered new fact, re-check users.
4552 Worklist.pushUsersToWorkList(GCR);
4553 }
4554 }
4555
4556 // If we have two copies of the same pointer in the statepoint argument
4557 // list, canonicalize to one. This may let us common gc.relocates.
4558 if (GCR.getBasePtr() == GCR.getDerivedPtr() &&
4559 GCR.getBasePtrIndex() != GCR.getDerivedPtrIndex()) {
4560 auto *OpIntTy = GCR.getOperand(2)->getType();
4561 GCR.setOperand(2, ConstantInt::get(OpIntTy, GCR.getBasePtrIndex()));
4562 }
4563
4564 // TODO: bitcast(relocate(p)) -> relocate(bitcast(p))
4565 // Canonicalize on the type from the uses to the defs
4566
4567 // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...)
4568 LiveGcValues.insert(BasePtr);
4569 LiveGcValues.insert(DerivedPtr);
4570 }
4571 std::optional<OperandBundleUse> Bundle =
4573 unsigned NumOfGCLives = LiveGcValues.size();
4574 if (!Bundle || NumOfGCLives == Bundle->Inputs.size())
4575 break;
4576 // We can reduce the size of gc live bundle.
4577 DenseMap<Value *, unsigned> Val2Idx;
4578 std::vector<Value *> NewLiveGc;
4579 for (Value *V : Bundle->Inputs) {
4580 auto [It, Inserted] = Val2Idx.try_emplace(V);
4581 if (!Inserted)
4582 continue;
4583 if (LiveGcValues.count(V)) {
4584 It->second = NewLiveGc.size();
4585 NewLiveGc.push_back(V);
4586 } else
4587 It->second = NumOfGCLives;
4588 }
4589 // Update all gc.relocates
4590 for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {
4591 GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);
4592 Value *BasePtr = GCR.getBasePtr();
4593 assert(Val2Idx.count(BasePtr) && Val2Idx[BasePtr] != NumOfGCLives &&
4594 "Missed live gc for base pointer");
4595 auto *OpIntTy1 = GCR.getOperand(1)->getType();
4596 GCR.setOperand(1, ConstantInt::get(OpIntTy1, Val2Idx[BasePtr]));
4597 Value *DerivedPtr = GCR.getDerivedPtr();
4598 assert(Val2Idx.count(DerivedPtr) && Val2Idx[DerivedPtr] != NumOfGCLives &&
4599 "Missed live gc for derived pointer");
4600 auto *OpIntTy2 = GCR.getOperand(2)->getType();
4601 GCR.setOperand(2, ConstantInt::get(OpIntTy2, Val2Idx[DerivedPtr]));
4602 }
4603 // Create new statepoint instruction.
4604 OperandBundleDef NewBundle("gc-live", NewLiveGc);
4605 return CallBase::Create(&Call, NewBundle);
4606 }
4607 default: { break; }
4608 }
4609
4610 return Changed ? &Call : nullptr;
4611}
4612
4613/// If the callee is a constexpr cast of a function, attempt to move the cast to
4614/// the arguments of the call/invoke.
4615/// CallBrInst is not supported.
4616bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
4617 auto *Callee =
4619 if (!Callee)
4620 return false;
4621
4623 "CallBr's don't have a single point after a def to insert at");
4624
4625 // Don't perform the transform for declarations, which may not be fully
4626 // accurate. For example, void @foo() is commonly used as a placeholder for
4627 // unknown prototypes.
4628 if (Callee->isDeclaration())
4629 return false;
4630
4631 // If this is a call to a thunk function, don't remove the cast. Thunks are
4632 // used to transparently forward all incoming parameters and outgoing return
4633 // values, so it's important to leave the cast in place.
4634 if (Callee->hasFnAttribute("thunk"))
4635 return false;
4636
4637 // If this is a call to a naked function, the assembly might be
4638 // using an argument, or otherwise rely on the frame layout,
4639 // the function prototype will mismatch.
4640 if (Callee->hasFnAttribute(Attribute::Naked))
4641 return false;
4642
4643 // If this is a musttail call, the callee's prototype must match the caller's
4644 // prototype with the exception of pointee types. The code below doesn't
4645 // implement that, so we can't do this transform.
4646 // TODO: Do the transform if it only requires adding pointer casts.
4647 if (Call.isMustTailCall())
4648 return false;
4649
4651 const AttributeList &CallerPAL = Call.getAttributes();
4652
4653 // Okay, this is a cast from a function to a different type. Unless doing so
4654 // would cause a type conversion of one of our arguments, change this call to
4655 // be a direct call with arguments casted to the appropriate types.
4656 FunctionType *FT = Callee->getFunctionType();
4657 Type *OldRetTy = Caller->getType();
4658 Type *NewRetTy = FT->getReturnType();
4659
4660 // Check to see if we are changing the return type...
4661 if (OldRetTy != NewRetTy) {
4662
4663 if (NewRetTy->isStructTy())
4664 return false; // TODO: Handle multiple return values.
4665
4666 if (!CastInst::isBitOrNoopPointerCastable(NewRetTy, OldRetTy, DL)) {
4667 if (!Caller->use_empty())
4668 return false; // Cannot transform this return value.
4669 }
4670
4671 if (!CallerPAL.isEmpty() && !Caller->use_empty()) {
4672 AttrBuilder RAttrs(FT->getContext(), CallerPAL.getRetAttrs());
4673 if (RAttrs.overlaps(AttributeFuncs::typeIncompatible(
4674 NewRetTy, CallerPAL.getRetAttrs())))
4675 return false; // Attribute not compatible with transformed value.
4676 }
4677
4678 // If the callbase is an invoke instruction, and the return value is
4679 // used by a PHI node in a successor, we cannot change the return type of
4680 // the call because there is no place to put the cast instruction (without
4681 // breaking the critical edge). Bail out in this case.
4682 if (!Caller->use_empty()) {
4683 BasicBlock *PhisNotSupportedBlock = nullptr;
4684 if (auto *II = dyn_cast<InvokeInst>(Caller))
4685 PhisNotSupportedBlock = II->getNormalDest();
4686 if (PhisNotSupportedBlock)
4687 for (User *U : Caller->users())
4688 if (PHINode *PN = dyn_cast<PHINode>(U))
4689 if (PN->getParent() == PhisNotSupportedBlock)
4690 return false;
4691 }
4692 }
4693
4694 unsigned NumActualArgs = Call.arg_size();
4695 unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);
4696
4697 // Prevent us turning:
4698 // declare void @takes_i32_inalloca(i32* inalloca)
4699 // call void bitcast (void (i32*)* @takes_i32_inalloca to void (i32)*)(i32 0)
4700 //
4701 // into:
4702 // call void @takes_i32_inalloca(i32* null)
4703 //
4704 // Similarly, avoid folding away bitcasts of byval calls.
4705 if (Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) ||
4706 Callee->getAttributes().hasAttrSomewhere(Attribute::Preallocated))
4707 return false;
4708
4709 auto AI = Call.arg_begin();
4710 for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {
4711 Type *ParamTy = FT->getParamType(i);
4712 Type *ActTy = (*AI)->getType();
4713
4714 if (!CastInst::isBitOrNoopPointerCastable(ActTy, ParamTy, DL))
4715 return false; // Cannot transform this parameter value.
4716
4717 // Check if there are any incompatible attributes we cannot drop safely.
4718 if (AttrBuilder(FT->getContext(), CallerPAL.getParamAttrs(i))
4719 .overlaps(AttributeFuncs::typeIncompatible(
4720 ParamTy, CallerPAL.getParamAttrs(i),
4721 AttributeFuncs::ASK_UNSAFE_TO_DROP)))
4722 return false; // Attribute not compatible with transformed value.
4723
4724 if (Call.isInAllocaArgument(i) ||
4725 CallerPAL.hasParamAttr(i, Attribute::Preallocated))
4726 return false; // Cannot transform to and from inalloca/preallocated.
4727
4728 if (CallerPAL.hasParamAttr(i, Attribute::SwiftError))
4729 return false;
4730
4731 if (CallerPAL.hasParamAttr(i, Attribute::ByVal) !=
4732 Callee->getAttributes().hasParamAttr(i, Attribute::ByVal))
4733 return false; // Cannot transform to or from byval.
4734 }
4735
4736 if (FT->getNumParams() < NumActualArgs && FT->isVarArg() &&
4737 !CallerPAL.isEmpty()) {
4738 // In this case we have more arguments than the new function type, but we
4739 // won't be dropping them. Check that these extra arguments have attributes
4740 // that are compatible with being a vararg call argument.
4741 unsigned SRetIdx;
4742 if (CallerPAL.hasAttrSomewhere(Attribute::StructRet, &SRetIdx) &&
4743 SRetIdx - AttributeList::FirstArgIndex >= FT->getNumParams())
4744 return false;
4745 }
4746
4747 // Okay, we decided that this is a safe thing to do: go ahead and start
4748 // inserting cast instructions as necessary.
4749 SmallVector<Value *, 8> Args;
4751 Args.reserve(NumActualArgs);
4752 ArgAttrs.reserve(NumActualArgs);
4753
4754 // Get any return attributes.
4755 AttrBuilder RAttrs(FT->getContext(), CallerPAL.getRetAttrs());
4756
4757 // If the return value is not being used, the type may not be compatible
4758 // with the existing attributes. Wipe out any problematic attributes.
4759 RAttrs.remove(
4760 AttributeFuncs::typeIncompatible(NewRetTy, CallerPAL.getRetAttrs()));
4761
4762 LLVMContext &Ctx = Call.getContext();
4763 AI = Call.arg_begin();
4764 for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
4765 Type *ParamTy = FT->getParamType(i);
4766
4767 Value *NewArg = *AI;
4768 if ((*AI)->getType() != ParamTy)
4769 NewArg = Builder.CreateBitOrPointerCast(*AI, ParamTy);
4770 Args.push_back(NewArg);
4771
4772 // Add any parameter attributes except the ones incompatible with the new
4773 // type. Note that we made sure all incompatible ones are safe to drop.
4774 AttributeMask IncompatibleAttrs = AttributeFuncs::typeIncompatible(
4775 ParamTy, CallerPAL.getParamAttrs(i), AttributeFuncs::ASK_SAFE_TO_DROP);
4776 ArgAttrs.push_back(
4777 CallerPAL.getParamAttrs(i).removeAttributes(Ctx, IncompatibleAttrs));
4778 }
4779
4780 // If the function takes more arguments than the call was taking, add them
4781 // now.
4782 for (unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i) {
4783 Args.push_back(Constant::getNullValue(FT->getParamType(i)));
4784 ArgAttrs.push_back(AttributeSet());
4785 }
4786
4787 // If we are removing arguments to the function, emit an obnoxious warning.
4788 if (FT->getNumParams() < NumActualArgs) {
4789 // TODO: if (!FT->isVarArg()) this call may be unreachable. PR14722
4790 if (FT->isVarArg()) {
4791 // Add all of the arguments in their promoted form to the arg list.
4792 for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {
4793 Type *PTy = getPromotedType((*AI)->getType());
4794 Value *NewArg = *AI;
4795 if (PTy != (*AI)->getType()) {
4796 // Must promote to pass through va_arg area!
4797 Instruction::CastOps opcode =
4798 CastInst::getCastOpcode(*AI, false, PTy, false);
4799 NewArg = Builder.CreateCast(opcode, *AI, PTy);
4800 }
4801 Args.push_back(NewArg);
4802
4803 // Add any parameter attributes.
4804 ArgAttrs.push_back(CallerPAL.getParamAttrs(i));
4805 }
4806 }
4807 }
4808
4809 AttributeSet FnAttrs = CallerPAL.getFnAttrs();
4810
4811 if (NewRetTy->isVoidTy())
4812 Caller->setName(""); // Void type should not have a name.
4813
4814 assert((ArgAttrs.size() == FT->getNumParams() || FT->isVarArg()) &&
4815 "missing argument attributes");
4816 AttributeList NewCallerPAL = AttributeList::get(
4817 Ctx, FnAttrs, AttributeSet::get(Ctx, RAttrs), ArgAttrs);
4818
4820 Call.getOperandBundlesAsDefs(OpBundles);
4821
4822 CallBase *NewCall;
4823 if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
4824 NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(),
4825 II->getUnwindDest(), Args, OpBundles);
4826 } else {
4827 NewCall = Builder.CreateCall(Callee, Args, OpBundles);
4828 cast<CallInst>(NewCall)->setTailCallKind(
4829 cast<CallInst>(Caller)->getTailCallKind());
4830 }
4831 NewCall->takeName(Caller);
4833 NewCall->setAttributes(NewCallerPAL);
4834
4835 // Preserve prof metadata if any.
4836 NewCall->copyMetadata(*Caller, {LLVMContext::MD_prof});
4837
4838 // Insert a cast of the return type as necessary.
4839 Instruction *NC = NewCall;
4840 Value *NV = NC;
4841 if (OldRetTy != NV->getType() && !Caller->use_empty()) {
4842 assert(!NV->getType()->isVoidTy());
4844 NC->setDebugLoc(Caller->getDebugLoc());
4845
4846 auto OptInsertPt = NewCall->getInsertionPointAfterDef();
4847 assert(OptInsertPt && "No place to insert cast");
4848 InsertNewInstBefore(NC, *OptInsertPt);
4849 Worklist.pushUsersToWorkList(*Caller);
4850 }
4851
4852 if (!Caller->use_empty())
4853 replaceInstUsesWith(*Caller, NV);
4854 else if (Caller->hasValueHandle()) {
4855 if (OldRetTy == NV->getType())
4857 else
4858 // We cannot call ValueIsRAUWd with a different type, and the
4859 // actual tracked value will disappear.
4861 }
4862
4863 eraseInstFromFunction(*Caller);
4864 return true;
4865}
4866
4867/// Turn a call to a function created by init_trampoline / adjust_trampoline
4868/// intrinsic pair into a direct call to the underlying function.
4870InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call,
4871 IntrinsicInst &Tramp) {
4872 FunctionType *FTy = Call.getFunctionType();
4873 AttributeList Attrs = Call.getAttributes();
4874
4875 // If the call already has the 'nest' attribute somewhere then give up -
4876 // otherwise 'nest' would occur twice after splicing in the chain.
4877 if (Attrs.hasAttrSomewhere(Attribute::Nest))
4878 return nullptr;
4879
4881 FunctionType *NestFTy = NestF->getFunctionType();
4882
4883 AttributeList NestAttrs = NestF->getAttributes();
4884 if (!NestAttrs.isEmpty()) {
4885 unsigned NestArgNo = 0;
4886 Type *NestTy = nullptr;
4887 AttributeSet NestAttr;
4888
4889 // Look for a parameter marked with the 'nest' attribute.
4890 for (FunctionType::param_iterator I = NestFTy->param_begin(),
4891 E = NestFTy->param_end();
4892 I != E; ++NestArgNo, ++I) {
4893 AttributeSet AS = NestAttrs.getParamAttrs(NestArgNo);
4894 if (AS.hasAttribute(Attribute::Nest)) {
4895 // Record the parameter type and any other attributes.
4896 NestTy = *I;
4897 NestAttr = AS;
4898 break;
4899 }
4900 }
4901
4902 if (NestTy) {
4903 std::vector<Value*> NewArgs;
4904 std::vector<AttributeSet> NewArgAttrs;
4905 NewArgs.reserve(Call.arg_size() + 1);
4906 NewArgAttrs.reserve(Call.arg_size());
4907
4908 // Insert the nest argument into the call argument list, which may
4909 // mean appending it. Likewise for attributes.
4910
4911 {
4912 unsigned ArgNo = 0;
4913 auto I = Call.arg_begin(), E = Call.arg_end();
4914 do {
4915 if (ArgNo == NestArgNo) {
4916 // Add the chain argument and attributes.
4917 Value *NestVal = Tramp.getArgOperand(2);
4918 if (NestVal->getType() != NestTy)
4919 NestVal = Builder.CreateBitCast(NestVal, NestTy, "nest");
4920 NewArgs.push_back(NestVal);
4921 NewArgAttrs.push_back(NestAttr);
4922 }
4923
4924 if (I == E)
4925 break;
4926
4927 // Add the original argument and attributes.
4928 NewArgs.push_back(*I);
4929 NewArgAttrs.push_back(Attrs.getParamAttrs(ArgNo));
4930
4931 ++ArgNo;
4932 ++I;
4933 } while (true);
4934 }
4935
4936 // The trampoline may have been bitcast to a bogus type (FTy).
4937 // Handle this by synthesizing a new function type, equal to FTy
4938 // with the chain parameter inserted.
4939
4940 std::vector<Type*> NewTypes;
4941 NewTypes.reserve(FTy->getNumParams()+1);
4942
4943 // Insert the chain's type into the list of parameter types, which may
4944 // mean appending it.
4945 {
4946 unsigned ArgNo = 0;
4947 FunctionType::param_iterator I = FTy->param_begin(),
4948 E = FTy->param_end();
4949
4950 do {
4951 if (ArgNo == NestArgNo)
4952 // Add the chain's type.
4953 NewTypes.push_back(NestTy);
4954
4955 if (I == E)
4956 break;
4957
4958 // Add the original type.
4959 NewTypes.push_back(*I);
4960
4961 ++ArgNo;
4962 ++I;
4963 } while (true);
4964 }
4965
4966 // Replace the trampoline call with a direct call. Let the generic
4967 // code sort out any function type mismatches.
4968 FunctionType *NewFTy =
4969 FunctionType::get(FTy->getReturnType(), NewTypes, FTy->isVarArg());
4970 AttributeList NewPAL =
4971 AttributeList::get(FTy->getContext(), Attrs.getFnAttrs(),
4972 Attrs.getRetAttrs(), NewArgAttrs);
4973
4975 Call.getOperandBundlesAsDefs(OpBundles);
4976
4977 Instruction *NewCaller;
4978 if (InvokeInst *II = dyn_cast<InvokeInst>(&Call)) {
4979 NewCaller = InvokeInst::Create(NewFTy, NestF, II->getNormalDest(),
4980 II->getUnwindDest(), NewArgs, OpBundles);
4981 cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv());
4982 cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);
4983 } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&Call)) {
4984 NewCaller =
4985 CallBrInst::Create(NewFTy, NestF, CBI->getDefaultDest(),
4986 CBI->getIndirectDests(), NewArgs, OpBundles);
4987 cast<CallBrInst>(NewCaller)->setCallingConv(CBI->getCallingConv());
4988 cast<CallBrInst>(NewCaller)->setAttributes(NewPAL);
4989 } else {
4990 NewCaller = CallInst::Create(NewFTy, NestF, NewArgs, OpBundles);
4991 cast<CallInst>(NewCaller)->setTailCallKind(
4992 cast<CallInst>(Call).getTailCallKind());
4993 cast<CallInst>(NewCaller)->setCallingConv(
4994 cast<CallInst>(Call).getCallingConv());
4995 cast<CallInst>(NewCaller)->setAttributes(NewPAL);
4996 }
4997 NewCaller->setDebugLoc(Call.getDebugLoc());
4998
4999 return NewCaller;
5000 }
5001 }
5002
5003 // Replace the trampoline call with a direct call. Since there is no 'nest'
5004 // parameter, there is no need to adjust the argument list. Let the generic
5005 // code sort out any function type mismatches.
5006 Call.setCalledFunction(FTy, NestF);
5007 return &Call;
5008}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
BitTracker BT
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static SDValue foldBitOrderCrossLogicOp(SDNode *N, SelectionDAG &DAG)
#define DEBUG_TYPE
IRTranslator LLVM IR MI
static Type * getPromotedType(Type *Ty)
Return the specified type promoted as it would be to pass though a va_arg area.
static Instruction * createOverflowTuple(IntrinsicInst *II, Value *Result, Constant *Overflow)
Creates a result tuple for an overflow intrinsic II with a given Result and a constant Overflow value...
static IntrinsicInst * findInitTrampolineFromAlloca(Value *TrampMem)
static bool removeTriviallyEmptyRange(IntrinsicInst &EndI, InstCombinerImpl &IC, std::function< bool(const IntrinsicInst &)> IsStart)
static bool inputDenormalIsDAZ(const Function &F, const Type *Ty)
static Instruction * reassociateMinMaxWithConstantInOperand(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If this min/max has a matching min/max operand with a constant, try to push the constant operand into...
static bool isIdempotentBinaryIntrinsic(Intrinsic::ID IID)
Helper to match idempotent binary intrinsics, namely, intrinsics where f(f(x, y), y) == f(x,...
static bool signBitMustBeTheSame(Value *Op0, Value *Op1, const SimplifyQuery &SQ)
Return true if two values Op0 and Op1 are known to have the same sign.
static Instruction * moveAddAfterMinMax(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0.
static Instruction * simplifyInvariantGroupIntrinsic(IntrinsicInst &II, InstCombinerImpl &IC)
This function transforms launder.invariant.group and strip.invariant.group like: launder(launder(x)) ...
static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E, unsigned NumOperands)
static std::optional< bool > getKnownSign(Value *Op, const SimplifyQuery &SQ)
static cl::opt< unsigned > GuardWideningWindow("instcombine-guard-widening-window", cl::init(3), cl::desc("How wide an instruction window to bypass looking for " "another guard"))
static bool hasUndefSource(AnyMemTransferInst *MI)
Recognize a memcpy/memmove from a trivially otherwise unused alloca.
static Instruction * factorizeMinMaxTree(IntrinsicInst *II)
Reduce a sequence of min/max intrinsics with a common operand.
static Value * simplifyNeonTbl1(const IntrinsicInst &II, InstCombiner::BuilderTy &Builder)
Convert a table lookup to shufflevector if the mask is constant.
static Instruction * foldClampRangeOfTwo(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If we have a clamp pattern like max (min X, 42), 41 – where the output can only be one of two possibl...
static Value * simplifyReductionOperand(Value *Arg, bool CanReorderLanes)
static IntrinsicInst * findInitTrampolineFromBB(IntrinsicInst *AdjustTramp, Value *TrampMem)
static Value * foldIntrinsicUsingDistributiveLaws(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
static std::optional< bool > getKnownSignOrZero(Value *Op, const SimplifyQuery &SQ)
static Value * foldMinimumOverTrailingOrLeadingZeroCount(Value *I0, Value *I1, const DataLayout &DL, InstCombiner::BuilderTy &Builder)
Fold an unsigned minimum of trailing or leading zero bits counts: umin(cttz(CtOp, ZeroUndef),...
static Value * foldIdempotentBinaryIntrinsicRecurrence(InstCombinerImpl &IC, IntrinsicInst *II)
Attempt to simplify value-accumulating recurrences of kind: umax.acc = phi i8 [ umax,...
static Instruction * foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC)
static Instruction * foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC)
static IntrinsicInst * findInitTrampoline(Value *Callee)
static FCmpInst::Predicate fpclassTestIsFCmp0(FPClassTest Mask, const Function &F, Type *Ty)
static bool leftDistributesOverRight(Instruction::BinaryOps LOp, bool HasNUW, bool HasNSW, Intrinsic::ID ROp)
Return whether "X LOp (Y ROp Z)" is always equal to "(X LOp Y) ROp (X LOp Z)".
static Value * reassociateMinMaxWithConstants(IntrinsicInst *II, IRBuilderBase &Builder, const SimplifyQuery &SQ)
If this min/max has a constant operand and an operand that is a matching min/max with a constant oper...
static CallInst * canonicalizeConstantArg0ToArg1(CallInst &Call)
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
static bool hasNoSignedWrap(BinaryOperator &I)
static bool inputDenormalIsIEEE(DenormalMode Mode)
Return true if it's possible to assume IEEE treatment of input denormals in F for Val.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
static const Function * getCalledFunction(const Value *V)
This file contains the declarations for metadata subclasses.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod
const SmallVectorImpl< MachineOperand > & Cond
This file implements the SmallBitVector class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:114
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition Debug.h:72
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition VPlanSLP.cpp:247
Value * RHS
Value * LHS
bool isNegative() const
Definition APFloat.h:1449
void clearSign()
Definition APFloat.h:1298
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:234
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:229
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition APInt.h:1201
LLVM_ABI APInt usub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1948
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1182
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:380
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1666
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1488
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1111
LLVM_ABI APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1928
LLVM_ABI APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1935
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition APInt.cpp:651
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:219
LLVM_ABI APInt uadd_sat(const APInt &RHS) const
Definition APInt.cpp:2036
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:334
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:306
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:200
LLVM_ABI APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1941
static APSInt getMinValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the minimum integer value with the given bit width and signedness.
Definition APSInt.h:312
static APSInt getMaxValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the maximum integer value with the given bit width and signedness.
Definition APSInt.h:304
This class represents any memset intrinsic.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
static LLVM_ABI AttributeSet get(LLVMContext &C, const AttrBuilder &B)
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static LLVM_ABI Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
InstListType::reverse_iterator reverse_iterator
Definition BasicBlock.h:172
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI bool isSigned() const
Whether the intrinsic is signed or unsigned.
LLVM_ABI Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
static BinaryOperator * CreateFAddFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:236
static LLVM_ABI BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
static BinaryOperator * CreateNSW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition InstrTypes.h:279
static LLVM_ABI BinaryOperator * CreateNot(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition InstrTypes.h:294
static BinaryOperator * CreateFMulFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:244
static BinaryOperator * CreateFDivFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:248
static BinaryOperator * CreateFSubFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:240
static LLVM_ABI BinaryOperator * CreateNSWNeg(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
void setDoesNotThrow()
MaybeAlign getRetAlign() const
Extract the alignment of the return value.
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
OperandBundleUse getOperandBundleAt(unsigned Index) const
Return the operand bundle at a specific index.
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool isInAllocaArgument(unsigned ArgNo) const
Determine whether this argument is passed in an alloca.
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
unsigned getNumOperandBundles() const
Return the number of operand bundles associated with this User.
uint64_t getParamDereferenceableBytes(unsigned i) const
Extract the number of dereferenceable bytes for a call or parameter (0=unknown).
CallingConv::ID getCallingConv() const
LLVM_ABI bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
void setNotConvergent()
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
bool doesNotThrow() const
Determine if the call cannot unwind.
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Value * getReturnedArgOperand() const
If one of the arguments has the 'returned' attribute, returns its operand value.
static LLVM_ABI CallBase * Create(CallBase *CB, ArrayRef< OperandBundleDef > Bundles, InsertPosition InsertPt=nullptr)
Create a clone of CB with a different set of operand bundles and insert it before InsertPt.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
static LLVM_ABI CallBase * removeOperandBundle(CallBase *CB, uint32_t ID, InsertPosition InsertPt=nullptr)
Create a clone of CB with operand bundle ID removed.
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
static CallBrInst * Create(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, ArrayRef< BasicBlock * > IndirectDests, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
This class represents a function call, abstracting a target machine's calling convention.
bool isNoTailCall() const
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
bool isMustTailCall() const
static LLVM_ABI Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
static LLVM_ABI CastInst * CreateIntegerCast(Value *S, Type *Ty, bool isSigned, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a ZExt, BitCast, or Trunc for int -> int casts.
static LLVM_ABI bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
static LLVM_ABI CastInst * CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:678
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:681
@ ICMP_SLT
signed less than
Definition InstrTypes.h:707
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:708
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:684
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:682
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:683
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:701
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:705
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:686
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:689
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:703
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:685
@ ICMP_NE
not equal
Definition InstrTypes.h:700
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:694
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:829
Predicate getNonStrictPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
Definition InstrTypes.h:873
Predicate getUnorderedPredicate() const
Definition InstrTypes.h:813
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
static LLVM_ABI Constant * getInfinity(Type *Ty, bool Negative=false)
static LLVM_ABI Constant * getZero(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition Constants.h:264
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:163
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:154
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI ConstantPtrAuth * get(Constant *Ptr, ConstantInt *Key, ConstantInt *Disc, Constant *AddrDisc)
Return a pointer signed with the specified parameters.
This class represents a range of values.
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI bool icmp(CmpInst::Predicate Pred, const ConstantRange &Other) const
Does the predicate Pred hold between ranges this and Other?
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
Definition Constant.h:43
static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
Record of a variable value-assignment, aka a non instruction representation of the dbg....
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:237
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:163
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
static FMFSource intersect(Value *A, Value *B)
Intersect the FMF from two instructions.
Definition IRBuilder.h:107
This class represents an extension of floating point types.
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
void setNoSignedZeros(bool B=true)
Definition FMF.h:84
bool allowReassoc() const
Flag queries.
Definition FMF.h:64
An instruction for ordering other memory operations.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Class to represent function types.
Type::subtype_iterator param_iterator
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
bool isConvergent() const
Determine if the call is convergent.
Definition Function.h:610
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:270
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:352
bool doesNotThrow() const
Determine if the function cannot unwind.
Definition Function.h:594
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition Function.h:249
LLVM_ABI Value * getBasePtr() const
unsigned getBasePtrIndex() const
The index into the associate statepoint's argument list which contains the base pointer of the pointe...
LLVM_ABI Value * getDerivedPtr() const
unsigned getDerivedPtrIndex() const
The index into the associate statepoint's argument list which contains the pointer whose relocation t...
std::vector< const GCRelocateInst * > getGCRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition Statepoint.h:206
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
Definition Value.h:576
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition Globals.cpp:328
PointerType * getType() const
Global values are always pointers.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
LLVM_ABI Value * CreateLaunderInvariantGroup(Value *Ptr)
Create a launder.invariant.group intrinsic call.
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition IRBuilder.h:502
LLVM_ABI Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1420
LLVM_ABI CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition IRBuilder.h:2082
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition IRBuilder.h:507
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2439
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2209
LLVM_ABI Value * CreateStripInvariantGroup(Value *Ptr)
Create a strip.invariant.group intrinsic call.
static InsertValueInst * Create(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
KnownFPClass computeKnownFPClass(Value *Val, FastMathFlags FMF, FPClassTest Interested=fcAllFlags, const Instruction *CtxI=nullptr, unsigned Depth=0) const
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &PoisonElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
bool SimplifyDemandedBits(Instruction *I, unsigned Op, const APInt &DemandedMask, KnownBits &Known, const SimplifyQuery &Q, unsigned Depth=0) override
This form of SimplifyDemandedBits simplifies the specified instruction operand if possible,...
Instruction * SimplifyAnyMemSet(AnyMemSetInst *MI)
Instruction * visitFree(CallInst &FI, Value *FreedOp)
Instruction * visitCallBrInst(CallBrInst &CBI)
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Value * foldReversedIntrinsicOperands(IntrinsicInst *II)
If all arguments of the intrinsic are reverses, try to pull the reverse after the intrinsic.
Value * tryGetLog2(Value *Op, bool AssumeNonZero)
Instruction * visitFenceInst(FenceInst &FI)
Instruction * foldShuffledIntrinsicOperands(IntrinsicInst *II)
If all arguments of the intrinsic are unary shuffles with the same mask, try to shuffle after the int...
Instruction * visitInvokeInst(InvokeInst &II)
bool SimplifyDemandedInstructionBits(Instruction &Inst)
Tries to simplify operands to an integer instruction based on its demanded bits.
void CreateNonTerminatorUnreachable(Instruction *InsertAt)
Create and insert the idiom we use to indicate a block is unreachable without having to rewrite the C...
Instruction * visitVAEndInst(VAEndInst &I)
Instruction * matchBSwapOrBitReverse(Instruction &I, bool MatchBSwaps, bool MatchBitReversals)
Given an initial instruction, check to see if it is the root of a bswap/bitreverse idiom.
Constant * unshuffleConstant(ArrayRef< int > ShMask, Constant *C, VectorType *NewCTy)
Find a constant NewC that has property: shuffle(NewC, ShMask) = C Returns nullptr if such a constant ...
Instruction * visitAllocSite(Instruction &FI)
Instruction * SimplifyAnyMemTransfer(AnyMemTransferInst *MI)
OverflowResult computeOverflow(Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, Instruction *CxtI) const
Instruction * visitCallInst(CallInst &CI)
CallInst simplification.
SimplifyQuery SQ
unsigned ComputeMaxSignificantBits(const Value *Op, const Instruction *CxtI=nullptr, unsigned Depth=0) const
IRBuilder< TargetFolder, IRBuilderCallbackInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
DominatorTree & getDominatorTree() const
BlockFrequencyInfo * BFI
TargetLibraryInfo & TLI
Instruction * InsertNewInstBefore(Instruction *New, BasicBlock::iterator Old)
Inserts an instruction New before instruction Old.
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
void replaceUse(Use &U, Value *NewValue)
Replace use and add the previously used value to the worklist.
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
const DataLayout & DL
DomConditionCache DC
void computeKnownBits(const Value *V, KnownBits &Known, const Instruction *CxtI, unsigned Depth=0) const
std::optional< Instruction * > targetInstCombineIntrinsic(IntrinsicInst &II)
AssumptionCache & AC
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const Instruction *CxtI=nullptr, unsigned Depth=0) const
DominatorTree & DT
ProfileSummaryInfo * PSI
BuilderTy & Builder
AssumptionCache & getAssumptionCache() const
OptimizationRemarkEmitter & ORE
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
const SimplifyQuery & getSimplifyQuery() const
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, const Instruction *CxtI=nullptr, unsigned Depth=0)
LLVM_ABI void setHasNoUnsignedWrap(bool b=true)
Set or clear the nuw flag on this instruction, which must be an operator which supports this flag.
LLVM_ABI bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
LLVM_ABI void setHasNoSignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
bool isTerminator() const
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI std::optional< InstListType::iterator > getInsertionPointAfterDef()
Get the first insertion point at which the result of this instruction is defined.
LLVM_ABI bool isIdenticalTo(const Instruction *I) const LLVM_READONLY
Return true if the specified instruction is exactly identical to the current one.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Invoke instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
An instruction for reading from memory.
Metadata node.
Definition Metadata.h:1078
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1569
static ICmpInst::Predicate getPredicate(Intrinsic::ID ID)
Returns the comparison predicate underlying the intrinsic.
ICmpInst::Predicate getPredicate() const
Returns the comparison predicate underlying the intrinsic.
bool isSigned() const
Whether the intrinsic is signed or unsigned.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
StringRef getName() const
Get a short "name" for the module.
Definition Module.h:269
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition Operator.h:43
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition Operator.h:78
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
Definition Operator.h:111
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
Definition Operator.h:105
bool isCommutative() const
Return true if the instruction is commutative.
Definition Operator.h:128
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Represents a saturating add/sub intrinsic.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, Instruction *MDFrom=nullptr)
This instruction constructs a fixed permutation of two input vectors.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
SmallBitVector & set()
bool test(unsigned Idx) const
bool all() const
Returns true if all bits are set.
size_type size() const
Definition SmallPtrSet.h:99
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
void setVolatile(bool V)
Specify whether this is a volatile store or not.
void setAlignment(Align Align)
void setOrdering(AtomicOrdering Ordering)
Sets the ordering constraint of this store instruction.
Class to represent struct types.
static LLVM_ABI bool isCallingConvCCompatible(CallBase *CI)
Returns true if call site / callee has cdecl-compatible calling conventions.
Provides information about what library functions are available for the current target.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:298
LLVM_ABI unsigned getIntegerBitWidth() const
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:297
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:267
LLVM_ABI bool canLosslesslyBitCastTo(Type *Ty) const
Return true if this type could be converted with a lossless BitCast to type 'Ty'.
Definition Type.cpp:154
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:261
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:139
static UnaryOperator * CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Definition InstrTypes.h:139
static UnaryOperator * CreateFNegFMF(Value *Op, Instruction *FMFSource, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Definition InstrTypes.h:147
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
Definition Use.cpp:35
void setOperand(unsigned i, Value *Val)
Definition User.h:237
Value * getOperand(unsigned i) const
Definition User.h:232
This represents the llvm.va_end intrinsic.
static LLVM_ABI void ValueIsDeleted(Value *V)
Definition Value.cpp:1226
static LLVM_ABI void ValueIsRAUWd(Value *Old, Value *New)
Definition Value.cpp:1279
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
static constexpr uint64_t MaximumAlignment
Definition Value.h:830
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
iterator_range< user_iterator > users()
Definition Value.h:426
static LLVM_ABI void dropDroppableUse(Use &U)
Remove the droppable use U.
Definition Value.cpp:226
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:701
bool use_empty() const
Definition Value.h:346
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
static constexpr unsigned MaxAlignmentExponent
The maximum alignment for instructions.
Definition Value.h:829
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:396
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:217
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:224
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
CallInst * Call
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
class_match< PoisonValue > m_Poison()
Match an arbitrary poison constant.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
m_Intrinsic_Ty< Opnd0 >::Ty m_BitReverse(const Opnd0 &Op0)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWSub(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
OverflowingBinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWNeg(const ValTy &V)
Matches a 'Neg' as 'sub nsw 0, V'.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
IntrinsicID_match m_Intrinsic()
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
cstfp_pred_ty< is_neg_zero_fp > m_NegZeroFP()
Match a floating-point negative zero.
specific_fpval m_SpecificFP(double V)
Match a specific floating point value or vector with all elements equal to the value.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
match_combine_or< match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > >, OpTy > m_ZExtOrSExtOrSelf(const OpTy &Op)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
cst_pred_ty< is_strictlypositive > m_StrictlyPositive()
Match an integer or vector of strictly positive values.
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWShl(const LHS &L, const RHS &R)
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWMul(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
cst_pred_ty< is_negated_power2 > m_NegatedPower2()
Match a integer or vector negated power-of-2.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
cst_pred_ty< custom_checkfn< APInt > > m_CheckedInt(function_ref< bool(const APInt &)> CheckFn)
Match an integer or vector where CheckFn(ele) for each element is true.
m_Intrinsic_Ty< Opnd0, Opnd1, Opnd2 >::Ty m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2)
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty, true >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty, true > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty, true >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty, true > > > m_c_MaxOrMin(const LHS &L, const RHS &R)
class_match< UnaryOperator > m_UnOp()
Match an arbitrary unary operation and ignore it.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Sub, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWSub(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap >, DisjointOr_match< LHS, RHS > > m_NSWAddLike(const LHS &L, const RHS &R)
Match either "add nsw" or "or disjoint".
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
Exact_match< T > m_Exact(const T &SubPattern)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
cstfp_pred_ty< is_pos_zero_fp > m_PosZeroFP()
Match a floating-point positive zero.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
m_Intrinsic_Ty< Opnd0, Opnd1, Opnd2 >::Ty m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2)
BinaryOp_match< LHS, RHS, Instruction::SRem > m_SRem(const LHS &L, const RHS &R)
auto m_Undef()
Match an arbitrary undef constant.
m_Intrinsic_Ty< Opnd0 >::Ty m_BSwap(const Opnd0 &Op0)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap >, DisjointOr_match< LHS, RHS > > m_NUWAddLike(const LHS &L, const RHS &R)
Match either "add nuw" or "or disjoint".
BinOpPred_match< LHS, RHS, is_bitwiselogic_op > m_BitwiseLogic(const LHS &L, const RHS &R)
Matches bitwise logic operations.
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
m_Intrinsic_Ty< Opnd0, Opnd1 >::Ty m_CopySign(const Opnd0 &Op0, const Opnd1 &Op1)
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
MatchFunctor< Val, Pattern > match_fn(const Pattern &P)
A match functor that can be used as a UnaryPredicate in functional algorithms like all_of.
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition LLVMContext.h:55
@ System
Synchronized with respect to all concurrently executing threads.
Definition LLVMContext.h:58
SmallVector< DbgVariableRecord * > getDVRAssignmentMarkers(const Instruction *Inst)
Return a range of dbg_assign records for which Inst performs the assignment they encode.
Definition DebugInfo.h:201
initializer< Ty > init(const Ty &Val)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:667
constexpr double e
Definition MathExtras.h:47
DiagnosticInfoOptimizationBase::Argument NV
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI cl::opt< bool > EnableKnowledgeRetention
LLVM_ABI Intrinsic::ID getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:355
@ Offset
Definition DWP.cpp:477
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
LLVM_ABI Value * simplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior=fp::ebIgnore, RoundingMode Rounding=RoundingMode::NearestTiesToEven)
Given operands for an FMul, fold the result or return null.
LLVM_ABI bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, const DominatorTree *DT=nullptr, bool AllowEphemerals=false)
Return true if it is valid to use the assumptions provided by an assume intrinsic,...
LLVM_ABI APInt possiblyDemandedEltsInMask(Value *Mask)
Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y) for each lane which may be ...
LLVM_ABI RetainedKnowledge simplifyRetainedKnowledge(AssumeInst *Assume, RetainedKnowledge RK, AssumptionCache *AC, DominatorTree *DT)
canonicalize the RetainedKnowledge RK.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:644
LLVM_ABI bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
LLVM_ABI Value * lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed)
Try to turn a call to @llvm.objectsize into an integer value of the given Type.
LLVM_ABI Value * getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI)
Gets the alignment argument for an aligned_alloc-like function, using either built-in knowledge based...
LLVM_ABI RetainedKnowledge getKnowledgeFromOperandInAssume(AssumeInst &Assume, unsigned Idx)
Retreive the information help by Assume on the operand at index Idx.
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition APFloat.h:1643
LLVM_ABI Value * simplifyCall(CallBase *Call, Value *Callee, ArrayRef< Value * > Args, const SimplifyQuery &Q)
Given a callsite, callee, and arguments, fold the result or return null.
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:557
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:293
LLVM_ABI bool isAssumeWithEmptyBundle(const AssumeInst &Assume)
Return true iff the operand bundles of the provided llvm.assume doesn't contain any valuable informat...
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
constexpr T MinAlign(U A, V B)
A and B are either alignments or offsets.
Definition MathExtras.h:368
LLVM_ABI RetainedKnowledge getKnowledgeFromBundle(AssumeInst &Assume, const CallBase::BundleOpInfo &BOI)
This extracts the Knowledge from an element of an operand bundle.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:754
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition Local.h:252
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
Definition APFloat.h:1598
LLVM_ABI FPClassTest fneg(FPClassTest Mask)
Return the test mask which returns true if the value's sign bit is flipped.
SelectPatternFlavor
Specific patterns of select instructions we can match.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
LLVM_ABI Constant * getLosslessUnsignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:288
bool isModSet(const ModRefInfo MRI)
Definition ModRef.h:49
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
LLVM_ABI bool matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I, PHINode *&P, Value *&Init, Value *&OtherOp)
Attempt to match a simple value-accumulating recurrence of the form: llvm.intrinsic....
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
auto find_if_not(R &&Range, UnaryPredicate P)
Definition STLExtras.h:1743
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1719
bool isAtLeastOrStrongerThan(AtomicOrdering AO, AtomicOrdering Other)
LLVM_ABI Constant * getLosslessSignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
LLVM_ABI AssumeInst * buildAssumeFromKnowledge(ArrayRef< RetainedKnowledge > Knowledge, Instruction *CtxI, AssumptionCache *AC=nullptr, DominatorTree *DT=nullptr)
Build and return a new assume created from the provided knowledge if the knowledge in the assume is f...
LLVM_ABI FPClassTest inverse_fabs(FPClassTest Mask)
Return the test mask which returns true after fabs is applied to the value.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
LLVM_ABI bool isNotCrossLaneOperation(const Instruction *I)
Return true if the instruction doesn't potentially cross vector lanes.
LLVM_ABI bool maskIsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr int PoisonMaskElem
@ Mod
The access may modify the value stored in memory.
Definition ModRef.h:34
LLVM_ABI Value * simplifyFMAFMul(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior=fp::ebIgnore, RoundingMode Rounding=RoundingMode::NearestTiesToEven)
Given operands for the multiplication of a FMA, fold the result or return null.
@ Other
Any other memory.
Definition ModRef.h:68
FunctionAddr VTableAddr uintptr_t uintptr_t Data
Definition InstrProf.h:189
LLVM_ABI Value * simplifyConstrainedFPCall(CallBase *Call, const SimplifyQuery &Q)
Given a constrained FP intrinsic call, tries to compute its simplified version.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 minNum semantics.
Definition APFloat.h:1579
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
@ Add
Sum of integers.
LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI ConstantRange computeConstantRangeIncludingKnownBits(const WithCache< const Value * > &V, bool ForSigned, const SimplifyQuery &SQ)
Combine constant ranges from computeConstantRange() and computeKnownBits().
FunctionAddr VTableAddr Next
Definition InstrProf.h:141
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I, bool IgnoreUBImplyingAttrs=true)
Don't use information from its non-constant operands.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
constexpr unsigned BitWidth
LLVM_ABI bool isDereferenceablePointer(const Value *V, Type *Ty, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if this is always a dereferenceable pointer.
Definition Loads.cpp:249
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:560
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1877
LLVM_ABI std::optional< APInt > getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, function_ref< const Value *(const Value *)> Mapper=[](const Value *V) { return V;})
Return the size of the requested allocation.
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:197
LLVM_ABI bool maskContainsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if any of the elements of this predicate mask are known to be ...
LLVM_ABI std::optional< bool > isImpliedByDomCondition(const Value *Cond, const Instruction *ContextI, const DataLayout &DL)
Return the boolean condition value in the context of the given instruction if it is known based on do...
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
Definition APFloat.h:1616
LLVM_ABI bool isKnownNegation(const Value *X, const Value *Y, bool NeedNSW=false, bool AllowPoison=true)
Return true if the two given values are negation.
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
LLVM_ABI bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
LLVM_ABI bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
LLVM_ABI std::optional< bool > computeKnownFPSignBit(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return false if we can prove that the specified FP value's sign bit is 0.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:872
#define NC
Definition regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition Metadata.h:761
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
@ IEEE
IEEE-754 denormal numbers preserved.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition KnownBits.h:108
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition KnownBits.h:242
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition KnownBits.h:274
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition KnownBits.h:289
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
bool isNonZero() const
Returns true if this value is known to be non-zero.
Definition KnownBits.h:111
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:248
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:105
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
Definition KnownBits.h:280
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition KnownBits.h:286
bool isAllOnes() const
Returns true if value is all one bits.
Definition KnownBits.h:83
FPClassTest KnownFPClasses
Floating-point classes the value could be one of.
Matching combinators.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition Alignment.h:130
A lightweight accessor for an operand bundle meant to be passed around by value.
StringRef getTagName() const
Return the tag of this operand bundle as a string.
uint32_t getTagID() const
Return the tag of this operand bundle as an integer.
ArrayRef< Use > Inputs
Represent one information held inside an operand bundle of an llvm.assume.
Attribute::AttrKind AttrKind
SelectPatternFlavor Flavor
const DataLayout & DL
const Instruction * CxtI
SimplifyQuery getWithInstruction(const Instruction *I) const