LLVM 22.0.0git
InstCombineCalls.cpp
Go to the documentation of this file.
1//===- InstCombineCalls.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitCall, visitInvoke, and visitCallBr functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/Statistic.h"
26#include "llvm/Analysis/Loads.h"
31#include "llvm/IR/Attributes.h"
32#include "llvm/IR/BasicBlock.h"
33#include "llvm/IR/Constant.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/DebugInfo.h"
38#include "llvm/IR/Function.h"
40#include "llvm/IR/InlineAsm.h"
41#include "llvm/IR/InstrTypes.h"
42#include "llvm/IR/Instruction.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsAArch64.h"
47#include "llvm/IR/IntrinsicsAMDGPU.h"
48#include "llvm/IR/IntrinsicsARM.h"
49#include "llvm/IR/IntrinsicsHexagon.h"
50#include "llvm/IR/LLVMContext.h"
51#include "llvm/IR/Metadata.h"
53#include "llvm/IR/Statepoint.h"
54#include "llvm/IR/Type.h"
55#include "llvm/IR/User.h"
56#include "llvm/IR/Value.h"
57#include "llvm/IR/ValueHandle.h"
62#include "llvm/Support/Debug.h"
73#include <algorithm>
74#include <cassert>
75#include <cstdint>
76#include <optional>
77#include <utility>
78#include <vector>
79
80#define DEBUG_TYPE "instcombine"
82
83using namespace llvm;
84using namespace PatternMatch;
85
86STATISTIC(NumSimplified, "Number of library calls simplified");
87
89 "instcombine-guard-widening-window",
90 cl::init(3),
91 cl::desc("How wide an instruction window to bypass looking for "
92 "another guard"));
93
94/// Return the specified type promoted as it would be to pass though a va_arg
95/// area.
97 if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
98 if (ITy->getBitWidth() < 32)
99 return Type::getInt32Ty(Ty->getContext());
100 }
101 return Ty;
102}
103
104/// Recognize a memcpy/memmove from a trivially otherwise unused alloca.
105/// TODO: This should probably be integrated with visitAllocSites, but that
106/// requires a deeper change to allow either unread or unwritten objects.
108 auto *Src = MI->getRawSource();
109 while (isa<GetElementPtrInst>(Src)) {
110 if (!Src->hasOneUse())
111 return false;
112 Src = cast<Instruction>(Src)->getOperand(0);
113 }
114 return isa<AllocaInst>(Src) && Src->hasOneUse();
115}
116
118 Align DstAlign = getKnownAlignment(MI->getRawDest(), DL, MI, &AC, &DT);
119 MaybeAlign CopyDstAlign = MI->getDestAlign();
120 if (!CopyDstAlign || *CopyDstAlign < DstAlign) {
121 MI->setDestAlignment(DstAlign);
122 return MI;
123 }
124
125 Align SrcAlign = getKnownAlignment(MI->getRawSource(), DL, MI, &AC, &DT);
126 MaybeAlign CopySrcAlign = MI->getSourceAlign();
127 if (!CopySrcAlign || *CopySrcAlign < SrcAlign) {
128 MI->setSourceAlignment(SrcAlign);
129 return MI;
130 }
131
132 // If we have a store to a location which is known constant, we can conclude
133 // that the store must be storing the constant value (else the memory
134 // wouldn't be constant), and this must be a noop.
135 if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {
136 // Set the size of the copy to 0, it will be deleted on the next iteration.
137 MI->setLength((uint64_t)0);
138 return MI;
139 }
140
141 // If the source is provably undef, the memcpy/memmove doesn't do anything
142 // (unless the transfer is volatile).
143 if (hasUndefSource(MI) && !MI->isVolatile()) {
144 // Set the size of the copy to 0, it will be deleted on the next iteration.
145 MI->setLength((uint64_t)0);
146 return MI;
147 }
148
149 // If MemCpyInst length is 1/2/4/8 bytes then replace memcpy with
150 // load/store.
151 ConstantInt *MemOpLength = dyn_cast<ConstantInt>(MI->getLength());
152 if (!MemOpLength) return nullptr;
153
154 // Source and destination pointer types are always "i8*" for intrinsic. See
155 // if the size is something we can handle with a single primitive load/store.
156 // A single load+store correctly handles overlapping memory in the memmove
157 // case.
158 uint64_t Size = MemOpLength->getLimitedValue();
159 assert(Size && "0-sized memory transferring should be removed already.");
160
161 if (Size > 8 || (Size&(Size-1)))
162 return nullptr; // If not 1/2/4/8 bytes, exit.
163
164 // If it is an atomic and alignment is less than the size then we will
165 // introduce the unaligned memory access which will be later transformed
166 // into libcall in CodeGen. This is not evident performance gain so disable
167 // it now.
168 if (MI->isAtomic())
169 if (*CopyDstAlign < Size || *CopySrcAlign < Size)
170 return nullptr;
171
172 // Use an integer load+store unless we can find something better.
173 IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
174
175 // If the memcpy has metadata describing the members, see if we can get the
176 // TBAA, scope and noalias tags describing our copy.
177 AAMDNodes AACopyMD = MI->getAAMetadata().adjustForAccess(Size);
178
179 Value *Src = MI->getArgOperand(1);
180 Value *Dest = MI->getArgOperand(0);
181 LoadInst *L = Builder.CreateLoad(IntType, Src);
182 // Alignment from the mem intrinsic will be better, so use it.
183 L->setAlignment(*CopySrcAlign);
184 L->setAAMetadata(AACopyMD);
185 MDNode *LoopMemParallelMD =
186 MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
187 if (LoopMemParallelMD)
188 L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
189 MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);
190 if (AccessGroupMD)
191 L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
192
193 StoreInst *S = Builder.CreateStore(L, Dest);
194 // Alignment from the mem intrinsic will be better, so use it.
195 S->setAlignment(*CopyDstAlign);
196 S->setAAMetadata(AACopyMD);
197 if (LoopMemParallelMD)
198 S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
199 if (AccessGroupMD)
200 S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
201 S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
202
203 if (auto *MT = dyn_cast<MemTransferInst>(MI)) {
204 // non-atomics can be volatile
205 L->setVolatile(MT->isVolatile());
206 S->setVolatile(MT->isVolatile());
207 }
208 if (MI->isAtomic()) {
209 // atomics have to be unordered
210 L->setOrdering(AtomicOrdering::Unordered);
212 }
213
214 // Set the size of the copy to 0, it will be deleted on the next iteration.
215 MI->setLength((uint64_t)0);
216 return MI;
217}
218
220 const Align KnownAlignment =
221 getKnownAlignment(MI->getDest(), DL, MI, &AC, &DT);
222 MaybeAlign MemSetAlign = MI->getDestAlign();
223 if (!MemSetAlign || *MemSetAlign < KnownAlignment) {
224 MI->setDestAlignment(KnownAlignment);
225 return MI;
226 }
227
228 // If we have a store to a location which is known constant, we can conclude
229 // that the store must be storing the constant value (else the memory
230 // wouldn't be constant), and this must be a noop.
231 if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {
232 // Set the size of the copy to 0, it will be deleted on the next iteration.
233 MI->setLength((uint64_t)0);
234 return MI;
235 }
236
237 // Remove memset with an undef value.
238 // FIXME: This is technically incorrect because it might overwrite a poison
239 // value. Change to PoisonValue once #52930 is resolved.
240 if (isa<UndefValue>(MI->getValue())) {
241 // Set the size of the copy to 0, it will be deleted on the next iteration.
242 MI->setLength((uint64_t)0);
243 return MI;
244 }
245
246 // Extract the length and alignment and fill if they are constant.
247 ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());
248 ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue());
249 if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8))
250 return nullptr;
251 const uint64_t Len = LenC->getLimitedValue();
252 assert(Len && "0-sized memory setting should be removed already.");
253 const Align Alignment = MI->getDestAlign().valueOrOne();
254
255 // If it is an atomic and alignment is less than the size then we will
256 // introduce the unaligned memory access which will be later transformed
257 // into libcall in CodeGen. This is not evident performance gain so disable
258 // it now.
259 if (MI->isAtomic() && Alignment < Len)
260 return nullptr;
261
262 // memset(s,c,n) -> store s, c (for n=1,2,4,8)
263 if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
264 Value *Dest = MI->getDest();
265
266 // Extract the fill value and store.
267 Constant *FillVal = ConstantInt::get(
268 MI->getContext(), APInt::getSplat(Len * 8, FillC->getValue()));
269 StoreInst *S = Builder.CreateStore(FillVal, Dest, MI->isVolatile());
270 S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
271 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(S)) {
272 if (llvm::is_contained(DbgAssign->location_ops(), FillC))
273 DbgAssign->replaceVariableLocationOp(FillC, FillVal);
274 }
275
276 S->setAlignment(Alignment);
277 if (MI->isAtomic())
279
280 // Set the size of the copy to 0, it will be deleted on the next iteration.
281 MI->setLength((uint64_t)0);
282 return MI;
283 }
284
285 return nullptr;
286}
287
288// TODO, Obvious Missing Transforms:
289// * Narrow width by halfs excluding zero/undef lanes
290Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
291 Value *LoadPtr = II.getArgOperand(0);
292 const Align Alignment = II.getParamAlign(0).valueOrOne();
293
294 // If the mask is all ones or undefs, this is a plain vector load of the 1st
295 // argument.
296 if (maskIsAllOneOrUndef(II.getArgOperand(1))) {
297 LoadInst *L = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
298 "unmaskedload");
299 L->copyMetadata(II);
300 return L;
301 }
302
303 // If we can unconditionally load from this address, replace with a
304 // load/select idiom. TODO: use DT for context sensitive query
305 if (isDereferenceablePointer(LoadPtr, II.getType(),
306 II.getDataLayout(), &II, &AC)) {
307 LoadInst *LI = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
308 "unmaskedload");
309 LI->copyMetadata(II);
310 return Builder.CreateSelect(II.getArgOperand(1), LI, II.getArgOperand(2));
311 }
312
313 return nullptr;
314}
315
316// TODO, Obvious Missing Transforms:
317// * Single constant active lane -> store
318// * Narrow width by halfs excluding zero/undef lanes
319Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) {
320 Value *StorePtr = II.getArgOperand(1);
321 Align Alignment = II.getParamAlign(1).valueOrOne();
322 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(2));
323 if (!ConstMask)
324 return nullptr;
325
326 // If the mask is all zeros, this instruction does nothing.
327 if (maskIsAllZeroOrUndef(ConstMask))
329
330 // If the mask is all ones, this is a plain vector store of the 1st argument.
331 if (maskIsAllOneOrUndef(ConstMask)) {
332 StoreInst *S =
333 new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);
334 S->copyMetadata(II);
335 return S;
336 }
337
338 if (isa<ScalableVectorType>(ConstMask->getType()))
339 return nullptr;
340
341 // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
342 APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
343 APInt PoisonElts(DemandedElts.getBitWidth(), 0);
344 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts,
345 PoisonElts))
346 return replaceOperand(II, 0, V);
347
348 return nullptr;
349}
350
351// TODO, Obvious Missing Transforms:
352// * Single constant active lane load -> load
353// * Dereferenceable address & few lanes -> scalarize speculative load/selects
354// * Adjacent vector addresses -> masked.load
355// * Narrow width by halfs excluding zero/undef lanes
356// * Vector incrementing address -> vector masked load
357Instruction *InstCombinerImpl::simplifyMaskedGather(IntrinsicInst &II) {
358 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(1));
359 if (!ConstMask)
360 return nullptr;
361
362 // Vector splat address w/known mask -> scalar load
363 // Fold the gather to load the source vector first lane
364 // because it is reloading the same value each time
365 if (ConstMask->isAllOnesValue())
366 if (auto *SplatPtr = getSplatValue(II.getArgOperand(0))) {
367 auto *VecTy = cast<VectorType>(II.getType());
368 const Align Alignment = II.getParamAlign(0).valueOrOne();
369 LoadInst *L = Builder.CreateAlignedLoad(VecTy->getElementType(), SplatPtr,
370 Alignment, "load.scalar");
371 Value *Shuf =
372 Builder.CreateVectorSplat(VecTy->getElementCount(), L, "broadcast");
374 }
375
376 return nullptr;
377}
378
379// TODO, Obvious Missing Transforms:
380// * Single constant active lane -> store
381// * Adjacent vector addresses -> masked.store
382// * Narrow store width by halfs excluding zero/undef lanes
383// * Vector incrementing address -> vector masked store
384Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
385 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(2));
386 if (!ConstMask)
387 return nullptr;
388
389 // If the mask is all zeros, a scatter does nothing.
390 if (maskIsAllZeroOrUndef(ConstMask))
392
393 // Vector splat address -> scalar store
394 if (auto *SplatPtr = getSplatValue(II.getArgOperand(1))) {
395 // scatter(splat(value), splat(ptr), non-zero-mask) -> store value, ptr
396 if (auto *SplatValue = getSplatValue(II.getArgOperand(0))) {
397 if (maskContainsAllOneOrUndef(ConstMask)) {
398 Align Alignment = II.getParamAlign(1).valueOrOne();
399 StoreInst *S = new StoreInst(SplatValue, SplatPtr, /*IsVolatile=*/false,
400 Alignment);
401 S->copyMetadata(II);
402 return S;
403 }
404 }
405 // scatter(vector, splat(ptr), splat(true)) -> store extract(vector,
406 // lastlane), ptr
407 if (ConstMask->isAllOnesValue()) {
408 Align Alignment = II.getParamAlign(1).valueOrOne();
409 VectorType *WideLoadTy = cast<VectorType>(II.getArgOperand(1)->getType());
410 ElementCount VF = WideLoadTy->getElementCount();
411 Value *RunTimeVF = Builder.CreateElementCount(Builder.getInt32Ty(), VF);
412 Value *LastLane = Builder.CreateSub(RunTimeVF, Builder.getInt32(1));
413 Value *Extract =
414 Builder.CreateExtractElement(II.getArgOperand(0), LastLane);
415 StoreInst *S =
416 new StoreInst(Extract, SplatPtr, /*IsVolatile=*/false, Alignment);
417 S->copyMetadata(II);
418 return S;
419 }
420 }
421 if (isa<ScalableVectorType>(ConstMask->getType()))
422 return nullptr;
423
424 // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
425 APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
426 APInt PoisonElts(DemandedElts.getBitWidth(), 0);
427 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts,
428 PoisonElts))
429 return replaceOperand(II, 0, V);
430 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts,
431 PoisonElts))
432 return replaceOperand(II, 1, V);
433
434 return nullptr;
435}
436
437/// This function transforms launder.invariant.group and strip.invariant.group
438/// like:
439/// launder(launder(%x)) -> launder(%x) (the result is not the argument)
440/// launder(strip(%x)) -> launder(%x)
441/// strip(strip(%x)) -> strip(%x) (the result is not the argument)
442/// strip(launder(%x)) -> strip(%x)
443/// This is legal because it preserves the most recent information about
444/// the presence or absence of invariant.group.
446 InstCombinerImpl &IC) {
447 auto *Arg = II.getArgOperand(0);
448 auto *StrippedArg = Arg->stripPointerCasts();
449 auto *StrippedInvariantGroupsArg = StrippedArg;
450 while (auto *Intr = dyn_cast<IntrinsicInst>(StrippedInvariantGroupsArg)) {
451 if (Intr->getIntrinsicID() != Intrinsic::launder_invariant_group &&
452 Intr->getIntrinsicID() != Intrinsic::strip_invariant_group)
453 break;
454 StrippedInvariantGroupsArg = Intr->getArgOperand(0)->stripPointerCasts();
455 }
456 if (StrippedArg == StrippedInvariantGroupsArg)
457 return nullptr; // No launders/strips to remove.
458
459 Value *Result = nullptr;
460
461 if (II.getIntrinsicID() == Intrinsic::launder_invariant_group)
462 Result = IC.Builder.CreateLaunderInvariantGroup(StrippedInvariantGroupsArg);
463 else if (II.getIntrinsicID() == Intrinsic::strip_invariant_group)
464 Result = IC.Builder.CreateStripInvariantGroup(StrippedInvariantGroupsArg);
465 else
467 "simplifyInvariantGroupIntrinsic only handles launder and strip");
468 if (Result->getType()->getPointerAddressSpace() !=
469 II.getType()->getPointerAddressSpace())
470 Result = IC.Builder.CreateAddrSpaceCast(Result, II.getType());
471
472 return cast<Instruction>(Result);
473}
474
476 assert((II.getIntrinsicID() == Intrinsic::cttz ||
477 II.getIntrinsicID() == Intrinsic::ctlz) &&
478 "Expected cttz or ctlz intrinsic");
479 bool IsTZ = II.getIntrinsicID() == Intrinsic::cttz;
480 Value *Op0 = II.getArgOperand(0);
481 Value *Op1 = II.getArgOperand(1);
482 Value *X;
483 // ctlz(bitreverse(x)) -> cttz(x)
484 // cttz(bitreverse(x)) -> ctlz(x)
485 if (match(Op0, m_BitReverse(m_Value(X)))) {
486 Intrinsic::ID ID = IsTZ ? Intrinsic::ctlz : Intrinsic::cttz;
487 Function *F =
488 Intrinsic::getOrInsertDeclaration(II.getModule(), ID, II.getType());
489 return CallInst::Create(F, {X, II.getArgOperand(1)});
490 }
491
492 if (II.getType()->isIntOrIntVectorTy(1)) {
493 // ctlz/cttz i1 Op0 --> not Op0
494 if (match(Op1, m_Zero()))
495 return BinaryOperator::CreateNot(Op0);
496 // If zero is poison, then the input can be assumed to be "true", so the
497 // instruction simplifies to "false".
498 assert(match(Op1, m_One()) && "Expected ctlz/cttz operand to be 0 or 1");
499 return IC.replaceInstUsesWith(II, ConstantInt::getNullValue(II.getType()));
500 }
501
502 // If ctlz/cttz is only used as a shift amount, set is_zero_poison to true.
503 if (II.hasOneUse() && match(Op1, m_Zero()) &&
504 match(II.user_back(), m_Shift(m_Value(), m_Specific(&II)))) {
505 II.dropUBImplyingAttrsAndMetadata();
506 return IC.replaceOperand(II, 1, IC.Builder.getTrue());
507 }
508
509 Constant *C;
510
511 if (IsTZ) {
512 // cttz(-x) -> cttz(x)
513 if (match(Op0, m_Neg(m_Value(X))))
514 return IC.replaceOperand(II, 0, X);
515
516 // cttz(-x & x) -> cttz(x)
517 if (match(Op0, m_c_And(m_Neg(m_Value(X)), m_Deferred(X))))
518 return IC.replaceOperand(II, 0, X);
519
520 // cttz(sext(x)) -> cttz(zext(x))
521 if (match(Op0, m_OneUse(m_SExt(m_Value(X))))) {
522 auto *Zext = IC.Builder.CreateZExt(X, II.getType());
523 auto *CttzZext =
524 IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, Zext, Op1);
525 return IC.replaceInstUsesWith(II, CttzZext);
526 }
527
528 // Zext doesn't change the number of trailing zeros, so narrow:
529 // cttz(zext(x)) -> zext(cttz(x)) if the 'ZeroIsPoison' parameter is 'true'.
530 if (match(Op0, m_OneUse(m_ZExt(m_Value(X)))) && match(Op1, m_One())) {
531 auto *Cttz = IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, X,
532 IC.Builder.getTrue());
533 auto *ZextCttz = IC.Builder.CreateZExt(Cttz, II.getType());
534 return IC.replaceInstUsesWith(II, ZextCttz);
535 }
536
537 // cttz(abs(x)) -> cttz(x)
538 // cttz(nabs(x)) -> cttz(x)
539 Value *Y;
541 if (SPF == SPF_ABS || SPF == SPF_NABS)
542 return IC.replaceOperand(II, 0, X);
543
545 return IC.replaceOperand(II, 0, X);
546
547 // cttz(shl(%const, %val), 1) --> add(cttz(%const, 1), %val)
548 if (match(Op0, m_Shl(m_ImmConstant(C), m_Value(X))) &&
549 match(Op1, m_One())) {
550 Value *ConstCttz =
551 IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1);
552 return BinaryOperator::CreateAdd(ConstCttz, X);
553 }
554
555 // cttz(lshr exact (%const, %val), 1) --> sub(cttz(%const, 1), %val)
556 if (match(Op0, m_Exact(m_LShr(m_ImmConstant(C), m_Value(X)))) &&
557 match(Op1, m_One())) {
558 Value *ConstCttz =
559 IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1);
560 return BinaryOperator::CreateSub(ConstCttz, X);
561 }
562
563 // cttz(add(lshr(UINT_MAX, %val), 1)) --> sub(width, %val)
564 if (match(Op0, m_Add(m_LShr(m_AllOnes(), m_Value(X)), m_One()))) {
565 Value *Width =
566 ConstantInt::get(II.getType(), II.getType()->getScalarSizeInBits());
567 return BinaryOperator::CreateSub(Width, X);
568 }
569 } else {
570 // ctlz(lshr(%const, %val), 1) --> add(ctlz(%const, 1), %val)
571 if (match(Op0, m_LShr(m_ImmConstant(C), m_Value(X))) &&
572 match(Op1, m_One())) {
573 Value *ConstCtlz =
574 IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1);
575 return BinaryOperator::CreateAdd(ConstCtlz, X);
576 }
577
578 // ctlz(shl nuw (%const, %val), 1) --> sub(ctlz(%const, 1), %val)
579 if (match(Op0, m_NUWShl(m_ImmConstant(C), m_Value(X))) &&
580 match(Op1, m_One())) {
581 Value *ConstCtlz =
582 IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1);
583 return BinaryOperator::CreateSub(ConstCtlz, X);
584 }
585 }
586
587 // cttz(Pow2) -> Log2(Pow2)
588 // ctlz(Pow2) -> BitWidth - 1 - Log2(Pow2)
589 if (auto *R = IC.tryGetLog2(Op0, match(Op1, m_One()))) {
590 if (IsTZ)
591 return IC.replaceInstUsesWith(II, R);
592 BinaryOperator *BO = BinaryOperator::CreateSub(
593 ConstantInt::get(R->getType(), R->getType()->getScalarSizeInBits() - 1),
594 R);
595 BO->setHasNoSignedWrap();
597 return BO;
598 }
599
600 KnownBits Known = IC.computeKnownBits(Op0, &II);
601
602 // Create a mask for bits above (ctlz) or below (cttz) the first known one.
603 unsigned PossibleZeros = IsTZ ? Known.countMaxTrailingZeros()
604 : Known.countMaxLeadingZeros();
605 unsigned DefiniteZeros = IsTZ ? Known.countMinTrailingZeros()
606 : Known.countMinLeadingZeros();
607
608 // If all bits above (ctlz) or below (cttz) the first known one are known
609 // zero, this value is constant.
610 // FIXME: This should be in InstSimplify because we're replacing an
611 // instruction with a constant.
612 if (PossibleZeros == DefiniteZeros) {
613 auto *C = ConstantInt::get(Op0->getType(), DefiniteZeros);
614 return IC.replaceInstUsesWith(II, C);
615 }
616
617 // If the input to cttz/ctlz is known to be non-zero,
618 // then change the 'ZeroIsPoison' parameter to 'true'
619 // because we know the zero behavior can't affect the result.
620 if (!Known.One.isZero() ||
622 if (!match(II.getArgOperand(1), m_One()))
623 return IC.replaceOperand(II, 1, IC.Builder.getTrue());
624 }
625
626 // Add range attribute since known bits can't completely reflect what we know.
627 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
628 if (BitWidth != 1 && !II.hasRetAttr(Attribute::Range) &&
629 !II.getMetadata(LLVMContext::MD_range)) {
630 ConstantRange Range(APInt(BitWidth, DefiniteZeros),
631 APInt(BitWidth, PossibleZeros + 1));
632 II.addRangeRetAttr(Range);
633 return &II;
634 }
635
636 return nullptr;
637}
638
640 assert(II.getIntrinsicID() == Intrinsic::ctpop &&
641 "Expected ctpop intrinsic");
642 Type *Ty = II.getType();
643 unsigned BitWidth = Ty->getScalarSizeInBits();
644 Value *Op0 = II.getArgOperand(0);
645 Value *X, *Y;
646
647 // ctpop(bitreverse(x)) -> ctpop(x)
648 // ctpop(bswap(x)) -> ctpop(x)
649 if (match(Op0, m_BitReverse(m_Value(X))) || match(Op0, m_BSwap(m_Value(X))))
650 return IC.replaceOperand(II, 0, X);
651
652 // ctpop(rot(x)) -> ctpop(x)
653 if ((match(Op0, m_FShl(m_Value(X), m_Value(Y), m_Value())) ||
654 match(Op0, m_FShr(m_Value(X), m_Value(Y), m_Value()))) &&
655 X == Y)
656 return IC.replaceOperand(II, 0, X);
657
658 // ctpop(x | -x) -> bitwidth - cttz(x, false)
659 if (Op0->hasOneUse() &&
660 match(Op0, m_c_Or(m_Value(X), m_Neg(m_Deferred(X))))) {
661 auto *Cttz = IC.Builder.CreateIntrinsic(Intrinsic::cttz, Ty,
662 {X, IC.Builder.getFalse()});
663 auto *Bw = ConstantInt::get(Ty, APInt(BitWidth, BitWidth));
664 return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Bw, Cttz));
665 }
666
667 // ctpop(~x & (x - 1)) -> cttz(x, false)
668 if (match(Op0,
670 Function *F =
671 Intrinsic::getOrInsertDeclaration(II.getModule(), Intrinsic::cttz, Ty);
672 return CallInst::Create(F, {X, IC.Builder.getFalse()});
673 }
674
675 // Zext doesn't change the number of set bits, so narrow:
676 // ctpop (zext X) --> zext (ctpop X)
677 if (match(Op0, m_OneUse(m_ZExt(m_Value(X))))) {
678 Value *NarrowPop = IC.Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, X);
679 return CastInst::Create(Instruction::ZExt, NarrowPop, Ty);
680 }
681
682 KnownBits Known(BitWidth);
683 IC.computeKnownBits(Op0, Known, &II);
684
685 // If all bits are zero except for exactly one fixed bit, then the result
686 // must be 0 or 1, and we can get that answer by shifting to LSB:
687 // ctpop (X & 32) --> (X & 32) >> 5
688 // TODO: Investigate removing this as its likely unnecessary given the below
689 // `isKnownToBeAPowerOfTwo` check.
690 if ((~Known.Zero).isPowerOf2())
691 return BinaryOperator::CreateLShr(
692 Op0, ConstantInt::get(Ty, (~Known.Zero).exactLogBase2()));
693
694 // More generally we can also handle non-constant power of 2 patterns such as
695 // shl/shr(Pow2, X), (X & -X), etc... by transforming:
696 // ctpop(Pow2OrZero) --> icmp ne X, 0
697 if (IC.isKnownToBeAPowerOfTwo(Op0, /* OrZero */ true))
698 return CastInst::Create(Instruction::ZExt,
701 Ty);
702
703 // Add range attribute since known bits can't completely reflect what we know.
704 if (BitWidth != 1) {
705 ConstantRange OldRange =
706 II.getRange().value_or(ConstantRange::getFull(BitWidth));
707
708 unsigned Lower = Known.countMinPopulation();
709 unsigned Upper = Known.countMaxPopulation() + 1;
710
711 if (Lower == 0 && OldRange.contains(APInt::getZero(BitWidth)) &&
713 Lower = 1;
714
716 Range = Range.intersectWith(OldRange, ConstantRange::Unsigned);
717
718 if (Range != OldRange) {
719 II.addRangeRetAttr(Range);
720 return &II;
721 }
722 }
723
724 return nullptr;
725}
726
727/// Convert a table lookup to shufflevector if the mask is constant.
728/// This could benefit tbl1 if the mask is { 7,6,5,4,3,2,1,0 }, in
729/// which case we could lower the shufflevector with rev64 instructions
730/// as it's actually a byte reverse.
732 InstCombiner::BuilderTy &Builder) {
733 // Bail out if the mask is not a constant.
734 auto *C = dyn_cast<Constant>(II.getArgOperand(1));
735 if (!C)
736 return nullptr;
737
738 auto *VecTy = cast<FixedVectorType>(II.getType());
739 unsigned NumElts = VecTy->getNumElements();
740
741 // Only perform this transformation for <8 x i8> vector types.
742 if (!VecTy->getElementType()->isIntegerTy(8) || NumElts != 8)
743 return nullptr;
744
745 int Indexes[8];
746
747 for (unsigned I = 0; I < NumElts; ++I) {
748 Constant *COp = C->getAggregateElement(I);
749
750 if (!COp || !isa<ConstantInt>(COp))
751 return nullptr;
752
753 Indexes[I] = cast<ConstantInt>(COp)->getLimitedValue();
754
755 // Make sure the mask indices are in range.
756 if ((unsigned)Indexes[I] >= NumElts)
757 return nullptr;
758 }
759
760 auto *V1 = II.getArgOperand(0);
761 auto *V2 = Constant::getNullValue(V1->getType());
762 return Builder.CreateShuffleVector(V1, V2, ArrayRef(Indexes));
763}
764
765// Returns true iff the 2 intrinsics have the same operands, limiting the
766// comparison to the first NumOperands.
767static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E,
768 unsigned NumOperands) {
769 assert(I.arg_size() >= NumOperands && "Not enough operands");
770 assert(E.arg_size() >= NumOperands && "Not enough operands");
771 for (unsigned i = 0; i < NumOperands; i++)
772 if (I.getArgOperand(i) != E.getArgOperand(i))
773 return false;
774 return true;
775}
776
777// Remove trivially empty start/end intrinsic ranges, i.e. a start
778// immediately followed by an end (ignoring debuginfo or other
779// start/end intrinsics in between). As this handles only the most trivial
780// cases, tracking the nesting level is not needed:
781//
782// call @llvm.foo.start(i1 0)
783// call @llvm.foo.start(i1 0) ; This one won't be skipped: it will be removed
784// call @llvm.foo.end(i1 0)
785// call @llvm.foo.end(i1 0) ; &I
786static bool
788 std::function<bool(const IntrinsicInst &)> IsStart) {
789 // We start from the end intrinsic and scan backwards, so that InstCombine
790 // has already processed (and potentially removed) all the instructions
791 // before the end intrinsic.
792 BasicBlock::reverse_iterator BI(EndI), BE(EndI.getParent()->rend());
793 for (; BI != BE; ++BI) {
794 if (auto *I = dyn_cast<IntrinsicInst>(&*BI)) {
795 if (I->isDebugOrPseudoInst() ||
796 I->getIntrinsicID() == EndI.getIntrinsicID())
797 continue;
798 if (IsStart(*I)) {
799 if (haveSameOperands(EndI, *I, EndI.arg_size())) {
801 IC.eraseInstFromFunction(EndI);
802 return true;
803 }
804 // Skip start intrinsics that don't pair with this end intrinsic.
805 continue;
806 }
807 }
808 break;
809 }
810
811 return false;
812}
813
815 removeTriviallyEmptyRange(I, *this, [&I](const IntrinsicInst &II) {
816 // Bail out on the case where the source va_list of a va_copy is destroyed
817 // immediately by a follow-up va_end.
818 return II.getIntrinsicID() == Intrinsic::vastart ||
819 (II.getIntrinsicID() == Intrinsic::vacopy &&
820 I.getArgOperand(0) != II.getArgOperand(1));
821 });
822 return nullptr;
823}
824
826 assert(Call.arg_size() > 1 && "Need at least 2 args to swap");
827 Value *Arg0 = Call.getArgOperand(0), *Arg1 = Call.getArgOperand(1);
828 if (isa<Constant>(Arg0) && !isa<Constant>(Arg1)) {
829 Call.setArgOperand(0, Arg1);
830 Call.setArgOperand(1, Arg0);
831 return &Call;
832 }
833 return nullptr;
834}
835
836/// Creates a result tuple for an overflow intrinsic \p II with a given
837/// \p Result and a constant \p Overflow value.
839 Constant *Overflow) {
840 Constant *V[] = {PoisonValue::get(Result->getType()), Overflow};
841 StructType *ST = cast<StructType>(II->getType());
843 return InsertValueInst::Create(Struct, Result, 0);
844}
845
847InstCombinerImpl::foldIntrinsicWithOverflowCommon(IntrinsicInst *II) {
848 WithOverflowInst *WO = cast<WithOverflowInst>(II);
849 Value *OperationResult = nullptr;
850 Constant *OverflowResult = nullptr;
851 if (OptimizeOverflowCheck(WO->getBinaryOp(), WO->isSigned(), WO->getLHS(),
852 WO->getRHS(), *WO, OperationResult, OverflowResult))
853 return createOverflowTuple(WO, OperationResult, OverflowResult);
854
855 // See whether we can optimize the overflow check with assumption information.
856 for (User *U : WO->users()) {
857 if (!match(U, m_ExtractValue<1>(m_Value())))
858 continue;
859
860 for (auto &AssumeVH : AC.assumptionsFor(U)) {
861 if (!AssumeVH)
862 continue;
863 CallInst *I = cast<CallInst>(AssumeVH);
864 if (!match(I->getArgOperand(0), m_Not(m_Specific(U))))
865 continue;
866 if (!isValidAssumeForContext(I, II, /*DT=*/nullptr,
867 /*AllowEphemerals=*/true))
868 continue;
869 Value *Result =
870 Builder.CreateBinOp(WO->getBinaryOp(), WO->getLHS(), WO->getRHS());
871 Result->takeName(WO);
872 if (auto *Inst = dyn_cast<Instruction>(Result)) {
873 if (WO->isSigned())
874 Inst->setHasNoSignedWrap();
875 else
876 Inst->setHasNoUnsignedWrap();
877 }
878 return createOverflowTuple(WO, Result,
879 ConstantInt::getFalse(U->getType()));
880 }
881 }
882
883 return nullptr;
884}
885
886static bool inputDenormalIsIEEE(const Function &F, const Type *Ty) {
887 Ty = Ty->getScalarType();
888 return F.getDenormalMode(Ty->getFltSemantics()).Input == DenormalMode::IEEE;
889}
890
891static bool inputDenormalIsDAZ(const Function &F, const Type *Ty) {
892 Ty = Ty->getScalarType();
893 return F.getDenormalMode(Ty->getFltSemantics()).inputsAreZero();
894}
895
896/// \returns the compare predicate type if the test performed by
897/// llvm.is.fpclass(x, \p Mask) is equivalent to fcmp o__ x, 0.0 with the
898/// floating-point environment assumed for \p F for type \p Ty
900 const Function &F, Type *Ty) {
901 switch (static_cast<unsigned>(Mask)) {
902 case fcZero:
903 if (inputDenormalIsIEEE(F, Ty))
904 return FCmpInst::FCMP_OEQ;
905 break;
906 case fcZero | fcSubnormal:
907 if (inputDenormalIsDAZ(F, Ty))
908 return FCmpInst::FCMP_OEQ;
909 break;
910 case fcPositive | fcNegZero:
911 if (inputDenormalIsIEEE(F, Ty))
912 return FCmpInst::FCMP_OGE;
913 break;
915 if (inputDenormalIsDAZ(F, Ty))
916 return FCmpInst::FCMP_OGE;
917 break;
919 if (inputDenormalIsIEEE(F, Ty))
920 return FCmpInst::FCMP_OGT;
921 break;
922 case fcNegative | fcPosZero:
923 if (inputDenormalIsIEEE(F, Ty))
924 return FCmpInst::FCMP_OLE;
925 break;
927 if (inputDenormalIsDAZ(F, Ty))
928 return FCmpInst::FCMP_OLE;
929 break;
931 if (inputDenormalIsIEEE(F, Ty))
932 return FCmpInst::FCMP_OLT;
933 break;
934 case fcPosNormal | fcPosInf:
935 if (inputDenormalIsDAZ(F, Ty))
936 return FCmpInst::FCMP_OGT;
937 break;
938 case fcNegNormal | fcNegInf:
939 if (inputDenormalIsDAZ(F, Ty))
940 return FCmpInst::FCMP_OLT;
941 break;
942 case ~fcZero & ~fcNan:
943 if (inputDenormalIsIEEE(F, Ty))
944 return FCmpInst::FCMP_ONE;
945 break;
946 case ~(fcZero | fcSubnormal) & ~fcNan:
947 if (inputDenormalIsDAZ(F, Ty))
948 return FCmpInst::FCMP_ONE;
949 break;
950 default:
951 break;
952 }
953
955}
956
957Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) {
958 Value *Src0 = II.getArgOperand(0);
959 Value *Src1 = II.getArgOperand(1);
960 const ConstantInt *CMask = cast<ConstantInt>(Src1);
961 FPClassTest Mask = static_cast<FPClassTest>(CMask->getZExtValue());
962 const bool IsUnordered = (Mask & fcNan) == fcNan;
963 const bool IsOrdered = (Mask & fcNan) == fcNone;
964 const FPClassTest OrderedMask = Mask & ~fcNan;
965 const FPClassTest OrderedInvertedMask = ~OrderedMask & ~fcNan;
966
967 const bool IsStrict =
968 II.getFunction()->getAttributes().hasFnAttr(Attribute::StrictFP);
969
970 Value *FNegSrc;
971 if (match(Src0, m_FNeg(m_Value(FNegSrc)))) {
972 // is.fpclass (fneg x), mask -> is.fpclass x, (fneg mask)
973
974 II.setArgOperand(1, ConstantInt::get(Src1->getType(), fneg(Mask)));
975 return replaceOperand(II, 0, FNegSrc);
976 }
977
978 Value *FAbsSrc;
979 if (match(Src0, m_FAbs(m_Value(FAbsSrc)))) {
980 II.setArgOperand(1, ConstantInt::get(Src1->getType(), inverse_fabs(Mask)));
981 return replaceOperand(II, 0, FAbsSrc);
982 }
983
984 if ((OrderedMask == fcInf || OrderedInvertedMask == fcInf) &&
985 (IsOrdered || IsUnordered) && !IsStrict) {
986 // is.fpclass(x, fcInf) -> fcmp oeq fabs(x), +inf
987 // is.fpclass(x, ~fcInf) -> fcmp one fabs(x), +inf
988 // is.fpclass(x, fcInf|fcNan) -> fcmp ueq fabs(x), +inf
989 // is.fpclass(x, ~(fcInf|fcNan)) -> fcmp une fabs(x), +inf
993 if (OrderedInvertedMask == fcInf)
994 Pred = IsUnordered ? FCmpInst::FCMP_UNE : FCmpInst::FCMP_ONE;
995
996 Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Src0);
997 Value *CmpInf = Builder.CreateFCmp(Pred, Fabs, Inf);
998 CmpInf->takeName(&II);
999 return replaceInstUsesWith(II, CmpInf);
1000 }
1001
1002 if ((OrderedMask == fcPosInf || OrderedMask == fcNegInf) &&
1003 (IsOrdered || IsUnordered) && !IsStrict) {
1004 // is.fpclass(x, fcPosInf) -> fcmp oeq x, +inf
1005 // is.fpclass(x, fcNegInf) -> fcmp oeq x, -inf
1006 // is.fpclass(x, fcPosInf|fcNan) -> fcmp ueq x, +inf
1007 // is.fpclass(x, fcNegInf|fcNan) -> fcmp ueq x, -inf
1008 Constant *Inf =
1009 ConstantFP::getInfinity(Src0->getType(), OrderedMask == fcNegInf);
1010 Value *EqInf = IsUnordered ? Builder.CreateFCmpUEQ(Src0, Inf)
1011 : Builder.CreateFCmpOEQ(Src0, Inf);
1012
1013 EqInf->takeName(&II);
1014 return replaceInstUsesWith(II, EqInf);
1015 }
1016
1017 if ((OrderedInvertedMask == fcPosInf || OrderedInvertedMask == fcNegInf) &&
1018 (IsOrdered || IsUnordered) && !IsStrict) {
1019 // is.fpclass(x, ~fcPosInf) -> fcmp one x, +inf
1020 // is.fpclass(x, ~fcNegInf) -> fcmp one x, -inf
1021 // is.fpclass(x, ~fcPosInf|fcNan) -> fcmp une x, +inf
1022 // is.fpclass(x, ~fcNegInf|fcNan) -> fcmp une x, -inf
1024 OrderedInvertedMask == fcNegInf);
1025 Value *NeInf = IsUnordered ? Builder.CreateFCmpUNE(Src0, Inf)
1026 : Builder.CreateFCmpONE(Src0, Inf);
1027 NeInf->takeName(&II);
1028 return replaceInstUsesWith(II, NeInf);
1029 }
1030
1031 if (Mask == fcNan && !IsStrict) {
1032 // Equivalent of isnan. Replace with standard fcmp if we don't care about FP
1033 // exceptions.
1034 Value *IsNan =
1035 Builder.CreateFCmpUNO(Src0, ConstantFP::getZero(Src0->getType()));
1036 IsNan->takeName(&II);
1037 return replaceInstUsesWith(II, IsNan);
1038 }
1039
1040 if (Mask == (~fcNan & fcAllFlags) && !IsStrict) {
1041 // Equivalent of !isnan. Replace with standard fcmp.
1042 Value *FCmp =
1043 Builder.CreateFCmpORD(Src0, ConstantFP::getZero(Src0->getType()));
1044 FCmp->takeName(&II);
1045 return replaceInstUsesWith(II, FCmp);
1046 }
1047
1049
1050 // Try to replace with an fcmp with 0
1051 //
1052 // is.fpclass(x, fcZero) -> fcmp oeq x, 0.0
1053 // is.fpclass(x, fcZero | fcNan) -> fcmp ueq x, 0.0
1054 // is.fpclass(x, ~fcZero & ~fcNan) -> fcmp one x, 0.0
1055 // is.fpclass(x, ~fcZero) -> fcmp une x, 0.0
1056 //
1057 // is.fpclass(x, fcPosSubnormal | fcPosNormal | fcPosInf) -> fcmp ogt x, 0.0
1058 // is.fpclass(x, fcPositive | fcNegZero) -> fcmp oge x, 0.0
1059 //
1060 // is.fpclass(x, fcNegSubnormal | fcNegNormal | fcNegInf) -> fcmp olt x, 0.0
1061 // is.fpclass(x, fcNegative | fcPosZero) -> fcmp ole x, 0.0
1062 //
1063 if (!IsStrict && (IsOrdered || IsUnordered) &&
1064 (PredType = fpclassTestIsFCmp0(OrderedMask, *II.getFunction(),
1065 Src0->getType())) !=
1068 // Equivalent of == 0.
1069 Value *FCmp = Builder.CreateFCmp(
1070 IsUnordered ? FCmpInst::getUnorderedPredicate(PredType) : PredType,
1071 Src0, Zero);
1072
1073 FCmp->takeName(&II);
1074 return replaceInstUsesWith(II, FCmp);
1075 }
1076
1077 KnownFPClass Known = computeKnownFPClass(Src0, Mask, &II);
1078
1079 // Clear test bits we know must be false from the source value.
1080 // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
1081 // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other
1082 if ((Mask & Known.KnownFPClasses) != Mask) {
1083 II.setArgOperand(
1084 1, ConstantInt::get(Src1->getType(), Mask & Known.KnownFPClasses));
1085 return &II;
1086 }
1087
1088 // If none of the tests which can return false are possible, fold to true.
1089 // fp_class (nnan x), ~(qnan|snan) -> true
1090 // fp_class (ninf x), ~(ninf|pinf) -> true
1091 if (Mask == Known.KnownFPClasses)
1092 return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
1093
1094 return nullptr;
1095}
1096
1097static std::optional<bool> getKnownSign(Value *Op, const SimplifyQuery &SQ) {
1098 KnownBits Known = computeKnownBits(Op, SQ);
1099 if (Known.isNonNegative())
1100 return false;
1101 if (Known.isNegative())
1102 return true;
1103
1104 Value *X, *Y;
1105 if (match(Op, m_NSWSub(m_Value(X), m_Value(Y))))
1107
1108 return std::nullopt;
1109}
1110
1111static std::optional<bool> getKnownSignOrZero(Value *Op,
1112 const SimplifyQuery &SQ) {
1113 if (std::optional<bool> Sign = getKnownSign(Op, SQ))
1114 return Sign;
1115
1116 Value *X, *Y;
1117 if (match(Op, m_NSWSub(m_Value(X), m_Value(Y))))
1119
1120 return std::nullopt;
1121}
1122
1123/// Return true if two values \p Op0 and \p Op1 are known to have the same sign.
1124static bool signBitMustBeTheSame(Value *Op0, Value *Op1,
1125 const SimplifyQuery &SQ) {
1126 std::optional<bool> Known1 = getKnownSign(Op1, SQ);
1127 if (!Known1)
1128 return false;
1129 std::optional<bool> Known0 = getKnownSign(Op0, SQ);
1130 if (!Known0)
1131 return false;
1132 return *Known0 == *Known1;
1133}
1134
1135/// Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0. This
1136/// can trigger other combines.
1138 InstCombiner::BuilderTy &Builder) {
1139 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1140 assert((MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin ||
1141 MinMaxID == Intrinsic::umax || MinMaxID == Intrinsic::umin) &&
1142 "Expected a min or max intrinsic");
1143
1144 // TODO: Match vectors with undef elements, but undef may not propagate.
1145 Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);
1146 Value *X;
1147 const APInt *C0, *C1;
1148 if (!match(Op0, m_OneUse(m_Add(m_Value(X), m_APInt(C0)))) ||
1149 !match(Op1, m_APInt(C1)))
1150 return nullptr;
1151
1152 // Check for necessary no-wrap and overflow constraints.
1153 bool IsSigned = MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin;
1154 auto *Add = cast<BinaryOperator>(Op0);
1155 if ((IsSigned && !Add->hasNoSignedWrap()) ||
1156 (!IsSigned && !Add->hasNoUnsignedWrap()))
1157 return nullptr;
1158
1159 // If the constant difference overflows, then instsimplify should reduce the
1160 // min/max to the add or C1.
1161 bool Overflow;
1162 APInt CDiff =
1163 IsSigned ? C1->ssub_ov(*C0, Overflow) : C1->usub_ov(*C0, Overflow);
1164 assert(!Overflow && "Expected simplify of min/max");
1165
1166 // min/max (add X, C0), C1 --> add (min/max X, C1 - C0), C0
1167 // Note: the "mismatched" no-overflow setting does not propagate.
1168 Constant *NewMinMaxC = ConstantInt::get(II->getType(), CDiff);
1169 Value *NewMinMax = Builder.CreateBinaryIntrinsic(MinMaxID, X, NewMinMaxC);
1170 return IsSigned ? BinaryOperator::CreateNSWAdd(NewMinMax, Add->getOperand(1))
1171 : BinaryOperator::CreateNUWAdd(NewMinMax, Add->getOperand(1));
1172}
1173/// Match a sadd_sat or ssub_sat which is using min/max to clamp the value.
1174Instruction *InstCombinerImpl::matchSAddSubSat(IntrinsicInst &MinMax1) {
1175 Type *Ty = MinMax1.getType();
1176
1177 // We are looking for a tree of:
1178 // max(INT_MIN, min(INT_MAX, add(sext(A), sext(B))))
1179 // Where the min and max could be reversed
1180 Instruction *MinMax2;
1181 BinaryOperator *AddSub;
1182 const APInt *MinValue, *MaxValue;
1183 if (match(&MinMax1, m_SMin(m_Instruction(MinMax2), m_APInt(MaxValue)))) {
1184 if (!match(MinMax2, m_SMax(m_BinOp(AddSub), m_APInt(MinValue))))
1185 return nullptr;
1186 } else if (match(&MinMax1,
1187 m_SMax(m_Instruction(MinMax2), m_APInt(MinValue)))) {
1188 if (!match(MinMax2, m_SMin(m_BinOp(AddSub), m_APInt(MaxValue))))
1189 return nullptr;
1190 } else
1191 return nullptr;
1192
1193 // Check that the constants clamp a saturate, and that the new type would be
1194 // sensible to convert to.
1195 if (!(*MaxValue + 1).isPowerOf2() || -*MinValue != *MaxValue + 1)
1196 return nullptr;
1197 // In what bitwidth can this be treated as saturating arithmetics?
1198 unsigned NewBitWidth = (*MaxValue + 1).logBase2() + 1;
1199 // FIXME: This isn't quite right for vectors, but using the scalar type is a
1200 // good first approximation for what should be done there.
1201 if (!shouldChangeType(Ty->getScalarType()->getIntegerBitWidth(), NewBitWidth))
1202 return nullptr;
1203
1204 // Also make sure that the inner min/max and the add/sub have one use.
1205 if (!MinMax2->hasOneUse() || !AddSub->hasOneUse())
1206 return nullptr;
1207
1208 // Create the new type (which can be a vector type)
1209 Type *NewTy = Ty->getWithNewBitWidth(NewBitWidth);
1210
1211 Intrinsic::ID IntrinsicID;
1212 if (AddSub->getOpcode() == Instruction::Add)
1213 IntrinsicID = Intrinsic::sadd_sat;
1214 else if (AddSub->getOpcode() == Instruction::Sub)
1215 IntrinsicID = Intrinsic::ssub_sat;
1216 else
1217 return nullptr;
1218
1219 // The two operands of the add/sub must be nsw-truncatable to the NewTy. This
1220 // is usually achieved via a sext from a smaller type.
1221 if (ComputeMaxSignificantBits(AddSub->getOperand(0), AddSub) > NewBitWidth ||
1222 ComputeMaxSignificantBits(AddSub->getOperand(1), AddSub) > NewBitWidth)
1223 return nullptr;
1224
1225 // Finally create and return the sat intrinsic, truncated to the new type
1226 Value *AT = Builder.CreateTrunc(AddSub->getOperand(0), NewTy);
1227 Value *BT = Builder.CreateTrunc(AddSub->getOperand(1), NewTy);
1228 Value *Sat = Builder.CreateIntrinsic(IntrinsicID, NewTy, {AT, BT});
1229 return CastInst::Create(Instruction::SExt, Sat, Ty);
1230}
1231
1232
1233/// If we have a clamp pattern like max (min X, 42), 41 -- where the output
1234/// can only be one of two possible constant values -- turn that into a select
1235/// of constants.
1237 InstCombiner::BuilderTy &Builder) {
1238 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1239 Value *X;
1240 const APInt *C0, *C1;
1241 if (!match(I1, m_APInt(C1)) || !I0->hasOneUse())
1242 return nullptr;
1243
1245 switch (II->getIntrinsicID()) {
1246 case Intrinsic::smax:
1247 if (match(I0, m_SMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)
1248 Pred = ICmpInst::ICMP_SGT;
1249 break;
1250 case Intrinsic::smin:
1251 if (match(I0, m_SMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)
1252 Pred = ICmpInst::ICMP_SLT;
1253 break;
1254 case Intrinsic::umax:
1255 if (match(I0, m_UMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)
1256 Pred = ICmpInst::ICMP_UGT;
1257 break;
1258 case Intrinsic::umin:
1259 if (match(I0, m_UMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)
1260 Pred = ICmpInst::ICMP_ULT;
1261 break;
1262 default:
1263 llvm_unreachable("Expected min/max intrinsic");
1264 }
1265 if (Pred == CmpInst::BAD_ICMP_PREDICATE)
1266 return nullptr;
1267
1268 // max (min X, 42), 41 --> X > 41 ? 42 : 41
1269 // min (max X, 42), 43 --> X < 43 ? 42 : 43
1270 Value *Cmp = Builder.CreateICmp(Pred, X, I1);
1271 return SelectInst::Create(Cmp, ConstantInt::get(II->getType(), *C0), I1);
1272}
1273
1274/// If this min/max has a constant operand and an operand that is a matching
1275/// min/max with a constant operand, constant-fold the 2 constant operands.
1277 IRBuilderBase &Builder,
1278 const SimplifyQuery &SQ) {
1279 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1280 auto *LHS = dyn_cast<MinMaxIntrinsic>(II->getArgOperand(0));
1281 if (!LHS)
1282 return nullptr;
1283
1284 Constant *C0, *C1;
1285 if (!match(LHS->getArgOperand(1), m_ImmConstant(C0)) ||
1286 !match(II->getArgOperand(1), m_ImmConstant(C1)))
1287 return nullptr;
1288
1289 // max (max X, C0), C1 --> max X, (max C0, C1)
1290 // min (min X, C0), C1 --> min X, (min C0, C1)
1291 // umax (smax X, nneg C0), nneg C1 --> smax X, (umax C0, C1)
1292 // smin (umin X, nneg C0), nneg C1 --> umin X, (smin C0, C1)
1293 Intrinsic::ID InnerMinMaxID = LHS->getIntrinsicID();
1294 if (InnerMinMaxID != MinMaxID &&
1295 !(((MinMaxID == Intrinsic::umax && InnerMinMaxID == Intrinsic::smax) ||
1296 (MinMaxID == Intrinsic::smin && InnerMinMaxID == Intrinsic::umin)) &&
1297 isKnownNonNegative(C0, SQ) && isKnownNonNegative(C1, SQ)))
1298 return nullptr;
1299
1301 Value *CondC = Builder.CreateICmp(Pred, C0, C1);
1302 Value *NewC = Builder.CreateSelect(CondC, C0, C1);
1303 return Builder.CreateIntrinsic(InnerMinMaxID, II->getType(),
1304 {LHS->getArgOperand(0), NewC});
1305}
1306
1307/// If this min/max has a matching min/max operand with a constant, try to push
1308/// the constant operand into this instruction. This can enable more folds.
1309static Instruction *
1311 InstCombiner::BuilderTy &Builder) {
1312 // Match and capture a min/max operand candidate.
1313 Value *X, *Y;
1314 Constant *C;
1315 Instruction *Inner;
1317 m_Instruction(Inner),
1319 m_Value(Y))))
1320 return nullptr;
1321
1322 // The inner op must match. Check for constants to avoid infinite loops.
1323 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1324 auto *InnerMM = dyn_cast<IntrinsicInst>(Inner);
1325 if (!InnerMM || InnerMM->getIntrinsicID() != MinMaxID ||
1327 return nullptr;
1328
1329 // max (max X, C), Y --> max (max X, Y), C
1331 MinMaxID, II->getType());
1332 Value *NewInner = Builder.CreateBinaryIntrinsic(MinMaxID, X, Y);
1333 NewInner->takeName(Inner);
1334 return CallInst::Create(MinMax, {NewInner, C});
1335}
1336
1337/// Reduce a sequence of min/max intrinsics with a common operand.
1339 // Match 3 of the same min/max ops. Example: umin(umin(), umin()).
1340 auto *LHS = dyn_cast<IntrinsicInst>(II->getArgOperand(0));
1341 auto *RHS = dyn_cast<IntrinsicInst>(II->getArgOperand(1));
1342 Intrinsic::ID MinMaxID = II->getIntrinsicID();
1343 if (!LHS || !RHS || LHS->getIntrinsicID() != MinMaxID ||
1344 RHS->getIntrinsicID() != MinMaxID ||
1345 (!LHS->hasOneUse() && !RHS->hasOneUse()))
1346 return nullptr;
1347
1348 Value *A = LHS->getArgOperand(0);
1349 Value *B = LHS->getArgOperand(1);
1350 Value *C = RHS->getArgOperand(0);
1351 Value *D = RHS->getArgOperand(1);
1352
1353 // Look for a common operand.
1354 Value *MinMaxOp = nullptr;
1355 Value *ThirdOp = nullptr;
1356 if (LHS->hasOneUse()) {
1357 // If the LHS is only used in this chain and the RHS is used outside of it,
1358 // reuse the RHS min/max because that will eliminate the LHS.
1359 if (D == A || C == A) {
1360 // min(min(a, b), min(c, a)) --> min(min(c, a), b)
1361 // min(min(a, b), min(a, d)) --> min(min(a, d), b)
1362 MinMaxOp = RHS;
1363 ThirdOp = B;
1364 } else if (D == B || C == B) {
1365 // min(min(a, b), min(c, b)) --> min(min(c, b), a)
1366 // min(min(a, b), min(b, d)) --> min(min(b, d), a)
1367 MinMaxOp = RHS;
1368 ThirdOp = A;
1369 }
1370 } else {
1371 assert(RHS->hasOneUse() && "Expected one-use operand");
1372 // Reuse the LHS. This will eliminate the RHS.
1373 if (D == A || D == B) {
1374 // min(min(a, b), min(c, a)) --> min(min(a, b), c)
1375 // min(min(a, b), min(c, b)) --> min(min(a, b), c)
1376 MinMaxOp = LHS;
1377 ThirdOp = C;
1378 } else if (C == A || C == B) {
1379 // min(min(a, b), min(b, d)) --> min(min(a, b), d)
1380 // min(min(a, b), min(c, b)) --> min(min(a, b), d)
1381 MinMaxOp = LHS;
1382 ThirdOp = D;
1383 }
1384 }
1385
1386 if (!MinMaxOp || !ThirdOp)
1387 return nullptr;
1388
1389 Module *Mod = II->getModule();
1390 Function *MinMax =
1391 Intrinsic::getOrInsertDeclaration(Mod, MinMaxID, II->getType());
1392 return CallInst::Create(MinMax, { MinMaxOp, ThirdOp });
1393}
1394
1395/// If all arguments of the intrinsic are unary shuffles with the same mask,
1396/// try to shuffle after the intrinsic.
1399 if (!isTriviallyVectorizable(II->getIntrinsicID()) ||
1400 !II->getCalledFunction()->isSpeculatable())
1401 return nullptr;
1402
1403 Value *X;
1404 Constant *C;
1405 ArrayRef<int> Mask;
1406 auto *NonConstArg = find_if_not(II->args(), [&II](Use &Arg) {
1407 return isa<Constant>(Arg.get()) ||
1408 isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
1409 Arg.getOperandNo(), nullptr);
1410 });
1411 if (!NonConstArg ||
1412 !match(NonConstArg, m_Shuffle(m_Value(X), m_Poison(), m_Mask(Mask))))
1413 return nullptr;
1414
1415 // At least 1 operand must be a shuffle with 1 use because we are creating 2
1416 // instructions.
1417 if (none_of(II->args(), match_fn(m_OneUse(m_Shuffle(m_Value(), m_Value())))))
1418 return nullptr;
1419
1420 // See if all arguments are shuffled with the same mask.
1422 Type *SrcTy = X->getType();
1423 for (Use &Arg : II->args()) {
1424 if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
1425 Arg.getOperandNo(), nullptr))
1426 NewArgs.push_back(Arg);
1427 else if (match(&Arg,
1428 m_Shuffle(m_Value(X), m_Poison(), m_SpecificMask(Mask))) &&
1429 X->getType() == SrcTy)
1430 NewArgs.push_back(X);
1431 else if (match(&Arg, m_ImmConstant(C))) {
1432 // If it's a constant, try find the constant that would be shuffled to C.
1433 if (Constant *ShuffledC =
1434 unshuffleConstant(Mask, C, cast<VectorType>(SrcTy)))
1435 NewArgs.push_back(ShuffledC);
1436 else
1437 return nullptr;
1438 } else
1439 return nullptr;
1440 }
1441
1442 // intrinsic (shuf X, M), (shuf Y, M), ... --> shuf (intrinsic X, Y, ...), M
1443 Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
1444 // Result type might be a different vector width.
1445 // TODO: Check that the result type isn't widened?
1446 VectorType *ResTy =
1447 VectorType::get(II->getType()->getScalarType(), cast<VectorType>(SrcTy));
1448 Value *NewIntrinsic =
1449 Builder.CreateIntrinsic(ResTy, II->getIntrinsicID(), NewArgs, FPI);
1450 return new ShuffleVectorInst(NewIntrinsic, Mask);
1451}
1452
1453/// If all arguments of the intrinsic are reverses, try to pull the reverse
1454/// after the intrinsic.
1456 if (!isTriviallyVectorizable(II->getIntrinsicID()))
1457 return nullptr;
1458
1459 // At least 1 operand must be a reverse with 1 use because we are creating 2
1460 // instructions.
1461 if (none_of(II->args(), [](Value *V) {
1462 return match(V, m_OneUse(m_VecReverse(m_Value())));
1463 }))
1464 return nullptr;
1465
1466 Value *X;
1467 Constant *C;
1468 SmallVector<Value *> NewArgs;
1469 for (Use &Arg : II->args()) {
1470 if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
1471 Arg.getOperandNo(), nullptr))
1472 NewArgs.push_back(Arg);
1473 else if (match(&Arg, m_VecReverse(m_Value(X))))
1474 NewArgs.push_back(X);
1475 else if (isSplatValue(Arg))
1476 NewArgs.push_back(Arg);
1477 else if (match(&Arg, m_ImmConstant(C)))
1478 NewArgs.push_back(Builder.CreateVectorReverse(C));
1479 else
1480 return nullptr;
1481 }
1482
1483 // intrinsic (reverse X), (reverse Y), ... --> reverse (intrinsic X, Y, ...)
1484 Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
1485 Instruction *NewIntrinsic = Builder.CreateIntrinsic(
1486 II->getType(), II->getIntrinsicID(), NewArgs, FPI);
1487 return Builder.CreateVectorReverse(NewIntrinsic);
1488}
1489
1490/// Fold the following cases and accepts bswap and bitreverse intrinsics:
1491/// bswap(logic_op(bswap(x), y)) --> logic_op(x, bswap(y))
1492/// bswap(logic_op(bswap(x), bswap(y))) --> logic_op(x, y) (ignores multiuse)
1493template <Intrinsic::ID IntrID>
1495 InstCombiner::BuilderTy &Builder) {
1496 static_assert(IntrID == Intrinsic::bswap || IntrID == Intrinsic::bitreverse,
1497 "This helper only supports BSWAP and BITREVERSE intrinsics");
1498
1499 Value *X, *Y;
1500 // Find bitwise logic op. Check that it is a BinaryOperator explicitly so we
1501 // don't match ConstantExpr that aren't meaningful for this transform.
1504 Value *OldReorderX, *OldReorderY;
1506
1507 // If both X and Y are bswap/bitreverse, the transform reduces the number
1508 // of instructions even if there's multiuse.
1509 // If only one operand is bswap/bitreverse, we need to ensure the operand
1510 // have only one use.
1511 if (match(X, m_Intrinsic<IntrID>(m_Value(OldReorderX))) &&
1512 match(Y, m_Intrinsic<IntrID>(m_Value(OldReorderY)))) {
1513 return BinaryOperator::Create(Op, OldReorderX, OldReorderY);
1514 }
1515
1516 if (match(X, m_OneUse(m_Intrinsic<IntrID>(m_Value(OldReorderX))))) {
1517 Value *NewReorder = Builder.CreateUnaryIntrinsic(IntrID, Y);
1518 return BinaryOperator::Create(Op, OldReorderX, NewReorder);
1519 }
1520
1521 if (match(Y, m_OneUse(m_Intrinsic<IntrID>(m_Value(OldReorderY))))) {
1522 Value *NewReorder = Builder.CreateUnaryIntrinsic(IntrID, X);
1523 return BinaryOperator::Create(Op, NewReorder, OldReorderY);
1524 }
1525 }
1526 return nullptr;
1527}
1528
1529/// Helper to match idempotent binary intrinsics, namely, intrinsics where
1530/// `f(f(x, y), y) == f(x, y)` holds.
1532 switch (IID) {
1533 case Intrinsic::smax:
1534 case Intrinsic::smin:
1535 case Intrinsic::umax:
1536 case Intrinsic::umin:
1537 case Intrinsic::maximum:
1538 case Intrinsic::minimum:
1539 case Intrinsic::maximumnum:
1540 case Intrinsic::minimumnum:
1541 case Intrinsic::maxnum:
1542 case Intrinsic::minnum:
1543 return true;
1544 default:
1545 return false;
1546 }
1547}
1548
1549/// Attempt to simplify value-accumulating recurrences of kind:
1550/// %umax.acc = phi i8 [ %umax, %backedge ], [ %a, %entry ]
1551/// %umax = call i8 @llvm.umax.i8(i8 %umax.acc, i8 %b)
1552/// And let the idempotent binary intrinsic be hoisted, when the operands are
1553/// known to be loop-invariant.
1555 IntrinsicInst *II) {
1556 PHINode *PN;
1557 Value *Init, *OtherOp;
1558
1559 // A binary intrinsic recurrence with loop-invariant operands is equivalent to
1560 // `call @llvm.binary.intrinsic(Init, OtherOp)`.
1561 auto IID = II->getIntrinsicID();
1562 if (!isIdempotentBinaryIntrinsic(IID) ||
1564 !IC.getDominatorTree().dominates(OtherOp, PN))
1565 return nullptr;
1566
1567 auto *InvariantBinaryInst =
1568 IC.Builder.CreateBinaryIntrinsic(IID, Init, OtherOp);
1569 if (isa<FPMathOperator>(InvariantBinaryInst))
1570 cast<Instruction>(InvariantBinaryInst)->copyFastMathFlags(II);
1571 return InvariantBinaryInst;
1572}
1573
1574static Value *simplifyReductionOperand(Value *Arg, bool CanReorderLanes) {
1575 if (!CanReorderLanes)
1576 return nullptr;
1577
1578 Value *V;
1579 if (match(Arg, m_VecReverse(m_Value(V))))
1580 return V;
1581
1582 ArrayRef<int> Mask;
1583 if (!isa<FixedVectorType>(Arg->getType()) ||
1584 !match(Arg, m_Shuffle(m_Value(V), m_Undef(), m_Mask(Mask))) ||
1585 !cast<ShuffleVectorInst>(Arg)->isSingleSource())
1586 return nullptr;
1587
1588 int Sz = Mask.size();
1589 SmallBitVector UsedIndices(Sz);
1590 for (int Idx : Mask) {
1591 if (Idx == PoisonMaskElem || UsedIndices.test(Idx))
1592 return nullptr;
1593 UsedIndices.set(Idx);
1594 }
1595
1596 // Can remove shuffle iff just shuffled elements, no repeats, undefs, or
1597 // other changes.
1598 return UsedIndices.all() ? V : nullptr;
1599}
1600
1601/// Fold an unsigned minimum of trailing or leading zero bits counts:
1602/// umin(cttz(CtOp, ZeroUndef), ConstOp) --> cttz(CtOp | (1 << ConstOp))
1603/// umin(ctlz(CtOp, ZeroUndef), ConstOp) --> ctlz(CtOp | (SignedMin
1604/// >> ConstOp))
1605template <Intrinsic::ID IntrID>
1606static Value *
1608 const DataLayout &DL,
1609 InstCombiner::BuilderTy &Builder) {
1610 static_assert(IntrID == Intrinsic::cttz || IntrID == Intrinsic::ctlz,
1611 "This helper only supports cttz and ctlz intrinsics");
1612
1613 Value *CtOp;
1614 Value *ZeroUndef;
1615 if (!match(I0,
1616 m_OneUse(m_Intrinsic<IntrID>(m_Value(CtOp), m_Value(ZeroUndef)))))
1617 return nullptr;
1618
1619 unsigned BitWidth = I1->getType()->getScalarSizeInBits();
1620 auto LessBitWidth = [BitWidth](auto &C) { return C.ult(BitWidth); };
1621 if (!match(I1, m_CheckedInt(LessBitWidth)))
1622 // We have a constant >= BitWidth (which can be handled by CVP)
1623 // or a non-splat vector with elements < and >= BitWidth
1624 return nullptr;
1625
1626 Type *Ty = I1->getType();
1628 IntrID == Intrinsic::cttz ? Instruction::Shl : Instruction::LShr,
1629 IntrID == Intrinsic::cttz
1630 ? ConstantInt::get(Ty, 1)
1631 : ConstantInt::get(Ty, APInt::getSignedMinValue(BitWidth)),
1632 cast<Constant>(I1), DL);
1633 return Builder.CreateBinaryIntrinsic(
1634 IntrID, Builder.CreateOr(CtOp, NewConst),
1635 ConstantInt::getTrue(ZeroUndef->getType()));
1636}
1637
1638/// Return whether "X LOp (Y ROp Z)" is always equal to
1639/// "(X LOp Y) ROp (X LOp Z)".
1641 bool HasNSW, Intrinsic::ID ROp) {
1642 switch (ROp) {
1643 case Intrinsic::umax:
1644 case Intrinsic::umin:
1645 if (HasNUW && LOp == Instruction::Add)
1646 return true;
1647 if (HasNUW && LOp == Instruction::Shl)
1648 return true;
1649 return false;
1650 case Intrinsic::smax:
1651 case Intrinsic::smin:
1652 return HasNSW && LOp == Instruction::Add;
1653 default:
1654 return false;
1655 }
1656}
1657
1658// Attempts to factorise a common term
1659// in an instruction that has the form "(A op' B) op (C op' D)
1660// where op is an intrinsic and op' is a binop
1661static Value *
1663 InstCombiner::BuilderTy &Builder) {
1664 Value *LHS = II->getOperand(0), *RHS = II->getOperand(1);
1665 Intrinsic::ID TopLevelOpcode = II->getIntrinsicID();
1666
1669
1670 if (!Op0 || !Op1)
1671 return nullptr;
1672
1673 if (Op0->getOpcode() != Op1->getOpcode())
1674 return nullptr;
1675
1676 if (!Op0->hasOneUse() || !Op1->hasOneUse())
1677 return nullptr;
1678
1679 Instruction::BinaryOps InnerOpcode =
1680 static_cast<Instruction::BinaryOps>(Op0->getOpcode());
1681 bool HasNUW = Op0->hasNoUnsignedWrap() && Op1->hasNoUnsignedWrap();
1682 bool HasNSW = Op0->hasNoSignedWrap() && Op1->hasNoSignedWrap();
1683
1684 if (!leftDistributesOverRight(InnerOpcode, HasNUW, HasNSW, TopLevelOpcode))
1685 return nullptr;
1686
1687 Value *A = Op0->getOperand(0);
1688 Value *B = Op0->getOperand(1);
1689 Value *C = Op1->getOperand(0);
1690 Value *D = Op1->getOperand(1);
1691
1692 // Attempts to swap variables such that A equals C or B equals D,
1693 // if the inner operation is commutative.
1694 if (Op0->isCommutative() && A != C && B != D) {
1695 if (A == D || B == C)
1696 std::swap(C, D);
1697 else
1698 return nullptr;
1699 }
1700
1701 BinaryOperator *NewBinop;
1702 if (A == C) {
1703 Value *NewIntrinsic = Builder.CreateBinaryIntrinsic(TopLevelOpcode, B, D);
1704 NewBinop =
1705 cast<BinaryOperator>(Builder.CreateBinOp(InnerOpcode, A, NewIntrinsic));
1706 } else if (B == D) {
1707 Value *NewIntrinsic = Builder.CreateBinaryIntrinsic(TopLevelOpcode, A, C);
1708 NewBinop =
1709 cast<BinaryOperator>(Builder.CreateBinOp(InnerOpcode, NewIntrinsic, B));
1710 } else {
1711 return nullptr;
1712 }
1713
1714 NewBinop->setHasNoUnsignedWrap(HasNUW);
1715 NewBinop->setHasNoSignedWrap(HasNSW);
1716
1717 return NewBinop;
1718}
1719
1720/// CallInst simplification. This mostly only handles folding of intrinsic
1721/// instructions. For normal calls, it allows visitCallBase to do the heavy
1722/// lifting.
1724 // Don't try to simplify calls without uses. It will not do anything useful,
1725 // but will result in the following folds being skipped.
1726 if (!CI.use_empty()) {
1727 SmallVector<Value *, 8> Args(CI.args());
1728 if (Value *V = simplifyCall(&CI, CI.getCalledOperand(), Args,
1729 SQ.getWithInstruction(&CI)))
1730 return replaceInstUsesWith(CI, V);
1731 }
1732
1733 if (Value *FreedOp = getFreedOperand(&CI, &TLI))
1734 return visitFree(CI, FreedOp);
1735
1736 // If the caller function (i.e. us, the function that contains this CallInst)
1737 // is nounwind, mark the call as nounwind, even if the callee isn't.
1738 if (CI.getFunction()->doesNotThrow() && !CI.doesNotThrow()) {
1739 CI.setDoesNotThrow();
1740 return &CI;
1741 }
1742
1744 if (!II)
1745 return visitCallBase(CI);
1746
1747 // Intrinsics cannot occur in an invoke or a callbr, so handle them here
1748 // instead of in visitCallBase.
1749 if (auto *MI = dyn_cast<AnyMemIntrinsic>(II)) {
1750 if (auto NumBytes = MI->getLengthInBytes()) {
1751 // memmove/cpy/set of zero bytes is a noop.
1752 if (NumBytes->isZero())
1753 return eraseInstFromFunction(CI);
1754
1755 // For atomic unordered mem intrinsics if len is not a positive or
1756 // not a multiple of element size then behavior is undefined.
1757 if (MI->isAtomic() &&
1758 (NumBytes->isNegative() ||
1759 (NumBytes->getZExtValue() % MI->getElementSizeInBytes() != 0))) {
1761 assert(MI->getType()->isVoidTy() &&
1762 "non void atomic unordered mem intrinsic");
1763 return eraseInstFromFunction(*MI);
1764 }
1765 }
1766
1767 // No other transformations apply to volatile transfers.
1768 if (MI->isVolatile())
1769 return nullptr;
1770
1772 // memmove(x,x,size) -> noop.
1773 if (MTI->getSource() == MTI->getDest())
1774 return eraseInstFromFunction(CI);
1775 }
1776
1777 auto IsPointerUndefined = [MI](Value *Ptr) {
1778 return isa<ConstantPointerNull>(Ptr) &&
1780 MI->getFunction(),
1781 cast<PointerType>(Ptr->getType())->getAddressSpace());
1782 };
1783 bool SrcIsUndefined = false;
1784 // If we can determine a pointer alignment that is bigger than currently
1785 // set, update the alignment.
1786 if (auto *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
1788 return I;
1789 SrcIsUndefined = IsPointerUndefined(MTI->getRawSource());
1790 } else if (auto *MSI = dyn_cast<AnyMemSetInst>(MI)) {
1791 if (Instruction *I = SimplifyAnyMemSet(MSI))
1792 return I;
1793 }
1794
1795 // If src/dest is null, this memory intrinsic must be a noop.
1796 if (SrcIsUndefined || IsPointerUndefined(MI->getRawDest())) {
1797 Builder.CreateAssumption(Builder.CreateIsNull(MI->getLength()));
1798 return eraseInstFromFunction(CI);
1799 }
1800
1801 // If we have a memmove and the source operation is a constant global,
1802 // then the source and dest pointers can't alias, so we can change this
1803 // into a call to memcpy.
1804 if (auto *MMI = dyn_cast<AnyMemMoveInst>(MI)) {
1805 if (GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource()))
1806 if (GVSrc->isConstant()) {
1807 Module *M = CI.getModule();
1808 Intrinsic::ID MemCpyID =
1809 MMI->isAtomic()
1810 ? Intrinsic::memcpy_element_unordered_atomic
1811 : Intrinsic::memcpy;
1812 Type *Tys[3] = { CI.getArgOperand(0)->getType(),
1813 CI.getArgOperand(1)->getType(),
1814 CI.getArgOperand(2)->getType() };
1816 Intrinsic::getOrInsertDeclaration(M, MemCpyID, Tys));
1817 return II;
1818 }
1819 }
1820 }
1821
1822 // For fixed width vector result intrinsics, use the generic demanded vector
1823 // support.
1824 if (auto *IIFVTy = dyn_cast<FixedVectorType>(II->getType())) {
1825 auto VWidth = IIFVTy->getNumElements();
1826 APInt PoisonElts(VWidth, 0);
1827 APInt AllOnesEltMask(APInt::getAllOnes(VWidth));
1828 if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, PoisonElts)) {
1829 if (V != II)
1830 return replaceInstUsesWith(*II, V);
1831 return II;
1832 }
1833 }
1834
1835 if (II->isCommutative()) {
1836 if (auto Pair = matchSymmetricPair(II->getOperand(0), II->getOperand(1))) {
1837 replaceOperand(*II, 0, Pair->first);
1838 replaceOperand(*II, 1, Pair->second);
1839 return II;
1840 }
1841
1842 if (CallInst *NewCall = canonicalizeConstantArg0ToArg1(CI))
1843 return NewCall;
1844 }
1845
1846 // Unused constrained FP intrinsic calls may have declared side effect, which
1847 // prevents it from being removed. In some cases however the side effect is
1848 // actually absent. To detect this case, call SimplifyConstrainedFPCall. If it
1849 // returns a replacement, the call may be removed.
1850 if (CI.use_empty() && isa<ConstrainedFPIntrinsic>(CI)) {
1851 if (simplifyConstrainedFPCall(&CI, SQ.getWithInstruction(&CI)))
1852 return eraseInstFromFunction(CI);
1853 }
1854
1855 Intrinsic::ID IID = II->getIntrinsicID();
1856 switch (IID) {
1857 case Intrinsic::objectsize: {
1858 SmallVector<Instruction *> InsertedInstructions;
1859 if (Value *V = lowerObjectSizeCall(II, DL, &TLI, AA, /*MustSucceed=*/false,
1860 &InsertedInstructions)) {
1861 for (Instruction *Inserted : InsertedInstructions)
1862 Worklist.add(Inserted);
1863 return replaceInstUsesWith(CI, V);
1864 }
1865 return nullptr;
1866 }
1867 case Intrinsic::abs: {
1868 Value *IIOperand = II->getArgOperand(0);
1869 bool IntMinIsPoison = cast<Constant>(II->getArgOperand(1))->isOneValue();
1870
1871 // abs(-x) -> abs(x)
1872 Value *X;
1873 if (match(IIOperand, m_Neg(m_Value(X)))) {
1874 if (cast<Instruction>(IIOperand)->hasNoSignedWrap() || IntMinIsPoison)
1875 replaceOperand(*II, 1, Builder.getTrue());
1876 return replaceOperand(*II, 0, X);
1877 }
1878 if (match(IIOperand, m_c_Select(m_Neg(m_Value(X)), m_Deferred(X))))
1879 return replaceOperand(*II, 0, X);
1880
1881 Value *Y;
1882 // abs(a * abs(b)) -> abs(a * b)
1883 if (match(IIOperand,
1886 bool NSW =
1887 cast<Instruction>(IIOperand)->hasNoSignedWrap() && IntMinIsPoison;
1888 auto *XY = NSW ? Builder.CreateNSWMul(X, Y) : Builder.CreateMul(X, Y);
1889 return replaceOperand(*II, 0, XY);
1890 }
1891
1892 if (std::optional<bool> Known =
1893 getKnownSignOrZero(IIOperand, SQ.getWithInstruction(II))) {
1894 // abs(x) -> x if x >= 0 (include abs(x-y) --> x - y where x >= y)
1895 // abs(x) -> x if x > 0 (include abs(x-y) --> x - y where x > y)
1896 if (!*Known)
1897 return replaceInstUsesWith(*II, IIOperand);
1898
1899 // abs(x) -> -x if x < 0
1900 // abs(x) -> -x if x < = 0 (include abs(x-y) --> y - x where x <= y)
1901 if (IntMinIsPoison)
1902 return BinaryOperator::CreateNSWNeg(IIOperand);
1903 return BinaryOperator::CreateNeg(IIOperand);
1904 }
1905
1906 // abs (sext X) --> zext (abs X*)
1907 // Clear the IsIntMin (nsw) bit on the abs to allow narrowing.
1908 if (match(IIOperand, m_OneUse(m_SExt(m_Value(X))))) {
1909 Value *NarrowAbs =
1910 Builder.CreateBinaryIntrinsic(Intrinsic::abs, X, Builder.getFalse());
1911 return CastInst::Create(Instruction::ZExt, NarrowAbs, II->getType());
1912 }
1913
1914 // Match a complicated way to check if a number is odd/even:
1915 // abs (srem X, 2) --> and X, 1
1916 const APInt *C;
1917 if (match(IIOperand, m_SRem(m_Value(X), m_APInt(C))) && *C == 2)
1918 return BinaryOperator::CreateAnd(X, ConstantInt::get(II->getType(), 1));
1919
1920 break;
1921 }
1922 case Intrinsic::umin: {
1923 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1924 // umin(x, 1) == zext(x != 0)
1925 if (match(I1, m_One())) {
1926 assert(II->getType()->getScalarSizeInBits() != 1 &&
1927 "Expected simplify of umin with max constant");
1928 Value *Zero = Constant::getNullValue(I0->getType());
1929 Value *Cmp = Builder.CreateICmpNE(I0, Zero);
1930 return CastInst::Create(Instruction::ZExt, Cmp, II->getType());
1931 }
1932 // umin(cttz(x), const) --> cttz(x | (1 << const))
1933 if (Value *FoldedCttz =
1935 I0, I1, DL, Builder))
1936 return replaceInstUsesWith(*II, FoldedCttz);
1937 // umin(ctlz(x), const) --> ctlz(x | (SignedMin >> const))
1938 if (Value *FoldedCtlz =
1940 I0, I1, DL, Builder))
1941 return replaceInstUsesWith(*II, FoldedCtlz);
1942 [[fallthrough]];
1943 }
1944 case Intrinsic::umax: {
1945 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1946 Value *X, *Y;
1947 if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_ZExt(m_Value(Y))) &&
1948 (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
1949 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
1950 return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());
1951 }
1952 Constant *C;
1953 if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_Constant(C)) &&
1954 I0->hasOneUse()) {
1955 if (Constant *NarrowC = getLosslessUnsignedTrunc(C, X->getType(), DL)) {
1956 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);
1957 return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());
1958 }
1959 }
1960 // If C is not 0:
1961 // umax(nuw_shl(x, C), x + 1) -> x == 0 ? 1 : nuw_shl(x, C)
1962 // If C is not 0 or 1:
1963 // umax(nuw_mul(x, C), x + 1) -> x == 0 ? 1 : nuw_mul(x, C)
1964 auto foldMaxMulShift = [&](Value *A, Value *B) -> Instruction * {
1965 const APInt *C;
1966 Value *X;
1967 if (!match(A, m_NUWShl(m_Value(X), m_APInt(C))) &&
1968 !(match(A, m_NUWMul(m_Value(X), m_APInt(C))) && !C->isOne()))
1969 return nullptr;
1970 if (C->isZero())
1971 return nullptr;
1972 if (!match(B, m_OneUse(m_Add(m_Specific(X), m_One()))))
1973 return nullptr;
1974
1975 Value *Cmp = Builder.CreateICmpEQ(X, ConstantInt::get(X->getType(), 0));
1976 Value *NewSelect =
1977 Builder.CreateSelect(Cmp, ConstantInt::get(X->getType(), 1), A);
1978 return replaceInstUsesWith(*II, NewSelect);
1979 };
1980
1981 if (IID == Intrinsic::umax) {
1982 if (Instruction *I = foldMaxMulShift(I0, I1))
1983 return I;
1984 if (Instruction *I = foldMaxMulShift(I1, I0))
1985 return I;
1986 }
1987
1988 // If both operands of unsigned min/max are sign-extended, it is still ok
1989 // to narrow the operation.
1990 [[fallthrough]];
1991 }
1992 case Intrinsic::smax:
1993 case Intrinsic::smin: {
1994 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1995 Value *X, *Y;
1996 if (match(I0, m_SExt(m_Value(X))) && match(I1, m_SExt(m_Value(Y))) &&
1997 (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
1998 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
1999 return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());
2000 }
2001
2002 Constant *C;
2003 if (match(I0, m_SExt(m_Value(X))) && match(I1, m_Constant(C)) &&
2004 I0->hasOneUse()) {
2005 if (Constant *NarrowC = getLosslessSignedTrunc(C, X->getType(), DL)) {
2006 Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);
2007 return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());
2008 }
2009 }
2010
2011 // smax(smin(X, MinC), MaxC) -> smin(smax(X, MaxC), MinC) if MinC s>= MaxC
2012 // umax(umin(X, MinC), MaxC) -> umin(umax(X, MaxC), MinC) if MinC u>= MaxC
2013 const APInt *MinC, *MaxC;
2014 auto CreateCanonicalClampForm = [&](bool IsSigned) {
2015 auto MaxIID = IsSigned ? Intrinsic::smax : Intrinsic::umax;
2016 auto MinIID = IsSigned ? Intrinsic::smin : Intrinsic::umin;
2017 Value *NewMax = Builder.CreateBinaryIntrinsic(
2018 MaxIID, X, ConstantInt::get(X->getType(), *MaxC));
2019 return replaceInstUsesWith(
2020 *II, Builder.CreateBinaryIntrinsic(
2021 MinIID, NewMax, ConstantInt::get(X->getType(), *MinC)));
2022 };
2023 if (IID == Intrinsic::smax &&
2025 m_APInt(MinC)))) &&
2026 match(I1, m_APInt(MaxC)) && MinC->sgt(*MaxC))
2027 return CreateCanonicalClampForm(true);
2028 if (IID == Intrinsic::umax &&
2030 m_APInt(MinC)))) &&
2031 match(I1, m_APInt(MaxC)) && MinC->ugt(*MaxC))
2032 return CreateCanonicalClampForm(false);
2033
2034 // umin(i1 X, i1 Y) -> and i1 X, Y
2035 // smax(i1 X, i1 Y) -> and i1 X, Y
2036 if ((IID == Intrinsic::umin || IID == Intrinsic::smax) &&
2037 II->getType()->isIntOrIntVectorTy(1)) {
2038 return BinaryOperator::CreateAnd(I0, I1);
2039 }
2040
2041 // umax(i1 X, i1 Y) -> or i1 X, Y
2042 // smin(i1 X, i1 Y) -> or i1 X, Y
2043 if ((IID == Intrinsic::umax || IID == Intrinsic::smin) &&
2044 II->getType()->isIntOrIntVectorTy(1)) {
2045 return BinaryOperator::CreateOr(I0, I1);
2046 }
2047
2048 // smin(smax(X, -1), 1) -> scmp(X, 0)
2049 // smax(smin(X, 1), -1) -> scmp(X, 0)
2050 // At this point, smax(smin(X, 1), -1) is changed to smin(smax(X, -1)
2051 // And i1's have been changed to and/ors
2052 // So we only need to check for smin
2053 if (IID == Intrinsic::smin) {
2054 if (match(I0, m_OneUse(m_SMax(m_Value(X), m_AllOnes()))) &&
2055 match(I1, m_One())) {
2056 Value *Zero = ConstantInt::get(X->getType(), 0);
2057 return replaceInstUsesWith(
2058 CI,
2059 Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {X, Zero}));
2060 }
2061 }
2062
2063 if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
2064 // smax (neg nsw X), (neg nsw Y) --> neg nsw (smin X, Y)
2065 // smin (neg nsw X), (neg nsw Y) --> neg nsw (smax X, Y)
2066 // TODO: Canonicalize neg after min/max if I1 is constant.
2067 if (match(I0, m_NSWNeg(m_Value(X))) && match(I1, m_NSWNeg(m_Value(Y))) &&
2068 (I0->hasOneUse() || I1->hasOneUse())) {
2070 Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, Y);
2071 return BinaryOperator::CreateNSWNeg(InvMaxMin);
2072 }
2073 }
2074
2075 // (umax X, (xor X, Pow2))
2076 // -> (or X, Pow2)
2077 // (umin X, (xor X, Pow2))
2078 // -> (and X, ~Pow2)
2079 // (smax X, (xor X, Pos_Pow2))
2080 // -> (or X, Pos_Pow2)
2081 // (smin X, (xor X, Pos_Pow2))
2082 // -> (and X, ~Pos_Pow2)
2083 // (smax X, (xor X, Neg_Pow2))
2084 // -> (and X, ~Neg_Pow2)
2085 // (smin X, (xor X, Neg_Pow2))
2086 // -> (or X, Neg_Pow2)
2087 if ((match(I0, m_c_Xor(m_Specific(I1), m_Value(X))) ||
2088 match(I1, m_c_Xor(m_Specific(I0), m_Value(X)))) &&
2089 isKnownToBeAPowerOfTwo(X, /* OrZero */ true)) {
2090 bool UseOr = IID == Intrinsic::smax || IID == Intrinsic::umax;
2091 bool UseAndN = IID == Intrinsic::smin || IID == Intrinsic::umin;
2092
2093 if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
2094 auto KnownSign = getKnownSign(X, SQ.getWithInstruction(II));
2095 if (KnownSign == std::nullopt) {
2096 UseOr = false;
2097 UseAndN = false;
2098 } else if (*KnownSign /* true is Signed. */) {
2099 UseOr ^= true;
2100 UseAndN ^= true;
2101 Type *Ty = I0->getType();
2102 // Negative power of 2 must be IntMin. It's possible to be able to
2103 // prove negative / power of 2 without actually having known bits, so
2104 // just get the value by hand.
2106 Ty, APInt::getSignedMinValue(Ty->getScalarSizeInBits()));
2107 }
2108 }
2109 if (UseOr)
2110 return BinaryOperator::CreateOr(I0, X);
2111 else if (UseAndN)
2112 return BinaryOperator::CreateAnd(I0, Builder.CreateNot(X));
2113 }
2114
2115 // If we can eliminate ~A and Y is free to invert:
2116 // max ~A, Y --> ~(min A, ~Y)
2117 //
2118 // Examples:
2119 // max ~A, ~Y --> ~(min A, Y)
2120 // max ~A, C --> ~(min A, ~C)
2121 // max ~A, (max ~Y, ~Z) --> ~min( A, (min Y, Z))
2122 auto moveNotAfterMinMax = [&](Value *X, Value *Y) -> Instruction * {
2123 Value *A;
2124 if (match(X, m_OneUse(m_Not(m_Value(A)))) &&
2125 !isFreeToInvert(A, A->hasOneUse())) {
2126 if (Value *NotY = getFreelyInverted(Y, Y->hasOneUse(), &Builder)) {
2128 Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, A, NotY);
2129 return BinaryOperator::CreateNot(InvMaxMin);
2130 }
2131 }
2132 return nullptr;
2133 };
2134
2135 if (Instruction *I = moveNotAfterMinMax(I0, I1))
2136 return I;
2137 if (Instruction *I = moveNotAfterMinMax(I1, I0))
2138 return I;
2139
2141 return I;
2142
2143 // minmax (X & NegPow2C, Y & NegPow2C) --> minmax(X, Y) & NegPow2C
2144 const APInt *RHSC;
2145 if (match(I0, m_OneUse(m_And(m_Value(X), m_NegatedPower2(RHSC)))) &&
2146 match(I1, m_OneUse(m_And(m_Value(Y), m_SpecificInt(*RHSC)))))
2147 return BinaryOperator::CreateAnd(Builder.CreateBinaryIntrinsic(IID, X, Y),
2148 ConstantInt::get(II->getType(), *RHSC));
2149
2150 // smax(X, -X) --> abs(X)
2151 // smin(X, -X) --> -abs(X)
2152 // umax(X, -X) --> -abs(X)
2153 // umin(X, -X) --> abs(X)
2154 if (isKnownNegation(I0, I1)) {
2155 // We can choose either operand as the input to abs(), but if we can
2156 // eliminate the only use of a value, that's better for subsequent
2157 // transforms/analysis.
2158 if (I0->hasOneUse() && !I1->hasOneUse())
2159 std::swap(I0, I1);
2160
2161 // This is some variant of abs(). See if we can propagate 'nsw' to the abs
2162 // operation and potentially its negation.
2163 bool IntMinIsPoison = isKnownNegation(I0, I1, /* NeedNSW */ true);
2164 Value *Abs = Builder.CreateBinaryIntrinsic(
2165 Intrinsic::abs, I0,
2166 ConstantInt::getBool(II->getContext(), IntMinIsPoison));
2167
2168 // We don't have a "nabs" intrinsic, so negate if needed based on the
2169 // max/min operation.
2170 if (IID == Intrinsic::smin || IID == Intrinsic::umax)
2171 Abs = Builder.CreateNeg(Abs, "nabs", IntMinIsPoison);
2172 return replaceInstUsesWith(CI, Abs);
2173 }
2174
2176 return Sel;
2177
2178 if (Instruction *SAdd = matchSAddSubSat(*II))
2179 return SAdd;
2180
2181 if (Value *NewMinMax = reassociateMinMaxWithConstants(II, Builder, SQ))
2182 return replaceInstUsesWith(*II, NewMinMax);
2183
2185 return R;
2186
2187 if (Instruction *NewMinMax = factorizeMinMaxTree(II))
2188 return NewMinMax;
2189
2190 // Try to fold minmax with constant RHS based on range information
2191 if (match(I1, m_APIntAllowPoison(RHSC))) {
2192 ICmpInst::Predicate Pred =
2194 bool IsSigned = MinMaxIntrinsic::isSigned(IID);
2196 I0, IsSigned, SQ.getWithInstruction(II));
2197 if (!LHS_CR.isFullSet()) {
2198 if (LHS_CR.icmp(Pred, *RHSC))
2199 return replaceInstUsesWith(*II, I0);
2200 if (LHS_CR.icmp(ICmpInst::getSwappedPredicate(Pred), *RHSC))
2201 return replaceInstUsesWith(*II,
2202 ConstantInt::get(II->getType(), *RHSC));
2203 }
2204 }
2205
2207 return replaceInstUsesWith(*II, V);
2208
2209 break;
2210 }
2211 case Intrinsic::scmp: {
2212 Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
2213 Value *LHS, *RHS;
2214 if (match(I0, m_NSWSub(m_Value(LHS), m_Value(RHS))) && match(I1, m_Zero()))
2215 return replaceInstUsesWith(
2216 CI,
2217 Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {LHS, RHS}));
2218 break;
2219 }
2220 case Intrinsic::bitreverse: {
2221 Value *IIOperand = II->getArgOperand(0);
2222 // bitrev (zext i1 X to ?) --> X ? SignBitC : 0
2223 Value *X;
2224 if (match(IIOperand, m_ZExt(m_Value(X))) &&
2225 X->getType()->isIntOrIntVectorTy(1)) {
2226 Type *Ty = II->getType();
2227 APInt SignBit = APInt::getSignMask(Ty->getScalarSizeInBits());
2228 return SelectInst::Create(X, ConstantInt::get(Ty, SignBit),
2230 }
2231
2232 if (Instruction *crossLogicOpFold =
2234 return crossLogicOpFold;
2235
2236 break;
2237 }
2238 case Intrinsic::bswap: {
2239 Value *IIOperand = II->getArgOperand(0);
2240
2241 // Try to canonicalize bswap-of-logical-shift-by-8-bit-multiple as
2242 // inverse-shift-of-bswap:
2243 // bswap (shl X, Y) --> lshr (bswap X), Y
2244 // bswap (lshr X, Y) --> shl (bswap X), Y
2245 Value *X, *Y;
2246 if (match(IIOperand, m_OneUse(m_LogicalShift(m_Value(X), m_Value(Y))))) {
2247 unsigned BitWidth = IIOperand->getType()->getScalarSizeInBits();
2249 Value *NewSwap = Builder.CreateUnaryIntrinsic(Intrinsic::bswap, X);
2250 BinaryOperator::BinaryOps InverseShift =
2251 cast<BinaryOperator>(IIOperand)->getOpcode() == Instruction::Shl
2252 ? Instruction::LShr
2253 : Instruction::Shl;
2254 return BinaryOperator::Create(InverseShift, NewSwap, Y);
2255 }
2256 }
2257
2258 KnownBits Known = computeKnownBits(IIOperand, II);
2259 uint64_t LZ = alignDown(Known.countMinLeadingZeros(), 8);
2260 uint64_t TZ = alignDown(Known.countMinTrailingZeros(), 8);
2261 unsigned BW = Known.getBitWidth();
2262
2263 // bswap(x) -> shift(x) if x has exactly one "active byte"
2264 if (BW - LZ - TZ == 8) {
2265 assert(LZ != TZ && "active byte cannot be in the middle");
2266 if (LZ > TZ) // -> shl(x) if the "active byte" is in the low part of x
2267 return BinaryOperator::CreateNUWShl(
2268 IIOperand, ConstantInt::get(IIOperand->getType(), LZ - TZ));
2269 // -> lshr(x) if the "active byte" is in the high part of x
2270 return BinaryOperator::CreateExactLShr(
2271 IIOperand, ConstantInt::get(IIOperand->getType(), TZ - LZ));
2272 }
2273
2274 // bswap(trunc(bswap(x))) -> trunc(lshr(x, c))
2275 if (match(IIOperand, m_Trunc(m_BSwap(m_Value(X))))) {
2276 unsigned C = X->getType()->getScalarSizeInBits() - BW;
2277 Value *CV = ConstantInt::get(X->getType(), C);
2278 Value *V = Builder.CreateLShr(X, CV);
2279 return new TruncInst(V, IIOperand->getType());
2280 }
2281
2282 if (Instruction *crossLogicOpFold =
2284 return crossLogicOpFold;
2285 }
2286
2287 // Try to fold into bitreverse if bswap is the root of the expression tree.
2288 if (Instruction *BitOp = matchBSwapOrBitReverse(*II, /*MatchBSwaps*/ false,
2289 /*MatchBitReversals*/ true))
2290 return BitOp;
2291 break;
2292 }
2293 case Intrinsic::masked_load:
2294 if (Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II))
2295 return replaceInstUsesWith(CI, SimplifiedMaskedOp);
2296 break;
2297 case Intrinsic::masked_store:
2298 return simplifyMaskedStore(*II);
2299 case Intrinsic::masked_gather:
2300 return simplifyMaskedGather(*II);
2301 case Intrinsic::masked_scatter:
2302 return simplifyMaskedScatter(*II);
2303 case Intrinsic::launder_invariant_group:
2304 case Intrinsic::strip_invariant_group:
2305 if (auto *SkippedBarrier = simplifyInvariantGroupIntrinsic(*II, *this))
2306 return replaceInstUsesWith(*II, SkippedBarrier);
2307 break;
2308 case Intrinsic::powi:
2309 if (ConstantInt *Power = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
2310 // 0 and 1 are handled in instsimplify
2311 // powi(x, -1) -> 1/x
2312 if (Power->isMinusOne())
2313 return BinaryOperator::CreateFDivFMF(ConstantFP::get(CI.getType(), 1.0),
2314 II->getArgOperand(0), II);
2315 // powi(x, 2) -> x*x
2316 if (Power->equalsInt(2))
2317 return BinaryOperator::CreateFMulFMF(II->getArgOperand(0),
2318 II->getArgOperand(0), II);
2319
2320 if (!Power->getValue()[0]) {
2321 Value *X;
2322 // If power is even:
2323 // powi(-x, p) -> powi(x, p)
2324 // powi(fabs(x), p) -> powi(x, p)
2325 // powi(copysign(x, y), p) -> powi(x, p)
2326 if (match(II->getArgOperand(0), m_FNeg(m_Value(X))) ||
2327 match(II->getArgOperand(0), m_FAbs(m_Value(X))) ||
2328 match(II->getArgOperand(0),
2330 return replaceOperand(*II, 0, X);
2331 }
2332 }
2333 break;
2334
2335 case Intrinsic::cttz:
2336 case Intrinsic::ctlz:
2337 if (auto *I = foldCttzCtlz(*II, *this))
2338 return I;
2339 break;
2340
2341 case Intrinsic::ctpop:
2342 if (auto *I = foldCtpop(*II, *this))
2343 return I;
2344 break;
2345
2346 case Intrinsic::fshl:
2347 case Intrinsic::fshr: {
2348 Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);
2349 Type *Ty = II->getType();
2350 unsigned BitWidth = Ty->getScalarSizeInBits();
2351 Constant *ShAmtC;
2352 if (match(II->getArgOperand(2), m_ImmConstant(ShAmtC))) {
2353 // Canonicalize a shift amount constant operand to modulo the bit-width.
2354 Constant *WidthC = ConstantInt::get(Ty, BitWidth);
2355 Constant *ModuloC =
2356 ConstantFoldBinaryOpOperands(Instruction::URem, ShAmtC, WidthC, DL);
2357 if (!ModuloC)
2358 return nullptr;
2359 if (ModuloC != ShAmtC)
2360 return replaceOperand(*II, 2, ModuloC);
2361
2363 ShAmtC, DL),
2364 m_One()) &&
2365 "Shift amount expected to be modulo bitwidth");
2366
2367 // Canonicalize funnel shift right by constant to funnel shift left. This
2368 // is not entirely arbitrary. For historical reasons, the backend may
2369 // recognize rotate left patterns but miss rotate right patterns.
2370 if (IID == Intrinsic::fshr) {
2371 // fshr X, Y, C --> fshl X, Y, (BitWidth - C) if C is not zero.
2372 if (!isKnownNonZero(ShAmtC, SQ.getWithInstruction(II)))
2373 return nullptr;
2374
2375 Constant *LeftShiftC = ConstantExpr::getSub(WidthC, ShAmtC);
2376 Module *Mod = II->getModule();
2377 Function *Fshl =
2378 Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::fshl, Ty);
2379 return CallInst::Create(Fshl, { Op0, Op1, LeftShiftC });
2380 }
2381 assert(IID == Intrinsic::fshl &&
2382 "All funnel shifts by simple constants should go left");
2383
2384 // fshl(X, 0, C) --> shl X, C
2385 // fshl(X, undef, C) --> shl X, C
2386 if (match(Op1, m_ZeroInt()) || match(Op1, m_Undef()))
2387 return BinaryOperator::CreateShl(Op0, ShAmtC);
2388
2389 // fshl(0, X, C) --> lshr X, (BW-C)
2390 // fshl(undef, X, C) --> lshr X, (BW-C)
2391 if (match(Op0, m_ZeroInt()) || match(Op0, m_Undef()))
2392 return BinaryOperator::CreateLShr(Op1,
2393 ConstantExpr::getSub(WidthC, ShAmtC));
2394
2395 // fshl i16 X, X, 8 --> bswap i16 X (reduce to more-specific form)
2396 if (Op0 == Op1 && BitWidth == 16 && match(ShAmtC, m_SpecificInt(8))) {
2397 Module *Mod = II->getModule();
2398 Function *Bswap =
2399 Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::bswap, Ty);
2400 return CallInst::Create(Bswap, { Op0 });
2401 }
2402 if (Instruction *BitOp =
2403 matchBSwapOrBitReverse(*II, /*MatchBSwaps*/ true,
2404 /*MatchBitReversals*/ true))
2405 return BitOp;
2406
2407 // R = fshl(X, X, C2)
2408 // fshl(R, R, C1) --> fshl(X, X, (C1 + C2) % bitsize)
2409 Value *InnerOp;
2410 const APInt *ShAmtInnerC, *ShAmtOuterC;
2411 if (match(Op0, m_FShl(m_Value(InnerOp), m_Deferred(InnerOp),
2412 m_APInt(ShAmtInnerC))) &&
2413 match(ShAmtC, m_APInt(ShAmtOuterC)) && Op0 == Op1) {
2414 APInt Sum = *ShAmtOuterC + *ShAmtInnerC;
2415 APInt Modulo = Sum.urem(APInt(Sum.getBitWidth(), BitWidth));
2416 if (Modulo.isZero())
2417 return replaceInstUsesWith(*II, InnerOp);
2418 Constant *ModuloC = ConstantInt::get(Ty, Modulo);
2420 {InnerOp, InnerOp, ModuloC});
2421 }
2422 }
2423
2424 // fshl(X, X, Neg(Y)) --> fshr(X, X, Y)
2425 // fshr(X, X, Neg(Y)) --> fshl(X, X, Y)
2426 // if BitWidth is a power-of-2
2427 Value *Y;
2428 if (Op0 == Op1 && isPowerOf2_32(BitWidth) &&
2429 match(II->getArgOperand(2), m_Neg(m_Value(Y)))) {
2430 Module *Mod = II->getModule();
2432 Mod, IID == Intrinsic::fshl ? Intrinsic::fshr : Intrinsic::fshl, Ty);
2433 return CallInst::Create(OppositeShift, {Op0, Op1, Y});
2434 }
2435
2436 // fshl(X, 0, Y) --> shl(X, and(Y, BitWidth - 1)) if bitwidth is a
2437 // power-of-2
2438 if (IID == Intrinsic::fshl && isPowerOf2_32(BitWidth) &&
2439 match(Op1, m_ZeroInt())) {
2440 Value *Op2 = II->getArgOperand(2);
2441 Value *And = Builder.CreateAnd(Op2, ConstantInt::get(Ty, BitWidth - 1));
2442 return BinaryOperator::CreateShl(Op0, And);
2443 }
2444
2445 // Left or right might be masked.
2447 return &CI;
2448
2449 // The shift amount (operand 2) of a funnel shift is modulo the bitwidth,
2450 // so only the low bits of the shift amount are demanded if the bitwidth is
2451 // a power-of-2.
2452 if (!isPowerOf2_32(BitWidth))
2453 break;
2455 KnownBits Op2Known(BitWidth);
2456 if (SimplifyDemandedBits(II, 2, Op2Demanded, Op2Known))
2457 return &CI;
2458 break;
2459 }
2460 case Intrinsic::ptrmask: {
2461 unsigned BitWidth = DL.getPointerTypeSizeInBits(II->getType());
2462 KnownBits Known(BitWidth);
2464 return II;
2465
2466 Value *InnerPtr, *InnerMask;
2467 bool Changed = false;
2468 // Combine:
2469 // (ptrmask (ptrmask p, A), B)
2470 // -> (ptrmask p, (and A, B))
2471 if (match(II->getArgOperand(0),
2473 m_Value(InnerMask))))) {
2474 assert(II->getArgOperand(1)->getType() == InnerMask->getType() &&
2475 "Mask types must match");
2476 // TODO: If InnerMask == Op1, we could copy attributes from inner
2477 // callsite -> outer callsite.
2478 Value *NewMask = Builder.CreateAnd(II->getArgOperand(1), InnerMask);
2479 replaceOperand(CI, 0, InnerPtr);
2480 replaceOperand(CI, 1, NewMask);
2481 Changed = true;
2482 }
2483
2484 // See if we can deduce non-null.
2485 if (!CI.hasRetAttr(Attribute::NonNull) &&
2486 (Known.isNonZero() ||
2487 isKnownNonZero(II, getSimplifyQuery().getWithInstruction(II)))) {
2488 CI.addRetAttr(Attribute::NonNull);
2489 Changed = true;
2490 }
2491
2492 unsigned NewAlignmentLog =
2494 std::min(BitWidth - 1, Known.countMinTrailingZeros()));
2495 // Known bits will capture if we had alignment information associated with
2496 // the pointer argument.
2497 if (NewAlignmentLog > Log2(CI.getRetAlign().valueOrOne())) {
2499 CI.getContext(), Align(uint64_t(1) << NewAlignmentLog)));
2500 Changed = true;
2501 }
2502 if (Changed)
2503 return &CI;
2504 break;
2505 }
2506 case Intrinsic::uadd_with_overflow:
2507 case Intrinsic::sadd_with_overflow: {
2508 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2509 return I;
2510
2511 // Given 2 constant operands whose sum does not overflow:
2512 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
2513 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
2514 Value *X;
2515 const APInt *C0, *C1;
2516 Value *Arg0 = II->getArgOperand(0);
2517 Value *Arg1 = II->getArgOperand(1);
2518 bool IsSigned = IID == Intrinsic::sadd_with_overflow;
2519 bool HasNWAdd = IsSigned
2520 ? match(Arg0, m_NSWAddLike(m_Value(X), m_APInt(C0)))
2521 : match(Arg0, m_NUWAddLike(m_Value(X), m_APInt(C0)));
2522 if (HasNWAdd && match(Arg1, m_APInt(C1))) {
2523 bool Overflow;
2524 APInt NewC =
2525 IsSigned ? C1->sadd_ov(*C0, Overflow) : C1->uadd_ov(*C0, Overflow);
2526 if (!Overflow)
2527 return replaceInstUsesWith(
2528 *II, Builder.CreateBinaryIntrinsic(
2529 IID, X, ConstantInt::get(Arg1->getType(), NewC)));
2530 }
2531 break;
2532 }
2533
2534 case Intrinsic::umul_with_overflow:
2535 case Intrinsic::smul_with_overflow:
2536 case Intrinsic::usub_with_overflow:
2537 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2538 return I;
2539 break;
2540
2541 case Intrinsic::ssub_with_overflow: {
2542 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2543 return I;
2544
2545 Constant *C;
2546 Value *Arg0 = II->getArgOperand(0);
2547 Value *Arg1 = II->getArgOperand(1);
2548 // Given a constant C that is not the minimum signed value
2549 // for an integer of a given bit width:
2550 //
2551 // ssubo X, C -> saddo X, -C
2552 if (match(Arg1, m_Constant(C)) && C->isNotMinSignedValue()) {
2553 Value *NegVal = ConstantExpr::getNeg(C);
2554 // Build a saddo call that is equivalent to the discovered
2555 // ssubo call.
2556 return replaceInstUsesWith(
2557 *II, Builder.CreateBinaryIntrinsic(Intrinsic::sadd_with_overflow,
2558 Arg0, NegVal));
2559 }
2560
2561 break;
2562 }
2563
2564 case Intrinsic::uadd_sat:
2565 case Intrinsic::sadd_sat:
2566 case Intrinsic::usub_sat:
2567 case Intrinsic::ssub_sat: {
2569 Type *Ty = SI->getType();
2570 Value *Arg0 = SI->getLHS();
2571 Value *Arg1 = SI->getRHS();
2572
2573 // Make use of known overflow information.
2574 OverflowResult OR = computeOverflow(SI->getBinaryOp(), SI->isSigned(),
2575 Arg0, Arg1, SI);
2576 switch (OR) {
2578 break;
2580 if (SI->isSigned())
2581 return BinaryOperator::CreateNSW(SI->getBinaryOp(), Arg0, Arg1);
2582 else
2583 return BinaryOperator::CreateNUW(SI->getBinaryOp(), Arg0, Arg1);
2585 unsigned BitWidth = Ty->getScalarSizeInBits();
2586 APInt Min = APSInt::getMinValue(BitWidth, !SI->isSigned());
2587 return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Min));
2588 }
2590 unsigned BitWidth = Ty->getScalarSizeInBits();
2591 APInt Max = APSInt::getMaxValue(BitWidth, !SI->isSigned());
2592 return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Max));
2593 }
2594 }
2595
2596 // usub_sat((sub nuw C, A), C1) -> usub_sat(usub_sat(C, C1), A)
2597 // which after that:
2598 // usub_sat((sub nuw C, A), C1) -> usub_sat(C - C1, A) if C1 u< C
2599 // usub_sat((sub nuw C, A), C1) -> 0 otherwise
2600 Constant *C, *C1;
2601 Value *A;
2602 if (IID == Intrinsic::usub_sat &&
2603 match(Arg0, m_NUWSub(m_ImmConstant(C), m_Value(A))) &&
2604 match(Arg1, m_ImmConstant(C1))) {
2605 auto *NewC = Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, C, C1);
2606 auto *NewSub =
2607 Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, NewC, A);
2608 return replaceInstUsesWith(*SI, NewSub);
2609 }
2610
2611 // ssub.sat(X, C) -> sadd.sat(X, -C) if C != MIN
2612 if (IID == Intrinsic::ssub_sat && match(Arg1, m_Constant(C)) &&
2613 C->isNotMinSignedValue()) {
2614 Value *NegVal = ConstantExpr::getNeg(C);
2615 return replaceInstUsesWith(
2616 *II, Builder.CreateBinaryIntrinsic(
2617 Intrinsic::sadd_sat, Arg0, NegVal));
2618 }
2619
2620 // sat(sat(X + Val2) + Val) -> sat(X + (Val+Val2))
2621 // sat(sat(X - Val2) - Val) -> sat(X - (Val+Val2))
2622 // if Val and Val2 have the same sign
2623 if (auto *Other = dyn_cast<IntrinsicInst>(Arg0)) {
2624 Value *X;
2625 const APInt *Val, *Val2;
2626 APInt NewVal;
2627 bool IsUnsigned =
2628 IID == Intrinsic::uadd_sat || IID == Intrinsic::usub_sat;
2629 if (Other->getIntrinsicID() == IID &&
2630 match(Arg1, m_APInt(Val)) &&
2631 match(Other->getArgOperand(0), m_Value(X)) &&
2632 match(Other->getArgOperand(1), m_APInt(Val2))) {
2633 if (IsUnsigned)
2634 NewVal = Val->uadd_sat(*Val2);
2635 else if (Val->isNonNegative() == Val2->isNonNegative()) {
2636 bool Overflow;
2637 NewVal = Val->sadd_ov(*Val2, Overflow);
2638 if (Overflow) {
2639 // Both adds together may add more than SignedMaxValue
2640 // without saturating the final result.
2641 break;
2642 }
2643 } else {
2644 // Cannot fold saturated addition with different signs.
2645 break;
2646 }
2647
2648 return replaceInstUsesWith(
2649 *II, Builder.CreateBinaryIntrinsic(
2650 IID, X, ConstantInt::get(II->getType(), NewVal)));
2651 }
2652 }
2653 break;
2654 }
2655
2656 case Intrinsic::minnum:
2657 case Intrinsic::maxnum:
2658 case Intrinsic::minimum:
2659 case Intrinsic::maximum: {
2660 Value *Arg0 = II->getArgOperand(0);
2661 Value *Arg1 = II->getArgOperand(1);
2662 Value *X, *Y;
2663 if (match(Arg0, m_FNeg(m_Value(X))) && match(Arg1, m_FNeg(m_Value(Y))) &&
2664 (Arg0->hasOneUse() || Arg1->hasOneUse())) {
2665 // If both operands are negated, invert the call and negate the result:
2666 // min(-X, -Y) --> -(max(X, Y))
2667 // max(-X, -Y) --> -(min(X, Y))
2668 Intrinsic::ID NewIID;
2669 switch (IID) {
2670 case Intrinsic::maxnum:
2671 NewIID = Intrinsic::minnum;
2672 break;
2673 case Intrinsic::minnum:
2674 NewIID = Intrinsic::maxnum;
2675 break;
2676 case Intrinsic::maximum:
2677 NewIID = Intrinsic::minimum;
2678 break;
2679 case Intrinsic::minimum:
2680 NewIID = Intrinsic::maximum;
2681 break;
2682 default:
2683 llvm_unreachable("unexpected intrinsic ID");
2684 }
2685 Value *NewCall = Builder.CreateBinaryIntrinsic(NewIID, X, Y, II);
2686 Instruction *FNeg = UnaryOperator::CreateFNeg(NewCall);
2687 FNeg->copyIRFlags(II);
2688 return FNeg;
2689 }
2690
2691 // m(m(X, C2), C1) -> m(X, C)
2692 const APFloat *C1, *C2;
2693 if (auto *M = dyn_cast<IntrinsicInst>(Arg0)) {
2694 if (M->getIntrinsicID() == IID && match(Arg1, m_APFloat(C1)) &&
2695 ((match(M->getArgOperand(0), m_Value(X)) &&
2696 match(M->getArgOperand(1), m_APFloat(C2))) ||
2697 (match(M->getArgOperand(1), m_Value(X)) &&
2698 match(M->getArgOperand(0), m_APFloat(C2))))) {
2699 APFloat Res(0.0);
2700 switch (IID) {
2701 case Intrinsic::maxnum:
2702 Res = maxnum(*C1, *C2);
2703 break;
2704 case Intrinsic::minnum:
2705 Res = minnum(*C1, *C2);
2706 break;
2707 case Intrinsic::maximum:
2708 Res = maximum(*C1, *C2);
2709 break;
2710 case Intrinsic::minimum:
2711 Res = minimum(*C1, *C2);
2712 break;
2713 default:
2714 llvm_unreachable("unexpected intrinsic ID");
2715 }
2716 // TODO: Conservatively intersecting FMF. If Res == C2, the transform
2717 // was a simplification (so Arg0 and its original flags could
2718 // propagate?)
2719 Value *V = Builder.CreateBinaryIntrinsic(
2720 IID, X, ConstantFP::get(Arg0->getType(), Res),
2722 return replaceInstUsesWith(*II, V);
2723 }
2724 }
2725
2726 // m((fpext X), (fpext Y)) -> fpext (m(X, Y))
2727 if (match(Arg0, m_OneUse(m_FPExt(m_Value(X)))) &&
2728 match(Arg1, m_OneUse(m_FPExt(m_Value(Y)))) &&
2729 X->getType() == Y->getType()) {
2730 Value *NewCall =
2731 Builder.CreateBinaryIntrinsic(IID, X, Y, II, II->getName());
2732 return new FPExtInst(NewCall, II->getType());
2733 }
2734
2735 // max X, -X --> fabs X
2736 // min X, -X --> -(fabs X)
2737 // TODO: Remove one-use limitation? That is obviously better for max,
2738 // hence why we don't check for one-use for that. However,
2739 // it would be an extra instruction for min (fnabs), but
2740 // that is still likely better for analysis and codegen.
2741 auto IsMinMaxOrXNegX = [IID, &X](Value *Op0, Value *Op1) {
2742 if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_Specific(X)))
2743 return Op0->hasOneUse() ||
2744 (IID != Intrinsic::minimum && IID != Intrinsic::minnum);
2745 return false;
2746 };
2747
2748 if (IsMinMaxOrXNegX(Arg0, Arg1) || IsMinMaxOrXNegX(Arg1, Arg0)) {
2749 Value *R = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II);
2750 if (IID == Intrinsic::minimum || IID == Intrinsic::minnum)
2751 R = Builder.CreateFNegFMF(R, II);
2752 return replaceInstUsesWith(*II, R);
2753 }
2754
2755 break;
2756 }
2757 case Intrinsic::matrix_multiply: {
2758 // Optimize negation in matrix multiplication.
2759
2760 // -A * -B -> A * B
2761 Value *A, *B;
2762 if (match(II->getArgOperand(0), m_FNeg(m_Value(A))) &&
2763 match(II->getArgOperand(1), m_FNeg(m_Value(B)))) {
2764 replaceOperand(*II, 0, A);
2765 replaceOperand(*II, 1, B);
2766 return II;
2767 }
2768
2769 Value *Op0 = II->getOperand(0);
2770 Value *Op1 = II->getOperand(1);
2771 Value *OpNotNeg, *NegatedOp;
2772 unsigned NegatedOpArg, OtherOpArg;
2773 if (match(Op0, m_FNeg(m_Value(OpNotNeg)))) {
2774 NegatedOp = Op0;
2775 NegatedOpArg = 0;
2776 OtherOpArg = 1;
2777 } else if (match(Op1, m_FNeg(m_Value(OpNotNeg)))) {
2778 NegatedOp = Op1;
2779 NegatedOpArg = 1;
2780 OtherOpArg = 0;
2781 } else
2782 // Multiplication doesn't have a negated operand.
2783 break;
2784
2785 // Only optimize if the negated operand has only one use.
2786 if (!NegatedOp->hasOneUse())
2787 break;
2788
2789 Value *OtherOp = II->getOperand(OtherOpArg);
2790 VectorType *RetTy = cast<VectorType>(II->getType());
2791 VectorType *NegatedOpTy = cast<VectorType>(NegatedOp->getType());
2792 VectorType *OtherOpTy = cast<VectorType>(OtherOp->getType());
2793 ElementCount NegatedCount = NegatedOpTy->getElementCount();
2794 ElementCount OtherCount = OtherOpTy->getElementCount();
2795 ElementCount RetCount = RetTy->getElementCount();
2796 // (-A) * B -> A * (-B), if it is cheaper to negate B and vice versa.
2797 if (ElementCount::isKnownGT(NegatedCount, OtherCount) &&
2798 ElementCount::isKnownLT(OtherCount, RetCount)) {
2799 Value *InverseOtherOp = Builder.CreateFNeg(OtherOp);
2800 replaceOperand(*II, NegatedOpArg, OpNotNeg);
2801 replaceOperand(*II, OtherOpArg, InverseOtherOp);
2802 return II;
2803 }
2804 // (-A) * B -> -(A * B), if it is cheaper to negate the result
2805 if (ElementCount::isKnownGT(NegatedCount, RetCount)) {
2806 SmallVector<Value *, 5> NewArgs(II->args());
2807 NewArgs[NegatedOpArg] = OpNotNeg;
2808 Instruction *NewMul =
2809 Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);
2810 return replaceInstUsesWith(*II, Builder.CreateFNegFMF(NewMul, II));
2811 }
2812 break;
2813 }
2814 case Intrinsic::fmuladd: {
2815 // Try to simplify the underlying FMul.
2816 if (Value *V =
2817 simplifyFMulInst(II->getArgOperand(0), II->getArgOperand(1),
2818 II->getFastMathFlags(), SQ.getWithInstruction(II)))
2819 return BinaryOperator::CreateFAddFMF(V, II->getArgOperand(2),
2820 II->getFastMathFlags());
2821
2822 [[fallthrough]];
2823 }
2824 case Intrinsic::fma: {
2825 // fma fneg(x), fneg(y), z -> fma x, y, z
2826 Value *Src0 = II->getArgOperand(0);
2827 Value *Src1 = II->getArgOperand(1);
2828 Value *Src2 = II->getArgOperand(2);
2829 Value *X, *Y;
2830 if (match(Src0, m_FNeg(m_Value(X))) && match(Src1, m_FNeg(m_Value(Y)))) {
2831 replaceOperand(*II, 0, X);
2832 replaceOperand(*II, 1, Y);
2833 return II;
2834 }
2835
2836 // fma fabs(x), fabs(x), z -> fma x, x, z
2837 if (match(Src0, m_FAbs(m_Value(X))) &&
2838 match(Src1, m_FAbs(m_Specific(X)))) {
2839 replaceOperand(*II, 0, X);
2840 replaceOperand(*II, 1, X);
2841 return II;
2842 }
2843
2844 // Try to simplify the underlying FMul. We can only apply simplifications
2845 // that do not require rounding.
2846 if (Value *V = simplifyFMAFMul(Src0, Src1, II->getFastMathFlags(),
2847 SQ.getWithInstruction(II)))
2848 return BinaryOperator::CreateFAddFMF(V, Src2, II->getFastMathFlags());
2849
2850 // fma x, y, 0 -> fmul x, y
2851 // This is always valid for -0.0, but requires nsz for +0.0 as
2852 // -0.0 + 0.0 = 0.0, which would not be the same as the fmul on its own.
2853 if (match(Src2, m_NegZeroFP()) ||
2854 (match(Src2, m_PosZeroFP()) && II->getFastMathFlags().noSignedZeros()))
2855 return BinaryOperator::CreateFMulFMF(Src0, Src1, II);
2856
2857 // fma x, -1.0, y -> fsub y, x
2858 if (match(Src1, m_SpecificFP(-1.0)))
2859 return BinaryOperator::CreateFSubFMF(Src2, Src0, II);
2860
2861 break;
2862 }
2863 case Intrinsic::copysign: {
2864 Value *Mag = II->getArgOperand(0), *Sign = II->getArgOperand(1);
2865 if (std::optional<bool> KnownSignBit = computeKnownFPSignBit(
2866 Sign, getSimplifyQuery().getWithInstruction(II))) {
2867 if (*KnownSignBit) {
2868 // If we know that the sign argument is negative, reduce to FNABS:
2869 // copysign Mag, -Sign --> fneg (fabs Mag)
2870 Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);
2871 return replaceInstUsesWith(*II, Builder.CreateFNegFMF(Fabs, II));
2872 }
2873
2874 // If we know that the sign argument is positive, reduce to FABS:
2875 // copysign Mag, +Sign --> fabs Mag
2876 Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);
2877 return replaceInstUsesWith(*II, Fabs);
2878 }
2879
2880 // Propagate sign argument through nested calls:
2881 // copysign Mag, (copysign ?, X) --> copysign Mag, X
2882 Value *X;
2884 Value *CopySign =
2885 Builder.CreateCopySign(Mag, X, FMFSource::intersect(II, Sign));
2886 return replaceInstUsesWith(*II, CopySign);
2887 }
2888
2889 // Clear sign-bit of constant magnitude:
2890 // copysign -MagC, X --> copysign MagC, X
2891 // TODO: Support constant folding for fabs
2892 const APFloat *MagC;
2893 if (match(Mag, m_APFloat(MagC)) && MagC->isNegative()) {
2894 APFloat PosMagC = *MagC;
2895 PosMagC.clearSign();
2896 return replaceOperand(*II, 0, ConstantFP::get(Mag->getType(), PosMagC));
2897 }
2898
2899 // Peek through changes of magnitude's sign-bit. This call rewrites those:
2900 // copysign (fabs X), Sign --> copysign X, Sign
2901 // copysign (fneg X), Sign --> copysign X, Sign
2902 if (match(Mag, m_FAbs(m_Value(X))) || match(Mag, m_FNeg(m_Value(X))))
2903 return replaceOperand(*II, 0, X);
2904
2905 break;
2906 }
2907 case Intrinsic::fabs: {
2908 Value *Cond, *TVal, *FVal;
2909 Value *Arg = II->getArgOperand(0);
2910 Value *X;
2911 // fabs (-X) --> fabs (X)
2912 if (match(Arg, m_FNeg(m_Value(X)))) {
2913 CallInst *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II);
2914 return replaceInstUsesWith(CI, Fabs);
2915 }
2916
2917 if (match(Arg, m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))) {
2918 // fabs (select Cond, TrueC, FalseC) --> select Cond, AbsT, AbsF
2919 if (Arg->hasOneUse() ? (isa<Constant>(TVal) || isa<Constant>(FVal))
2920 : (isa<Constant>(TVal) && isa<Constant>(FVal))) {
2921 CallInst *AbsT = Builder.CreateCall(II->getCalledFunction(), {TVal});
2922 CallInst *AbsF = Builder.CreateCall(II->getCalledFunction(), {FVal});
2923 SelectInst *SI = SelectInst::Create(Cond, AbsT, AbsF);
2924 FastMathFlags FMF1 = II->getFastMathFlags();
2925 FastMathFlags FMF2 = cast<SelectInst>(Arg)->getFastMathFlags();
2926 FMF2.setNoSignedZeros(false);
2927 SI->setFastMathFlags(FMF1 | FMF2);
2928 return SI;
2929 }
2930 // fabs (select Cond, -FVal, FVal) --> fabs FVal
2931 if (match(TVal, m_FNeg(m_Specific(FVal))))
2932 return replaceOperand(*II, 0, FVal);
2933 // fabs (select Cond, TVal, -TVal) --> fabs TVal
2934 if (match(FVal, m_FNeg(m_Specific(TVal))))
2935 return replaceOperand(*II, 0, TVal);
2936 }
2937
2938 Value *Magnitude, *Sign;
2939 if (match(II->getArgOperand(0),
2940 m_CopySign(m_Value(Magnitude), m_Value(Sign)))) {
2941 // fabs (copysign x, y) -> (fabs x)
2942 CallInst *AbsSign =
2943 Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Magnitude, II);
2944 return replaceInstUsesWith(*II, AbsSign);
2945 }
2946
2947 [[fallthrough]];
2948 }
2949 case Intrinsic::ceil:
2950 case Intrinsic::floor:
2951 case Intrinsic::round:
2952 case Intrinsic::roundeven:
2953 case Intrinsic::nearbyint:
2954 case Intrinsic::rint:
2955 case Intrinsic::trunc: {
2956 Value *ExtSrc;
2957 if (match(II->getArgOperand(0), m_OneUse(m_FPExt(m_Value(ExtSrc))))) {
2958 // Narrow the call: intrinsic (fpext x) -> fpext (intrinsic x)
2959 Value *NarrowII = Builder.CreateUnaryIntrinsic(IID, ExtSrc, II);
2960 return new FPExtInst(NarrowII, II->getType());
2961 }
2962 break;
2963 }
2964 case Intrinsic::cos:
2965 case Intrinsic::amdgcn_cos: {
2966 Value *X, *Sign;
2967 Value *Src = II->getArgOperand(0);
2968 if (match(Src, m_FNeg(m_Value(X))) || match(Src, m_FAbs(m_Value(X))) ||
2969 match(Src, m_CopySign(m_Value(X), m_Value(Sign)))) {
2970 // cos(-x) --> cos(x)
2971 // cos(fabs(x)) --> cos(x)
2972 // cos(copysign(x, y)) --> cos(x)
2973 return replaceOperand(*II, 0, X);
2974 }
2975 break;
2976 }
2977 case Intrinsic::sin:
2978 case Intrinsic::amdgcn_sin: {
2979 Value *X;
2980 if (match(II->getArgOperand(0), m_OneUse(m_FNeg(m_Value(X))))) {
2981 // sin(-x) --> -sin(x)
2982 Value *NewSin = Builder.CreateUnaryIntrinsic(IID, X, II);
2983 return UnaryOperator::CreateFNegFMF(NewSin, II);
2984 }
2985 break;
2986 }
2987 case Intrinsic::ldexp: {
2988 // ldexp(ldexp(x, a), b) -> ldexp(x, a + b)
2989 //
2990 // The danger is if the first ldexp would overflow to infinity or underflow
2991 // to zero, but the combined exponent avoids it. We ignore this with
2992 // reassoc.
2993 //
2994 // It's also safe to fold if we know both exponents are >= 0 or <= 0 since
2995 // it would just double down on the overflow/underflow which would occur
2996 // anyway.
2997 //
2998 // TODO: Could do better if we had range tracking for the input value
2999 // exponent. Also could broaden sign check to cover == 0 case.
3000 Value *Src = II->getArgOperand(0);
3001 Value *Exp = II->getArgOperand(1);
3002 Value *InnerSrc;
3003 Value *InnerExp;
3005 m_Value(InnerSrc), m_Value(InnerExp)))) &&
3006 Exp->getType() == InnerExp->getType()) {
3007 FastMathFlags FMF = II->getFastMathFlags();
3008 FastMathFlags InnerFlags = cast<FPMathOperator>(Src)->getFastMathFlags();
3009
3010 if ((FMF.allowReassoc() && InnerFlags.allowReassoc()) ||
3011 signBitMustBeTheSame(Exp, InnerExp, SQ.getWithInstruction(II))) {
3012 // TODO: Add nsw/nuw probably safe if integer type exceeds exponent
3013 // width.
3014 Value *NewExp = Builder.CreateAdd(InnerExp, Exp);
3015 II->setArgOperand(1, NewExp);
3016 II->setFastMathFlags(InnerFlags); // Or the inner flags.
3017 return replaceOperand(*II, 0, InnerSrc);
3018 }
3019 }
3020
3021 // ldexp(x, zext(i1 y)) -> fmul x, (select y, 2.0, 1.0)
3022 // ldexp(x, sext(i1 y)) -> fmul x, (select y, 0.5, 1.0)
3023 Value *ExtSrc;
3024 if (match(Exp, m_ZExt(m_Value(ExtSrc))) &&
3025 ExtSrc->getType()->getScalarSizeInBits() == 1) {
3026 Value *Select =
3027 Builder.CreateSelect(ExtSrc, ConstantFP::get(II->getType(), 2.0),
3028 ConstantFP::get(II->getType(), 1.0));
3030 }
3031 if (match(Exp, m_SExt(m_Value(ExtSrc))) &&
3032 ExtSrc->getType()->getScalarSizeInBits() == 1) {
3033 Value *Select =
3034 Builder.CreateSelect(ExtSrc, ConstantFP::get(II->getType(), 0.5),
3035 ConstantFP::get(II->getType(), 1.0));
3037 }
3038
3039 // ldexp(x, c ? exp : 0) -> c ? ldexp(x, exp) : x
3040 // ldexp(x, c ? 0 : exp) -> c ? x : ldexp(x, exp)
3041 ///
3042 // TODO: If we cared, should insert a canonicalize for x
3043 Value *SelectCond, *SelectLHS, *SelectRHS;
3044 if (match(II->getArgOperand(1),
3045 m_OneUse(m_Select(m_Value(SelectCond), m_Value(SelectLHS),
3046 m_Value(SelectRHS))))) {
3047 Value *NewLdexp = nullptr;
3048 Value *Select = nullptr;
3049 if (match(SelectRHS, m_ZeroInt())) {
3050 NewLdexp = Builder.CreateLdexp(Src, SelectLHS, II);
3051 Select = Builder.CreateSelect(SelectCond, NewLdexp, Src);
3052 } else if (match(SelectLHS, m_ZeroInt())) {
3053 NewLdexp = Builder.CreateLdexp(Src, SelectRHS, II);
3054 Select = Builder.CreateSelect(SelectCond, Src, NewLdexp);
3055 }
3056
3057 if (NewLdexp) {
3058 Select->takeName(II);
3059 return replaceInstUsesWith(*II, Select);
3060 }
3061 }
3062
3063 break;
3064 }
3065 case Intrinsic::ptrauth_auth:
3066 case Intrinsic::ptrauth_resign: {
3067 // (sign|resign) + (auth|resign) can be folded by omitting the middle
3068 // sign+auth component if the key and discriminator match.
3069 bool NeedSign = II->getIntrinsicID() == Intrinsic::ptrauth_resign;
3070 Value *Ptr = II->getArgOperand(0);
3071 Value *Key = II->getArgOperand(1);
3072 Value *Disc = II->getArgOperand(2);
3073
3074 // AuthKey will be the key we need to end up authenticating against in
3075 // whatever we replace this sequence with.
3076 Value *AuthKey = nullptr, *AuthDisc = nullptr, *BasePtr;
3077 if (const auto *CI = dyn_cast<CallBase>(Ptr)) {
3078 BasePtr = CI->getArgOperand(0);
3079 if (CI->getIntrinsicID() == Intrinsic::ptrauth_sign) {
3080 if (CI->getArgOperand(1) != Key || CI->getArgOperand(2) != Disc)
3081 break;
3082 } else if (CI->getIntrinsicID() == Intrinsic::ptrauth_resign) {
3083 if (CI->getArgOperand(3) != Key || CI->getArgOperand(4) != Disc)
3084 break;
3085 AuthKey = CI->getArgOperand(1);
3086 AuthDisc = CI->getArgOperand(2);
3087 } else
3088 break;
3089 } else if (const auto *PtrToInt = dyn_cast<PtrToIntOperator>(Ptr)) {
3090 // ptrauth constants are equivalent to a call to @llvm.ptrauth.sign for
3091 // our purposes, so check for that too.
3092 const auto *CPA = dyn_cast<ConstantPtrAuth>(PtrToInt->getOperand(0));
3093 if (!CPA || !CPA->isKnownCompatibleWith(Key, Disc, DL))
3094 break;
3095
3096 // resign(ptrauth(p,ks,ds),ks,ds,kr,dr) -> ptrauth(p,kr,dr)
3097 if (NeedSign && isa<ConstantInt>(II->getArgOperand(4))) {
3098 auto *SignKey = cast<ConstantInt>(II->getArgOperand(3));
3099 auto *SignDisc = cast<ConstantInt>(II->getArgOperand(4));
3100 auto *SignAddrDisc = ConstantPointerNull::get(Builder.getPtrTy());
3101 auto *NewCPA = ConstantPtrAuth::get(CPA->getPointer(), SignKey,
3102 SignDisc, SignAddrDisc);
3104 *II, ConstantExpr::getPointerCast(NewCPA, II->getType()));
3105 return eraseInstFromFunction(*II);
3106 }
3107
3108 // auth(ptrauth(p,k,d),k,d) -> p
3109 BasePtr = Builder.CreatePtrToInt(CPA->getPointer(), II->getType());
3110 } else
3111 break;
3112
3113 unsigned NewIntrin;
3114 if (AuthKey && NeedSign) {
3115 // resign(0,1) + resign(1,2) = resign(0, 2)
3116 NewIntrin = Intrinsic::ptrauth_resign;
3117 } else if (AuthKey) {
3118 // resign(0,1) + auth(1) = auth(0)
3119 NewIntrin = Intrinsic::ptrauth_auth;
3120 } else if (NeedSign) {
3121 // sign(0) + resign(0, 1) = sign(1)
3122 NewIntrin = Intrinsic::ptrauth_sign;
3123 } else {
3124 // sign(0) + auth(0) = nop
3125 replaceInstUsesWith(*II, BasePtr);
3126 return eraseInstFromFunction(*II);
3127 }
3128
3129 SmallVector<Value *, 4> CallArgs;
3130 CallArgs.push_back(BasePtr);
3131 if (AuthKey) {
3132 CallArgs.push_back(AuthKey);
3133 CallArgs.push_back(AuthDisc);
3134 }
3135
3136 if (NeedSign) {
3137 CallArgs.push_back(II->getArgOperand(3));
3138 CallArgs.push_back(II->getArgOperand(4));
3139 }
3140
3141 Function *NewFn =
3142 Intrinsic::getOrInsertDeclaration(II->getModule(), NewIntrin);
3143 return CallInst::Create(NewFn, CallArgs);
3144 }
3145 case Intrinsic::arm_neon_vtbl1:
3146 case Intrinsic::aarch64_neon_tbl1:
3147 if (Value *V = simplifyNeonTbl1(*II, Builder))
3148 return replaceInstUsesWith(*II, V);
3149 break;
3150
3151 case Intrinsic::arm_neon_vmulls:
3152 case Intrinsic::arm_neon_vmullu:
3153 case Intrinsic::aarch64_neon_smull:
3154 case Intrinsic::aarch64_neon_umull: {
3155 Value *Arg0 = II->getArgOperand(0);
3156 Value *Arg1 = II->getArgOperand(1);
3157
3158 // Handle mul by zero first:
3160 return replaceInstUsesWith(CI, ConstantAggregateZero::get(II->getType()));
3161 }
3162
3163 // Check for constant LHS & RHS - in this case we just simplify.
3164 bool Zext = (IID == Intrinsic::arm_neon_vmullu ||
3165 IID == Intrinsic::aarch64_neon_umull);
3166 VectorType *NewVT = cast<VectorType>(II->getType());
3167 if (Constant *CV0 = dyn_cast<Constant>(Arg0)) {
3168 if (Constant *CV1 = dyn_cast<Constant>(Arg1)) {
3169 Value *V0 = Builder.CreateIntCast(CV0, NewVT, /*isSigned=*/!Zext);
3170 Value *V1 = Builder.CreateIntCast(CV1, NewVT, /*isSigned=*/!Zext);
3171 return replaceInstUsesWith(CI, Builder.CreateMul(V0, V1));
3172 }
3173
3174 // Couldn't simplify - canonicalize constant to the RHS.
3175 std::swap(Arg0, Arg1);
3176 }
3177
3178 // Handle mul by one:
3179 if (Constant *CV1 = dyn_cast<Constant>(Arg1))
3180 if (ConstantInt *Splat =
3181 dyn_cast_or_null<ConstantInt>(CV1->getSplatValue()))
3182 if (Splat->isOne())
3183 return CastInst::CreateIntegerCast(Arg0, II->getType(),
3184 /*isSigned=*/!Zext);
3185
3186 break;
3187 }
3188 case Intrinsic::arm_neon_aesd:
3189 case Intrinsic::arm_neon_aese:
3190 case Intrinsic::aarch64_crypto_aesd:
3191 case Intrinsic::aarch64_crypto_aese:
3192 case Intrinsic::aarch64_sve_aesd:
3193 case Intrinsic::aarch64_sve_aese: {
3194 Value *DataArg = II->getArgOperand(0);
3195 Value *KeyArg = II->getArgOperand(1);
3196
3197 // Accept zero on either operand.
3198 if (!match(KeyArg, m_ZeroInt()))
3199 std::swap(KeyArg, DataArg);
3200
3201 // Try to use the builtin XOR in AESE and AESD to eliminate a prior XOR
3202 Value *Data, *Key;
3203 if (match(KeyArg, m_ZeroInt()) &&
3204 match(DataArg, m_Xor(m_Value(Data), m_Value(Key)))) {
3205 replaceOperand(*II, 0, Data);
3206 replaceOperand(*II, 1, Key);
3207 return II;
3208 }
3209 break;
3210 }
3211 case Intrinsic::hexagon_V6_vandvrt:
3212 case Intrinsic::hexagon_V6_vandvrt_128B: {
3213 // Simplify Q -> V -> Q conversion.
3214 if (auto Op0 = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
3215 Intrinsic::ID ID0 = Op0->getIntrinsicID();
3216 if (ID0 != Intrinsic::hexagon_V6_vandqrt &&
3217 ID0 != Intrinsic::hexagon_V6_vandqrt_128B)
3218 break;
3219 Value *Bytes = Op0->getArgOperand(1), *Mask = II->getArgOperand(1);
3220 uint64_t Bytes1 = computeKnownBits(Bytes, Op0).One.getZExtValue();
3221 uint64_t Mask1 = computeKnownBits(Mask, II).One.getZExtValue();
3222 // Check if every byte has common bits in Bytes and Mask.
3223 uint64_t C = Bytes1 & Mask1;
3224 if ((C & 0xFF) && (C & 0xFF00) && (C & 0xFF0000) && (C & 0xFF000000))
3225 return replaceInstUsesWith(*II, Op0->getArgOperand(0));
3226 }
3227 break;
3228 }
3229 case Intrinsic::stackrestore: {
3230 enum class ClassifyResult {
3231 None,
3232 Alloca,
3233 StackRestore,
3234 CallWithSideEffects,
3235 };
3236 auto Classify = [](const Instruction *I) {
3237 if (isa<AllocaInst>(I))
3238 return ClassifyResult::Alloca;
3239
3240 if (auto *CI = dyn_cast<CallInst>(I)) {
3241 if (auto *II = dyn_cast<IntrinsicInst>(CI)) {
3242 if (II->getIntrinsicID() == Intrinsic::stackrestore)
3243 return ClassifyResult::StackRestore;
3244
3245 if (II->mayHaveSideEffects())
3246 return ClassifyResult::CallWithSideEffects;
3247 } else {
3248 // Consider all non-intrinsic calls to be side effects
3249 return ClassifyResult::CallWithSideEffects;
3250 }
3251 }
3252
3253 return ClassifyResult::None;
3254 };
3255
3256 // If the stacksave and the stackrestore are in the same BB, and there is
3257 // no intervening call, alloca, or stackrestore of a different stacksave,
3258 // remove the restore. This can happen when variable allocas are DCE'd.
3259 if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
3260 if (SS->getIntrinsicID() == Intrinsic::stacksave &&
3261 SS->getParent() == II->getParent()) {
3262 BasicBlock::iterator BI(SS);
3263 bool CannotRemove = false;
3264 for (++BI; &*BI != II; ++BI) {
3265 switch (Classify(&*BI)) {
3266 case ClassifyResult::None:
3267 // So far so good, look at next instructions.
3268 break;
3269
3270 case ClassifyResult::StackRestore:
3271 // If we found an intervening stackrestore for a different
3272 // stacksave, we can't remove the stackrestore. Otherwise, continue.
3273 if (cast<IntrinsicInst>(*BI).getArgOperand(0) != SS)
3274 CannotRemove = true;
3275 break;
3276
3277 case ClassifyResult::Alloca:
3278 case ClassifyResult::CallWithSideEffects:
3279 // If we found an alloca, a non-intrinsic call, or an intrinsic
3280 // call with side effects, we can't remove the stackrestore.
3281 CannotRemove = true;
3282 break;
3283 }
3284 if (CannotRemove)
3285 break;
3286 }
3287
3288 if (!CannotRemove)
3289 return eraseInstFromFunction(CI);
3290 }
3291 }
3292
3293 // Scan down this block to see if there is another stack restore in the
3294 // same block without an intervening call/alloca.
3296 Instruction *TI = II->getParent()->getTerminator();
3297 bool CannotRemove = false;
3298 for (++BI; &*BI != TI; ++BI) {
3299 switch (Classify(&*BI)) {
3300 case ClassifyResult::None:
3301 // So far so good, look at next instructions.
3302 break;
3303
3304 case ClassifyResult::StackRestore:
3305 // If there is a stackrestore below this one, remove this one.
3306 return eraseInstFromFunction(CI);
3307
3308 case ClassifyResult::Alloca:
3309 case ClassifyResult::CallWithSideEffects:
3310 // If we found an alloca, a non-intrinsic call, or an intrinsic call
3311 // with side effects (such as llvm.stacksave and llvm.read_register),
3312 // we can't remove the stack restore.
3313 CannotRemove = true;
3314 break;
3315 }
3316 if (CannotRemove)
3317 break;
3318 }
3319
3320 // If the stack restore is in a return, resume, or unwind block and if there
3321 // are no allocas or calls between the restore and the return, nuke the
3322 // restore.
3323 if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI)))
3324 return eraseInstFromFunction(CI);
3325 break;
3326 }
3327 case Intrinsic::lifetime_end:
3328 // Asan needs to poison memory to detect invalid access which is possible
3329 // even for empty lifetime range.
3330 if (II->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
3331 II->getFunction()->hasFnAttribute(Attribute::SanitizeMemory) ||
3332 II->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
3333 break;
3334
3335 if (removeTriviallyEmptyRange(*II, *this, [](const IntrinsicInst &I) {
3336 return I.getIntrinsicID() == Intrinsic::lifetime_start;
3337 }))
3338 return nullptr;
3339 break;
3340 case Intrinsic::assume: {
3341 Value *IIOperand = II->getArgOperand(0);
3343 II->getOperandBundlesAsDefs(OpBundles);
3344
3345 /// This will remove the boolean Condition from the assume given as
3346 /// argument and remove the assume if it becomes useless.
3347 /// always returns nullptr for use as a return values.
3348 auto RemoveConditionFromAssume = [&](Instruction *Assume) -> Instruction * {
3349 assert(isa<AssumeInst>(Assume));
3351 return eraseInstFromFunction(CI);
3352 replaceUse(II->getOperandUse(0), ConstantInt::getTrue(II->getContext()));
3353 return nullptr;
3354 };
3355 // Remove an assume if it is followed by an identical assume.
3356 // TODO: Do we need this? Unless there are conflicting assumptions, the
3357 // computeKnownBits(IIOperand) below here eliminates redundant assumes.
3358 Instruction *Next = II->getNextNode();
3360 return RemoveConditionFromAssume(Next);
3361
3362 // Canonicalize assume(a && b) -> assume(a); assume(b);
3363 // Note: New assumption intrinsics created here are registered by
3364 // the InstCombineIRInserter object.
3365 FunctionType *AssumeIntrinsicTy = II->getFunctionType();
3366 Value *AssumeIntrinsic = II->getCalledOperand();
3367 Value *A, *B;
3368 if (match(IIOperand, m_LogicalAnd(m_Value(A), m_Value(B)))) {
3369 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, A, OpBundles,
3370 II->getName());
3371 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, B, II->getName());
3372 return eraseInstFromFunction(*II);
3373 }
3374 // assume(!(a || b)) -> assume(!a); assume(!b);
3375 if (match(IIOperand, m_Not(m_LogicalOr(m_Value(A), m_Value(B))))) {
3376 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
3377 Builder.CreateNot(A), OpBundles, II->getName());
3378 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
3379 Builder.CreateNot(B), II->getName());
3380 return eraseInstFromFunction(*II);
3381 }
3382
3383 // assume( (load addr) != null ) -> add 'nonnull' metadata to load
3384 // (if assume is valid at the load)
3385 Instruction *LHS;
3387 m_Zero())) &&
3388 LHS->getOpcode() == Instruction::Load &&
3389 LHS->getType()->isPointerTy() &&
3390 isValidAssumeForContext(II, LHS, &DT)) {
3391 MDNode *MD = MDNode::get(II->getContext(), {});
3392 LHS->setMetadata(LLVMContext::MD_nonnull, MD);
3393 LHS->setMetadata(LLVMContext::MD_noundef, MD);
3394 return RemoveConditionFromAssume(II);
3395
3396 // TODO: apply nonnull return attributes to calls and invokes
3397 // TODO: apply range metadata for range check patterns?
3398 }
3399
3400 for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
3401 OperandBundleUse OBU = II->getOperandBundleAt(Idx);
3402
3403 // Separate storage assumptions apply to the underlying allocations, not
3404 // any particular pointer within them. When evaluating the hints for AA
3405 // purposes we getUnderlyingObject them; by precomputing the answers here
3406 // we can avoid having to do so repeatedly there.
3407 if (OBU.getTagName() == "separate_storage") {
3408 assert(OBU.Inputs.size() == 2);
3409 auto MaybeSimplifyHint = [&](const Use &U) {
3410 Value *Hint = U.get();
3411 // Not having a limit is safe because InstCombine removes unreachable
3412 // code.
3413 Value *UnderlyingObject = getUnderlyingObject(Hint, /*MaxLookup*/ 0);
3414 if (Hint != UnderlyingObject)
3415 replaceUse(const_cast<Use &>(U), UnderlyingObject);
3416 };
3417 MaybeSimplifyHint(OBU.Inputs[0]);
3418 MaybeSimplifyHint(OBU.Inputs[1]);
3419 }
3420
3421 // Try to remove redundant alignment assumptions.
3422 if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) {
3424 *cast<AssumeInst>(II), II->arg_size() + Idx);
3425 if (!RK || RK.AttrKind != Attribute::Alignment ||
3427 continue;
3428
3429 // Remove align 1 bundles; they don't add any useful information.
3430 if (RK.ArgValue == 1)
3432
3433 // Don't try to remove align assumptions for pointers derived from
3434 // arguments. We might lose information if the function gets inline and
3435 // the align argument attribute disappears.
3437 if (!UO || isa<Argument>(UO))
3438 continue;
3439
3440 // Compute known bits for the pointer, passing nullptr as context to
3441 // avoid computeKnownBits using the assumption we are about to remove
3442 // for reasoning.
3443 KnownBits Known = computeKnownBits(RK.WasOn, /*CtxI=*/nullptr);
3444 unsigned TZ = std::min(Known.countMinTrailingZeros(),
3446 if ((1ULL << TZ) < RK.ArgValue)
3447 continue;
3449 }
3450 }
3451
3452 // Convert nonnull assume like:
3453 // %A = icmp ne i32* %PTR, null
3454 // call void @llvm.assume(i1 %A)
3455 // into
3456 // call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ]
3458 match(IIOperand,
3460 A->getType()->isPointerTy()) {
3461 if (auto *Replacement = buildAssumeFromKnowledge(
3462 {RetainedKnowledge{Attribute::NonNull, 0, A}}, Next, &AC, &DT)) {
3463
3464 Replacement->insertBefore(Next->getIterator());
3465 AC.registerAssumption(Replacement);
3466 return RemoveConditionFromAssume(II);
3467 }
3468 }
3469
3470 // Convert alignment assume like:
3471 // %B = ptrtoint i32* %A to i64
3472 // %C = and i64 %B, Constant
3473 // %D = icmp eq i64 %C, 0
3474 // call void @llvm.assume(i1 %D)
3475 // into
3476 // call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 Constant + 1)]
3477 uint64_t AlignMask = 1;
3479 (match(IIOperand, m_Not(m_Trunc(m_Value(A)))) ||
3480 match(IIOperand,
3482 m_And(m_Value(A), m_ConstantInt(AlignMask)),
3483 m_Zero())))) {
3484 if (isPowerOf2_64(AlignMask + 1)) {
3485 uint64_t Offset = 0;
3487 if (match(A, m_PtrToInt(m_Value(A)))) {
3488 /// Note: this doesn't preserve the offset information but merges
3489 /// offset and alignment.
3490 /// TODO: we can generate a GEP instead of merging the alignment with
3491 /// the offset.
3492 RetainedKnowledge RK{Attribute::Alignment,
3493 (unsigned)MinAlign(Offset, AlignMask + 1), A};
3494 if (auto *Replacement =
3496
3497 Replacement->insertAfter(II->getIterator());
3498 AC.registerAssumption(Replacement);
3499 }
3500 return RemoveConditionFromAssume(II);
3501 }
3502 }
3503 }
3504
3505 /// Canonicalize Knowledge in operand bundles.
3506 if (EnableKnowledgeRetention && II->hasOperandBundles()) {
3507 for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
3508 auto &BOI = II->bundle_op_info_begin()[Idx];
3511 if (BOI.End - BOI.Begin > 2)
3512 continue; // Prevent reducing knowledge in an align with offset since
3513 // extracting a RetainedKnowledge from them looses offset
3514 // information
3515 RetainedKnowledge CanonRK =
3518 &getDominatorTree());
3519 if (CanonRK == RK)
3520 continue;
3521 if (!CanonRK) {
3522 if (BOI.End - BOI.Begin > 0) {
3523 Worklist.pushValue(II->op_begin()[BOI.Begin]);
3524 Value::dropDroppableUse(II->op_begin()[BOI.Begin]);
3525 }
3526 continue;
3527 }
3528 assert(RK.AttrKind == CanonRK.AttrKind);
3529 if (BOI.End - BOI.Begin > 0)
3530 II->op_begin()[BOI.Begin].set(CanonRK.WasOn);
3531 if (BOI.End - BOI.Begin > 1)
3532 II->op_begin()[BOI.Begin + 1].set(ConstantInt::get(
3533 Type::getInt64Ty(II->getContext()), CanonRK.ArgValue));
3534 if (RK.WasOn)
3535 Worklist.pushValue(RK.WasOn);
3536 return II;
3537 }
3538 }
3539
3540 // If there is a dominating assume with the same condition as this one,
3541 // then this one is redundant, and should be removed.
3542 KnownBits Known(1);
3543 computeKnownBits(IIOperand, Known, II);
3545 return eraseInstFromFunction(*II);
3546
3547 // assume(false) is unreachable.
3548 if (match(IIOperand, m_CombineOr(m_Zero(), m_Undef()))) {
3550 return eraseInstFromFunction(*II);
3551 }
3552
3553 // Update the cache of affected values for this assumption (we might be
3554 // here because we just simplified the condition).
3555 AC.updateAffectedValues(cast<AssumeInst>(II));
3556 break;
3557 }
3558 case Intrinsic::experimental_guard: {
3559 // Is this guard followed by another guard? We scan forward over a small
3560 // fixed window of instructions to handle common cases with conditions
3561 // computed between guards.
3562 Instruction *NextInst = II->getNextNode();
3563 for (unsigned i = 0; i < GuardWideningWindow; i++) {
3564 // Note: Using context-free form to avoid compile time blow up
3565 if (!isSafeToSpeculativelyExecute(NextInst))
3566 break;
3567 NextInst = NextInst->getNextNode();
3568 }
3569 Value *NextCond = nullptr;
3570 if (match(NextInst,
3572 Value *CurrCond = II->getArgOperand(0);
3573
3574 // Remove a guard that it is immediately preceded by an identical guard.
3575 // Otherwise canonicalize guard(a); guard(b) -> guard(a & b).
3576 if (CurrCond != NextCond) {
3577 Instruction *MoveI = II->getNextNode();
3578 while (MoveI != NextInst) {
3579 auto *Temp = MoveI;
3580 MoveI = MoveI->getNextNode();
3581 Temp->moveBefore(II->getIterator());
3582 }
3583 replaceOperand(*II, 0, Builder.CreateAnd(CurrCond, NextCond));
3584 }
3585 eraseInstFromFunction(*NextInst);
3586 return II;
3587 }
3588 break;
3589 }
3590 case Intrinsic::vector_insert: {
3591 Value *Vec = II->getArgOperand(0);
3592 Value *SubVec = II->getArgOperand(1);
3593 Value *Idx = II->getArgOperand(2);
3594 auto *DstTy = dyn_cast<FixedVectorType>(II->getType());
3595 auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
3596 auto *SubVecTy = dyn_cast<FixedVectorType>(SubVec->getType());
3597
3598 // Only canonicalize if the destination vector, Vec, and SubVec are all
3599 // fixed vectors.
3600 if (DstTy && VecTy && SubVecTy) {
3601 unsigned DstNumElts = DstTy->getNumElements();
3602 unsigned VecNumElts = VecTy->getNumElements();
3603 unsigned SubVecNumElts = SubVecTy->getNumElements();
3604 unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
3605
3606 // An insert that entirely overwrites Vec with SubVec is a nop.
3607 if (VecNumElts == SubVecNumElts)
3608 return replaceInstUsesWith(CI, SubVec);
3609
3610 // Widen SubVec into a vector of the same width as Vec, since
3611 // shufflevector requires the two input vectors to be the same width.
3612 // Elements beyond the bounds of SubVec within the widened vector are
3613 // undefined.
3614 SmallVector<int, 8> WidenMask;
3615 unsigned i;
3616 for (i = 0; i != SubVecNumElts; ++i)
3617 WidenMask.push_back(i);
3618 for (; i != VecNumElts; ++i)
3619 WidenMask.push_back(PoisonMaskElem);
3620
3621 Value *WidenShuffle = Builder.CreateShuffleVector(SubVec, WidenMask);
3622
3624 for (unsigned i = 0; i != IdxN; ++i)
3625 Mask.push_back(i);
3626 for (unsigned i = DstNumElts; i != DstNumElts + SubVecNumElts; ++i)
3627 Mask.push_back(i);
3628 for (unsigned i = IdxN + SubVecNumElts; i != DstNumElts; ++i)
3629 Mask.push_back(i);
3630
3631 Value *Shuffle = Builder.CreateShuffleVector(Vec, WidenShuffle, Mask);
3632 return replaceInstUsesWith(CI, Shuffle);
3633 }
3634 break;
3635 }
3636 case Intrinsic::vector_extract: {
3637 Value *Vec = II->getArgOperand(0);
3638 Value *Idx = II->getArgOperand(1);
3639
3640 Type *ReturnType = II->getType();
3641 // (extract_vector (insert_vector InsertTuple, InsertValue, InsertIdx),
3642 // ExtractIdx)
3643 unsigned ExtractIdx = cast<ConstantInt>(Idx)->getZExtValue();
3644 Value *InsertTuple, *InsertIdx, *InsertValue;
3646 m_Value(InsertValue),
3647 m_Value(InsertIdx))) &&
3648 InsertValue->getType() == ReturnType) {
3649 unsigned Index = cast<ConstantInt>(InsertIdx)->getZExtValue();
3650 // Case where we get the same index right after setting it.
3651 // extract.vector(insert.vector(InsertTuple, InsertValue, Idx), Idx) -->
3652 // InsertValue
3653 if (ExtractIdx == Index)
3654 return replaceInstUsesWith(CI, InsertValue);
3655 // If we are getting a different index than what was set in the
3656 // insert.vector intrinsic. We can just set the input tuple to the one up
3657 // in the chain. extract.vector(insert.vector(InsertTuple, InsertValue,
3658 // InsertIndex), ExtractIndex)
3659 // --> extract.vector(InsertTuple, ExtractIndex)
3660 else
3661 return replaceOperand(CI, 0, InsertTuple);
3662 }
3663
3664 auto *DstTy = dyn_cast<VectorType>(ReturnType);
3665 auto *VecTy = dyn_cast<VectorType>(Vec->getType());
3666
3667 if (DstTy && VecTy) {
3668 auto DstEltCnt = DstTy->getElementCount();
3669 auto VecEltCnt = VecTy->getElementCount();
3670 unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
3671
3672 // Extracting the entirety of Vec is a nop.
3673 if (DstEltCnt == VecTy->getElementCount()) {
3674 replaceInstUsesWith(CI, Vec);
3675 return eraseInstFromFunction(CI);
3676 }
3677
3678 // Only canonicalize to shufflevector if the destination vector and
3679 // Vec are fixed vectors.
3680 if (VecEltCnt.isScalable() || DstEltCnt.isScalable())
3681 break;
3682
3684 for (unsigned i = 0; i != DstEltCnt.getKnownMinValue(); ++i)
3685 Mask.push_back(IdxN + i);
3686
3687 Value *Shuffle = Builder.CreateShuffleVector(Vec, Mask);
3688 return replaceInstUsesWith(CI, Shuffle);
3689 }
3690 break;
3691 }
3692 case Intrinsic::experimental_vp_reverse: {
3693 Value *X;
3694 Value *Vec = II->getArgOperand(0);
3695 Value *Mask = II->getArgOperand(1);
3696 if (!match(Mask, m_AllOnes()))
3697 break;
3698 Value *EVL = II->getArgOperand(2);
3699 // TODO: Canonicalize experimental.vp.reverse after unop/binops?
3700 // rev(unop rev(X)) --> unop X
3701 if (match(Vec,
3703 m_Value(X), m_AllOnes(), m_Specific(EVL)))))) {
3704 auto *OldUnOp = cast<UnaryOperator>(Vec);
3706 OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(),
3707 II->getIterator());
3708 return replaceInstUsesWith(CI, NewUnOp);
3709 }
3710 break;
3711 }
3712 case Intrinsic::vector_reduce_or:
3713 case Intrinsic::vector_reduce_and: {
3714 // Canonicalize logical or/and reductions:
3715 // Or reduction for i1 is represented as:
3716 // %val = bitcast <ReduxWidth x i1> to iReduxWidth
3717 // %res = cmp ne iReduxWidth %val, 0
3718 // And reduction for i1 is represented as:
3719 // %val = bitcast <ReduxWidth x i1> to iReduxWidth
3720 // %res = cmp eq iReduxWidth %val, 11111
3721 Value *Arg = II->getArgOperand(0);
3722 Value *Vect;
3723
3724 if (Value *NewOp =
3725 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3726 replaceUse(II->getOperandUse(0), NewOp);
3727 return II;
3728 }
3729
3730 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3731 if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
3732 if (FTy->getElementType() == Builder.getInt1Ty()) {
3733 Value *Res = Builder.CreateBitCast(
3734 Vect, Builder.getIntNTy(FTy->getNumElements()));
3735 if (IID == Intrinsic::vector_reduce_and) {
3736 Res = Builder.CreateICmpEQ(
3738 } else {
3739 assert(IID == Intrinsic::vector_reduce_or &&
3740 "Expected or reduction.");
3741 Res = Builder.CreateIsNotNull(Res);
3742 }
3743 if (Arg != Vect)
3744 Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
3745 II->getType());
3746 return replaceInstUsesWith(CI, Res);
3747 }
3748 }
3749 [[fallthrough]];
3750 }
3751 case Intrinsic::vector_reduce_add: {
3752 if (IID == Intrinsic::vector_reduce_add) {
3753 // Convert vector_reduce_add(ZExt(<n x i1>)) to
3754 // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
3755 // Convert vector_reduce_add(SExt(<n x i1>)) to
3756 // -ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
3757 // Convert vector_reduce_add(<n x i1>) to
3758 // Trunc(ctpop(bitcast <n x i1> to in)).
3759 Value *Arg = II->getArgOperand(0);
3760 Value *Vect;
3761
3762 if (Value *NewOp =
3763 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3764 replaceUse(II->getOperandUse(0), NewOp);
3765 return II;
3766 }
3767
3768 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3769 if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
3770 if (FTy->getElementType() == Builder.getInt1Ty()) {
3771 Value *V = Builder.CreateBitCast(
3772 Vect, Builder.getIntNTy(FTy->getNumElements()));
3773 Value *Res = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, V);
3774 if (Res->getType() != II->getType())
3775 Res = Builder.CreateZExtOrTrunc(Res, II->getType());
3776 if (Arg != Vect &&
3777 cast<Instruction>(Arg)->getOpcode() == Instruction::SExt)
3778 Res = Builder.CreateNeg(Res);
3779 return replaceInstUsesWith(CI, Res);
3780 }
3781 }
3782
3783 // vector.reduce.add.vNiM(splat(%x)) -> mul(%x, N)
3784 if (Value *Splat = getSplatValue(Arg)) {
3785 ElementCount VecToReduceCount =
3786 cast<VectorType>(Arg->getType())->getElementCount();
3787 if (VecToReduceCount.isFixed()) {
3788 unsigned VectorSize = VecToReduceCount.getFixedValue();
3789 return BinaryOperator::CreateMul(
3790 Splat, ConstantInt::get(Splat->getType(), VectorSize));
3791 }
3792 }
3793 }
3794 [[fallthrough]];
3795 }
3796 case Intrinsic::vector_reduce_xor: {
3797 if (IID == Intrinsic::vector_reduce_xor) {
3798 // Exclusive disjunction reduction over the vector with
3799 // (potentially-extended) i1 element type is actually a
3800 // (potentially-extended) arithmetic `add` reduction over the original
3801 // non-extended value:
3802 // vector_reduce_xor(?ext(<n x i1>))
3803 // -->
3804 // ?ext(vector_reduce_add(<n x i1>))
3805 Value *Arg = II->getArgOperand(0);
3806 Value *Vect;
3807
3808 if (Value *NewOp =
3809 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3810 replaceUse(II->getOperandUse(0), NewOp);
3811 return II;
3812 }
3813
3814 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3815 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3816 if (VTy->getElementType() == Builder.getInt1Ty()) {
3817 Value *Res = Builder.CreateAddReduce(Vect);
3818 if (Arg != Vect)
3819 Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
3820 II->getType());
3821 return replaceInstUsesWith(CI, Res);
3822 }
3823 }
3824 }
3825 [[fallthrough]];
3826 }
3827 case Intrinsic::vector_reduce_mul: {
3828 if (IID == Intrinsic::vector_reduce_mul) {
3829 // Multiplicative reduction over the vector with (potentially-extended)
3830 // i1 element type is actually a (potentially zero-extended)
3831 // logical `and` reduction over the original non-extended value:
3832 // vector_reduce_mul(?ext(<n x i1>))
3833 // -->
3834 // zext(vector_reduce_and(<n x i1>))
3835 Value *Arg = II->getArgOperand(0);
3836 Value *Vect;
3837
3838 if (Value *NewOp =
3839 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3840 replaceUse(II->getOperandUse(0), NewOp);
3841 return II;
3842 }
3843
3844 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3845 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3846 if (VTy->getElementType() == Builder.getInt1Ty()) {
3847 Value *Res = Builder.CreateAndReduce(Vect);
3848 if (Res->getType() != II->getType())
3849 Res = Builder.CreateZExt(Res, II->getType());
3850 return replaceInstUsesWith(CI, Res);
3851 }
3852 }
3853 }
3854 [[fallthrough]];
3855 }
3856 case Intrinsic::vector_reduce_umin:
3857 case Intrinsic::vector_reduce_umax: {
3858 if (IID == Intrinsic::vector_reduce_umin ||
3859 IID == Intrinsic::vector_reduce_umax) {
3860 // UMin/UMax reduction over the vector with (potentially-extended)
3861 // i1 element type is actually a (potentially-extended)
3862 // logical `and`/`or` reduction over the original non-extended value:
3863 // vector_reduce_u{min,max}(?ext(<n x i1>))
3864 // -->
3865 // ?ext(vector_reduce_{and,or}(<n x i1>))
3866 Value *Arg = II->getArgOperand(0);
3867 Value *Vect;
3868
3869 if (Value *NewOp =
3870 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3871 replaceUse(II->getOperandUse(0), NewOp);
3872 return II;
3873 }
3874
3875 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3876 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3877 if (VTy->getElementType() == Builder.getInt1Ty()) {
3878 Value *Res = IID == Intrinsic::vector_reduce_umin
3879 ? Builder.CreateAndReduce(Vect)
3880 : Builder.CreateOrReduce(Vect);
3881 if (Arg != Vect)
3882 Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
3883 II->getType());
3884 return replaceInstUsesWith(CI, Res);
3885 }
3886 }
3887 }
3888 [[fallthrough]];
3889 }
3890 case Intrinsic::vector_reduce_smin:
3891 case Intrinsic::vector_reduce_smax: {
3892 if (IID == Intrinsic::vector_reduce_smin ||
3893 IID == Intrinsic::vector_reduce_smax) {
3894 // SMin/SMax reduction over the vector with (potentially-extended)
3895 // i1 element type is actually a (potentially-extended)
3896 // logical `and`/`or` reduction over the original non-extended value:
3897 // vector_reduce_s{min,max}(<n x i1>)
3898 // -->
3899 // vector_reduce_{or,and}(<n x i1>)
3900 // and
3901 // vector_reduce_s{min,max}(sext(<n x i1>))
3902 // -->
3903 // sext(vector_reduce_{or,and}(<n x i1>))
3904 // and
3905 // vector_reduce_s{min,max}(zext(<n x i1>))
3906 // -->
3907 // zext(vector_reduce_{and,or}(<n x i1>))
3908 Value *Arg = II->getArgOperand(0);
3909 Value *Vect;
3910
3911 if (Value *NewOp =
3912 simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
3913 replaceUse(II->getOperandUse(0), NewOp);
3914 return II;
3915 }
3916
3917 if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
3918 if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
3919 if (VTy->getElementType() == Builder.getInt1Ty()) {
3920 Instruction::CastOps ExtOpc = Instruction::CastOps::CastOpsEnd;
3921 if (Arg != Vect)
3922 ExtOpc = cast<CastInst>(Arg)->getOpcode();
3923 Value *Res = ((IID == Intrinsic::vector_reduce_smin) ==
3924 (ExtOpc == Instruction::CastOps::ZExt))
3925 ? Builder.CreateAndReduce(Vect)
3926 : Builder.CreateOrReduce(Vect);
3927 if (Arg != Vect)
3928 Res = Builder.CreateCast(ExtOpc, Res, II->getType());
3929 return replaceInstUsesWith(CI, Res);
3930 }
3931 }
3932 }
3933 [[fallthrough]];
3934 }
3935 case Intrinsic::vector_reduce_fmax:
3936 case Intrinsic::vector_reduce_fmin:
3937 case Intrinsic::vector_reduce_fadd:
3938 case Intrinsic::vector_reduce_fmul: {
3939 bool CanReorderLanes = (IID != Intrinsic::vector_reduce_fadd &&
3940 IID != Intrinsic::vector_reduce_fmul) ||
3941 II->hasAllowReassoc();
3942 const unsigned ArgIdx = (IID == Intrinsic::vector_reduce_fadd ||
3943 IID == Intrinsic::vector_reduce_fmul)
3944 ? 1
3945 : 0;
3946 Value *Arg = II->getArgOperand(ArgIdx);
3947 if (Value *NewOp = simplifyReductionOperand(Arg, CanReorderLanes)) {
3948 replaceUse(II->getOperandUse(ArgIdx), NewOp);
3949 return nullptr;
3950 }
3951 break;
3952 }
3953 case Intrinsic::is_fpclass: {
3954 if (Instruction *I = foldIntrinsicIsFPClass(*II))
3955 return I;
3956 break;
3957 }
3958 case Intrinsic::threadlocal_address: {
3959 Align MinAlign = getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);
3960 MaybeAlign Align = II->getRetAlign();
3961 if (MinAlign > Align.valueOrOne()) {
3962 II->addRetAttr(Attribute::getWithAlignment(II->getContext(), MinAlign));
3963 return II;
3964 }
3965 break;
3966 }
3967 case Intrinsic::frexp: {
3968 Value *X;
3969 // The first result is idempotent with the added complication of the struct
3970 // return, and the second result is zero because the value is already
3971 // normalized.
3972 if (match(II->getArgOperand(0), m_ExtractValue<0>(m_Value(X)))) {
3974 X = Builder.CreateInsertValue(
3975 X, Constant::getNullValue(II->getType()->getStructElementType(1)),
3976 1);
3977 return replaceInstUsesWith(*II, X);
3978 }
3979 }
3980 break;
3981 }
3982 case Intrinsic::get_active_lane_mask: {
3983 const APInt *Op0, *Op1;
3984 if (match(II->getOperand(0), m_StrictlyPositive(Op0)) &&
3985 match(II->getOperand(1), m_APInt(Op1))) {
3986 Type *OpTy = II->getOperand(0)->getType();
3987 return replaceInstUsesWith(
3988 *II, Builder.CreateIntrinsic(
3989 II->getType(), Intrinsic::get_active_lane_mask,
3990 {Constant::getNullValue(OpTy),
3991 ConstantInt::get(OpTy, Op1->usub_sat(*Op0))}));
3992 }
3993 break;
3994 }
3995 default: {
3996 // Handle target specific intrinsics
3997 std::optional<Instruction *> V = targetInstCombineIntrinsic(*II);
3998 if (V)
3999 return *V;
4000 break;
4001 }
4002 }
4003
4004 // Try to fold intrinsic into select/phi operands. This is legal if:
4005 // * The intrinsic is speculatable.
4006 // * The select condition is not a vector, or the intrinsic does not
4007 // perform cross-lane operations.
4010 for (Value *Op : II->args()) {
4011 if (auto *Sel = dyn_cast<SelectInst>(Op))
4012 if (Instruction *R = FoldOpIntoSelect(*II, Sel))
4013 return R;
4014 if (auto *Phi = dyn_cast<PHINode>(Op))
4015 if (Instruction *R = foldOpIntoPhi(*II, Phi))
4016 return R;
4017 }
4018
4020 return Shuf;
4021
4023 return replaceInstUsesWith(*II, Reverse);
4024
4026 return replaceInstUsesWith(*II, Res);
4027
4028 // Some intrinsics (like experimental_gc_statepoint) can be used in invoke
4029 // context, so it is handled in visitCallBase and we should trigger it.
4030 return visitCallBase(*II);
4031}
4032
4033// Fence instruction simplification
4035 auto *NFI = dyn_cast<FenceInst>(FI.getNextNode());
4036 // This check is solely here to handle arbitrary target-dependent syncscopes.
4037 // TODO: Can remove if does not matter in practice.
4038 if (NFI && FI.isIdenticalTo(NFI))
4039 return eraseInstFromFunction(FI);
4040
4041 // Returns true if FI1 is identical or stronger fence than FI2.
4042 auto isIdenticalOrStrongerFence = [](FenceInst *FI1, FenceInst *FI2) {
4043 auto FI1SyncScope = FI1->getSyncScopeID();
4044 // Consider same scope, where scope is global or single-thread.
4045 if (FI1SyncScope != FI2->getSyncScopeID() ||
4046 (FI1SyncScope != SyncScope::System &&
4047 FI1SyncScope != SyncScope::SingleThread))
4048 return false;
4049
4050 return isAtLeastOrStrongerThan(FI1->getOrdering(), FI2->getOrdering());
4051 };
4052 if (NFI && isIdenticalOrStrongerFence(NFI, &FI))
4053 return eraseInstFromFunction(FI);
4054
4055 if (auto *PFI = dyn_cast_or_null<FenceInst>(FI.getPrevNode()))
4056 if (isIdenticalOrStrongerFence(PFI, &FI))
4057 return eraseInstFromFunction(FI);
4058 return nullptr;
4059}
4060
4061// InvokeInst simplification
4063 return visitCallBase(II);
4064}
4065
4066// CallBrInst simplification
4068 return visitCallBase(CBI);
4069}
4070
4071Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
4072 if (!CI->getCalledFunction()) return nullptr;
4073
4074 // Skip optimizing notail and musttail calls so
4075 // LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.
4076 // LibCallSimplifier::optimizeCall should try to preserve tail calls though.
4077 if (CI->isMustTailCall() || CI->isNoTailCall())
4078 return nullptr;
4079
4080 auto InstCombineRAUW = [this](Instruction *From, Value *With) {
4081 replaceInstUsesWith(*From, With);
4082 };
4083 auto InstCombineErase = [this](Instruction *I) {
4085 };
4086 LibCallSimplifier Simplifier(DL, &TLI, &DT, &DC, &AC, ORE, BFI, PSI,
4087 InstCombineRAUW, InstCombineErase);
4088 if (Value *With = Simplifier.optimizeCall(CI, Builder)) {
4089 ++NumSimplified;
4090 return CI->use_empty() ? CI : replaceInstUsesWith(*CI, With);
4091 }
4092
4093 return nullptr;
4094}
4095
4097 // Strip off at most one level of pointer casts, looking for an alloca. This
4098 // is good enough in practice and simpler than handling any number of casts.
4099 Value *Underlying = TrampMem->stripPointerCasts();
4100 if (Underlying != TrampMem &&
4101 (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem))
4102 return nullptr;
4103 if (!isa<AllocaInst>(Underlying))
4104 return nullptr;
4105
4106 IntrinsicInst *InitTrampoline = nullptr;
4107 for (User *U : TrampMem->users()) {
4109 if (!II)
4110 return nullptr;
4111 if (II->getIntrinsicID() == Intrinsic::init_trampoline) {
4112 if (InitTrampoline)
4113 // More than one init_trampoline writes to this value. Give up.
4114 return nullptr;
4115 InitTrampoline = II;
4116 continue;
4117 }
4118 if (II->getIntrinsicID() == Intrinsic::adjust_trampoline)
4119 // Allow any number of calls to adjust.trampoline.
4120 continue;
4121 return nullptr;
4122 }
4123
4124 // No call to init.trampoline found.
4125 if (!InitTrampoline)
4126 return nullptr;
4127
4128 // Check that the alloca is being used in the expected way.
4129 if (InitTrampoline->getOperand(0) != TrampMem)
4130 return nullptr;
4131
4132 return InitTrampoline;
4133}
4134
4136 Value *TrampMem) {
4137 // Visit all the previous instructions in the basic block, and try to find a
4138 // init.trampoline which has a direct path to the adjust.trampoline.
4139 for (BasicBlock::iterator I = AdjustTramp->getIterator(),
4140 E = AdjustTramp->getParent()->begin();
4141 I != E;) {
4142 Instruction *Inst = &*--I;
4144 if (II->getIntrinsicID() == Intrinsic::init_trampoline &&
4145 II->getOperand(0) == TrampMem)
4146 return II;
4147 if (Inst->mayWriteToMemory())
4148 return nullptr;
4149 }
4150 return nullptr;
4151}
4152
4153// Given a call to llvm.adjust.trampoline, find and return the corresponding
4154// call to llvm.init.trampoline if the call to the trampoline can be optimized
4155// to a direct call to a function. Otherwise return NULL.
4157 Callee = Callee->stripPointerCasts();
4158 IntrinsicInst *AdjustTramp = dyn_cast<IntrinsicInst>(Callee);
4159 if (!AdjustTramp ||
4160 AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline)
4161 return nullptr;
4162
4163 Value *TrampMem = AdjustTramp->getOperand(0);
4164
4166 return IT;
4167 if (IntrinsicInst *IT = findInitTrampolineFromBB(AdjustTramp, TrampMem))
4168 return IT;
4169 return nullptr;
4170}
4171
4172Instruction *InstCombinerImpl::foldPtrAuthIntrinsicCallee(CallBase &Call) {
4173 const Value *Callee = Call.getCalledOperand();
4174 const auto *IPC = dyn_cast<IntToPtrInst>(Callee);
4175 if (!IPC || !IPC->isNoopCast(DL))
4176 return nullptr;
4177
4178 const auto *II = dyn_cast<IntrinsicInst>(IPC->getOperand(0));
4179 if (!II)
4180 return nullptr;
4181
4182 Intrinsic::ID IIID = II->getIntrinsicID();
4183 if (IIID != Intrinsic::ptrauth_resign && IIID != Intrinsic::ptrauth_sign)
4184 return nullptr;
4185
4186 // Isolate the ptrauth bundle from the others.
4187 std::optional<OperandBundleUse> PtrAuthBundleOrNone;
4189 for (unsigned BI = 0, BE = Call.getNumOperandBundles(); BI != BE; ++BI) {
4190 OperandBundleUse Bundle = Call.getOperandBundleAt(BI);
4191 if (Bundle.getTagID() == LLVMContext::OB_ptrauth)
4192 PtrAuthBundleOrNone = Bundle;
4193 else
4194 NewBundles.emplace_back(Bundle);
4195 }
4196
4197 if (!PtrAuthBundleOrNone)
4198 return nullptr;
4199
4200 Value *NewCallee = nullptr;
4201 switch (IIID) {
4202 // call(ptrauth.resign(p)), ["ptrauth"()] -> call p, ["ptrauth"()]
4203 // assuming the call bundle and the sign operands match.
4204 case Intrinsic::ptrauth_resign: {
4205 // Resign result key should match bundle.
4206 if (II->getOperand(3) != PtrAuthBundleOrNone->Inputs[0])
4207 return nullptr;
4208 // Resign result discriminator should match bundle.
4209 if (II->getOperand(4) != PtrAuthBundleOrNone->Inputs[1])
4210 return nullptr;
4211
4212 // Resign input (auth) key should also match: we can't change the key on
4213 // the new call we're generating, because we don't know what keys are valid.
4214 if (II->getOperand(1) != PtrAuthBundleOrNone->Inputs[0])
4215 return nullptr;
4216
4217 Value *NewBundleOps[] = {II->getOperand(1), II->getOperand(2)};
4218 NewBundles.emplace_back("ptrauth", NewBundleOps);
4219 NewCallee = II->getOperand(0);
4220 break;
4221 }
4222
4223 // call(ptrauth.sign(p)), ["ptrauth"()] -> call p
4224 // assuming the call bundle and the sign operands match.
4225 // Non-ptrauth indirect calls are undesirable, but so is ptrauth.sign.
4226 case Intrinsic::ptrauth_sign: {
4227 // Sign key should match bundle.
4228 if (II->getOperand(1) != PtrAuthBundleOrNone->Inputs[0])
4229 return nullptr;
4230 // Sign discriminator should match bundle.
4231 if (II->getOperand(2) != PtrAuthBundleOrNone->Inputs[1])
4232 return nullptr;
4233 NewCallee = II->getOperand(0);
4234 break;
4235 }
4236 default:
4237 llvm_unreachable("unexpected intrinsic ID");
4238 }
4239
4240 if (!NewCallee)
4241 return nullptr;
4242
4243 NewCallee = Builder.CreateBitOrPointerCast(NewCallee, Callee->getType());
4244 CallBase *NewCall = CallBase::Create(&Call, NewBundles);
4245 NewCall->setCalledOperand(NewCallee);
4246 return NewCall;
4247}
4248
4249Instruction *InstCombinerImpl::foldPtrAuthConstantCallee(CallBase &Call) {
4251 if (!CPA)
4252 return nullptr;
4253
4254 auto *CalleeF = dyn_cast<Function>(CPA->getPointer());
4255 // If the ptrauth constant isn't based on a function pointer, bail out.
4256 if (!CalleeF)
4257 return nullptr;
4258
4259 // Inspect the call ptrauth bundle to check it matches the ptrauth constant.
4261 if (!PAB)
4262 return nullptr;
4263
4264 auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
4265 Value *Discriminator = PAB->Inputs[1];
4266
4267 // If the bundle doesn't match, this is probably going to fail to auth.
4268 if (!CPA->isKnownCompatibleWith(Key, Discriminator, DL))
4269 return nullptr;
4270
4271 // If the bundle matches the constant, proceed in making this a direct call.
4273 NewCall->setCalledOperand(CalleeF);
4274 return NewCall;
4275}
4276
4277bool InstCombinerImpl::annotateAnyAllocSite(CallBase &Call,
4278 const TargetLibraryInfo *TLI) {
4279 // Note: We only handle cases which can't be driven from generic attributes
4280 // here. So, for example, nonnull and noalias (which are common properties
4281 // of some allocation functions) are expected to be handled via annotation
4282 // of the respective allocator declaration with generic attributes.
4283 bool Changed = false;
4284
4285 if (!Call.getType()->isPointerTy())
4286 return Changed;
4287
4288 std::optional<APInt> Size = getAllocSize(&Call, TLI);
4289 if (Size && *Size != 0) {
4290 // TODO: We really should just emit deref_or_null here and then
4291 // let the generic inference code combine that with nonnull.
4292 if (Call.hasRetAttr(Attribute::NonNull)) {
4293 Changed = !Call.hasRetAttr(Attribute::Dereferenceable);
4295 Call.getContext(), Size->getLimitedValue()));
4296 } else {
4297 Changed = !Call.hasRetAttr(Attribute::DereferenceableOrNull);
4299 Call.getContext(), Size->getLimitedValue()));
4300 }
4301 }
4302
4303 // Add alignment attribute if alignment is a power of two constant.
4304 Value *Alignment = getAllocAlignment(&Call, TLI);
4305 if (!Alignment)
4306 return Changed;
4307
4308 ConstantInt *AlignOpC = dyn_cast<ConstantInt>(Alignment);
4309 if (AlignOpC && AlignOpC->getValue().ult(llvm::Value::MaximumAlignment)) {
4310 uint64_t AlignmentVal = AlignOpC->getZExtValue();
4311 if (llvm::isPowerOf2_64(AlignmentVal)) {
4312 Align ExistingAlign = Call.getRetAlign().valueOrOne();
4313 Align NewAlign = Align(AlignmentVal);
4314 if (NewAlign > ExistingAlign) {
4317 Changed = true;
4318 }
4319 }
4320 }
4321 return Changed;
4322}
4323
4324/// Improvements for call, callbr and invoke instructions.
4325Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
4326 bool Changed = annotateAnyAllocSite(Call, &TLI);
4327
4328 // Mark any parameters that are known to be non-null with the nonnull
4329 // attribute. This is helpful for inlining calls to functions with null
4330 // checks on their arguments.
4331 SmallVector<unsigned, 4> ArgNos;
4332 unsigned ArgNo = 0;
4333
4334 for (Value *V : Call.args()) {
4335 if (V->getType()->isPointerTy()) {
4336 // Simplify the nonnull operand if the parameter is known to be nonnull.
4337 // Otherwise, try to infer nonnull for it.
4338 bool HasDereferenceable = Call.getParamDereferenceableBytes(ArgNo) > 0;
4339 if (Call.paramHasAttr(ArgNo, Attribute::NonNull) ||
4340 (HasDereferenceable &&
4342 V->getType()->getPointerAddressSpace()))) {
4343 if (Value *Res = simplifyNonNullOperand(V, HasDereferenceable)) {
4344 replaceOperand(Call, ArgNo, Res);
4345 Changed = true;
4346 }
4347 } else if (isKnownNonZero(V,
4348 getSimplifyQuery().getWithInstruction(&Call))) {
4349 ArgNos.push_back(ArgNo);
4350 }
4351 }
4352 ArgNo++;
4353 }
4354
4355 assert(ArgNo == Call.arg_size() && "Call arguments not processed correctly.");
4356
4357 if (!ArgNos.empty()) {
4358 AttributeList AS = Call.getAttributes();
4359 LLVMContext &Ctx = Call.getContext();
4360 AS = AS.addParamAttribute(Ctx, ArgNos,
4361 Attribute::get(Ctx, Attribute::NonNull));
4362 Call.setAttributes(AS);
4363 Changed = true;
4364 }
4365
4366 // If the callee is a pointer to a function, attempt to move any casts to the
4367 // arguments of the call/callbr/invoke.
4369 Function *CalleeF = dyn_cast<Function>(Callee);
4370 if ((!CalleeF || CalleeF->getFunctionType() != Call.getFunctionType()) &&
4371 transformConstExprCastCall(Call))
4372 return nullptr;
4373
4374 if (CalleeF) {
4375 // Remove the convergent attr on calls when the callee is not convergent.
4376 if (Call.isConvergent() && !CalleeF->isConvergent() &&
4377 !CalleeF->isIntrinsic()) {
4378 LLVM_DEBUG(dbgs() << "Removing convergent attr from instr " << Call
4379 << "\n");
4381 return &Call;
4382 }
4383
4384 // If the call and callee calling conventions don't match, and neither one
4385 // of the calling conventions is compatible with C calling convention
4386 // this call must be unreachable, as the call is undefined.
4387 if ((CalleeF->getCallingConv() != Call.getCallingConv() &&
4388 !(CalleeF->getCallingConv() == llvm::CallingConv::C &&
4392 // Only do this for calls to a function with a body. A prototype may
4393 // not actually end up matching the implementation's calling conv for a
4394 // variety of reasons (e.g. it may be written in assembly).
4395 !CalleeF->isDeclaration()) {
4396 Instruction *OldCall = &Call;
4398 // If OldCall does not return void then replaceInstUsesWith poison.
4399 // This allows ValueHandlers and custom metadata to adjust itself.
4400 if (!OldCall->getType()->isVoidTy())
4401 replaceInstUsesWith(*OldCall, PoisonValue::get(OldCall->getType()));
4402 if (isa<CallInst>(OldCall))
4403 return eraseInstFromFunction(*OldCall);
4404
4405 // We cannot remove an invoke or a callbr, because it would change thexi
4406 // CFG, just change the callee to a null pointer.
4407 cast<CallBase>(OldCall)->setCalledFunction(
4408 CalleeF->getFunctionType(),
4409 Constant::getNullValue(CalleeF->getType()));
4410 return nullptr;
4411 }
4412 }
4413
4414 // Calling a null function pointer is undefined if a null address isn't
4415 // dereferenceable.
4416 if ((isa<ConstantPointerNull>(Callee) &&
4418 isa<UndefValue>(Callee)) {
4419 // If Call does not return void then replaceInstUsesWith poison.
4420 // This allows ValueHandlers and custom metadata to adjust itself.
4421 if (!Call.getType()->isVoidTy())
4423
4424 if (Call.isTerminator()) {
4425 // Can't remove an invoke or callbr because we cannot change the CFG.
4426 return nullptr;
4427 }
4428
4429 // This instruction is not reachable, just remove it.
4432 }
4433
4434 if (IntrinsicInst *II = findInitTrampoline(Callee))
4435 return transformCallThroughTrampoline(Call, *II);
4436
4437 // Combine calls involving pointer authentication intrinsics.
4438 if (Instruction *NewCall = foldPtrAuthIntrinsicCallee(Call))
4439 return NewCall;
4440
4441 // Combine calls to ptrauth constants.
4442 if (Instruction *NewCall = foldPtrAuthConstantCallee(Call))
4443 return NewCall;
4444
4445 if (isa<InlineAsm>(Callee) && !Call.doesNotThrow()) {
4446 InlineAsm *IA = cast<InlineAsm>(Callee);
4447 if (!IA->canThrow()) {
4448 // Normal inline asm calls cannot throw - mark them
4449 // 'nounwind'.
4451 Changed = true;
4452 }
4453 }
4454
4455 // Try to optimize the call if possible, we require DataLayout for most of
4456 // this. None of these calls are seen as possibly dead so go ahead and
4457 // delete the instruction now.
4458 if (CallInst *CI = dyn_cast<CallInst>(&Call)) {
4459 Instruction *I = tryOptimizeCall(CI);
4460 // If we changed something return the result, etc. Otherwise let
4461 // the fallthrough check.
4462 if (I) return eraseInstFromFunction(*I);
4463 }
4464
4465 if (!Call.use_empty() && !Call.isMustTailCall())
4466 if (Value *ReturnedArg = Call.getReturnedArgOperand()) {
4467 Type *CallTy = Call.getType();
4468 Type *RetArgTy = ReturnedArg->getType();
4469 if (RetArgTy->canLosslesslyBitCastTo(CallTy))
4470 return replaceInstUsesWith(
4471 Call, Builder.CreateBitOrPointerCast(ReturnedArg, CallTy));
4472 }
4473
4474 // Drop unnecessary callee_type metadata from calls that were converted
4475 // into direct calls.
4476 if (Call.getMetadata(LLVMContext::MD_callee_type) && !Call.isIndirectCall()) {
4477 Call.setMetadata(LLVMContext::MD_callee_type, nullptr);
4478 Changed = true;
4479 }
4480
4481 // Drop unnecessary kcfi operand bundles from calls that were converted
4482 // into direct calls.
4484 if (Bundle && !Call.isIndirectCall()) {
4485 DEBUG_WITH_TYPE(DEBUG_TYPE "-kcfi", {
4486 if (CalleeF) {
4487 ConstantInt *FunctionType = nullptr;
4488 ConstantInt *ExpectedType = cast<ConstantInt>(Bundle->Inputs[0]);
4489
4490 if (MDNode *MD = CalleeF->getMetadata(LLVMContext::MD_kcfi_type))
4491 FunctionType = mdconst::extract<ConstantInt>(MD->getOperand(0));
4492
4493 if (FunctionType &&
4494 FunctionType->getZExtValue() != ExpectedType->getZExtValue())
4495 dbgs() << Call.getModule()->getName()
4496 << ": warning: kcfi: " << Call.getCaller()->getName()
4497 << ": call to " << CalleeF->getName()
4498 << " using a mismatching function pointer type\n";
4499 }
4500 });
4501
4503 }
4504
4505 if (isRemovableAlloc(&Call, &TLI))
4506 return visitAllocSite(Call);
4507
4508 // Handle intrinsics which can be used in both call and invoke context.
4509 switch (Call.getIntrinsicID()) {
4510 case Intrinsic::experimental_gc_statepoint: {
4511 GCStatepointInst &GCSP = *cast<GCStatepointInst>(&Call);
4512 SmallPtrSet<Value *, 32> LiveGcValues;
4513 for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {
4514 GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);
4515
4516 // Remove the relocation if unused.
4517 if (GCR.use_empty()) {
4519 continue;
4520 }
4521
4522 Value *DerivedPtr = GCR.getDerivedPtr();
4523 Value *BasePtr = GCR.getBasePtr();
4524
4525 // Undef is undef, even after relocation.
4526 if (isa<UndefValue>(DerivedPtr) || isa<UndefValue>(BasePtr)) {
4529 continue;
4530 }
4531
4532 if (auto *PT = dyn_cast<PointerType>(GCR.getType())) {
4533 // The relocation of null will be null for most any collector.
4534 // TODO: provide a hook for this in GCStrategy. There might be some
4535 // weird collector this property does not hold for.
4536 if (isa<ConstantPointerNull>(DerivedPtr)) {
4537 // Use null-pointer of gc_relocate's type to replace it.
4540 continue;
4541 }
4542
4543 // isKnownNonNull -> nonnull attribute
4544 if (!GCR.hasRetAttr(Attribute::NonNull) &&
4545 isKnownNonZero(DerivedPtr,
4546 getSimplifyQuery().getWithInstruction(&Call))) {
4547 GCR.addRetAttr(Attribute::NonNull);
4548 // We discovered new fact, re-check users.
4549 Worklist.pushUsersToWorkList(GCR);
4550 }
4551 }
4552
4553 // If we have two copies of the same pointer in the statepoint argument
4554 // list, canonicalize to one. This may let us common gc.relocates.
4555 if (GCR.getBasePtr() == GCR.getDerivedPtr() &&
4556 GCR.getBasePtrIndex() != GCR.getDerivedPtrIndex()) {
4557 auto *OpIntTy = GCR.getOperand(2)->getType();
4558 GCR.setOperand(2, ConstantInt::get(OpIntTy, GCR.getBasePtrIndex()));
4559 }
4560
4561 // TODO: bitcast(relocate(p)) -> relocate(bitcast(p))
4562 // Canonicalize on the type from the uses to the defs
4563
4564 // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...)
4565 LiveGcValues.insert(BasePtr);
4566 LiveGcValues.insert(DerivedPtr);
4567 }
4568 std::optional<OperandBundleUse> Bundle =
4570 unsigned NumOfGCLives = LiveGcValues.size();
4571 if (!Bundle || NumOfGCLives == Bundle->Inputs.size())
4572 break;
4573 // We can reduce the size of gc live bundle.
4574 DenseMap<Value *, unsigned> Val2Idx;
4575 std::vector<Value *> NewLiveGc;
4576 for (Value *V : Bundle->Inputs) {
4577 auto [It, Inserted] = Val2Idx.try_emplace(V);
4578 if (!Inserted)
4579 continue;
4580 if (LiveGcValues.count(V)) {
4581 It->second = NewLiveGc.size();
4582 NewLiveGc.push_back(V);
4583 } else
4584 It->second = NumOfGCLives;
4585 }
4586 // Update all gc.relocates
4587 for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {
4588 GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);
4589 Value *BasePtr = GCR.getBasePtr();
4590 assert(Val2Idx.count(BasePtr) && Val2Idx[BasePtr] != NumOfGCLives &&
4591 "Missed live gc for base pointer");
4592 auto *OpIntTy1 = GCR.getOperand(1)->getType();
4593 GCR.setOperand(1, ConstantInt::get(OpIntTy1, Val2Idx[BasePtr]));
4594 Value *DerivedPtr = GCR.getDerivedPtr();
4595 assert(Val2Idx.count(DerivedPtr) && Val2Idx[DerivedPtr] != NumOfGCLives &&
4596 "Missed live gc for derived pointer");
4597 auto *OpIntTy2 = GCR.getOperand(2)->getType();
4598 GCR.setOperand(2, ConstantInt::get(OpIntTy2, Val2Idx[DerivedPtr]));
4599 }
4600 // Create new statepoint instruction.
4601 OperandBundleDef NewBundle("gc-live", NewLiveGc);
4602 return CallBase::Create(&Call, NewBundle);
4603 }
4604 default: { break; }
4605 }
4606
4607 return Changed ? &Call : nullptr;
4608}
4609
4610/// If the callee is a constexpr cast of a function, attempt to move the cast to
4611/// the arguments of the call/invoke.
4612/// CallBrInst is not supported.
4613bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
4614 auto *Callee =
4616 if (!Callee)
4617 return false;
4618
4620 "CallBr's don't have a single point after a def to insert at");
4621
4622 // Don't perform the transform for declarations, which may not be fully
4623 // accurate. For example, void @foo() is commonly used as a placeholder for
4624 // unknown prototypes.
4625 if (Callee->isDeclaration())
4626 return false;
4627
4628 // If this is a call to a thunk function, don't remove the cast. Thunks are
4629 // used to transparently forward all incoming parameters and outgoing return
4630 // values, so it's important to leave the cast in place.
4631 if (Callee->hasFnAttribute("thunk"))
4632 return false;
4633
4634 // If this is a call to a naked function, the assembly might be
4635 // using an argument, or otherwise rely on the frame layout,
4636 // the function prototype will mismatch.
4637 if (Callee->hasFnAttribute(Attribute::Naked))
4638 return false;
4639
4640 // If this is a musttail call, the callee's prototype must match the caller's
4641 // prototype with the exception of pointee types. The code below doesn't
4642 // implement that, so we can't do this transform.
4643 // TODO: Do the transform if it only requires adding pointer casts.
4644 if (Call.isMustTailCall())
4645 return false;
4646
4648 const AttributeList &CallerPAL = Call.getAttributes();
4649
4650 // Okay, this is a cast from a function to a different type. Unless doing so
4651 // would cause a type conversion of one of our arguments, change this call to
4652 // be a direct call with arguments casted to the appropriate types.
4653 FunctionType *FT = Callee->getFunctionType();
4654 Type *OldRetTy = Caller->getType();
4655 Type *NewRetTy = FT->getReturnType();
4656
4657 // Check to see if we are changing the return type...
4658 if (OldRetTy != NewRetTy) {
4659
4660 if (NewRetTy->isStructTy())
4661 return false; // TODO: Handle multiple return values.
4662
4663 if (!CastInst::isBitOrNoopPointerCastable(NewRetTy, OldRetTy, DL)) {
4664 if (!Caller->use_empty())
4665 return false; // Cannot transform this return value.
4666 }
4667
4668 if (!CallerPAL.isEmpty() && !Caller->use_empty()) {
4669 AttrBuilder RAttrs(FT->getContext(), CallerPAL.getRetAttrs());
4670 if (RAttrs.overlaps(AttributeFuncs::typeIncompatible(
4671 NewRetTy, CallerPAL.getRetAttrs())))
4672 return false; // Attribute not compatible with transformed value.
4673 }
4674
4675 // If the callbase is an invoke instruction, and the return value is
4676 // used by a PHI node in a successor, we cannot change the return type of
4677 // the call because there is no place to put the cast instruction (without
4678 // breaking the critical edge). Bail out in this case.
4679 if (!Caller->use_empty()) {
4680 BasicBlock *PhisNotSupportedBlock = nullptr;
4681 if (auto *II = dyn_cast<InvokeInst>(Caller))
4682 PhisNotSupportedBlock = II->getNormalDest();
4683 if (PhisNotSupportedBlock)
4684 for (User *U : Caller->users())
4685 if (PHINode *PN = dyn_cast<PHINode>(U))
4686 if (PN->getParent() == PhisNotSupportedBlock)
4687 return false;
4688 }
4689 }
4690
4691 unsigned NumActualArgs = Call.arg_size();
4692 unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);
4693
4694 // Prevent us turning:
4695 // declare void @takes_i32_inalloca(i32* inalloca)
4696 // call void bitcast (void (i32*)* @takes_i32_inalloca to void (i32)*)(i32 0)
4697 //
4698 // into:
4699 // call void @takes_i32_inalloca(i32* null)
4700 //
4701 // Similarly, avoid folding away bitcasts of byval calls.
4702 if (Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) ||
4703 Callee->getAttributes().hasAttrSomewhere(Attribute::Preallocated))
4704 return false;
4705
4706 auto AI = Call.arg_begin();
4707 for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {
4708 Type *ParamTy = FT->getParamType(i);
4709 Type *ActTy = (*AI)->getType();
4710
4711 if (!CastInst::isBitOrNoopPointerCastable(ActTy, ParamTy, DL))
4712 return false; // Cannot transform this parameter value.
4713
4714 // Check if there are any incompatible attributes we cannot drop safely.
4715 if (AttrBuilder(FT->getContext(), CallerPAL.getParamAttrs(i))
4716 .overlaps(AttributeFuncs::typeIncompatible(
4717 ParamTy, CallerPAL.getParamAttrs(i),
4718 AttributeFuncs::ASK_UNSAFE_TO_DROP)))
4719 return false; // Attribute not compatible with transformed value.
4720
4721 if (Call.isInAllocaArgument(i) ||
4722 CallerPAL.hasParamAttr(i, Attribute::Preallocated))
4723 return false; // Cannot transform to and from inalloca/preallocated.
4724
4725 if (CallerPAL.hasParamAttr(i, Attribute::SwiftError))
4726 return false;
4727
4728 if (CallerPAL.hasParamAttr(i, Attribute::ByVal) !=
4729 Callee->getAttributes().hasParamAttr(i, Attribute::ByVal))
4730 return false; // Cannot transform to or from byval.
4731 }
4732
4733 if (FT->getNumParams() < NumActualArgs && FT->isVarArg() &&
4734 !CallerPAL.isEmpty()) {
4735 // In this case we have more arguments than the new function type, but we
4736 // won't be dropping them. Check that these extra arguments have attributes
4737 // that are compatible with being a vararg call argument.
4738 unsigned SRetIdx;
4739 if (CallerPAL.hasAttrSomewhere(Attribute::StructRet, &SRetIdx) &&
4740 SRetIdx - AttributeList::FirstArgIndex >= FT->getNumParams())
4741 return false;
4742 }
4743
4744 // Okay, we decided that this is a safe thing to do: go ahead and start
4745 // inserting cast instructions as necessary.
4746 SmallVector<Value *, 8> Args;
4748 Args.reserve(NumActualArgs);
4749 ArgAttrs.reserve(NumActualArgs);
4750
4751 // Get any return attributes.
4752 AttrBuilder RAttrs(FT->getContext(), CallerPAL.getRetAttrs());
4753
4754 // If the return value is not being used, the type may not be compatible
4755 // with the existing attributes. Wipe out any problematic attributes.
4756 RAttrs.remove(
4757 AttributeFuncs::typeIncompatible(NewRetTy, CallerPAL.getRetAttrs()));
4758
4759 LLVMContext &Ctx = Call.getContext();
4760 AI = Call.arg_begin();
4761 for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
4762 Type *ParamTy = FT->getParamType(i);
4763
4764 Value *NewArg = *AI;
4765 if ((*AI)->getType() != ParamTy)
4766 NewArg = Builder.CreateBitOrPointerCast(*AI, ParamTy);
4767 Args.push_back(NewArg);
4768
4769 // Add any parameter attributes except the ones incompatible with the new
4770 // type. Note that we made sure all incompatible ones are safe to drop.
4771 AttributeMask IncompatibleAttrs = AttributeFuncs::typeIncompatible(
4772 ParamTy, CallerPAL.getParamAttrs(i), AttributeFuncs::ASK_SAFE_TO_DROP);
4773 ArgAttrs.push_back(
4774 CallerPAL.getParamAttrs(i).removeAttributes(Ctx, IncompatibleAttrs));
4775 }
4776
4777 // If the function takes more arguments than the call was taking, add them
4778 // now.
4779 for (unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i) {
4780 Args.push_back(Constant::getNullValue(FT->getParamType(i)));
4781 ArgAttrs.push_back(AttributeSet());
4782 }
4783
4784 // If we are removing arguments to the function, emit an obnoxious warning.
4785 if (FT->getNumParams() < NumActualArgs) {
4786 // TODO: if (!FT->isVarArg()) this call may be unreachable. PR14722
4787 if (FT->isVarArg()) {
4788 // Add all of the arguments in their promoted form to the arg list.
4789 for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {
4790 Type *PTy = getPromotedType((*AI)->getType());
4791 Value *NewArg = *AI;
4792 if (PTy != (*AI)->getType()) {
4793 // Must promote to pass through va_arg area!
4794 Instruction::CastOps opcode =
4795 CastInst::getCastOpcode(*AI, false, PTy, false);
4796 NewArg = Builder.CreateCast(opcode, *AI, PTy);
4797 }
4798 Args.push_back(NewArg);
4799
4800 // Add any parameter attributes.
4801 ArgAttrs.push_back(CallerPAL.getParamAttrs(i));
4802 }
4803 }
4804 }
4805
4806 AttributeSet FnAttrs = CallerPAL.getFnAttrs();
4807
4808 if (NewRetTy->isVoidTy())
4809 Caller->setName(""); // Void type should not have a name.
4810
4811 assert((ArgAttrs.size() == FT->getNumParams() || FT->isVarArg()) &&
4812 "missing argument attributes");
4813 AttributeList NewCallerPAL = AttributeList::get(
4814 Ctx, FnAttrs, AttributeSet::get(Ctx, RAttrs), ArgAttrs);
4815
4817 Call.getOperandBundlesAsDefs(OpBundles);
4818
4819 CallBase *NewCall;
4820 if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
4821 NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(),
4822 II->getUnwindDest(), Args, OpBundles);
4823 } else {
4824 NewCall = Builder.CreateCall(Callee, Args, OpBundles);
4825 cast<CallInst>(NewCall)->setTailCallKind(
4826 cast<CallInst>(Caller)->getTailCallKind());
4827 }
4828 NewCall->takeName(Caller);
4830 NewCall->setAttributes(NewCallerPAL);
4831
4832 // Preserve prof metadata if any.
4833 NewCall->copyMetadata(*Caller, {LLVMContext::MD_prof});
4834
4835 // Insert a cast of the return type as necessary.
4836 Instruction *NC = NewCall;
4837 Value *NV = NC;
4838 if (OldRetTy != NV->getType() && !Caller->use_empty()) {
4839 assert(!NV->getType()->isVoidTy());
4841 NC->setDebugLoc(Caller->getDebugLoc());
4842
4843 auto OptInsertPt = NewCall->getInsertionPointAfterDef();
4844 assert(OptInsertPt && "No place to insert cast");
4845 InsertNewInstBefore(NC, *OptInsertPt);
4846 Worklist.pushUsersToWorkList(*Caller);
4847 }
4848
4849 if (!Caller->use_empty())
4850 replaceInstUsesWith(*Caller, NV);
4851 else if (Caller->hasValueHandle()) {
4852 if (OldRetTy == NV->getType())
4854 else
4855 // We cannot call ValueIsRAUWd with a different type, and the
4856 // actual tracked value will disappear.
4858 }
4859
4860 eraseInstFromFunction(*Caller);
4861 return true;
4862}
4863
4864/// Turn a call to a function created by init_trampoline / adjust_trampoline
4865/// intrinsic pair into a direct call to the underlying function.
4867InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call,
4868 IntrinsicInst &Tramp) {
4869 FunctionType *FTy = Call.getFunctionType();
4870 AttributeList Attrs = Call.getAttributes();
4871
4872 // If the call already has the 'nest' attribute somewhere then give up -
4873 // otherwise 'nest' would occur twice after splicing in the chain.
4874 if (Attrs.hasAttrSomewhere(Attribute::Nest))
4875 return nullptr;
4876
4878 FunctionType *NestFTy = NestF->getFunctionType();
4879
4880 AttributeList NestAttrs = NestF->getAttributes();
4881 if (!NestAttrs.isEmpty()) {
4882 unsigned NestArgNo = 0;
4883 Type *NestTy = nullptr;
4884 AttributeSet NestAttr;
4885
4886 // Look for a parameter marked with the 'nest' attribute.
4887 for (FunctionType::param_iterator I = NestFTy->param_begin(),
4888 E = NestFTy->param_end();
4889 I != E; ++NestArgNo, ++I) {
4890 AttributeSet AS = NestAttrs.getParamAttrs(NestArgNo);
4891 if (AS.hasAttribute(Attribute::Nest)) {
4892 // Record the parameter type and any other attributes.
4893 NestTy = *I;
4894 NestAttr = AS;
4895 break;
4896 }
4897 }
4898
4899 if (NestTy) {
4900 std::vector<Value*> NewArgs;
4901 std::vector<AttributeSet> NewArgAttrs;
4902 NewArgs.reserve(Call.arg_size() + 1);
4903 NewArgAttrs.reserve(Call.arg_size());
4904
4905 // Insert the nest argument into the call argument list, which may
4906 // mean appending it. Likewise for attributes.
4907
4908 {
4909 unsigned ArgNo = 0;
4910 auto I = Call.arg_begin(), E = Call.arg_end();
4911 do {
4912 if (ArgNo == NestArgNo) {
4913 // Add the chain argument and attributes.
4914 Value *NestVal = Tramp.getArgOperand(2);
4915 if (NestVal->getType() != NestTy)
4916 NestVal = Builder.CreateBitCast(NestVal, NestTy, "nest");
4917 NewArgs.push_back(NestVal);
4918 NewArgAttrs.push_back(NestAttr);
4919 }
4920
4921 if (I == E)
4922 break;
4923
4924 // Add the original argument and attributes.
4925 NewArgs.push_back(*I);
4926 NewArgAttrs.push_back(Attrs.getParamAttrs(ArgNo));
4927
4928 ++ArgNo;
4929 ++I;
4930 } while (true);
4931 }
4932
4933 // The trampoline may have been bitcast to a bogus type (FTy).
4934 // Handle this by synthesizing a new function type, equal to FTy
4935 // with the chain parameter inserted.
4936
4937 std::vector<Type*> NewTypes;
4938 NewTypes.reserve(FTy->getNumParams()+1);
4939
4940 // Insert the chain's type into the list of parameter types, which may
4941 // mean appending it.
4942 {
4943 unsigned ArgNo = 0;
4944 FunctionType::param_iterator I = FTy->param_begin(),
4945 E = FTy->param_end();
4946
4947 do {
4948 if (ArgNo == NestArgNo)
4949 // Add the chain's type.
4950 NewTypes.push_back(NestTy);
4951
4952 if (I == E)
4953 break;
4954
4955 // Add the original type.
4956 NewTypes.push_back(*I);
4957
4958 ++ArgNo;
4959 ++I;
4960 } while (true);
4961 }
4962
4963 // Replace the trampoline call with a direct call. Let the generic
4964 // code sort out any function type mismatches.
4965 FunctionType *NewFTy =
4966 FunctionType::get(FTy->getReturnType(), NewTypes, FTy->isVarArg());
4967 AttributeList NewPAL =
4968 AttributeList::get(FTy->getContext(), Attrs.getFnAttrs(),
4969 Attrs.getRetAttrs(), NewArgAttrs);
4970
4972 Call.getOperandBundlesAsDefs(OpBundles);
4973
4974 Instruction *NewCaller;
4975 if (InvokeInst *II = dyn_cast<InvokeInst>(&Call)) {
4976 NewCaller = InvokeInst::Create(NewFTy, NestF, II->getNormalDest(),
4977 II->getUnwindDest(), NewArgs, OpBundles);
4978 cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv());
4979 cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);
4980 } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&Call)) {
4981 NewCaller =
4982 CallBrInst::Create(NewFTy, NestF, CBI->getDefaultDest(),
4983 CBI->getIndirectDests(), NewArgs, OpBundles);
4984 cast<CallBrInst>(NewCaller)->setCallingConv(CBI->getCallingConv());
4985 cast<CallBrInst>(NewCaller)->setAttributes(NewPAL);
4986 } else {
4987 NewCaller = CallInst::Create(NewFTy, NestF, NewArgs, OpBundles);
4988 cast<CallInst>(NewCaller)->setTailCallKind(
4989 cast<CallInst>(Call).getTailCallKind());
4990 cast<CallInst>(NewCaller)->setCallingConv(
4991 cast<CallInst>(Call).getCallingConv());
4992 cast<CallInst>(NewCaller)->setAttributes(NewPAL);
4993 }
4994 NewCaller->setDebugLoc(Call.getDebugLoc());
4995
4996 return NewCaller;
4997 }
4998 }
4999
5000 // Replace the trampoline call with a direct call. Since there is no 'nest'
5001 // parameter, there is no need to adjust the argument list. Let the generic
5002 // code sort out any function type mismatches.
5003 Call.setCalledFunction(FTy, NestF);
5004 return &Call;
5005}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
BitTracker BT
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static SDValue foldBitOrderCrossLogicOp(SDNode *N, SelectionDAG &DAG)
#define DEBUG_TYPE
IRTranslator LLVM IR MI
static Type * getPromotedType(Type *Ty)
Return the specified type promoted as it would be to pass though a va_arg area.
static Instruction * createOverflowTuple(IntrinsicInst *II, Value *Result, Constant *Overflow)
Creates a result tuple for an overflow intrinsic II with a given Result and a constant Overflow value...
static IntrinsicInst * findInitTrampolineFromAlloca(Value *TrampMem)
static bool removeTriviallyEmptyRange(IntrinsicInst &EndI, InstCombinerImpl &IC, std::function< bool(const IntrinsicInst &)> IsStart)
static bool inputDenormalIsDAZ(const Function &F, const Type *Ty)
static Instruction * reassociateMinMaxWithConstantInOperand(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If this min/max has a matching min/max operand with a constant, try to push the constant operand into...
static bool isIdempotentBinaryIntrinsic(Intrinsic::ID IID)
Helper to match idempotent binary intrinsics, namely, intrinsics where f(f(x, y), y) == f(x,...
static bool signBitMustBeTheSame(Value *Op0, Value *Op1, const SimplifyQuery &SQ)
Return true if two values Op0 and Op1 are known to have the same sign.
static Instruction * moveAddAfterMinMax(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0.
static Instruction * simplifyInvariantGroupIntrinsic(IntrinsicInst &II, InstCombinerImpl &IC)
This function transforms launder.invariant.group and strip.invariant.group like: launder(launder(x)) ...
static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E, unsigned NumOperands)
static std::optional< bool > getKnownSign(Value *Op, const SimplifyQuery &SQ)
static cl::opt< unsigned > GuardWideningWindow("instcombine-guard-widening-window", cl::init(3), cl::desc("How wide an instruction window to bypass looking for " "another guard"))
static bool hasUndefSource(AnyMemTransferInst *MI)
Recognize a memcpy/memmove from a trivially otherwise unused alloca.
static Instruction * factorizeMinMaxTree(IntrinsicInst *II)
Reduce a sequence of min/max intrinsics with a common operand.
static Value * simplifyNeonTbl1(const IntrinsicInst &II, InstCombiner::BuilderTy &Builder)
Convert a table lookup to shufflevector if the mask is constant.
static Instruction * foldClampRangeOfTwo(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If we have a clamp pattern like max (min X, 42), 41 – where the output can only be one of two possibl...
static Value * simplifyReductionOperand(Value *Arg, bool CanReorderLanes)
static IntrinsicInst * findInitTrampolineFromBB(IntrinsicInst *AdjustTramp, Value *TrampMem)
static Value * foldIntrinsicUsingDistributiveLaws(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
static std::optional< bool > getKnownSignOrZero(Value *Op, const SimplifyQuery &SQ)
static Value * foldMinimumOverTrailingOrLeadingZeroCount(Value *I0, Value *I1, const DataLayout &DL, InstCombiner::BuilderTy &Builder)
Fold an unsigned minimum of trailing or leading zero bits counts: umin(cttz(CtOp, ZeroUndef),...
static Value * foldIdempotentBinaryIntrinsicRecurrence(InstCombinerImpl &IC, IntrinsicInst *II)
Attempt to simplify value-accumulating recurrences of kind: umax.acc = phi i8 [ umax,...
static Instruction * foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC)
static Instruction * foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC)
static IntrinsicInst * findInitTrampoline(Value *Callee)
static FCmpInst::Predicate fpclassTestIsFCmp0(FPClassTest Mask, const Function &F, Type *Ty)
static bool leftDistributesOverRight(Instruction::BinaryOps LOp, bool HasNUW, bool HasNSW, Intrinsic::ID ROp)
Return whether "X LOp (Y ROp Z)" is always equal to "(X LOp Y) ROp (X LOp Z)".
static Value * reassociateMinMaxWithConstants(IntrinsicInst *II, IRBuilderBase &Builder, const SimplifyQuery &SQ)
If this min/max has a constant operand and an operand that is a matching min/max with a constant oper...
static CallInst * canonicalizeConstantArg0ToArg1(CallInst &Call)
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
static bool hasNoSignedWrap(BinaryOperator &I)
static bool inputDenormalIsIEEE(DenormalMode Mode)
Return true if it's possible to assume IEEE treatment of input denormals in F for Val.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
static const Function * getCalledFunction(const Value *V)
This file contains the declarations for metadata subclasses.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod
const SmallVectorImpl< MachineOperand > & Cond
This file implements the SmallBitVector class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:114
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition Debug.h:72
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition VPlanSLP.cpp:247
Value * RHS
Value * LHS
bool isNegative() const
Definition APFloat.h:1431
void clearSign()
Definition APFloat.h:1280
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:234
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:229
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition APInt.h:1201
LLVM_ABI APInt usub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1948
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1182
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:380
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1666
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1488
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1111
LLVM_ABI APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1928
LLVM_ABI APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1935
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition APInt.cpp:651
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:219
LLVM_ABI APInt uadd_sat(const APInt &RHS) const
Definition APInt.cpp:2036
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:334
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:306
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:200
LLVM_ABI APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1941
static APSInt getMinValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the minimum integer value with the given bit width and signedness.
Definition APSInt.h:312
static APSInt getMaxValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the maximum integer value with the given bit width and signedness.
Definition APSInt.h:304
This class represents any memset intrinsic.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
static LLVM_ABI AttributeSet get(LLVMContext &C, const AttrBuilder &B)
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static LLVM_ABI Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
InstListType::reverse_iterator reverse_iterator
Definition BasicBlock.h:172
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI bool isSigned() const
Whether the intrinsic is signed or unsigned.
LLVM_ABI Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
static BinaryOperator * CreateFAddFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:236
static LLVM_ABI BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
static BinaryOperator * CreateNSW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition InstrTypes.h:279
static LLVM_ABI BinaryOperator * CreateNot(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition InstrTypes.h:294
static BinaryOperator * CreateFMulFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:244
static BinaryOperator * CreateFDivFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:248
static BinaryOperator * CreateFSubFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:240
static LLVM_ABI BinaryOperator * CreateNSWNeg(Value *Op, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
void setDoesNotThrow()
MaybeAlign getRetAlign() const
Extract the alignment of the return value.
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
OperandBundleUse getOperandBundleAt(unsigned Index) const
Return the operand bundle at a specific index.
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool isInAllocaArgument(unsigned ArgNo) const
Determine whether this argument is passed in an alloca.
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
unsigned getNumOperandBundles() const
Return the number of operand bundles associated with this User.
uint64_t getParamDereferenceableBytes(unsigned i) const
Extract the number of dereferenceable bytes for a call or parameter (0=unknown).
CallingConv::ID getCallingConv() const
LLVM_ABI bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
void setNotConvergent()
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
bool doesNotThrow() const
Determine if the call cannot unwind.
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Value * getReturnedArgOperand() const
If one of the arguments has the 'returned' attribute, returns its operand value.
static LLVM_ABI CallBase * Create(CallBase *CB, ArrayRef< OperandBundleDef > Bundles, InsertPosition InsertPt=nullptr)
Create a clone of CB with a different set of operand bundles and insert it before InsertPt.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
static LLVM_ABI CallBase * removeOperandBundle(CallBase *CB, uint32_t ID, InsertPosition InsertPt=nullptr)
Create a clone of CB with operand bundle ID removed.
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
static CallBrInst * Create(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, ArrayRef< BasicBlock * > IndirectDests, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
This class represents a function call, abstracting a target machine's calling convention.
bool isNoTailCall() const
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
bool isMustTailCall() const
static LLVM_ABI Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
static LLVM_ABI CastInst * CreateIntegerCast(Value *S, Type *Ty, bool isSigned, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a ZExt, BitCast, or Trunc for int -> int casts.
static LLVM_ABI bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
static LLVM_ABI CastInst * CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:676
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:679
@ ICMP_SLT
signed less than
Definition InstrTypes.h:705
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:706
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:682
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:680
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:681
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:699
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:703
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:684
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:687
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:701
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:683
@ ICMP_NE
not equal
Definition InstrTypes.h:698
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:692
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:827
Predicate getNonStrictPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
Definition InstrTypes.h:871
Predicate getUnorderedPredicate() const
Definition InstrTypes.h:811
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
static LLVM_ABI Constant * getInfinity(Type *Ty, bool Negative=false)
static LLVM_ABI Constant * getZero(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition Constants.h:264
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:163
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:154
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI ConstantPtrAuth * get(Constant *Ptr, ConstantInt *Key, ConstantInt *Disc, Constant *AddrDisc)
Return a pointer signed with the specified parameters.
This class represents a range of values.
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI bool icmp(CmpInst::Predicate Pred, const ConstantRange &Other) const
Does the predicate Pred hold between ranges this and Other?
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
Definition Constant.h:43
static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
Record of a variable value-assignment, aka a non instruction representation of the dbg....
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:237
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:163
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
static FMFSource intersect(Value *A, Value *B)
Intersect the FMF from two instructions.
Definition IRBuilder.h:107
This class represents an extension of floating point types.
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
void setNoSignedZeros(bool B=true)
Definition FMF.h:84
bool allowReassoc() const
Flag queries.
Definition FMF.h:64
An instruction for ordering other memory operations.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Class to represent function types.
Type::subtype_iterator param_iterator
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
bool isConvergent() const
Determine if the call is convergent.
Definition Function.h:610
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:270
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:352
bool doesNotThrow() const
Determine if the function cannot unwind.
Definition Function.h:594
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition Function.h:249
LLVM_ABI Value * getBasePtr() const
unsigned getBasePtrIndex() const
The index into the associate statepoint's argument list which contains the base pointer of the pointe...
LLVM_ABI Value * getDerivedPtr() const
unsigned getDerivedPtrIndex() const
The index into the associate statepoint's argument list which contains the pointer whose relocation t...
std::vector< const GCRelocateInst * > getGCRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition Statepoint.h:206
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
Definition Value.h:576
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition Globals.cpp:328
PointerType * getType() const
Global values are always pointers.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
LLVM_ABI Value * CreateLaunderInvariantGroup(Value *Ptr)
Create a launder.invariant.group intrinsic call.
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition IRBuilder.h:502
LLVM_ABI Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1420
LLVM_ABI CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition IRBuilder.h:2085
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition IRBuilder.h:507
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2442
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2212
LLVM_ABI Value * CreateStripInvariantGroup(Value *Ptr)
Create a strip.invariant.group intrinsic call.
static InsertValueInst * Create(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
KnownFPClass computeKnownFPClass(Value *Val, FastMathFlags FMF, FPClassTest Interested=fcAllFlags, const Instruction *CtxI=nullptr, unsigned Depth=0) const
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &PoisonElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
bool SimplifyDemandedBits(Instruction *I, unsigned Op, const APInt &DemandedMask, KnownBits &Known, const SimplifyQuery &Q, unsigned Depth=0) override
This form of SimplifyDemandedBits simplifies the specified instruction operand if possible,...
Instruction * SimplifyAnyMemSet(AnyMemSetInst *MI)
Instruction * visitFree(CallInst &FI, Value *FreedOp)
Instruction * visitCallBrInst(CallBrInst &CBI)
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Value * foldReversedIntrinsicOperands(IntrinsicInst *II)
If all arguments of the intrinsic are reverses, try to pull the reverse after the intrinsic.
Value * tryGetLog2(Value *Op, bool AssumeNonZero)
Instruction * visitFenceInst(FenceInst &FI)
Instruction * foldShuffledIntrinsicOperands(IntrinsicInst *II)
If all arguments of the intrinsic are unary shuffles with the same mask, try to shuffle after the int...
Instruction * visitInvokeInst(InvokeInst &II)
bool SimplifyDemandedInstructionBits(Instruction &Inst)
Tries to simplify operands to an integer instruction based on its demanded bits.
void CreateNonTerminatorUnreachable(Instruction *InsertAt)
Create and insert the idiom we use to indicate a block is unreachable without having to rewrite the C...
Instruction * visitVAEndInst(VAEndInst &I)
Instruction * matchBSwapOrBitReverse(Instruction &I, bool MatchBSwaps, bool MatchBitReversals)
Given an initial instruction, check to see if it is the root of a bswap/bitreverse idiom.
Constant * unshuffleConstant(ArrayRef< int > ShMask, Constant *C, VectorType *NewCTy)
Find a constant NewC that has property: shuffle(NewC, ShMask) = C Returns nullptr if such a constant ...
Instruction * visitAllocSite(Instruction &FI)
Instruction * SimplifyAnyMemTransfer(AnyMemTransferInst *MI)
OverflowResult computeOverflow(Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, Instruction *CxtI) const
Instruction * visitCallInst(CallInst &CI)
CallInst simplification.
SimplifyQuery SQ
unsigned ComputeMaxSignificantBits(const Value *Op, const Instruction *CxtI=nullptr, unsigned Depth=0) const
IRBuilder< TargetFolder, IRBuilderCallbackInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
bool isFreeToInvert(Value *V, bool WillInvertAllUses, bool &DoesConsume)
Return true if the specified value is free to invert (apply ~ to).
DominatorTree & getDominatorTree() const
BlockFrequencyInfo * BFI
TargetLibraryInfo & TLI
Instruction * InsertNewInstBefore(Instruction *New, BasicBlock::iterator Old)
Inserts an instruction New before instruction Old.
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
void replaceUse(Use &U, Value *NewValue)
Replace use and add the previously used value to the worklist.
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
const DataLayout & DL
DomConditionCache DC
void computeKnownBits(const Value *V, KnownBits &Known, const Instruction *CxtI, unsigned Depth=0) const
std::optional< Instruction * > targetInstCombineIntrinsic(IntrinsicInst &II)
AssumptionCache & AC
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const Instruction *CxtI=nullptr, unsigned Depth=0) const
DominatorTree & DT
ProfileSummaryInfo * PSI
BuilderTy & Builder
AssumptionCache & getAssumptionCache() const
OptimizationRemarkEmitter & ORE
Value * getFreelyInverted(Value *V, bool WillInvertAllUses, BuilderTy *Builder, bool &DoesConsume)
const SimplifyQuery & getSimplifyQuery() const
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, const Instruction *CxtI=nullptr, unsigned Depth=0)
LLVM_ABI void setHasNoUnsignedWrap(bool b=true)
Set or clear the nuw flag on this instruction, which must be an operator which supports this flag.
LLVM_ABI bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
LLVM_ABI void setHasNoSignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
bool isTerminator() const
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI std::optional< InstListType::iterator > getInsertionPointAfterDef()
Get the first insertion point at which the result of this instruction is defined.
LLVM_ABI bool isIdenticalTo(const Instruction *I) const LLVM_READONLY
Return true if the specified instruction is exactly identical to the current one.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Invoke instruction.
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, InsertPosition InsertBefore=nullptr)
An instruction for reading from memory.
Metadata node.
Definition Metadata.h:1078
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1569
static ICmpInst::Predicate getPredicate(Intrinsic::ID ID)
Returns the comparison predicate underlying the intrinsic.
ICmpInst::Predicate getPredicate() const
Returns the comparison predicate underlying the intrinsic.
bool isSigned() const
Whether the intrinsic is signed or unsigned.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
StringRef getName() const
Get a short "name" for the module.
Definition Module.h:269
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition Operator.h:43
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition Operator.h:78
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
Definition Operator.h:111
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
Definition Operator.h:105
bool isCommutative() const
Return true if the instruction is commutative.
Definition Operator.h:128
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Represents a saturating add/sub intrinsic.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
This instruction constructs a fixed permutation of two input vectors.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
SmallBitVector & set()
bool test(unsigned Idx) const
bool all() const
Returns true if all bits are set.
size_type size() const
Definition SmallPtrSet.h:99
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
void setVolatile(bool V)
Specify whether this is a volatile store or not.
void setAlignment(Align Align)
void setOrdering(AtomicOrdering Ordering)
Sets the ordering constraint of this store instruction.
Class to represent struct types.
static LLVM_ABI bool isCallingConvCCompatible(CallBase *CI)
Returns true if call site / callee has cdecl-compatible calling conventions.
Provides information about what library functions are available for the current target.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:298
LLVM_ABI unsigned getIntegerBitWidth() const
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:297
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:267
LLVM_ABI bool canLosslesslyBitCastTo(Type *Ty) const
Return true if this type could be converted with a lossless BitCast to type 'Ty'.
Definition Type.cpp:154
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:261
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:139
static UnaryOperator * CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Definition InstrTypes.h:139
static UnaryOperator * CreateFNegFMF(Value *Op, Instruction *FMFSource, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Definition InstrTypes.h:147
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
Definition Use.cpp:35
void setOperand(unsigned i, Value *Val)
Definition User.h:237
Value * getOperand(unsigned i) const
Definition User.h:232
This represents the llvm.va_end intrinsic.
static LLVM_ABI void ValueIsDeleted(Value *V)
Definition Value.cpp:1226
static LLVM_ABI void ValueIsRAUWd(Value *Old, Value *New)
Definition Value.cpp:1279
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
static constexpr uint64_t MaximumAlignment
Definition Value.h:830
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
iterator_range< user_iterator > users()
Definition Value.h:426
static LLVM_ABI void dropDroppableUse(Use &U)
Remove the droppable use U.
Definition Value.cpp:226
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:701
bool use_empty() const
Definition Value.h:346
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
static constexpr unsigned MaxAlignmentExponent
The maximum alignment for instructions.
Definition Value.h:829
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:396
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:217
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:224
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
CallInst * Call
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
class_match< PoisonValue > m_Poison()
Match an arbitrary poison constant.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
m_Intrinsic_Ty< Opnd0 >::Ty m_BitReverse(const Opnd0 &Op0)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWSub(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
OverflowingBinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWNeg(const ValTy &V)
Matches a 'Neg' as 'sub nsw 0, V'.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
IntrinsicID_match m_Intrinsic()
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
cstfp_pred_ty< is_neg_zero_fp > m_NegZeroFP()
Match a floating-point negative zero.
specific_fpval m_SpecificFP(double V)
Match a specific floating point value or vector with all elements equal to the value.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
match_combine_or< match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > >, OpTy > m_ZExtOrSExtOrSelf(const OpTy &Op)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
cst_pred_ty< is_strictlypositive > m_StrictlyPositive()
Match an integer or vector of strictly positive values.
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWShl(const LHS &L, const RHS &R)
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWMul(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
cst_pred_ty< is_negated_power2 > m_NegatedPower2()
Match a integer or vector negated power-of-2.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
cst_pred_ty< custom_checkfn< APInt > > m_CheckedInt(function_ref< bool(const APInt &)> CheckFn)
Match an integer or vector where CheckFn(ele) for each element is true.
m_Intrinsic_Ty< Opnd0, Opnd1, Opnd2 >::Ty m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2)
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty, true >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty, true > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty, true >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty, true > > > m_c_MaxOrMin(const LHS &L, const RHS &R)
class_match< UnaryOperator > m_UnOp()
Match an arbitrary unary operation and ignore it.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Sub, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWSub(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap >, DisjointOr_match< LHS, RHS > > m_NSWAddLike(const LHS &L, const RHS &R)
Match either "add nsw" or "or disjoint".
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
Exact_match< T > m_Exact(const T &SubPattern)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
cstfp_pred_ty< is_pos_zero_fp > m_PosZeroFP()
Match a floating-point positive zero.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
m_Intrinsic_Ty< Opnd0 >::Ty m_VecReverse(const Opnd0 &Op0)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
match_combine_or< match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > >, match_combine_or< MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty >, MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > > > m_MaxOrMin(const LHS &L, const RHS &R)
m_Intrinsic_Ty< Opnd0, Opnd1, Opnd2 >::Ty m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2)
BinaryOp_match< LHS, RHS, Instruction::SRem > m_SRem(const LHS &L, const RHS &R)
auto m_Undef()
Match an arbitrary undef constant.
m_Intrinsic_Ty< Opnd0 >::Ty m_BSwap(const Opnd0 &Op0)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap >, DisjointOr_match< LHS, RHS > > m_NUWAddLike(const LHS &L, const RHS &R)
Match either "add nuw" or "or disjoint".
BinOpPred_match< LHS, RHS, is_bitwiselogic_op > m_BitwiseLogic(const LHS &L, const RHS &R)
Matches bitwise logic operations.
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
m_Intrinsic_Ty< Opnd0, Opnd1 >::Ty m_CopySign(const Opnd0 &Op0, const Opnd1 &Op1)
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
MatchFunctor< Val, Pattern > match_fn(const Pattern &P)
A match functor that can be used as a UnaryPredicate in functional algorithms like all_of.
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition LLVMContext.h:55
@ System
Synchronized with respect to all concurrently executing threads.
Definition LLVMContext.h:58
SmallVector< DbgVariableRecord * > getDVRAssignmentMarkers(const Instruction *Inst)
Return a range of dbg_assign records for which Inst performs the assignment they encode.
Definition DebugInfo.h:201
initializer< Ty > init(const Ty &Val)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:667
constexpr double e
DiagnosticInfoOptimizationBase::Argument NV
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI cl::opt< bool > EnableKnowledgeRetention
LLVM_ABI Intrinsic::ID getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:344
@ Offset
Definition DWP.cpp:477
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
LLVM_ABI Value * simplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior=fp::ebIgnore, RoundingMode Rounding=RoundingMode::NearestTiesToEven)
Given operands for an FMul, fold the result or return null.
LLVM_ABI bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, const DominatorTree *DT=nullptr, bool AllowEphemerals=false)
Return true if it is valid to use the assumptions provided by an assume intrinsic,...
LLVM_ABI APInt possiblyDemandedEltsInMask(Value *Mask)
Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y) for each lane which may be ...
LLVM_ABI RetainedKnowledge simplifyRetainedKnowledge(AssumeInst *Assume, RetainedKnowledge RK, AssumptionCache *AC, DominatorTree *DT)
canonicalize the RetainedKnowledge RK.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
LLVM_ABI Value * lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed)
Try to turn a call to @llvm.objectsize into an integer value of the given Type.
LLVM_ABI Value * getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI)
Gets the alignment argument for an aligned_alloc-like function, using either built-in knowledge based...
LLVM_ABI RetainedKnowledge getKnowledgeFromOperandInAssume(AssumeInst &Assume, unsigned Idx)
Retreive the information help by Assume on the operand at index Idx.
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition APFloat.h:1625
LLVM_ABI Value * simplifyCall(CallBase *Call, Value *Callee, ArrayRef< Value * > Args, const SimplifyQuery &Q)
Given a callsite, callee, and arguments, fold the result or return null.
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:284
LLVM_ABI bool isAssumeWithEmptyBundle(const AssumeInst &Assume)
Return true iff the operand bundles of the provided llvm.assume doesn't contain any valuable informat...
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
constexpr T MinAlign(U A, V B)
A and B are either alignments or offsets.
Definition MathExtras.h:357
LLVM_ABI RetainedKnowledge getKnowledgeFromBundle(AssumeInst &Assume, const CallBase::BundleOpInfo &BOI)
This extracts the Knowledge from an element of an operand bundle.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition Local.h:252
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
Definition APFloat.h:1580
LLVM_ABI FPClassTest fneg(FPClassTest Mask)
Return the test mask which returns true if the value's sign bit is flipped.
SelectPatternFlavor
Specific patterns of select instructions we can match.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
LLVM_ABI Constant * getLosslessUnsignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:279
bool isModSet(const ModRefInfo MRI)
Definition ModRef.h:49
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
LLVM_ABI bool matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I, PHINode *&P, Value *&Init, Value *&OtherOp)
Attempt to match a simple value-accumulating recurrence of the form: llvm.intrinsic....
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
auto find_if_not(R &&Range, UnaryPredicate P)
Definition STLExtras.h:1763
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1739
bool isAtLeastOrStrongerThan(AtomicOrdering AO, AtomicOrdering Other)
LLVM_ABI Constant * getLosslessSignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
LLVM_ABI AssumeInst * buildAssumeFromKnowledge(ArrayRef< RetainedKnowledge > Knowledge, Instruction *CtxI, AssumptionCache *AC=nullptr, DominatorTree *DT=nullptr)
Build and return a new assume created from the provided knowledge if the knowledge in the assume is f...
LLVM_ABI FPClassTest inverse_fabs(FPClassTest Mask)
Return the test mask which returns true after fabs is applied to the value.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI bool isNotCrossLaneOperation(const Instruction *I)
Return true if the instruction doesn't potentially cross vector lanes.
LLVM_ABI bool maskIsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
constexpr int PoisonMaskElem
@ Mod
The access may modify the value stored in memory.
Definition ModRef.h:34
LLVM_ABI Value * simplifyFMAFMul(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior=fp::ebIgnore, RoundingMode Rounding=RoundingMode::NearestTiesToEven)
Given operands for the multiplication of a FMA, fold the result or return null.
@ Other
Any other memory.
Definition ModRef.h:68
FunctionAddr VTableAddr uintptr_t uintptr_t Data
Definition InstrProf.h:189
LLVM_ABI Value * simplifyConstrainedFPCall(CallBase *Call, const SimplifyQuery &Q)
Given a constrained FP intrinsic call, tries to compute its simplified version.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 minNum semantics.
Definition APFloat.h:1561
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
@ Add
Sum of integers.
LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI ConstantRange computeConstantRangeIncludingKnownBits(const WithCache< const Value * > &V, bool ForSigned, const SimplifyQuery &SQ)
Combine constant ranges from computeConstantRange() and computeKnownBits().
FunctionAddr VTableAddr Next
Definition InstrProf.h:141
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I, bool IgnoreUBImplyingAttrs=true)
Don't use information from its non-constant operands.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
constexpr unsigned BitWidth
LLVM_ABI bool isDereferenceablePointer(const Value *V, Type *Ty, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if this is always a dereferenceable pointer.
Definition Loads.cpp:249
LLVM_ABI bool maskIsAllZeroOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1897
LLVM_ABI std::optional< APInt > getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, function_ref< const Value *(const Value *)> Mapper=[](const Value *V) { return V;})
Return the size of the requested allocation.
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:197
LLVM_ABI bool maskContainsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if any of the elements of this predicate mask are known to be ...
LLVM_ABI std::optional< bool > isImpliedByDomCondition(const Value *Cond, const Instruction *ContextI, const DataLayout &DL)
Return the boolean condition value in the context of the given instruction if it is known based on do...
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
Definition APFloat.h:1598
LLVM_ABI bool isKnownNegation(const Value *X, const Value *Y, bool NeedNSW=false, bool AllowPoison=true)
Return true if the two given values are negation.
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
LLVM_ABI bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
LLVM_ABI bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
LLVM_ABI std::optional< bool > computeKnownFPSignBit(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return false if we can prove that the specified FP value's sign bit is 0.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:869
#define NC
Definition regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition Metadata.h:761
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
@ IEEE
IEEE-754 denormal numbers preserved.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition KnownBits.h:108
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition KnownBits.h:242
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition KnownBits.h:274
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition KnownBits.h:289
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
bool isNonZero() const
Returns true if this value is known to be non-zero.
Definition KnownBits.h:111
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:248
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:105
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
Definition KnownBits.h:280
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition KnownBits.h:286
bool isAllOnes() const
Returns true if value is all one bits.
Definition KnownBits.h:83
FPClassTest KnownFPClasses
Floating-point classes the value could be one of.
Matching combinators.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition Alignment.h:130
A lightweight accessor for an operand bundle meant to be passed around by value.
StringRef getTagName() const
Return the tag of this operand bundle as a string.
uint32_t getTagID() const
Return the tag of this operand bundle as an integer.
ArrayRef< Use > Inputs
Represent one information held inside an operand bundle of an llvm.assume.
Attribute::AttrKind AttrKind
SelectPatternFlavor Flavor
const DataLayout & DL
const Instruction * CxtI
SimplifyQuery getWithInstruction(const Instruction *I) const