LLVM 24.0.0git
AttributorAttributes.cpp
Go to the documentation of this file.
1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
14#include "IPOOptions.h"
17
18#include "llvm/ADT/APInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
22#include "llvm/ADT/MapVector.h"
24#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
29#include "llvm/ADT/Statistic.h"
42#include "llvm/IR/Argument.h"
43#include "llvm/IR/Assumptions.h"
45#include "llvm/IR/Attributes.h"
46#include "llvm/IR/BasicBlock.h"
47#include "llvm/IR/Constant.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DataLayout.h"
51#include "llvm/IR/GlobalValue.h"
52#include "llvm/IR/IRBuilder.h"
53#include "llvm/IR/InlineAsm.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
58#include "llvm/IR/IntrinsicsAMDGPU.h"
59#include "llvm/IR/IntrinsicsNVPTX.h"
60#include "llvm/IR/LLVMContext.h"
61#include "llvm/IR/MDBuilder.h"
62#include "llvm/IR/NoFolder.h"
63#include "llvm/IR/Value.h"
64#include "llvm/IR/ValueHandle.h"
78#include <cassert>
79#include <numeric>
80#include <optional>
81#include <string>
82
83using namespace llvm;
84
85#define DEBUG_TYPE "attributor"
86
87STATISTIC(NumAAs, "Number of abstract attributes created");
88STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");
89
90// Some helper macros to deal with statistics tracking.
91//
92// Usage:
93// For simple IR attribute tracking overload trackStatistics in the abstract
94// attribute and choose the right STATS_DECLTRACK_********* macro,
95// e.g.,:
96// void trackStatistics() const override {
97// STATS_DECLTRACK_ARG_ATTR(returned)
98// }
99// If there is a single "increment" side one can use the macro
100// STATS_DECLTRACK with a custom message. If there are multiple increment
101// sides, STATS_DECL and STATS_TRACK can also be used separately.
102//
103#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
104 ("Number of " #TYPE " marked '" #NAME "'")
105#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
106#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
107#define STATS_DECL(NAME, TYPE, MSG) \
108 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
109#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
110#define STATS_DECLTRACK(NAME, TYPE, MSG) \
111 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
112#define STATS_DECLTRACK_ARG_ATTR(NAME) \
113 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
114#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
115 STATS_DECLTRACK(NAME, CSArguments, \
116 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
117#define STATS_DECLTRACK_FN_ATTR(NAME) \
118 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
119#define STATS_DECLTRACK_CS_ATTR(NAME) \
120 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
121#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
122 STATS_DECLTRACK(NAME, FunctionReturn, \
123 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
124#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
125 STATS_DECLTRACK(NAME, CSReturn, \
126 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
127#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
128 STATS_DECLTRACK(NAME, Floating, \
129 ("Number of floating values known to be '" #NAME "'"))
130
131// Specialization of the operator<< for abstract attributes subclasses. This
132// disambiguates situations where multiple operators are applicable.
133namespace llvm {
134#define PIPE_OPERATOR(CLASS) \
135 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
136 return OS << static_cast<const AbstractAttribute &>(AA); \
137 }
138
178
179#undef PIPE_OPERATOR
180
181template <>
183 const DerefState &R) {
184 ChangeStatus CS0 =
185 clampStateAndIndicateChange(S.DerefBytesState, R.DerefBytesState);
186 ChangeStatus CS1 = clampStateAndIndicateChange(S.GlobalState, R.GlobalState);
187 return CS0 | CS1;
188}
189
190} // namespace llvm
191
192static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
193 bool HeaderOnly, CycleRef *CPtr = nullptr) {
194 if (!CI)
195 return true;
196 auto *BB = I->getParent();
197 CycleRef C = CI->getCycle(BB);
198 if (!C)
199 return false;
200 if (CPtr)
201 *CPtr = C;
202 return !HeaderOnly || BB == CI->getHeader(C);
203}
204
205/// Checks if a type could have padding bytes.
206static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
207 // There is no size information, so be conservative.
208 if (!Ty->isSized())
209 return false;
210
211 // If the alloc size is not equal to the storage size, then there are padding
212 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
213 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
214 return false;
215
216 // FIXME: This isn't the right way to check for padding in vectors with
217 // non-byte-size elements.
218 if (VectorType *SeqTy = dyn_cast<VectorType>(Ty))
219 return isDenselyPacked(SeqTy->getElementType(), DL);
220
221 // For array types, check for padding within members.
222 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty))
223 return isDenselyPacked(SeqTy->getElementType(), DL);
224
225 if (!isa<StructType>(Ty))
226 return true;
227
228 // Check for padding within and between elements of a struct.
229 StructType *StructTy = cast<StructType>(Ty);
230 const StructLayout *Layout = DL.getStructLayout(StructTy);
231 uint64_t StartPos = 0;
232 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
233 Type *ElTy = StructTy->getElementType(I);
234 if (!isDenselyPacked(ElTy, DL))
235 return false;
236 if (StartPos != Layout->getElementOffsetInBits(I))
237 return false;
238 StartPos += DL.getTypeAllocSizeInBits(ElTy);
239 }
240
241 // Check for tail padding.
242 return StartPos == Layout->getSizeInBits();
243}
244
245/// Get pointer operand of memory accessing instruction. If \p I is
246/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
247/// is set to false and the instruction is volatile, return nullptr.
249 bool AllowVolatile) {
250 if (!AllowVolatile && I->isVolatile())
251 return nullptr;
252
253 if (auto *LI = dyn_cast<LoadInst>(I)) {
254 return LI->getPointerOperand();
255 }
256
257 if (auto *SI = dyn_cast<StoreInst>(I)) {
258 return SI->getPointerOperand();
259 }
260
261 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I)) {
262 return CXI->getPointerOperand();
263 }
264
265 if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
266 return RMWI->getPointerOperand();
267 }
268
269 return nullptr;
270}
271
272/// Helper function to create a pointer based on \p Ptr, and advanced by \p
273/// Offset bytes.
274static Value *constructPointer(Value *Ptr, int64_t Offset,
275 IRBuilder<NoFolder> &IRB) {
276 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
277 << "-bytes\n");
278
279 if (Offset)
281 Ptr, Offset, Ptr->getName() + ".b" + Twine(Offset));
282 return Ptr;
283}
284
285static const Value *
287 const Value *Val, const DataLayout &DL, APInt &Offset,
288 bool GetMinOffset, bool AllowNonInbounds,
289 bool UseAssumed = false) {
290
291 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
292 const IRPosition &Pos = IRPosition::value(V);
293 // Only track dependence if we are going to use the assumed info.
294 const AAValueConstantRange *ValueConstantRangeAA =
295 A.getAAFor<AAValueConstantRange>(QueryingAA, Pos,
296 UseAssumed ? DepClassTy::OPTIONAL
298 if (!ValueConstantRangeAA)
299 return false;
300 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
301 : ValueConstantRangeAA->getKnown();
302 if (Range.isFullSet())
303 return false;
304
305 // We can only use the lower part of the range because the upper part can
306 // be higher than what the value can really be.
307 if (GetMinOffset)
308 ROffset = Range.getSignedMin();
309 else
310 ROffset = Range.getSignedMax();
311 return true;
312 };
313
314 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
315 /* AllowInvariant */ true,
316 AttributorAnalysis);
317}
318
319static const Value *
321 const Value *Ptr, int64_t &BytesOffset,
322 const DataLayout &DL, bool AllowNonInbounds = false) {
323 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
324 const Value *Base =
325 stripAndAccumulateOffsets(A, QueryingAA, Ptr, DL, OffsetAPInt,
326 /* GetMinOffset */ true, AllowNonInbounds);
327
328 BytesOffset = OffsetAPInt.getSExtValue();
329 return Base;
330}
331
332/// Clamp the information known for all returned values of a function
333/// (identified by \p QueryingAA) into \p S.
334template <typename AAType, typename StateType = typename AAType::StateType,
335 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
336 bool RecurseForSelectAndPHI = true>
338 Attributor &A, const AAType &QueryingAA, StateType &S,
339 const IRPosition::CallBaseContext *CBContext = nullptr) {
340 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
341 << QueryingAA << " into " << S << "\n");
342
343 assert((QueryingAA.getIRPosition().getPositionKind() ==
345 QueryingAA.getIRPosition().getPositionKind() ==
347 "Can only clamp returned value states for a function returned or call "
348 "site returned position!");
349
350 // Use an optional state as there might not be any return values and we want
351 // to join (IntegerState::operator&) the state of all there are.
352 std::optional<StateType> T;
353
354 // Callback for each possibly returned value.
355 auto CheckReturnValue = [&](Value &RV) -> bool {
356 const IRPosition &RVPos = IRPosition::value(RV, CBContext);
357 // If possible, use the hasAssumedIRAttr interface.
358 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
359 bool IsKnown;
361 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
362 }
363
364 const AAType *AA =
365 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
366 if (!AA)
367 return false;
368 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
369 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
370 const StateType &AAS = AA->getState();
371 if (!T)
372 T = StateType::getBestState(AAS);
373 *T &= AAS;
374 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
375 << "\n");
376 return T->isValidState();
377 };
378
379 if (!A.checkForAllReturnedValues(CheckReturnValue, QueryingAA,
381 RecurseForSelectAndPHI))
382 S.indicatePessimisticFixpoint();
383 else if (T)
384 S ^= *T;
385}
386
387namespace {
388/// Helper class for generic deduction: return value -> returned position.
389template <typename AAType, typename BaseType,
390 typename StateType = typename BaseType::StateType,
391 bool PropagateCallBaseContext = false,
392 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
393 bool RecurseForSelectAndPHI = true>
394struct AAReturnedFromReturnedValues : public BaseType {
395 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
396 : BaseType(IRP, A) {}
397
398 ChangeStatus updateImpl(Attributor &A) override {
399 StateType S(StateType::getBestState(this->getState()));
400 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
401 RecurseForSelectAndPHI>(
402 A, *this, S,
403 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
404 // TODO: If we know we visited all returned values, thus no are assumed
405 // dead, we can take the known information from the state T.
406 return clampStateAndIndicateChange<StateType>(this->getState(), S);
407 }
408};
409
410/// Clamp the information known at all call sites for a given argument
411/// (identified by \p QueryingAA) into \p S.
412template <typename AAType, typename StateType = typename AAType::StateType,
413 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
414static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
415 StateType &S) {
416 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
417 << QueryingAA << " into " << S << "\n");
418
419 assert(QueryingAA.getIRPosition().getPositionKind() ==
421 "Can only clamp call site argument states for an argument position!");
422
423 // Use an optional state as there might not be any return values and we want
424 // to join (IntegerState::operator&) the state of all there are.
425 std::optional<StateType> T;
426
427 // The argument number which is also the call site argument number.
428 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
429
430 auto CallSiteCheck = [&](AbstractCallSite ACS) {
431 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
432 // Check if a coresponding argument was found or if it is on not associated
433 // (which can happen for callback calls).
434 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
435 return false;
436
437 // If possible, use the hasAssumedIRAttr interface.
438 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
439 bool IsKnown;
441 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
442 }
443
444 const AAType *AA =
445 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
446 if (!AA)
447 return false;
448 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
449 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
450 << "\n");
451 const StateType &AAS = AA->getState();
452 if (!T)
453 T = StateType::getBestState(AAS);
454 *T &= AAS;
455 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
456 << "\n");
457 return T->isValidState();
458 };
459
460 bool UsedAssumedInformation = false;
461 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
462 UsedAssumedInformation))
463 S.indicatePessimisticFixpoint();
464 else if (T)
465 S ^= *T;
466}
467
468/// This function is the bridge between argument position and the call base
469/// context.
470template <typename AAType, typename BaseType,
471 typename StateType = typename AAType::StateType,
472 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
473bool getArgumentStateFromCallBaseContext(Attributor &A,
474 BaseType &QueryingAttribute,
475 IRPosition &Pos, StateType &State) {
477 "Expected an 'argument' position !");
478 const CallBase *CBContext = Pos.getCallBaseContext();
479 if (!CBContext)
480 return false;
481
482 int ArgNo = Pos.getCallSiteArgNo();
483 assert(ArgNo >= 0 && "Invalid Arg No!");
484 const IRPosition CBArgPos = IRPosition::callsite_argument(*CBContext, ArgNo);
485
486 // If possible, use the hasAssumedIRAttr interface.
487 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
488 bool IsKnown;
490 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
491 }
492
493 const auto *AA =
494 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
495 if (!AA)
496 return false;
497 const StateType &CBArgumentState =
498 static_cast<const StateType &>(AA->getState());
499
500 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
501 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
502 << "\n");
503
504 // NOTE: If we want to do call site grouping it should happen here.
505 State ^= CBArgumentState;
506 return true;
507}
508
509/// Helper class for generic deduction: call site argument -> argument position.
510template <typename AAType, typename BaseType,
511 typename StateType = typename AAType::StateType,
512 bool BridgeCallBaseContext = false,
513 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
514struct AAArgumentFromCallSiteArguments : public BaseType {
515 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
516 : BaseType(IRP, A) {}
517
518 ChangeStatus updateImpl(Attributor &A) override {
519 StateType S = StateType::getBestState(this->getState());
520
521 if (BridgeCallBaseContext) {
522 bool Success =
523 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
524 IRAttributeKind>(
525 A, *this, this->getIRPosition(), S);
526 if (Success)
527 return clampStateAndIndicateChange<StateType>(this->getState(), S);
528 }
529 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
530 S);
531
532 // TODO: If we know we visited all incoming values, thus no are assumed
533 // dead, we can take the known information from the state T.
534 return clampStateAndIndicateChange<StateType>(this->getState(), S);
535 }
536};
537
538/// Helper class for generic replication: function returned -> cs returned.
539template <typename AAType, typename BaseType,
540 typename StateType = typename BaseType::StateType,
541 bool IntroduceCallBaseContext = false,
542 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
543struct AACalleeToCallSite : public BaseType {
544 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
545
546 ChangeStatus updateImpl(Attributor &A) override {
547 auto IRPKind = this->getIRPosition().getPositionKind();
549 IRPKind == IRPosition::IRP_CALL_SITE) &&
550 "Can only wrap function returned positions for call site "
551 "returned positions!");
552 auto &S = this->getState();
553
554 CallBase &CB = cast<CallBase>(this->getAnchorValue());
555 if (IntroduceCallBaseContext)
556 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
557 << "\n");
558
559 ChangeStatus Changed = ChangeStatus::UNCHANGED;
560 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
561 for (const Function *Callee : Callees) {
562 // The returned position of the callee does not describe the call if
563 // the call uses a different return type.
564 if (IRPKind == IRPosition::IRP_CALL_SITE_RETURNED &&
565 Callee->getReturnType() != CB.getType())
566 return false;
567 IRPosition FnPos =
569 ? IRPosition::returned(*Callee,
570 IntroduceCallBaseContext ? &CB : nullptr)
571 : IRPosition::function(
572 *Callee, IntroduceCallBaseContext ? &CB : nullptr);
573 // If possible, use the hasAssumedIRAttr interface.
574 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
575 bool IsKnown;
577 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
578 return false;
579 continue;
580 }
581
582 const AAType *AA =
583 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
584 if (!AA)
585 return false;
586 Changed |= clampStateAndIndicateChange(S, AA->getState());
587 if (S.isAtFixpoint())
588 return S.isValidState();
589 }
590 return true;
591 };
592 if (!A.checkForAllCallees(CalleePred, *this, CB))
593 return S.indicatePessimisticFixpoint();
594 return Changed;
595 }
596};
597
598/// Helper function to accumulate uses.
599template <class AAType, typename StateType = typename AAType::StateType>
600static void followUsesInContext(AAType &AA, Attributor &A,
602 const Instruction *CtxI,
604 StateType &State) {
605 for (unsigned u = 0; u < Uses.size(); ++u) {
606 const Use *U = Uses[u];
607 if (const Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {
608 bool Found = Explorer.isInContextOf(UserI, CtxI);
609 if (Found && AA.followUseInMBEC(A, U, UserI, State))
610 Uses.insert_range(llvm::make_pointer_range(UserI->uses()));
611 }
612 }
613}
614
615/// Use the must-be-executed-context around \p I to add information into \p S.
616/// The AAType class is required to have `followUseInMBEC` method with the
617/// following signature and behaviour:
618///
619/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
620/// U - Underlying use.
621/// I - The user of the \p U.
622/// Returns true if the value should be tracked transitively.
623///
624template <class AAType, typename StateType = typename AAType::StateType>
625static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
626 Instruction &CtxI) {
627 if (!A.canUseBodyFor(AA.getIRPosition()))
628 return;
629
630 const Value &Val = AA.getIRPosition().getAssociatedValue();
631 if (isa<ConstantData>(Val))
632 return;
633
635 A.getInfoCache().getMustBeExecutedContextExplorer();
636 if (!Explorer)
637 return;
638
639 // Container for (transitive) uses of the associated value.
641 for (const Use &U : Val.uses())
642 Uses.insert(&U);
643
644 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
645
646 if (S.isAtFixpoint())
647 return;
648
650 auto Pred = [&](const Instruction *I) {
651 if (const CondBrInst *Br = dyn_cast<CondBrInst>(I))
652 BrInsts.push_back(Br);
653 return true;
654 };
655
656 // Here, accumulate conditional branch instructions in the context. We
657 // explore the child paths and collect the known states. The disjunction of
658 // those states can be merged to its own state. Let ParentState_i be a state
659 // to indicate the known information for an i-th branch instruction in the
660 // context. ChildStates are created for its successors respectively.
661 //
662 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
663 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
664 // ...
665 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
666 //
667 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
668 //
669 // FIXME: Currently, recursive branches are not handled. For example, we
670 // can't deduce that ptr must be dereferenced in below function.
671 //
672 // void f(int a, int c, int *ptr) {
673 // if(a)
674 // if (b) {
675 // *ptr = 0;
676 // } else {
677 // *ptr = 1;
678 // }
679 // else {
680 // if (b) {
681 // *ptr = 0;
682 // } else {
683 // *ptr = 1;
684 // }
685 // }
686 // }
687
688 Explorer->checkForAllTerminatorsInContext(&CtxI, Pred);
689 for (const CondBrInst *Br : BrInsts) {
690 StateType ParentState;
691
692 // The known state of the parent state is a conjunction of children's
693 // known states so it is initialized with a best state.
694 ParentState.indicateOptimisticFixpoint();
695
696 for (const BasicBlock *BB : Br->successors()) {
697 StateType ChildState;
698
699 size_t BeforeSize = Uses.size();
700 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
701
702 // Erase uses which only appear in the child.
703 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
704 It = Uses.erase(It);
705
706 ParentState &= ChildState;
707 }
708
709 // Use only known state.
710 S += ParentState;
711 }
712}
713} // namespace
714
715/// ------------------------ PointerInfo ---------------------------------------
716
717namespace llvm {
718namespace AA {
719namespace PointerInfo {
720
721struct State;
722
723} // namespace PointerInfo
724} // namespace AA
725
726/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
727template <>
730 static unsigned getHashValue(const Access &A);
731 static bool isEqual(const Access &LHS, const Access &RHS);
732};
733
734/// Helper that allows RangeTy as a key in a DenseMap.
735template <> struct DenseMapInfo<AA::RangeTy> {
741
742 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
743 return A == B;
744 }
745};
746
747} // namespace llvm
748
749/// A type to track pointer/struct usage and accesses for AAPointerInfo.
751 /// Return the best possible representable state.
752 static State getBestState(const State &SIS) { return State(); }
753
754 /// Return the worst possible representable state.
755 static State getWorstState(const State &SIS) {
756 State R;
757 R.indicatePessimisticFixpoint();
758 return R;
759 }
760
761 State() = default;
762 State(State &&SIS) = default;
763
764 const State &getAssumed() const { return *this; }
765
766 bool isValidState() const override { return BS.isValidState(); }
767
768 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
769
771 BS.indicateOptimisticFixpoint();
773 }
774
776 BS.indicatePessimisticFixpoint();
778 }
779
780 State &operator=(const State &R) {
781 if (this == &R)
782 return *this;
783 BS = R.BS;
784 AccessList = R.AccessList;
785 OffsetBins = R.OffsetBins;
786 RemoteIMap = R.RemoteIMap;
787 ReturnedOffsets = R.ReturnedOffsets;
788 return *this;
789 }
790
792 if (this == &R)
793 return *this;
794 std::swap(BS, R.BS);
795 std::swap(AccessList, R.AccessList);
796 std::swap(OffsetBins, R.OffsetBins);
797 std::swap(RemoteIMap, R.RemoteIMap);
798 std::swap(ReturnedOffsets, R.ReturnedOffsets);
799 return *this;
800 }
801
802 /// Add a new Access to the state at offset \p Offset and with size \p Size.
803 /// The access is associated with \p I, writes \p Content (if anything), and
804 /// is of kind \p Kind. If an Access already exists for the same \p I and same
805 /// \p RemoteI, the two are combined, potentially losing information about
806 /// offset and size. The resulting access must now be moved from its original
807 /// OffsetBin to the bin for its new offset.
808 ///
809 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
811 Instruction &I, std::optional<Value *> Content,
813 Instruction *RemoteI = nullptr);
814
817 int64_t numOffsetBins() const { return OffsetBins.size(); }
818
819 const AAPointerInfo::Access &getAccess(unsigned Index) const {
820 return AccessList[Index];
821 }
822
823protected:
824 // Every memory instruction results in an Access object. We maintain a list of
825 // all Access objects that we own, along with the following maps:
826 //
827 // - OffsetBins: RangeTy -> { Access }
828 // - RemoteIMap: RemoteI x LocalI -> Access
829 //
830 // A RemoteI is any instruction that accesses memory. RemoteI is different
831 // from LocalI if and only if LocalI is a call; then RemoteI is some
832 // instruction in the callgraph starting from LocalI. Multiple paths in the
833 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
834 // are all combined into a single Access object. This may result in loss of
835 // information in RangeTy in the Access object.
839
840 /// Flag to determine if the underlying pointer is reaching a return statement
841 /// in the associated function or not. Returns in other functions cause
842 /// invalidation.
844
845 /// See AAPointerInfo::forallInterferingAccesses.
846 template <typename F>
848 if (!isValidState() || !ReturnedOffsets.isUnassigned())
849 return false;
850
851 for (const auto &It : OffsetBins) {
852 AA::RangeTy ItRange = It.getFirst();
853 if (!Range.mayOverlap(ItRange))
854 continue;
855 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
856 for (auto Index : It.getSecond()) {
857 auto &Access = AccessList[Index];
858 if (!CB(Access, IsExact))
859 return false;
860 }
861 }
862 return true;
863 }
864
865 /// See AAPointerInfo::forallInterferingAccesses.
866 template <typename F>
868 AA::RangeTy &Range) const {
869 if (!isValidState() || !ReturnedOffsets.isUnassigned())
870 return false;
871
872 auto LocalList = RemoteIMap.find(&I);
873 if (LocalList == RemoteIMap.end()) {
874 return true;
875 }
876
877 for (unsigned Index : LocalList->getSecond()) {
878 for (auto &R : AccessList[Index]) {
879 Range &= R;
880 if (Range.offsetAndSizeAreUnknown())
881 break;
882 }
883 }
885 }
886
887private:
888 /// State to track fixpoint and validity.
889 BooleanState BS;
890};
891
894 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
895 Instruction *RemoteI) {
896 RemoteI = RemoteI ? RemoteI : &I;
897
898 // Check if we have an access for this instruction, if not, simply add it.
899 auto &LocalList = RemoteIMap[RemoteI];
900 bool AccExists = false;
901 unsigned AccIndex = AccessList.size();
902 for (auto Index : LocalList) {
903 auto &A = AccessList[Index];
904 if (A.getLocalInst() == &I) {
905 AccExists = true;
906 AccIndex = Index;
907 break;
908 }
909 }
910
911 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
912 LLVM_DEBUG(if (ToAdd.size()) dbgs()
913 << "[AAPointerInfo] Inserting access in new offset bins\n";);
914
915 for (auto Key : ToAdd) {
916 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
917 OffsetBins[Key].insert(AccIndex);
918 }
919 };
920
921 if (!AccExists) {
922 AccessList.emplace_back(&I, RemoteI, Ranges, Content, Kind, Ty);
923 assert((AccessList.size() == AccIndex + 1) &&
924 "New Access should have been at AccIndex");
925 LocalList.push_back(AccIndex);
926 AddToBins(AccessList[AccIndex].getRanges());
928 }
929
930 // Combine the new Access with the existing Access, and then update the
931 // mapping in the offset bins.
932 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
933 auto &Current = AccessList[AccIndex];
934 auto Before = Current;
935 Current &= Acc;
936 if (Current == Before)
938
939 auto &ExistingRanges = Before.getRanges();
940 auto &NewRanges = Current.getRanges();
941
942 // Ranges that are in the old access but not the new access need to be removed
943 // from the offset bins.
945 AAPointerInfo::RangeList::set_difference(ExistingRanges, NewRanges, ToRemove);
946 LLVM_DEBUG(if (ToRemove.size()) dbgs()
947 << "[AAPointerInfo] Removing access from old offset bins\n";);
948
949 for (auto Key : ToRemove) {
950 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
951 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
952 auto &Bin = OffsetBins[Key];
953 assert(Bin.count(AccIndex) &&
954 "Expected bin to actually contain the Access.");
955 Bin.erase(AccIndex);
956 }
957
958 // Ranges that are in the new access but not the old access need to be added
959 // to the offset bins.
961 AAPointerInfo::RangeList::set_difference(NewRanges, ExistingRanges, ToAdd);
962 AddToBins(ToAdd);
964}
965
966namespace {
967
968#ifndef NDEBUG
970 const AAPointerInfo::OffsetInfo &OI) {
971 OS << llvm::interleaved_array(OI);
972 return OS;
973}
974#endif // NDEBUG
975
976struct AAPointerInfoImpl
977 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
979 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
980
981 const std::string getAsStr(Attributor *A) const override {
982 return std::string("PointerInfo ") +
983 (isValidState() ? (std::string("#") +
984 std::to_string(OffsetBins.size()) + " bins")
985 : "<invalid>") +
986 (reachesReturn()
987 ? (" (returned:" +
988 join(map_range(ReturnedOffsets,
989 [](int64_t O) { return std::to_string(O); }),
990 ", ") +
991 ")")
992 : "");
993 }
994
995 ChangeStatus manifest(Attributor &A) override {
996 return AAPointerInfo::manifest(A);
997 }
998
999 const_bin_iterator begin() const override { return State::begin(); }
1000 const_bin_iterator end() const override { return State::end(); }
1001 int64_t numOffsetBins() const override { return State::numOffsetBins(); }
1002 bool reachesReturn() const override {
1003 return !ReturnedOffsets.isUnassigned();
1004 }
1005 void addReturnedOffsetsTo(OffsetInfo &OI) const override {
1006 if (ReturnedOffsets.isUnknown()) {
1007 OI.setUnknown();
1008 return;
1009 }
1010
1011 OffsetInfo MergedOI;
1012 for (auto Offset : ReturnedOffsets) {
1013 OffsetInfo TmpOI = OI;
1014 TmpOI.addToAll(Offset);
1015 MergedOI.merge(TmpOI);
1016 }
1017 OI = std::move(MergedOI);
1018 }
1019
1020 ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
1021 if (ReturnedOffsets.isUnknown())
1022 return ChangeStatus::UNCHANGED;
1023 if (ReachedReturnedOffsets.isUnknown()) {
1024 ReturnedOffsets.setUnknown();
1025 return ChangeStatus::CHANGED;
1026 }
1027 if (ReturnedOffsets.merge(ReachedReturnedOffsets))
1028 return ChangeStatus::CHANGED;
1029 return ChangeStatus::UNCHANGED;
1030 }
1031
1032 bool forallInterferingAccesses(
1033 AA::RangeTy Range,
1034 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1035 const override {
1036 return State::forallInterferingAccesses(Range, CB);
1037 }
1038
1039 bool forallInterferingAccesses(
1040 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1041 bool FindInterferingWrites, bool FindInterferingReads,
1042 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1043 AA::RangeTy &Range,
1044 function_ref<bool(const Access &)> SkipCB) const override {
1045 HasBeenWrittenTo = false;
1046
1047 SmallPtrSet<const Access *, 8> DominatingWrites;
1048 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1049
1050 Function &Scope = *I.getFunction();
1051 bool IsKnownNoSync;
1052 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1053 A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1054 IsKnownNoSync);
1055 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1056 IRPosition::function(Scope), &QueryingAA, DepClassTy::NONE);
1057 bool AllInSameNoSyncFn = IsAssumedNoSync;
1058 bool InstIsExecutedByInitialThreadOnly =
1059 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1060
1061 // If the function is not ending in aligned barriers, we need the stores to
1062 // be in aligned barriers. The load being in one is not sufficient since the
1063 // store might be executed by a thread that disappears after, causing the
1064 // aligned barrier guarding the load to unblock and the load to read a value
1065 // that has no CFG path to the load.
1066 bool InstIsExecutedInAlignedRegion =
1067 FindInterferingReads && ExecDomainAA &&
1068 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1069
1070 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1071 A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1072
1073 InformationCache &InfoCache = A.getInfoCache();
1074 bool IsThreadLocalObj =
1075 AA::isAssumedThreadLocalObject(A, getAssociatedValue(), *this);
1076
1077 // Helper to determine if we need to consider threading, which we cannot
1078 // right now. However, if the function is (assumed) nosync or the thread
1079 // executing all instructions is the main thread only we can ignore
1080 // threading. Also, thread-local objects do not require threading reasoning.
1081 // Finally, we can ignore threading if either access is executed in an
1082 // aligned region.
1083 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1084 if (IsThreadLocalObj || AllInSameNoSyncFn)
1085 return true;
1086 const auto *FnExecDomainAA =
1087 I.getFunction() == &Scope
1088 ? ExecDomainAA
1089 : A.lookupAAFor<AAExecutionDomain>(
1090 IRPosition::function(*I.getFunction()), &QueryingAA,
1091 DepClassTy::NONE);
1092 if (!FnExecDomainAA)
1093 return false;
1094 if (InstIsExecutedInAlignedRegion ||
1095 (FindInterferingWrites &&
1096 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1097 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1098 return true;
1099 }
1100 if (InstIsExecutedByInitialThreadOnly &&
1101 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1102 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1103 return true;
1104 }
1105 return false;
1106 };
1107
1108 // Helper to determine if the access is executed by the same thread as the
1109 // given instruction, for now it is sufficient to avoid any potential
1110 // threading effects as we cannot deal with them anyway.
1111 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1112 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1113 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1114 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1115 };
1116
1117 // TODO: Use inter-procedural reachability and dominance.
1118 bool IsKnownNoRecurse;
1120 A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1121 IsKnownNoRecurse);
1122
1123 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1124 // AAExecutionDomain) such that we allow scopes other than kernels as long
1125 // as the reaching kernels are disjoint.
1126 bool InstInKernel = A.getInfoCache().isKernel(Scope);
1127 bool ObjHasKernelLifetime = false;
1128 const bool UseDominanceReasoning =
1129 FindInterferingWrites && IsKnownNoRecurse;
1130 const DominatorTree *DT =
1131 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(Scope);
1132
1133 // Helper to check if a value has "kernel lifetime", that is it will not
1134 // outlive a GPU kernel. This is true for shared, constant, and local
1135 // globals on AMD and NVIDIA GPUs.
1136 auto HasKernelLifetime = [&](Value *V, Module &M) {
1137 if (!AA::isGPU(M))
1138 return false;
1139 unsigned VAS = V->getType()->getPointerAddressSpace();
1140 return AA::isGPUSharedAddressSpace(M, VAS) ||
1143 };
1144
1145 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1146 // to determine if we should look at reachability from the callee. For
1147 // certain pointers we know the lifetime and we do not have to step into the
1148 // callee to determine reachability as the pointer would be dead in the
1149 // callee. See the conditional initialization below.
1150 std::function<bool(const Function &)> IsLiveInCalleeCB;
1151
1152 if (auto *AI = dyn_cast<AllocaInst>(&getAssociatedValue())) {
1153 // If the alloca containing function is not recursive the alloca
1154 // must be dead in the callee.
1155 const Function *AIFn = AI->getFunction();
1156 ObjHasKernelLifetime = A.getInfoCache().isKernel(*AIFn);
1157 bool IsKnownNoRecurse;
1159 A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,
1160 IsKnownNoRecurse)) {
1161 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1162 }
1163 } else if (auto *GV = dyn_cast<GlobalValue>(&getAssociatedValue())) {
1164 // If the global has kernel lifetime we can stop if we reach a kernel
1165 // as it is "dead" in the (unknown) callees.
1166 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1167 if (ObjHasKernelLifetime)
1168 IsLiveInCalleeCB = [&A](const Function &Fn) {
1169 return !A.getInfoCache().isKernel(Fn);
1170 };
1171 }
1172
1173 // Set of accesses/instructions that will overwrite the result and are
1174 // therefore blockers in the reachability traversal.
1175 AA::InstExclusionSetTy ExclusionSet;
1176
1177 auto AccessCB = [&](const Access &Acc, bool Exact) {
1178 Function *AccScope = Acc.getRemoteInst()->getFunction();
1179 bool AccInSameScope = AccScope == &Scope;
1180
1181 // If the object has kernel lifetime we can ignore accesses only reachable
1182 // by other kernels. For now we only skip accesses *in* other kernels.
1183 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1184 A.getInfoCache().isKernel(*AccScope))
1185 return true;
1186
1187 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1188 if (Acc.isWrite() || (isa<LoadInst>(I) && Acc.isWriteOrAssumption()))
1189 ExclusionSet.insert(Acc.getRemoteInst());
1190 }
1191
1192 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1193 (!FindInterferingReads || !Acc.isRead()))
1194 return true;
1195
1196 bool Dominates = FindInterferingWrites && DT && Exact &&
1197 Acc.isMustAccess() && AccInSameScope &&
1198 DT->dominates(Acc.getRemoteInst(), &I);
1199 if (Dominates)
1200 DominatingWrites.insert(&Acc);
1201
1202 // Track if all interesting accesses are in the same `nosync` function as
1203 // the given instruction.
1204 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1205
1206 InterferingAccesses.push_back({&Acc, Exact});
1207 return true;
1208 };
1209 if (!State::forallInterferingAccesses(I, AccessCB, Range))
1210 return false;
1211
1212 HasBeenWrittenTo = !DominatingWrites.empty();
1213
1214 // Dominating writes form a chain, find the least/lowest member.
1215 Instruction *LeastDominatingWriteInst = nullptr;
1216 for (const Access *Acc : DominatingWrites) {
1217 if (!LeastDominatingWriteInst) {
1218 LeastDominatingWriteInst = Acc->getRemoteInst();
1219 } else if (DT->dominates(LeastDominatingWriteInst,
1220 Acc->getRemoteInst())) {
1221 LeastDominatingWriteInst = Acc->getRemoteInst();
1222 }
1223 }
1224
1225 // Helper to determine if we can skip a specific write access.
1226 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1227 if (SkipCB && SkipCB(Acc))
1228 return true;
1229 if (!CanIgnoreThreading(Acc))
1230 return false;
1231
1232 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1233 // If we successfully excluded all effects we are interested in, the
1234 // access can be skipped.
1235 bool ReadChecked = !FindInterferingReads;
1236 bool WriteChecked = !FindInterferingWrites;
1237
1238 // If the instruction cannot reach the access, the former does not
1239 // interfere with what the access reads.
1240 if (!ReadChecked) {
1241 if (!AA::isPotentiallyReachable(A, I, *Acc.getRemoteInst(), QueryingAA,
1242 &ExclusionSet, IsLiveInCalleeCB))
1243 ReadChecked = true;
1244 }
1245 // If the instruction cannot be reach from the access, the latter does not
1246 // interfere with what the instruction reads.
1247 if (!WriteChecked) {
1248 if (!AA::isPotentiallyReachable(A, *Acc.getRemoteInst(), I, QueryingAA,
1249 &ExclusionSet, IsLiveInCalleeCB))
1250 WriteChecked = true;
1251 }
1252
1253 // If we still might be affected by the write of the access but there are
1254 // dominating writes in the function of the instruction
1255 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1256 // by them. This would have happend above if they are all in the same
1257 // function, so we only check the inter-procedural case. Effectively, we
1258 // want to show that there is no call after the dominting write that might
1259 // reach the access, and when it returns reach the instruction with the
1260 // updated value. To this end, we iterate all call sites, check if they
1261 // might reach the instruction without going through another access
1262 // (ExclusionSet) and at the same time might reach the access. However,
1263 // that is all part of AAInterFnReachability.
1264 if (!WriteChecked && HasBeenWrittenTo &&
1265 Acc.getRemoteInst()->getFunction() != &Scope) {
1266
1267 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1268 QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL);
1269 if (FnReachabilityAA) {
1270 // Without going backwards in the call tree, can we reach the access
1271 // from the least dominating write. Do not allow to pass the
1272 // instruction itself either.
1273 bool Inserted = ExclusionSet.insert(&I).second;
1274
1275 if (!FnReachabilityAA->instructionCanReach(
1276 A, *LeastDominatingWriteInst,
1277 *Acc.getRemoteInst()->getFunction(), &ExclusionSet))
1278 WriteChecked = true;
1279
1280 if (Inserted)
1281 ExclusionSet.erase(&I);
1282 }
1283 }
1284
1285 if (ReadChecked && WriteChecked)
1286 return true;
1287
1288 if (!DT || !UseDominanceReasoning)
1289 return false;
1290 if (!DominatingWrites.count(&Acc))
1291 return false;
1292 return LeastDominatingWriteInst != Acc.getRemoteInst();
1293 };
1294
1295 // Run the user callback on all accesses we cannot skip and return if
1296 // that succeeded for all or not.
1297 for (auto &It : InterferingAccesses) {
1298 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1299 !CanSkipAccess(*It.first, It.second)) {
1300 if (!UserCB(*It.first, It.second))
1301 return false;
1302 }
1303 }
1304 return true;
1305 }
1306
1307 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1308 const AAPointerInfo &OtherAA,
1309 CallBase &CB) {
1310 using namespace AA::PointerInfo;
1311 if (!OtherAA.getState().isValidState() || !isValidState())
1312 return indicatePessimisticFixpoint();
1313
1314 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1315 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1316 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1317 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1318
1319 // Combine the accesses bin by bin.
1320 const auto &State = OtherAAImpl.getState();
1321 for (const auto &It : State) {
1322 for (auto Index : It.getSecond()) {
1323 const auto &RAcc = State.getAccess(Index);
1324 if (IsByval && !RAcc.isRead())
1325 continue;
1326 bool UsedAssumedInformation = false;
1327 AccessKind AK = RAcc.getKind();
1328 auto Content = A.translateArgumentToCallSiteContent(
1329 RAcc.getContent(), CB, *this, UsedAssumedInformation);
1330 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1331 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1332
1333 Changed |= addAccess(A, RAcc.getRanges(), CB, Content, AK,
1334 RAcc.getType(), RAcc.getRemoteInst());
1335 }
1336 }
1337 return Changed;
1338 }
1339
1340 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1341 const OffsetInfo &Offsets, CallBase &CB,
1342 bool IsMustAcc) {
1343 using namespace AA::PointerInfo;
1344 if (!OtherAA.getState().isValidState() || !isValidState())
1345 return indicatePessimisticFixpoint();
1346
1347 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1348
1349 // Combine the accesses bin by bin.
1350 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1351 const auto &State = OtherAAImpl.getState();
1352 for (const auto &It : State) {
1353 for (auto Index : It.getSecond()) {
1354 const auto &RAcc = State.getAccess(Index);
1355 if (!IsMustAcc && RAcc.isAssumption())
1356 continue;
1357 for (auto Offset : Offsets) {
1358 auto NewRanges = Offset == AA::RangeTy::Unknown
1360 : RAcc.getRanges();
1361 if (!NewRanges.isUnknown()) {
1362 NewRanges.addToAllOffsets(Offset);
1363 }
1364 AccessKind AK = RAcc.getKind();
1365 if (!IsMustAcc)
1366 AK = AccessKind((AK & ~AK_MUST) | AK_MAY);
1367 Changed |= addAccess(A, NewRanges, CB, RAcc.getContent(), AK,
1368 RAcc.getType(), RAcc.getRemoteInst());
1369 }
1370 }
1371 }
1372 return Changed;
1373 }
1374
1375 /// Statistic tracking for all AAPointerInfo implementations.
1376 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1377
1378 /// Dump the state into \p O.
1379 void dumpState(raw_ostream &O) {
1380 for (auto &It : OffsetBins) {
1381 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1382 << "] : " << It.getSecond().size() << "\n";
1383 for (auto AccIndex : It.getSecond()) {
1384 auto &Acc = AccessList[AccIndex];
1385 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1386 if (Acc.getLocalInst() != Acc.getRemoteInst())
1387 O << " --> " << *Acc.getRemoteInst()
1388 << "\n";
1389 if (!Acc.isWrittenValueYetUndetermined()) {
1390 if (isa_and_nonnull<Function>(Acc.getWrittenValue()))
1391 O << " - c: func " << Acc.getWrittenValue()->getName()
1392 << "\n";
1393 else if (Acc.getWrittenValue())
1394 O << " - c: " << *Acc.getWrittenValue() << "\n";
1395 else
1396 O << " - c: <unknown>\n";
1397 }
1398 }
1399 }
1400 }
1401};
1402
1403struct AAPointerInfoFloating : public AAPointerInfoImpl {
1405 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1406 : AAPointerInfoImpl(IRP, A) {}
1407
1408 /// Deal with an access and signal if it was handled successfully.
1409 bool handleAccess(Attributor &A, Instruction &I,
1410 std::optional<Value *> Content, AccessKind Kind,
1411 OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
1412 Type &Ty) {
1413 using namespace AA::PointerInfo;
1415 const DataLayout &DL = A.getDataLayout();
1416 TypeSize AccessSize = DL.getTypeStoreSize(&Ty);
1417 if (!AccessSize.isScalable())
1418 Size = AccessSize.getFixedValue();
1419
1420 // Make a strictly ascending list of offsets as required by addAccess()
1421 SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
1422 llvm::sort(OffsetsSorted);
1423
1425 if (!VT || VT->getElementCount().isScalable() ||
1426 !Content.value_or(nullptr) || !isa<Constant>(*Content) ||
1427 (*Content)->getType() != VT ||
1428 DL.getTypeStoreSize(VT->getElementType()).isScalable()) {
1429 Changed =
1430 Changed | addAccess(A, {OffsetsSorted, Size}, I, Content, Kind, &Ty);
1431 } else {
1432 // Handle vector stores with constant content element-wise.
1433 // TODO: We could look for the elements or create instructions
1434 // representing them.
1435 // TODO: We need to push the Content into the range abstraction
1436 // (AA::RangeTy) to allow different content values for different
1437 // ranges. ranges. Hence, support vectors storing different values.
1438 Type *ElementType = VT->getElementType();
1439 int64_t ElementSize = DL.getTypeStoreSize(ElementType).getFixedValue();
1440 auto *ConstContent = cast<Constant>(*Content);
1441 Type *Int32Ty = Type::getInt32Ty(ElementType->getContext());
1442 SmallVector<int64_t> ElementOffsets(OffsetsSorted.begin(),
1443 OffsetsSorted.end());
1444
1445 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1446 Value *ElementContent = ConstantExpr::getExtractElement(
1447 ConstContent, ConstantInt::get(Int32Ty, i));
1448
1449 // Add the element access.
1450 Changed = Changed | addAccess(A, {ElementOffsets, ElementSize}, I,
1451 ElementContent, Kind, ElementType);
1452
1453 // Advance the offsets for the next element.
1454 for (auto &ElementOffset : ElementOffsets)
1455 ElementOffset += ElementSize;
1456 }
1457 }
1458 return true;
1459 };
1460
1461 ChangeStatus updateImpl(Attributor &A) override;
1462
1463 /// If the indices to \p GEP can be traced to constants, incorporate all
1464 /// of these into \p UsrOI.
1465 ///
1466 /// \return true iff \p UsrOI is updated.
1467 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1468 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1469 const GEPOperator *GEP);
1470
1471 void trackStatistics() const override {
1472 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1473 }
1474};
1475
1476bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1477 const DataLayout &DL,
1478 OffsetInfo &UsrOI,
1479 const OffsetInfo &PtrOI,
1480 const GEPOperator *GEP) {
1481 unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());
1482 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1483 APInt ConstantOffset(BitWidth, 0);
1484
1485 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1486 "Don't look for constant values if the offset has already been "
1487 "determined to be unknown.");
1488
1489 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1490 UsrOI.setUnknown();
1491 return true;
1492 }
1493
1494 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1495 << (VariableOffsets.empty() ? "" : "not") << " constant "
1496 << *GEP << "\n");
1497
1498 auto Union = PtrOI;
1499 Union.addToAll(ConstantOffset.getSExtValue());
1500
1501 // Each VI in VariableOffsets has a set of potential constant values. Every
1502 // combination of elements, picked one each from these sets, is separately
1503 // added to the original set of offsets, thus resulting in more offsets.
1504 for (const auto &VI : VariableOffsets) {
1505 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1506 *this, IRPosition::value(*VI.first), DepClassTy::OPTIONAL);
1507 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1508 UsrOI.setUnknown();
1509 return true;
1510 }
1511
1512 // UndefValue is treated as a zero, which leaves Union as is.
1513 if (PotentialConstantsAA->undefIsContained())
1514 continue;
1515
1516 // We need at least one constant in every set to compute an actual offset.
1517 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1518 // don't actually exist. In other words, the absence of constant values
1519 // implies that the operation can be assumed dead for now.
1520 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1521 if (AssumedSet.empty())
1522 return false;
1523
1524 OffsetInfo Product;
1525 for (const auto &ConstOffset : AssumedSet) {
1526 auto CopyPerOffset = Union;
1527 CopyPerOffset.addToAll(ConstOffset.getSExtValue() *
1528 VI.second.getZExtValue());
1529 Product.merge(CopyPerOffset);
1530 }
1531 Union = Product;
1532 }
1533
1534 UsrOI = std::move(Union);
1535 return true;
1536}
1537
1538ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1539 using namespace AA::PointerInfo;
1541 const DataLayout &DL = A.getDataLayout();
1542 Value &AssociatedValue = getAssociatedValue();
1543
1544 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1545 OffsetInfoMap[&AssociatedValue].insert(0);
1546
1547 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1548 // One does not simply walk into a map and assign a reference to a possibly
1549 // new location. That can cause an invalidation before the assignment
1550 // happens, like so:
1551 //
1552 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1553 //
1554 // The RHS is a reference that may be invalidated by an insertion caused by
1555 // the LHS. So we ensure that the side-effect of the LHS happens first.
1556
1557 assert(OffsetInfoMap.contains(CurPtr) &&
1558 "CurPtr does not exist in the map!");
1559
1560 auto &UsrOI = OffsetInfoMap[Usr];
1561 auto &PtrOI = OffsetInfoMap[CurPtr];
1562 assert(!PtrOI.isUnassigned() &&
1563 "Cannot pass through if the input Ptr was not visited!");
1564 UsrOI.merge(PtrOI);
1565 Follow = true;
1566 return true;
1567 };
1568
1569 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1570 Value *CurPtr = U.get();
1571 User *Usr = U.getUser();
1572 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1573 << "\n");
1574 assert(OffsetInfoMap.count(CurPtr) &&
1575 "The current pointer offset should have been seeded!");
1576 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1577 "Current pointer should be assigned");
1578
1579 // Only address space casts are followed; other casts of a pointer (e.g.,
1580 // ptrtoint, ptrtoaddr, or a bitcast to a byte type) do not produce a
1581 // pointer that can be tracked.
1583 return HandlePassthroughUser(Usr, CurPtr, Follow);
1584
1585 if (auto *GEP = dyn_cast<GEPOperator>(Usr)) {
1586 // Note the order here, the Usr access might change the map, CurPtr is
1587 // already in it though.
1588 auto &UsrOI = OffsetInfoMap[Usr];
1589 auto &PtrOI = OffsetInfoMap[CurPtr];
1590
1591 if (UsrOI.isUnknown())
1592 return true;
1593
1594 if (PtrOI.isUnknown()) {
1595 Follow = true;
1596 UsrOI.setUnknown();
1597 return true;
1598 }
1599
1600 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1601 return true;
1602 }
1603 if (isa<SelectInst>(Usr))
1604 return HandlePassthroughUser(Usr, CurPtr, Follow);
1605 // Returns are allowed if they are in the associated functions. Users can
1606 // then check the call site return. Returns from other functions can't be
1607 // tracked and are cause for invalidation.
1608 if (auto *RI = dyn_cast<ReturnInst>(Usr)) {
1609 if (RI->getFunction() == getAssociatedFunction()) {
1610 auto &PtrOI = OffsetInfoMap[CurPtr];
1611 Changed |= setReachesReturn(PtrOI);
1612 return true;
1613 }
1614 return false;
1615 }
1616
1617 // For PHIs we need to take care of the recurrence explicitly as the value
1618 // might change while we iterate through a loop. For now, we give up if
1619 // the PHI is not invariant.
1620 if (auto *PHI = dyn_cast<PHINode>(Usr)) {
1621 // Note the order here, the Usr access might change the map, CurPtr is
1622 // already in it though.
1623 auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(PHI);
1624 auto &UsrOI = PhiIt->second;
1625 auto &PtrOI = OffsetInfoMap[CurPtr];
1626
1627 // Check if the PHI operand has already an unknown offset as we can't
1628 // improve on that anymore.
1629 if (PtrOI.isUnknown()) {
1630 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1631 << *CurPtr << " in " << *PHI << "\n");
1632 Follow = !UsrOI.isUnknown();
1633 UsrOI.setUnknown();
1634 return true;
1635 }
1636
1637 // Check if the PHI is invariant (so far).
1638 if (UsrOI == PtrOI) {
1639 assert(!PtrOI.isUnassigned() &&
1640 "Cannot assign if the current Ptr was not visited!");
1641 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1642 return true;
1643 }
1644
1645 // Check if the PHI operand can be traced back to AssociatedValue.
1646 APInt Offset(
1647 DL.getIndexSizeInBits(CurPtr->getType()->getPointerAddressSpace()),
1648 0);
1649 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1650 DL, Offset, /* AllowNonInbounds */ true);
1651 auto It = OffsetInfoMap.find(CurPtrBase);
1652 if (It == OffsetInfoMap.end()) {
1653 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1654 << *CurPtr << " in " << *PHI
1655 << " (base: " << *CurPtrBase << ")\n");
1656 UsrOI.setUnknown();
1657 Follow = true;
1658 return true;
1659 }
1660
1661 // Check if the PHI operand is not dependent on the PHI itself. Every
1662 // recurrence is a cyclic net of PHIs in the data flow, and has an
1663 // equivalent Cycle in the control flow. One of those PHIs must be in the
1664 // header of that control flow Cycle. This is independent of the choice of
1665 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1666 // every Cycle header; if such a node is marked unknown, this will
1667 // eventually propagate through the whole net of PHIs in the recurrence.
1668 const auto *CI =
1669 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1670 *PHI->getFunction());
1671 if (mayBeInCycle(CI, cast<Instruction>(Usr), /* HeaderOnly */ true)) {
1672 auto BaseOI = It->getSecond();
1673 BaseOI.addToAll(Offset.getZExtValue());
1674 if (IsFirstPHIUser || BaseOI == UsrOI) {
1675 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1676 << " in " << *Usr << "\n");
1677 return HandlePassthroughUser(Usr, CurPtr, Follow);
1678 }
1679
1680 LLVM_DEBUG(
1681 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1682 << *CurPtr << " in " << *PHI << "\n");
1683 UsrOI.setUnknown();
1684 Follow = true;
1685 return true;
1686 }
1687
1688 UsrOI.merge(PtrOI);
1689 Follow = true;
1690 return true;
1691 }
1692
1693 if (auto *LoadI = dyn_cast<LoadInst>(Usr)) {
1694 // If the access is to a pointer that may or may not be the associated
1695 // value, e.g. due to a PHI, we cannot assume it will be read.
1696 AccessKind AK = AccessKind::AK_R;
1697 if (getUnderlyingObject(CurPtr) == &AssociatedValue)
1698 AK = AccessKind(AK | AccessKind::AK_MUST);
1699 else
1700 AK = AccessKind(AK | AccessKind::AK_MAY);
1701 if (!handleAccess(A, *LoadI, /* Content */ nullptr, AK,
1702 OffsetInfoMap[CurPtr].Offsets, Changed,
1703 *LoadI->getType()))
1704 return false;
1705
1706 auto IsAssumption = [](Instruction &I) {
1707 if (auto *II = dyn_cast<IntrinsicInst>(&I))
1708 return II->isAssumeLikeIntrinsic();
1709 return false;
1710 };
1711
1712 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1713 // Check if the assumption and the load are executed together without
1714 // memory modification.
1715 do {
1716 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1717 return true;
1718 FromI = FromI->getNextNode();
1719 } while (FromI && FromI != ToI);
1720 return false;
1721 };
1722
1723 BasicBlock *BB = LoadI->getParent();
1724 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1725 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1726 return false;
1727 BasicBlock *IntrBB = IntrI.getParent();
1728 if (IntrI.getParent() == BB) {
1729 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1730 return false;
1731 } else {
1732 auto PredIt = pred_begin(IntrBB);
1733 if (PredIt == pred_end(IntrBB))
1734 return false;
1735 if ((*PredIt) != BB)
1736 return false;
1737 if (++PredIt != pred_end(IntrBB))
1738 return false;
1739 for (auto *SuccBB : successors(BB)) {
1740 if (SuccBB == IntrBB)
1741 continue;
1742 if (isa<UnreachableInst>(SuccBB->getTerminator()))
1743 continue;
1744 return false;
1745 }
1746 if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))
1747 return false;
1748 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1749 return false;
1750 }
1751 return true;
1752 };
1753
1754 std::pair<Value *, IntrinsicInst *> Assumption;
1755 for (const Use &LoadU : LoadI->uses()) {
1756 if (auto *CmpI = dyn_cast<CmpInst>(LoadU.getUser())) {
1757 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1758 continue;
1759 for (const Use &CmpU : CmpI->uses()) {
1760 if (auto *IntrI = dyn_cast<IntrinsicInst>(CmpU.getUser())) {
1761 if (!IsValidAssume(*IntrI))
1762 continue;
1763 int Idx = CmpI->getOperandUse(0) == LoadU;
1764 Assumption = {CmpI->getOperand(Idx), IntrI};
1765 break;
1766 }
1767 }
1768 }
1769 if (Assumption.first)
1770 break;
1771 }
1772
1773 // Check if we found an assumption associated with this load.
1774 if (!Assumption.first || !Assumption.second)
1775 return true;
1776
1777 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1778 << *Assumption.second << ": " << *LoadI
1779 << " == " << *Assumption.first << "\n");
1780 bool UsedAssumedInformation = false;
1781 std::optional<Value *> Content = nullptr;
1782 if (Assumption.first)
1783 Content =
1784 A.getAssumedSimplified(*Assumption.first, *this,
1785 UsedAssumedInformation, AA::Interprocedural);
1786 return handleAccess(
1787 A, *Assumption.second, Content, AccessKind::AK_ASSUMPTION,
1788 OffsetInfoMap[CurPtr].Offsets, Changed, *LoadI->getType());
1789 }
1790
1791 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1792 ArrayRef<Value *> OtherOps, AccessKind AK) {
1793 for (auto *OtherOp : OtherOps) {
1794 if (OtherOp == CurPtr) {
1795 LLVM_DEBUG(
1796 dbgs()
1797 << "[AAPointerInfo] Escaping use in store like instruction " << I
1798 << "\n");
1799 return false;
1800 }
1801 }
1802
1803 // If the access is to a pointer that may or may not be the associated
1804 // value, e.g. due to a PHI, we cannot assume it will be written.
1805 if (getUnderlyingObject(CurPtr) == &AssociatedValue)
1806 AK = AccessKind(AK | AccessKind::AK_MUST);
1807 else
1808 AK = AccessKind(AK | AccessKind::AK_MAY);
1809 bool UsedAssumedInformation = false;
1810 std::optional<Value *> Content = nullptr;
1811 if (ValueOp)
1812 Content = A.getAssumedSimplified(
1813 *ValueOp, *this, UsedAssumedInformation, AA::Interprocedural);
1814 return handleAccess(A, I, Content, AK, OffsetInfoMap[CurPtr].Offsets,
1815 Changed, ValueTy);
1816 };
1817
1818 if (auto *StoreI = dyn_cast<StoreInst>(Usr))
1819 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1820 *StoreI->getValueOperand()->getType(),
1821 {StoreI->getValueOperand()}, AccessKind::AK_W);
1822 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Usr))
1823 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1824 {RMWI->getValOperand()}, AccessKind::AK_RW);
1825 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Usr))
1826 return HandleStoreLike(
1827 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1828 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1829 AccessKind::AK_RW);
1830
1831 if (auto *CB = dyn_cast<CallBase>(Usr)) {
1832 if (CB->isLifetimeStartOrEnd())
1833 return true;
1834 const auto *TLI =
1835 A.getInfoCache().getTargetLibraryInfoForFunction(*CB->getFunction());
1836 if (getFreedOperand(CB, TLI) == U)
1837 return true;
1838 if (CB->isArgOperand(&U)) {
1839 unsigned ArgNo = CB->getArgOperandNo(&U);
1840 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1841 *this, IRPosition::callsite_argument(*CB, ArgNo),
1843 if (!CSArgPI)
1844 return false;
1845 bool IsArgMustAcc = (getUnderlyingObject(CurPtr) == &AssociatedValue);
1846 Changed = translateAndAddState(A, *CSArgPI, OffsetInfoMap[CurPtr], *CB,
1847 IsArgMustAcc) |
1848 Changed;
1849 if (!CSArgPI->reachesReturn())
1850 return isValidState();
1851
1853 if (!Callee || Callee->arg_size() <= ArgNo)
1854 return false;
1855 bool UsedAssumedInformation = false;
1856 auto ReturnedValue = A.getAssumedSimplified(
1857 IRPosition::returned(*Callee), *this, UsedAssumedInformation,
1859 auto *ReturnedArg =
1860 dyn_cast_or_null<Argument>(ReturnedValue.value_or(nullptr));
1861 auto *Arg = Callee->getArg(ArgNo);
1862 if (ReturnedArg && Arg != ReturnedArg)
1863 return true;
1864 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1865 const auto *CSRetPI = A.getAAFor<AAPointerInfo>(
1867 if (!CSRetPI)
1868 return false;
1869 OffsetInfo OI = OffsetInfoMap[CurPtr];
1870 CSArgPI->addReturnedOffsetsTo(OI);
1871 Changed =
1872 translateAndAddState(A, *CSRetPI, OI, *CB, IsRetMustAcc) | Changed;
1873 return isValidState();
1874 }
1875 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1876 << "\n");
1877 return false;
1878 }
1879
1880 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1881 return false;
1882 };
1883 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1884 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1885 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1886 if (OffsetInfoMap.count(NewU)) {
1887 LLVM_DEBUG({
1888 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1889 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1890 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1891 << "\n";
1892 }
1893 });
1894 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1895 }
1896 bool Unused;
1897 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1898 };
1899 if (!A.checkForAllUses(UsePred, *this, AssociatedValue,
1900 /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,
1901 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1902 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1903 return indicatePessimisticFixpoint();
1904 }
1905
1906 LLVM_DEBUG({
1907 dbgs() << "Accesses by bin after update:\n";
1908 dumpState(dbgs());
1909 });
1910
1911 return Changed;
1912}
1913
1914struct AAPointerInfoReturned final : AAPointerInfoImpl {
1915 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1916 : AAPointerInfoImpl(IRP, A) {}
1917
1918 ChangeStatus updateImpl(Attributor &A) override {
1919 return indicatePessimisticFixpoint();
1920 }
1921
1922 void trackStatistics() const override {
1923 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1924 }
1925};
1926
1927struct AAPointerInfoArgument final : AAPointerInfoFloating {
1928 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1929 : AAPointerInfoFloating(IRP, A) {}
1930
1931 void trackStatistics() const override {
1932 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1933 }
1934};
1935
1936struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1937 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1938 : AAPointerInfoFloating(IRP, A) {}
1939
1940 ChangeStatus updateImpl(Attributor &A) override {
1941 using namespace AA::PointerInfo;
1942 // We handle memory intrinsics explicitly, at least the first (=
1943 // destination) and second (=source) arguments as we know how they are
1944 // accessed.
1945 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(getCtxI())) {
1946 int64_t LengthVal = AA::RangeTy::Unknown;
1947 if (auto Length = MI->getLengthInBytes())
1948 LengthVal = Length->getSExtValue();
1949 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1950 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1951 if (ArgNo > 1) {
1952 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1953 << *MI << "\n");
1954 return indicatePessimisticFixpoint();
1955 } else {
1956 auto Kind =
1957 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1958 Changed =
1959 Changed | addAccess(A, {0, LengthVal}, *MI, nullptr, Kind, nullptr);
1960 }
1961 LLVM_DEBUG({
1962 dbgs() << "Accesses by bin after update:\n";
1963 dumpState(dbgs());
1964 });
1965
1966 return Changed;
1967 }
1968
1969 // TODO: Once we have call site specific value information we can provide
1970 // call site specific liveness information and then it makes
1971 // sense to specialize attributes for call sites arguments instead of
1972 // redirecting requests to the callee argument.
1973 Argument *Arg = getAssociatedArgument();
1974 if (Arg) {
1975 const IRPosition &ArgPos = IRPosition::argument(*Arg);
1976 auto *ArgAA =
1977 A.getAAFor<AAPointerInfo>(*this, ArgPos, DepClassTy::REQUIRED);
1978 if (ArgAA && ArgAA->getState().isValidState())
1979 return translateAndAddStateFromCallee(A, *ArgAA,
1980 *cast<CallBase>(getCtxI()));
1981 if (!Arg->getParent()->isDeclaration())
1982 return indicatePessimisticFixpoint();
1983 }
1984
1985 bool IsKnownNoCapture;
1987 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
1988 return indicatePessimisticFixpoint();
1989
1990 bool IsKnown = false;
1991 if (AA::isAssumedReadNone(A, getIRPosition(), *this, IsKnown))
1992 return ChangeStatus::UNCHANGED;
1993 bool ReadOnly = AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown);
1994 auto Kind =
1995 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
1996 return addAccess(A, AA::RangeTy::getUnknown(), *getCtxI(), nullptr, Kind,
1997 nullptr);
1998 }
1999
2000 void trackStatistics() const override {
2001 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2002 }
2003};
2004
2005struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2006 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2007 : AAPointerInfoFloating(IRP, A) {}
2008
2009 void trackStatistics() const override {
2010 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2011 }
2012};
2013} // namespace
2014
2015/// -----------------------NoUnwind Function Attribute--------------------------
2016
2017namespace {
2018struct AANoUnwindImpl : AANoUnwind {
2019 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2020
2021 void initialize(Attributor &A) override {
2022 bool IsKnown;
2024 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2025 (void)IsKnown;
2026 }
2027
2028 const std::string getAsStr(Attributor *A) const override {
2029 return getAssumed() ? "nounwind" : "may-unwind";
2030 }
2031
2032 ChangeStatus updateImpl(Attributor &A) override {
2033 auto Opcodes = {
2034 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2035 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2036 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2037
2038 auto CheckForNoUnwind = [&](Instruction &I) {
2039 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2040 return true;
2041
2042 if (const auto *CB = dyn_cast<CallBase>(&I)) {
2043 bool IsKnownNoUnwind;
2045 A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,
2046 IsKnownNoUnwind);
2047 }
2048 return false;
2049 };
2050
2051 bool UsedAssumedInformation = false;
2052 if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes,
2053 UsedAssumedInformation))
2054 return indicatePessimisticFixpoint();
2055
2056 return ChangeStatus::UNCHANGED;
2057 }
2058};
2059
2060struct AANoUnwindFunction final : public AANoUnwindImpl {
2061 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2062 : AANoUnwindImpl(IRP, A) {}
2063
2064 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2065};
2066
2067/// NoUnwind attribute deduction for a call sites.
2068struct AANoUnwindCallSite final
2069 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2070 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2071 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2072
2073 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2074};
2075} // namespace
2076
2077/// ------------------------ NoSync Function Attribute -------------------------
2078
2079bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2080 switch (CB.getIntrinsicID()) {
2081 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2082 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2083 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2084 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2085 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2086 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2087 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2088 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2089 return true;
2090 case Intrinsic::amdgcn_s_barrier:
2091 if (ExecutedAligned)
2092 return true;
2093 break;
2094 default:
2095 break;
2096 }
2097 return hasAssumption(CB, KnownAssumptionString("ompx_aligned_barrier"));
2098}
2099
2101 if (!I->isAtomic())
2102 return false;
2103
2104 if (auto *FI = dyn_cast<FenceInst>(I))
2105 // All legal orderings for fence are stronger than monotonic.
2106 return FI->getSyncScopeID() != SyncScope::SingleThread;
2107 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
2108 // Unordered is not a legal ordering for cmpxchg.
2109 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2110 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2111 }
2112
2113 AtomicOrdering Ordering;
2114 switch (I->getOpcode()) {
2115 case Instruction::AtomicRMW:
2116 Ordering = cast<AtomicRMWInst>(I)->getOrdering();
2117 break;
2118 case Instruction::Store:
2119 Ordering = cast<StoreInst>(I)->getOrdering();
2120 break;
2121 case Instruction::Load:
2122 Ordering = cast<LoadInst>(I)->getOrdering();
2123 break;
2124 default:
2126 "New atomic operations need to be known in the attributor.");
2127 }
2128
2129 return (Ordering != AtomicOrdering::Unordered &&
2130 Ordering != AtomicOrdering::Monotonic);
2131}
2132
2133namespace {
2134struct AANoSyncImpl : AANoSync {
2135 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2136
2137 void initialize(Attributor &A) override {
2138 bool IsKnown;
2139 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2140 DepClassTy::NONE, IsKnown));
2141 (void)IsKnown;
2142 }
2143
2144 const std::string getAsStr(Attributor *A) const override {
2145 return getAssumed() ? "nosync" : "may-sync";
2146 }
2147
2148 ChangeStatus updateImpl(Attributor &A) override;
2149};
2150
2151ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2152
2153 auto CheckRWInstForNoSync = [&](Instruction &I) {
2154 return AA::isNoSyncInst(A, I, *this);
2155 };
2156
2157 auto CheckForNoSync = [&](Instruction &I) {
2158 // At this point we handled all read/write effects and they are all
2159 // nosync, so they can be skipped.
2160 if (I.mayReadOrWriteMemory())
2161 return true;
2162
2163 bool IsKnown;
2164 CallBase &CB = cast<CallBase>(I);
2167 IsKnown))
2168 return true;
2169
2170 // non-convergent and readnone imply nosync.
2171 return !CB.isConvergent();
2172 };
2173
2174 bool UsedAssumedInformation = false;
2175 if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this,
2176 UsedAssumedInformation) ||
2177 !A.checkForAllCallLikeInstructions(CheckForNoSync, *this,
2178 UsedAssumedInformation))
2179 return indicatePessimisticFixpoint();
2180
2182}
2183
2184struct AANoSyncFunction final : public AANoSyncImpl {
2185 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2186 : AANoSyncImpl(IRP, A) {}
2187
2188 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2189};
2190
2191/// NoSync attribute deduction for a call sites.
2192struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2193 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2194 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2195
2196 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2197};
2198} // namespace
2199
2200/// ------------------------ No-Free Attributes ----------------------------
2201
2202namespace {
2203struct AANoFreeImpl : public AANoFree {
2204 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2205
2206 void initialize(Attributor &A) override {
2207 bool IsKnown;
2208 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2209 DepClassTy::NONE, IsKnown));
2210 (void)IsKnown;
2211 }
2212
2213 ChangeStatus updateImpl(Attributor &A) override {
2214 auto CheckForNoFree = [&](Instruction &I) {
2215 if (auto *CB = dyn_cast<CallBase>(&I)) {
2216 bool IsKnown;
2218 A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,
2219 IsKnown);
2220 }
2221 // Make sure that synchronization cannot establish happens-before with a
2222 // free on another thread.
2223 return AA::isNoSyncInst(A, I, *this);
2224 };
2225
2226 bool UsedAssumedInformation = false;
2227 if (!A.checkForAllReadWriteInstructions(CheckForNoFree, *this,
2228 UsedAssumedInformation) ||
2229 !A.checkForAllCallLikeInstructions(CheckForNoFree, *this,
2230 UsedAssumedInformation))
2231 return indicatePessimisticFixpoint();
2232
2233 return ChangeStatus::UNCHANGED;
2234 }
2235
2236 const std::string getAsStr(Attributor *A) const override {
2237 return getAssumed() ? "nofree" : "may-free";
2238 }
2239};
2240
2241struct AANoFreeFunction final : public AANoFreeImpl {
2242 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2243 : AANoFreeImpl(IRP, A) {}
2244
2245 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2246};
2247
2248/// NoFree attribute deduction for a call sites.
2249struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2250 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2251 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2252
2253 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2254};
2255
2256/// NoFree attribute for floating values.
2257struct AANoFreeFloating : AANoFreeImpl {
2258 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2259 : AANoFreeImpl(IRP, A) {}
2260
2261 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2262
2263 /// See Abstract Attribute::updateImpl(...).
2264 ChangeStatus updateImpl(Attributor &A) override {
2265 const IRPosition &IRP = getIRPosition();
2266
2267 bool IsKnown;
2270 DepClassTy::OPTIONAL, IsKnown))
2271 return ChangeStatus::UNCHANGED;
2272
2273 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2274 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2275 Instruction *UserI = cast<Instruction>(U.getUser());
2276 if (auto *CB = dyn_cast<CallBase>(UserI)) {
2277 if (CB->isBundleOperand(&U))
2278 return false;
2279 if (!CB->isArgOperand(&U))
2280 return true;
2281 unsigned ArgNo = CB->getArgOperandNo(&U);
2282
2283 // Even if the argument is nofree, we still need to check for nocapture,
2284 // as the call may capture the argument without freeing it, and the
2285 // captured argument is freed later.
2286 bool IsKnown;
2288 A, this, IRPosition::callsite_argument(*CB, ArgNo),
2289 DepClassTy::REQUIRED, IsKnown))
2290 return false;
2291
2292 const AANoCapture *NoCaptureAA = nullptr;
2294 A, this, IRPosition::callsite_argument(*CB, ArgNo),
2295 DepClassTy::REQUIRED, IsKnown,
2296 /*IgnoreSubsumingPositions=*/false, &NoCaptureAA)) {
2297 if (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
2298 Follow = true;
2299 return true;
2300 }
2301 return false;
2302 }
2303
2304 return true;
2305 }
2306
2307 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
2308 if (!capturesAnyProvenance(CI))
2309 return true;
2311 Follow = true;
2312 return true;
2313 }
2314
2315 if (isa<ReturnInst>(UserI) && getIRPosition().isArgumentPosition())
2316 return true;
2317
2318 // Capturing user.
2319 return false;
2320 };
2321 if (!A.checkForAllUses(Pred, *this, AssociatedValue))
2322 return indicatePessimisticFixpoint();
2323
2324 return ChangeStatus::UNCHANGED;
2325 }
2326};
2327
2328/// NoFree attribute for a call site argument.
2329struct AANoFreeArgument final : AANoFreeFloating {
2330 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2331 : AANoFreeFloating(IRP, A) {}
2332
2333 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2334};
2335
2336/// NoFree attribute for call site arguments.
2337struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2338 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2339 : AANoFreeFloating(IRP, A) {}
2340
2341 ChangeStatus updateImpl(Attributor &A) override {
2342 // TODO: Once we have call site specific value information we can provide
2343 // call site specific liveness information and then it makes
2344 // sense to specialize attributes for call sites arguments instead of
2345 // redirecting requests to the callee argument.
2346 Argument *Arg = getAssociatedArgument();
2347 if (!Arg)
2348 return indicatePessimisticFixpoint();
2349 const IRPosition &ArgPos = IRPosition::argument(*Arg);
2350 bool IsKnown;
2352 DepClassTy::REQUIRED, IsKnown))
2353 return ChangeStatus::UNCHANGED;
2354 return indicatePessimisticFixpoint();
2355 }
2356
2357 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };
2358};
2359
2360/// NoFree attribute for function return value.
2361struct AANoFreeReturned final : AANoFreeFloating {
2362 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2363 : AANoFreeFloating(IRP, A) {
2364 llvm_unreachable("NoFree is not applicable to function returns!");
2365 }
2366
2367 void initialize(Attributor &A) override {
2368 llvm_unreachable("NoFree is not applicable to function returns!");
2369 }
2370
2371 ChangeStatus updateImpl(Attributor &A) override {
2372 llvm_unreachable("NoFree is not applicable to function returns!");
2373 }
2374
2375 void trackStatistics() const override {}
2376};
2377
2378/// NoFree attribute deduction for a call site return value.
2379struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2380 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2381 : AANoFreeFloating(IRP, A) {}
2382
2383 ChangeStatus manifest(Attributor &A) override {
2384 return ChangeStatus::UNCHANGED;
2385 }
2386 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2387};
2388} // namespace
2389
2390/// ------------------------ NonNull Argument Attribute ------------------------
2391
2393 Attribute::AttrKind ImpliedAttributeKind,
2394 bool IgnoreSubsumingPositions) {
2396 AttrKinds.push_back(Attribute::NonNull);
2399 AttrKinds.push_back(Attribute::Dereferenceable);
2400 if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))
2401 return true;
2402
2403 if (!A.canUseBodyFor(IRP))
2404 return false;
2405
2406 DominatorTree *DT = nullptr;
2407 AssumptionCache *AC = nullptr;
2408 InformationCache &InfoCache = A.getInfoCache();
2409 if (const Function *Fn = IRP.getAnchorScope()) {
2410 if (!Fn->isDeclaration()) {
2413 }
2414 }
2415
2417 if (IRP.getPositionKind() != IRP_RETURNED) {
2418 Worklist.push_back({IRP.getAssociatedValue(), IRP.getCtxI()});
2419 } else {
2420 bool UsedAssumedInformation = false;
2421 if (!A.checkForAllInstructions(
2422 [&](Instruction &I) {
2423 Worklist.push_back({*cast<ReturnInst>(I).getReturnValue(), &I});
2424 return true;
2425 },
2426 IRP.getAssociatedFunction(), nullptr, {Instruction::Ret},
2427 UsedAssumedInformation, false, /*CheckPotentiallyDead=*/true))
2428 return false;
2429 }
2430
2431 if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {
2432 return !isKnownNonZero(
2433 VAC.getValue(),
2434 SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2435 }))
2436 return false;
2437
2438 A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(),
2439 Attribute::NonNull)});
2440 return true;
2441}
2442
2443namespace {
2444static int64_t getKnownNonNullAndDerefBytesForUse(
2445 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2446 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2447 TrackUse = false;
2448
2449 const Value *UseV = U->get();
2450 if (!UseV->getType()->isPointerTy())
2451 return 0;
2452
2453 // We need to follow common pointer manipulation uses to the accesses they
2454 // feed into. We can try to be smart to avoid looking through things we do not
2455 // like for now, e.g., non-inbounds GEPs.
2456 if (isa<CastInst>(I)) {
2457 TrackUse = true;
2458 return 0;
2459 }
2460
2462 TrackUse = true;
2463 return 0;
2464 }
2465
2466 Type *PtrTy = UseV->getType();
2467 const Function *F = I->getFunction();
2470 const DataLayout &DL = A.getInfoCache().getDL();
2471 if (const auto *CB = dyn_cast<CallBase>(I)) {
2472 if (CB->isBundleOperand(U)) {
2473 auto OBU = CB->getOperandBundleForOperand(U->getOperandNo());
2474 switch (getBundleAttrFromOBU(OBU)) {
2475 case BundleAttr::NonNull:
2476 IsNonNull = true;
2477 break;
2478 case BundleAttr::Dereferenceable: {
2479 IsNonNull |= !NullPointerIsDefined;
2481 if (auto *CI = dyn_cast<ConstantInt>(Count))
2482 return CI->getZExtValue();
2483 } break;
2484 default:
2485 break;
2486 }
2487 return 0;
2488 }
2489
2490 if (CB->isCallee(U)) {
2491 IsNonNull |= !NullPointerIsDefined;
2492 return 0;
2493 }
2494
2495 unsigned ArgNo = CB->getArgOperandNo(U);
2496 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
2497 // As long as we only use known information there is no need to track
2498 // dependences here.
2499 bool IsKnownNonNull;
2501 DepClassTy::NONE, IsKnownNonNull);
2502 IsNonNull |= IsKnownNonNull;
2503 auto *DerefAA =
2504 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClassTy::NONE);
2505 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2506 }
2507
2508 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
2509 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2510 Loc->Size.isScalable() || I->isVolatile())
2511 return 0;
2512
2513 int64_t Offset;
2514 const Value *Base =
2515 getMinimalBaseOfPointer(A, QueryingAA, Loc->Ptr, Offset, DL);
2516 if (Base && Base == &AssociatedValue) {
2517 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2518 IsNonNull |= !NullPointerIsDefined;
2519 return std::max(int64_t(0), DerefBytes);
2520 }
2521
2522 /// Corner case when an offset is 0.
2524 /*AllowNonInbounds*/ true);
2525 if (Base && Base == &AssociatedValue && Offset == 0) {
2526 int64_t DerefBytes = Loc->Size.getValue();
2527 IsNonNull |= !NullPointerIsDefined;
2528 return std::max(int64_t(0), DerefBytes);
2529 }
2530
2531 return 0;
2532}
2533
2534struct AANonNullImpl : AANonNull {
2535 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2536
2537 void initialize(Attributor &A) override {
2538 Value &V = *getAssociatedValue().stripPointerCasts();
2540 indicatePessimisticFixpoint();
2541 return;
2542 }
2543
2544 if (Instruction *CtxI = getCtxI())
2545 followUsesInMBEC(*this, A, getState(), *CtxI);
2546 }
2547
2548 /// See followUsesInMBEC
2549 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2550 AANonNull::StateType &State) {
2551 bool IsNonNull = false;
2552 bool TrackUse = false;
2553 getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I,
2554 IsNonNull, TrackUse);
2555 State.setKnown(IsNonNull);
2556 return TrackUse;
2557 }
2558
2559 const std::string getAsStr(Attributor *A) const override {
2560 return getAssumed() ? "nonnull" : "may-null";
2561 }
2562};
2563
2564/// NonNull attribute for a floating value.
2565struct AANonNullFloating : public AANonNullImpl {
2566 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2567 : AANonNullImpl(IRP, A) {}
2568
2569 ChangeStatus updateImpl(Attributor &A) override {
2570 auto CheckIRP = [&](const IRPosition &IRP) {
2571 bool IsKnownNonNull;
2573 A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);
2574 };
2575
2576 bool Stripped;
2577 bool UsedAssumedInformation = false;
2578 Value *AssociatedValue = &getAssociatedValue();
2580 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
2581 AA::AnyScope, UsedAssumedInformation))
2582 Stripped = false;
2583 else
2584 Stripped =
2585 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2586
2587 if (!Stripped) {
2588 bool IsKnown;
2589 if (auto *PHI = dyn_cast<PHINode>(AssociatedValue))
2590 if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) {
2591 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2592 A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,
2593 IsKnown);
2594 }))
2595 return ChangeStatus::UNCHANGED;
2596 if (auto *Select = dyn_cast<SelectInst>(AssociatedValue))
2598 A, this, IRPosition::value(*Select->getFalseValue()),
2599 DepClassTy::OPTIONAL, IsKnown) &&
2601 A, this, IRPosition::value(*Select->getTrueValue()),
2602 DepClassTy::OPTIONAL, IsKnown))
2603 return ChangeStatus::UNCHANGED;
2604
2605 // If we haven't stripped anything we might still be able to use a
2606 // different AA, but only if the IRP changes. Effectively when we
2607 // interpret this not as a call site value but as a floating/argument
2608 // value.
2609 const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
2610 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2611 return indicatePessimisticFixpoint();
2612 return ChangeStatus::UNCHANGED;
2613 }
2614
2615 for (const auto &VAC : Values)
2616 if (!CheckIRP(IRPosition::value(*VAC.getValue())))
2617 return indicatePessimisticFixpoint();
2618
2619 return ChangeStatus::UNCHANGED;
2620 }
2621
2622 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2623};
2624
2625/// NonNull attribute for function return value.
2626struct AANonNullReturned final
2627 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2628 false, AANonNull::IRAttributeKind, false> {
2629 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2630 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2631 false, Attribute::NonNull, false>(IRP, A) {
2632 }
2633
2634 const std::string getAsStr(Attributor *A) const override {
2635 return getAssumed() ? "nonnull" : "may-null";
2636 }
2637
2638 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2639};
2640
2641/// NonNull attribute for function argument.
2642struct AANonNullArgument final
2643 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2644 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2645 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2646
2647 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2648};
2649
2650struct AANonNullCallSiteArgument final : AANonNullFloating {
2651 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2652 : AANonNullFloating(IRP, A) {}
2653
2654 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2655};
2656
2657/// NonNull attribute for a call site return position.
2658struct AANonNullCallSiteReturned final
2659 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2660 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2661 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2662
2663 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2664};
2665} // namespace
2666
2667/// ------------------------ Must-Progress Attributes --------------------------
2668namespace {
2669struct AAMustProgressImpl : public AAMustProgress {
2670 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2671 : AAMustProgress(IRP, A) {}
2672
2673 void initialize(Attributor &A) override {
2674 bool IsKnown;
2676 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2677 (void)IsKnown;
2678 }
2679
2680 const std::string getAsStr(Attributor *A) const override {
2681 return getAssumed() ? "mustprogress" : "may-not-progress";
2682 }
2683};
2684
2685struct AAMustProgressFunction final : AAMustProgressImpl {
2686 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2687 : AAMustProgressImpl(IRP, A) {}
2688
2689 ChangeStatus updateImpl(Attributor &A) override {
2690 bool IsKnown;
2692 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {
2693 if (IsKnown)
2694 return indicateOptimisticFixpoint();
2695 return ChangeStatus::UNCHANGED;
2696 }
2697
2698 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2699 IRPosition IPos = IRPosition::callsite_function(*ACS.getInstruction());
2700 bool IsKnownMustProgress;
2702 A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,
2703 /* IgnoreSubsumingPositions */ true);
2704 };
2705
2706 bool AllCallSitesKnown = true;
2707 if (!A.checkForAllCallSites(CheckForMustProgress, *this,
2708 /* RequireAllCallSites */ true,
2709 AllCallSitesKnown))
2710 return indicatePessimisticFixpoint();
2711
2712 return ChangeStatus::UNCHANGED;
2713 }
2714
2715 void trackStatistics() const override {
2716 STATS_DECLTRACK_FN_ATTR(mustprogress)
2717 }
2718};
2719
2720/// MustProgress attribute deduction for a call sites.
2721struct AAMustProgressCallSite final : AAMustProgressImpl {
2722 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2723 : AAMustProgressImpl(IRP, A) {}
2724
2725 ChangeStatus updateImpl(Attributor &A) override {
2726 // TODO: Once we have call site specific value information we can provide
2727 // call site specific liveness information and then it makes
2728 // sense to specialize attributes for call sites arguments instead of
2729 // redirecting requests to the callee argument.
2730 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
2731 bool IsKnownMustProgress;
2733 A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))
2734 return indicatePessimisticFixpoint();
2735 return ChangeStatus::UNCHANGED;
2736 }
2737
2738 void trackStatistics() const override {
2739 STATS_DECLTRACK_CS_ATTR(mustprogress);
2740 }
2741};
2742} // namespace
2743
2744/// ------------------------ No-Recurse Attributes ----------------------------
2745
2746namespace {
2747struct AANoRecurseImpl : public AANoRecurse {
2748 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2749
2750 void initialize(Attributor &A) override {
2751 bool IsKnown;
2753 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2754 (void)IsKnown;
2755 }
2756
2757 const std::string getAsStr(Attributor *A) const override {
2758 return getAssumed() ? "norecurse" : "may-recurse";
2759 }
2760};
2761
2762struct AANoRecurseFunction final : AANoRecurseImpl {
2763 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2764 : AANoRecurseImpl(IRP, A) {}
2765
2766 ChangeStatus updateImpl(Attributor &A) override {
2767
2768 // If all live call sites are known to be no-recurse, we are as well.
2769 auto CallSitePred = [&](AbstractCallSite ACS) {
2770 bool IsKnownNoRecurse;
2772 A, this,
2773 IRPosition::function(*ACS.getInstruction()->getFunction()),
2774 DepClassTy::NONE, IsKnownNoRecurse))
2775 return false;
2776 return IsKnownNoRecurse;
2777 };
2778 bool UsedAssumedInformation = false;
2779 if (A.checkForAllCallSites(CallSitePred, *this, true,
2780 UsedAssumedInformation)) {
2781 // If we know all call sites and all are known no-recurse, we are done.
2782 // If all known call sites, which might not be all that exist, are known
2783 // to be no-recurse, we are not done but we can continue to assume
2784 // no-recurse. If one of the call sites we have not visited will become
2785 // live, another update is triggered.
2786 if (!UsedAssumedInformation)
2787 indicateOptimisticFixpoint();
2788 return ChangeStatus::UNCHANGED;
2789 }
2790
2791 const AAInterFnReachability *EdgeReachability =
2792 A.getAAFor<AAInterFnReachability>(*this, getIRPosition(),
2793 DepClassTy::REQUIRED);
2794 if (EdgeReachability && EdgeReachability->canReach(A, *getAnchorScope()))
2795 return indicatePessimisticFixpoint();
2796 return ChangeStatus::UNCHANGED;
2797 }
2798
2799 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2800};
2801
2802/// NoRecurse attribute deduction for a call sites.
2803struct AANoRecurseCallSite final
2804 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2805 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2806 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2807
2808 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2809};
2810} // namespace
2811
2812/// ------------------------ No-Convergent Attribute --------------------------
2813
2814namespace {
2815struct AANonConvergentImpl : public AANonConvergent {
2816 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2817 : AANonConvergent(IRP, A) {}
2818
2819 const std::string getAsStr(Attributor *A) const override {
2820 return getAssumed() ? "non-convergent" : "may-be-convergent";
2821 }
2822};
2823
2824struct AANonConvergentFunction final : AANonConvergentImpl {
2825 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2826 : AANonConvergentImpl(IRP, A) {}
2827
2828 ChangeStatus updateImpl(Attributor &A) override {
2829 // If all function calls are known to not be convergent, we are not
2830 // convergent.
2831 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2832 CallBase &CB = cast<CallBase>(Inst);
2834 if (!Callee || Callee->isIntrinsic()) {
2835 return false;
2836 }
2837 if (Callee->isDeclaration()) {
2838 return !Callee->hasFnAttribute(Attribute::Convergent);
2839 }
2840 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2841 *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);
2842 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2843 };
2844
2845 bool UsedAssumedInformation = false;
2846 if (!A.checkForAllCallLikeInstructions(CalleeIsNotConvergent, *this,
2847 UsedAssumedInformation)) {
2848 return indicatePessimisticFixpoint();
2849 }
2850 return ChangeStatus::UNCHANGED;
2851 }
2852
2853 ChangeStatus manifest(Attributor &A) override {
2854 if (isKnownNotConvergent() &&
2855 A.hasAttr(getIRPosition(), Attribute::Convergent)) {
2856 A.removeAttrs(getIRPosition(), {Attribute::Convergent});
2857 return ChangeStatus::CHANGED;
2858 }
2859 return ChangeStatus::UNCHANGED;
2860 }
2861
2862 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2863};
2864} // namespace
2865
2866/// -------------------- Undefined-Behavior Attributes ------------------------
2867
2868namespace {
2869struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2870 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2871 : AAUndefinedBehavior(IRP, A) {}
2872
2873 struct UBInfo {
2874 enum Kind {
2875 NullPtrAccess,
2876 UndefPtrAccess,
2877 UndefBranchCondition,
2878 UndefReturnValue,
2879 NullReturnViolatesNonNull,
2880 UndefCallArgument,
2881 NullArgViolatesNonNull,
2882 };
2883
2884 Kind K;
2885 std::optional<unsigned> ArgNo;
2886
2887 UBInfo(Kind K) : K(K), ArgNo(std::nullopt) {}
2888
2889 UBInfo(Kind K, std::optional<unsigned> ArgNo) : K(K), ArgNo(ArgNo) {}
2890 };
2891
2892 ChangeStatus updateImpl(Attributor &A) override {
2893 const size_t UBPrevSize = KnownUBInsts.size();
2894 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2895
2896 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2897 // Volatile accesses on null are not necessarily UB.
2898 if (I.isVolatile())
2899 return true;
2900
2901 // Skip instructions that are already saved.
2902 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2903 return true;
2904
2905 // If we reach here, we know we have an instruction
2906 // that accesses memory through a pointer operand,
2907 // for which getPointerOperand() should give it to us.
2908 Value *PtrOp =
2909 const_cast<Value *>(getPointerOperand(&I, /* AllowVolatile */ true));
2910 assert(PtrOp &&
2911 "Expected pointer operand of memory accessing instruction");
2912
2913 // Either we stopped and the appropriate action was taken,
2914 // or we got back a simplified value to continue.
2915 std::optional<Value *> SimplifiedPtrOp =
2916 stopOnUndefOrAssumed(A, PtrOp, &I, UBInfo::UndefPtrAccess);
2917 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2918 return true;
2919 const Value *PtrOpVal = *SimplifiedPtrOp;
2920
2921 // A memory access through a pointer is considered UB
2922 // only if the pointer has constant null value.
2923 // TODO: Expand it to not only check constant values.
2924 if (!isa<ConstantPointerNull>(PtrOpVal)) {
2925 AssumedNoUBInsts.insert(&I);
2926 return true;
2927 }
2928 const Type *PtrTy = PtrOpVal->getType();
2929
2930 // Because we only consider instructions inside functions,
2931 // assume that a parent function exists.
2932 const Function *F = I.getFunction();
2933
2934 // A memory access using constant null pointer is only considered UB
2935 // if null pointer is _not_ defined for the target platform.
2937 AssumedNoUBInsts.insert(&I);
2938 else
2939 KnownUBInsts.try_emplace(&I, UBInfo::NullPtrAccess);
2940 return true;
2941 };
2942
2943 auto InspectBrInstForUB = [&](Instruction &I) {
2944 // A conditional branch instruction is considered UB if it has `undef`
2945 // condition.
2946
2947 // Skip instructions that are already saved.
2948 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2949 return true;
2950
2951 // We know we have a branch instruction.
2952 auto *BrInst = cast<CondBrInst>(&I);
2953
2954 // Either we stopped and the appropriate action was taken,
2955 // or we got back a simplified value to continue.
2956 std::optional<Value *> SimplifiedCond = stopOnUndefOrAssumed(
2957 A, BrInst->getCondition(), BrInst, UBInfo::UndefBranchCondition);
2958 if (!SimplifiedCond || !*SimplifiedCond)
2959 return true;
2960 AssumedNoUBInsts.insert(&I);
2961 return true;
2962 };
2963
2964 auto InspectCallSiteForUB = [&](Instruction &I) {
2965 // Check whether a callsite always cause UB or not
2966
2967 // Skip instructions that are already saved.
2968 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2969 return true;
2970
2971 // Check nonnull and noundef argument attribute violation for each
2972 // callsite.
2973 CallBase &CB = cast<CallBase>(I);
2975 if (!Callee)
2976 return true;
2977 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
2978 // If current argument is known to be simplified to null pointer and the
2979 // corresponding argument position is known to have nonnull attribute,
2980 // the argument is poison. Furthermore, if the argument is poison and
2981 // the position is known to have noundef attriubte, this callsite is
2982 // considered UB.
2983 if (idx >= Callee->arg_size())
2984 break;
2985 Value *ArgVal = CB.getArgOperand(idx);
2986 if (!ArgVal)
2987 continue;
2988 // Here, we handle three cases.
2989 // (1) Not having a value means it is dead. (we can replace the value
2990 // with undef)
2991 // (2) Simplified to undef. The argument violate noundef attriubte.
2992 // (3) Simplified to null pointer where known to be nonnull.
2993 // The argument is a poison value and violate noundef attribute.
2994 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, idx);
2995 bool IsKnownNoUndef;
2997 A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);
2998 if (!IsKnownNoUndef)
2999 continue;
3000 bool UsedAssumedInformation = false;
3001 std::optional<Value *> SimplifiedVal =
3002 A.getAssumedSimplified(IRPosition::value(*ArgVal), *this,
3003 UsedAssumedInformation, AA::Interprocedural);
3004 if (UsedAssumedInformation)
3005 continue;
3006 if (SimplifiedVal && !*SimplifiedVal)
3007 return true;
3008 if (!SimplifiedVal || isa<UndefValue>(**SimplifiedVal)) {
3009 KnownUBInsts.try_emplace(&I, UBInfo(UBInfo::UndefCallArgument, idx));
3010 continue;
3011 }
3012 if (!ArgVal->getType()->isPointerTy() ||
3013 !isa<ConstantPointerNull>(**SimplifiedVal))
3014 continue;
3015 bool IsKnownNonNull;
3017 A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);
3018 if (IsKnownNonNull)
3019 KnownUBInsts.try_emplace(&I,
3020 UBInfo(UBInfo::NullArgViolatesNonNull, idx));
3021 }
3022 return true;
3023 };
3024
3025 auto InspectReturnInstForUB = [&](Instruction &I) {
3026 auto &RI = cast<ReturnInst>(I);
3027 // Either we stopped and the appropriate action was taken,
3028 // or we got back a simplified return value to continue.
3029 std::optional<Value *> SimplifiedRetValue = stopOnUndefOrAssumed(
3030 A, RI.getReturnValue(), &I, UBInfo::UndefReturnValue);
3031 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3032 return true;
3033
3034 // Check if a return instruction always cause UB or not
3035 // Note: It is guaranteed that the returned position of the anchor
3036 // scope has noundef attribute when this is called.
3037 // We also ensure the return position is not "assumed dead"
3038 // because the returned value was then potentially simplified to
3039 // `undef` in AAReturnedValues without removing the `noundef`
3040 // attribute yet.
3041
3042 // When the returned position has noundef attriubte, UB occurs in the
3043 // following cases.
3044 // (1) Returned value is known to be undef.
3045 // (2) The value is known to be a null pointer and the returned
3046 // position has nonnull attribute (because the returned value is
3047 // poison).
3048 if (isa<ConstantPointerNull>(*SimplifiedRetValue)) {
3049 bool IsKnownNonNull;
3051 A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE,
3052 IsKnownNonNull);
3053 if (IsKnownNonNull)
3054 KnownUBInsts.try_emplace(&I, UBInfo::NullReturnViolatesNonNull);
3055 }
3056
3057 return true;
3058 };
3059
3060 bool UsedAssumedInformation = false;
3061 A.checkForAllInstructions(InspectMemAccessInstForUB, *this,
3062 {Instruction::Load, Instruction::Store,
3063 Instruction::AtomicCmpXchg,
3064 Instruction::AtomicRMW},
3065 UsedAssumedInformation,
3066 /* CheckBBLivenessOnly */ true);
3067 A.checkForAllInstructions(InspectBrInstForUB, *this, {Instruction::CondBr},
3068 UsedAssumedInformation,
3069 /* CheckBBLivenessOnly */ true);
3070 A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *this,
3071 UsedAssumedInformation);
3072
3073 // If the returned position of the anchor scope has noundef attriubte, check
3074 // all returned instructions.
3075 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3076 const IRPosition &ReturnIRP = IRPosition::returned(*getAnchorScope());
3077 if (!A.isAssumedDead(ReturnIRP, this, nullptr, UsedAssumedInformation)) {
3078 bool IsKnownNoUndef;
3080 A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);
3081 if (IsKnownNoUndef)
3082 A.checkForAllInstructions(InspectReturnInstForUB, *this,
3083 {Instruction::Ret}, UsedAssumedInformation,
3084 /* CheckBBLivenessOnly */ true);
3085 }
3086 }
3087
3088 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3089 UBPrevSize != KnownUBInsts.size())
3090 return ChangeStatus::CHANGED;
3091 return ChangeStatus::UNCHANGED;
3092 }
3093
3094 bool isKnownToCauseUB(Instruction *I) const override {
3095 return KnownUBInsts.count(I);
3096 }
3097
3098 bool isAssumedToCauseUB(Instruction *I) const override {
3099 // In simple words, if an instruction is not in the assumed to _not_
3100 // cause UB, then it is assumed UB (that includes those
3101 // in the KnownUBInsts set). The rest is boilerplate
3102 // is to ensure that it is one of the instructions we test
3103 // for UB.
3104
3105 switch (I->getOpcode()) {
3106 case Instruction::Load:
3107 case Instruction::Store:
3108 case Instruction::AtomicCmpXchg:
3109 case Instruction::AtomicRMW:
3110 case Instruction::CondBr:
3111 return !AssumedNoUBInsts.count(I);
3112 default:
3113 return false;
3114 }
3115 return false;
3116 }
3117
3118 /// Emit an optimization remark explaining why \p I is known to cause UB,
3119 /// per \p Info, right before it is replaced with 'unreachable'.
3120 static void emitUBRemark(Attributor &A, Instruction *I, const UBInfo &Info) {
3121 auto Remark = [&](OptimizationRemark OR) {
3122 switch (Info.K) {
3123 case UBInfo::NullPtrAccess:
3124 case UBInfo::UndefPtrAccess: {
3125 return OR << "Memory access through a pointer known to be "
3126 << ore::NV("Pointer",
3127 getPointerOperand(I, /*AllowVolatile*/ true))
3128 << " is undefined behavior; replacing with 'unreachable'.";
3129 }
3130 case UBInfo::UndefBranchCondition:
3131 return OR << "Branch condition known to be "
3132 << ore::NV("Condition", cast<CondBrInst>(I)->getCondition())
3133 << " is undefined behavior; replacing with 'unreachable'.";
3134 case UBInfo::UndefReturnValue:
3135 case UBInfo::NullReturnViolatesNonNull:
3136 return OR << "Value returned known to be "
3137 << ore::NV("ReturnValue",
3138 cast<ReturnInst>(I)->getReturnValue())
3139 << " is undefined behavior; replacing with 'unreachable'.";
3140 case UBInfo::UndefCallArgument:
3141 case UBInfo::NullArgViolatesNonNull: {
3142 bool IsUndef = Info.K == UBInfo::UndefCallArgument;
3143 CallBase &CB = *cast<CallBase>(I);
3144 OR << "Argument " << ore::NV("ArgNo", *Info.ArgNo)
3145 << " passed to parameter of ";
3146 if (auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand()))
3147 OR << ore::NV("Callee", Callee);
3148 else
3149 OR << "the callee";
3150 return OR << " known to be "
3151 << ore::NV("Argument", IsUndef ? "undef" : "null")
3152 << " is undefined behavior; replacing with 'unreachable'.";
3153 }
3154 }
3155 llvm_unreachable("Unknown UBInfo::Kind");
3156 };
3157 A.emitRemark<OptimizationRemark>(I, "UndefinedBehavior", Remark);
3158 }
3159
3160 ChangeStatus manifest(Attributor &A) override {
3161 if (KnownUBInsts.empty())
3162 return ChangeStatus::UNCHANGED;
3163 for (const auto &[I, Info] : KnownUBInsts) {
3164 emitUBRemark(A, I, Info);
3165 A.changeToUnreachableAfterManifest(I);
3166 }
3167 return ChangeStatus::CHANGED;
3168 }
3169
3170 const std::string getAsStr(Attributor *A) const override {
3171 return getAssumed() ? "undefined-behavior" : "no-ub";
3172 }
3173
3174 /// Note: The correctness of this analysis depends on the fact that the
3175 /// following 2 sets will stop changing after some point.
3176 /// "Change" here means that their size changes.
3177 /// The size of each set is monotonically increasing
3178 /// (we only add items to them) and it is upper bounded by the number of
3179 /// instructions in the processed function (we can never save more
3180 /// elements in either set than this number). Hence, at some point,
3181 /// they will stop increasing.
3182 /// Consequently, at some point, both sets will have stopped
3183 /// changing, effectively making the analysis reach a fixpoint.
3184
3185 /// Note: These 2 sets are disjoint and an instruction can be considered
3186 /// one of 3 things:
3187 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3188 /// the KnownUBInsts set.
3189 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3190 /// has a reason to assume it).
3191 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3192 /// could not find a reason to assume or prove that it can cause UB,
3193 /// hence it assumes it doesn't. We have a set for these instructions
3194 /// so that we don't reprocess them in every update.
3195 /// Note however that instructions in this set may cause UB.
3196
3197protected:
3198 /// A map from all live instructions _known_ to cause UB to the reason why,
3199 /// used to build actionable optimization remarks in manifest().
3200 MapVector<Instruction *, UBInfo> KnownUBInsts;
3201
3202private:
3203 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3204 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3205
3206 // Should be called on updates in which if we're processing an instruction
3207 // \p I that depends on a value \p V, one of the following has to happen:
3208 // - If the value is assumed, then stop.
3209 // - If the value is known but undef, then consider it UB for \p K.
3210 // - Otherwise, do specific processing with the simplified value.
3211 // We return std::nullopt in the first 2 cases to signify that an appropriate
3212 // action was taken and the caller should stop.
3213 // Otherwise, we return the simplified value that the caller should
3214 // use for specific processing.
3215 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3216 Instruction *I, UBInfo::Kind K) {
3217 bool UsedAssumedInformation = false;
3218 std::optional<Value *> SimplifiedV =
3219 A.getAssumedSimplified(IRPosition::value(*V), *this,
3220 UsedAssumedInformation, AA::Interprocedural);
3221 if (!UsedAssumedInformation) {
3222 // Don't depend on assumed values.
3223 if (!SimplifiedV) {
3224 // If it is known (which we tested above) but it doesn't have a value,
3225 // then we can assume `undef` and hence the instruction is UB.
3226 KnownUBInsts.try_emplace(I, K);
3227 return std::nullopt;
3228 }
3229 if (!*SimplifiedV)
3230 return nullptr;
3231 V = *SimplifiedV;
3232 }
3233 if (isa<UndefValue>(V)) {
3234 KnownUBInsts.try_emplace(I, K);
3235 return std::nullopt;
3236 }
3237 return V;
3238 }
3239};
3240
3241struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3242 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3243 : AAUndefinedBehaviorImpl(IRP, A) {}
3244
3245 void trackStatistics() const override {
3246 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3247 "Number of instructions known to have UB");
3248 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3249 KnownUBInsts.size();
3250 }
3251};
3252} // namespace
3253
3254/// ------------------------ Will-Return Attributes ----------------------------
3255
3256namespace {
3257// Helper function that checks whether a function has any cycle which we don't
3258// know if it is bounded or not.
3259// Loops with maximum trip count are considered bounded, any other cycle not.
3260static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3261 ScalarEvolution *SE =
3262 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3263 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3264 // If either SCEV or LoopInfo is not available for the function then we assume
3265 // any cycle to be unbounded cycle.
3266 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3267 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3268 if (!SE || !LI) {
3269 for (scc_iterator<Function *> SCCI = scc_begin(&F); !SCCI.isAtEnd(); ++SCCI)
3270 if (SCCI.hasCycle())
3271 return true;
3272 return false;
3273 }
3274
3275 // If there's irreducible control, the function may contain non-loop cycles.
3277 return true;
3278
3279 // Any loop that does not have a max trip count is considered unbounded cycle.
3280 for (auto *L : LI->getLoopsInPreorder()) {
3281 if (!SE->getSmallConstantMaxTripCount(L))
3282 return true;
3283 }
3284 return false;
3285}
3286
3287struct AAWillReturnImpl : public AAWillReturn {
3288 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3289 : AAWillReturn(IRP, A) {}
3290
3291 void initialize(Attributor &A) override {
3292 bool IsKnown;
3294 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3295 (void)IsKnown;
3296 }
3297
3298 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3299 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3300 if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress}))
3301 return false;
3302
3303 bool IsKnown;
3304 if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
3305 return IsKnown || !KnownOnly;
3306 return false;
3307 }
3308
3309 ChangeStatus updateImpl(Attributor &A) override {
3310 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3311 return ChangeStatus::UNCHANGED;
3312
3313 auto CheckForWillReturn = [&](Instruction &I) {
3315 bool IsKnown;
3317 A, this, IPos, DepClassTy::REQUIRED, IsKnown)) {
3318 if (IsKnown)
3319 return true;
3320 } else {
3321 return false;
3322 }
3323 bool IsKnownNoRecurse;
3325 A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);
3326 };
3327
3328 bool UsedAssumedInformation = false;
3329 if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this,
3330 UsedAssumedInformation))
3331 return indicatePessimisticFixpoint();
3332
3333 auto CheckForVolatile = [&](Instruction &I) {
3334 // Volatile operations are not willreturn.
3335 return !I.isVolatile();
3336 };
3337 if (!A.checkForAllInstructions(CheckForVolatile, *this,
3338 {Instruction::Load, Instruction::Store,
3339 Instruction::AtomicCmpXchg,
3340 Instruction::AtomicRMW},
3341 UsedAssumedInformation))
3342 return indicatePessimisticFixpoint();
3343
3344 return ChangeStatus::UNCHANGED;
3345 }
3346
3347 const std::string getAsStr(Attributor *A) const override {
3348 return getAssumed() ? "willreturn" : "may-noreturn";
3349 }
3350};
3351
3352struct AAWillReturnFunction final : AAWillReturnImpl {
3353 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3354 : AAWillReturnImpl(IRP, A) {}
3355
3356 void initialize(Attributor &A) override {
3357 AAWillReturnImpl::initialize(A);
3358
3359 Function *F = getAnchorScope();
3360 assert(F && "Did expect an anchor function");
3361 if (F->isDeclaration() || mayContainUnboundedCycle(*F, A))
3362 indicatePessimisticFixpoint();
3363 }
3364
3365 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3366};
3367
3368/// WillReturn attribute deduction for a call sites.
3369struct AAWillReturnCallSite final
3370 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3371 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3372 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3373
3374 ChangeStatus updateImpl(Attributor &A) override {
3375 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3376 return ChangeStatus::UNCHANGED;
3377
3378 return AACalleeToCallSite::updateImpl(A);
3379 }
3380
3381 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3382};
3383} // namespace
3384
3385/// -------------------AAIntraFnReachability Attribute--------------------------
3386
3387/// All information associated with a reachability query. This boilerplate code
3388/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3389/// different \p ToTy values.
3390template <typename ToTy> struct ReachabilityQueryInfo {
3391 enum class Reachable {
3394 };
3395
3396 /// Start here,
3397 const Instruction *From = nullptr;
3398 /// reach this place,
3399 const ToTy *To = nullptr;
3400 /// without going through any of these instructions,
3402 /// and remember if it worked:
3404
3405 /// Precomputed hash for this RQI.
3406 unsigned Hash = 0;
3407
3408 unsigned computeHashValue() const {
3409 assert(Hash == 0 && "Computed hash twice!");
3412 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3413 detail::combineHashValue(PairDMI ::getHashValue({From, To}),
3414 InstSetDMI::getHashValue(ExclusionSet));
3415 }
3416
3418 : From(From), To(To) {}
3419
3420 /// Constructor replacement to ensure unique and stable sets are used for the
3421 /// cache.
3423 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3424 : From(&From), To(&To), ExclusionSet(ES) {
3425
3426 if (!ES || ES->empty()) {
3427 ExclusionSet = nullptr;
3428 } else if (MakeUnique) {
3429 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(ES);
3430 }
3431 }
3432
3435};
3436
3437namespace llvm {
3438template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3441
3442 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3443 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3444 }
3445 static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3446 const ReachabilityQueryInfo<ToTy> *RHS) {
3447 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3448 return false;
3449 return InstSetDMI::isEqual(LHS->ExclusionSet, RHS->ExclusionSet);
3450 }
3451};
3452
3453} // namespace llvm
3454
3455namespace {
3456
3457template <typename BaseTy, typename ToTy>
3458struct CachedReachabilityAA : public BaseTy {
3459 using RQITy = ReachabilityQueryInfo<ToTy>;
3460
3461 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3462
3463 bool isQueryAA() const override { return true; }
3464
3465 ChangeStatus updateImpl(Attributor &A) override {
3466 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3467 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3468 RQITy *RQI = QueryVector[u];
3469 if (RQI->Result == RQITy::Reachable::No &&
3470 isReachableImpl(A, *RQI, /*IsTemporaryRQI=*/false))
3471 Changed = ChangeStatus::CHANGED;
3472 }
3473 return Changed;
3474 }
3475
3476 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3477 bool IsTemporaryRQI) = 0;
3478
3479 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3480 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3481 RQI.Result = Result;
3482
3483 // Remove the temporary RQI from the cache.
3484 if (IsTemporaryRQI)
3485 QueryCache.erase(&RQI);
3486
3487 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3488 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3489 // this query. 2) We did not use the exclusion set, potentially because
3490 // there is none.
3491 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3492 RQITy PlainRQI(RQI.From, RQI.To);
3493 if (!QueryCache.count(&PlainRQI)) {
3494 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3495 RQIPtr->Result = Result;
3496 QueryVector.push_back(RQIPtr);
3497 QueryCache.insert(RQIPtr);
3498 }
3499 }
3500
3501 // Check if we need to insert a new permanent RQI with the exclusion set.
3502 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3503 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3504 "Did not expect empty set!");
3505 RQITy *RQIPtr = new (A.Allocator)
3506 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3507 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3508 RQIPtr->Result = Result;
3509 assert(!QueryCache.count(RQIPtr));
3510 QueryVector.push_back(RQIPtr);
3511 QueryCache.insert(RQIPtr);
3512 }
3513
3514 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3515 A.registerForUpdate(*this);
3516 return Result == RQITy::Reachable::Yes;
3517 }
3518
3519 const std::string getAsStr(Attributor *A) const override {
3520 // TODO: Return the number of reachable queries.
3521 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3522 }
3523
3524 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3525 typename RQITy::Reachable &Result) {
3526 if (!this->getState().isValidState()) {
3527 Result = RQITy::Reachable::Yes;
3528 return true;
3529 }
3530
3531 // If we have an exclusion set we might be able to find our answer by
3532 // ignoring it first.
3533 if (StackRQI.ExclusionSet) {
3534 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3535 auto It = QueryCache.find(&PlainRQI);
3536 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3537 Result = RQITy::Reachable::No;
3538 return true;
3539 }
3540 }
3541
3542 auto It = QueryCache.find(&StackRQI);
3543 if (It != QueryCache.end()) {
3544 Result = (*It)->Result;
3545 return true;
3546 }
3547
3548 // Insert a temporary for recursive queries. We will replace it with a
3549 // permanent entry later.
3550 QueryCache.insert(&StackRQI);
3551 return false;
3552 }
3553
3554private:
3555 SmallVector<RQITy *> QueryVector;
3556 DenseSet<RQITy *> QueryCache;
3557};
3558
3559struct AAIntraFnReachabilityFunction final
3560 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3561 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3562 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3563 : Base(IRP, A) {
3564 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3565 *IRP.getAssociatedFunction());
3566 }
3567
3568 bool isAssumedReachable(
3569 Attributor &A, const Instruction &From, const Instruction &To,
3570 const AA::InstExclusionSetTy *ExclusionSet) const override {
3571 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3572 if (&From == &To)
3573 return true;
3574
3575 RQITy StackRQI(A, From, To, ExclusionSet, false);
3576 RQITy::Reachable Result;
3577 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3578 return NonConstThis->isReachableImpl(A, StackRQI,
3579 /*IsTemporaryRQI=*/true);
3580 return Result == RQITy::Reachable::Yes;
3581 }
3582
3583 ChangeStatus updateImpl(Attributor &A) override {
3584 // We only depend on liveness. DeadEdges is all we care about, check if any
3585 // of them changed.
3586 auto *LivenessAA =
3587 A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
3588 if (LivenessAA &&
3589 llvm::all_of(DeadEdges,
3590 [&](const auto &DeadEdge) {
3591 return LivenessAA->isEdgeDead(DeadEdge.first,
3592 DeadEdge.second);
3593 }) &&
3594 llvm::all_of(DeadBlocks, [&](const BasicBlock *BB) {
3595 return LivenessAA->isAssumedDead(BB);
3596 })) {
3597 return ChangeStatus::UNCHANGED;
3598 }
3599 DeadEdges.clear();
3600 DeadBlocks.clear();
3601 return Base::updateImpl(A);
3602 }
3603
3604 bool isReachableImpl(Attributor &A, RQITy &RQI,
3605 bool IsTemporaryRQI) override {
3606 const Instruction *Origin = RQI.From;
3607 bool UsedExclusionSet = false;
3608
3609 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3610 const AA::InstExclusionSetTy *ExclusionSet) {
3611 const Instruction *IP = &From;
3612 while (IP && IP != &To) {
3613 if (ExclusionSet && IP != Origin && ExclusionSet->count(IP)) {
3614 UsedExclusionSet = true;
3615 break;
3616 }
3617 IP = IP->getNextNode();
3618 }
3619 return IP == &To;
3620 };
3621
3622 const BasicBlock *FromBB = RQI.From->getParent();
3623 const BasicBlock *ToBB = RQI.To->getParent();
3624 assert(FromBB->getParent() == ToBB->getParent() &&
3625 "Not an intra-procedural query!");
3626
3627 // Check intra-block reachability, however, other reaching paths are still
3628 // possible.
3629 if (FromBB == ToBB &&
3630 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3631 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3632 IsTemporaryRQI);
3633
3634 // Check if reaching the ToBB block is sufficient or if even that would not
3635 // ensure reaching the target. In the latter case we are done.
3636 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3637 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3638 IsTemporaryRQI);
3639
3640 const Function *Fn = FromBB->getParent();
3641 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3642 if (RQI.ExclusionSet)
3643 for (auto *I : *RQI.ExclusionSet)
3644 if (I->getFunction() == Fn)
3645 ExclusionBlocks.insert(I->getParent());
3646
3647 // Check if we make it out of the FromBB block at all.
3648 if (ExclusionBlocks.count(FromBB) &&
3649 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3650 RQI.ExclusionSet))
3651 return rememberResult(A, RQITy::Reachable::No, RQI, true, IsTemporaryRQI);
3652
3653 auto *LivenessAA =
3654 A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
3655 if (LivenessAA && LivenessAA->isAssumedDead(ToBB)) {
3656 DeadBlocks.insert(ToBB);
3657 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3658 IsTemporaryRQI);
3659 }
3660
3661 SmallPtrSet<const BasicBlock *, 16> Visited;
3663 Worklist.push_back(FromBB);
3664
3665 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3666 while (!Worklist.empty()) {
3667 const BasicBlock *BB = Worklist.pop_back_val();
3668 if (!Visited.insert(BB).second)
3669 continue;
3670 for (const BasicBlock *SuccBB : successors(BB)) {
3671 if (LivenessAA && LivenessAA->isEdgeDead(BB, SuccBB)) {
3672 LocalDeadEdges.insert({BB, SuccBB});
3673 continue;
3674 }
3675 // We checked before if we just need to reach the ToBB block.
3676 if (SuccBB == ToBB)
3677 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3678 IsTemporaryRQI);
3679 if (DT && ExclusionBlocks.empty() && DT->dominates(BB, ToBB))
3680 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3681 IsTemporaryRQI);
3682
3683 if (ExclusionBlocks.count(SuccBB)) {
3684 UsedExclusionSet = true;
3685 continue;
3686 }
3687 Worklist.push_back(SuccBB);
3688 }
3689 }
3690
3691 DeadEdges.insert_range(LocalDeadEdges);
3692 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3693 IsTemporaryRQI);
3694 }
3695
3696 void trackStatistics() const override {}
3697
3698private:
3699 // Set of assumed dead blocks we used in the last query. If any changes we
3700 // update the state.
3701 DenseSet<const BasicBlock *> DeadBlocks;
3702
3703 // Set of assumed dead edges we used in the last query. If any changes we
3704 // update the state.
3705 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3706
3707 /// The dominator tree of the function to short-circuit reasoning.
3708 const DominatorTree *DT = nullptr;
3709};
3710} // namespace
3711
3712/// ------------------------ NoAlias Argument Attribute ------------------------
3713
3715 Attribute::AttrKind ImpliedAttributeKind,
3716 bool IgnoreSubsumingPositions) {
3717 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3718 "Unexpected attribute kind");
3719 Value *Val = &IRP.getAssociatedValue();
3721 if (isa<AllocaInst>(Val))
3722 return true;
3723 } else {
3724 IgnoreSubsumingPositions = true;
3725 }
3726
3727 if (isa<UndefValue>(Val))
3728 return true;
3729
3730 if (isa<ConstantPointerNull>(Val) &&
3733 return true;
3734
3735 if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},
3736 IgnoreSubsumingPositions, Attribute::NoAlias))
3737 return true;
3738
3739 return false;
3740}
3741
3742namespace {
3743struct AANoAliasImpl : AANoAlias {
3744 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3745 assert(getAssociatedType()->isPointerTy() &&
3746 "Noalias is a pointer attribute");
3747 }
3748
3749 const std::string getAsStr(Attributor *A) const override {
3750 return getAssumed() ? "noalias" : "may-alias";
3751 }
3752};
3753
3754/// NoAlias attribute for a floating value.
3755struct AANoAliasFloating final : AANoAliasImpl {
3756 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3757 : AANoAliasImpl(IRP, A) {}
3758
3759 ChangeStatus updateImpl(Attributor &A) override {
3760 // TODO: Implement this.
3761 return indicatePessimisticFixpoint();
3762 }
3763
3764 void trackStatistics() const override {
3766 }
3767};
3768
3769/// NoAlias attribute for an argument.
3770struct AANoAliasArgument final
3771 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3772 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3773 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3774
3775 ChangeStatus updateImpl(Attributor &A) override {
3776 // We have to make sure no-alias on the argument does not break
3777 // synchronization when this is a callback argument, see also [1] below.
3778 // If synchronization cannot be affected, we delegate to the base updateImpl
3779 // function, otherwise we give up for now.
3780
3781 // If the function is no-sync, no-alias cannot break synchronization.
3782 bool IsKnownNoSycn;
3784 A, this, IRPosition::function_scope(getIRPosition()),
3785 DepClassTy::OPTIONAL, IsKnownNoSycn))
3786 return Base::updateImpl(A);
3787
3788 // If the argument is read-only, no-alias cannot break synchronization.
3789 bool IsKnown;
3790 if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
3791 return Base::updateImpl(A);
3792
3793 // If the argument is never passed through callbacks, no-alias cannot break
3794 // synchronization.
3795 bool UsedAssumedInformation = false;
3796 if (A.checkForAllCallSites(
3797 [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this,
3798 true, UsedAssumedInformation))
3799 return Base::updateImpl(A);
3800
3801 // TODO: add no-alias but make sure it doesn't break synchronization by
3802 // introducing fake uses. See:
3803 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3804 // International Workshop on OpenMP 2018,
3805 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3806
3807 return indicatePessimisticFixpoint();
3808 }
3809
3810 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3811};
3812
3813struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3814 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3815 : AANoAliasImpl(IRP, A) {}
3816
3817 /// Determine if the underlying value may alias with the call site argument
3818 /// \p OtherArgNo of \p ICS (= the underlying call site).
3819 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3820 const AAMemoryBehavior &MemBehaviorAA,
3821 const CallBase &CB, unsigned OtherArgNo) {
3822 // We do not need to worry about aliasing with the underlying IRP.
3823 if (this->getCallSiteArgNo() == (int)OtherArgNo)
3824 return false;
3825
3826 // If it is not a pointer or pointer vector we do not alias.
3827 const Value *ArgOp = CB.getArgOperand(OtherArgNo);
3828 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3829 return false;
3830
3831 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3832 *this, IRPosition::callsite_argument(CB, OtherArgNo), DepClassTy::NONE);
3833
3834 // If the argument is readnone, there is no read-write aliasing.
3835 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3836 A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
3837 return false;
3838 }
3839
3840 // If the argument is readonly and the underlying value is readonly, there
3841 // is no read-write aliasing.
3842 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3843 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3844 IsReadOnly) {
3845 A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL);
3846 A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
3847 return false;
3848 }
3849
3850 // We have to utilize actual alias analysis queries so we need the object.
3851 if (!AAR)
3852 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3853 *getAnchorScope());
3854
3855 // Try to rule it out at the call site.
3856 bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp);
3857 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3858 "callsite arguments: "
3859 << getAssociatedValue() << " " << *ArgOp << " => "
3860 << (IsAliasing ? "" : "no-") << "alias \n");
3861
3862 return IsAliasing;
3863 }
3864
3865 bool isKnownNoAliasDueToNoAliasPreservation(
3866 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3867 // We can deduce "noalias" if the following conditions hold.
3868 // (i) Associated value is assumed to be noalias in the definition.
3869 // (ii) Associated value is assumed to be no-capture in all the uses
3870 // possibly executed before this callsite.
3871 // (iii) There is no other pointer argument which could alias with the
3872 // value.
3873
3874 const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
3875 const Function *ScopeFn = VIRP.getAnchorScope();
3876 // Check whether the value is captured in the scope using AANoCapture.
3877 // Look at CFG and check only uses possibly executed before this
3878 // callsite.
3879 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3880 Instruction *UserI = cast<Instruction>(U.getUser());
3881
3882 // If UserI is the curr instruction and there is a single potential use of
3883 // the value in UserI we allow the use.
3884 // TODO: We should inspect the operands and allow those that cannot alias
3885 // with the value.
3886 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3887 return true;
3888
3889 if (ScopeFn) {
3890 if (auto *CB = dyn_cast<CallBase>(UserI)) {
3891 if (CB->isArgOperand(&U)) {
3892
3893 unsigned ArgNo = CB->getArgOperandNo(&U);
3894
3895 bool IsKnownNoCapture;
3897 A, this, IRPosition::callsite_argument(*CB, ArgNo),
3898 DepClassTy::OPTIONAL, IsKnownNoCapture))
3899 return true;
3900 }
3901 }
3902
3904 A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr,
3905 [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3906 return true;
3907 }
3908
3909 // TODO: We should track the capturing uses in AANoCapture but the problem
3910 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3911 // a value in the module slice.
3912 // TODO(captures): Make this more precise.
3913 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
3914 if (capturesNothing(CI))
3915 return true;
3916 if (CI.isPassthrough()) {
3917 Follow = true;
3918 return true;
3919 }
3920 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");
3921 return false;
3922 };
3923
3924 bool IsKnownNoCapture;
3925 const AANoCapture *NoCaptureAA = nullptr;
3926 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
3927 A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);
3928 if (!IsAssumedNoCapture &&
3929 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
3930 if (!A.checkForAllUses(UsePred, *this, getAssociatedValue())) {
3931 LLVM_DEBUG(
3932 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
3933 << " cannot be noalias as it is potentially captured\n");
3934 return false;
3935 }
3936 }
3937 if (NoCaptureAA)
3938 A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL);
3939
3940 // Check there is no other pointer argument which could alias with the
3941 // value passed at this call site.
3942 // TODO: AbstractCallSite
3943 const auto &CB = cast<CallBase>(getAnchorValue());
3944 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
3945 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
3946 return false;
3947
3948 return true;
3949 }
3950
3951 ChangeStatus updateImpl(Attributor &A) override {
3952 // If the argument is readnone we are done as there are no accesses via the
3953 // argument.
3954 auto *MemBehaviorAA =
3955 A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
3956 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
3957 A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
3958 return ChangeStatus::UNCHANGED;
3959 }
3960
3961 bool IsKnownNoAlias;
3962 const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
3964 A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
3965 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
3966 << " is not no-alias at the definition\n");
3967 return indicatePessimisticFixpoint();
3968 }
3969
3970 AAResults *AAR = nullptr;
3971 if (MemBehaviorAA &&
3972 isKnownNoAliasDueToNoAliasPreservation(A, AAR, *MemBehaviorAA)) {
3973 LLVM_DEBUG(
3974 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
3975 return ChangeStatus::UNCHANGED;
3976 }
3977
3978 return indicatePessimisticFixpoint();
3979 }
3980
3981 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
3982};
3983
3984/// NoAlias attribute for function return value.
3985struct AANoAliasReturned final : AANoAliasImpl {
3986 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
3987 : AANoAliasImpl(IRP, A) {}
3988
3989 ChangeStatus updateImpl(Attributor &A) override {
3990
3991 auto CheckReturnValue = [&](Value &RV) -> bool {
3992 if (Constant *C = dyn_cast<Constant>(&RV))
3993 if (C->isNullValue() || isa<UndefValue>(C))
3994 return true;
3995
3996 /// For now, we can only deduce noalias if we have call sites.
3997 /// FIXME: add more support.
3998 if (!isa<CallBase>(&RV))
3999 return false;
4000
4001 const IRPosition &RVPos = IRPosition::value(RV);
4002 bool IsKnownNoAlias;
4004 A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))
4005 return false;
4006
4007 bool IsKnownNoCapture;
4008 const AANoCapture *NoCaptureAA = nullptr;
4009 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4010 A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
4011 &NoCaptureAA);
4012 return IsAssumedNoCapture ||
4013 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4014 };
4015
4016 if (!A.checkForAllReturnedValues(CheckReturnValue, *this))
4017 return indicatePessimisticFixpoint();
4018
4019 return ChangeStatus::UNCHANGED;
4020 }
4021
4022 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4023};
4024
4025/// NoAlias attribute deduction for a call site return value.
4026struct AANoAliasCallSiteReturned final
4027 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4028 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4029 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4030
4031 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4032};
4033} // namespace
4034
4035/// -------------------AAIsDead Function Attribute-----------------------
4036
4037namespace {
4038struct AAIsDeadValueImpl : public AAIsDead {
4039 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4040
4041 bool isAssumedDead() const override { return isAssumed(IS_DEAD); }
4042
4043 bool isKnownDead() const override { return isKnown(IS_DEAD); }
4044
4045 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4046
4047 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4048
4049 bool isAssumedDead(const Instruction *I) const override {
4050 return I == getCtxI() && isAssumedDead();
4051 }
4052
4053 bool isKnownDead(const Instruction *I) const override {
4054 return isAssumedDead(I) && isKnownDead();
4055 }
4056
4057 const std::string getAsStr(Attributor *A) const override {
4058 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4059 }
4060
4061 /// Check if all uses are assumed dead.
4062 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4063 // Callers might not check the type, void has no uses.
4064 if (V.getType()->isVoidTy() || V.use_empty())
4065 return true;
4066
4067 // If we replace a value with a constant there are no uses left afterwards.
4068 if (!isa<Constant>(V)) {
4069 if (auto *I = dyn_cast<Instruction>(&V))
4070 if (!A.isRunOn(*I->getFunction()))
4071 return false;
4072 bool UsedAssumedInformation = false;
4073 std::optional<Constant *> C =
4074 A.getAssumedConstant(V, *this, UsedAssumedInformation);
4075 if (!C || *C)
4076 return true;
4077 }
4078
4079 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4080 // Explicitly set the dependence class to required because we want a long
4081 // chain of N dependent instructions to be considered live as soon as one is
4082 // without going through N update cycles. This is not required for
4083 // correctness.
4084 return A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ false,
4085 DepClassTy::REQUIRED,
4086 /* IgnoreDroppableUses */ false);
4087 }
4088
4089 /// Determine if \p I is assumed to be side-effect free.
4090 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4092 return true;
4093
4094 if (!I->isTerminator() && !I->mayHaveSideEffects())
4095 return true;
4096
4097 auto *CB = dyn_cast<CallBase>(I);
4098 if (!CB || isa<IntrinsicInst>(CB))
4099 return false;
4100
4101 const IRPosition &CallIRP = IRPosition::callsite_function(*CB);
4102
4103 bool IsKnownNoUnwind;
4105 A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))
4106 return false;
4107
4108 bool IsKnown;
4109 return AA::isAssumedReadOnly(A, CallIRP, *this, IsKnown);
4110 }
4111};
4112
4113struct AAIsDeadFloating : public AAIsDeadValueImpl {
4114 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4115 : AAIsDeadValueImpl(IRP, A) {}
4116
4117 void initialize(Attributor &A) override {
4118 AAIsDeadValueImpl::initialize(A);
4119
4120 if (isa<UndefValue>(getAssociatedValue())) {
4121 indicatePessimisticFixpoint();
4122 return;
4123 }
4124
4125 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4126 if (!isAssumedSideEffectFree(A, I)) {
4128 indicatePessimisticFixpoint();
4129 else
4130 removeAssumedBits(HAS_NO_EFFECT);
4131 }
4132 }
4133
4134 bool isDeadFence(Attributor &A, FenceInst &FI) {
4135 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4136 IRPosition::function(*FI.getFunction()), *this, DepClassTy::NONE);
4137 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4138 return false;
4139 A.recordDependence(*ExecDomainAA, *this, DepClassTy::OPTIONAL);
4140 return true;
4141 }
4142
4143 bool isDeadStore(Attributor &A, StoreInst &SI,
4144 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4145 // Lang ref now states volatile store is not UB/dead, let's skip them.
4146 if (SI.isVolatile())
4147 return false;
4148
4149 // If we are collecting assumes to be deleted we are in the manifest stage.
4150 // It's problematic to collect the potential copies again now so we use the
4151 // cached ones.
4152 bool UsedAssumedInformation = false;
4153 if (!AssumeOnlyInst) {
4154 PotentialCopies.clear();
4155 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, *this,
4156 UsedAssumedInformation)) {
4157 LLVM_DEBUG(
4158 dbgs()
4159 << "[AAIsDead] Could not determine potential copies of store!\n");
4160 return false;
4161 }
4162 }
4163 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4164 << " potential copies.\n");
4165
4166 InformationCache &InfoCache = A.getInfoCache();
4167 return llvm::all_of(PotentialCopies, [&](Value *V) {
4168 if (A.isAssumedDead(IRPosition::value(*V), this, nullptr,
4169 UsedAssumedInformation))
4170 return true;
4171 if (auto *LI = dyn_cast<LoadInst>(V)) {
4172 if (llvm::all_of(LI->uses(), [&](const Use &U) {
4173 auto &UserI = cast<Instruction>(*U.getUser());
4174 if (InfoCache.isOnlyUsedByAssume(UserI)) {
4175 if (AssumeOnlyInst)
4176 AssumeOnlyInst->insert(&UserI);
4177 return true;
4178 }
4179 return A.isAssumedDead(U, this, nullptr, UsedAssumedInformation);
4180 })) {
4181 return true;
4182 }
4183 }
4184 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4185 << " is assumed live!\n");
4186 return false;
4187 });
4188 }
4189
4190 const std::string getAsStr(Attributor *A) const override {
4191 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4193 if (isValidState())
4194 return "assumed-dead-store";
4196 if (isValidState())
4197 return "assumed-dead-fence";
4198 return AAIsDeadValueImpl::getAsStr(A);
4199 }
4200
4201 ChangeStatus updateImpl(Attributor &A) override {
4202 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4203 if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {
4204 if (!isDeadStore(A, *SI))
4205 return indicatePessimisticFixpoint();
4206 } else if (auto *FI = dyn_cast_or_null<FenceInst>(I)) {
4207 if (!isDeadFence(A, *FI))
4208 return indicatePessimisticFixpoint();
4209 } else {
4210 if (!isAssumedSideEffectFree(A, I))
4211 return indicatePessimisticFixpoint();
4212 if (!areAllUsesAssumedDead(A, getAssociatedValue()))
4213 return indicatePessimisticFixpoint();
4214 }
4216 }
4217
4218 bool isRemovableStore() const override {
4219 return isAssumed(IS_REMOVABLE) && isa<StoreInst>(&getAssociatedValue());
4220 }
4221
4222 ChangeStatus manifest(Attributor &A) override {
4223 Value &V = getAssociatedValue();
4224 if (auto *I = dyn_cast<Instruction>(&V)) {
4225 // If we get here we basically know the users are all dead. We check if
4226 // isAssumedSideEffectFree returns true here again because it might not be
4227 // the case and only the users are dead but the instruction (=call) is
4228 // still needed.
4229 if (auto *SI = dyn_cast<StoreInst>(I)) {
4230 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4231 bool IsDead = isDeadStore(A, *SI, &AssumeOnlyInst);
4232 (void)IsDead;
4233 assert(IsDead && "Store was assumed to be dead!");
4234 A.deleteAfterManifest(*I);
4235 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4236 Instruction *AOI = AssumeOnlyInst[i];
4237 for (auto *Usr : AOI->users())
4238 AssumeOnlyInst.insert(cast<Instruction>(Usr));
4239 A.deleteAfterManifest(*AOI);
4240 }
4241 return ChangeStatus::CHANGED;
4242 }
4243 if (auto *FI = dyn_cast<FenceInst>(I)) {
4244 assert(isDeadFence(A, *FI));
4245 A.deleteAfterManifest(*FI);
4246 return ChangeStatus::CHANGED;
4247 }
4248 if (isAssumedSideEffectFree(A, I) && !I->isTerminator()) {
4249 A.deleteAfterManifest(*I);
4250 return ChangeStatus::CHANGED;
4251 }
4252 }
4254 }
4255
4256 void trackStatistics() const override {
4258 }
4259
4260private:
4261 // The potential copies of a dead store, used for deletion during manifest.
4262 SmallSetVector<Value *, 4> PotentialCopies;
4263};
4264
4265struct AAIsDeadArgument : public AAIsDeadFloating {
4266 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4267 : AAIsDeadFloating(IRP, A) {}
4268
4269 ChangeStatus manifest(Attributor &A) override {
4270 Argument &Arg = *getAssociatedArgument();
4271 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4272 if (A.registerFunctionSignatureRewrite(
4273 Arg, /* ReplacementTypes */ {},
4276 return ChangeStatus::CHANGED;
4277 }
4278 return ChangeStatus::UNCHANGED;
4279 }
4280
4281 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4282};
4283
4284struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4285 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4286 : AAIsDeadValueImpl(IRP, A) {}
4287
4288 void initialize(Attributor &A) override {
4289 AAIsDeadValueImpl::initialize(A);
4290 if (isa<UndefValue>(getAssociatedValue()))
4291 indicatePessimisticFixpoint();
4292 }
4293
4294 ChangeStatus updateImpl(Attributor &A) override {
4295 // TODO: Once we have call site specific value information we can provide
4296 // call site specific liveness information and then it makes
4297 // sense to specialize attributes for call sites arguments instead of
4298 // redirecting requests to the callee argument.
4299 Argument *Arg = getAssociatedArgument();
4300 if (!Arg)
4301 return indicatePessimisticFixpoint();
4302 const IRPosition &ArgPos = IRPosition::argument(*Arg);
4303 auto *ArgAA = A.getAAFor<AAIsDead>(*this, ArgPos, DepClassTy::REQUIRED);
4304 if (!ArgAA)
4305 return indicatePessimisticFixpoint();
4306 return clampStateAndIndicateChange(getState(), ArgAA->getState());
4307 }
4308
4309 ChangeStatus manifest(Attributor &A) override {
4310 CallBase &CB = cast<CallBase>(getAnchorValue());
4311 Use &U = CB.getArgOperandUse(getCallSiteArgNo());
4312 assert(!isa<UndefValue>(U.get()) &&
4313 "Expected undef values to be filtered out!");
4314 UndefValue &UV = *UndefValue::get(U->getType());
4315 if (A.changeUseAfterManifest(U, UV))
4316 return ChangeStatus::CHANGED;
4317 return ChangeStatus::UNCHANGED;
4318 }
4319
4320 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4321};
4322
4323struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4324 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4325 : AAIsDeadFloating(IRP, A) {}
4326
4327 bool isAssumedDead() const override {
4328 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4329 }
4330
4331 void initialize(Attributor &A) override {
4332 AAIsDeadFloating::initialize(A);
4333 if (isa<UndefValue>(getAssociatedValue())) {
4334 indicatePessimisticFixpoint();
4335 return;
4336 }
4337
4338 // We track this separately as a secondary state.
4339 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, getCtxI());
4340 }
4341
4342 ChangeStatus updateImpl(Attributor &A) override {
4343 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4344 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, getCtxI())) {
4345 IsAssumedSideEffectFree = false;
4346 Changed = ChangeStatus::CHANGED;
4347 }
4348 if (!areAllUsesAssumedDead(A, getAssociatedValue()))
4349 return indicatePessimisticFixpoint();
4350 return Changed;
4351 }
4352
4353 void trackStatistics() const override {
4354 if (IsAssumedSideEffectFree)
4356 else
4357 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4358 }
4359
4360 const std::string getAsStr(Attributor *A) const override {
4361 return isAssumedDead()
4362 ? "assumed-dead"
4363 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4364 }
4365
4366private:
4367 bool IsAssumedSideEffectFree = true;
4368};
4369
4370struct AAIsDeadReturned : public AAIsDeadValueImpl {
4371 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4372 : AAIsDeadValueImpl(IRP, A) {}
4373
4374 ChangeStatus updateImpl(Attributor &A) override {
4375
4376 bool UsedAssumedInformation = false;
4377 A.checkForAllInstructions([](Instruction &) { return true; }, *this,
4378 {Instruction::Ret}, UsedAssumedInformation);
4379
4380 auto PredForCallSite = [&](AbstractCallSite ACS) {
4381 if (ACS.isCallbackCall() || !ACS.getInstruction())
4382 return false;
4383 return areAllUsesAssumedDead(A, *ACS.getInstruction());
4384 };
4385
4386 if (!A.checkForAllCallSites(PredForCallSite, *this, true,
4387 UsedAssumedInformation))
4388 return indicatePessimisticFixpoint();
4389
4390 return ChangeStatus::UNCHANGED;
4391 }
4392
4393 ChangeStatus manifest(Attributor &A) override {
4394 // TODO: Rewrite the signature to return void?
4395 bool AnyChange = false;
4396 UndefValue &UV = *UndefValue::get(getAssociatedFunction()->getReturnType());
4397 auto RetInstPred = [&](Instruction &I) {
4398 ReturnInst &RI = cast<ReturnInst>(I);
4400 AnyChange |= A.changeUseAfterManifest(RI.getOperandUse(0), UV);
4401 return true;
4402 };
4403 bool UsedAssumedInformation = false;
4404 A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
4405 UsedAssumedInformation);
4406 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4407 }
4408
4409 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4410};
4411
4412struct AAIsDeadFunction : public AAIsDead {
4413 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4414
4415 void initialize(Attributor &A) override {
4416 Function *F = getAnchorScope();
4417 assert(F && "Did expect an anchor function");
4418 if (!isAssumedDeadInternalFunction(A)) {
4419 ToBeExploredFrom.insert(&F->getEntryBlock().front());
4420 assumeLive(A, F->getEntryBlock());
4421 }
4422 }
4423
4424 bool isAssumedDeadInternalFunction(Attributor &A) {
4425 if (!getAnchorScope()->hasLocalLinkage())
4426 return false;
4427 // An AA that will not be updated is fixed pessimistically right after
4428 // initialization, so an assumed-dead result would be discarded. Checking
4429 // the call sites recursively initializes the liveness of every internal
4430 // caller, which in CGSCC cleanup repeats for each callee SCC.
4431 if (!A.shouldUpdateAA<AAIsDead>(getIRPosition()))
4432 return false;
4433 bool UsedAssumedInformation = false;
4434 return A.checkForAllCallSites([](AbstractCallSite) { return false; }, *this,
4435 true, UsedAssumedInformation);
4436 }
4437
4438 const std::string getAsStr(Attributor *A) const override {
4439 return "Live[#BB " + std::to_string(AssumedLiveBlocks.size()) + "/" +
4440 std::to_string(getAnchorScope()->size()) + "][#TBEP " +
4441 std::to_string(ToBeExploredFrom.size()) + "][#KDE " +
4442 std::to_string(KnownDeadEnds.size()) + "]";
4443 }
4444
4445 ChangeStatus manifest(Attributor &A) override {
4446 assert(getState().isValidState() &&
4447 "Attempted to manifest an invalid state!");
4448
4449 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4450 Function &F = *getAnchorScope();
4451
4452 if (AssumedLiveBlocks.empty()) {
4453 A.deleteAfterManifest(F);
4454 return ChangeStatus::CHANGED;
4455 }
4456
4457 // Flag to determine if we can change an invoke to a call assuming the
4458 // callee is nounwind. This is not possible if the personality of the
4459 // function allows to catch asynchronous exceptions.
4460 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4461
4462 KnownDeadEnds.set_union(ToBeExploredFrom);
4463 for (const Instruction *DeadEndI : KnownDeadEnds) {
4464 auto *CB = dyn_cast<CallBase>(DeadEndI);
4465 if (!CB)
4466 continue;
4467 bool IsKnownNoReturn;
4469 A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL,
4470 IsKnownNoReturn);
4471 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(CB)))
4472 continue;
4473
4474 if (auto *II = dyn_cast<InvokeInst>(DeadEndI))
4475 A.registerInvokeWithDeadSuccessor(const_cast<InvokeInst &>(*II));
4476 else
4477 A.changeToUnreachableAfterManifest(
4478 const_cast<Instruction *>(DeadEndI->getNextNode()));
4479 HasChanged = ChangeStatus::CHANGED;
4480 }
4481
4482 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4483 for (BasicBlock &BB : F)
4484 if (!AssumedLiveBlocks.count(&BB)) {
4485 A.deleteAfterManifest(BB);
4486 ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4487 HasChanged = ChangeStatus::CHANGED;
4488 }
4489
4490 return HasChanged;
4491 }
4492
4493 ChangeStatus updateImpl(Attributor &A) override;
4494
4495 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4496 assert(From->getParent() == getAnchorScope() &&
4497 To->getParent() == getAnchorScope() &&
4498 "Used AAIsDead of the wrong function");
4499 return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To));
4500 }
4501
4502 void trackStatistics() const override {}
4503
4504 /// Returns true if the function is assumed dead.
4505 bool isAssumedDead() const override { return false; }
4506
4507 bool isKnownDead() const override { return false; }
4508
4509 bool isAssumedDead(const BasicBlock *BB) const override {
4510 assert(BB->getParent() == getAnchorScope() &&
4511 "BB must be in the same anchor scope function.");
4512
4513 if (!getAssumed())
4514 return false;
4515 return !AssumedLiveBlocks.count(BB);
4516 }
4517
4518 bool isKnownDead(const BasicBlock *BB) const override {
4519 return getKnown() && isAssumedDead(BB);
4520 }
4521
4522 bool isAssumedDead(const Instruction *I) const override {
4523 assert(I->getParent()->getParent() == getAnchorScope() &&
4524 "Instruction must be in the same anchor scope function.");
4525
4526 if (!getAssumed())
4527 return false;
4528
4529 // If it is not in AssumedLiveBlocks then it for sure dead.
4530 // Otherwise, it can still be after noreturn call in a live block.
4531 if (!AssumedLiveBlocks.count(I->getParent()))
4532 return true;
4533
4534 // If it is not after a liveness barrier it is live.
4535 const Instruction *PrevI = I->getPrevNode();
4536 while (PrevI) {
4537 if (KnownDeadEnds.count(PrevI) || ToBeExploredFrom.count(PrevI))
4538 return true;
4539 PrevI = PrevI->getPrevNode();
4540 }
4541 return false;
4542 }
4543
4544 bool isKnownDead(const Instruction *I) const override {
4545 return getKnown() && isAssumedDead(I);
4546 }
4547
4548 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4549 /// that internal function called from \p BB should now be looked at.
4550 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4551 if (!AssumedLiveBlocks.insert(&BB).second)
4552 return false;
4553
4554 if (!A.isDuringDeduction())
4555 return true;
4556
4557 // We assume that all of BB is (probably) live now and if there are calls to
4558 // internal functions we will assume that those are now live as well. This
4559 // is a performance optimization for blocks with calls to a lot of internal
4560 // functions. It can however cause dead functions to be treated as live.
4561 for (const Instruction &I : BB)
4562 if (const auto *CB = dyn_cast<CallBase>(&I))
4564 if (F->hasLocalLinkage()) {
4565 LLVM_DEBUG({
4566 dbgs() << "[AAIsDead] Seeding live internal callee ";
4567 F->printAsOperand(dbgs(), /*PrintType=*/false);
4568 dbgs() << " from ";
4569 BB.getParent()->printAsOperand(dbgs(), /*PrintType=*/false);
4570 dbgs() << "\n";
4571 });
4572 A.markLiveInternalFunction(*F);
4573 }
4574 return true;
4575 }
4576
4577 /// Collection of instructions that need to be explored again, e.g., we
4578 /// did assume they do not transfer control to (one of their) successors.
4579 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4580
4581 /// Collection of instructions that are known to not transfer control.
4582 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4583
4584 /// Collection of all assumed live edges
4585 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4586
4587 /// Collection of all assumed live BasicBlocks.
4588 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4589};
4590
4591static bool
4592identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4593 AbstractAttribute &AA,
4594 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4595 const IRPosition &IPos = IRPosition::callsite_function(CB);
4596
4597 bool IsKnownNoReturn;
4599 A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn))
4600 return !IsKnownNoReturn;
4601 if (CB.isTerminator())
4602 AliveSuccessors.push_back(&CB.getSuccessor(0)->front());
4603 else
4604 AliveSuccessors.push_back(CB.getNextNode());
4605 return false;
4606}
4607
4608static bool
4609identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4610 AbstractAttribute &AA,
4611 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4612 bool UsedAssumedInformation =
4613 identifyAliveSuccessors(A, cast<CallBase>(II), AA, AliveSuccessors);
4614
4615 // First, determine if we can change an invoke to a call assuming the
4616 // callee is nounwind. This is not possible if the personality of the
4617 // function allows to catch asynchronous exceptions.
4618 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(*II.getFunction())) {
4619 AliveSuccessors.push_back(&II.getUnwindDest()->front());
4620 } else {
4621 const IRPosition &IPos = IRPosition::callsite_function(II);
4622
4623 bool IsKnownNoUnwind;
4625 A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
4626 UsedAssumedInformation |= !IsKnownNoUnwind;
4627 } else {
4628 AliveSuccessors.push_back(&II.getUnwindDest()->front());
4629 }
4630 }
4631 return UsedAssumedInformation;
4632}
4633
4634static bool
4635identifyAliveSuccessors(Attributor &, const UncondBrInst &BI,
4636 AbstractAttribute &,
4637 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4638 AliveSuccessors.push_back(&BI.getSuccessor()->front());
4639 return false;
4640}
4641
4642static bool
4643identifyAliveSuccessors(Attributor &A, const CondBrInst &BI,
4644 AbstractAttribute &AA,
4645 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4646 bool UsedAssumedInformation = false;
4647 std::optional<Constant *> C =
4648 A.getAssumedConstant(*BI.getCondition(), AA, UsedAssumedInformation);
4649 if (!C || isa_and_nonnull<UndefValue>(*C)) {
4650 // No value yet, assume both edges are dead.
4651 } else if (isa_and_nonnull<ConstantInt>(*C)) {
4652 const BasicBlock *SuccBB =
4653 BI.getSuccessor(1 - cast<ConstantInt>(*C)->getValue().getZExtValue());
4654 AliveSuccessors.push_back(&SuccBB->front());
4655 } else {
4656 AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
4657 AliveSuccessors.push_back(&BI.getSuccessor(1)->front());
4658 UsedAssumedInformation = false;
4659 }
4660 return UsedAssumedInformation;
4661}
4662
4663static bool
4664identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4665 AbstractAttribute &AA,
4666 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4667 bool UsedAssumedInformation = false;
4669 if (!A.getAssumedSimplifiedValues(IRPosition::value(*SI.getCondition()), &AA,
4671 UsedAssumedInformation)) {
4672 // Something went wrong, assume all successors are live.
4673 for (const BasicBlock *SuccBB : successors(SI.getParent()))
4674 AliveSuccessors.push_back(&SuccBB->front());
4675 return false;
4676 }
4677
4678 if (Values.empty() ||
4679 (Values.size() == 1 &&
4680 isa_and_nonnull<UndefValue>(Values.front().getValue()))) {
4681 // No valid value yet, assume all edges are dead.
4682 return UsedAssumedInformation;
4683 }
4684
4685 Type &Ty = *SI.getCondition()->getType();
4686 SmallPtrSet<ConstantInt *, 8> Constants;
4687 auto CheckForConstantInt = [&](Value *V) {
4688 if (auto *CI = dyn_cast_if_present<ConstantInt>(AA::getWithType(*V, Ty))) {
4689 Constants.insert(CI);
4690 return true;
4691 }
4692 return false;
4693 };
4694
4695 if (!all_of(Values, [&](AA::ValueAndContext &VAC) {
4696 return CheckForConstantInt(VAC.getValue());
4697 })) {
4698 for (const BasicBlock *SuccBB : successors(SI.getParent()))
4699 AliveSuccessors.push_back(&SuccBB->front());
4700 return UsedAssumedInformation;
4701 }
4702
4703 unsigned MatchedCases = 0;
4704 for (const auto &CaseIt : SI.cases()) {
4705 if (Constants.count(CaseIt.getCaseValue())) {
4706 ++MatchedCases;
4707 AliveSuccessors.push_back(&CaseIt.getCaseSuccessor()->front());
4708 }
4709 }
4710
4711 // If all potential values have been matched, we will not visit the default
4712 // case.
4713 if (MatchedCases < Constants.size())
4714 AliveSuccessors.push_back(&SI.getDefaultDest()->front());
4715 return UsedAssumedInformation;
4716}
4717
4718ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4720
4721 if (AssumedLiveBlocks.empty()) {
4722 if (isAssumedDeadInternalFunction(A))
4724
4725 Function *F = getAnchorScope();
4726 ToBeExploredFrom.insert(&F->getEntryBlock().front());
4727 assumeLive(A, F->getEntryBlock());
4728 Change = ChangeStatus::CHANGED;
4729 }
4730
4731 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4732 << getAnchorScope()->size() << "] BBs and "
4733 << ToBeExploredFrom.size() << " exploration points and "
4734 << KnownDeadEnds.size() << " known dead ends\n");
4735
4736 // Copy and clear the list of instructions we need to explore from. It is
4737 // refilled with instructions the next update has to look at.
4738 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4739 ToBeExploredFrom.end());
4740 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4741
4743 while (!Worklist.empty()) {
4744 const Instruction *I = Worklist.pop_back_val();
4745 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4746
4747 // Fast forward for uninteresting instructions. We could look for UB here
4748 // though.
4749 while (!I->isTerminator() && !isa<CallBase>(I))
4750 I = I->getNextNode();
4751
4752 AliveSuccessors.clear();
4753
4754 bool UsedAssumedInformation = false;
4755 switch (I->getOpcode()) {
4756 // TODO: look for (assumed) UB to backwards propagate "deadness".
4757 default:
4758 assert(I->isTerminator() &&
4759 "Expected non-terminators to be handled already!");
4760 for (const BasicBlock *SuccBB : successors(I->getParent()))
4761 AliveSuccessors.push_back(&SuccBB->front());
4762 break;
4763 case Instruction::Call:
4764 UsedAssumedInformation = identifyAliveSuccessors(A, cast<CallInst>(*I),
4765 *this, AliveSuccessors);
4766 break;
4767 case Instruction::Invoke:
4768 UsedAssumedInformation = identifyAliveSuccessors(A, cast<InvokeInst>(*I),
4769 *this, AliveSuccessors);
4770 break;
4771 case Instruction::UncondBr:
4772 UsedAssumedInformation = identifyAliveSuccessors(
4773 A, cast<UncondBrInst>(*I), *this, AliveSuccessors);
4774 break;
4775 case Instruction::CondBr:
4776 UsedAssumedInformation = identifyAliveSuccessors(A, cast<CondBrInst>(*I),
4777 *this, AliveSuccessors);
4778 break;
4779 case Instruction::Switch:
4780 UsedAssumedInformation = identifyAliveSuccessors(A, cast<SwitchInst>(*I),
4781 *this, AliveSuccessors);
4782 break;
4783 }
4784
4785 if (UsedAssumedInformation) {
4786 NewToBeExploredFrom.insert(I);
4787 } else if (AliveSuccessors.empty() ||
4788 (I->isTerminator() &&
4789 AliveSuccessors.size() < I->getNumSuccessors())) {
4790 if (KnownDeadEnds.insert(I))
4791 Change = ChangeStatus::CHANGED;
4792 }
4793
4794 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4795 << AliveSuccessors.size() << " UsedAssumedInformation: "
4796 << UsedAssumedInformation << "\n");
4797
4798 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4799 if (!I->isTerminator()) {
4800 assert(AliveSuccessors.size() == 1 &&
4801 "Non-terminator expected to have a single successor!");
4802 Worklist.push_back(AliveSuccessor);
4803 } else {
4804 // record the assumed live edge
4805 auto Edge = std::make_pair(I->getParent(), AliveSuccessor->getParent());
4806 if (AssumedLiveEdges.insert(Edge).second)
4807 Change = ChangeStatus::CHANGED;
4808 if (assumeLive(A, *AliveSuccessor->getParent()))
4809 Worklist.push_back(AliveSuccessor);
4810 }
4811 }
4812 }
4813
4814 // Check if the content of ToBeExploredFrom changed, ignore the order.
4815 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4816 llvm::any_of(NewToBeExploredFrom, [&](const Instruction *I) {
4817 return !ToBeExploredFrom.count(I);
4818 })) {
4819 Change = ChangeStatus::CHANGED;
4820 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4821 }
4822
4823 // If we know everything is live there is no need to query for liveness.
4824 // Instead, indicating a pessimistic fixpoint will cause the state to be
4825 // "invalid" and all queries to be answered conservatively without lookups.
4826 // To be in this state we have to (1) finished the exploration and (3) not
4827 // discovered any non-trivial dead end and (2) not ruled unreachable code
4828 // dead.
4829 if (ToBeExploredFrom.empty() &&
4830 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4831 llvm::all_of(KnownDeadEnds, [](const Instruction *DeadEndI) {
4832 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4833 }))
4834 return indicatePessimisticFixpoint();
4835 return Change;
4836}
4837
4838/// Liveness information for a call sites.
4839struct AAIsDeadCallSite final : AAIsDeadFunction {
4840 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4841 : AAIsDeadFunction(IRP, A) {}
4842
4843 void initialize(Attributor &A) override {
4844 // TODO: Once we have call site specific value information we can provide
4845 // call site specific liveness information and then it makes
4846 // sense to specialize attributes for call sites instead of
4847 // redirecting requests to the callee.
4848 llvm_unreachable("Abstract attributes for liveness are not "
4849 "supported for call sites yet!");
4850 }
4851
4852 ChangeStatus updateImpl(Attributor &A) override {
4853 return indicatePessimisticFixpoint();
4854 }
4855
4856 void trackStatistics() const override {}
4857};
4858} // namespace
4859
4860/// -------------------- Dereferenceable Argument Attribute --------------------
4861
4862namespace {
4863struct AADereferenceableImpl : AADereferenceable {
4864 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4865 : AADereferenceable(IRP, A) {}
4866 using StateType = DerefState;
4867
4868 void initialize(Attributor &A) override {
4869 Value &V = *getAssociatedValue().stripPointerCasts();
4871 A.getAttrs(getIRPosition(),
4872 {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4873 Attrs, /* IgnoreSubsumingPositions */ false);
4874 for (const Attribute &Attr : Attrs)
4875 takeKnownDerefBytesMaximum(Attr.getValueAsInt());
4876
4877 // Ensure we initialize the non-null AA (if necessary).
4878 bool IsKnownNonNull;
4880 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);
4881
4882 bool CanBeNull;
4883 takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes(
4884 A.getDataLayout(), CanBeNull, /*CanBeFreed=*/nullptr));
4885
4886 if (Instruction *CtxI = getCtxI())
4887 followUsesInMBEC(*this, A, getState(), *CtxI);
4888 }
4889
4890 StateType &getState() override { return *this; }
4891 const StateType &getState() const override { return *this; }
4892
4893 /// Helper function for collecting accessed bytes in must-be-executed-context
4894 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
4895 DerefState &State) {
4896 const Value *UseV = U->get();
4897 if (!UseV->getType()->isPointerTy())
4898 return;
4899
4900 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
4901 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
4902 return;
4903
4904 int64_t Offset;
4906 Loc->Ptr, Offset, A.getDataLayout(), /*AllowNonInbounds*/ true);
4907 if (Base && Base == &getAssociatedValue())
4908 State.addAccessedBytes(Offset, Loc->Size.getValue());
4909 }
4910
4911 /// See followUsesInMBEC
4912 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
4913 AADereferenceable::StateType &State) {
4914 bool IsNonNull = false;
4915 bool TrackUse = false;
4916 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4917 A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse);
4918 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
4919 << " for instruction " << *I << "\n");
4920
4921 addAccessedBytesForUse(A, U, I, State);
4922 State.takeKnownDerefBytesMaximum(DerefBytes);
4923 return TrackUse;
4924 }
4925
4926 ChangeStatus manifest(Attributor &A) override {
4927 ChangeStatus Change = AADereferenceable::manifest(A);
4928 bool IsKnownNonNull;
4929 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4930 A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4931 if (IsAssumedNonNull &&
4932 A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {
4933 A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});
4934 return ChangeStatus::CHANGED;
4935 }
4936 return Change;
4937 }
4938
4939 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
4940 SmallVectorImpl<Attribute> &Attrs) const override {
4941 // TODO: Add *_globally support
4942 bool IsKnownNonNull;
4943 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4944 A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4945 if (IsAssumedNonNull)
4946 Attrs.emplace_back(Attribute::getWithDereferenceableBytes(
4947 Ctx, getAssumedDereferenceableBytes()));
4948 else
4949 Attrs.emplace_back(Attribute::getWithDereferenceableOrNullBytes(
4950 Ctx, getAssumedDereferenceableBytes()));
4951 }
4952
4953 const std::string getAsStr(Attributor *A) const override {
4954 if (!getAssumedDereferenceableBytes())
4955 return "unknown-dereferenceable";
4956 bool IsKnownNonNull;
4957 bool IsAssumedNonNull = false;
4958 if (A)
4960 *A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4961 return std::string("dereferenceable") +
4962 (IsAssumedNonNull ? "" : "_or_null") +
4963 (isAssumedGlobal() ? "_globally" : "") + "<" +
4964 std::to_string(getKnownDereferenceableBytes()) + "-" +
4965 std::to_string(getAssumedDereferenceableBytes()) + ">" +
4966 (!A ? " [non-null is unknown]" : "");
4967 }
4968};
4969
4970/// Dereferenceable attribute for a floating value.
4971struct AADereferenceableFloating : AADereferenceableImpl {
4972 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
4973 : AADereferenceableImpl(IRP, A) {}
4974
4975 ChangeStatus updateImpl(Attributor &A) override {
4976 bool Stripped;
4977 bool UsedAssumedInformation = false;
4979 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
4980 AA::AnyScope, UsedAssumedInformation)) {
4981 Values.push_back({getAssociatedValue(), getCtxI()});
4982 Stripped = false;
4983 } else {
4984 Stripped = Values.size() != 1 ||
4985 Values.front().getValue() != &getAssociatedValue();
4986 }
4987
4988 const DataLayout &DL = A.getDataLayout();
4989 DerefState T;
4990
4991 auto VisitValueCB = [&](const Value &V) -> bool {
4992 unsigned IdxWidth =
4993 DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace());
4994 APInt Offset(IdxWidth, 0);
4996 A, *this, &V, DL, Offset, /* GetMinOffset */ false,
4997 /* AllowNonInbounds */ true);
4998
4999 const auto *AA = A.getAAFor<AADereferenceable>(
5000 *this, IRPosition::value(*Base), DepClassTy::REQUIRED);
5001 int64_t DerefBytes = 0;
5002 if (!AA || (!Stripped && this == AA)) {
5003 // Use IR information if we did not strip anything.
5004 // TODO: track globally.
5005 bool CanBeNull;
5006 DerefBytes = Base->getPointerDereferenceableBytes(
5007 DL, CanBeNull, /*CanBeFreed=*/nullptr);
5008 T.GlobalState.indicatePessimisticFixpoint();
5009 } else {
5010 const DerefState &DS = AA->getState();
5011 DerefBytes = DS.DerefBytesState.getAssumed();
5012 T.GlobalState &= DS.GlobalState;
5013 }
5014
5015 // For now we do not try to "increase" dereferenceability due to negative
5016 // indices as we first have to come up with code to deal with loops and
5017 // for overflows of the dereferenceable bytes.
5018 int64_t OffsetSExt = Offset.getSExtValue();
5019 if (OffsetSExt < 0)
5020 OffsetSExt = 0;
5021
5022 T.takeAssumedDerefBytesMinimum(
5023 std::max(int64_t(0), DerefBytes - OffsetSExt));
5024
5025 if (this == AA) {
5026 if (!Stripped) {
5027 // If nothing was stripped IR information is all we got.
5028 T.takeKnownDerefBytesMaximum(
5029 std::max(int64_t(0), DerefBytes - OffsetSExt));
5030 T.indicatePessimisticFixpoint();
5031 } else if (OffsetSExt > 0) {
5032 // If something was stripped but there is circular reasoning we look
5033 // for the offset. If it is positive we basically decrease the
5034 // dereferenceable bytes in a circular loop now, which will simply
5035 // drive them down to the known value in a very slow way which we
5036 // can accelerate.
5037 T.indicatePessimisticFixpoint();
5038 }
5039 }
5040
5041 return T.isValidState();
5042 };
5043
5044 for (const auto &VAC : Values)
5045 if (!VisitValueCB(*VAC.getValue()))
5046 return indicatePessimisticFixpoint();
5047
5048 return clampStateAndIndicateChange(getState(), T);
5049 }
5050
5051 void trackStatistics() const override {
5052 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5053 }
5054};
5055
5056/// Dereferenceable attribute for a return value.
5057struct AADereferenceableReturned final
5058 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5059 using Base =
5060 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5061 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5062 : Base(IRP, A) {}
5063
5064 void trackStatistics() const override {
5065 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5066 }
5067};
5068
5069/// Dereferenceable attribute for an argument
5070struct AADereferenceableArgument final
5071 : AAArgumentFromCallSiteArguments<AADereferenceable,
5072 AADereferenceableImpl> {
5073 using Base =
5074 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5075 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5076 : Base(IRP, A) {}
5077
5078 void trackStatistics() const override {
5079 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5080 }
5081};
5082
5083/// Dereferenceable attribute for a call site argument.
5084struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5085 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5086 : AADereferenceableFloating(IRP, A) {}
5087
5088 void trackStatistics() const override {
5089 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5090 }
5091};
5092
5093/// Dereferenceable attribute deduction for a call site return value.
5094struct AADereferenceableCallSiteReturned final
5095 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5096 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5097 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5098 : Base(IRP, A) {}
5099
5100 void trackStatistics() const override {
5101 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5102 }
5103};
5104} // namespace
5105
5106// ------------------------ Align Argument Attribute ------------------------
5107
5108namespace {
5109
5110static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5111 Value &AssociatedValue, const Use *U,
5112 const Instruction *I, bool &TrackUse) {
5113 // We need to follow common pointer manipulation uses to the accesses they
5114 // feed into.
5115 if (isa<CastInst>(I)) {
5116 // Follow all but ptr2int casts.
5117 TrackUse = !isa<PtrToIntInst>(I);
5118 return 0;
5119 }
5120 if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
5121 if (GEP->hasAllConstantIndices())
5122 TrackUse = true;
5123 return 0;
5124 }
5125 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
5126 switch (II->getIntrinsicID()) {
5127 case Intrinsic::ptrmask: {
5128 // Is it appropriate to pull attribute in initialization?
5129 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5130 QueryingAA, IRPosition::value(*II->getOperand(1)), DepClassTy::NONE);
5131 const auto *AlignAA = A.getAAFor<AAAlign>(
5132 QueryingAA, IRPosition::value(*II), DepClassTy::NONE);
5133 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5134 unsigned ShiftValue = std::min(ConstVals->getAssumedMinTrailingZeros(),
5136 Align ConstAlign = Align::fromLog2(ShiftValue);
5137 if (ConstAlign >= AlignAA->getKnownAlign())
5138 return Align(1).value();
5139 }
5140 if (AlignAA)
5141 return AlignAA->getKnownAlign().value();
5142 break;
5143 }
5144 case Intrinsic::amdgcn_make_buffer_rsrc: {
5145 const auto *AlignAA = A.getAAFor<AAAlign>(
5146 QueryingAA, IRPosition::value(*II), DepClassTy::NONE);
5147 if (AlignAA)
5148 return AlignAA->getKnownAlign().value();
5149 break;
5150 }
5151 default:
5152 break;
5153 }
5154
5155 MaybeAlign MA;
5156 if (const auto *CB = dyn_cast<CallBase>(I)) {
5157 if (CB->isBundleOperand(U) || CB->isCallee(U))
5158 return 0;
5159
5160 unsigned ArgNo = CB->getArgOperandNo(U);
5161 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
5162 // As long as we only use known information there is no need to track
5163 // dependences here.
5164 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE);
5165 if (AlignAA)
5166 MA = MaybeAlign(AlignAA->getKnownAlign());
5167 }
5168
5169 const DataLayout &DL = A.getDataLayout();
5170 const Value *UseV = U->get();
5171 if (auto *SI = dyn_cast<StoreInst>(I)) {
5172 if (SI->getPointerOperand() == UseV)
5173 MA = SI->getAlign();
5174 } else if (auto *LI = dyn_cast<LoadInst>(I)) {
5175 if (LI->getPointerOperand() == UseV)
5176 MA = LI->getAlign();
5177 } else if (auto *AI = dyn_cast<AtomicRMWInst>(I)) {
5178 if (AI->getPointerOperand() == UseV)
5179 MA = AI->getAlign();
5180 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
5181 if (AI->getPointerOperand() == UseV)
5182 MA = AI->getAlign();
5183 }
5184
5185 if (!MA || *MA <= QueryingAA.getKnownAlign())
5186 return 0;
5187
5188 unsigned Alignment = MA->value();
5189 int64_t Offset;
5190
5191 if (const Value *Base = GetPointerBaseWithConstantOffset(UseV, Offset, DL)) {
5192 if (Base == &AssociatedValue) {
5193 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5194 // So we can say that the maximum power of two which is a divisor of
5195 // gcd(Offset, Alignment) is an alignment.
5196
5197 uint32_t gcd = std::gcd(uint32_t(abs((int32_t)Offset)), Alignment);
5199 }
5200 }
5201
5202 return Alignment;
5203}
5204
5205struct AAAlignImpl : AAAlign {
5206 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5207
5208 void initialize(Attributor &A) override {
5210 A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs);
5211 for (const Attribute &Attr : Attrs)
5212 takeKnownMaximum(Attr.getValueAsInt());
5213
5214 Value &V = *getAssociatedValue().stripPointerCasts();
5215 takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value());
5216
5217 if (Instruction *CtxI = getCtxI())
5218 followUsesInMBEC(*this, A, getState(), *CtxI);
5219 }
5220
5221 ChangeStatus manifest(Attributor &A) override {
5222 ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;
5223
5224 // Check for users that allow alignment annotations.
5225 Value &AssociatedValue = getAssociatedValue();
5226 if (isa<ConstantData>(AssociatedValue))
5227 return ChangeStatus::UNCHANGED;
5228
5229 for (const Use &U : AssociatedValue.uses()) {
5230 if (auto *SI = dyn_cast<StoreInst>(U.getUser())) {
5231 if (SI->getPointerOperand() == &AssociatedValue)
5232 if (SI->getAlign() < getAssumedAlign()) {
5233 STATS_DECLTRACK(AAAlign, Store,
5234 "Number of times alignment added to a store");
5235 SI->setAlignment(getAssumedAlign());
5236 InstrChanged = ChangeStatus::CHANGED;
5237 }
5238 } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {
5239 if (LI->getPointerOperand() == &AssociatedValue)
5240 if (LI->getAlign() < getAssumedAlign()) {
5241 LI->setAlignment(getAssumedAlign());
5242 STATS_DECLTRACK(AAAlign, Load,
5243 "Number of times alignment added to a load");
5244 InstrChanged = ChangeStatus::CHANGED;
5245 }
5246 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(U.getUser())) {
5247 if (RMW->getPointerOperand() == &AssociatedValue) {
5248 if (RMW->getAlign() < getAssumedAlign()) {
5249 STATS_DECLTRACK(AAAlign, AtomicRMW,
5250 "Number of times alignment added to atomicrmw");
5251
5252 RMW->setAlignment(getAssumedAlign());
5253 InstrChanged = ChangeStatus::CHANGED;
5254 }
5255 }
5256 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(U.getUser())) {
5257 if (CAS->getPointerOperand() == &AssociatedValue) {
5258 if (CAS->getAlign() < getAssumedAlign()) {
5259 STATS_DECLTRACK(AAAlign, AtomicCmpXchg,
5260 "Number of times alignment added to cmpxchg");
5261 CAS->setAlignment(getAssumedAlign());
5262 InstrChanged = ChangeStatus::CHANGED;
5263 }
5264 }
5265 }
5266 }
5267
5268 ChangeStatus Changed = AAAlign::manifest(A);
5269
5270 Align InheritAlign =
5271 getAssociatedValue().getPointerAlignment(A.getDataLayout());
5272 if (InheritAlign >= getAssumedAlign())
5273 return InstrChanged;
5274 return Changed | InstrChanged;
5275 }
5276
5277 // TODO: Provide a helper to determine the implied ABI alignment and check in
5278 // the existing manifest method and a new one for AAAlignImpl that value
5279 // to avoid making the alignment explicit if it did not improve.
5280
5281 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5282 SmallVectorImpl<Attribute> &Attrs) const override {
5283 if (getAssumedAlign() > 1)
5284 Attrs.emplace_back(
5285 Attribute::getWithAlignment(Ctx, Align(getAssumedAlign())));
5286 }
5287
5288 /// See followUsesInMBEC
5289 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5290 AAAlign::StateType &State) {
5291 bool TrackUse = false;
5292
5293 unsigned int KnownAlign =
5294 getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse);
5295 State.takeKnownMaximum(KnownAlign);
5296
5297 return TrackUse;
5298 }
5299
5300 const std::string getAsStr(Attributor *A) const override {
5301 return "align<" + std::to_string(getKnownAlign().value()) + "-" +
5302 std::to_string(getAssumedAlign().value()) + ">";
5303 }
5304};
5305
5306/// Align attribute for a floating value.
5307struct AAAlignFloating : AAAlignImpl {
5308 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5309
5310 ChangeStatus updateImpl(Attributor &A) override {
5311 const DataLayout &DL = A.getDataLayout();
5312
5313 bool Stripped;
5314 bool UsedAssumedInformation = false;
5316 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
5317 AA::AnyScope, UsedAssumedInformation)) {
5318 Values.push_back({getAssociatedValue(), getCtxI()});
5319 Stripped = false;
5320 } else {
5321 Stripped = Values.size() != 1 ||
5322 Values.front().getValue() != &getAssociatedValue();
5323 }
5324
5325 StateType T;
5326 auto VisitValueCB = [&](Value &V) -> bool {
5328 return true;
5329 const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V),
5330 DepClassTy::REQUIRED);
5331 if (!AA || (!Stripped && this == AA)) {
5332 int64_t Offset;
5333 unsigned Alignment = 1;
5334 if (const Value *Base =
5336 // TODO: Use AAAlign for the base too.
5337 Align PA = Base->getPointerAlignment(DL);
5338 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5339 // So we can say that the maximum power of two which is a divisor of
5340 // gcd(Offset, Alignment) is an alignment.
5341
5342 uint32_t gcd =
5343 std::gcd(uint32_t(abs((int32_t)Offset)), uint32_t(PA.value()));
5345 } else {
5346 Alignment = V.getPointerAlignment(DL).value();
5347 }
5348 // Use only IR information if we did not strip anything.
5349 T.takeKnownMaximum(Alignment);
5350 T.indicatePessimisticFixpoint();
5351 } else {
5352 // Use abstract attribute information.
5353 const AAAlign::StateType &DS = AA->getState();
5354 T ^= DS;
5355 }
5356 return T.isValidState();
5357 };
5358
5359 for (const auto &VAC : Values) {
5360 if (!VisitValueCB(*VAC.getValue()))
5361 return indicatePessimisticFixpoint();
5362 }
5363
5364 // TODO: If we know we visited all incoming values, thus no are assumed
5365 // dead, we can take the known information from the state T.
5366 return clampStateAndIndicateChange(getState(), T);
5367 }
5368
5369 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5370};
5371
5372/// Align attribute for function return value.
5373struct AAAlignReturned final
5374 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5375 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5376 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5377
5378 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5379};
5380
5381/// Align attribute for function argument.
5382struct AAAlignArgument final
5383 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5384 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5385 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5386
5387 ChangeStatus manifest(Attributor &A) override {
5388 // If the associated argument is involved in a must-tail call we give up
5389 // because we would need to keep the argument alignments of caller and
5390 // callee in-sync. Just does not seem worth the trouble right now.
5391 if (A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))
5392 return ChangeStatus::UNCHANGED;
5393 return Base::manifest(A);
5394 }
5395
5396 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5397};
5398
5399struct AAAlignCallSiteArgument final : AAAlignFloating {
5400 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5401 : AAAlignFloating(IRP, A) {}
5402
5403 ChangeStatus manifest(Attributor &A) override {
5404 // If the associated argument is involved in a must-tail call we give up
5405 // because we would need to keep the argument alignments of caller and
5406 // callee in-sync. Just does not seem worth the trouble right now.
5407 if (Argument *Arg = getAssociatedArgument())
5408 if (A.getInfoCache().isInvolvedInMustTailCall(*Arg))
5409 return ChangeStatus::UNCHANGED;
5410 ChangeStatus Changed = AAAlignImpl::manifest(A);
5411 Align InheritAlign =
5412 getAssociatedValue().getPointerAlignment(A.getDataLayout());
5413 if (InheritAlign >= getAssumedAlign())
5414 Changed = ChangeStatus::UNCHANGED;
5415 return Changed;
5416 }
5417
5418 ChangeStatus updateImpl(Attributor &A) override {
5419 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5420 if (Argument *Arg = getAssociatedArgument()) {
5421 // We only take known information from the argument
5422 // so we do not need to track a dependence.
5423 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5424 *this, IRPosition::argument(*Arg), DepClassTy::NONE);
5425 if (ArgAlignAA)
5426 takeKnownMaximum(ArgAlignAA->getKnownAlign().value());
5427 }
5428 return Changed;
5429 }
5430
5431 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5432};
5433
5434/// Align attribute deduction for a call site return value.
5435struct AAAlignCallSiteReturned final
5436 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5437 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5438 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5439 : Base(IRP, A) {}
5440
5441 ChangeStatus updateImpl(Attributor &A) override {
5442 Instruction *I = getIRPosition().getCtxI();
5443 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
5444 switch (II->getIntrinsicID()) {
5445 case Intrinsic::ptrmask: {
5447 bool Valid = false;
5448
5449 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5450 *this, IRPosition::value(*II->getOperand(1)), DepClassTy::REQUIRED);
5451 if (ConstVals && ConstVals->isValidState()) {
5452 unsigned ShiftValue =
5453 std::min(ConstVals->getAssumedMinTrailingZeros(),
5454 Value::MaxAlignmentExponent);
5455 Alignment = Align::fromLog2(ShiftValue);
5456 Valid = true;
5457 }
5458
5459 const auto *AlignAA =
5460 A.getAAFor<AAAlign>(*this, IRPosition::value(*(II->getOperand(0))),
5461 DepClassTy::REQUIRED);
5462 if (AlignAA) {
5463 Alignment = std::max(AlignAA->getAssumedAlign(), Alignment);
5464 Valid = true;
5465 }
5466
5467 if (Valid)
5469 this->getState(),
5470 std::min(this->getAssumedAlign(), Alignment).value());
5471 break;
5472 }
5473 // FIXME: Should introduce target specific sub-attributes and letting
5474 // getAAfor<AAAlign> lead to create sub-attribute to handle target
5475 // specific intrinsics.
5476 case Intrinsic::amdgcn_make_buffer_rsrc: {
5477 const auto *AlignAA =
5478 A.getAAFor<AAAlign>(*this, IRPosition::value(*(II->getOperand(0))),
5479 DepClassTy::REQUIRED);
5480 if (AlignAA)
5482 this->getState(), AlignAA->getAssumedAlign().value());
5483 break;
5484 }
5485 default:
5486 break;
5487 }
5488 }
5489 return Base::updateImpl(A);
5490 };
5491 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5492};
5493} // namespace
5494
5495/// ------------------ Function No-Return Attribute ----------------------------
5496namespace {
5497struct AANoReturnImpl : public AANoReturn {
5498 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5499
5500 void initialize(Attributor &A) override {
5501 bool IsKnown;
5503 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5504 (void)IsKnown;
5505 }
5506
5507 const std::string getAsStr(Attributor *A) const override {
5508 return getAssumed() ? "noreturn" : "may-return";
5509 }
5510
5511 ChangeStatus updateImpl(Attributor &A) override {
5512 auto CheckForNoReturn = [](Instruction &) { return false; };
5513 bool UsedAssumedInformation = false;
5514 if (!A.checkForAllInstructions(CheckForNoReturn, *this,
5515 {(unsigned)Instruction::Ret},
5516 UsedAssumedInformation))
5517 return indicatePessimisticFixpoint();
5518 return ChangeStatus::UNCHANGED;
5519 }
5520};
5521
5522struct AANoReturnFunction final : AANoReturnImpl {
5523 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5524 : AANoReturnImpl(IRP, A) {}
5525
5526 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5527};
5528
5529/// NoReturn attribute deduction for a call sites.
5530struct AANoReturnCallSite final
5531 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5532 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5533 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5534
5535 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5536};
5537} // namespace
5538
5539/// ----------------------- Instance Info ---------------------------------
5540
5541namespace {
5542/// A class to hold the state of for no-capture attributes.
5543struct AAInstanceInfoImpl : public AAInstanceInfo {
5544 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5545 : AAInstanceInfo(IRP, A) {}
5546
5547 void initialize(Attributor &A) override {
5548 Value &V = getAssociatedValue();
5549 if (auto *C = dyn_cast<Constant>(&V)) {
5550 if (C->isThreadDependent())
5551 indicatePessimisticFixpoint();
5552 else
5553 indicateOptimisticFixpoint();
5554 return;
5555 }
5556 if (auto *CB = dyn_cast<CallBase>(&V))
5557 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5558 !CB->mayReadFromMemory()) {
5559 indicateOptimisticFixpoint();
5560 return;
5561 }
5562 if (auto *I = dyn_cast<Instruction>(&V)) {
5563 const auto *CI =
5564 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5565 *I->getFunction());
5566 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5567 indicatePessimisticFixpoint();
5568 return;
5569 }
5570 }
5571 }
5572
5573 ChangeStatus updateImpl(Attributor &A) override {
5574 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5575
5576 Value &V = getAssociatedValue();
5577 const Function *Scope = nullptr;
5578 if (auto *I = dyn_cast<Instruction>(&V))
5579 Scope = I->getFunction();
5580 if (auto *A = dyn_cast<Argument>(&V)) {
5581 Scope = A->getParent();
5582 if (!Scope->hasLocalLinkage())
5583 return Changed;
5584 }
5585 if (!Scope)
5586 return indicateOptimisticFixpoint();
5587
5588 bool IsKnownNoRecurse;
5590 A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL,
5591 IsKnownNoRecurse))
5592 return Changed;
5593
5594 auto UsePred = [&](const Use &U, bool &Follow) {
5595 const Instruction *UserI = dyn_cast<Instruction>(U.getUser());
5596 if (!UserI || isa<GetElementPtrInst>(UserI) || isa<CastInst>(UserI) ||
5597 isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
5598 Follow = true;
5599 return true;
5600 }
5601 if (isa<LoadInst>(UserI) || isa<CmpInst>(UserI) ||
5602 (isa<StoreInst>(UserI) &&
5603 cast<StoreInst>(UserI)->getValueOperand() != U.get()))
5604 return true;
5605 if (auto *CB = dyn_cast<CallBase>(UserI)) {
5606 // This check is not guaranteeing uniqueness but for now that we cannot
5607 // end up with two versions of \p U thinking it was one.
5609 if (!Callee || !Callee->hasLocalLinkage())
5610 return true;
5611 if (!CB->isArgOperand(&U))
5612 return false;
5613 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5615 DepClassTy::OPTIONAL);
5616 if (!ArgInstanceInfoAA ||
5617 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5618 return false;
5619 // If this call base might reach the scope again we might forward the
5620 // argument back here. This is very conservative.
5622 A, *CB, *Scope, *this, /* ExclusionSet */ nullptr,
5623 [Scope](const Function &Fn) { return &Fn != Scope; }))
5624 return false;
5625 return true;
5626 }
5627 return false;
5628 };
5629
5630 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5631 if (auto *SI = dyn_cast<StoreInst>(OldU.getUser())) {
5632 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5633 if ((isa<AllocaInst>(Ptr) || isNoAliasCall(Ptr)) &&
5634 AA::isDynamicallyUnique(A, *this, *Ptr))
5635 return true;
5636 }
5637 return false;
5638 };
5639
5640 if (!A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ true,
5641 DepClassTy::OPTIONAL,
5642 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5643 return indicatePessimisticFixpoint();
5644
5645 return Changed;
5646 }
5647
5648 const std::string getAsStr(Attributor *A) const override {
5649 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5650 }
5651
5652 void trackStatistics() const override {}
5653};
5654
5655/// InstanceInfo attribute for floating values.
5656struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5657 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5658 : AAInstanceInfoImpl(IRP, A) {}
5659};
5660
5661/// NoCapture attribute for function arguments.
5662struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5663 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5664 : AAInstanceInfoFloating(IRP, A) {}
5665};
5666
5667/// InstanceInfo attribute for call site arguments.
5668struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5669 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5670 : AAInstanceInfoImpl(IRP, A) {}
5671
5672 ChangeStatus updateImpl(Attributor &A) override {
5673 // TODO: Once we have call site specific value information we can provide
5674 // call site specific liveness information and then it makes
5675 // sense to specialize attributes for call sites arguments instead of
5676 // redirecting requests to the callee argument.
5677 Argument *Arg = getAssociatedArgument();
5678 if (!Arg)
5679 return indicatePessimisticFixpoint();
5680 const IRPosition &ArgPos = IRPosition::argument(*Arg);
5681 auto *ArgAA =
5682 A.getAAFor<AAInstanceInfo>(*this, ArgPos, DepClassTy::REQUIRED);
5683 if (!ArgAA)
5684 return indicatePessimisticFixpoint();
5685 return clampStateAndIndicateChange(getState(), ArgAA->getState());
5686 }
5687};
5688
5689/// InstanceInfo attribute for function return value.
5690struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5691 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5692 : AAInstanceInfoImpl(IRP, A) {
5693 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5694 }
5695
5696 void initialize(Attributor &A) override {
5697 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5698 }
5699
5700 ChangeStatus updateImpl(Attributor &A) override {
5701 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5702 }
5703};
5704
5705/// InstanceInfo attribute deduction for a call site return value.
5706struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5707 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5708 : AAInstanceInfoFloating(IRP, A) {}
5709};
5710} // namespace
5711
5712/// ----------------------- Variable Capturing ---------------------------------
5714 Attribute::AttrKind ImpliedAttributeKind,
5715 bool IgnoreSubsumingPositions) {
5716 assert(ImpliedAttributeKind == Attribute::Captures &&
5717 "Unexpected attribute kind");
5718 Value &V = IRP.getAssociatedValue();
5719 if (!isa<Constant>(V) && !IRP.isArgumentPosition())
5720 return V.use_empty();
5721
5722 // You cannot "capture" null if it is not a valid pointer.
5723 if (isa<UndefValue>(V) ||
5727 return true;
5728 }
5729
5731 A.getAttrs(IRP, {Attribute::Captures}, Attrs,
5732 /* IgnoreSubsumingPositions */ true);
5733 for (const Attribute &Attr : Attrs)
5734 if (capturesNothing(Attr.getCaptureInfo()))
5735 return true;
5736
5738 if (Argument *Arg = IRP.getAssociatedArgument()) {
5740 A.getAttrs(IRPosition::argument(*Arg),
5741 {Attribute::Captures, Attribute::ByVal}, Attrs,
5742 /* IgnoreSubsumingPositions */ true);
5743 bool ArgNoCapture = any_of(Attrs, [](Attribute Attr) {
5744 return Attr.getKindAsEnum() == Attribute::ByVal ||
5746 });
5747 if (ArgNoCapture) {
5748 A.manifestAttrs(IRP, Attribute::getWithCaptureInfo(
5749 V.getContext(), CaptureInfo::none()));
5750 return true;
5751 }
5752 }
5753
5754 if (const Function *F = IRP.getAssociatedFunction()) {
5755 // Check what state the associated function can actually capture.
5758 if (State.isKnown(NO_CAPTURE)) {
5759 A.manifestAttrs(IRP, Attribute::getWithCaptureInfo(V.getContext(),
5761 return true;
5762 }
5763 }
5764
5765 return false;
5766}
5767
5768/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5769/// depending on the ability of the function associated with \p IRP to capture
5770/// state in memory and through "returning/throwing", respectively.
5772 const Function &F,
5773 BitIntegerState &State) {
5774 // TODO: Once we have memory behavior attributes we should use them here.
5775
5776 // If we know we cannot communicate or write to memory, we do not care about
5777 // ptr2int anymore. The value can still be captured by whether the function
5778 // returns.
5779 bool ReadOnly = F.onlyReadsMemory();
5780 bool NoThrow = F.doesNotThrow();
5781 bool WillReturn = F.willReturn();
5782 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5783 if (ReadOnly && NoThrow && WillReturn && IsVoidReturn) {
5784 State.addKnownBits(NO_CAPTURE);
5785 return;
5786 }
5787
5788 // A function cannot capture state in memory if it only reads memory, it can
5789 // however return/throw state and the state might be influenced by the
5790 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5791 if (ReadOnly)
5792 State.addKnownBits(NOT_CAPTURED_IN_MEM);
5793
5794 // A function cannot communicate state back if it does not through
5795 // exceptions and doesn not return values.
5796 if (NoThrow && IsVoidReturn)
5797 State.addKnownBits(NOT_CAPTURED_IN_RET);
5798
5799 // Check existing "returned" attributes.
5800 int ArgNo = IRP.getCalleeArgNo();
5801 if (!NoThrow || ArgNo < 0 ||
5802 !F.getAttributes().hasAttrSomewhere(Attribute::Returned))
5803 return;
5804
5805 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5806 if (F.hasParamAttribute(U, Attribute::Returned)) {
5807 if (U == unsigned(ArgNo))
5808 State.removeAssumedBits(NOT_CAPTURED_IN_RET);
5809 else if (ReadOnly && WillReturn)
5810 State.addKnownBits(NO_CAPTURE);
5811 else
5812 State.addKnownBits(NOT_CAPTURED_IN_RET);
5813 break;
5814 }
5815}
5816
5817namespace {
5818/// A class to hold the state of for no-capture attributes.
5819struct AANoCaptureImpl : public AANoCapture {
5820 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5821
5822 void initialize(Attributor &A) override {
5823 bool IsKnown;
5825 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5826 (void)IsKnown;
5827 }
5828
5829 ChangeStatus updateImpl(Attributor &A) override;
5830
5831 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5832 SmallVectorImpl<Attribute> &Attrs) const override {
5833 if (!isAssumedNoCaptureMaybeReturned())
5834 return;
5835
5836 if (isArgumentPosition()) {
5837 if (isAssumedNoCapture())
5838 Attrs.emplace_back(Attribute::get(Ctx, Attribute::Captures));
5839 else if (A.Opts.attributor_manifest_internal)
5840 Attrs.emplace_back(Attribute::get(Ctx, "no-capture-maybe-returned"));
5841 }
5842 }
5843
5844 const std::string getAsStr(Attributor *A) const override {
5845 if (isKnownNoCapture())
5846 return "known not-captured";
5847 if (isAssumedNoCapture())
5848 return "assumed not-captured";
5849 if (isKnownNoCaptureMaybeReturned())
5850 return "known not-captured-maybe-returned";
5851 if (isAssumedNoCaptureMaybeReturned())
5852 return "assumed not-captured-maybe-returned";
5853 return "assumed-captured";
5854 }
5855
5856 /// Check the use \p U and update \p State accordingly. Return true if we
5857 /// should continue to update the state.
5858 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
5859 bool &Follow) {
5860 Instruction *UInst = cast<Instruction>(U.getUser());
5861 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
5862 << *UInst << "\n");
5863
5864 // Deal with ptr2int by following uses.
5865 if (isa<PtrToIntInst>(UInst)) {
5866 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
5867 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5868 /* Return */ true);
5869 }
5870
5871 // For stores we already checked if we can follow them, if they make it
5872 // here we give up.
5873 if (isa<StoreInst>(UInst))
5874 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5875 /* Return */ true);
5876
5877 // Explicitly catch return instructions.
5878 if (isa<ReturnInst>(UInst)) {
5879 if (UInst->getFunction() == getAnchorScope())
5880 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5881 /* Return */ true);
5882 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5883 /* Return */ true);
5884 }
5885
5886 // For now we only use special logic for call sites. However, the tracker
5887 // itself knows about a lot of other non-capturing cases already.
5888 auto *CB = dyn_cast<CallBase>(UInst);
5889 if (!CB || !CB->isArgOperand(&U))
5890 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5891 /* Return */ true);
5892
5893 unsigned ArgNo = CB->getArgOperandNo(&U);
5894 const IRPosition &CSArgPos = IRPosition::callsite_argument(*CB, ArgNo);
5895 // If we have a abstract no-capture attribute for the argument we can use
5896 // it to justify a non-capture attribute here. This allows recursion!
5897 bool IsKnownNoCapture;
5898 const AANoCapture *ArgNoCaptureAA = nullptr;
5899 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
5900 A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
5901 &ArgNoCaptureAA);
5902 if (IsAssumedNoCapture)
5903 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5904 /* Return */ false);
5905 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
5906 Follow = true;
5907 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5908 /* Return */ false);
5909 }
5910
5911 // Lastly, we could not find a reason no-capture can be assumed so we don't.
5912 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5913 /* Return */ true);
5914 }
5915
5916 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
5917 /// \p CapturedInRet, then return true if we should continue updating the
5918 /// state.
5919 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
5920 bool CapturedInInt, bool CapturedInRet) {
5921 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
5922 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
5923 if (CapturedInMem)
5924 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM);
5925 if (CapturedInInt)
5926 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT);
5927 if (CapturedInRet)
5928 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET);
5929 return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
5930 }
5931};
5932
5933ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
5934 const IRPosition &IRP = getIRPosition();
5935 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
5936 : &IRP.getAssociatedValue();
5937 if (!V)
5938 return indicatePessimisticFixpoint();
5939
5940 const Function *F =
5941 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
5942
5943 // TODO: Is the checkForAllUses below useful for constants?
5944 if (!F)
5945 return indicatePessimisticFixpoint();
5946
5948 const IRPosition &FnPos = IRPosition::function(*F);
5949
5950 // Readonly means we cannot capture through memory.
5951 bool IsKnown;
5952 if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) {
5953 T.addKnownBits(NOT_CAPTURED_IN_MEM);
5954 if (IsKnown)
5955 addKnownBits(NOT_CAPTURED_IN_MEM);
5956 }
5957
5958 // Make sure all returned values are different than the underlying value.
5959 // TODO: we could do this in a more sophisticated way inside
5960 // AAReturnedValues, e.g., track all values that escape through returns
5961 // directly somehow.
5962 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
5964 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*F), this, Values,
5966 UsedAssumedInformation))
5967 return false;
5968 bool SeenConstant = false;
5969 for (const AA::ValueAndContext &VAC : Values) {
5970 if (isa<Constant>(VAC.getValue())) {
5971 if (SeenConstant)
5972 return false;
5973 SeenConstant = true;
5974 } else if (!isa<Argument>(VAC.getValue()) ||
5975 VAC.getValue() == getAssociatedArgument())
5976 return false;
5977 }
5978 return true;
5979 };
5980
5981 bool IsKnownNoUnwind;
5983 A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
5984 bool IsVoidTy = F->getReturnType()->isVoidTy();
5985 bool UsedAssumedInformation = false;
5986 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
5987 T.addKnownBits(NOT_CAPTURED_IN_RET);
5988 // If the function only reads memory and does not return the value, the
5989 // value can still be captured by whether the function returns.
5990 bool IsKnownWillReturn;
5991 bool IsAssumedWillReturn = AA::hasAssumedIRAttr<Attribute::WillReturn>(
5992 A, this, FnPos, DepClassTy::OPTIONAL, IsKnownWillReturn);
5993 if (IsAssumedWillReturn && T.isKnown(NOT_CAPTURED_IN_MEM))
5995 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
5996 addKnownBits(NOT_CAPTURED_IN_RET);
5997 if (IsKnownWillReturn && isKnown(NOT_CAPTURED_IN_MEM))
5998 return indicateOptimisticFixpoint();
5999 }
6000 }
6001 }
6002
6003 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
6004 // TODO(captures): Make this more precise.
6005 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
6006 if (capturesNothing(CI))
6007 return true;
6008 if (CI.isPassthrough()) {
6009 Follow = true;
6010 return true;
6011 }
6012 return checkUse(A, T, U, Follow);
6013 };
6014
6015 if (!A.checkForAllUses(UseCheck, *this, *V))
6016 return indicatePessimisticFixpoint();
6017
6018 AANoCapture::StateType &S = getState();
6019 auto Assumed = S.getAssumed();
6020 S.intersectAssumedBits(T.getAssumed());
6021 if (!isAssumedNoCaptureMaybeReturned())
6022 return indicatePessimisticFixpoint();
6023 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
6025}
6026
6027/// NoCapture attribute for function arguments.
6028struct AANoCaptureArgument final : AANoCaptureImpl {
6029 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6030 : AANoCaptureImpl(IRP, A) {}
6031
6032 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6033};
6034
6035/// NoCapture attribute for call site arguments.
6036struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6037 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6038 : AANoCaptureImpl(IRP, A) {}
6039
6040 ChangeStatus updateImpl(Attributor &A) override {
6041 // TODO: Once we have call site specific value information we can provide
6042 // call site specific liveness information and then it makes
6043 // sense to specialize attributes for call sites arguments instead of
6044 // redirecting requests to the callee argument.
6045 Argument *Arg = getAssociatedArgument();
6046 if (!Arg)
6047 return indicatePessimisticFixpoint();
6048 const IRPosition &ArgPos = IRPosition::argument(*Arg);
6049 bool IsKnownNoCapture;
6050 const AANoCapture *ArgAA = nullptr;
6052 A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
6053 &ArgAA))
6054 return ChangeStatus::UNCHANGED;
6055 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6056 return indicatePessimisticFixpoint();
6057 return clampStateAndIndicateChange(getState(), ArgAA->getState());
6058 }
6059
6060 void trackStatistics() const override {
6062 };
6063};
6064
6065/// NoCapture attribute for floating values.
6066struct AANoCaptureFloating final : AANoCaptureImpl {
6067 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6068 : AANoCaptureImpl(IRP, A) {}
6069
6070 void trackStatistics() const override {
6072 }
6073};
6074
6075/// NoCapture attribute for function return value.
6076struct AANoCaptureReturned final : AANoCaptureImpl {
6077 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6078 : AANoCaptureImpl(IRP, A) {
6079 llvm_unreachable("NoCapture is not applicable to function returns!");
6080 }
6081
6082 void initialize(Attributor &A) override {
6083 llvm_unreachable("NoCapture is not applicable to function returns!");
6084 }
6085
6086 ChangeStatus updateImpl(Attributor &A) override {
6087 llvm_unreachable("NoCapture is not applicable to function returns!");
6088 }
6089
6090 void trackStatistics() const override {}
6091};
6092
6093/// NoCapture attribute deduction for a call site return value.
6094struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6095 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6096 : AANoCaptureImpl(IRP, A) {}
6097
6098 void initialize(Attributor &A) override {
6099 const Function *F = getAnchorScope();
6100 // Check what state the associated function can actually capture.
6101 determineFunctionCaptureCapabilities(getIRPosition(), *F, *this);
6102 }
6103
6104 void trackStatistics() const override {
6106 }
6107};
6108} // namespace
6109
6110/// ------------------ Value Simplify Attribute ----------------------------
6111
6112bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6113 // FIXME: Add a typecast support.
6116 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6117 return false;
6118
6119 LLVM_DEBUG({
6121 dbgs() << "[ValueSimplify] is assumed to be "
6122 << **SimplifiedAssociatedValue << "\n";
6123 else
6124 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6125 });
6126 return true;
6127}
6128
6129namespace {
6130struct AAValueSimplifyImpl : AAValueSimplify {
6131 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6132 : AAValueSimplify(IRP, A) {}
6133
6134 void initialize(Attributor &A) override {
6135 if (getAssociatedValue().getType()->isVoidTy())
6136 indicatePessimisticFixpoint();
6137 if (A.hasSimplificationCallback(getIRPosition()))
6138 indicatePessimisticFixpoint();
6139 }
6140
6141 const std::string getAsStr(Attributor *A) const override {
6142 LLVM_DEBUG({
6143 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6144 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6145 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6146 });
6147 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6148 : "not-simple";
6149 }
6150
6151 void trackStatistics() const override {}
6152
6153 std::optional<Value *>
6154 getAssumedSimplifiedValue(Attributor &A) const override {
6155 return SimplifiedAssociatedValue;
6156 }
6157
6158 /// Ensure the return value is \p V with type \p Ty, if not possible return
6159 /// nullptr. If \p Check is true we will only verify such an operation would
6160 /// suceed and return a non-nullptr value if that is the case. No IR is
6161 /// generated or modified.
6162 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6163 bool Check) {
6164 if (auto *TypedV = AA::getWithType(V, Ty))
6165 return TypedV;
6166 if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty))
6167 return Check ? &V
6168 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6169 &V, &Ty, "", CtxI->getIterator());
6170 return nullptr;
6171 }
6172
6173 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6174 /// If \p Check is true we will only verify such an operation would suceed and
6175 /// return a non-nullptr value if that is the case. No IR is generated or
6176 /// modified.
6177 static Value *reproduceInst(Attributor &A,
6178 const AbstractAttribute &QueryingAA,
6179 Instruction &I, Type &Ty, Instruction *CtxI,
6180 bool Check, ValueToValueMapTy &VMap) {
6181 assert(CtxI && "Cannot reproduce an instruction without context!");
6182 if (Check && (I.mayReadFromMemory() ||
6183 !isSafeToSpeculativelyExecute(&I, CtxI, /* DT */ nullptr,
6184 /* TLI */ nullptr)))
6185 return nullptr;
6186 for (Value *Op : I.operands()) {
6187 Value *NewOp = reproduceValue(A, QueryingAA, *Op, Ty, CtxI, Check, VMap);
6188 if (!NewOp) {
6189 assert(Check && "Manifest of new value unexpectedly failed!");
6190 return nullptr;
6191 }
6192 if (!Check)
6193 VMap[Op] = NewOp;
6194 }
6195 if (Check)
6196 return &I;
6197
6198 Instruction *CloneI = I.clone();
6199 // TODO: Try to salvage debug information here.
6200 CloneI->setDebugLoc(DebugLoc());
6201 VMap[&I] = CloneI;
6202 CloneI->insertBefore(CtxI->getIterator());
6203 RemapInstruction(CloneI, VMap);
6204 return CloneI;
6205 }
6206
6207 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6208 /// If \p Check is true we will only verify such an operation would suceed and
6209 /// return a non-nullptr value if that is the case. No IR is generated or
6210 /// modified.
6211 static Value *reproduceValue(Attributor &A,
6212 const AbstractAttribute &QueryingAA, Value &V,
6213 Type &Ty, Instruction *CtxI, bool Check,
6214 ValueToValueMapTy &VMap) {
6215 if (const auto &NewV = VMap.lookup(&V))
6216 return NewV;
6217 bool UsedAssumedInformation = false;
6218 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6219 V, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
6220 if (!SimpleV.has_value())
6221 return PoisonValue::get(&Ty);
6222 Value *EffectiveV = &V;
6223 if (*SimpleV)
6224 EffectiveV = *SimpleV;
6225 if (auto *C = dyn_cast<Constant>(EffectiveV))
6226 return C;
6227 if (CtxI && AA::isValidAtPosition(AA::ValueAndContext(*EffectiveV, *CtxI),
6228 A.getInfoCache()))
6229 return ensureType(A, *EffectiveV, Ty, CtxI, Check);
6230 if (auto *I = dyn_cast<Instruction>(EffectiveV))
6231 if (Value *NewV = reproduceInst(A, QueryingAA, *I, Ty, CtxI, Check, VMap))
6232 return ensureType(A, *NewV, Ty, CtxI, Check);
6233 return nullptr;
6234 }
6235
6236 /// Return a value we can use as replacement for the associated one, or
6237 /// nullptr if we don't have one that makes sense.
6238 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6239 Value *NewV = SimplifiedAssociatedValue
6240 ? *SimplifiedAssociatedValue
6241 : UndefValue::get(getAssociatedType());
6242 if (NewV && NewV != &getAssociatedValue()) {
6243 ValueToValueMapTy VMap;
6244 // First verify we can reprduce the value with the required type at the
6245 // context location before we actually start modifying the IR.
6246 if (reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
6247 /* CheckOnly */ true, VMap))
6248 return reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
6249 /* CheckOnly */ false, VMap);
6250 }
6251 return nullptr;
6252 }
6253
6254 /// Helper function for querying AAValueSimplify and updating candidate.
6255 /// \param IRP The value position we are trying to unify with SimplifiedValue
6256 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6257 const IRPosition &IRP, bool Simplify = true) {
6258 bool UsedAssumedInformation = false;
6259 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6260 if (Simplify)
6261 QueryingValueSimplified = A.getAssumedSimplified(
6262 IRP, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
6263 return unionAssumed(QueryingValueSimplified);
6264 }
6265
6266 /// Returns a candidate is found or not
6267 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6268 if (!getAssociatedValue().getType()->isIntegerTy())
6269 return false;
6270
6271 // This will also pass the call base context.
6272 const auto *AA =
6273 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6274 if (!AA)
6275 return false;
6276
6277 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6278
6279 if (!COpt) {
6280 SimplifiedAssociatedValue = std::nullopt;
6281 A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
6282 return true;
6283 }
6284 if (auto *C = *COpt) {
6285 SimplifiedAssociatedValue = C;
6286 A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
6287 return true;
6288 }
6289 return false;
6290 }
6291
6292 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6293 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6294 return true;
6295 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6296 return true;
6297 return false;
6298 }
6299
6300 ChangeStatus manifest(Attributor &A) override {
6301 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6302 for (auto &U : getAssociatedValue().uses()) {
6303 // Check if we need to adjust the insertion point to make sure the IR is
6304 // valid.
6305 Instruction *IP = dyn_cast<Instruction>(U.getUser());
6306 if (auto *PHI = dyn_cast_or_null<PHINode>(IP))
6307 IP = PHI->getIncomingBlock(U)->getTerminator();
6308 if (auto *NewV = manifestReplacementValue(A, IP)) {
6309 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6310 << " -> " << *NewV << " :: " << *this << "\n");
6311 if (A.changeUseAfterManifest(U, *NewV))
6312 Changed = ChangeStatus::CHANGED;
6313 }
6314 }
6315
6316 return Changed | AAValueSimplify::manifest(A);
6317 }
6318
6319 ChangeStatus indicatePessimisticFixpoint() override {
6320 SimplifiedAssociatedValue = &getAssociatedValue();
6321 return AAValueSimplify::indicatePessimisticFixpoint();
6322 }
6323};
6324
6325struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6326 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6327 : AAValueSimplifyImpl(IRP, A) {}
6328
6329 void initialize(Attributor &A) override {
6330 AAValueSimplifyImpl::initialize(A);
6331 if (A.hasAttr(getIRPosition(),
6332 {Attribute::InAlloca, Attribute::Preallocated,
6333 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6334 /* IgnoreSubsumingPositions */ true))
6335 indicatePessimisticFixpoint();
6336 }
6337
6338 ChangeStatus updateImpl(Attributor &A) override {
6339 // Byval is only replacable if it is readonly otherwise we would write into
6340 // the replaced value and not the copy that byval creates implicitly.
6341 Argument *Arg = getAssociatedArgument();
6342 if (Arg->hasByValAttr()) {
6343 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6344 // there is no race by not copying a constant byval.
6345 bool IsKnown;
6346 if (!AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
6347 return indicatePessimisticFixpoint();
6348 }
6349
6350 auto Before = SimplifiedAssociatedValue;
6351
6352 auto PredForCallSite = [&](AbstractCallSite ACS) {
6353 const IRPosition &ACSArgPos =
6354 IRPosition::callsite_argument(ACS, getCallSiteArgNo());
6355 // Check if a coresponding argument was found or if it is on not
6356 // associated (which can happen for callback calls).
6357 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6358 return false;
6359
6360 // Simplify the argument operand explicitly and check if the result is
6361 // valid in the current scope. This avoids refering to simplified values
6362 // in other functions, e.g., we don't want to say a an argument in a
6363 // static function is actually an argument in a different function.
6364 bool UsedAssumedInformation = false;
6365 std::optional<Constant *> SimpleArgOp =
6366 A.getAssumedConstant(ACSArgPos, *this, UsedAssumedInformation);
6367 if (!SimpleArgOp)
6368 return true;
6369 if (!*SimpleArgOp)
6370 return false;
6371 if (!AA::isDynamicallyUnique(A, *this, **SimpleArgOp))
6372 return false;
6373 return unionAssumed(*SimpleArgOp);
6374 };
6375
6376 // Generate a answer specific to a call site context.
6377 bool Success;
6378 bool UsedAssumedInformation = false;
6379 if (hasCallBaseContext() &&
6380 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6381 Success = PredForCallSite(
6382 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6383 else
6384 Success = A.checkForAllCallSites(PredForCallSite, *this, true,
6385 UsedAssumedInformation);
6386
6387 if (!Success)
6388 if (!askSimplifiedValueForOtherAAs(A))
6389 return indicatePessimisticFixpoint();
6390
6391 // If a candidate was found in this update, return CHANGED.
6392 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6393 : ChangeStatus ::CHANGED;
6394 }
6395
6396 void trackStatistics() const override {
6397 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6398 }
6399};
6400
6401struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6402 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6403 : AAValueSimplifyImpl(IRP, A) {}
6404
6405 std::optional<Value *>
6406 getAssumedSimplifiedValue(Attributor &A) const override {
6407 if (!isValidState())
6408 return nullptr;
6409 return SimplifiedAssociatedValue;
6410 }
6411
6412 ChangeStatus updateImpl(Attributor &A) override {
6413 auto Before = SimplifiedAssociatedValue;
6414
6415 auto ReturnInstCB = [&](Instruction &I) {
6416 auto &RI = cast<ReturnInst>(I);
6417 return checkAndUpdate(
6418 A, *this,
6419 IRPosition::value(*RI.getReturnValue(), getCallBaseContext()));
6420 };
6421
6422 bool UsedAssumedInformation = false;
6423 if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret},
6424 UsedAssumedInformation))
6425 if (!askSimplifiedValueForOtherAAs(A))
6426 return indicatePessimisticFixpoint();
6427
6428 // If a candidate was found in this update, return CHANGED.
6429 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6430 : ChangeStatus ::CHANGED;
6431 }
6432
6433 ChangeStatus manifest(Attributor &A) override {
6434 // We queried AAValueSimplify for the returned values so they will be
6435 // replaced if a simplified form was found. Nothing to do here.
6436 return ChangeStatus::UNCHANGED;
6437 }
6438
6439 void trackStatistics() const override {
6440 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6441 }
6442};
6443
6444struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6445 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6446 : AAValueSimplifyImpl(IRP, A) {}
6447
6448 void initialize(Attributor &A) override {
6449 AAValueSimplifyImpl::initialize(A);
6450 Value &V = getAnchorValue();
6451
6452 // TODO: add other stuffs
6453 if (isa<Constant>(V))
6454 indicatePessimisticFixpoint();
6455 }
6456
6457 ChangeStatus updateImpl(Attributor &A) override {
6458 auto Before = SimplifiedAssociatedValue;
6459 if (!askSimplifiedValueForOtherAAs(A))
6460 return indicatePessimisticFixpoint();
6461
6462 // If a candidate was found in this update, return CHANGED.
6463 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6464 : ChangeStatus ::CHANGED;
6465 }
6466
6467 void trackStatistics() const override {
6468 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6469 }
6470};
6471
6472struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6473 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6474 : AAValueSimplifyImpl(IRP, A) {}
6475
6476 void initialize(Attributor &A) override {
6477 SimplifiedAssociatedValue = nullptr;
6478 indicateOptimisticFixpoint();
6479 }
6480 ChangeStatus updateImpl(Attributor &A) override {
6482 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6483 }
6484 void trackStatistics() const override {
6485 STATS_DECLTRACK_FN_ATTR(value_simplify)
6486 }
6487};
6488
6489struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6490 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6491 : AAValueSimplifyFunction(IRP, A) {}
6492 void trackStatistics() const override {
6493 STATS_DECLTRACK_CS_ATTR(value_simplify)
6494 }
6495};
6496
6497struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6498 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6499 : AAValueSimplifyImpl(IRP, A) {}
6500
6501 void initialize(Attributor &A) override {
6502 AAValueSimplifyImpl::initialize(A);
6503 Function *Fn = getAssociatedFunction();
6504 assert(Fn && "Did expect an associted function");
6505 for (Argument &Arg : Fn->args()) {
6506 if (Arg.hasReturnedAttr()) {
6507 auto IRP = IRPosition::callsite_argument(*cast<CallBase>(getCtxI()),
6508 Arg.getArgNo());
6510 checkAndUpdate(A, *this, IRP))
6511 indicateOptimisticFixpoint();
6512 else
6513 indicatePessimisticFixpoint();
6514 return;
6515 }
6516 }
6517 }
6518
6519 ChangeStatus updateImpl(Attributor &A) override {
6520 return indicatePessimisticFixpoint();
6521 }
6522
6523 void trackStatistics() const override {
6524 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6525 }
6526};
6527
6528struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6529 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6530 : AAValueSimplifyFloating(IRP, A) {}
6531
6532 ChangeStatus manifest(Attributor &A) override {
6533 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6534 // TODO: We should avoid simplification duplication to begin with.
6535 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6536 IRPosition::value(getAssociatedValue()), this, DepClassTy::NONE);
6537 if (FloatAA && FloatAA->getState().isValidState())
6538 return Changed;
6539
6540 if (auto *NewV = manifestReplacementValue(A, getCtxI())) {
6541 Use &U = cast<CallBase>(&getAnchorValue())
6542 ->getArgOperandUse(getCallSiteArgNo());
6543 if (A.changeUseAfterManifest(U, *NewV))
6544 Changed = ChangeStatus::CHANGED;
6545 }
6546
6547 return Changed | AAValueSimplify::manifest(A);
6548 }
6549
6550 void trackStatistics() const override {
6551 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6552 }
6553};
6554} // namespace
6555
6556/// Collect the non-musttail tail calls in the function of \p QueryingAA. Tail
6557/// calls cannot access allocas of the caller.
6558/// TODO: Be smarter about new allocas escaping into tail calls.
6559static bool collectTailCalls(Attributor &A, const AbstractAttribute &QueryingAA,
6560 SmallVectorImpl<CallInst *> &TailCalls) {
6561 bool UsedAssumedInformation = false;
6562 return A.checkForAllInstructions(
6563 [&](Instruction &I) {
6564 auto &CI = cast<CallInst>(I);
6565 if (CI.isTailCall() && !CI.isMustTailCall())
6566 TailCalls.push_back(&CI);
6567 return true;
6568 },
6569 QueryingAA, {Instruction::Call}, UsedAssumedInformation);
6570}
6571
6572/// ----------------------- Heap-To-Stack Conversion ---------------------------
6573namespace {
6574struct AAHeapToStackFunction final : public AAHeapToStack {
6575
6576 static bool isGlobalizedLocal(const CallBase &CB) {
6577 Attribute A = CB.getFnAttr("alloc-family");
6578 return A.isValid() && A.getValueAsString() == "__kmpc_alloc_shared";
6579 }
6580
6581 struct AllocationInfo {
6582 /// The call that allocates the memory.
6583 CallBase *const CB;
6584
6585 /// Whether this allocation is an OpenMP globalized local variable.
6586 bool IsGlobalizedLocal = false;
6587
6588 /// The status wrt. a rewrite.
6589 enum {
6590 STACK_DUE_TO_USE,
6591 STACK_DUE_TO_FREE,
6592 INVALID,
6593 } Status = STACK_DUE_TO_USE;
6594
6595 /// Flag to indicate if we encountered a use that might free this allocation
6596 /// but which is not in the deallocation infos.
6597 bool HasPotentiallyFreeingUnknownUses = false;
6598
6599 /// Flag to indicate that we should place the new alloca in the function
6600 /// entry block rather than where the call site (CB) is.
6601 bool MoveAllocaIntoEntry = true;
6602
6603 /// The set of free calls that use this allocation.
6604 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6605 };
6606
6607 struct DeallocationInfo {
6608 /// The call that deallocates the memory.
6609 CallBase *const CB;
6610 /// The value freed by the call.
6611 Value *FreedOp;
6612
6613 /// Flag to indicate if we don't know all objects this deallocation might
6614 /// free.
6615 bool MightFreeUnknownObjects = false;
6616
6617 /// The set of allocation calls that are potentially freed.
6618 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6619 };
6620
6621 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6622 : AAHeapToStack(IRP, A) {}
6623
6624 ~AAHeapToStackFunction() override {
6625 // Ensure we call the destructor so we release any memory allocated in the
6626 // sets.
6627 for (auto &It : AllocationInfos)
6628 It.second->~AllocationInfo();
6629 for (auto &It : DeallocationInfos)
6630 It.second->~DeallocationInfo();
6631 }
6632
6633 void initialize(Attributor &A) override {
6634 AAHeapToStack::initialize(A);
6635
6636 const Function *F = getAnchorScope();
6637 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6638
6639 auto AllocationIdentifierCB = [&](Instruction &I) {
6640 CallBase *CB = dyn_cast<CallBase>(&I);
6641 if (!CB)
6642 return true;
6643 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6644 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{CB, FreedOp};
6645 return true;
6646 }
6647 // To do heap to stack, we need to know that the allocation itself is
6648 // removable once uses are rewritten, and that we can initialize the
6649 // alloca to the same pattern as the original allocation result.
6650 if (isRemovableAlloc(CB, TLI)) {
6651 auto *I8Ty = Type::getInt8Ty(CB->getParent()->getContext());
6652 if (nullptr != getInitialValueOfAllocation(CB, TLI, I8Ty)) {
6653 AllocationInfo *AI = new (A.Allocator) AllocationInfo{CB};
6654 AllocationInfos[CB] = AI;
6655 AI->IsGlobalizedLocal = isGlobalizedLocal(*CB);
6656 }
6657 }
6658 return true;
6659 };
6660
6661 bool UsedAssumedInformation = false;
6662 bool Success = A.checkForAllCallLikeInstructions(
6663 AllocationIdentifierCB, *this, UsedAssumedInformation,
6664 /* CheckBBLivenessOnly */ false,
6665 /* CheckPotentiallyDead */ true);
6666 (void)Success;
6667 assert(Success && "Did not expect the call base visit callback to fail!");
6668
6670 [](const IRPosition &, const AbstractAttribute *,
6671 bool &) -> std::optional<Value *> { return nullptr; };
6672 for (const auto &It : AllocationInfos)
6673 A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
6674 SCB);
6675 for (const auto &It : DeallocationInfos)
6676 A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
6677 SCB);
6678 }
6679
6680 const std::string getAsStr(Attributor *A) const override {
6681 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6682 for (const auto &It : AllocationInfos) {
6683 if (It.second->Status == AllocationInfo::INVALID)
6684 ++NumInvalidMallocs;
6685 else
6686 ++NumH2SMallocs;
6687 }
6688 return "[H2S] Mallocs Good/Bad: " + std::to_string(NumH2SMallocs) + "/" +
6689 std::to_string(NumInvalidMallocs);
6690 }
6691
6692 void trackStatistics() const override {
6693 STATS_DECL(
6694 MallocCalls, Function,
6695 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6696 for (const auto &It : AllocationInfos)
6697 if (It.second->Status != AllocationInfo::INVALID)
6698 ++BUILD_STAT_NAME(MallocCalls, Function);
6699 }
6700
6701 bool isAssumedHeapToStack(const CallBase &CB) const override {
6702 if (isValidState())
6703 if (AllocationInfo *AI =
6704 AllocationInfos.lookup(const_cast<CallBase *>(&CB)))
6705 return AI->Status != AllocationInfo::INVALID;
6706 return false;
6707 }
6708
6709 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6710 if (!isValidState())
6711 return false;
6712
6713 for (const auto &It : AllocationInfos) {
6714 AllocationInfo &AI = *It.second;
6715 if (AI.Status == AllocationInfo::INVALID)
6716 continue;
6717
6718 if (AI.PotentialFreeCalls.count(&CB))
6719 return true;
6720 }
6721
6722 return false;
6723 }
6724
6725 ChangeStatus manifest(Attributor &A) override {
6726 assert(getState().isValidState() &&
6727 "Attempted to manifest an invalid state!");
6728
6729 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6730 Function *F = getAnchorScope();
6731 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6732
6733 // The new allocas may escape into tail calls.
6735 if (any_of(AllocationInfos,
6736 [](const std::pair<CallBase *, AllocationInfo *> &It) {
6737 return It.second->Status != AllocationInfo::INVALID;
6738 }) &&
6739 !collectTailCalls(A, *this, TailCalls))
6740 return ChangeStatus::UNCHANGED;
6741
6742 for (auto &It : AllocationInfos) {
6743 AllocationInfo &AI = *It.second;
6744 if (AI.Status == AllocationInfo::INVALID)
6745 continue;
6746
6747 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6748 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6749 A.deleteAfterManifest(*FreeCall);
6750 HasChanged = ChangeStatus::CHANGED;
6751 }
6752
6753 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6754 << "\n");
6755
6756 auto Remark = [&](OptimizationRemark OR) {
6757 if (AI.IsGlobalizedLocal)
6758 return OR << "Moving globalized variable to the stack.";
6759 return OR << "Moving memory allocation from the heap to the stack.";
6760 };
6761 if (AI.IsGlobalizedLocal)
6762 A.emitRemark<OptimizationRemark>(AI.CB, "OMP110", Remark);
6763 else
6764 A.emitRemark<OptimizationRemark>(AI.CB, "HeapToStack", Remark);
6765
6766 const DataLayout &DL = A.getInfoCache().getDL();
6767 Value *Size;
6768 std::optional<APInt> SizeAPI = getSize(A, *this, AI);
6769 if (SizeAPI) {
6770 Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);
6771 } else {
6772 ObjectSizeOpts Opts;
6773 ObjectSizeOffsetEvaluator Eval(*AI.CB->getModule(), TLI, Opts);
6774 SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);
6775 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6776 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6777 Size = SizeOffsetPair.Size;
6778 }
6779
6780 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6781 ? F->getEntryBlock().begin()
6782 : AI.CB->getIterator();
6783
6784 Align Alignment(1);
6785 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6786 Alignment = std::max(Alignment, *RetAlign);
6787 if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
6788 std::optional<APInt> AlignmentAPI = getAPInt(A, *this, *Align);
6789 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6790 "Expected an alignment during manifest!");
6791 Alignment =
6792 std::max(Alignment, assumeAligned(AlignmentAPI->getZExtValue()));
6793 }
6794
6795 // TODO: Hoist the alloca towards the function entry.
6796 unsigned AS = DL.getAllocaAddrSpace();
6797 Instruction *Alloca =
6798 new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment,
6799 AI.CB->getName() + ".h2s", IP);
6800
6801 if (Alloca->getType() != AI.CB->getType())
6802 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6803 Alloca, AI.CB->getType(), "malloc_cast", AI.CB->getIterator());
6804
6805 auto *I8Ty = Type::getInt8Ty(F->getContext());
6806 auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty);
6807 assert(InitVal &&
6808 "Must be able to materialize initial memory state of allocation");
6809
6810 A.changeAfterManifest(IRPosition::inst(*AI.CB), *Alloca);
6811
6812 if (auto *II = dyn_cast<InvokeInst>(AI.CB)) {
6813 auto *NBB = II->getNormalDest();
6814 UncondBrInst::Create(NBB, AI.CB->getParent());
6815 A.deleteAfterManifest(*AI.CB);
6816 } else {
6817 A.deleteAfterManifest(*AI.CB);
6818 }
6819
6820 // Initialize the alloca with the same value as used by the allocation
6821 // function. We can skip undef as the initial value of an alloc is
6822 // undef, and the memset would simply end up being DSEd.
6823 if (!isa<UndefValue>(InitVal)) {
6824 IRBuilder<> Builder(Alloca->getNextNode());
6825 // TODO: Use alignment above if align!=1
6826 Builder.CreateMemSet(Alloca, InitVal, Size, std::nullopt);
6827 }
6828 HasChanged = ChangeStatus::CHANGED;
6829 }
6830
6831 for (CallInst *CI : TailCalls)
6832 CI->setTailCall(false);
6833
6834 return HasChanged;
6835 }
6836
6837 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6838 Value &V) {
6839 bool UsedAssumedInformation = false;
6840 std::optional<Constant *> SimpleV =
6841 A.getAssumedConstant(V, AA, UsedAssumedInformation);
6842 if (!SimpleV)
6843 return APInt(64, 0);
6844 if (auto *CI = dyn_cast_or_null<ConstantInt>(*SimpleV))
6845 return CI->getValue();
6846 return std::nullopt;
6847 }
6848
6849 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
6850 AllocationInfo &AI) {
6851 auto Mapper = [&](const Value *V) -> const Value * {
6852 bool UsedAssumedInformation = false;
6853 if (std::optional<Constant *> SimpleV =
6854 A.getAssumedConstant(*V, AA, UsedAssumedInformation))
6855 if (*SimpleV)
6856 return *SimpleV;
6857 return V;
6858 };
6859
6860 const Function *F = getAnchorScope();
6861 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6862 return getAllocSize(AI.CB, TLI, Mapper);
6863 }
6864
6865 /// Collection of all malloc-like calls in a function with associated
6866 /// information.
6867 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
6868
6869 /// Collection of all free-like calls in a function with associated
6870 /// information.
6871 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
6872
6873 ChangeStatus updateImpl(Attributor &A) override;
6874};
6875
6876ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
6878 const Function *F = getAnchorScope();
6879 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6880
6881 const auto *LivenessAA =
6882 A.getAAFor<AAIsDead>(*this, IRPosition::function(*F), DepClassTy::NONE);
6883
6884 MustBeExecutedContextExplorer *Explorer =
6885 A.getInfoCache().getMustBeExecutedContextExplorer();
6886
6887 bool StackIsAccessibleByOtherThreads =
6888 A.getInfoCache().stackIsAccessibleByOtherThreads();
6889
6890 LoopInfo *LI =
6891 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F);
6892 std::optional<bool> MayContainIrreducibleControl;
6893 auto IsInLoop = [&](BasicBlock &BB) {
6894 if (&F->getEntryBlock() == &BB)
6895 return false;
6896 if (!MayContainIrreducibleControl.has_value())
6897 MayContainIrreducibleControl = mayContainIrreducibleControl(*F, LI);
6898 if (*MayContainIrreducibleControl)
6899 return true;
6900 if (!LI)
6901 return true;
6902 return LI->getLoopFor(&BB) != nullptr;
6903 };
6904
6905 // Flag to ensure we update our deallocation information at most once per
6906 // updateImpl call and only if we use the free check reasoning.
6907 bool HasUpdatedFrees = false;
6908
6909 auto UpdateFrees = [&]() {
6910 HasUpdatedFrees = true;
6911
6912 for (auto &It : DeallocationInfos) {
6913 DeallocationInfo &DI = *It.second;
6914 // For now we cannot use deallocations that have unknown inputs, skip
6915 // them.
6916 if (DI.MightFreeUnknownObjects)
6917 continue;
6918
6919 // No need to analyze dead calls, ignore them instead.
6920 bool UsedAssumedInformation = false;
6921 if (A.isAssumedDead(*DI.CB, this, LivenessAA, UsedAssumedInformation,
6922 /* CheckBBLivenessOnly */ true))
6923 continue;
6924
6925 // Use the non-optimistic version to get the freed object.
6926 Value *Obj = getUnderlyingObject(DI.FreedOp);
6927 if (!Obj) {
6928 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
6929 DI.MightFreeUnknownObjects = true;
6930 continue;
6931 }
6932
6933 // Free of null and undef can be ignored as no-ops (or UB in the latter
6934 // case).
6936 continue;
6937
6938 CallBase *ObjCB = dyn_cast<CallBase>(Obj);
6939 if (!ObjCB) {
6940 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
6941 << "\n");
6942 DI.MightFreeUnknownObjects = true;
6943 continue;
6944 }
6945
6946 AllocationInfo *AI = AllocationInfos.lookup(ObjCB);
6947 if (!AI) {
6948 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
6949 << "\n");
6950 DI.MightFreeUnknownObjects = true;
6951 continue;
6952 }
6953
6954 DI.PotentialAllocationCalls.insert(ObjCB);
6955 }
6956 };
6957
6958 auto FreeCheck = [&](AllocationInfo &AI) {
6959 // If the stack is not accessible by other threads, the "must-free" logic
6960 // doesn't apply as the pointer could be shared and needs to be places in
6961 // "shareable" memory.
6962 if (!StackIsAccessibleByOtherThreads) {
6963 bool IsKnownNoSycn;
6965 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) {
6966 LLVM_DEBUG(
6967 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
6968 "other threads and function is not nosync:\n");
6969 return false;
6970 }
6971 }
6972 if (!HasUpdatedFrees)
6973 UpdateFrees();
6974
6975 // TODO: Allow multi exit functions that have different free calls.
6976 if (AI.PotentialFreeCalls.size() != 1) {
6977 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
6978 << AI.PotentialFreeCalls.size() << "\n");
6979 return false;
6980 }
6981 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
6982 DeallocationInfo *DI = DeallocationInfos.lookup(UniqueFree);
6983 if (!DI) {
6984 LLVM_DEBUG(
6985 dbgs() << "[H2S] unique free call was not known as deallocation call "
6986 << *UniqueFree << "\n");
6987 return false;
6988 }
6989 if (DI->MightFreeUnknownObjects) {
6990 LLVM_DEBUG(
6991 dbgs() << "[H2S] unique free call might free unknown allocations\n");
6992 return false;
6993 }
6994 if (DI->PotentialAllocationCalls.empty())
6995 return true;
6996 if (DI->PotentialAllocationCalls.size() > 1) {
6997 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
6998 << DI->PotentialAllocationCalls.size()
6999 << " different allocations\n");
7000 return false;
7001 }
7002 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
7003 LLVM_DEBUG(
7004 dbgs()
7005 << "[H2S] unique free call not known to free this allocation but "
7006 << **DI->PotentialAllocationCalls.begin() << "\n");
7007 return false;
7008 }
7009
7010 // __kmpc_alloc_shared and __kmpc_free_shared are by construction matched.
7011 if (!AI.IsGlobalizedLocal) {
7012 Instruction *CtxI = isa<InvokeInst>(AI.CB) ? AI.CB : AI.CB->getNextNode();
7013 if (!Explorer || !Explorer->isInContextOf(UniqueFree, CtxI)) {
7014 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
7015 "with the allocation "
7016 << *UniqueFree << "\n");
7017 return false;
7018 }
7019 }
7020 return true;
7021 };
7022
7023 auto UsesCheck = [&](AllocationInfo &AI) {
7024 bool ValidUsesOnly = true;
7025
7026 auto Pred = [&](const Use &U, bool &Follow) -> bool {
7027 Instruction *UserI = cast<Instruction>(U.getUser());
7028 if (isa<LoadInst>(UserI))
7029 return true;
7030 if (auto *SI = dyn_cast<StoreInst>(UserI)) {
7031 if (SI->getValueOperand() == U.get()) {
7033 << "[H2S] escaping store to memory: " << *UserI << "\n");
7034 ValidUsesOnly = false;
7035 } else {
7036 // A store into the malloc'ed memory is fine.
7037 }
7038 return true;
7039 }
7040 if (auto *CB = dyn_cast<CallBase>(UserI)) {
7041 // A musttail call cannot access allocas of the caller, and its marker
7042 // cannot be removed.
7043 if (CB->isMustTailCall()) {
7044 LLVM_DEBUG(dbgs() << "[H2S] musttail user: " << *UserI << "\n");
7045 ValidUsesOnly = false;
7046 return true;
7047 }
7048
7049 if (!CB->isArgOperand(&U) || CB->isLifetimeStartOrEnd())
7050 return true;
7051 if (DeallocationInfos.count(CB)) {
7052 AI.PotentialFreeCalls.insert(CB);
7053 return true;
7054 }
7055
7056 unsigned ArgNo = CB->getArgOperandNo(&U);
7057 auto CBIRP = IRPosition::callsite_argument(*CB, ArgNo);
7058
7059 bool IsKnownNoCapture;
7060 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7061 A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);
7062
7063 // If a call site argument use is nofree, we are fine.
7064 bool IsKnownNoFree;
7065 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7066 A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree);
7067
7068 if (!IsAssumedNoCapture ||
7069 (!AI.IsGlobalizedLocal && !IsAssumedNoFree)) {
7070 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7071
7072 // Emit a missed remark if this is missed OpenMP globalization.
7073 auto Remark = [&](OptimizationRemarkMissed ORM) {
7074 return ORM
7075 << "Could not move globalized variable to the stack. "
7076 "Variable is potentially captured in call. Mark "
7077 "parameter as `__attribute__((noescape))` to override.";
7078 };
7079
7080 if (ValidUsesOnly && AI.IsGlobalizedLocal)
7081 A.emitRemark<OptimizationRemarkMissed>(CB, "OMP113", Remark);
7082
7083 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7084 ValidUsesOnly = false;
7085 }
7086 return true;
7087 }
7088
7089 if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
7090 isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
7091 Follow = true;
7092 return true;
7093 }
7094 // Unknown user for which we can not track uses further (in a way that
7095 // makes sense).
7096 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7097 ValidUsesOnly = false;
7098 return true;
7099 };
7100 if (!A.checkForAllUses(Pred, *this, *AI.CB, /* CheckBBLivenessOnly */ false,
7101 DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7102 [&](const Use &OldU, const Use &NewU) {
7103 auto *SI = dyn_cast<StoreInst>(OldU.getUser());
7104 return !SI || StackIsAccessibleByOtherThreads ||
7105 AA::isAssumedThreadLocalObject(
7106 A, *SI->getPointerOperand(), *this);
7107 }))
7108 return false;
7109 return ValidUsesOnly;
7110 };
7111
7112 // The actual update starts here. We look at all allocations and depending on
7113 // their status perform the appropriate check(s).
7114 for (auto &It : AllocationInfos) {
7115 AllocationInfo &AI = *It.second;
7116 if (AI.Status == AllocationInfo::INVALID)
7117 continue;
7118
7119 if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
7120 std::optional<APInt> APAlign = getAPInt(A, *this, *Align);
7121 if (!APAlign) {
7122 // Can't generate an alloca which respects the required alignment
7123 // on the allocation.
7124 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7125 << "\n");
7126 AI.Status = AllocationInfo::INVALID;
7128 continue;
7129 }
7130 if (APAlign->ugt(llvm::Value::MaximumAlignment) ||
7131 !APAlign->isPowerOf2()) {
7132 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7133 << "\n");
7134 AI.Status = AllocationInfo::INVALID;
7136 continue;
7137 }
7138 }
7139
7140 std::optional<APInt> Size = getSize(A, *this, AI);
7141 if (!AI.IsGlobalizedLocal && A.Opts.max_heap_to_stack_size != -1) {
7142 if (!Size || Size->ugt(A.Opts.max_heap_to_stack_size)) {
7143 LLVM_DEBUG({
7144 if (!Size)
7145 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7146 else
7147 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7148 << A.Opts.max_heap_to_stack_size << "\n";
7149 });
7150
7151 AI.Status = AllocationInfo::INVALID;
7153 continue;
7154 }
7155 }
7156
7157 switch (AI.Status) {
7158 case AllocationInfo::STACK_DUE_TO_USE:
7159 if (UsesCheck(AI))
7160 break;
7161 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7162 [[fallthrough]];
7163 case AllocationInfo::STACK_DUE_TO_FREE:
7164 if (FreeCheck(AI))
7165 break;
7166 AI.Status = AllocationInfo::INVALID;
7168 break;
7169 case AllocationInfo::INVALID:
7170 llvm_unreachable("Invalid allocations should never reach this point!");
7171 };
7172
7173 // Check if we still think we can move it into the entry block. If the
7174 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7175 // ignore the potential complications associated with loops.
7176 bool IsGlobalizedLocal = AI.IsGlobalizedLocal;
7177 if (AI.MoveAllocaIntoEntry &&
7178 (!Size.has_value() ||
7179 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7180 AI.MoveAllocaIntoEntry = false;
7181 }
7182
7183 return Changed;
7184}
7185} // namespace
7186
7187/// ----------------------- Privatizable Pointers ------------------------------
7188namespace {
7189struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7190 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7191 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7192
7193 ChangeStatus indicatePessimisticFixpoint() override {
7194 AAPrivatizablePtr::indicatePessimisticFixpoint();
7195 PrivatizableType = nullptr;
7196 return ChangeStatus::CHANGED;
7197 }
7198
7199 /// Identify the type we can chose for a private copy of the underlying
7200 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7201 /// none.
7202 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7203
7204 /// Return a privatizable type that encloses both T0 and T1.
7205 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7206 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7207 std::optional<Type *> T1) {
7208 if (!T0)
7209 return T1;
7210 if (!T1)
7211 return T0;
7212 if (T0 == T1)
7213 return T0;
7214 return nullptr;
7215 }
7216
7217 std::optional<Type *> getPrivatizableType() const override {
7218 return PrivatizableType;
7219 }
7220
7221 const std::string getAsStr(Attributor *A) const override {
7222 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7223 }
7224
7225protected:
7226 std::optional<Type *> PrivatizableType;
7227};
7228
7229// TODO: Do this for call site arguments (probably also other values) as well.
7230
7231struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7232 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7233 : AAPrivatizablePtrImpl(IRP, A) {}
7234
7235 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7236 // If this is a byval argument and we know all the call sites (so we can
7237 // rewrite them), there is no need to check them explicitly.
7238 bool UsedAssumedInformation = false;
7240 A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs,
7241 /* IgnoreSubsumingPositions */ true);
7242 if (!Attrs.empty() &&
7243 A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *this,
7244 true, UsedAssumedInformation))
7245 return Attrs[0].getValueAsType();
7246
7247 std::optional<Type *> Ty;
7248 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7249
7250 // Make sure the associated call site argument has the same type at all call
7251 // sites and it is an allocation we know is safe to privatize, for now that
7252 // means we only allow alloca instructions.
7253 // TODO: We can additionally analyze the accesses in the callee to create
7254 // the type from that information instead. That is a little more
7255 // involved and will be done in a follow up patch.
7256 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7257 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7258 // Check if a coresponding argument was found or if it is one not
7259 // associated (which can happen for callback calls).
7260 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7261 return false;
7262
7263 // Check that all call sites agree on a type.
7264 auto *PrivCSArgAA =
7265 A.getAAFor<AAPrivatizablePtr>(*this, ACSArgPos, DepClassTy::REQUIRED);
7266 if (!PrivCSArgAA)
7267 return false;
7268 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7269
7270 LLVM_DEBUG({
7271 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7272 if (CSTy && *CSTy)
7273 (*CSTy)->print(dbgs());
7274 else if (CSTy)
7275 dbgs() << "<nullptr>";
7276 else
7277 dbgs() << "<none>";
7278 });
7279
7280 Ty = combineTypes(Ty, CSTy);
7281
7282 LLVM_DEBUG({
7283 dbgs() << " : New Type: ";
7284 if (Ty && *Ty)
7285 (*Ty)->print(dbgs());
7286 else if (Ty)
7287 dbgs() << "<nullptr>";
7288 else
7289 dbgs() << "<none>";
7290 dbgs() << "\n";
7291 });
7292
7293 return !Ty || *Ty;
7294 };
7295
7296 if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
7297 UsedAssumedInformation))
7298 return nullptr;
7299 return Ty;
7300 }
7301
7302 ChangeStatus updateImpl(Attributor &A) override {
7303 PrivatizableType = identifyPrivatizableType(A);
7304 if (!PrivatizableType)
7305 return ChangeStatus::UNCHANGED;
7306 if (!*PrivatizableType)
7307 return indicatePessimisticFixpoint();
7308
7309 // The dependence is optional so we don't give up once we give up on the
7310 // alignment.
7311 A.getAAFor<AAAlign>(*this, IRPosition::value(getAssociatedValue()),
7312 DepClassTy::OPTIONAL);
7313
7314 // Avoid arguments with padding for now.
7315 if (!A.hasAttr(getIRPosition(), Attribute::ByVal) &&
7316 !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) {
7317 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7318 return indicatePessimisticFixpoint();
7319 }
7320
7321 // Collect the types that will replace the privatizable type in the function
7322 // signature.
7323 SmallVector<Type *, 16> ReplacementTypes;
7324 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7325
7326 // Verify callee and caller agree on how the promoted argument would be
7327 // passed.
7328 Function &Fn = *getIRPosition().getAnchorScope();
7329 const auto *TTI =
7330 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);
7331 if (!TTI) {
7332 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7333 << Fn.getName() << "\n");
7334 return indicatePessimisticFixpoint();
7335 }
7336
7337 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7338 CallBase *CB = ACS.getInstruction();
7339 return TTI->areTypesABICompatible(
7340 CB->getCaller(),
7342 ReplacementTypes);
7343 };
7344 bool UsedAssumedInformation = false;
7345 if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
7346 UsedAssumedInformation)) {
7347 LLVM_DEBUG(
7348 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7349 << Fn.getName() << "\n");
7350 return indicatePessimisticFixpoint();
7351 }
7352
7353 // Register a rewrite of the argument.
7354 Argument *Arg = getAssociatedArgument();
7355 if (!A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {
7356 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7357 return indicatePessimisticFixpoint();
7358 }
7359
7360 unsigned ArgNo = Arg->getArgNo();
7361
7362 // Helper to check if for the given call site the associated argument is
7363 // passed to a callback where the privatization would be different.
7364 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7365 SmallVector<const Use *, 4> CallbackUses;
7366 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7367 for (const Use *U : CallbackUses) {
7368 AbstractCallSite CBACS(U);
7369 assert(CBACS && CBACS.isCallbackCall());
7370 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7371 int CBArgNo = CBACS.getCallArgOperandNo(CBArg);
7372
7373 LLVM_DEBUG({
7374 dbgs()
7375 << "[AAPrivatizablePtr] Argument " << *Arg
7376 << "check if can be privatized in the context of its parent ("
7377 << Arg->getParent()->getName()
7378 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7379 "callback ("
7380 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7381 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7382 << CBACS.getCallArgOperand(CBArg) << " vs "
7383 << CB.getArgOperand(ArgNo) << "\n"
7384 << "[AAPrivatizablePtr] " << CBArg << " : "
7385 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7386 });
7387
7388 if (CBArgNo != int(ArgNo))
7389 continue;
7390 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7391 *this, IRPosition::argument(CBArg), DepClassTy::REQUIRED);
7392 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7393 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7394 if (!CBArgPrivTy)
7395 continue;
7396 if (*CBArgPrivTy == PrivatizableType)
7397 continue;
7398 }
7399
7400 LLVM_DEBUG({
7401 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7402 << " cannot be privatized in the context of its parent ("
7403 << Arg->getParent()->getName()
7404 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7405 "callback ("
7406 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7407 << ").\n[AAPrivatizablePtr] for which the argument "
7408 "privatization is not compatible.\n";
7409 });
7410 return false;
7411 }
7412 }
7413 return true;
7414 };
7415
7416 // Helper to check if for the given call site the associated argument is
7417 // passed to a direct call where the privatization would be different.
7418 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7419 CallBase *DC = cast<CallBase>(ACS.getInstruction());
7420 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7421 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7422 "Expected a direct call operand for callback call operand");
7423
7424 Function *DCCallee =
7426 LLVM_DEBUG({
7427 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7428 << " check if be privatized in the context of its parent ("
7429 << Arg->getParent()->getName()
7430 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7431 "direct call of ("
7432 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7433 });
7434
7435 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7436 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7437 *this, IRPosition::argument(*DCCallee->getArg(DCArgNo)),
7438 DepClassTy::REQUIRED);
7439 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7440 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7441 if (!DCArgPrivTy)
7442 return true;
7443 if (*DCArgPrivTy == PrivatizableType)
7444 return true;
7445 }
7446 }
7447
7448 LLVM_DEBUG({
7449 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7450 << " cannot be privatized in the context of its parent ("
7451 << Arg->getParent()->getName()
7452 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7453 "direct call of ("
7455 << ").\n[AAPrivatizablePtr] for which the argument "
7456 "privatization is not compatible.\n";
7457 });
7458 return false;
7459 };
7460
7461 // Helper to check if the associated argument is used at the given abstract
7462 // call site in a way that is incompatible with the privatization assumed
7463 // here.
7464 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7465 if (ACS.isDirectCall())
7466 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7467 if (ACS.isCallbackCall())
7468 return IsCompatiblePrivArgOfDirectCS(ACS);
7469 return false;
7470 };
7471
7472 if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true,
7473 UsedAssumedInformation))
7474 return indicatePessimisticFixpoint();
7475
7476 return ChangeStatus::UNCHANGED;
7477 }
7478
7479 /// Given a type to private \p PrivType, collect the constituates (which are
7480 /// used) in \p ReplacementTypes.
7481 static void
7482 identifyReplacementTypes(Type *PrivType,
7483 SmallVectorImpl<Type *> &ReplacementTypes) {
7484 // TODO: For now we expand the privatization type to the fullest which can
7485 // lead to dead arguments that need to be removed later.
7486 assert(PrivType && "Expected privatizable type!");
7487
7488 // Traverse the type, extract constituate types on the outermost level.
7489 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7490 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7491 ReplacementTypes.push_back(PrivStructType->getElementType(u));
7492 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7493 ReplacementTypes.append(PrivArrayType->getNumElements(),
7494 PrivArrayType->getElementType());
7495 } else {
7496 ReplacementTypes.push_back(PrivType);
7497 }
7498 }
7499
7500 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7501 /// The values needed are taken from the arguments of \p F starting at
7502 /// position \p ArgNo.
7503 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7504 unsigned ArgNo, BasicBlock::iterator IP) {
7505 assert(PrivType && "Expected privatizable type!");
7506
7507 IRBuilder<NoFolder> IRB(IP);
7508 const DataLayout &DL = F.getDataLayout();
7509
7510 // Traverse the type, build GEPs and stores.
7511 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7512 const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
7513 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7514 Value *Ptr =
7515 constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB);
7516 new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
7517 }
7518 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7519 Type *PointeeTy = PrivArrayType->getElementType();
7520 uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
7521 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7522 Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB);
7523 new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
7524 }
7525 } else {
7526 new StoreInst(F.getArg(ArgNo), &Base, IP);
7527 }
7528 }
7529
7530 /// Extract values from \p Base according to the type \p PrivType at the
7531 /// call position \p ACS. The values are appended to \p ReplacementValues.
7532 void createReplacementValues(Align Alignment, Type *PrivType,
7533 AbstractCallSite ACS, Value *Base,
7534 SmallVectorImpl<Value *> &ReplacementValues) {
7535 assert(Base && "Expected base value!");
7536 assert(PrivType && "Expected privatizable type!");
7537 Instruction *IP = ACS.getInstruction();
7538
7539 IRBuilder<NoFolder> IRB(IP);
7540 const DataLayout &DL = IP->getDataLayout();
7541
7542 // Traverse the type, build GEPs and loads.
7543 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7544 const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
7545 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7546 Type *PointeeTy = PrivStructType->getElementType(u);
7547 Value *Ptr =
7548 constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB);
7549 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7550 L->setAlignment(Alignment);
7551 ReplacementValues.push_back(L);
7552 }
7553 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7554 Type *PointeeTy = PrivArrayType->getElementType();
7555 uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
7556 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7557 Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB);
7558 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7559 L->setAlignment(Alignment);
7560 ReplacementValues.push_back(L);
7561 }
7562 } else {
7563 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7564 L->setAlignment(Alignment);
7565 ReplacementValues.push_back(L);
7566 }
7567 }
7568
7569 ChangeStatus manifest(Attributor &A) override {
7570 if (!PrivatizableType)
7571 return ChangeStatus::UNCHANGED;
7572 assert(*PrivatizableType && "Expected privatizable type!");
7573
7574 Argument *Arg = getAssociatedArgument();
7575 // Query AAAlign attribute for alignment of associated argument to
7576 // determine the best alignment of loads.
7577 const auto *AlignAA =
7578 A.getAAFor<AAAlign>(*this, IRPosition::value(*Arg), DepClassTy::NONE);
7579
7580 // Callback to repair the associated function. A new alloca is placed at the
7581 // beginning and initialized with the values passed through arguments. The
7582 // new alloca replaces the use of the old pointer argument.
7584 [=](const Attributor::ArgumentReplacementInfo &ARI,
7585 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7586 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7588 const DataLayout &DL = IP->getDataLayout();
7589 unsigned AS = DL.getAllocaAddrSpace();
7590 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7591 Arg->getName() + ".priv", IP);
7592 createInitialization(*PrivatizableType, *AI, ReplacementFn,
7593 ArgIt->getArgNo(), IP);
7594
7595 if (AI->getType() != Arg->getType())
7596 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7597 AI, Arg->getType(), "", IP);
7598 Arg->replaceAllUsesWith(AI);
7599
7600 // The new alloca may escape into tail calls. Find them here, as calls
7601 // may have been replaced or deleted since manifest.
7602 for (Instruction &I : instructions(ReplacementFn)) {
7603 auto *CI = dyn_cast<CallInst>(&I);
7604 if (CI && CI->isTailCall() && !CI->isMustTailCall())
7605 CI->setTailCall(false);
7606 }
7607 };
7608
7609 // Callback to repair a call site of the associated function. The elements
7610 // of the privatizable type are loaded prior to the call and passed to the
7611 // new function version.
7613 [=](const Attributor::ArgumentReplacementInfo &ARI,
7614 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7615 // When no alignment is specified for the load instruction,
7616 // natural alignment is assumed.
7617 createReplacementValues(
7618 AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7619 *PrivatizableType, ACS,
7620 ACS.getCallArgOperand(ARI.getReplacedArg().getArgNo()),
7621 NewArgOperands);
7622 };
7623
7624 // Collect the types that will replace the privatizable type in the function
7625 // signature.
7626 SmallVector<Type *, 16> ReplacementTypes;
7627 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7628
7629 // Register a rewrite of the argument.
7630 if (A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes,
7631 std::move(FnRepairCB),
7632 std::move(ACSRepairCB)))
7633 return ChangeStatus::CHANGED;
7634 return ChangeStatus::UNCHANGED;
7635 }
7636
7637 void trackStatistics() const override {
7638 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7639 }
7640};
7641
7642struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7643 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7644 : AAPrivatizablePtrImpl(IRP, A) {}
7645
7646 void initialize(Attributor &A) override {
7647 // TODO: We can privatize more than arguments.
7648 indicatePessimisticFixpoint();
7649 }
7650
7651 ChangeStatus updateImpl(Attributor &A) override {
7652 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7653 "updateImpl will not be called");
7654 }
7655
7656 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7657 Value *Obj = getUnderlyingObject(&getAssociatedValue());
7658 if (!Obj) {
7659 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7660 return nullptr;
7661 }
7662
7663 if (auto *AI = dyn_cast<AllocaInst>(Obj))
7664 if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
7665 if (CI->isOne())
7666 return AI->getAllocatedType();
7667 if (auto *Arg = dyn_cast<Argument>(Obj)) {
7668 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7669 *this, IRPosition::argument(*Arg), DepClassTy::REQUIRED);
7670 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7671 return PrivArgAA->getPrivatizableType();
7672 }
7673
7674 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7675 "alloca nor privatizable argument: "
7676 << *Obj << "!\n");
7677 return nullptr;
7678 }
7679
7680 void trackStatistics() const override {
7681 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7682 }
7683};
7684
7685struct AAPrivatizablePtrCallSiteArgument final
7686 : public AAPrivatizablePtrFloating {
7687 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7688 : AAPrivatizablePtrFloating(IRP, A) {}
7689
7690 void initialize(Attributor &A) override {
7691 if (A.hasAttr(getIRPosition(), Attribute::ByVal))
7692 indicateOptimisticFixpoint();
7693 }
7694
7695 ChangeStatus updateImpl(Attributor &A) override {
7696 PrivatizableType = identifyPrivatizableType(A);
7697 if (!PrivatizableType)
7698 return ChangeStatus::UNCHANGED;
7699 if (!*PrivatizableType)
7700 return indicatePessimisticFixpoint();
7701
7702 const IRPosition &IRP = getIRPosition();
7703 bool IsKnownNoCapture;
7704 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7705 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
7706 if (!IsAssumedNoCapture) {
7707 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7708 return indicatePessimisticFixpoint();
7709 }
7710
7711 bool IsKnownNoAlias;
7713 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
7714 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7715 return indicatePessimisticFixpoint();
7716 }
7717
7718 bool IsKnown;
7719 if (!AA::isAssumedReadOnly(A, IRP, *this, IsKnown)) {
7720 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7721 return indicatePessimisticFixpoint();
7722 }
7723
7724 return ChangeStatus::UNCHANGED;
7725 }
7726
7727 void trackStatistics() const override {
7728 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7729 }
7730};
7731
7732struct AAPrivatizablePtrCallSiteReturned final
7733 : public AAPrivatizablePtrFloating {
7734 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7735 : AAPrivatizablePtrFloating(IRP, A) {}
7736
7737 void initialize(Attributor &A) override {
7738 // TODO: We can privatize more than arguments.
7739 indicatePessimisticFixpoint();
7740 }
7741
7742 void trackStatistics() const override {
7743 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7744 }
7745};
7746
7747struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7748 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7749 : AAPrivatizablePtrFloating(IRP, A) {}
7750
7751 void initialize(Attributor &A) override {
7752 // TODO: We can privatize more than arguments.
7753 indicatePessimisticFixpoint();
7754 }
7755
7756 void trackStatistics() const override {
7757 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7758 }
7759};
7760} // namespace
7761
7762/// -------------------- Memory Behavior Attributes ----------------------------
7763/// Includes read-none, read-only, and write-only.
7764/// ----------------------------------------------------------------------------
7765namespace {
7766struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7767 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7768 : AAMemoryBehavior(IRP, A) {}
7769
7770 void initialize(Attributor &A) override {
7771 intersectAssumedBits(BEST_STATE);
7772 getKnownStateFromValue(A, getIRPosition(), getState());
7773 AAMemoryBehavior::initialize(A);
7774 }
7775
7776 /// Return the memory behavior information encoded in the IR for \p IRP.
7777 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7778 BitIntegerState &State,
7779 bool IgnoreSubsumingPositions = false) {
7781 A.getAttrs(IRP, AttrKinds, Attrs, IgnoreSubsumingPositions);
7782 for (const Attribute &Attr : Attrs) {
7783 switch (Attr.getKindAsEnum()) {
7784 case Attribute::ReadNone:
7785 State.addKnownBits(NO_ACCESSES);
7786 break;
7787 case Attribute::ReadOnly:
7788 State.addKnownBits(NO_WRITES);
7789 break;
7790 case Attribute::WriteOnly:
7791 State.addKnownBits(NO_READS);
7792 break;
7793 default:
7794 llvm_unreachable("Unexpected attribute!");
7795 }
7796 }
7797
7798 // Only a call site anchor performs the accesses of the position. A floating
7799 // value is only defined by its anchor, e.g., a loaded pointer may be
7800 // written through even though the load does not write.
7802 if (Kind == IRPosition::IRP_CALL_SITE ||
7804 if (auto *CB = dyn_cast<CallBase>(&IRP.getAnchorValue())) {
7806 if (!isRefSet(MR))
7807 State.addKnownBits(NO_READS);
7808 if (!isModSet(MR))
7809 State.addKnownBits(NO_WRITES);
7810 }
7811 }
7812 }
7813
7814 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7815 SmallVectorImpl<Attribute> &Attrs) const override {
7816 assert(Attrs.size() == 0);
7817 if (isAssumedReadNone())
7818 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));
7819 else if (isAssumedReadOnly())
7820 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));
7821 else if (isAssumedWriteOnly())
7822 Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));
7823 assert(Attrs.size() <= 1);
7824 }
7825
7826 ChangeStatus manifest(Attributor &A) override {
7827 const IRPosition &IRP = getIRPosition();
7828
7829 if (A.hasAttr(IRP, Attribute::ReadNone,
7830 /* IgnoreSubsumingPositions */ true))
7831 return ChangeStatus::UNCHANGED;
7832
7833 // Check if we would improve the existing attributes first.
7834 SmallVector<Attribute, 4> DeducedAttrs;
7835 getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
7836 if (llvm::all_of(DeducedAttrs, [&](const Attribute &Attr) {
7837 return A.hasAttr(IRP, Attr.getKindAsEnum(),
7838 /* IgnoreSubsumingPositions */ true);
7839 }))
7840 return ChangeStatus::UNCHANGED;
7841
7842 // Clear existing attributes.
7843 A.removeAttrs(IRP, AttrKinds);
7844 // Clear conflicting writable attribute.
7845 if (isAssumedReadOnly())
7846 A.removeAttrs(IRP, Attribute::Writable);
7847
7848 // Use the generic manifest method.
7849 return IRAttribute::manifest(A);
7850 }
7851
7852 const std::string getAsStr(Attributor *A) const override {
7853 if (isAssumedReadNone())
7854 return "readnone";
7855 if (isAssumedReadOnly())
7856 return "readonly";
7857 if (isAssumedWriteOnly())
7858 return "writeonly";
7859 return "may-read/write";
7860 }
7861
7862 /// The set of IR attributes AAMemoryBehavior deals with.
7863 static const Attribute::AttrKind AttrKinds[3];
7864};
7865
7866const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
7867 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7868
7869/// Memory behavior attribute for a floating value.
7870struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7871 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
7872 : AAMemoryBehaviorImpl(IRP, A) {}
7873
7874 ChangeStatus updateImpl(Attributor &A) override;
7875
7876 void trackStatistics() const override {
7877 if (isAssumedReadNone())
7879 else if (isAssumedReadOnly())
7881 else if (isAssumedWriteOnly())
7883 }
7884
7885private:
7886 /// Return true if users of \p UserI might access the underlying
7887 /// variable/location described by \p U and should therefore be analyzed.
7888 bool followUsersOfUseIn(Attributor &A, const Use &U,
7889 const Instruction *UserI);
7890
7891 /// Update the state according to the effect of use \p U in \p UserI.
7892 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
7893};
7894
7895/// Memory behavior attribute for function argument.
7896struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
7897 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
7898 : AAMemoryBehaviorFloating(IRP, A) {}
7899
7900 void initialize(Attributor &A) override {
7901 intersectAssumedBits(BEST_STATE);
7902 const IRPosition &IRP = getIRPosition();
7903 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
7904 // can query it when we use has/getAttr. That would allow us to reuse the
7905 // initialize of the base class here.
7906 bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal},
7907 /* IgnoreSubsumingPositions */ true);
7908 getKnownStateFromValue(A, IRP, getState(),
7909 /* IgnoreSubsumingPositions */ HasByVal);
7910 }
7911
7912 ChangeStatus manifest(Attributor &A) override {
7913 // TODO: Pointer arguments are not supported on vectors of pointers yet.
7914 if (!getAssociatedValue().getType()->isPointerTy())
7915 return ChangeStatus::UNCHANGED;
7916
7917 // TODO: From readattrs.ll: "inalloca parameters are always
7918 // considered written"
7919 if (A.hasAttr(getIRPosition(),
7920 {Attribute::InAlloca, Attribute::Preallocated})) {
7921 removeKnownBits(NO_WRITES);
7922 removeAssumedBits(NO_WRITES);
7923 }
7924 A.removeAttrs(getIRPosition(), AttrKinds);
7925 return AAMemoryBehaviorFloating::manifest(A);
7926 }
7927
7928 void trackStatistics() const override {
7929 if (isAssumedReadNone())
7930 STATS_DECLTRACK_ARG_ATTR(readnone)
7931 else if (isAssumedReadOnly())
7932 STATS_DECLTRACK_ARG_ATTR(readonly)
7933 else if (isAssumedWriteOnly())
7934 STATS_DECLTRACK_ARG_ATTR(writeonly)
7935 }
7936};
7937
7938struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
7939 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
7940 : AAMemoryBehaviorArgument(IRP, A) {}
7941
7942 void initialize(Attributor &A) override {
7943 // If we don't have an associated attribute this is either a variadic call
7944 // or an indirect call, either way, nothing to do here.
7945 Argument *Arg = getAssociatedArgument();
7946 if (!Arg) {
7947 indicatePessimisticFixpoint();
7948 return;
7949 }
7950 if (Arg->hasByValAttr()) {
7951 addKnownBits(NO_WRITES);
7952 removeKnownBits(NO_READS);
7953 removeAssumedBits(NO_READS);
7954 }
7955 AAMemoryBehaviorArgument::initialize(A);
7956 if (getAssociatedFunction()->isDeclaration())
7957 indicatePessimisticFixpoint();
7958 }
7959
7960 ChangeStatus updateImpl(Attributor &A) override {
7961 // TODO: Once we have call site specific value information we can provide
7962 // call site specific liveness liveness information and then it makes
7963 // sense to specialize attributes for call sites arguments instead of
7964 // redirecting requests to the callee argument.
7965 Argument *Arg = getAssociatedArgument();
7966 const IRPosition &ArgPos = IRPosition::argument(*Arg);
7967 auto *ArgAA =
7968 A.getAAFor<AAMemoryBehavior>(*this, ArgPos, DepClassTy::REQUIRED);
7969 if (!ArgAA)
7970 return indicatePessimisticFixpoint();
7971 return clampStateAndIndicateChange(getState(), ArgAA->getState());
7972 }
7973
7974 void trackStatistics() const override {
7975 if (isAssumedReadNone())
7977 else if (isAssumedReadOnly())
7979 else if (isAssumedWriteOnly())
7981 }
7982};
7983
7984/// Memory behavior attribute for a call site return position.
7985struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
7986 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
7987 : AAMemoryBehaviorFloating(IRP, A) {}
7988
7989 void initialize(Attributor &A) override {
7990 AAMemoryBehaviorImpl::initialize(A);
7991 }
7992 ChangeStatus manifest(Attributor &A) override {
7993 // We do not annotate returned values.
7994 return ChangeStatus::UNCHANGED;
7995 }
7996
7997 void trackStatistics() const override {}
7998};
7999
8000/// An AA to represent the memory behavior function attributes.
8001struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
8002 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
8003 : AAMemoryBehaviorImpl(IRP, A) {}
8004
8005 ChangeStatus updateImpl(Attributor &A) override;
8006
8007 ChangeStatus manifest(Attributor &A) override {
8008 // TODO: It would be better to merge this with AAMemoryLocation, so that
8009 // we could determine read/write per location. This would also have the
8010 // benefit of only one place trying to manifest the memory attribute.
8011 Function &F = cast<Function>(getAnchorValue());
8013 if (isAssumedReadNone())
8014 ME = MemoryEffects::none();
8015 else if (isAssumedReadOnly())
8017 else if (isAssumedWriteOnly())
8019
8020 A.removeAttrs(getIRPosition(), AttrKinds);
8021 // Clear conflicting writable attribute.
8022 if (ME.onlyReadsMemory())
8023 for (Argument &Arg : F.args())
8024 A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);
8025 return A.manifestAttrs(getIRPosition(),
8026 Attribute::getWithMemoryEffects(F.getContext(), ME));
8027 }
8028
8029 void trackStatistics() const override {
8030 if (isAssumedReadNone())
8031 STATS_DECLTRACK_FN_ATTR(readnone)
8032 else if (isAssumedReadOnly())
8033 STATS_DECLTRACK_FN_ATTR(readonly)
8034 else if (isAssumedWriteOnly())
8035 STATS_DECLTRACK_FN_ATTR(writeonly)
8036 }
8037};
8038
8039/// AAMemoryBehavior attribute for call sites.
8040struct AAMemoryBehaviorCallSite final
8041 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8042 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8043 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8044
8045 ChangeStatus manifest(Attributor &A) override {
8046 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8047 CallBase &CB = cast<CallBase>(getAnchorValue());
8049 if (isAssumedReadNone())
8050 ME = MemoryEffects::none();
8051 else if (isAssumedReadOnly())
8053 else if (isAssumedWriteOnly())
8055
8056 A.removeAttrs(getIRPosition(), AttrKinds);
8057 // Clear conflicting writable attribute.
8058 if (ME.onlyReadsMemory())
8059 for (Use &U : CB.args())
8060 A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()),
8061 Attribute::Writable);
8062 return A.manifestAttrs(
8063 getIRPosition(), Attribute::getWithMemoryEffects(CB.getContext(), ME));
8064 }
8065
8066 void trackStatistics() const override {
8067 if (isAssumedReadNone())
8068 STATS_DECLTRACK_CS_ATTR(readnone)
8069 else if (isAssumedReadOnly())
8070 STATS_DECLTRACK_CS_ATTR(readonly)
8071 else if (isAssumedWriteOnly())
8072 STATS_DECLTRACK_CS_ATTR(writeonly)
8073 }
8074};
8075
8076ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8077
8078 // The current assumed state used to determine a change.
8079 auto AssumedState = getAssumed();
8080
8081 auto CheckRWInst = [&](Instruction &I) {
8082 // If the instruction has an own memory behavior state, use it to restrict
8083 // the local state. No further analysis is required as the other memory
8084 // state is as optimistic as it gets.
8085 if (const auto *CB = dyn_cast<CallBase>(&I)) {
8086 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8088 if (MemBehaviorAA) {
8089 intersectAssumedBits(MemBehaviorAA->getAssumed());
8090 return !isAtFixpoint();
8091 }
8092 }
8093
8094 // Remove access kind modifiers if necessary.
8095 if (I.mayReadFromMemory())
8096 removeAssumedBits(NO_READS);
8097 if (I.mayWriteToMemory())
8098 removeAssumedBits(NO_WRITES);
8099 return !isAtFixpoint();
8100 };
8101
8102 bool UsedAssumedInformation = false;
8103 if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
8104 UsedAssumedInformation))
8105 return indicatePessimisticFixpoint();
8106
8107 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8109}
8110
8111ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8112
8113 const IRPosition &IRP = getIRPosition();
8114 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8115 AAMemoryBehavior::StateType &S = getState();
8116
8117 // First, check the function scope. We take the known information and we avoid
8118 // work if the assumed information implies the current assumed information for
8119 // this attribute. This is a valid for all but byval arguments.
8120 Argument *Arg = IRP.getAssociatedArgument();
8121 AAMemoryBehavior::base_t FnMemAssumedState =
8123 if (!Arg || !Arg->hasByValAttr()) {
8124 const auto *FnMemAA =
8125 A.getAAFor<AAMemoryBehavior>(*this, FnPos, DepClassTy::OPTIONAL);
8126 if (FnMemAA) {
8127 FnMemAssumedState = FnMemAA->getAssumed();
8128 S.addKnownBits(FnMemAA->getKnown());
8129 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8131 }
8132 }
8133
8134 // The current assumed state used to determine a change.
8135 auto AssumedState = S.getAssumed();
8136
8137 // Make sure the value is not captured (except through "return"), if
8138 // it is, any information derived would be irrelevant anyway as we cannot
8139 // check the potential aliases introduced by the capture. However, no need
8140 // to fall back to anythign less optimistic than the function state.
8141 bool IsKnownNoCapture;
8142 const AANoCapture *ArgNoCaptureAA = nullptr;
8143 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
8144 A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,
8145 &ArgNoCaptureAA);
8146
8147 if (!IsAssumedNoCapture &&
8148 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8149 S.intersectAssumedBits(FnMemAssumedState);
8150 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8152 }
8153
8154 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8155 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8156 Instruction *UserI = cast<Instruction>(U.getUser());
8157 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8158 << " \n");
8159
8160 // Droppable users, e.g., llvm::assume does not actually perform any action.
8161 if (UserI->isDroppable())
8162 return true;
8163
8164 // Check if the users of UserI should also be visited.
8165 Follow = followUsersOfUseIn(A, U, UserI);
8166
8167 // If UserI might touch memory we analyze the use in detail.
8168 if (UserI->mayReadOrWriteMemory())
8169 analyzeUseIn(A, U, UserI);
8170
8171 return !isAtFixpoint();
8172 };
8173
8174 if (!A.checkForAllUses(UsePred, *this, getAssociatedValue()))
8175 return indicatePessimisticFixpoint();
8176
8177 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8179}
8180
8181bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8182 const Instruction *UserI) {
8183 // The loaded value is unrelated to the pointer argument, no need to
8184 // follow the users of the load.
8185 if (isa<LoadInst>(UserI) || isa<ReturnInst>(UserI))
8186 return false;
8187
8188 // By default we follow all uses assuming UserI might leak information on U,
8189 // we have special handling for call sites operands though.
8190 const auto *CB = dyn_cast<CallBase>(UserI);
8191 if (!CB || !CB->isArgOperand(&U))
8192 return true;
8193
8194 // If the use is a call argument known not to be captured, the users of
8195 // the call do not need to be visited because they have to be unrelated to
8196 // the input. Note that this check is not trivial even though we disallow
8197 // general capturing of the underlying argument. The reason is that the
8198 // call might the argument "through return", which we allow and for which we
8199 // need to check call users.
8200 if (U.get()->getType()->isPointerTy()) {
8201 unsigned ArgNo = CB->getArgOperandNo(&U);
8202 bool IsKnownNoCapture;
8204 A, this, IRPosition::callsite_argument(*CB, ArgNo),
8205 DepClassTy::OPTIONAL, IsKnownNoCapture);
8206 }
8207
8208 return true;
8209}
8210
8211void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8212 const Instruction *UserI) {
8213 assert(UserI->mayReadOrWriteMemory());
8214
8215 switch (UserI->getOpcode()) {
8216 default:
8217 // TODO: Handle all atomics and other side-effect operations we know of.
8218 break;
8219 case Instruction::Load:
8220 // Loads cause the NO_READS property to disappear.
8221 removeAssumedBits(NO_READS);
8222 return;
8223
8224 case Instruction::Store:
8225 // Stores cause the NO_WRITES property to disappear if the use is the
8226 // pointer operand. Note that while capturing was taken care of somewhere
8227 // else we need to deal with stores of the value that is not looked through.
8228 if (cast<StoreInst>(UserI)->getPointerOperand() == U.get())
8229 removeAssumedBits(NO_WRITES);
8230 else
8231 indicatePessimisticFixpoint();
8232 return;
8233
8234 case Instruction::Call:
8235 case Instruction::CallBr:
8236 case Instruction::Invoke: {
8237 // For call sites we look at the argument memory behavior attribute (this
8238 // could be recursive!) in order to restrict our own state.
8239 const auto *CB = cast<CallBase>(UserI);
8240
8241 // Give up on operand bundles.
8242 if (CB->isBundleOperand(&U)) {
8243 indicatePessimisticFixpoint();
8244 return;
8245 }
8246
8247 // Calling a function does read the function pointer, maybe write it if the
8248 // function is self-modifying.
8249 if (CB->isCallee(&U)) {
8250 removeAssumedBits(NO_READS);
8251 break;
8252 }
8253
8254 // Adjust the possible access behavior based on the information on the
8255 // argument.
8256 IRPosition Pos;
8257 if (U.get()->getType()->isPointerTy())
8259 else
8261 const auto *MemBehaviorAA =
8262 A.getAAFor<AAMemoryBehavior>(*this, Pos, DepClassTy::OPTIONAL);
8263 if (!MemBehaviorAA)
8264 break;
8265 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8266 // and at least "known".
8267 intersectAssumedBits(MemBehaviorAA->getAssumed());
8268 return;
8269 }
8270 };
8271
8272 // Generally, look at the "may-properties" and adjust the assumed state if we
8273 // did not trigger special handling before.
8274 if (UserI->mayReadFromMemory())
8275 removeAssumedBits(NO_READS);
8276 if (UserI->mayWriteToMemory())
8277 removeAssumedBits(NO_WRITES);
8278}
8279} // namespace
8280
8281/// -------------------- Memory Locations Attributes ---------------------------
8282/// Includes read-none, argmemonly, inaccessiblememonly,
8283/// inaccessiblememorargmemonly
8284/// ----------------------------------------------------------------------------
8285
8288 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8289 return "all memory";
8291 return "no memory";
8292 std::string S = "memory:";
8293 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8294 S += "stack,";
8295 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8296 S += "constant,";
8298 S += "internal global,";
8300 S += "external global,";
8301 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8302 S += "argument,";
8304 S += "inaccessible,";
8305 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8306 S += "malloced,";
8307 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8308 S += "unknown,";
8309 S.pop_back();
8310 return S;
8311}
8312
8313namespace {
8314struct AAMemoryLocationImpl : public AAMemoryLocation {
8315
8316 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8317 : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8318 AccessKind2Accesses.fill(nullptr);
8319 }
8320
8321 ~AAMemoryLocationImpl() override {
8322 // The AccessSets are allocated via a BumpPtrAllocator, we call
8323 // the destructor manually.
8324 for (AccessSet *AS : AccessKind2Accesses)
8325 if (AS)
8326 AS->~AccessSet();
8327 }
8328
8329 void initialize(Attributor &A) override {
8330 intersectAssumedBits(BEST_STATE);
8331 getKnownStateFromValue(A, getIRPosition(), getState());
8332 AAMemoryLocation::initialize(A);
8333 }
8334
8335 /// Return the memory behavior information encoded in the IR for \p IRP.
8336 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8337 BitIntegerState &State,
8338 bool IgnoreSubsumingPositions = false) {
8339 // For internal functions we ignore `argmemonly` and
8340 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8341 // constant propagation. It is unclear if this is the best way but it is
8342 // unlikely this will cause real performance problems. If we are deriving
8343 // attributes for the anchor function we even remove the attribute in
8344 // addition to ignoring it.
8345 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8346 // MemoryEffects::Other as a possible location.
8347 bool UseArgMemOnly = true;
8348 Function *AnchorFn = IRP.getAnchorScope();
8349 if (AnchorFn && A.isRunOn(*AnchorFn))
8350 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8351
8353 A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8354 for (const Attribute &Attr : Attrs) {
8355 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8356 MemoryEffects ME = Attr.getMemoryEffects();
8357 if (ME.doesNotAccessMemory()) {
8358 State.addKnownBits(NO_LOCAL_MEM | NO_CONST_MEM);
8359 continue;
8360 }
8361 if (ME.onlyAccessesInaccessibleMem()) {
8362 State.addKnownBits(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
8363 continue;
8364 }
8365 if (ME.onlyAccessesArgPointees()) {
8366 if (UseArgMemOnly)
8367 State.addKnownBits(inverseLocation(NO_ARGUMENT_MEM, true, true));
8368 else {
8369 // Remove location information, only keep read/write info.
8370 ME = MemoryEffects(ME.getModRef());
8371 A.manifestAttrs(IRP,
8372 Attribute::getWithMemoryEffects(
8373 IRP.getAnchorValue().getContext(), ME),
8374 /*ForceReplace*/ true);
8375 }
8376 continue;
8377 }
8379 if (UseArgMemOnly)
8380 State.addKnownBits(inverseLocation(
8381 NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
8382 else {
8383 // Remove location information, only keep read/write info.
8384 ME = MemoryEffects(ME.getModRef());
8385 A.manifestAttrs(IRP,
8386 Attribute::getWithMemoryEffects(
8387 IRP.getAnchorValue().getContext(), ME),
8388 /*ForceReplace*/ true);
8389 }
8390 continue;
8391 }
8392 }
8393 }
8394
8395 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8396 SmallVectorImpl<Attribute> &Attrs) const override {
8397 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8398 assert(Attrs.size() == 0);
8399 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8400 if (isAssumedReadNone())
8401 Attrs.push_back(
8402 Attribute::getWithMemoryEffects(Ctx, MemoryEffects::none()));
8403 else if (isAssumedInaccessibleMemOnly())
8404 Attrs.push_back(Attribute::getWithMemoryEffects(
8406 else if (isAssumedArgMemOnly())
8407 Attrs.push_back(
8408 Attribute::getWithMemoryEffects(Ctx, MemoryEffects::argMemOnly()));
8409 else if (isAssumedInaccessibleOrArgMemOnly())
8410 Attrs.push_back(Attribute::getWithMemoryEffects(
8412 }
8413 assert(Attrs.size() <= 1);
8414 }
8415
8416 ChangeStatus manifest(Attributor &A) override {
8417 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8418 // provide per-location modref information here.
8419 const IRPosition &IRP = getIRPosition();
8420
8421 SmallVector<Attribute, 1> DeducedAttrs;
8422 getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
8423 if (DeducedAttrs.size() != 1)
8424 return ChangeStatus::UNCHANGED;
8425 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8426
8427 // Clear conflicting writable attributes of the function arguments.
8429 !isModSet(ME.getModRef(IRMemLocation::ArgMem))) {
8430 for (Argument &Arg : IRP.getAnchorScope()->args())
8431 A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);
8432 }
8433
8434 return A.manifestAttrs(IRP, Attribute::getWithMemoryEffects(
8435 IRP.getAnchorValue().getContext(), ME));
8436 }
8437
8438 bool checkForAllAccessesToMemoryKind(
8439 function_ref<bool(const Instruction *, const Value *, AccessKind,
8440 MemoryLocationsKind)>
8441 Pred,
8442 MemoryLocationsKind RequestedMLK) const override {
8443 if (!isValidState())
8444 return false;
8445
8446 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8447 if (AssumedMLK == NO_LOCATIONS)
8448 return true;
8449
8450 unsigned Idx = 0;
8451 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8452 CurMLK *= 2, ++Idx) {
8453 if (CurMLK & RequestedMLK)
8454 continue;
8455
8456 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8457 for (const AccessInfo &AI : *Accesses)
8458 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8459 return false;
8460 }
8461
8462 return true;
8463 }
8464
8465 ChangeStatus indicatePessimisticFixpoint() override {
8466 // If we give up and indicate a pessimistic fixpoint this instruction will
8467 // become an access for all potential access kinds:
8468 // TODO: Add pointers for argmemonly and globals to improve the results of
8469 // checkForAllAccessesToMemoryKind.
8470 bool Changed = false;
8471 MemoryLocationsKind KnownMLK = getKnown();
8472 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
8473 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8474 if (!(CurMLK & KnownMLK))
8475 updateStateAndAccessesMap(getState(), CurMLK, I, nullptr, Changed,
8476 getAccessKindFromInst(I));
8477 return AAMemoryLocation::indicatePessimisticFixpoint();
8478 }
8479
8480protected:
8481 /// Helper struct to tie together an instruction that has a read or write
8482 /// effect with the pointer it accesses (if any).
8483 struct AccessInfo {
8484
8485 /// The instruction that caused the access.
8486 const Instruction *I;
8487
8488 /// The base pointer that is accessed, or null if unknown.
8489 const Value *Ptr;
8490
8491 /// The kind of access (read/write/read+write).
8493
8494 bool operator==(const AccessInfo &RHS) const {
8495 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8496 }
8497 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8498 if (LHS.I != RHS.I)
8499 return LHS.I < RHS.I;
8500 if (LHS.Ptr != RHS.Ptr)
8501 return LHS.Ptr < RHS.Ptr;
8502 if (LHS.Kind != RHS.Kind)
8503 return LHS.Kind < RHS.Kind;
8504 return false;
8505 }
8506 };
8507
8508 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8509 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8510 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8511 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8512 AccessKind2Accesses;
8513
8514 /// Categorize the pointer arguments of CB that might access memory in
8515 /// AccessedLoc and update the state and access map accordingly.
8516 void
8517 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8518 AAMemoryLocation::StateType &AccessedLocs,
8519 bool &Changed);
8520
8521 /// Return the kind(s) of location that may be accessed by \p V.
8523 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8524
8525 /// Return the access kind as determined by \p I.
8526 AccessKind getAccessKindFromInst(const Instruction *I) {
8527 AccessKind AK = READ_WRITE;
8528 if (I) {
8529 AK = I->mayReadFromMemory() ? READ : NONE;
8530 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8531 }
8532 return AK;
8533 }
8534
8535 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8536 /// an access of kind \p AK to a \p MLK memory location with the access
8537 /// pointer \p Ptr.
8538 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8539 MemoryLocationsKind MLK, const Instruction *I,
8540 const Value *Ptr, bool &Changed,
8541 AccessKind AK = READ_WRITE) {
8542
8543 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8544 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(MLK)];
8545 if (!Accesses)
8546 Accesses = new (Allocator) AccessSet();
8547 Changed |= Accesses->insert(AccessInfo{I, Ptr, AK}).second;
8548 if (MLK == NO_UNKOWN_MEM)
8549 MLK = NO_LOCATIONS;
8550 State.removeAssumedBits(MLK);
8551 }
8552
8553 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8554 /// arguments, and update the state and access map accordingly.
8555 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8556 AAMemoryLocation::StateType &State, bool &Changed,
8557 unsigned AccessAS = 0);
8558
8559 /// Used to allocate access sets.
8561};
8562
8563void AAMemoryLocationImpl::categorizePtrValue(
8564 Attributor &A, const Instruction &I, const Value &Ptr,
8565 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8566 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8567 << Ptr << " ["
8568 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8569
8570 auto Pred = [&](Value &Obj) {
8571 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8572 // TODO: recognize the TBAA used for constant accesses.
8573 MemoryLocationsKind MLK = NO_LOCATIONS;
8574
8575 // Filter accesses to constant (GPU) memory if we have an AS at the access
8576 // site or the object is known to actually have the associated AS.
8577 if (AA::isGPU(A.getModule())) {
8578 if (AA::isGPUConstantAddressSpace(A.getModule(), AccessAS) ||
8579 (AA::isGPUConstantAddressSpace(A.getModule(), ObjectAS) &&
8580 isIdentifiedObject(&Obj)))
8581 return true;
8582 }
8583
8584 if (isa<UndefValue>(&Obj))
8585 return true;
8586 if (isa<Argument>(&Obj)) {
8587 // TODO: For now we do not treat byval arguments as local copies performed
8588 // on the call edge, though, we should. To make that happen we need to
8589 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8590 // would also allow us to mark functions only accessing byval arguments as
8591 // readnone again, arguably their accesses have no effect outside of the
8592 // function, like accesses to allocas.
8593 MLK = NO_ARGUMENT_MEM;
8594 } else if (auto *GV = dyn_cast<GlobalValue>(&Obj)) {
8595 // Reading constant memory is not treated as a read "effect" by the
8596 // function attr pass so we won't neither. Constants defined by TBAA are
8597 // similar. (We know we do not write it because it is constant.)
8598 if (auto *GVar = dyn_cast<GlobalVariable>(GV))
8599 if (GVar->isConstant())
8600 return true;
8601
8602 if (GV->hasLocalLinkage())
8603 MLK = NO_GLOBAL_INTERNAL_MEM;
8604 else
8605 MLK = NO_GLOBAL_EXTERNAL_MEM;
8606 } else if (isa<ConstantPointerNull>(&Obj) && ObjectAS == AccessAS &&
8607 !NullPointerIsDefined(getAssociatedFunction(), AccessAS)) {
8608 // Accessing null is undefined, but not accessing a null pointer cast to
8609 // another address space, which may be a valid address there.
8610 return true;
8611 } else if (isa<AllocaInst>(&Obj)) {
8612 MLK = NO_LOCAL_MEM;
8613 } else if (const auto *CB = dyn_cast<CallBase>(&Obj)) {
8614 bool IsKnownNoAlias;
8617 IsKnownNoAlias))
8618 MLK = NO_MALLOCED_MEM;
8619 else
8620 MLK = NO_UNKOWN_MEM;
8621 } else {
8622 MLK = NO_UNKOWN_MEM;
8623 }
8624
8625 assert(MLK != NO_LOCATIONS && "No location specified!");
8626 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8627 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8628 updateStateAndAccessesMap(State, MLK, &I, &Obj, Changed,
8629 getAccessKindFromInst(&I));
8630
8631 return true;
8632 };
8633
8634 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8636 if (!AA || !AA->forallUnderlyingObjects(Pred, AA::Intraprocedural)) {
8637 LLVM_DEBUG(
8638 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8639 updateStateAndAccessesMap(State, NO_UNKOWN_MEM, &I, nullptr, Changed,
8640 getAccessKindFromInst(&I));
8641 return;
8642 }
8643
8644 LLVM_DEBUG(
8645 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8646 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8647}
8648
8649void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8650 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8651 bool &Changed) {
8652 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8653
8654 // Skip non-pointer arguments.
8655 const Value *ArgOp = CB.getArgOperand(ArgNo);
8656 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8657 continue;
8658
8659 // Skip readnone arguments.
8660 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8661 const auto *ArgOpMemLocationAA =
8662 A.getAAFor<AAMemoryBehavior>(*this, ArgOpIRP, DepClassTy::OPTIONAL);
8663
8664 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8665 continue;
8666
8667 // Categorize potentially accessed pointer arguments as if there was an
8668 // access instruction with them as pointer.
8669 categorizePtrValue(A, CB, *ArgOp, AccessedLocs, Changed);
8670 }
8671}
8672
8674AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8675 bool &Changed) {
8676 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8677 << I << "\n");
8678
8679 AAMemoryLocation::StateType AccessedLocs;
8680 AccessedLocs.intersectAssumedBits(NO_LOCATIONS);
8681
8682 if (auto *CB = dyn_cast<CallBase>(&I)) {
8683
8684 // First check if we assume any memory is access is visible.
8685 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8687 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8688 << " [" << CBMemLocationAA << "]\n");
8689 if (!CBMemLocationAA) {
8690 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr,
8691 Changed, getAccessKindFromInst(&I));
8692 return NO_UNKOWN_MEM;
8693 }
8694
8695 if (CBMemLocationAA->isAssumedReadNone())
8696 return NO_LOCATIONS;
8697
8698 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8699 updateStateAndAccessesMap(AccessedLocs, NO_INACCESSIBLE_MEM, &I, nullptr,
8700 Changed, getAccessKindFromInst(&I));
8701 return AccessedLocs.getAssumed();
8702 }
8703
8704 uint32_t CBAssumedNotAccessedLocs =
8705 CBMemLocationAA->getAssumedNotAccessedLocation();
8706
8707 // Set the argmemonly and global bit as we handle them separately below.
8708 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8709 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8710
8711 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8712 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8713 continue;
8714 updateStateAndAccessesMap(AccessedLocs, CurMLK, &I, nullptr, Changed,
8715 getAccessKindFromInst(&I));
8716 }
8717
8718 // Now handle global memory if it might be accessed. This is slightly tricky
8719 // as NO_GLOBAL_MEM has multiple bits set.
8720 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8721 if (HasGlobalAccesses) {
8722 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8723 AccessKind Kind, MemoryLocationsKind MLK) {
8724 updateStateAndAccessesMap(AccessedLocs, MLK, &I, Ptr, Changed,
8725 getAccessKindFromInst(&I));
8726 return true;
8727 };
8728 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8729 AccessPred, inverseLocation(NO_GLOBAL_MEM, false, false)))
8730 return AccessedLocs.getWorstState();
8731 }
8732
8733 LLVM_DEBUG(
8734 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8735 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8736
8737 // Now handle argument memory if it might be accessed.
8738 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8739 if (HasArgAccesses)
8740 categorizeArgumentPointerLocations(A, *CB, AccessedLocs, Changed);
8741
8742 LLVM_DEBUG(
8743 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8744 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8745
8746 return AccessedLocs.getAssumed();
8747 }
8748
8749 if (const Value *Ptr = getPointerOperand(&I, /* AllowVolatile */ true)) {
8750 LLVM_DEBUG(
8751 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8752 << I << " [" << *Ptr << "]\n");
8753 categorizePtrValue(A, I, *Ptr, AccessedLocs, Changed,
8754 Ptr->getType()->getPointerAddressSpace());
8755 return AccessedLocs.getAssumed();
8756 }
8757
8758 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8759 << I << "\n");
8760 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr, Changed,
8761 getAccessKindFromInst(&I));
8762 return AccessedLocs.getAssumed();
8763}
8764
8765/// An AA to represent the memory behavior function attributes.
8766struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8767 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8768 : AAMemoryLocationImpl(IRP, A) {}
8769
8770 ChangeStatus updateImpl(Attributor &A) override {
8771
8772 const auto *MemBehaviorAA =
8773 A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
8774 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8775 if (MemBehaviorAA->isKnownReadNone())
8776 return indicateOptimisticFixpoint();
8778 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8779 A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
8780 return ChangeStatus::UNCHANGED;
8781 }
8782
8783 // The current assumed state used to determine a change.
8784 auto AssumedState = getAssumed();
8785 bool Changed = false;
8786
8787 auto CheckRWInst = [&](Instruction &I) {
8788 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8789 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8790 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8791 removeAssumedBits(inverseLocation(MLK, false, false));
8792 // Stop once only the valid bit set in the *not assumed location*, thus
8793 // once we don't actually exclude any memory locations in the state.
8794 return getAssumedNotAccessedLocation() != VALID_STATE;
8795 };
8796
8797 bool UsedAssumedInformation = false;
8798 if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
8799 UsedAssumedInformation))
8800 return indicatePessimisticFixpoint();
8801
8802 Changed |= AssumedState != getAssumed();
8803 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8804 }
8805
8806 void trackStatistics() const override {
8807 if (isAssumedReadNone())
8808 STATS_DECLTRACK_FN_ATTR(readnone)
8809 else if (isAssumedArgMemOnly())
8810 STATS_DECLTRACK_FN_ATTR(argmemonly)
8811 else if (isAssumedInaccessibleMemOnly())
8812 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8813 else if (isAssumedInaccessibleOrArgMemOnly())
8814 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8815 }
8816};
8817
8818/// AAMemoryLocation attribute for call sites.
8819struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8820 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8821 : AAMemoryLocationImpl(IRP, A) {}
8822
8823 void initialize(Attributor &A) override {
8824 AAMemoryLocationImpl::initialize(A);
8825
8826 // Volatile calls may also access inaccessible memory, which the callee's
8827 // attributes do not say.
8828 auto &CB = cast<CallBase>(getAnchorValue());
8829 if (CB.isVolatile()) {
8830 bool Changed = false;
8831 removeKnownBits(NO_INACCESSIBLE_MEM);
8832 updateStateAndAccessesMap(getState(), NO_INACCESSIBLE_MEM, &CB, nullptr,
8833 Changed, getAccessKindFromInst(&CB));
8834 }
8835 }
8836
8837 ChangeStatus updateImpl(Attributor &A) override {
8838 // TODO: Once we have call site specific value information we can provide
8839 // call site specific liveness liveness information and then it makes
8840 // sense to specialize attributes for call sites arguments instead of
8841 // redirecting requests to the callee argument.
8842 Function *F = getAssociatedFunction();
8843 const IRPosition &FnPos = IRPosition::function(*F);
8844 auto *FnAA =
8845 A.getAAFor<AAMemoryLocation>(*this, FnPos, DepClassTy::REQUIRED);
8846 if (!FnAA)
8847 return indicatePessimisticFixpoint();
8848 bool Changed = false;
8849 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
8850 AccessKind Kind, MemoryLocationsKind MLK) {
8851 updateStateAndAccessesMap(getState(), MLK, I, Ptr, Changed,
8852 getAccessKindFromInst(I));
8853 return true;
8854 };
8855 if (!FnAA->checkForAllAccessesToMemoryKind(AccessPred, ALL_LOCATIONS))
8856 return indicatePessimisticFixpoint();
8857 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8858 }
8859
8860 void trackStatistics() const override {
8861 if (isAssumedReadNone())
8862 STATS_DECLTRACK_CS_ATTR(readnone)
8863 }
8864};
8865} // namespace
8866
8867/// ------------------ denormal-fp-math Attribute -------------------------
8868
8869namespace {
8870struct AADenormalFPMathImpl : public AADenormalFPMath {
8871 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
8872 : AADenormalFPMath(IRP, A) {}
8873
8874 const std::string getAsStr(Attributor *A) const override {
8875 std::string Str("AADenormalFPMath[");
8876 raw_string_ostream OS(Str);
8877
8878 DenormalState Known = getKnown();
8879 if (Known.Mode.isValid())
8880 OS << "denormal-fp-math=" << Known.Mode;
8881 else
8882 OS << "invalid";
8883
8884 if (Known.ModeF32.isValid())
8885 OS << " denormal-fp-math-f32=" << Known.ModeF32;
8886 OS << ']';
8887 return Str;
8888 }
8889};
8890
8891struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
8892 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
8893 : AADenormalFPMathImpl(IRP, A) {}
8894
8895 void initialize(Attributor &A) override {
8896 const Function *F = getAnchorScope();
8897 DenormalFPEnv DenormEnv = F->getDenormalFPEnv();
8898
8899 Known = DenormalState{DenormEnv.DefaultMode, DenormEnv.F32Mode};
8900 if (isModeFixed())
8901 indicateFixpoint();
8902 }
8903
8904 ChangeStatus updateImpl(Attributor &A) override {
8905 ChangeStatus Change = ChangeStatus::UNCHANGED;
8906
8907 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
8908 Function *Caller = CS.getInstruction()->getFunction();
8909 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
8910 << "->" << getAssociatedFunction()->getName() << '\n');
8911
8912 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
8913 *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
8914 if (!CallerInfo)
8915 return false;
8916
8917 Change = Change | clampStateAndIndicateChange(this->getState(),
8918 CallerInfo->getState());
8919 return true;
8920 };
8921
8922 bool AllCallSitesKnown = true;
8923 if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown))
8924 return indicatePessimisticFixpoint();
8925
8926 if (Change == ChangeStatus::CHANGED && isModeFixed())
8927 indicateFixpoint();
8928 return Change;
8929 }
8930
8931 ChangeStatus manifest(Attributor &A) override {
8932 LLVMContext &Ctx = getAssociatedFunction()->getContext();
8933
8934 SmallVector<Attribute, 2> AttrToAdd;
8936
8937 // TODO: Change to use DenormalFPEnv everywhere.
8938 DenormalFPEnv KnownEnv(Known.Mode, Known.ModeF32);
8939
8940 if (KnownEnv == DenormalFPEnv::getDefault()) {
8941 AttrToRemove.push_back(Attribute::DenormalFPEnv);
8942 } else {
8943 AttrToAdd.push_back(Attribute::get(
8944 Ctx, Attribute::DenormalFPEnv,
8945 DenormalFPEnv(Known.Mode, Known.ModeF32).toIntValue()));
8946 }
8947
8948 auto &IRP = getIRPosition();
8949
8950 // TODO: There should be a combined add and remove API.
8951 return A.removeAttrs(IRP, AttrToRemove) |
8952 A.manifestAttrs(IRP, AttrToAdd, /*ForceReplace=*/true);
8953 }
8954
8955 void trackStatistics() const override {
8956 STATS_DECLTRACK_FN_ATTR(denormal_fpenv)
8957 }
8958};
8959} // namespace
8960
8961/// ------------------ Value Constant Range Attribute -------------------------
8962
8963namespace {
8964struct AAValueConstantRangeImpl : AAValueConstantRange {
8965 using StateType = IntegerRangeState;
8966 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
8967 : AAValueConstantRange(IRP, A) {}
8968
8969 void initialize(Attributor &A) override {
8970 if (A.hasSimplificationCallback(getIRPosition())) {
8971 indicatePessimisticFixpoint();
8972 return;
8973 }
8974
8975 // Intersect a range given by SCEV.
8976 intersectKnown(getConstantRangeFromSCEV(A, getCtxI()));
8977
8978 // Intersect a range given by LVI.
8979 intersectKnown(getConstantRangeFromLVI(A, getCtxI()));
8980 }
8981
8982 const std::string getAsStr(Attributor *A) const override {
8983 std::string Str;
8984 llvm::raw_string_ostream OS(Str);
8985 OS << "range(" << getBitWidth() << ")<";
8986 getKnown().print(OS);
8987 OS << " / ";
8988 getAssumed().print(OS);
8989 OS << ">";
8990 return Str;
8991 }
8992
8993 /// Helper function to get a SCEV expr for the associated value at program
8994 /// point \p I.
8995 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
8996 if (!getAnchorScope())
8997 return nullptr;
8998
8999 ScalarEvolution *SE =
9000 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9001 *getAnchorScope());
9002
9003 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
9004 *getAnchorScope());
9005
9006 if (!SE || !LI)
9007 return nullptr;
9008
9009 const SCEV *S = SE->getSCEV(&getAssociatedValue());
9010 if (!I)
9011 return S;
9012
9013 return SE->getSCEVAtScope(S, LI->getLoopFor(I->getParent()));
9014 }
9015
9016 /// Helper function to get a range from SCEV for the associated value at
9017 /// program point \p I.
9018 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9019 const Instruction *I = nullptr) const {
9020 if (!getAnchorScope())
9021 return getWorstState(getBitWidth());
9022
9023 ScalarEvolution *SE =
9024 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9025 *getAnchorScope());
9026
9027 const SCEV *S = getSCEV(A, I);
9028 if (!SE || !S)
9029 return getWorstState(getBitWidth());
9030
9031 return SE->getUnsignedRange(S);
9032 }
9033
9034 /// Helper function to get a range from LVI for the associated value at
9035 /// program point \p I.
9036 ConstantRange
9037 getConstantRangeFromLVI(Attributor &A,
9038 const Instruction *CtxI = nullptr) const {
9039 if (!getAnchorScope())
9040 return getWorstState(getBitWidth());
9041
9042 LazyValueInfo *LVI =
9043 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9044 *getAnchorScope());
9045
9046 if (!LVI || !CtxI)
9047 return getWorstState(getBitWidth());
9048 return LVI->getConstantRange(&getAssociatedValue(),
9049 const_cast<Instruction *>(CtxI),
9050 /*UndefAllowed*/ false);
9051 }
9052
9053 /// Return true if \p CtxI is valid for querying outside analyses.
9054 /// This basically makes sure we do not ask intra-procedural analysis
9055 /// about a context in the wrong function or a context that violates
9056 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9057 /// if the original context of this AA is OK or should be considered invalid.
9058 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9059 const Instruction *CtxI,
9060 bool AllowAACtxI) const {
9061 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9062 return false;
9063
9064 // Our context might be in a different function, neither intra-procedural
9065 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9066 if (!AA::isValidInScope(getAssociatedValue(), CtxI->getFunction()))
9067 return false;
9068
9069 // If the context is not dominated by the value there are paths to the
9070 // context that do not define the value. This cannot be handled by
9071 // LazyValueInfo so we need to bail.
9072 if (auto *I = dyn_cast<Instruction>(&getAssociatedValue())) {
9073 InformationCache &InfoCache = A.getInfoCache();
9074 const DominatorTree *DT =
9075 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9076 *I->getFunction());
9077 return DT && DT->dominates(I, CtxI);
9078 }
9079
9080 return true;
9081 }
9082
9083 ConstantRange
9084 getAssumedConstantRange(Attributor &A,
9085 const Instruction *CtxI = nullptr) const override {
9086 // TODO: Make SCEV use Attributor assumption.
9087 // We may be able to bound a variable range via assumptions in
9088 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9089 // evolve to x^2 + x, then we can say that y is in [2, 12].
9090 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9091 /* AllowAACtxI */ false))
9092 return getAssumed();
9093
9094 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9095 ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
9096 return getAssumed().intersectWith(SCEVR).intersectWith(LVIR);
9097 }
9098
9099 /// Helper function to create MDNode for range metadata.
9100 static MDNode *
9101 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9102 const ConstantRange &AssumedConstantRange) {
9103 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(ConstantInt::get(
9104 Ty, AssumedConstantRange.getLower())),
9105 ConstantAsMetadata::get(ConstantInt::get(
9106 Ty, AssumedConstantRange.getUpper()))};
9107 return MDNode::get(Ctx, LowAndHigh);
9108 }
9109
9110 /// Return true if \p Assumed is included in ranges from instruction \p I.
9111 static bool isBetterRange(const ConstantRange &Assumed,
9112 const Instruction &I) {
9113 if (Assumed.isFullSet())
9114 return false;
9115
9116 std::optional<ConstantRange> Known;
9117
9118 if (const auto *CB = dyn_cast<CallBase>(&I)) {
9119 Known = CB->getRange();
9120 } else if (MDNode *KnownRanges = I.getMetadata(LLVMContext::MD_range)) {
9121 // If multiple ranges are annotated in IR, we give up to annotate assumed
9122 // range for now.
9123
9124 // TODO: If there exists a known range which containts assumed range, we
9125 // can say assumed range is better.
9126 if (KnownRanges->getNumOperands() > 2)
9127 return false;
9128
9129 ConstantInt *Lower =
9130 mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
9131 ConstantInt *Upper =
9132 mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
9133
9134 Known.emplace(Lower->getValue(), Upper->getValue());
9135 }
9136 return !Known || (*Known != Assumed && Known->contains(Assumed));
9137 }
9138
9139 /// Helper function to set range metadata.
9140 static bool
9141 setRangeMetadataIfisBetterRange(Instruction *I,
9142 const ConstantRange &AssumedConstantRange) {
9143 if (isBetterRange(AssumedConstantRange, *I)) {
9144 I->setMetadata(LLVMContext::MD_range,
9145 getMDNodeForConstantRange(I->getType(), I->getContext(),
9146 AssumedConstantRange));
9147 return true;
9148 }
9149 return false;
9150 }
9151 /// Helper function to set range return attribute.
9152 static bool
9153 setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
9154 Instruction *I,
9155 const ConstantRange &AssumedConstantRange) {
9156 if (isBetterRange(AssumedConstantRange, *I)) {
9157 A.manifestAttrs(IRP,
9158 Attribute::get(I->getContext(), Attribute::Range,
9159 AssumedConstantRange),
9160 /*ForceReplace*/ true);
9161 return true;
9162 }
9163 return false;
9164 }
9165
9166 ChangeStatus manifest(Attributor &A) override {
9167 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9168 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9169 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9170
9171 auto &V = getAssociatedValue();
9172 if (!AssumedConstantRange.isEmptySet() &&
9173 !AssumedConstantRange.isSingleElement()) {
9174 if (Instruction *I = dyn_cast<Instruction>(&V)) {
9175 assert(I == getCtxI() && "Should not annotate an instruction which is "
9176 "not the context instruction");
9177 if (isa<LoadInst>(I))
9178 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9179 Changed = ChangeStatus::CHANGED;
9180 if (isa<CallInst>(I))
9181 if (setRangeRetAttrIfisBetterRange(A, getIRPosition(), I,
9182 AssumedConstantRange))
9183 Changed = ChangeStatus::CHANGED;
9184 }
9185 }
9186
9187 return Changed;
9188 }
9189};
9190
9191struct AAValueConstantRangeArgument final
9192 : AAArgumentFromCallSiteArguments<
9193 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9194 true /* BridgeCallBaseContext */> {
9195 using Base = AAArgumentFromCallSiteArguments<
9196 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9197 true /* BridgeCallBaseContext */>;
9198 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9199 : Base(IRP, A) {}
9200
9201 void trackStatistics() const override {
9202 STATS_DECLTRACK_ARG_ATTR(value_range)
9203 }
9204};
9205
9206struct AAValueConstantRangeReturned
9207 : AAReturnedFromReturnedValues<AAValueConstantRange,
9208 AAValueConstantRangeImpl,
9209 AAValueConstantRangeImpl::StateType,
9210 /* PropagateCallBaseContext */ true> {
9211 using Base =
9212 AAReturnedFromReturnedValues<AAValueConstantRange,
9213 AAValueConstantRangeImpl,
9214 AAValueConstantRangeImpl::StateType,
9215 /* PropagateCallBaseContext */ true>;
9216 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9217 : Base(IRP, A) {}
9218
9219 void initialize(Attributor &A) override {
9220 if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
9221 indicatePessimisticFixpoint();
9222 }
9223
9224 void trackStatistics() const override {
9225 STATS_DECLTRACK_FNRET_ATTR(value_range)
9226 }
9227};
9228
9229struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9230 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9231 : AAValueConstantRangeImpl(IRP, A) {}
9232
9233 void initialize(Attributor &A) override {
9234 AAValueConstantRangeImpl::initialize(A);
9235 if (isAtFixpoint())
9236 return;
9237
9238 Value &V = getAssociatedValue();
9239
9240 if (auto *C = dyn_cast<ConstantInt>(&V)) {
9241 unionAssumed(ConstantRange(C->getValue()));
9242 indicateOptimisticFixpoint();
9243 return;
9244 }
9245
9246 if (isa<UndefValue>(&V)) {
9247 // Collapse the undef state to 0.
9248 unionAssumed(ConstantRange(APInt(getBitWidth(), 0)));
9249 indicateOptimisticFixpoint();
9250 return;
9251 }
9252
9253 if (isa<CallBase>(&V))
9254 return;
9255
9257 return;
9258
9259 // If it is a load instruction with range metadata, use it.
9260 if (LoadInst *LI = dyn_cast<LoadInst>(&V))
9261 if (auto *RangeMD = LI->getMetadata(LLVMContext::MD_range)) {
9262 intersectKnown(getConstantRangeFromMetadata(*RangeMD));
9263 return;
9264 }
9265
9266 // We can work with PHI and select instruction as we traverse their operands
9267 // during update.
9269 return;
9270
9271 // Otherwise we give up.
9272 indicatePessimisticFixpoint();
9273
9274 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9275 << getAssociatedValue() << "\n");
9276 }
9277
9278 bool calculateBinaryOperator(
9279 Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9280 const Instruction *CtxI,
9281 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9282 Value *LHS = BinOp->getOperand(0);
9283 Value *RHS = BinOp->getOperand(1);
9284
9285 // Simplify the operands first.
9286 bool UsedAssumedInformation = false;
9287 const auto &SimplifiedLHS = A.getAssumedSimplified(
9288 IRPosition::value(*LHS, getCallBaseContext()), *this,
9289 UsedAssumedInformation, AA::Interprocedural);
9290 if (!SimplifiedLHS.has_value())
9291 return true;
9292 if (!*SimplifiedLHS)
9293 return false;
9294 LHS = *SimplifiedLHS;
9295
9296 const auto &SimplifiedRHS = A.getAssumedSimplified(
9297 IRPosition::value(*RHS, getCallBaseContext()), *this,
9298 UsedAssumedInformation, AA::Interprocedural);
9299 if (!SimplifiedRHS.has_value())
9300 return true;
9301 if (!*SimplifiedRHS)
9302 return false;
9303 RHS = *SimplifiedRHS;
9304
9305 // TODO: Allow non integers as well.
9306 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9307 return false;
9308
9309 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9310 *this, IRPosition::value(*LHS, getCallBaseContext()),
9311 DepClassTy::REQUIRED);
9312 if (!LHSAA)
9313 return false;
9314 QuerriedAAs.push_back(LHSAA);
9315 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9316
9317 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9318 *this, IRPosition::value(*RHS, getCallBaseContext()),
9319 DepClassTy::REQUIRED);
9320 if (!RHSAA)
9321 return false;
9322 QuerriedAAs.push_back(RHSAA);
9323 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9324
9325 auto AssumedRange = LHSAARange.binaryOp(BinOp->getOpcode(), RHSAARange);
9326
9327 T.unionAssumed(AssumedRange);
9328
9329 // TODO: Track a known state too.
9330
9331 return T.isValidState();
9332 }
9333
9334 bool calculateCastInst(
9335 Attributor &A, CastInst *CastI, IntegerRangeState &T,
9336 const Instruction *CtxI,
9337 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9338 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9339 // TODO: Allow non integers as well.
9340 Value *OpV = CastI->getOperand(0);
9341
9342 // Simplify the operand first.
9343 bool UsedAssumedInformation = false;
9344 const auto &SimplifiedOpV = A.getAssumedSimplified(
9345 IRPosition::value(*OpV, getCallBaseContext()), *this,
9346 UsedAssumedInformation, AA::Interprocedural);
9347 if (!SimplifiedOpV.has_value())
9348 return true;
9349 if (!*SimplifiedOpV)
9350 return false;
9351 OpV = *SimplifiedOpV;
9352
9353 if (!OpV->getType()->isIntegerTy())
9354 return false;
9355
9356 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9357 *this, IRPosition::value(*OpV, getCallBaseContext()),
9358 DepClassTy::REQUIRED);
9359 if (!OpAA)
9360 return false;
9361 QuerriedAAs.push_back(OpAA);
9362 T.unionAssumed(OpAA->getAssumed().castOp(CastI->getOpcode(),
9363 getState().getBitWidth()));
9364 return T.isValidState();
9365 }
9366
9367 bool
9368 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9369 const Instruction *CtxI,
9370 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9371 Value *LHS = CmpI->getOperand(0);
9372 Value *RHS = CmpI->getOperand(1);
9373
9374 // Simplify the operands first.
9375 bool UsedAssumedInformation = false;
9376 const auto &SimplifiedLHS = A.getAssumedSimplified(
9377 IRPosition::value(*LHS, getCallBaseContext()), *this,
9378 UsedAssumedInformation, AA::Interprocedural);
9379 if (!SimplifiedLHS.has_value())
9380 return true;
9381 if (!*SimplifiedLHS)
9382 return false;
9383 LHS = *SimplifiedLHS;
9384
9385 const auto &SimplifiedRHS = A.getAssumedSimplified(
9386 IRPosition::value(*RHS, getCallBaseContext()), *this,
9387 UsedAssumedInformation, AA::Interprocedural);
9388 if (!SimplifiedRHS.has_value())
9389 return true;
9390 if (!*SimplifiedRHS)
9391 return false;
9392 RHS = *SimplifiedRHS;
9393
9394 // TODO: Allow non integers as well.
9395 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9396 return false;
9397
9398 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9399 *this, IRPosition::value(*LHS, getCallBaseContext()),
9400 DepClassTy::REQUIRED);
9401 if (!LHSAA)
9402 return false;
9403 QuerriedAAs.push_back(LHSAA);
9404 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9405 *this, IRPosition::value(*RHS, getCallBaseContext()),
9406 DepClassTy::REQUIRED);
9407 if (!RHSAA)
9408 return false;
9409 QuerriedAAs.push_back(RHSAA);
9410 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9411 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9412
9413 // If one of them is empty set, we can't decide.
9414 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9415 return true;
9416
9417 bool MustTrue = false, MustFalse = false;
9418
9419 auto AllowedRegion =
9421
9422 if (AllowedRegion.intersectWith(LHSAARange).isEmptySet())
9423 MustFalse = true;
9424
9425 if (LHSAARange.icmp(CmpI->getPredicate(), RHSAARange))
9426 MustTrue = true;
9427
9428 assert((!MustTrue || !MustFalse) &&
9429 "Either MustTrue or MustFalse should be false!");
9430
9431 if (MustTrue)
9432 T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9433 else if (MustFalse)
9434 T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9435 else
9436 T.unionAssumed(ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9437
9438 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9439 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9440 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9441 << *RHSAA);
9442
9443 // TODO: Track a known state too.
9444 return T.isValidState();
9445 }
9446
9447 ChangeStatus updateImpl(Attributor &A) override {
9448
9449 IntegerRangeState T(getBitWidth());
9450 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9452 if (!I || isa<CallBase>(I)) {
9453
9454 // Simplify the operand first.
9455 bool UsedAssumedInformation = false;
9456 const auto &SimplifiedOpV = A.getAssumedSimplified(
9457 IRPosition::value(V, getCallBaseContext()), *this,
9458 UsedAssumedInformation, AA::Interprocedural);
9459 if (!SimplifiedOpV.has_value())
9460 return true;
9461 if (!*SimplifiedOpV)
9462 return false;
9463 Value *VPtr = *SimplifiedOpV;
9464
9465 // If the value is not instruction, we query AA to Attributor.
9466 const auto *AA = A.getAAFor<AAValueConstantRange>(
9467 *this, IRPosition::value(*VPtr, getCallBaseContext()),
9468 DepClassTy::REQUIRED);
9469
9470 // Clamp operator is not used to utilize a program point CtxI.
9471 if (AA)
9472 T.unionAssumed(AA->getAssumedConstantRange(A, CtxI));
9473 else
9474 return false;
9475
9476 return T.isValidState();
9477 }
9478
9480 if (auto *BinOp = dyn_cast<BinaryOperator>(I)) {
9481 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9482 return false;
9483 } else if (auto *CmpI = dyn_cast<CmpInst>(I)) {
9484 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9485 return false;
9486 } else if (auto *CastI = dyn_cast<CastInst>(I)) {
9487 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9488 return false;
9489 } else {
9490 // Give up with other instructions.
9491 // TODO: Add other instructions
9492
9493 T.indicatePessimisticFixpoint();
9494 return false;
9495 }
9496
9497 // Catch circular reasoning in a pessimistic way for now.
9498 // TODO: Check how the range evolves and if we stripped anything, see also
9499 // AADereferenceable or AAAlign for similar situations.
9500 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9501 if (QueriedAA != this)
9502 continue;
9503 // If we are in a stady state we do not need to worry.
9504 if (T.getAssumed() == getState().getAssumed())
9505 continue;
9506 T.indicatePessimisticFixpoint();
9507 }
9508
9509 return T.isValidState();
9510 };
9511
9512 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9513 return indicatePessimisticFixpoint();
9514
9515 // Ensure that long def-use chains can't cause circular reasoning either by
9516 // introducing a cutoff below.
9517 if (clampStateAndIndicateChange(getState(), T) == ChangeStatus::UNCHANGED)
9518 return ChangeStatus::UNCHANGED;
9519 if (++NumChanges > MaxNumChanges) {
9520 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9521 << " but only " << MaxNumChanges
9522 << " are allowed to avoid cyclic reasoning.");
9523 return indicatePessimisticFixpoint();
9524 }
9525 return ChangeStatus::CHANGED;
9526 }
9527
9528 void trackStatistics() const override {
9530 }
9531
9532 /// Tracker to bail after too many widening steps of the constant range.
9533 int NumChanges = 0;
9534
9535 /// Upper bound for the number of allowed changes (=widening steps) for the
9536 /// constant range before we give up.
9537 static constexpr int MaxNumChanges = 5;
9538};
9539
9540struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9541 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9542 : AAValueConstantRangeImpl(IRP, A) {}
9543
9544 ChangeStatus updateImpl(Attributor &A) override {
9545 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9546 "not be called");
9547 }
9548
9549 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9550};
9551
9552struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9553 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9554 : AAValueConstantRangeFunction(IRP, A) {}
9555
9556 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9557};
9558
9559struct AAValueConstantRangeCallSiteReturned
9560 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9561 AAValueConstantRangeImpl::StateType,
9562 /* IntroduceCallBaseContext */ true> {
9563 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9564 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9565 AAValueConstantRangeImpl::StateType,
9566 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9567
9568 void initialize(Attributor &A) override {
9569 // If it is a call instruction with range attribute, use the range.
9570 if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue())) {
9571 if (std::optional<ConstantRange> Range = CI->getRange())
9572 intersectKnown(*Range);
9573 }
9574
9575 AAValueConstantRangeImpl::initialize(A);
9576 }
9577
9578 void trackStatistics() const override {
9579 STATS_DECLTRACK_CSRET_ATTR(value_range)
9580 }
9581};
9582struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9583 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9584 : AAValueConstantRangeFloating(IRP, A) {}
9585
9586 ChangeStatus manifest(Attributor &A) override {
9587 return ChangeStatus::UNCHANGED;
9588 }
9589
9590 void trackStatistics() const override {
9591 STATS_DECLTRACK_CSARG_ATTR(value_range)
9592 }
9593};
9594} // namespace
9595
9596/// ------------------ Potential Values Attribute -------------------------
9597
9599 Attributor &A)
9600 : Base(IRP, true, A.Opts.attributor_max_potential_values) {}
9601
9602namespace {
9603struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9604 using StateType = PotentialConstantIntValuesState;
9605
9606 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9607 : AAPotentialConstantValues(IRP, A) {}
9608
9609 void initialize(Attributor &A) override {
9610 if (A.hasSimplificationCallback(getIRPosition()))
9611 indicatePessimisticFixpoint();
9612 else
9614 }
9615
9616 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9617 bool &ContainsUndef, bool ForSelf) {
9619 bool UsedAssumedInformation = false;
9620 if (!A.getAssumedSimplifiedValues(IRP, *this, Values, AA::Interprocedural,
9621 UsedAssumedInformation)) {
9622 // Avoid recursion when the caller is computing constant values for this
9623 // IRP itself.
9624 if (ForSelf)
9625 return false;
9626 if (!IRP.getAssociatedType()->isIntegerTy())
9627 return false;
9628 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9629 *this, IRP, DepClassTy::REQUIRED);
9630 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9631 return false;
9632 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9633 S = PotentialValuesAA->getState().getAssumedSet();
9634 return true;
9635 }
9636
9637 // Copy all the constant values, except UndefValue. ContainsUndef is true
9638 // iff Values contains only UndefValue instances. If there are other known
9639 // constants, then UndefValue is dropped.
9640 ContainsUndef = false;
9641 for (auto &It : Values) {
9642 if (isa<UndefValue>(It.getValue())) {
9643 ContainsUndef = true;
9644 continue;
9645 }
9646 auto *CI = dyn_cast<ConstantInt>(It.getValue());
9647 if (!CI)
9648 return false;
9649 S.insert(CI->getValue());
9650 }
9651 ContainsUndef &= S.empty();
9652
9653 return true;
9654 }
9655
9656 const std::string getAsStr(Attributor *A) const override {
9657 std::string Str;
9658 llvm::raw_string_ostream OS(Str);
9659 OS << getState();
9660 return Str;
9661 }
9662
9663 ChangeStatus updateImpl(Attributor &A) override {
9664 return indicatePessimisticFixpoint();
9665 }
9666};
9667
9668struct AAPotentialConstantValuesArgument final
9669 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9670 AAPotentialConstantValuesImpl,
9671 PotentialConstantIntValuesState> {
9672 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9673 AAPotentialConstantValuesImpl,
9675 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9676 : Base(IRP, A) {}
9677
9678 void trackStatistics() const override {
9679 STATS_DECLTRACK_ARG_ATTR(potential_values)
9680 }
9681};
9682
9683struct AAPotentialConstantValuesReturned
9684 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9685 AAPotentialConstantValuesImpl> {
9686 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9687 AAPotentialConstantValuesImpl>;
9688 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9689 : Base(IRP, A) {}
9690
9691 void initialize(Attributor &A) override {
9692 if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
9693 indicatePessimisticFixpoint();
9694 Base::initialize(A);
9695 }
9696
9697 void trackStatistics() const override {
9698 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9699 }
9700};
9701
9702struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9703 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9704 : AAPotentialConstantValuesImpl(IRP, A) {}
9705
9706 void initialize(Attributor &A) override {
9707 AAPotentialConstantValuesImpl::initialize(A);
9708 if (isAtFixpoint())
9709 return;
9710
9711 Value &V = getAssociatedValue();
9712
9713 if (auto *C = dyn_cast<ConstantInt>(&V)) {
9714 unionAssumed(C->getValue());
9715 indicateOptimisticFixpoint();
9716 return;
9717 }
9718
9719 if (isa<UndefValue>(&V)) {
9720 unionAssumedWithUndef();
9721 indicateOptimisticFixpoint();
9722 return;
9723 }
9724
9726 return;
9727
9729 return;
9730
9731 indicatePessimisticFixpoint();
9732
9733 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9734 << getAssociatedValue() << "\n");
9735 }
9736
9737 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9738 const APInt &RHS) {
9739 return ICmpInst::compare(LHS, RHS, ICI->getPredicate());
9740 }
9741
9742 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9743 uint32_t ResultBitWidth) {
9744 Instruction::CastOps CastOp = CI->getOpcode();
9745 switch (CastOp) {
9746 default:
9747 llvm_unreachable("unsupported or not integer cast");
9748 case Instruction::Trunc:
9749 return Src.trunc(ResultBitWidth);
9750 case Instruction::SExt:
9751 return Src.sext(ResultBitWidth);
9752 case Instruction::ZExt:
9753 return Src.zext(ResultBitWidth);
9754 case Instruction::BitCast:
9755 return Src;
9756 }
9757 }
9758
9759 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9760 const APInt &LHS, const APInt &RHS,
9761 bool &SkipOperation, bool &Unsupported) {
9762 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9763 // Unsupported is set to true when the binary operator is not supported.
9764 // SkipOperation is set to true when UB occur with the given operand pair
9765 // (LHS, RHS).
9766 // TODO: we should look at nsw and nuw keywords to handle operations
9767 // that create poison or undef value.
9768 switch (BinOpcode) {
9769 default:
9770 Unsupported = true;
9771 return LHS;
9772 case Instruction::Add:
9773 return LHS + RHS;
9774 case Instruction::Sub:
9775 return LHS - RHS;
9776 case Instruction::Mul:
9777 return LHS * RHS;
9778 case Instruction::UDiv:
9779 if (RHS.isZero()) {
9780 SkipOperation = true;
9781 return LHS;
9782 }
9783 return LHS.udiv(RHS);
9784 case Instruction::SDiv:
9785 if (RHS.isZero()) {
9786 SkipOperation = true;
9787 return LHS;
9788 }
9789 return LHS.sdiv(RHS);
9790 case Instruction::URem:
9791 if (RHS.isZero()) {
9792 SkipOperation = true;
9793 return LHS;
9794 }
9795 return LHS.urem(RHS);
9796 case Instruction::SRem:
9797 if (RHS.isZero()) {
9798 SkipOperation = true;
9799 return LHS;
9800 }
9801 return LHS.srem(RHS);
9802 case Instruction::Shl:
9803 return LHS.shl(RHS);
9804 case Instruction::LShr:
9805 return LHS.lshr(RHS);
9806 case Instruction::AShr:
9807 return LHS.ashr(RHS);
9808 case Instruction::And:
9809 return LHS & RHS;
9810 case Instruction::Or:
9811 return LHS | RHS;
9812 case Instruction::Xor:
9813 return LHS ^ RHS;
9814 }
9815 }
9816
9817 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
9818 const APInt &LHS, const APInt &RHS) {
9819 bool SkipOperation = false;
9820 bool Unsupported = false;
9821 APInt Result =
9822 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
9823 if (Unsupported)
9824 return false;
9825 // If SkipOperation is true, we can ignore this operand pair (L, R).
9826 if (!SkipOperation)
9827 unionAssumed(Result);
9828 return isValidState();
9829 }
9830
9831 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
9832 auto AssumedBefore = getAssumed();
9833 Value *LHS = ICI->getOperand(0);
9834 Value *RHS = ICI->getOperand(1);
9835
9836 bool LHSContainsUndef = false, RHSContainsUndef = false;
9837 SetTy LHSAAPVS, RHSAAPVS;
9838 if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9839 LHSContainsUndef, /* ForSelf */ false) ||
9840 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9841 RHSContainsUndef, /* ForSelf */ false))
9842 return indicatePessimisticFixpoint();
9843
9844 // TODO: make use of undef flag to limit potential values aggressively.
9845 bool MaybeTrue = false, MaybeFalse = false;
9846 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
9847 if (LHSContainsUndef && RHSContainsUndef) {
9848 // The result of any comparison between undefs can be soundly replaced
9849 // with undef.
9850 unionAssumedWithUndef();
9851 } else if (LHSContainsUndef) {
9852 for (const APInt &R : RHSAAPVS) {
9853 bool CmpResult = calculateICmpInst(ICI, Zero, R);
9854 MaybeTrue |= CmpResult;
9855 MaybeFalse |= !CmpResult;
9856 if (MaybeTrue & MaybeFalse)
9857 return indicatePessimisticFixpoint();
9858 }
9859 } else if (RHSContainsUndef) {
9860 for (const APInt &L : LHSAAPVS) {
9861 bool CmpResult = calculateICmpInst(ICI, L, Zero);
9862 MaybeTrue |= CmpResult;
9863 MaybeFalse |= !CmpResult;
9864 if (MaybeTrue & MaybeFalse)
9865 return indicatePessimisticFixpoint();
9866 }
9867 } else {
9868 for (const APInt &L : LHSAAPVS) {
9869 for (const APInt &R : RHSAAPVS) {
9870 bool CmpResult = calculateICmpInst(ICI, L, R);
9871 MaybeTrue |= CmpResult;
9872 MaybeFalse |= !CmpResult;
9873 if (MaybeTrue & MaybeFalse)
9874 return indicatePessimisticFixpoint();
9875 }
9876 }
9877 }
9878 if (MaybeTrue)
9879 unionAssumed(APInt(/* numBits */ 1, /* val */ 1));
9880 if (MaybeFalse)
9881 unionAssumed(APInt(/* numBits */ 1, /* val */ 0));
9882 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9883 : ChangeStatus::CHANGED;
9884 }
9885
9886 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
9887 auto AssumedBefore = getAssumed();
9888 Value *LHS = SI->getTrueValue();
9889 Value *RHS = SI->getFalseValue();
9890
9891 bool UsedAssumedInformation = false;
9892 std::optional<Constant *> C = A.getAssumedConstant(
9893 *SI->getCondition(), *this, UsedAssumedInformation);
9894
9895 // Check if we only need one operand.
9896 bool OnlyLeft = false, OnlyRight = false;
9897 if (C && *C && (*C)->isOneValue())
9898 OnlyLeft = true;
9899 else if (C && *C && (*C)->isNullValue())
9900 OnlyRight = true;
9901
9902 bool LHSContainsUndef = false, RHSContainsUndef = false;
9903 SetTy LHSAAPVS, RHSAAPVS;
9904 if (!OnlyRight &&
9905 !fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9906 LHSContainsUndef, /* ForSelf */ false))
9907 return indicatePessimisticFixpoint();
9908
9909 if (!OnlyLeft &&
9910 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9911 RHSContainsUndef, /* ForSelf */ false))
9912 return indicatePessimisticFixpoint();
9913
9914 if (OnlyLeft || OnlyRight) {
9915 // select (true/false), lhs, rhs
9916 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
9917 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
9918
9919 if (Undef)
9920 unionAssumedWithUndef();
9921 else {
9922 for (const auto &It : *OpAA)
9923 unionAssumed(It);
9924 }
9925
9926 } else if (LHSContainsUndef && RHSContainsUndef) {
9927 // select i1 *, undef , undef => undef
9928 unionAssumedWithUndef();
9929 } else {
9930 for (const auto &It : LHSAAPVS)
9931 unionAssumed(It);
9932 for (const auto &It : RHSAAPVS)
9933 unionAssumed(It);
9934 }
9935 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9936 : ChangeStatus::CHANGED;
9937 }
9938
9939 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
9940 auto AssumedBefore = getAssumed();
9941 if (!CI->isIntegerCast())
9942 return indicatePessimisticFixpoint();
9943 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
9944 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
9945 Value *Src = CI->getOperand(0);
9946
9947 bool SrcContainsUndef = false;
9948 SetTy SrcPVS;
9949 if (!fillSetWithConstantValues(A, IRPosition::value(*Src), SrcPVS,
9950 SrcContainsUndef, /* ForSelf */ false))
9951 return indicatePessimisticFixpoint();
9952
9953 if (SrcContainsUndef)
9954 unionAssumedWithUndef();
9955 else {
9956 for (const APInt &S : SrcPVS) {
9957 APInt T = calculateCastInst(CI, S, ResultBitWidth);
9958 unionAssumed(T);
9959 }
9960 }
9961 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9962 : ChangeStatus::CHANGED;
9963 }
9964
9965 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
9966 auto AssumedBefore = getAssumed();
9967 Value *LHS = BinOp->getOperand(0);
9968 Value *RHS = BinOp->getOperand(1);
9969
9970 bool LHSContainsUndef = false, RHSContainsUndef = false;
9971 SetTy LHSAAPVS, RHSAAPVS;
9972 if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9973 LHSContainsUndef, /* ForSelf */ false) ||
9974 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9975 RHSContainsUndef, /* ForSelf */ false))
9976 return indicatePessimisticFixpoint();
9977
9978 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
9979
9980 // TODO: make use of undef flag to limit potential values aggressively.
9981 if (LHSContainsUndef && RHSContainsUndef) {
9982 if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, Zero))
9983 return indicatePessimisticFixpoint();
9984 } else if (LHSContainsUndef) {
9985 for (const APInt &R : RHSAAPVS) {
9986 if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, R))
9987 return indicatePessimisticFixpoint();
9988 }
9989 } else if (RHSContainsUndef) {
9990 for (const APInt &L : LHSAAPVS) {
9991 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, Zero))
9992 return indicatePessimisticFixpoint();
9993 }
9994 } else {
9995 for (const APInt &L : LHSAAPVS) {
9996 for (const APInt &R : RHSAAPVS) {
9997 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, R))
9998 return indicatePessimisticFixpoint();
9999 }
10000 }
10001 }
10002 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10003 : ChangeStatus::CHANGED;
10004 }
10005
10006 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10007 auto AssumedBefore = getAssumed();
10008 SetTy Incoming;
10009 bool ContainsUndef;
10010 if (!fillSetWithConstantValues(A, IRPosition::value(*Inst), Incoming,
10011 ContainsUndef, /* ForSelf */ true))
10012 return indicatePessimisticFixpoint();
10013 if (ContainsUndef) {
10014 unionAssumedWithUndef();
10015 } else {
10016 for (const auto &It : Incoming)
10017 unionAssumed(It);
10018 }
10019 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10020 : ChangeStatus::CHANGED;
10021 }
10022
10023 ChangeStatus updateImpl(Attributor &A) override {
10024 Value &V = getAssociatedValue();
10026
10027 if (auto *ICI = dyn_cast<ICmpInst>(I))
10028 return updateWithICmpInst(A, ICI);
10029
10030 if (auto *SI = dyn_cast<SelectInst>(I))
10031 return updateWithSelectInst(A, SI);
10032
10033 if (auto *CI = dyn_cast<CastInst>(I))
10034 return updateWithCastInst(A, CI);
10035
10036 if (auto *BinOp = dyn_cast<BinaryOperator>(I))
10037 return updateWithBinaryOperator(A, BinOp);
10038
10039 if (isa<PHINode>(I) || isa<LoadInst>(I))
10040 return updateWithInstruction(A, I);
10041
10042 return indicatePessimisticFixpoint();
10043 }
10044
10045 void trackStatistics() const override {
10046 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10047 }
10048};
10049
10050struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10051 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10052 : AAPotentialConstantValuesImpl(IRP, A) {}
10053
10054 ChangeStatus updateImpl(Attributor &A) override {
10056 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10057 "not be called");
10058 }
10059
10060 void trackStatistics() const override {
10061 STATS_DECLTRACK_FN_ATTR(potential_values)
10062 }
10063};
10064
10065struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10066 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10067 : AAPotentialConstantValuesFunction(IRP, A) {}
10068
10069 void trackStatistics() const override {
10070 STATS_DECLTRACK_CS_ATTR(potential_values)
10071 }
10072};
10073
10074struct AAPotentialConstantValuesCallSiteReturned
10075 : AACalleeToCallSite<AAPotentialConstantValues,
10076 AAPotentialConstantValuesImpl> {
10077 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10078 Attributor &A)
10079 : AACalleeToCallSite<AAPotentialConstantValues,
10080 AAPotentialConstantValuesImpl>(IRP, A) {}
10081
10082 void trackStatistics() const override {
10083 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10084 }
10085};
10086
10087struct AAPotentialConstantValuesCallSiteArgument
10088 : AAPotentialConstantValuesFloating {
10089 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10090 Attributor &A)
10091 : AAPotentialConstantValuesFloating(IRP, A) {}
10092
10093 void initialize(Attributor &A) override {
10094 AAPotentialConstantValuesImpl::initialize(A);
10095 if (isAtFixpoint())
10096 return;
10097
10098 Value &V = getAssociatedValue();
10099
10100 if (auto *C = dyn_cast<ConstantInt>(&V)) {
10101 unionAssumed(C->getValue());
10102 indicateOptimisticFixpoint();
10103 return;
10104 }
10105
10106 if (isa<UndefValue>(&V)) {
10107 unionAssumedWithUndef();
10108 indicateOptimisticFixpoint();
10109 return;
10110 }
10111 }
10112
10113 ChangeStatus updateImpl(Attributor &A) override {
10114 Value &V = getAssociatedValue();
10115 auto AssumedBefore = getAssumed();
10116 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10117 *this, IRPosition::value(V), DepClassTy::REQUIRED);
10118 if (!AA)
10119 return indicatePessimisticFixpoint();
10120 const auto &S = AA->getAssumed();
10121 unionAssumed(S);
10122 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10123 : ChangeStatus::CHANGED;
10124 }
10125
10126 void trackStatistics() const override {
10127 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10128 }
10129};
10130} // namespace
10131
10132/// ------------------------ NoUndef Attribute ---------------------------------
10134 Attribute::AttrKind ImpliedAttributeKind,
10135 bool IgnoreSubsumingPositions) {
10136 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10137 "Unexpected attribute kind");
10138 if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,
10139 Attribute::NoUndef))
10140 return true;
10141
10142 Value &Val = IRP.getAssociatedValue();
10145 LLVMContext &Ctx = Val.getContext();
10146 A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef));
10147 return true;
10148 }
10149
10150 return false;
10151}
10152
10153namespace {
10154struct AANoUndefImpl : AANoUndef {
10155 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10156
10157 void initialize(Attributor &A) override {
10158 Value &V = getAssociatedValue();
10159 if (isa<UndefValue>(V))
10160 indicatePessimisticFixpoint();
10161 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10162 }
10163
10164 /// See followUsesInMBEC
10165 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10166 AANoUndef::StateType &State) {
10167 const Value *UseV = U->get();
10168 const DominatorTree *DT = nullptr;
10169 AssumptionCache *AC = nullptr;
10170 InformationCache &InfoCache = A.getInfoCache();
10171 if (Function *F = getAnchorScope()) {
10172 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
10173 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
10174 }
10175 State.setKnown(isGuaranteedNotToBeUndefOrPoison(UseV, AC, I, DT));
10176 bool TrackUse = false;
10177 // Track use for instructions which must produce undef or poison bits when
10178 // at least one operand contains such bits.
10180 TrackUse = true;
10181 return TrackUse;
10182 }
10183
10184 const std::string getAsStr(Attributor *A) const override {
10185 return getAssumed() ? "noundef" : "may-undef-or-poison";
10186 }
10187
10188 ChangeStatus manifest(Attributor &A) override {
10189 // We don't manifest noundef attribute for dead positions because the
10190 // associated values with dead positions would be replaced with undef
10191 // values.
10192 bool UsedAssumedInformation = false;
10193 if (A.isAssumedDead(getIRPosition(), nullptr, nullptr,
10194 UsedAssumedInformation))
10195 return ChangeStatus::UNCHANGED;
10196 // A position whose simplified value does not have any value is
10197 // considered to be dead. We don't manifest noundef in such positions for
10198 // the same reason above.
10199 if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation,
10201 .has_value())
10202 return ChangeStatus::UNCHANGED;
10203 return AANoUndef::manifest(A);
10204 }
10205};
10206
10207struct AANoUndefFloating : public AANoUndefImpl {
10208 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10209 : AANoUndefImpl(IRP, A) {}
10210
10211 void initialize(Attributor &A) override {
10212 AANoUndefImpl::initialize(A);
10213 if (!getState().isAtFixpoint() && getAnchorScope() &&
10214 !getAnchorScope()->isDeclaration())
10215 if (Instruction *CtxI = getCtxI())
10216 followUsesInMBEC(*this, A, getState(), *CtxI);
10217 }
10218
10219 ChangeStatus updateImpl(Attributor &A) override {
10220 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10221 bool IsKnownNoUndef;
10223 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);
10224 };
10225
10226 bool Stripped;
10227 bool UsedAssumedInformation = false;
10228 Value *AssociatedValue = &getAssociatedValue();
10230 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
10231 AA::AnyScope, UsedAssumedInformation))
10232 Stripped = false;
10233 else
10234 Stripped =
10235 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10236
10237 if (!Stripped) {
10238 // If we haven't stripped anything we might still be able to use a
10239 // different AA, but only if the IRP changes. Effectively when we
10240 // interpret this not as a call site value but as a floating/argument
10241 // value.
10242 const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
10243 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10244 return indicatePessimisticFixpoint();
10245 return ChangeStatus::UNCHANGED;
10246 }
10247
10248 for (const auto &VAC : Values)
10249 if (!VisitValueCB(IRPosition::value(*VAC.getValue())))
10250 return indicatePessimisticFixpoint();
10251
10252 return ChangeStatus::UNCHANGED;
10253 }
10254
10255 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10256};
10257
10258/// Return true if the poison-generating attributes in \p Attrs are assumed to
10259/// hold for the value at \p Pos.
10260static bool checkPoisonGeneratingAttrsHold(Attributor &A,
10261 const AbstractAttribute &QueryingAA,
10262 AttributeSet Attrs,
10263 const IRPosition &Pos,
10264 const Instruction *CtxI) {
10265 AttributeMask AM = AttributeFuncs::getPoisonGeneratingAttributes();
10266 for (Attribute Attr : Attrs) {
10267 if (!AM.contains(Attr))
10268 continue;
10269
10270 switch (Attr.getKindAsEnum()) {
10271 case Attribute::NonNull: {
10272 bool IsKnown;
10274 A, &QueryingAA, Pos, DepClassTy::REQUIRED, IsKnown))
10275 return false;
10276 break;
10277 }
10278 case Attribute::Alignment: {
10279 const auto *AlignAA =
10280 A.getAAFor<AAAlign>(QueryingAA, Pos, DepClassTy::REQUIRED);
10281 if (!AlignAA || AlignAA->getAssumedAlign() < *Attr.getAlignment())
10282 return false;
10283 break;
10284 }
10285 case Attribute::Range: {
10286 const auto *RangeAA = A.getAAFor<AAValueConstantRange>(
10287 QueryingAA, Pos, DepClassTy::REQUIRED);
10288 if (!RangeAA ||
10289 !Attr.getRange().contains(RangeAA->getAssumedConstantRange(A, CtxI)))
10290 return false;
10291 break;
10292 }
10293 case Attribute::NoFPClass: {
10294 FPClassTest NoFPClass = Attr.getNoFPClass();
10295 const auto *NoFPClassAA =
10296 A.getAAFor<AANoFPClass>(QueryingAA, Pos, DepClassTy::REQUIRED);
10297 if (!NoFPClassAA ||
10298 (NoFPClassAA->getAssumedNoFPClass() & NoFPClass) != NoFPClass)
10299 return false;
10300 break;
10301 }
10302 default:
10303 // Conservatively assume unhandled attributes do not hold.
10304 return false;
10305 }
10306 }
10307
10308 return true;
10309}
10310
10311struct AANoUndefReturned final
10312 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10313 using Base = AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>;
10314 AANoUndefReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
10315
10316 void initialize(Attributor &A) override {
10317 Base::initialize(A);
10318 RetAttrs = getAssociatedFunction()->getAttributes().getRetAttrs();
10319 }
10320
10321 ChangeStatus updateImpl(Attributor &A) override {
10322 // A returned value that violates a poison-generating return attribute
10323 // becomes poison, so the attributes must hold for every returned value.
10324 if (RetAttrs.hasAttributes()) {
10325 auto AttrsHold = [&](Instruction &I) {
10326 Value &V = *cast<ReturnInst>(I).getReturnValue();
10327 return checkPoisonGeneratingAttrsHold(A, *this, RetAttrs,
10328 IRPosition::value(V), &I);
10329 };
10330 bool UsedAssumedInformation = false;
10331 if (!A.checkForAllInstructions(AttrsHold, *this, {Instruction::Ret},
10332 UsedAssumedInformation))
10333 return indicatePessimisticFixpoint();
10334 }
10335
10336 return Base::updateImpl(A);
10337 }
10338
10339 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10340
10341private:
10342 /// The function return attributes.
10343 AttributeSet RetAttrs;
10344};
10345
10346struct AANoUndefArgument final
10347 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10348 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10349 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10350
10351 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10352};
10353
10354struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10355 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10356 : AANoUndefFloating(IRP, A) {}
10357
10358 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10359};
10360
10361struct AANoUndefCallSiteReturned final
10362 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10363 using Base = AACalleeToCallSite<AANoUndef, AANoUndefImpl>;
10364 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10365 : Base(IRP, A) {}
10366
10367 void initialize(Attributor &A) override {
10368 Base::initialize(A);
10369 RetAttrs = cast<CallBase>(getAnchorValue()).getRetAttributes();
10370 }
10371
10372 ChangeStatus updateImpl(Attributor &A) override {
10373 // The call returns poison if the callee return value violates a
10374 // poison-generating call site return attribute, so the attributes must
10375 // hold for every callee.
10376 if (RetAttrs.hasAttributes()) {
10377 auto AttrsHold = [&](ArrayRef<const Function *> Callees) {
10378 return all_of(Callees, [&](const Function *Callee) {
10379 return checkPoisonGeneratingAttrsHold(
10380 A, *this, RetAttrs, IRPosition::returned(*Callee), nullptr);
10381 });
10382 };
10383 if (!A.checkForAllCallees(AttrsHold, *this,
10384 cast<CallBase>(getAnchorValue())))
10385 return indicatePessimisticFixpoint();
10386 }
10387
10388 return Base::updateImpl(A);
10389 }
10390
10391 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10392
10393private:
10394 /// The call site return attributes.
10395 AttributeSet RetAttrs;
10396};
10397
10398/// ------------------------ NoFPClass Attribute -------------------------------
10399
10400struct AANoFPClassImpl : AANoFPClass {
10401 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10402
10403 void initialize(Attributor &A) override {
10404 const IRPosition &IRP = getIRPosition();
10405
10406 Value &V = IRP.getAssociatedValue();
10407 if (isa<UndefValue>(V)) {
10408 indicateOptimisticFixpoint();
10409 return;
10410 }
10411
10413 A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false);
10414 for (const auto &Attr : Attrs) {
10415 addKnownBits(Attr.getNoFPClass());
10416 }
10417
10418 Instruction *CtxI = getCtxI();
10419
10420 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10421 const DataLayout &DL = A.getDataLayout();
10422 InformationCache &InfoCache = A.getInfoCache();
10423
10424 const DominatorTree *DT = nullptr;
10425 AssumptionCache *AC = nullptr;
10426 const TargetLibraryInfo *TLI = nullptr;
10427 Function *F = getAnchorScope();
10428 if (F) {
10430 if (!F->isDeclaration()) {
10431 DT =
10432 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
10433 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
10434 }
10435 }
10436
10437 SimplifyQuery Q(DL, TLI, DT, AC, CtxI);
10438
10439 KnownFPClass KnownFPClass = computeKnownFPClass(&V, fcAllFlags, Q);
10440 addKnownBits(~KnownFPClass.getKnownFPClasses());
10441 }
10442
10443 if (CtxI)
10444 followUsesInMBEC(*this, A, getState(), *CtxI);
10445 }
10446
10447 /// See followUsesInMBEC
10448 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10449 AANoFPClass::StateType &State) {
10450 // TODO: Determine what instructions can be looked through.
10451 auto *CB = dyn_cast<CallBase>(I);
10452 if (!CB)
10453 return false;
10454
10455 if (!CB->isArgOperand(U))
10456 return false;
10457
10458 unsigned ArgNo = CB->getArgOperandNo(U);
10459 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
10460 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(*this, IRP, DepClassTy::NONE))
10461 State.addKnownBits(NoFPAA->getState().getKnown());
10462 return false;
10463 }
10464
10465 const std::string getAsStr(Attributor *A) const override {
10466 std::string Result = "nofpclass";
10467 raw_string_ostream OS(Result);
10468 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10469 return Result;
10470 }
10471
10472 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10473 SmallVectorImpl<Attribute> &Attrs) const override {
10474 Attrs.emplace_back(Attribute::getWithNoFPClass(Ctx, getAssumedNoFPClass()));
10475 }
10476};
10477
10478struct AANoFPClassFloating : public AANoFPClassImpl {
10479 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10480 : AANoFPClassImpl(IRP, A) {}
10481
10482 ChangeStatus updateImpl(Attributor &A) override {
10484 bool UsedAssumedInformation = false;
10485 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
10486 AA::AnyScope, UsedAssumedInformation)) {
10487 Values.push_back({getAssociatedValue(), getCtxI()});
10488 }
10489
10490 StateType T;
10491 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10492 const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V),
10493 DepClassTy::REQUIRED);
10494 if (!AA || this == AA) {
10495 T.indicatePessimisticFixpoint();
10496 } else {
10497 const AANoFPClass::StateType &S =
10498 static_cast<const AANoFPClass::StateType &>(AA->getState());
10499 T ^= S;
10500 }
10501 return T.isValidState();
10502 };
10503
10504 for (const auto &VAC : Values)
10505 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10506 return indicatePessimisticFixpoint();
10507
10508 return clampStateAndIndicateChange(getState(), T);
10509 }
10510
10511 void trackStatistics() const override {
10513 }
10514};
10515
10516struct AANoFPClassReturned final
10517 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10518 AANoFPClassImpl::StateType, false,
10519 Attribute::None, false> {
10520 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10521 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10522 AANoFPClassImpl::StateType, false,
10523 Attribute::None, false>(IRP, A) {}
10524
10525 void trackStatistics() const override {
10527 }
10528};
10529
10530struct AANoFPClassArgument final
10531 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10532 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10533 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10534
10535 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10536};
10537
10538struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10539 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10540 : AANoFPClassFloating(IRP, A) {}
10541
10542 void trackStatistics() const override {
10544 }
10545};
10546
10547struct AANoFPClassCallSiteReturned final
10548 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10549 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10550 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10551
10552 void trackStatistics() const override {
10554 }
10555};
10556
10557struct AACallEdgesImpl : public AACallEdges {
10558 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10559
10560 const SetVector<Function *> &getOptimisticEdges() const override {
10561 return CalledFunctions;
10562 }
10563
10564 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10565
10566 bool hasNonAsmUnknownCallee() const override {
10567 return HasUnknownCalleeNonAsm;
10568 }
10569
10570 const std::string getAsStr(Attributor *A) const override {
10571 return "CallEdges[" + std::to_string(HasUnknownCallee) + "," +
10572 std::to_string(CalledFunctions.size()) + "]";
10573 }
10574
10575 void trackStatistics() const override {}
10576
10577protected:
10578 /// Return the callee of \p CB if it is a direct call whose callee has no
10579 /// callback metadata. Such a call only calls its callee.
10580 static Function *getOnlyCallee(const CallBase &CB) {
10582 if (Callee && !Callee->hasMetadata(LLVMContext::MD_callback))
10583 return Callee;
10584 return nullptr;
10585 }
10586
10587 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10588 if (CalledFunctions.insert(Fn)) {
10589 Change = ChangeStatus::CHANGED;
10590 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10591 << "\n");
10592 }
10593 }
10594
10595 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10596 if (!HasUnknownCallee)
10597 Change = ChangeStatus::CHANGED;
10598 if (NonAsm && !HasUnknownCalleeNonAsm)
10599 Change = ChangeStatus::CHANGED;
10600 HasUnknownCalleeNonAsm |= NonAsm;
10601 HasUnknownCallee = true;
10602 }
10603
10604private:
10605 /// Optimistic set of functions that might be called by this position.
10606 SetVector<Function *> CalledFunctions;
10607
10608 /// Is there any call with a unknown callee.
10609 bool HasUnknownCallee = false;
10610
10611 /// Is there any call with a unknown callee, excluding any inline asm.
10612 bool HasUnknownCalleeNonAsm = false;
10613};
10614
10615struct AACallEdgesCallSite : public AACallEdgesImpl {
10616 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10617 : AACallEdgesImpl(IRP, A) {}
10618
10619 void initialize(Attributor &A) override {
10620 // The only callee of a direct call without callbacks never changes.
10621 if (Function *Callee = getOnlyCallee(cast<CallBase>(getAnchorValue()))) {
10622 ChangeStatus Change = ChangeStatus::UNCHANGED;
10623 addCalledFunction(Callee, Change);
10624 indicateOptimisticFixpoint();
10625 }
10626 }
10627
10628 ChangeStatus updateImpl(Attributor &A) override {
10629 ChangeStatus Change = ChangeStatus::UNCHANGED;
10630
10631 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10632 if (Function *Fn = dyn_cast<Function>(&V)) {
10633 addCalledFunction(Fn, Change);
10634 } else {
10635 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10636 setHasUnknownCallee(true, Change);
10637 }
10638
10639 // Explore all values.
10640 return true;
10641 };
10642
10644 // Process any value that we might call.
10645 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10646 if (isa<Constant>(V)) {
10647 VisitValue(*V, CtxI);
10648 return;
10649 }
10650
10651 bool UsedAssumedInformation = false;
10652 Values.clear();
10653 if (!A.getAssumedSimplifiedValues(IRPosition::value(*V), *this, Values,
10654 AA::AnyScope, UsedAssumedInformation)) {
10655 Values.push_back({*V, CtxI});
10656 }
10657 for (auto &VAC : Values)
10658 VisitValue(*VAC.getValue(), VAC.getCtxI());
10659 };
10660
10661 CallBase *CB = cast<CallBase>(getCtxI());
10662
10663 if (auto *IA = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
10664 if (IA->hasSideEffects() &&
10665 !hasAssumption(*CB->getCaller(), "ompx_no_call_asm") &&
10666 !hasAssumption(*CB, "ompx_no_call_asm")) {
10667 setHasUnknownCallee(false, Change);
10668 }
10669 return Change;
10670 }
10671
10672 if (CB->isIndirectCall())
10673 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10674 *this, getIRPosition(), DepClassTy::OPTIONAL))
10675 if (IndirectCallAA->foreachCallee(
10676 [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10677 return Change;
10678
10679 // The most simple case.
10680 ProcessCalledOperand(CB->getCalledOperand(), CB);
10681
10682 // Process callback functions.
10683 SmallVector<const Use *, 4u> CallbackUses;
10684 AbstractCallSite::getCallbackUses(*CB, CallbackUses);
10685 for (const Use *U : CallbackUses)
10686 ProcessCalledOperand(U->get(), CB);
10687
10688 return Change;
10689 }
10690};
10691
10692struct AACallEdgesFunction : public AACallEdgesImpl {
10693 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10694 : AACallEdgesImpl(IRP, A) {}
10695
10696 ChangeStatus updateImpl(Attributor &A) override {
10697 ChangeStatus Change = ChangeStatus::UNCHANGED;
10698 bool AllCallsDirect = true;
10699
10700 auto ProcessCallInst = [&](Instruction &Inst) {
10701 CallBase &CB = cast<CallBase>(Inst);
10702
10703 // A direct call without callbacks only calls its callee; there is no
10704 // need for a call site attribute.
10705 if (Function *Callee = getOnlyCallee(CB)) {
10706 addCalledFunction(Callee, Change);
10707 return true;
10708 }
10709
10710 AllCallsDirect = false;
10711 auto *CBEdges = A.getAAFor<AACallEdges>(
10712 *this, IRPosition::callsite_function(CB), DepClassTy::REQUIRED);
10713 if (!CBEdges)
10714 return false;
10715 if (CBEdges->hasNonAsmUnknownCallee())
10716 setHasUnknownCallee(true, Change);
10717 if (CBEdges->hasUnknownCallee())
10718 setHasUnknownCallee(false, Change);
10719
10720 for (Function *F : CBEdges->getOptimisticEdges())
10721 addCalledFunction(F, Change);
10722
10723 return true;
10724 };
10725
10726 // Visit all callable instructions.
10727 bool UsedAssumedInformation = false;
10728 if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this,
10729 UsedAssumedInformation,
10730 /* CheckBBLivenessOnly */ true)) {
10731 // If we haven't looked at all call like instructions, assume that there
10732 // are unknown callees.
10733 setHasUnknownCallee(true, Change);
10734 }
10735
10736 // The callees of direct calls in code known to be live do not change.
10737 if (AllCallsDirect && !UsedAssumedInformation)
10738 indicateOptimisticFixpoint();
10739
10740 return Change;
10741 }
10742};
10743
10744/// -------------------AAInterFnReachability Attribute--------------------------
10745
10746struct AAInterFnReachabilityFunction
10747 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10748 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10749 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10750 : Base(IRP, A) {}
10751
10752 bool instructionCanReach(
10753 Attributor &A, const Instruction &From, const Function &To,
10754 const AA::InstExclusionSetTy *ExclusionSet) const override {
10755 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10756 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10757
10758 RQITy StackRQI(A, From, To, ExclusionSet, false);
10759 RQITy::Reachable Result;
10760 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10761 return NonConstThis->isReachableImpl(A, StackRQI,
10762 /*IsTemporaryRQI=*/true);
10763 return Result == RQITy::Reachable::Yes;
10764 }
10765
10766 bool isReachableImpl(Attributor &A, RQITy &RQI,
10767 bool IsTemporaryRQI) override {
10768 const Instruction *EntryI =
10769 &RQI.From->getFunction()->getEntryBlock().front();
10770 if (EntryI != RQI.From &&
10771 !instructionCanReach(A, *EntryI, *RQI.To, nullptr))
10772 return rememberResult(A, RQITy::Reachable::No, RQI, false,
10773 IsTemporaryRQI);
10774
10775 auto CheckReachableCallBase = [&](CallBase *CB) {
10776 auto *CBEdges = A.getAAFor<AACallEdges>(
10777 *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
10778 if (!CBEdges || !CBEdges->getState().isValidState())
10779 return false;
10780 // TODO Check To backwards in this case.
10781 if (CBEdges->hasUnknownCallee())
10782 return false;
10783
10784 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10785 if (Fn == RQI.To)
10786 return false;
10787
10788 if (Fn->isDeclaration()) {
10789 if (Fn->hasFnAttribute(Attribute::NoCallback))
10790 continue;
10791 // TODO Check To backwards in this case.
10792 return false;
10793 }
10794
10795 if (Fn == getAnchorScope()) {
10796 if (EntryI == RQI.From)
10797 continue;
10798 return false;
10799 }
10800
10801 const AAInterFnReachability *InterFnReachability =
10802 A.getAAFor<AAInterFnReachability>(*this, IRPosition::function(*Fn),
10803 DepClassTy::OPTIONAL);
10804
10805 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10806 if (!InterFnReachability ||
10807 InterFnReachability->instructionCanReach(A, FnFirstInst, *RQI.To,
10808 RQI.ExclusionSet))
10809 return false;
10810 }
10811 return true;
10812 };
10813
10814 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10815 *this, IRPosition::function(*RQI.From->getFunction()),
10816 DepClassTy::OPTIONAL);
10817
10818 // Determine call like instructions that we can reach from the inst.
10819 auto CheckCallBase = [&](Instruction &CBInst) {
10820 // There are usually less nodes in the call graph, check inter function
10821 // reachability first.
10822 if (CheckReachableCallBase(cast<CallBase>(&CBInst)))
10823 return true;
10824 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10825 A, *RQI.From, CBInst, RQI.ExclusionSet);
10826 };
10827
10828 bool UsedExclusionSet = /* conservative */ true;
10829 bool UsedAssumedInformation = false;
10830 if (!A.checkForAllCallLikeInstructions(CheckCallBase, *this,
10831 UsedAssumedInformation,
10832 /* CheckBBLivenessOnly */ true))
10833 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10834 IsTemporaryRQI);
10835
10836 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
10837 IsTemporaryRQI);
10838 }
10839
10840 void trackStatistics() const override {}
10841};
10842} // namespace
10843
10844template <typename AAType>
10845static std::optional<Constant *>
10847 const IRPosition &IRP, Type &Ty) {
10848 if (!Ty.isIntegerTy())
10849 return nullptr;
10850
10851 // This will also pass the call base context.
10852 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10853 if (!AA)
10854 return nullptr;
10855
10856 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10857
10858 if (!COpt.has_value()) {
10859 A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
10860 return std::nullopt;
10861 }
10862 if (auto *C = *COpt) {
10863 A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
10864 return C;
10865 }
10866 return nullptr;
10867}
10868
10870 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10872 Type &Ty = *IRP.getAssociatedType();
10873 std::optional<Value *> V;
10874 for (auto &It : Values) {
10875 V = AA::combineOptionalValuesInAAValueLatice(V, It.getValue(), &Ty);
10876 if (V.has_value() && !*V)
10877 break;
10878 }
10879 if (!V.has_value())
10880 return UndefValue::get(&Ty);
10881 return *V;
10882}
10883
10884namespace {
10885struct AAPotentialValuesImpl : AAPotentialValues {
10886 using StateType = PotentialLLVMValuesState;
10887
10888 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10889 : AAPotentialValues(IRP, A) {}
10890
10891 void initialize(Attributor &A) override {
10892 if (A.hasSimplificationCallback(getIRPosition())) {
10893 indicatePessimisticFixpoint();
10894 return;
10895 }
10896 Value *Stripped = getAssociatedValue().stripPointerCasts();
10897 if (isa<Constant>(Stripped) && !isa<ConstantExpr>(Stripped)) {
10898 addValue(A, getState(), *Stripped, getCtxI(), AA::AnyScope,
10899 getAnchorScope());
10900 indicateOptimisticFixpoint();
10901 return;
10902 }
10903 AAPotentialValues::initialize(A);
10904 }
10905
10906 const std::string getAsStr(Attributor *A) const override {
10907 std::string Str;
10908 llvm::raw_string_ostream OS(Str);
10909 OS << getState();
10910 return Str;
10911 }
10912
10913 template <typename AAType>
10914 static std::optional<Value *> askOtherAA(Attributor &A,
10915 const AbstractAttribute &AA,
10916 const IRPosition &IRP, Type &Ty) {
10918 return &IRP.getAssociatedValue();
10919 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
10920 if (!C)
10921 return std::nullopt;
10922 if (*C)
10923 if (auto *CC = AA::getWithType(**C, Ty))
10924 return CC;
10925 return nullptr;
10926 }
10927
10928 virtual void addValue(Attributor &A, StateType &State, Value &V,
10929 const Instruction *CtxI, AA::ValueScope S,
10930 Function *AnchorScope) const {
10931
10932 IRPosition ValIRP = IRPosition::value(V);
10933 if (auto *CB = dyn_cast_or_null<CallBase>(CtxI)) {
10934 for (const auto &U : CB->args()) {
10935 if (U.get() != &V)
10936 continue;
10937 ValIRP = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));
10938 break;
10939 }
10940 }
10941
10942 Value *VPtr = &V;
10943 if (ValIRP.getAssociatedType()->isIntegerTy()) {
10944 Type &Ty = *getAssociatedType();
10945 std::optional<Value *> SimpleV =
10946 askOtherAA<AAValueConstantRange>(A, *this, ValIRP, Ty);
10947 if (SimpleV.has_value() && !*SimpleV) {
10948 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
10949 *this, ValIRP, DepClassTy::OPTIONAL);
10950 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
10951 for (const auto &It : PotentialConstantsAA->getAssumedSet())
10952 State.unionAssumed({{*ConstantInt::get(&Ty, It), nullptr}, S});
10953 if (PotentialConstantsAA->undefIsContained())
10954 State.unionAssumed({{*UndefValue::get(&Ty), nullptr}, S});
10955 return;
10956 }
10957 }
10958 if (!SimpleV.has_value())
10959 return;
10960
10961 if (*SimpleV)
10962 VPtr = *SimpleV;
10963 }
10964
10965 if (isa<ConstantInt>(VPtr))
10966 CtxI = nullptr;
10967 if (!AA::isValidInScope(*VPtr, AnchorScope))
10969
10970 State.unionAssumed({{*VPtr, CtxI}, S});
10971 }
10972
10973 /// Helper struct to tie a value+context pair together with the scope for
10974 /// which this is the simplified version.
10975 struct ItemInfo {
10976 AA::ValueAndContext I;
10978
10979 bool operator==(const ItemInfo &II) const {
10980 return II.I == I && II.S == S;
10981 };
10982 bool operator<(const ItemInfo &II) const {
10983 return std::tie(I, S) < std::tie(II.I, II.S);
10984 };
10985 };
10986
10987 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
10988 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
10989 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
10990 if (!(CS & S))
10991 continue;
10992
10993 bool UsedAssumedInformation = false;
10995 if (!A.getAssumedSimplifiedValues(IRP, this, Values, CS,
10996 UsedAssumedInformation))
10997 return false;
10998
10999 for (auto &It : Values)
11000 ValueScopeMap[It] += CS;
11001 }
11002 for (auto &It : ValueScopeMap)
11003 addValue(A, getState(), *It.first.getValue(), It.first.getCtxI(),
11004 AA::ValueScope(It.second), getAnchorScope());
11005
11006 return true;
11007 }
11008
11009 void giveUpOnIntraprocedural(Attributor &A) {
11010 auto NewS = StateType::getBestState(getState());
11011 for (const auto &It : getAssumedSet()) {
11012 if (It.second == AA::Intraprocedural)
11013 continue;
11014 addValue(A, NewS, *It.first.getValue(), It.first.getCtxI(),
11015 AA::Interprocedural, getAnchorScope());
11016 }
11017 assert(!undefIsContained() && "Undef should be an explicit value!");
11018 addValue(A, NewS, getAssociatedValue(), getCtxI(), AA::Intraprocedural,
11019 getAnchorScope());
11020 getState() = NewS;
11021 }
11022
11023 ChangeStatus indicatePessimisticFixpoint() override {
11024 getState() = StateType::getBestState(getState());
11025 getState().unionAssumed({{getAssociatedValue(), getCtxI()}, AA::AnyScope});
11026 AAPotentialValues::indicateOptimisticFixpoint();
11027 return ChangeStatus::CHANGED;
11028 }
11029
11030 ChangeStatus updateImpl(Attributor &A) override {
11031 return indicatePessimisticFixpoint();
11032 }
11033
11034 ChangeStatus manifest(Attributor &A) override {
11037 Values.clear();
11038 if (!getAssumedSimplifiedValues(A, Values, S))
11039 continue;
11040 Value &OldV = getAssociatedValue();
11041 if (isa<UndefValue>(OldV))
11042 continue;
11043 Value *NewV = getSingleValue(A, *this, getIRPosition(), Values);
11044 if (!NewV || NewV == &OldV)
11045 continue;
11046 if (getCtxI() &&
11047 !AA::isValidAtPosition({*NewV, *getCtxI()}, A.getInfoCache()))
11048 continue;
11049 if (A.changeAfterManifest(getIRPosition(), *NewV))
11050 return ChangeStatus::CHANGED;
11051 }
11052 return ChangeStatus::UNCHANGED;
11053 }
11054
11055 bool getAssumedSimplifiedValues(
11056 Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
11057 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
11058 if (!isValidState())
11059 return false;
11060 bool UsedAssumedInformation = false;
11061 for (const auto &It : getAssumedSet())
11062 if (It.second & S) {
11063 if (RecurseForSelectAndPHI && (isa<PHINode>(It.first.getValue()) ||
11064 isa<SelectInst>(It.first.getValue()))) {
11065 if (A.getAssumedSimplifiedValues(
11066 IRPosition::inst(*cast<Instruction>(It.first.getValue())),
11067 this, Values, S, UsedAssumedInformation))
11068 continue;
11069 }
11070 Values.push_back(It.first);
11071 }
11072 assert(!undefIsContained() && "Undef should be an explicit value!");
11073 return true;
11074 }
11075};
11076
11077struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11078 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
11079 : AAPotentialValuesImpl(IRP, A) {}
11080
11081 ChangeStatus updateImpl(Attributor &A) override {
11082 auto AssumedBefore = getAssumed();
11083
11084 genericValueTraversal(A, &getAssociatedValue());
11085
11086 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11087 : ChangeStatus::CHANGED;
11088 }
11089
11090 /// Helper struct to remember which AAIsDead instances we actually used.
11091 struct LivenessInfo {
11092 const AAIsDead *LivenessAA = nullptr;
11093 bool AnyDead = false;
11094 };
11095
11096 /// Check if \p Cmp is a comparison we can simplify.
11097 ///
11098 /// We handle multiple cases, one in which at least one operand is an
11099 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
11100 /// operand. Return true if successful, in that case Worklist will be updated.
11101 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
11102 CmpInst::Predicate Pred, ItemInfo II,
11103 SmallVectorImpl<ItemInfo> &Worklist) {
11104
11105 // Simplify the operands first.
11106 bool UsedAssumedInformation = false;
11107 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11108 auto GetSimplifiedValues = [&](Value &V,
11110 if (!A.getAssumedSimplifiedValues(
11111 IRPosition::value(V, getCallBaseContext()), this, Values,
11112 AA::Intraprocedural, UsedAssumedInformation)) {
11113 Values.clear();
11114 Values.push_back(AA::ValueAndContext{V, II.I.getCtxI()});
11115 }
11116 return Values.empty();
11117 };
11118 if (GetSimplifiedValues(*LHS, LHSValues))
11119 return true;
11120 if (GetSimplifiedValues(*RHS, RHSValues))
11121 return true;
11122
11123 LLVMContext &Ctx = LHS->getContext();
11124
11125 InformationCache &InfoCache = A.getInfoCache();
11126 Instruction *CmpI = dyn_cast<Instruction>(&Cmp);
11127 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11128 const auto *DT =
11129 F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F)
11130 : nullptr;
11131 const auto *TLI =
11132 F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
11133 auto *AC =
11134 F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F)
11135 : nullptr;
11136
11137 const DataLayout &DL = A.getDataLayout();
11138 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11139
11140 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11141 if (isa<UndefValue>(LHSV) || isa<UndefValue>(RHSV)) {
11142 addValue(A, getState(), *UndefValue::get(Cmp.getType()),
11143 /* CtxI */ nullptr, II.S, getAnchorScope());
11144 return true;
11145 }
11146
11147 // Handle the trivial case first in which we don't even need to think
11148 // about null or non-null.
11149 if (&LHSV == &RHSV &&
11151 Constant *NewV = ConstantInt::get(Type::getInt1Ty(Ctx),
11153 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11154 getAnchorScope());
11155 return true;
11156 }
11157
11158 auto *TypedLHS = AA::getWithType(LHSV, *LHS->getType());
11159 auto *TypedRHS = AA::getWithType(RHSV, *RHS->getType());
11160 if (TypedLHS && TypedRHS) {
11161 Value *NewV = simplifyCmpInst(Pred, TypedLHS, TypedRHS, Q);
11162 if (NewV && NewV != &Cmp) {
11163 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11164 getAnchorScope());
11165 return true;
11166 }
11167 }
11168
11169 // From now on we only handle equalities (==, !=).
11171 return false;
11172
11173 bool LHSIsNull = isa<ConstantPointerNull>(LHSV);
11174 bool RHSIsNull = isa<ConstantPointerNull>(RHSV);
11175 if (!LHSIsNull && !RHSIsNull)
11176 return false;
11177
11178 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11179 // non-nullptr operand and if we assume it's non-null we can conclude the
11180 // result of the comparison.
11181 assert((LHSIsNull || RHSIsNull) &&
11182 "Expected nullptr versus non-nullptr comparison at this point");
11183
11184 // The index is the operand that we assume is not null.
11185 unsigned PtrIdx = LHSIsNull;
11186 bool IsKnownNonNull;
11187 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11188 A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)),
11189 DepClassTy::REQUIRED, IsKnownNonNull);
11190 if (!IsAssumedNonNull)
11191 return false;
11192
11193 // The new value depends on the predicate, true for != and false for ==.
11194 Constant *NewV =
11195 ConstantInt::get(Type::getInt1Ty(Ctx), Pred == CmpInst::ICMP_NE);
11196 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11197 getAnchorScope());
11198 return true;
11199 };
11200
11201 for (auto &LHSValue : LHSValues)
11202 for (auto &RHSValue : RHSValues)
11203 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11204 return false;
11205 return true;
11206 }
11207
11208 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11209 SmallVectorImpl<ItemInfo> &Worklist) {
11210 const Instruction *CtxI = II.I.getCtxI();
11211 bool UsedAssumedInformation = false;
11212
11213 std::optional<Constant *> C =
11214 A.getAssumedConstant(*SI.getCondition(), *this, UsedAssumedInformation);
11215 bool NoValueYet = !C.has_value();
11216 if (NoValueYet || isa_and_nonnull<UndefValue>(*C))
11217 return true;
11218 if (auto *CI = dyn_cast_or_null<ConstantInt>(*C)) {
11219 if (CI->isZero())
11220 Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
11221 else
11222 Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
11223 } else if (&SI == &getAssociatedValue()) {
11224 // We could not simplify the condition, assume both values.
11225 Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
11226 Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
11227 } else {
11228 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11229 IRPosition::inst(SI), *this, UsedAssumedInformation, II.S);
11230 if (!SimpleV.has_value())
11231 return true;
11232 if (*SimpleV) {
11233 addValue(A, getState(), **SimpleV, CtxI, II.S, getAnchorScope());
11234 return true;
11235 }
11236 return false;
11237 }
11238 return true;
11239 }
11240
11241 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11242 SmallVectorImpl<ItemInfo> &Worklist) {
11243 SmallSetVector<Value *, 4> PotentialCopies;
11244 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11245 bool UsedAssumedInformation = false;
11246 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialCopies,
11247 PotentialValueOrigins, *this,
11248 UsedAssumedInformation,
11249 /* OnlyExact */ true)) {
11250 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11251 "loaded values for load instruction "
11252 << LI << "\n");
11253 return false;
11254 }
11255
11256 // Do not simplify loads that are only used in llvm.assume if we cannot also
11257 // remove all stores that may feed into the load. The reason is that the
11258 // assume is probably worth something as long as the stores are around.
11259 InformationCache &InfoCache = A.getInfoCache();
11260 if (InfoCache.isOnlyUsedByAssume(LI)) {
11261 if (!llvm::all_of(PotentialValueOrigins, [&](Instruction *I) {
11262 if (!I || isa<AssumeInst>(I))
11263 return true;
11264 if (auto *SI = dyn_cast<StoreInst>(I))
11265 return A.isAssumedDead(SI->getOperandUse(0), this,
11266 /* LivenessAA */ nullptr,
11267 UsedAssumedInformation,
11268 /* CheckBBLivenessOnly */ false);
11269 return A.isAssumedDead(*I, this, /* LivenessAA */ nullptr,
11270 UsedAssumedInformation,
11271 /* CheckBBLivenessOnly */ false);
11272 })) {
11273 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11274 "and we cannot delete all the stores: "
11275 << LI << "\n");
11276 return false;
11277 }
11278 }
11279
11280 // Values have to be dynamically unique or we loose the fact that a
11281 // single llvm::Value might represent two runtime values (e.g.,
11282 // stack locations in different recursive calls).
11283 const Instruction *CtxI = II.I.getCtxI();
11284 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11285 bool AllLocal = ScopeIsLocal;
11286 bool DynamicallyUnique = llvm::all_of(PotentialCopies, [&](Value *PC) {
11287 AllLocal &= AA::isValidInScope(*PC, getAnchorScope());
11288 return AA::isDynamicallyUnique(A, *this, *PC);
11289 });
11290 if (!DynamicallyUnique) {
11291 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11292 "values are dynamically unique: "
11293 << LI << "\n");
11294 return false;
11295 }
11296
11297 for (auto *PotentialCopy : PotentialCopies) {
11298 if (AllLocal) {
11299 Worklist.push_back({{*PotentialCopy, CtxI}, II.S});
11300 } else {
11301 Worklist.push_back({{*PotentialCopy, CtxI}, AA::Interprocedural});
11302 }
11303 }
11304 if (!AllLocal && ScopeIsLocal)
11305 addValue(A, getState(), LI, CtxI, AA::Intraprocedural, getAnchorScope());
11306 return true;
11307 }
11308
11309 bool handlePHINode(
11310 Attributor &A, PHINode &PHI, ItemInfo II,
11311 SmallVectorImpl<ItemInfo> &Worklist,
11312 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11313 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11314 LivenessInfo &LI = LivenessAAs[&F];
11315 if (!LI.LivenessAA)
11316 LI.LivenessAA = A.getAAFor<AAIsDead>(*this, IRPosition::function(F),
11317 DepClassTy::NONE);
11318 return LI;
11319 };
11320
11321 if (&PHI == &getAssociatedValue()) {
11322 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11323 const auto *CI =
11324 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11325 *PHI.getFunction());
11326
11327 CycleRef C;
11328 bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);
11329 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11330 BasicBlock *IncomingBB = PHI.getIncomingBlock(u);
11331 if (LI.LivenessAA &&
11332 LI.LivenessAA->isEdgeDead(IncomingBB, PHI.getParent())) {
11333 LI.AnyDead = true;
11334 continue;
11335 }
11336 Value *V = PHI.getIncomingValue(u);
11337 if (V == &PHI)
11338 continue;
11339
11340 // If the incoming value is not the PHI but an instruction in the same
11341 // cycle we might have multiple versions of it flying around.
11342 if (CyclePHI && isa<Instruction>(V) &&
11343 (!C || CI->contains(C, cast<Instruction>(V)->getParent())))
11344 return false;
11345
11346 Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});
11347 }
11348 return true;
11349 }
11350
11351 bool UsedAssumedInformation = false;
11352 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11353 IRPosition::inst(PHI), *this, UsedAssumedInformation, II.S);
11354 if (!SimpleV.has_value())
11355 return true;
11356 if (!(*SimpleV))
11357 return false;
11358 addValue(A, getState(), **SimpleV, &PHI, II.S, getAnchorScope());
11359 return true;
11360 }
11361
11362 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11363 /// simplify any operand of the instruction \p I. Return true if successful,
11364 /// in that case Worklist will be updated.
11365 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11366 SmallVectorImpl<ItemInfo> &Worklist) {
11367 bool SomeSimplified = false;
11368 bool UsedAssumedInformation = false;
11369
11370 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11371 int Idx = 0;
11372 for (Value *Op : I.operands()) {
11373 const auto &SimplifiedOp = A.getAssumedSimplified(
11374 IRPosition::value(*Op, getCallBaseContext()), *this,
11375 UsedAssumedInformation, AA::Intraprocedural);
11376 // If we are not sure about any operand we are not sure about the entire
11377 // instruction, we'll wait.
11378 if (!SimplifiedOp.has_value())
11379 return true;
11380
11381 if (*SimplifiedOp)
11382 NewOps[Idx] = *SimplifiedOp;
11383 else
11384 NewOps[Idx] = Op;
11385
11386 SomeSimplified |= (NewOps[Idx] != Op);
11387 ++Idx;
11388 }
11389
11390 // We won't bother with the InstSimplify interface if we didn't simplify any
11391 // operand ourselves.
11392 if (!SomeSimplified)
11393 return false;
11394
11395 InformationCache &InfoCache = A.getInfoCache();
11396 Function *F = I.getFunction();
11397 const auto *DT =
11398 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
11399 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
11400 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
11401
11402 const DataLayout &DL = I.getDataLayout();
11403 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11404 Value *NewV = simplifyInstructionWithOperands(&I, NewOps, Q);
11405 if (!NewV || NewV == &I)
11406 return false;
11407
11408 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11409 << *NewV << "\n");
11410 Worklist.push_back({{*NewV, II.I.getCtxI()}, II.S});
11411 return true;
11412 }
11413
11415 Attributor &A, Instruction &I, ItemInfo II,
11416 SmallVectorImpl<ItemInfo> &Worklist,
11417 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11418 if (auto *CI = dyn_cast<CmpInst>(&I))
11419 return handleCmp(A, *CI, CI->getOperand(0), CI->getOperand(1),
11420 CI->getPredicate(), II, Worklist);
11421
11422 switch (I.getOpcode()) {
11423 case Instruction::Select:
11424 return handleSelectInst(A, cast<SelectInst>(I), II, Worklist);
11425 case Instruction::PHI:
11426 return handlePHINode(A, cast<PHINode>(I), II, Worklist, LivenessAAs);
11427 case Instruction::Load:
11428 return handleLoadInst(A, cast<LoadInst>(I), II, Worklist);
11429 default:
11430 return handleGenericInst(A, I, II, Worklist);
11431 };
11432 return false;
11433 }
11434
11435 void genericValueTraversal(Attributor &A, Value *InitialV) {
11436 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11437
11438 SmallSet<ItemInfo, 16> Visited;
11440 Worklist.push_back({{*InitialV, getCtxI()}, AA::AnyScope});
11441
11442 int Iteration = 0;
11443 do {
11444 ItemInfo II = Worklist.pop_back_val();
11445 Value *V = II.I.getValue();
11446 assert(V);
11447 const Instruction *CtxI = II.I.getCtxI();
11448 AA::ValueScope S = II.S;
11449
11450 // Check if we should process the current value. To prevent endless
11451 // recursion keep a record of the values we followed!
11452 if (!Visited.insert(II).second)
11453 continue;
11454
11455 // Make sure we limit the compile time for complex expressions.
11456 if (Iteration++ >= A.Opts.attributor_max_potential_values_iterations) {
11457 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11458 << Iteration << "!\n");
11459 addValue(A, getState(), *V, CtxI, S, getAnchorScope());
11460 continue;
11461 }
11462
11463 // Explicitly look through calls with a "returned" attribute if we do
11464 // not have a pointer as stripPointerCasts only works on them.
11465 Value *NewV = nullptr;
11466 if (V->getType()->isPointerTy()) {
11467 NewV = AA::getWithType(*V->stripPointerCasts(), *V->getType());
11468 } else {
11469 if (auto *CB = dyn_cast<CallBase>(V))
11470 if (auto *Callee =
11472 for (Argument &Arg : Callee->args())
11473 if (Arg.hasReturnedAttr()) {
11474 NewV = CB->getArgOperand(Arg.getArgNo());
11475 break;
11476 }
11477 }
11478 }
11479 if (NewV && NewV != V) {
11480 Worklist.push_back({{*NewV, CtxI}, S});
11481 continue;
11482 }
11483
11484 if (auto *I = dyn_cast<Instruction>(V)) {
11485 if (simplifyInstruction(A, *I, II, Worklist, LivenessAAs))
11486 continue;
11487 }
11488
11489 if (V != InitialV || isa<Argument>(V))
11490 if (recurseForValue(A, IRPosition::value(*V), II.S))
11491 continue;
11492
11493 // If we haven't stripped anything we give up.
11494 if (V == InitialV && CtxI == getCtxI()) {
11495 indicatePessimisticFixpoint();
11496 return;
11497 }
11498
11499 addValue(A, getState(), *V, CtxI, S, getAnchorScope());
11500 } while (!Worklist.empty());
11501
11502 // If we actually used liveness information so we have to record a
11503 // dependence.
11504 for (auto &It : LivenessAAs)
11505 if (It.second.AnyDead)
11506 A.recordDependence(*It.second.LivenessAA, *this, DepClassTy::OPTIONAL);
11507 }
11508
11509 void trackStatistics() const override {
11510 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11511 }
11512};
11513
11514struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11515 using Base = AAPotentialValuesImpl;
11516 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11517 : Base(IRP, A) {}
11518
11519 void initialize(Attributor &A) override {
11520 auto &Arg = cast<Argument>(getAssociatedValue());
11522 indicatePessimisticFixpoint();
11523 }
11524
11525 ChangeStatus updateImpl(Attributor &A) override {
11526 auto AssumedBefore = getAssumed();
11527
11528 unsigned ArgNo = getCalleeArgNo();
11529
11530 bool UsedAssumedInformation = false;
11532 auto CallSitePred = [&](AbstractCallSite ACS) {
11533 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11534 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11535 return false;
11536
11537 if (!A.getAssumedSimplifiedValues(CSArgIRP, this, Values,
11539 UsedAssumedInformation))
11540 return false;
11541
11542 return isValidState();
11543 };
11544
11545 if (!A.checkForAllCallSites(CallSitePred, *this,
11546 /* RequireAllCallSites */ true,
11547 UsedAssumedInformation))
11548 return indicatePessimisticFixpoint();
11549
11550 Function *Fn = getAssociatedFunction();
11551 bool AnyNonLocal = false;
11552 for (auto &It : Values) {
11553 if (isa<Constant>(It.getValue())) {
11554 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
11555 getAnchorScope());
11556 continue;
11557 }
11558 if (!AA::isDynamicallyUnique(A, *this, *It.getValue()))
11559 return indicatePessimisticFixpoint();
11560
11561 if (auto *Arg = dyn_cast<Argument>(It.getValue()))
11562 if (Arg->getParent() == Fn) {
11563 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
11564 getAnchorScope());
11565 continue;
11566 }
11567 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::Interprocedural,
11568 getAnchorScope());
11569 AnyNonLocal = true;
11570 }
11571 assert(!undefIsContained() && "Undef should be an explicit value!");
11572 if (AnyNonLocal)
11573 giveUpOnIntraprocedural(A);
11574
11575 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11576 : ChangeStatus::CHANGED;
11577 }
11578
11579 void trackStatistics() const override {
11580 STATS_DECLTRACK_ARG_ATTR(potential_values)
11581 }
11582};
11583
11584struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11585 using Base = AAPotentialValuesFloating;
11586 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11587 : Base(IRP, A) {}
11588
11589 void initialize(Attributor &A) override {
11590 Function *F = getAssociatedFunction();
11591 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11592 indicatePessimisticFixpoint();
11593 return;
11594 }
11595
11596 for (Argument &Arg : F->args())
11597 if (Arg.hasReturnedAttr()) {
11598 addValue(A, getState(), Arg, nullptr, AA::AnyScope, F);
11599 ReturnedArg = &Arg;
11600 break;
11601 }
11602 if (!A.isFunctionIPOAmendable(*F) ||
11603 A.hasSimplificationCallback(getIRPosition())) {
11604 if (!ReturnedArg)
11605 indicatePessimisticFixpoint();
11606 else
11607 indicateOptimisticFixpoint();
11608 }
11609 }
11610
11611 ChangeStatus updateImpl(Attributor &A) override {
11612 auto AssumedBefore = getAssumed();
11613 bool UsedAssumedInformation = false;
11614
11616 Function *AnchorScope = getAnchorScope();
11617 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11618 bool AddValues) {
11620 Values.clear();
11621 if (!A.getAssumedSimplifiedValues(IRPosition::value(V), this, Values, S,
11622 UsedAssumedInformation,
11623 /* RecurseForSelectAndPHI */ true))
11624 return false;
11625 if (!AddValues)
11626 continue;
11627
11628 bool AllInterAreIntra = false;
11629 if (S == AA::Interprocedural)
11630 AllInterAreIntra =
11631 llvm::all_of(Values, [&](const AA::ValueAndContext &VAC) {
11632 return AA::isValidInScope(*VAC.getValue(), AnchorScope);
11633 });
11634
11635 for (const AA::ValueAndContext &VAC : Values) {
11636 addValue(A, getState(), *VAC.getValue(),
11637 VAC.getCtxI() ? VAC.getCtxI() : CtxI,
11638 AllInterAreIntra ? AA::AnyScope : S, AnchorScope);
11639 }
11640 if (AllInterAreIntra)
11641 break;
11642 }
11643 return true;
11644 };
11645
11646 if (ReturnedArg) {
11647 HandleReturnedValue(*ReturnedArg, nullptr, true);
11648 } else {
11649 auto RetInstPred = [&](Instruction &RetI) {
11650 bool AddValues = true;
11651 if (isa<PHINode>(RetI.getOperand(0)) ||
11652 isa<SelectInst>(RetI.getOperand(0))) {
11653 addValue(A, getState(), *RetI.getOperand(0), &RetI, AA::AnyScope,
11654 AnchorScope);
11655 AddValues = false;
11656 }
11657 return HandleReturnedValue(*RetI.getOperand(0), &RetI, AddValues);
11658 };
11659
11660 if (!A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
11661 UsedAssumedInformation,
11662 /* CheckBBLivenessOnly */ true))
11663 return indicatePessimisticFixpoint();
11664 }
11665
11666 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11667 : ChangeStatus::CHANGED;
11668 }
11669
11670 ChangeStatus manifest(Attributor &A) override {
11671 if (ReturnedArg)
11672 return ChangeStatus::UNCHANGED;
11674 if (!getAssumedSimplifiedValues(A, Values, AA::ValueScope::Intraprocedural,
11675 /* RecurseForSelectAndPHI */ true))
11676 return ChangeStatus::UNCHANGED;
11677 Value *NewVal = getSingleValue(A, *this, getIRPosition(), Values);
11678 if (!NewVal)
11679 return ChangeStatus::UNCHANGED;
11680
11681 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11682 if (auto *Arg = dyn_cast<Argument>(NewVal)) {
11683 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11684 "Number of function with unique return");
11685 Changed |= A.manifestAttrs(
11687 {Attribute::get(Arg->getContext(), Attribute::Returned)});
11688 STATS_DECLTRACK_ARG_ATTR(returned);
11689 }
11690
11691 auto RetInstPred = [&](Instruction &RetI) {
11692 Value *RetOp = RetI.getOperand(0);
11693 if (isa<UndefValue>(RetOp) || RetOp == NewVal)
11694 return true;
11695 if (AA::isValidAtPosition({*NewVal, RetI}, A.getInfoCache()))
11696 if (A.changeUseAfterManifest(RetI.getOperandUse(0), *NewVal))
11697 Changed = ChangeStatus::CHANGED;
11698 return true;
11699 };
11700 bool UsedAssumedInformation = false;
11701 (void)A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
11702 UsedAssumedInformation,
11703 /* CheckBBLivenessOnly */ true);
11704 return Changed;
11705 }
11706
11707 ChangeStatus indicatePessimisticFixpoint() override {
11708 return AAPotentialValues::indicatePessimisticFixpoint();
11709 }
11710
11711 void trackStatistics() const override{
11712 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11713
11714 /// The argumented with an existing `returned` attribute.
11715 Argument *ReturnedArg = nullptr;
11716};
11717
11718struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11719 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11720 : AAPotentialValuesImpl(IRP, A) {}
11721
11722 ChangeStatus updateImpl(Attributor &A) override {
11723 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11724 "not be called");
11725 }
11726
11727 void trackStatistics() const override {
11728 STATS_DECLTRACK_FN_ATTR(potential_values)
11729 }
11730};
11731
11732struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11733 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11734 : AAPotentialValuesFunction(IRP, A) {}
11735
11736 void trackStatistics() const override {
11737 STATS_DECLTRACK_CS_ATTR(potential_values)
11738 }
11739};
11740
11741struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11742 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11743 : AAPotentialValuesImpl(IRP, A) {}
11744
11745 ChangeStatus updateImpl(Attributor &A) override {
11746 auto AssumedBefore = getAssumed();
11747
11748 Function *Callee = getAssociatedFunction();
11749 if (!Callee)
11750 return indicatePessimisticFixpoint();
11751
11752 bool UsedAssumedInformation = false;
11753 auto *CB = cast<CallBase>(getCtxI());
11754 if (CB->isMustTailCall() &&
11755 !A.isAssumedDead(IRPosition::inst(*CB), this, nullptr,
11756 UsedAssumedInformation))
11757 return indicatePessimisticFixpoint();
11758
11759 Function *Caller = CB->getCaller();
11760
11761 auto AddScope = [&](AA::ValueScope S) {
11763 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
11764 Values, S, UsedAssumedInformation))
11765 return false;
11766
11767 for (auto &It : Values) {
11768 Value *V = It.getValue();
11769 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11770 V, *CB, *this, UsedAssumedInformation);
11771 if (!CallerV.has_value()) {
11772 // Nothing to do as long as no value was determined.
11773 continue;
11774 }
11775 V = *CallerV ? *CallerV : V;
11776 if (*CallerV && AA::isDynamicallyUnique(A, *this, *V)) {
11777 if (recurseForValue(A, IRPosition::value(*V), S))
11778 continue;
11779 }
11780 if (S == AA::Intraprocedural && !AA::isValidInScope(*V, Caller)) {
11781 giveUpOnIntraprocedural(A);
11782 return true;
11783 }
11784 addValue(A, getState(), *V, CB, S, getAnchorScope());
11785 }
11786 return true;
11787 };
11788 if (!AddScope(AA::Intraprocedural))
11789 return indicatePessimisticFixpoint();
11790 if (!AddScope(AA::Interprocedural))
11791 return indicatePessimisticFixpoint();
11792 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11793 : ChangeStatus::CHANGED;
11794 }
11795
11796 ChangeStatus indicatePessimisticFixpoint() override {
11797 return AAPotentialValues::indicatePessimisticFixpoint();
11798 }
11799
11800 void trackStatistics() const override {
11801 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11802 }
11803};
11804
11805struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11806 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11807 : AAPotentialValuesFloating(IRP, A) {}
11808
11809 void trackStatistics() const override {
11810 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11811 }
11812};
11813} // namespace
11814
11815/// ---------------------- Assumption Propagation ------------------------------
11816namespace {
11817struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11818 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11819 const DenseSet<StringRef> &Known)
11820 : AAAssumptionInfo(IRP, A, Known) {}
11821
11822 ChangeStatus manifest(Attributor &A) override {
11823 // Don't manifest a universal set if it somehow made it here.
11824 if (getKnown().isUniversal())
11825 return ChangeStatus::UNCHANGED;
11826
11827 const IRPosition &IRP = getIRPosition();
11828 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11829 getAssumed().getSet().end());
11830 llvm::sort(Set);
11831 return A.manifestAttrs(IRP,
11832 Attribute::get(IRP.getAnchorValue().getContext(),
11834 llvm::join(Set, ",")),
11835 /*ForceReplace=*/true);
11836 }
11837
11838 bool hasAssumption(const StringRef Assumption) const override {
11839 return isValidState() && setContains(Assumption);
11840 }
11841
11842 const std::string getAsStr(Attributor *A) const override {
11843 const SetContents &Known = getKnown();
11844 const SetContents &Assumed = getAssumed();
11845
11846 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11847 llvm::sort(Set);
11848 const std::string KnownStr = llvm::join(Set, ",");
11849
11850 std::string AssumedStr = "Universal";
11851 if (!Assumed.isUniversal()) {
11852 Set.assign(Assumed.getSet().begin(), Assumed.getSet().end());
11853 AssumedStr = llvm::join(Set, ",");
11854 }
11855 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11856 }
11857};
11858
11859/// Propagates assumption information from parent functions to all of their
11860/// successors. An assumption can be propagated if the containing function
11861/// dominates the called function.
11862///
11863/// We start with a "known" set of assumptions already valid for the associated
11864/// function and an "assumed" set that initially contains all possible
11865/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11866/// contents as concrete values are known. The concrete values are seeded by the
11867/// first nodes that are either entries into the call graph, or contains no
11868/// assumptions. Each node is updated as the intersection of the assumed state
11869/// with all of its predecessors.
11870struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11871 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11872 : AAAssumptionInfoImpl(IRP, A,
11873 getAssumptions(*IRP.getAssociatedFunction())) {}
11874
11875 ChangeStatus updateImpl(Attributor &A) override {
11876 bool Changed = false;
11877
11878 auto CallSitePred = [&](AbstractCallSite ACS) {
11879 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11880 *this, IRPosition::callsite_function(*ACS.getInstruction()),
11881 DepClassTy::REQUIRED);
11882 if (!AssumptionAA)
11883 return false;
11884 // Get the set of assumptions shared by all of this function's callers.
11885 Changed |= getIntersection(AssumptionAA->getAssumed());
11886 return !getAssumed().empty() || !getKnown().empty();
11887 };
11888
11889 bool UsedAssumedInformation = false;
11890 // Get the intersection of all assumptions held by this node's predecessors.
11891 // If we don't know all the call sites then this is either an entry into the
11892 // call graph or an empty node. This node is known to only contain its own
11893 // assumptions and can be propagated to its successors.
11894 if (!A.checkForAllCallSites(CallSitePred, *this, true,
11895 UsedAssumedInformation))
11896 return indicatePessimisticFixpoint();
11897
11898 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11899 }
11900
11901 void trackStatistics() const override {}
11902};
11903
11904/// Assumption Info defined for call sites.
11905struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11906
11907 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
11908 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
11909
11910 void initialize(Attributor &A) override {
11911 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
11912 A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
11913 }
11914
11915 ChangeStatus updateImpl(Attributor &A) override {
11916 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
11917 auto *AssumptionAA =
11918 A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
11919 if (!AssumptionAA)
11920 return indicatePessimisticFixpoint();
11921 bool Changed = getIntersection(AssumptionAA->getAssumed());
11922 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11923 }
11924
11925 void trackStatistics() const override {}
11926
11927private:
11928 /// Helper to initialized the known set as all the assumptions this call and
11929 /// the callee contain.
11930 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
11931 const CallBase &CB = cast<CallBase>(IRP.getAssociatedValue());
11932 auto Assumptions = getAssumptions(CB);
11933 if (const Function *F = CB.getCaller())
11934 set_union(Assumptions, getAssumptions(*F));
11935 if (Function *F = IRP.getAssociatedFunction())
11936 set_union(Assumptions, getAssumptions(*F));
11937 return Assumptions;
11938 }
11939};
11940} // namespace
11941
11943 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
11944 A.getOrCreateAAFor<AACallEdges>(IRPosition::function(**I))));
11945}
11946
11948
11949/// ------------------------ UnderlyingObjects ---------------------------------
11950
11951namespace {
11952struct AAUnderlyingObjectsImpl
11953 : StateWrapper<BooleanState, AAUnderlyingObjects> {
11955 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
11956
11957 const std::string getAsStr(Attributor *A) const override {
11958 if (!isValidState())
11959 return "<invalid>";
11960 std::string Str;
11962 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
11963 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
11964 << " objects.\n";
11965 if (!InterAssumedUnderlyingObjects.empty()) {
11966 OS << "inter objects:\n";
11967 for (auto *Obj : InterAssumedUnderlyingObjects)
11968 OS << *Obj << '\n';
11969 }
11970 if (!IntraAssumedUnderlyingObjects.empty()) {
11971 OS << "intra objects:\n";
11972 for (auto *Obj : IntraAssumedUnderlyingObjects)
11973 OS << *Obj << '\n';
11974 }
11975 return Str;
11976 }
11977
11978 void trackStatistics() const override {}
11979
11980 ChangeStatus updateImpl(Attributor &A) override {
11981 auto &Ptr = getAssociatedValue();
11982
11983 bool UsedAssumedInformation = false;
11984 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
11986 SmallPtrSet<Value *, 8> SeenObjects;
11988
11989 if (!A.getAssumedSimplifiedValues(IRPosition::value(Ptr), *this, Values,
11990 Scope, UsedAssumedInformation))
11991 return UnderlyingObjects.insert(&Ptr);
11992
11993 bool Changed = false;
11994
11995 for (unsigned I = 0; I < Values.size(); ++I) {
11996 auto &VAC = Values[I];
11997 auto *Obj = VAC.getValue();
11998 Value *UO = getUnderlyingObject(Obj);
11999 if (!SeenObjects.insert(UO ? UO : Obj).second)
12000 continue;
12001 if (UO && UO != Obj) {
12002 if (isa<AllocaInst>(UO) || isa<GlobalValue>(UO)) {
12003 Changed |= UnderlyingObjects.insert(UO);
12004 continue;
12005 }
12006
12007 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
12008 *this, IRPosition::value(*UO), DepClassTy::OPTIONAL);
12009 auto Pred = [&](Value &V) {
12010 if (&V == UO)
12011 Changed |= UnderlyingObjects.insert(UO);
12012 else
12013 Values.emplace_back(V, nullptr);
12014 return true;
12015 };
12016
12017 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
12019 "The forall call should not return false at this position");
12020 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
12021 continue;
12022 }
12023
12024 if (isa<SelectInst>(Obj)) {
12025 Changed |= handleIndirect(A, *Obj, UnderlyingObjects, Scope,
12026 UsedAssumedInformation);
12027 continue;
12028 }
12029 if (auto *PHI = dyn_cast<PHINode>(Obj)) {
12030 // Explicitly look through PHIs as we do not care about dynamically
12031 // uniqueness.
12032 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
12033 Changed |=
12034 handleIndirect(A, *PHI->getIncomingValue(u), UnderlyingObjects,
12035 Scope, UsedAssumedInformation);
12036 }
12037 continue;
12038 }
12039
12040 Changed |= UnderlyingObjects.insert(Obj);
12041 }
12042
12043 return Changed;
12044 };
12045
12046 bool Changed = false;
12047 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
12048 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
12049 if (!UsedAssumedInformation)
12050 indicateOptimisticFixpoint();
12051 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12052 }
12053
12054 bool forallUnderlyingObjects(
12055 function_ref<bool(Value &)> Pred,
12056 AA::ValueScope Scope = AA::Interprocedural) const override {
12057 if (!isValidState())
12058 return Pred(getAssociatedValue());
12059
12060 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
12061 ? IntraAssumedUnderlyingObjects
12062 : InterAssumedUnderlyingObjects;
12063 for (Value *Obj : AssumedUnderlyingObjects)
12064 if (!Pred(*Obj))
12065 return false;
12066
12067 return true;
12068 }
12069
12070private:
12071 /// Handle the case where the value is not the actual underlying value, such
12072 /// as a phi node or a select instruction.
12073 bool handleIndirect(Attributor &A, Value &V,
12074 SmallSetVector<Value *, 8> &UnderlyingObjects,
12075 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12076 bool Changed = false;
12077 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12078 *this, IRPosition::value(V), DepClassTy::OPTIONAL);
12079 auto Pred = [&](Value &V) {
12080 Changed |= UnderlyingObjects.insert(&V);
12081 return true;
12082 };
12083 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12085 "The forall call should not return false at this position");
12086 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12087 return Changed;
12088 }
12089
12090 /// All the underlying objects collected so far via intra procedural scope.
12091 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12092 /// All the underlying objects collected so far via inter procedural scope.
12093 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12094};
12095
12096struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12097 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12098 : AAUnderlyingObjectsImpl(IRP, A) {}
12099};
12100
12101struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12102 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12103 : AAUnderlyingObjectsImpl(IRP, A) {}
12104};
12105
12106struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12107 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12108 : AAUnderlyingObjectsImpl(IRP, A) {}
12109};
12110
12111struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12112 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12113 : AAUnderlyingObjectsImpl(IRP, A) {}
12114};
12115
12116struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12117 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12118 : AAUnderlyingObjectsImpl(IRP, A) {}
12119};
12120
12121struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12122 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12123 : AAUnderlyingObjectsImpl(IRP, A) {}
12124};
12125
12126struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12127 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12128 : AAUnderlyingObjectsImpl(IRP, A) {}
12129};
12130} // namespace
12131
12132/// ------------------------ Global Value Info -------------------------------
12133namespace {
12134struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12135 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12136 : AAGlobalValueInfo(IRP, A) {}
12137
12138 void initialize(Attributor &A) override {}
12139
12140 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12141 SmallVectorImpl<const Value *> &Worklist) {
12142 Instruction *UInst = dyn_cast<Instruction>(U.getUser());
12143 if (!UInst) {
12144 // Outside a closed world, code outside the module can read an
12145 // externally visible global, so the value escapes through it.
12146 if (auto *GV = dyn_cast<GlobalValue>(U.getUser()))
12147 if (!GV->hasLocalLinkage() && !A.isClosedWorldModule())
12148 return false;
12149 Follow = true;
12150 return true;
12151 }
12152
12153 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12154 << *UInst << "\n");
12155
12156 if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
12157 int Idx = &Cmp->getOperandUse(0) == &U;
12158 if (isa<Constant>(Cmp->getOperand(Idx)))
12159 return true;
12160 return U == &getAnchorValue();
12161 }
12162
12163 // Explicitly catch return instructions.
12164 if (isa<ReturnInst>(UInst)) {
12165 auto CallSitePred = [&](AbstractCallSite ACS) {
12166 Worklist.push_back(ACS.getInstruction());
12167 return true;
12168 };
12169 bool UsedAssumedInformation = false;
12170 // TODO: We should traverse the uses or add a "non-call-site" CB.
12171 if (!A.checkForAllCallSites(CallSitePred, *UInst->getFunction(),
12172 /*RequireAllCallSites=*/true, this,
12173 UsedAssumedInformation))
12174 return false;
12175 return true;
12176 }
12177
12178 // For now we only use special logic for call sites. However, the tracker
12179 // itself knows about a lot of other non-capturing cases already.
12180 auto *CB = dyn_cast<CallBase>(UInst);
12181 if (!CB)
12182 return false;
12183 // Direct calls are OK uses.
12184 if (CB->isCallee(&U))
12185 return true;
12186 // Non-argument uses are scary.
12187 if (!CB->isArgOperand(&U))
12188 return false;
12189 // TODO: Iterate callees.
12190 auto *Fn = dyn_cast<Function>(CB->getCalledOperand());
12191 if (!Fn || !A.isFunctionIPOAmendable(*Fn))
12192 return false;
12193
12194 unsigned ArgNo = CB->getArgOperandNo(&U);
12195 Worklist.push_back(Fn->getArg(ArgNo));
12196 return true;
12197 }
12198
12199 ChangeStatus updateImpl(Attributor &A) override {
12200 unsigned NumUsesBefore = Uses.size();
12201
12202 SmallPtrSet<const Value *, 8> Visited;
12204 Worklist.push_back(&getAnchorValue());
12205
12206 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12207 Uses.insert(&U);
12208 // TODO(captures): Make this more precise.
12209 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
12210 if (CI.isPassthrough()) {
12211 Follow = true;
12212 return true;
12213 }
12214 return checkUse(A, U, Follow, Worklist);
12215 };
12216 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12217 Uses.insert(&OldU);
12218 return true;
12219 };
12220
12221 while (!Worklist.empty()) {
12222 const Value *V = Worklist.pop_back_val();
12223 if (!Visited.insert(V).second)
12224 continue;
12225 if (!A.checkForAllUses(UsePred, *this, *V,
12226 /* CheckBBLivenessOnly */ true,
12227 DepClassTy::OPTIONAL,
12228 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12229 return indicatePessimisticFixpoint();
12230 }
12231 }
12232
12233 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12234 : ChangeStatus::CHANGED;
12235 }
12236
12237 bool isPotentialUse(const Use &U) const override {
12238 return !isValidState() || Uses.contains(&U);
12239 }
12240
12241 ChangeStatus manifest(Attributor &A) override {
12242 return ChangeStatus::UNCHANGED;
12243 }
12244
12245 const std::string getAsStr(Attributor *A) const override {
12246 return "[" + std::to_string(Uses.size()) + " uses]";
12247 }
12248
12249 void trackStatistics() const override {
12250 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12251 }
12252
12253private:
12254 /// Set of (transitive) uses of this GlobalValue.
12255 SmallPtrSet<const Use *, 8> Uses;
12256};
12257} // namespace
12258
12259/// ------------------------ Indirect Call Info -------------------------------
12260namespace {
12261struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12262 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12263 : AAIndirectCallInfo(IRP, A) {}
12264
12265 void initialize(Attributor &A) override {
12266 auto *MD = getCtxI()->getMetadata(LLVMContext::MD_callees);
12267 if (!MD && !A.isClosedWorldModule())
12268 return;
12269
12270 if (MD) {
12271 for (const auto &Op : MD->operands())
12273 PotentialCallees.insert(Callee);
12274 } else if (A.isClosedWorldModule()) {
12275 ArrayRef<Function *> IndirectlyCallableFunctions =
12276 A.getInfoCache().getIndirectlyCallableFunctions(A);
12277 PotentialCallees.insert_range(IndirectlyCallableFunctions);
12278 }
12279
12280 if (PotentialCallees.empty())
12281 indicateOptimisticFixpoint();
12282 }
12283
12284 ChangeStatus updateImpl(Attributor &A) override {
12285 CallBase *CB = cast<CallBase>(getCtxI());
12286 const Use &CalleeUse = CB->getCalledOperandUse();
12287 Value *FP = CB->getCalledOperand();
12288
12289 SmallSetVector<Function *, 4> AssumedCalleesNow;
12290 bool AllCalleesKnownNow = AllCalleesKnown;
12291
12292 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12293 bool &UsedAssumedInformation) {
12294 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12295 *this, IRPosition::value(PotentialCallee), DepClassTy::OPTIONAL);
12296 if (!GIAA || GIAA->isPotentialUse(CalleeUse))
12297 return true;
12298 UsedAssumedInformation = !GIAA->isAtFixpoint();
12299 return false;
12300 };
12301
12302 auto AddPotentialCallees = [&]() {
12303 for (auto *PotentialCallee : PotentialCallees) {
12304 bool UsedAssumedInformation = false;
12305 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12306 AssumedCalleesNow.insert(PotentialCallee);
12307 }
12308 };
12309
12310 // Use simplification to find potential callees, if !callees was present,
12311 // fallback to that set if necessary.
12312 bool UsedAssumedInformation = false;
12314 if (!A.getAssumedSimplifiedValues(IRPosition::value(*FP), this, Values,
12315 AA::ValueScope::AnyScope,
12316 UsedAssumedInformation)) {
12317 if (PotentialCallees.empty())
12318 return indicatePessimisticFixpoint();
12319 AddPotentialCallees();
12320 }
12321
12322 // Try to find a reason for \p Fn not to be a potential callee. If none was
12323 // found, add it to the assumed callees set.
12324 auto CheckPotentialCallee = [&](Function &Fn) {
12325 if (!PotentialCallees.empty() && !PotentialCallees.count(&Fn))
12326 return false;
12327
12328 auto &CachedResult = FilterResults[&Fn];
12329 if (CachedResult.has_value())
12330 return CachedResult.value();
12331
12332 bool UsedAssumedInformation = false;
12333 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12334 if (!UsedAssumedInformation)
12335 CachedResult = false;
12336 return false;
12337 }
12338
12339 int NumFnArgs = Fn.arg_size();
12340 int NumCBArgs = CB->arg_size();
12341
12342 // Check if any excess argument (which we fill up with poison) is known to
12343 // be UB on undef.
12344 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12345 bool IsKnown = false;
12347 A, this, IRPosition::argument(*Fn.getArg(I)),
12348 DepClassTy::OPTIONAL, IsKnown)) {
12349 if (IsKnown)
12350 CachedResult = false;
12351 return false;
12352 }
12353 }
12354
12355 CachedResult = true;
12356 return true;
12357 };
12358
12359 // Check simplification result, prune known UB callees, also restrict it to
12360 // the !callees set, if present.
12361 for (auto &VAC : Values) {
12362 if (isa<UndefValue>(VAC.getValue()))
12363 continue;
12365 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12366 continue;
12367 // TODO: Check for known UB, e.g., poison + noundef.
12368 if (auto *VACFn = dyn_cast<Function>(VAC.getValue())) {
12369 if (CheckPotentialCallee(*VACFn))
12370 AssumedCalleesNow.insert(VACFn);
12371 continue;
12372 }
12373 if (!PotentialCallees.empty()) {
12374 AddPotentialCallees();
12375 break;
12376 }
12377 AllCalleesKnownNow = false;
12378 }
12379
12380 if (AssumedCalleesNow == AssumedCallees &&
12381 AllCalleesKnown == AllCalleesKnownNow)
12382 return ChangeStatus::UNCHANGED;
12383
12384 std::swap(AssumedCallees, AssumedCalleesNow);
12385 AllCalleesKnown = AllCalleesKnownNow;
12386 return ChangeStatus::CHANGED;
12387 }
12388
12389 ChangeStatus manifest(Attributor &A) override {
12390 // If we can't specialize at all, give up now.
12391 if (!AllCalleesKnown && AssumedCallees.empty())
12392 return ChangeStatus::UNCHANGED;
12393
12394 CallBase *CB = cast<CallBase>(getCtxI());
12395 bool UsedAssumedInformation = false;
12396 if (A.isAssumedDead(*CB, this, /*LivenessAA=*/nullptr,
12397 UsedAssumedInformation))
12398 return ChangeStatus::UNCHANGED;
12399
12400 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12401 unsigned ProgramAS = CB->getDataLayout().getProgramAddressSpace();
12402 Value *FP = CB->getCalledOperand();
12403 if (FP->getType()->getPointerAddressSpace() != ProgramAS)
12404 FP = new AddrSpaceCastInst(
12405 FP, PointerType::get(FP->getContext(), ProgramAS),
12406 FP->getName() + ".as" + Twine(ProgramAS), CB->getIterator());
12407
12408 bool CBIsVoid = CB->getType()->isVoidTy();
12410 FunctionType *CSFT = CB->getFunctionType();
12411 SmallVector<Value *> CSArgs(CB->args());
12412
12413 // If we know all callees and there are none, the call site is (effectively)
12414 // dead (or UB).
12415 if (AssumedCallees.empty()) {
12416 assert(AllCalleesKnown &&
12417 "Expected all callees to be known if there are none.");
12418 A.changeToUnreachableAfterManifest(CB);
12419 return ChangeStatus::CHANGED;
12420 }
12421
12422 // Special handling for the single callee case.
12423 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12424 auto *NewCallee = AssumedCallees.front();
12425 if (isLegalToPromote(*CB, NewCallee)) {
12426 promoteCall(*CB, NewCallee, nullptr);
12427 NumIndirectCallsPromoted++;
12428 return ChangeStatus::CHANGED;
12429 }
12430 Instruction *NewCall =
12431 CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
12432 CB->getName(), CB->getIterator());
12433 NewCall->setDebugLoc(CB->getDebugLoc());
12434 if (!CBIsVoid)
12435 A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewCall);
12436 A.deleteAfterManifest(*CB);
12437 return ChangeStatus::CHANGED;
12438 }
12439
12440 // For each potential value we create a conditional
12441 //
12442 // ```
12443 // if (ptr == value) value(args);
12444 // else ...
12445 // ```
12446 //
12447 bool SpecializedForAnyCallees = false;
12448 bool SpecializedForAllCallees = AllCalleesKnown;
12449 ICmpInst *LastCmp = nullptr;
12450 SmallVector<Function *, 8> SkippedAssumedCallees;
12452 for (Function *NewCallee : AssumedCallees) {
12453 if (!A.shouldSpecializeCallSiteForCallee(*this, *CB, *NewCallee,
12454 AssumedCallees.size())) {
12455 SkippedAssumedCallees.push_back(NewCallee);
12456 SpecializedForAllCallees = false;
12457 continue;
12458 }
12459 SpecializedForAnyCallees = true;
12460
12461 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12462 Instruction *ThenTI =
12463 SplitBlockAndInsertIfThen(LastCmp, IP, /* Unreachable */ false);
12464 BasicBlock *CBBB = CB->getParent();
12465 A.registerManifestAddedBasicBlock(*ThenTI->getParent());
12466 A.registerManifestAddedBasicBlock(*IP->getParent());
12467 auto *SplitTI = cast<CondBrInst>(LastCmp->getNextNode());
12468 BasicBlock *ElseBB;
12469 if (&*IP == CB) {
12470 ElseBB = BasicBlock::Create(ThenTI->getContext(), "",
12471 ThenTI->getFunction(), CBBB);
12472 A.registerManifestAddedBasicBlock(*ElseBB);
12473 IP = UncondBrInst::Create(CBBB, ElseBB)->getIterator();
12474 SplitTI->replaceUsesOfWith(CBBB, ElseBB);
12475 } else {
12476 ElseBB = IP->getParent();
12477 ThenTI->replaceUsesOfWith(ElseBB, CBBB);
12478 }
12479 CastInst *RetBC = nullptr;
12480 CallInst *NewCall = nullptr;
12481 if (isLegalToPromote(*CB, NewCallee)) {
12482 auto *CBClone = cast<CallBase>(CB->clone());
12483 CBClone->insertBefore(ThenTI->getIterator());
12484 NewCall = &cast<CallInst>(promoteCall(*CBClone, NewCallee, &RetBC));
12485 NumIndirectCallsPromoted++;
12486 } else {
12487 NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
12488 CB->getName(), ThenTI->getIterator());
12489 NewCall->setDebugLoc(CB->getDebugLoc());
12490 }
12491 NewCalls.push_back({NewCall, RetBC});
12492 }
12493
12494 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12495 if (!AllCalleesKnown)
12496 return ChangeStatus::UNCHANGED;
12497 MDBuilder MDB(IndirectCB.getContext());
12498 MDNode *Callees = MDB.createCallees(SkippedAssumedCallees);
12499 IndirectCB.setMetadata(LLVMContext::MD_callees, Callees);
12500 return ChangeStatus::CHANGED;
12501 };
12502
12503 if (!SpecializedForAnyCallees)
12504 return AttachCalleeMetadata(*CB);
12505
12506 // Check if we need the fallback indirect call still.
12507 if (SpecializedForAllCallees) {
12509 LastCmp->eraseFromParent();
12510 new UnreachableInst(IP->getContext(), IP);
12511 IP->eraseFromParent();
12512 } else {
12513 auto *CBClone = cast<CallInst>(CB->clone());
12514 CBClone->setName(CB->getName());
12515 CBClone->insertBefore(*IP->getParent(), IP);
12516 NewCalls.push_back({CBClone, nullptr});
12517 AttachCalleeMetadata(*CBClone);
12518 }
12519
12520 // Check if we need a PHI to merge the results.
12521 if (!CBIsVoid) {
12522 auto *PHI = PHINode::Create(CB->getType(), NewCalls.size(),
12523 CB->getName() + ".phi",
12524 CB->getParent()->getFirstInsertionPt());
12525 for (auto &It : NewCalls) {
12526 CallBase *NewCall = It.first;
12527 Instruction *CallRet = It.second ? It.second : It.first;
12528 if (CallRet->getType() == CB->getType())
12529 PHI->addIncoming(CallRet, CallRet->getParent());
12530 else if (NewCall->getType()->isVoidTy())
12531 PHI->addIncoming(PoisonValue::get(CB->getType()),
12532 NewCall->getParent());
12533 else
12534 llvm_unreachable("Call return should match or be void!");
12535 }
12536 A.changeAfterManifest(IRPosition::callsite_returned(*CB), *PHI);
12537 }
12538
12539 A.deleteAfterManifest(*CB);
12540 Changed = ChangeStatus::CHANGED;
12541
12542 return Changed;
12543 }
12544
12545 const std::string getAsStr(Attributor *A) const override {
12546 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12547 " indirect call site with " + std::to_string(AssumedCallees.size()) +
12548 " functions";
12549 }
12550
12551 void trackStatistics() const override {
12552 if (AllCalleesKnown) {
12554 Eliminated, CallSites,
12555 "Number of indirect call sites eliminated via specialization")
12556 } else {
12557 STATS_DECLTRACK(Specialized, CallSites,
12558 "Number of indirect call sites specialized")
12559 }
12560 }
12561
12562 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12563 return isValidState() && AllCalleesKnown && all_of(AssumedCallees, CB);
12564 }
12565
12566private:
12567 /// Map to remember filter results.
12568 DenseMap<Function *, std::optional<bool>> FilterResults;
12569
12570 /// If the !callee metadata was present, this set will contain all potential
12571 /// callees (superset).
12572 SmallSetVector<Function *, 4> PotentialCallees;
12573
12574 /// This set contains all currently assumed calllees, which might grow over
12575 /// time.
12576 SmallSetVector<Function *, 4> AssumedCallees;
12577
12578 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12579 /// if there could be others.
12580 bool AllCalleesKnown = true;
12581};
12582} // namespace
12583
12584/// --------------------- Invariant Load Pointer -------------------------------
12585namespace {
12586
12587struct AAInvariantLoadPointerImpl
12588 : public StateWrapper<BitIntegerState<uint8_t, 15>,
12589 AAInvariantLoadPointer> {
12590
12591 enum {
12592 // pointer does not alias within the bounds of the function
12593 IS_NOALIAS = 1 << 0,
12594 // pointer is not involved in any effectful instructions within the bounds
12595 // of the function
12596 IS_NOEFFECT = 1 << 1,
12597 // loads are invariant within the bounds of the function
12598 IS_LOCALLY_INVARIANT = 1 << 2,
12599 // memory lifetime is constrained within the bounds of the function
12600 IS_LOCALLY_CONSTRAINED = 1 << 3,
12601
12602 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12603 IS_LOCALLY_CONSTRAINED,
12604 };
12605 static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
12606
12607 using Base =
12608 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12609
12610 // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
12611 // pessimistic about IS_KNOWN_INVARIANT
12612 AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
12613 : Base(IRP) {}
12614
12615 bool isKnownInvariant() const final {
12616 return isKnownLocallyInvariant() && isKnown(IS_LOCALLY_CONSTRAINED);
12617 }
12618
12619 bool isKnownLocallyInvariant() const final {
12620 if (isKnown(IS_LOCALLY_INVARIANT))
12621 return true;
12622 return isKnown(IS_NOALIAS | IS_NOEFFECT);
12623 }
12624
12625 bool isAssumedInvariant() const final {
12626 return isAssumedLocallyInvariant() && isAssumed(IS_LOCALLY_CONSTRAINED);
12627 }
12628
12629 bool isAssumedLocallyInvariant() const final {
12630 if (isAssumed(IS_LOCALLY_INVARIANT))
12631 return true;
12632 return isAssumed(IS_NOALIAS | IS_NOEFFECT);
12633 }
12634
12635 ChangeStatus updateImpl(Attributor &A) override {
12636 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12637
12638 Changed |= updateNoAlias(A);
12639 if (requiresNoAlias() && !isAssumed(IS_NOALIAS))
12640 return indicatePessimisticFixpoint();
12641
12642 Changed |= updateNoEffect(A);
12643
12644 Changed |= updateLocalInvariance(A);
12645
12646 return Changed;
12647 }
12648
12649 ChangeStatus manifest(Attributor &A) override {
12650 if (!isKnownInvariant())
12651 return ChangeStatus::UNCHANGED;
12652
12653 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12654 const Value *Ptr = &getAssociatedValue();
12655 const auto TagInvariantLoads = [&](const Use &U, bool &) {
12656 if (U.get() != Ptr)
12657 return true;
12658 auto *I = dyn_cast<Instruction>(U.getUser());
12659 if (!I)
12660 return true;
12661
12662 // Ensure that we are only changing uses from the corresponding callgraph
12663 // SSC in the case that the AA isn't run on the entire module
12664 if (!A.isRunOn(I->getFunction()))
12665 return true;
12666
12667 if (I->hasMetadata(LLVMContext::MD_invariant_load))
12668 return true;
12669
12670 if (auto *LI = dyn_cast<LoadInst>(I)) {
12671 LI->setMetadata(LLVMContext::MD_invariant_load,
12672 MDNode::get(LI->getContext(), {}));
12673 Changed = ChangeStatus::CHANGED;
12674 }
12675 return true;
12676 };
12677
12678 (void)A.checkForAllUses(TagInvariantLoads, *this, *Ptr);
12679 return Changed;
12680 }
12681
12682 const std::string getAsStr(Attributor *) const override {
12683 if (isKnownInvariant())
12684 return "load-invariant pointer";
12685 return "non-invariant pointer";
12686 }
12687
12688 void trackStatistics() const override {}
12689
12690private:
12691 /// Indicate that noalias is required for the pointer to be invariant.
12692 bool requiresNoAlias() const {
12693 switch (getPositionKind()) {
12694 default:
12695 // Conservatively default to require noalias.
12696 return true;
12697 case IRP_FLOAT:
12698 case IRP_RETURNED:
12699 case IRP_CALL_SITE:
12700 return false;
12701 case IRP_CALL_SITE_RETURNED: {
12702 const auto &CB = cast<CallBase>(getAnchorValue());
12704 &CB, /*MustPreserveOffset=*/false);
12705 }
12706 case IRP_ARGUMENT: {
12707 const Function *F = getAssociatedFunction();
12708 assert(F && "no associated function for argument");
12709 return !isCallableCC(F->getCallingConv());
12710 }
12711 }
12712 }
12713
12714 bool isExternal() const {
12715 const Function *F = getAssociatedFunction();
12716 if (!F)
12717 return true;
12718 return isCallableCC(F->getCallingConv()) &&
12719 getPositionKind() != IRP_CALL_SITE_RETURNED;
12720 }
12721
12722 ChangeStatus updateNoAlias(Attributor &A) {
12723 if (isKnown(IS_NOALIAS) || !isAssumed(IS_NOALIAS))
12724 return ChangeStatus::UNCHANGED;
12725
12726 // Try to use AANoAlias.
12727 if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
12728 getIRPosition(), this, DepClassTy::REQUIRED)) {
12729 if (ANoAlias->isKnownNoAlias()) {
12730 addKnownBits(IS_NOALIAS);
12731 return ChangeStatus::CHANGED;
12732 }
12733
12734 if (!ANoAlias->isAssumedNoAlias()) {
12735 removeAssumedBits(IS_NOALIAS);
12736 return ChangeStatus::CHANGED;
12737 }
12738
12739 return ChangeStatus::UNCHANGED;
12740 }
12741
12742 // Try to infer noalias from argument attribute, since it is applicable for
12743 // the duration of the function.
12744 if (const Argument *Arg = getAssociatedArgument()) {
12745 if (Arg->hasNoAliasAttr()) {
12746 addKnownBits(IS_NOALIAS);
12747 return ChangeStatus::UNCHANGED;
12748 }
12749
12750 // Noalias information is not provided, and cannot be inferred,
12751 // so we conservatively assume the pointer aliases.
12752 removeAssumedBits(IS_NOALIAS);
12753 return ChangeStatus::CHANGED;
12754 }
12755
12756 return ChangeStatus::UNCHANGED;
12757 }
12758
12759 ChangeStatus updateNoEffect(Attributor &A) {
12760 if (isKnown(IS_NOEFFECT) || !isAssumed(IS_NOEFFECT))
12761 return ChangeStatus::UNCHANGED;
12762
12763 if (!getAssociatedFunction())
12764 return indicatePessimisticFixpoint();
12765
12766 if (isa<AllocaInst>(&getAssociatedValue()))
12767 return indicatePessimisticFixpoint();
12768
12769 const auto HasNoEffectLoads = [&](const Use &U, bool &) {
12770 const auto *LI = dyn_cast<LoadInst>(U.getUser());
12771 return !LI || !LI->mayHaveSideEffects();
12772 };
12773 if (!A.checkForAllUses(HasNoEffectLoads, *this, getAssociatedValue()))
12774 return indicatePessimisticFixpoint();
12775
12776 if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
12777 getIRPosition(), this, DepClassTy::REQUIRED)) {
12778 // For non-instructions, try to use AAMemoryBehavior to infer the readonly
12779 // attribute
12780 if (!AMemoryBehavior->isAssumedReadOnly())
12781 return indicatePessimisticFixpoint();
12782
12783 if (AMemoryBehavior->isKnownReadOnly()) {
12784 addKnownBits(IS_NOEFFECT);
12785 return ChangeStatus::UNCHANGED;
12786 }
12787
12788 return ChangeStatus::UNCHANGED;
12789 }
12790
12791 if (const Argument *Arg = getAssociatedArgument()) {
12792 if (Arg->onlyReadsMemory()) {
12793 addKnownBits(IS_NOEFFECT);
12794 return ChangeStatus::UNCHANGED;
12795 }
12796
12797 // Readonly information is not provided, and cannot be inferred from
12798 // AAMemoryBehavior.
12799 return indicatePessimisticFixpoint();
12800 }
12801
12802 return ChangeStatus::UNCHANGED;
12803 }
12804
12805 ChangeStatus updateLocalInvariance(Attributor &A) {
12806 if (isKnown(IS_LOCALLY_INVARIANT) || !isAssumed(IS_LOCALLY_INVARIANT))
12807 return ChangeStatus::UNCHANGED;
12808
12809 // try to infer invariance from underlying objects
12810 const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
12811 getIRPosition(), this, DepClassTy::REQUIRED);
12812 if (!AUO)
12813 return ChangeStatus::UNCHANGED;
12814
12815 bool UsedAssumedInformation = false;
12816 const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
12817 if (!V.getType()->isPointerTy())
12818 return true;
12819 const auto *IsInvariantLoadPointer =
12820 A.getOrCreateAAFor<AAInvariantLoadPointer>(IRPosition::value(V), this,
12821 DepClassTy::REQUIRED);
12822 // Conservatively fail if invariance cannot be inferred.
12823 if (!IsInvariantLoadPointer)
12824 return false;
12825
12826 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12827 return true;
12828 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12829 return false;
12830
12831 UsedAssumedInformation = true;
12832 return true;
12833 };
12834 if (!AUO->forallUnderlyingObjects(IsLocallyInvariantLoadIfPointer))
12835 return indicatePessimisticFixpoint();
12836
12837 if (const auto *CB = dyn_cast<CallBase>(&getAnchorValue())) {
12839 CB, /*MustPreserveOffset=*/false)) {
12840 for (const Value *Arg : CB->args()) {
12841 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12842 return indicatePessimisticFixpoint();
12843 }
12844 }
12845 }
12846
12847 if (!UsedAssumedInformation) {
12848 // Pointer is known and not just assumed to be locally invariant.
12849 addKnownBits(IS_LOCALLY_INVARIANT);
12850 return ChangeStatus::CHANGED;
12851 }
12852
12853 return ChangeStatus::UNCHANGED;
12854 }
12855};
12856
12857struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12858 AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
12859 : AAInvariantLoadPointerImpl(IRP, A) {}
12860};
12861
12862struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12863 AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
12864 : AAInvariantLoadPointerImpl(IRP, A) {}
12865
12866 void initialize(Attributor &) override {
12867 removeAssumedBits(IS_LOCALLY_CONSTRAINED);
12868 }
12869};
12870
12871struct AAInvariantLoadPointerCallSiteReturned final
12872 : AAInvariantLoadPointerImpl {
12873 AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
12874 : AAInvariantLoadPointerImpl(IRP, A) {}
12875
12876 void initialize(Attributor &A) override {
12877 // Nothing is known about the result of an indirect call.
12878 const Function *F = getAssociatedFunction();
12879 if (!F) {
12880 indicatePessimisticFixpoint();
12881 return;
12882 }
12883
12884 if (!F->isDeclaration() && !F->isIntrinsic())
12885 return AAInvariantLoadPointerImpl::initialize(A);
12886
12887 const auto &CB = cast<CallBase>(getAnchorValue());
12889 &CB, /*MustPreserveOffset=*/false))
12890 return AAInvariantLoadPointerImpl::initialize(A);
12891
12892 if (F->onlyReadsMemory() && F->hasNoSync())
12893 return AAInvariantLoadPointerImpl::initialize(A);
12894
12895 // At this point, the function is opaque, so we conservatively assume
12896 // non-invariance.
12897 indicatePessimisticFixpoint();
12898 }
12899};
12900
12901struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
12902 AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
12903 : AAInvariantLoadPointerImpl(IRP, A) {}
12904
12905 void initialize(Attributor &) override {
12906 const Function *F = getAssociatedFunction();
12907 assert(F && "no associated function for argument");
12908
12909 if (!isCallableCC(F->getCallingConv())) {
12910 addKnownBits(IS_LOCALLY_CONSTRAINED);
12911 return;
12912 }
12913
12914 if (!F->hasLocalLinkage())
12915 removeAssumedBits(IS_LOCALLY_CONSTRAINED);
12916 }
12917};
12918
12919struct AAInvariantLoadPointerCallSiteArgument final
12920 : AAInvariantLoadPointerImpl {
12921 AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
12922 : AAInvariantLoadPointerImpl(IRP, A) {}
12923};
12924} // namespace
12925
12926/// ------------------------ Address Space ------------------------------------
12927namespace {
12928
12929template <typename InstType>
12930static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,
12931 Value *OriginalValue, PointerType *NewPtrTy,
12932 bool UseOriginalValue) {
12933 if (U.getOperandNo() != InstType::getPointerOperandIndex())
12934 return false;
12935
12936 if (MemInst->isVolatile()) {
12937 auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
12938 *MemInst->getFunction());
12939 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
12940 if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))
12941 return false;
12942 }
12943
12944 if (UseOriginalValue) {
12945 A.changeUseAfterManifest(const_cast<Use &>(U), *OriginalValue);
12946 return true;
12947 }
12948
12949 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
12950 CastInst->insertBefore(MemInst->getIterator());
12951 A.changeUseAfterManifest(const_cast<Use &>(U), *CastInst);
12952 return true;
12953}
12954
12955struct AAAddressSpaceImpl : public AAAddressSpace {
12956 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
12957 : AAAddressSpace(IRP, A) {}
12958
12959 uint32_t getAddressSpace() const override {
12960 assert(isValidState() && "the AA is invalid");
12961 return AssumedAddressSpace;
12962 }
12963
12964 void initialize(Attributor &A) override {
12965 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
12966 "Associated value is not a pointer");
12967
12968 if (!A.getInfoCache().getFlatAddressSpace().has_value()) {
12969 indicatePessimisticFixpoint();
12970 return;
12971 }
12972
12973 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
12974 unsigned AS = getAssociatedType()->getPointerAddressSpace();
12975 if (AS != FlatAS) {
12976 [[maybe_unused]] bool R = takeAddressSpace(AS);
12977 assert(R && "The take should happen");
12978 indicateOptimisticFixpoint();
12979 }
12980 }
12981
12982 ChangeStatus updateImpl(Attributor &A) override {
12983 uint32_t OldAddressSpace = AssumedAddressSpace;
12984 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
12985
12986 auto CheckAddressSpace = [&](Value &Obj) {
12987 // Ignore undef.
12988 if (isa<UndefValue>(&Obj))
12989 return true;
12990
12991 // If the object already has a non-flat address space, we simply take it.
12992 unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
12993 if (ObjAS != FlatAS)
12994 return takeAddressSpace(ObjAS);
12995
12996 // At this point, we know Obj is in the flat address space. For a final
12997 // attempt, we want to use getAssumedAddrSpace, but first we must get the
12998 // associated function, if possible.
12999 Function *F = nullptr;
13000 if (auto *Arg = dyn_cast<Argument>(&Obj))
13001 F = Arg->getParent();
13002 else if (auto *I = dyn_cast<Instruction>(&Obj))
13003 F = I->getFunction();
13004
13005 // Use getAssumedAddrSpace if the associated function exists.
13006 if (F) {
13007 auto *TTI =
13008 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(*F);
13009 unsigned AssumedAS = TTI->getAssumedAddrSpace(&Obj);
13010 if (AssumedAS != ~0U)
13011 return takeAddressSpace(AssumedAS);
13012 }
13013
13014 // Now we can't do anything else but to take the flat AS.
13015 return takeAddressSpace(FlatAS);
13016 };
13017
13018 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(), this,
13019 DepClassTy::REQUIRED);
13020 if (!AUO->forallUnderlyingObjects(CheckAddressSpace))
13021 return indicatePessimisticFixpoint();
13022
13023 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13024 : ChangeStatus::CHANGED;
13025 }
13026
13027 ChangeStatus manifest(Attributor &A) override {
13028 unsigned NewAS = getAddressSpace();
13029
13030 if (NewAS == InvalidAddressSpace ||
13031 NewAS == getAssociatedType()->getPointerAddressSpace())
13032 return ChangeStatus::UNCHANGED;
13033
13034 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13035
13036 Value *AssociatedValue = &getAssociatedValue();
13037 Value *OriginalValue = peelAddrspacecast(AssociatedValue, FlatAS);
13038
13039 PointerType *NewPtrTy =
13040 PointerType::get(getAssociatedType()->getContext(), NewAS);
13041 bool UseOriginalValue =
13042 OriginalValue->getType()->getPointerAddressSpace() == NewAS;
13043
13044 bool Changed = false;
13045
13046 auto Pred = [&](const Use &U, bool &) {
13047 if (U.get() != AssociatedValue)
13048 return true;
13049 auto *Inst = dyn_cast<Instruction>(U.getUser());
13050 if (!Inst)
13051 return true;
13052 // This is a WA to make sure we only change uses from the corresponding
13053 // CGSCC if the AA is run on CGSCC instead of the entire module.
13054 if (!A.isRunOn(Inst->getFunction()))
13055 return true;
13056 if (auto *LI = dyn_cast<LoadInst>(Inst)) {
13057 Changed |=
13058 makeChange(A, LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13059 } else if (auto *SI = dyn_cast<StoreInst>(Inst)) {
13060 Changed |=
13061 makeChange(A, SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13062 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Inst)) {
13063 Changed |=
13064 makeChange(A, RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13065 } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {
13066 Changed |=
13067 makeChange(A, CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13068 }
13069 return true;
13070 };
13071
13072 // It doesn't matter if we can't check all uses as we can simply
13073 // conservatively ignore those that can not be visited.
13074 (void)A.checkForAllUses(Pred, *this, getAssociatedValue(),
13075 /* CheckBBLivenessOnly */ true);
13076
13077 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13078 }
13079
13080 const std::string getAsStr(Attributor *A) const override {
13081 if (!isValidState())
13082 return "addrspace(<invalid>)";
13083 return "addrspace(" +
13084 (AssumedAddressSpace == InvalidAddressSpace
13085 ? "none"
13086 : std::to_string(AssumedAddressSpace)) +
13087 ")";
13088 }
13089
13090private:
13091 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13092
13093 bool takeAddressSpace(uint32_t AS) {
13094 if (AssumedAddressSpace == InvalidAddressSpace) {
13095 AssumedAddressSpace = AS;
13096 return true;
13097 }
13098 return AssumedAddressSpace == AS;
13099 }
13100
13101 static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {
13102 if (auto *I = dyn_cast<AddrSpaceCastInst>(V)) {
13103 assert(I->getSrcAddressSpace() != FlatAS &&
13104 "there should not be flat AS -> non-flat AS");
13105 return I->getPointerOperand();
13106 }
13107 if (auto *C = dyn_cast<ConstantExpr>(V))
13108 if (C->getOpcode() == Instruction::AddrSpaceCast) {
13109 assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=
13110 FlatAS &&
13111 "there should not be flat AS -> non-flat AS X");
13112 return C->getOperand(0);
13113 }
13114 return V;
13115 }
13116};
13117
13118struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13119 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
13120 : AAAddressSpaceImpl(IRP, A) {}
13121
13122 void trackStatistics() const override {
13124 }
13125};
13126
13127struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13128 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
13129 : AAAddressSpaceImpl(IRP, A) {}
13130
13131 void initialize(Attributor &A) override {
13132 // TODO: we don't rewrite function argument for now because it will need to
13133 // rewrite the function signature and all call sites.
13134 (void)indicatePessimisticFixpoint();
13135 }
13136
13137 void trackStatistics() const override {
13138 STATS_DECLTRACK_FNRET_ATTR(addrspace);
13139 }
13140};
13141
13142struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13143 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13144 : AAAddressSpaceImpl(IRP, A) {}
13145
13146 void trackStatistics() const override {
13147 STATS_DECLTRACK_CSRET_ATTR(addrspace);
13148 }
13149};
13150
13151struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13152 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
13153 : AAAddressSpaceImpl(IRP, A) {}
13154
13155 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
13156};
13157
13158struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13159 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13160 : AAAddressSpaceImpl(IRP, A) {}
13161
13162 void initialize(Attributor &A) override {
13163 // TODO: we don't rewrite call site argument for now because it will need to
13164 // rewrite the function signature of the callee.
13165 (void)indicatePessimisticFixpoint();
13166 }
13167
13168 void trackStatistics() const override {
13169 STATS_DECLTRACK_CSARG_ATTR(addrspace);
13170 }
13171};
13172} // namespace
13173
13174/// ------------------------ No Alias Address Space ---------------------------
13175// This attribute assumes flat address space can alias all other address space
13176
13177// TODO: this is similar to AAAddressSpace, most of the code should be merged.
13178// But merging it created failing cased on gateway test that cannot be
13179// reproduced locally. So should open a separated PR to handle the merge of
13180// AANoAliasAddrSpace and AAAddressSpace attribute
13181
13182namespace {
13183struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
13184 AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
13185 : AANoAliasAddrSpace(IRP, A) {}
13186
13187 void initialize(Attributor &A) override {
13188 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13189 "Associated value is not a pointer");
13190
13191 resetASRanges(A);
13192
13193 std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
13194 if (!FlatAS.has_value()) {
13195 indicatePessimisticFixpoint();
13196 return;
13197 }
13198
13199 removeAS(*FlatAS);
13200
13201 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13202 if (AS != *FlatAS) {
13203 removeAS(AS);
13204 indicateOptimisticFixpoint();
13205 }
13206 }
13207
13208 ChangeStatus updateImpl(Attributor &A) override {
13209 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13210 uint32_t OldAssumed = getAssumed();
13211
13212 auto CheckAddressSpace = [&](Value &Obj) {
13213 if (isa<PoisonValue>(&Obj))
13214 return true;
13215
13216 unsigned AS = Obj.getType()->getPointerAddressSpace();
13217 if (AS == FlatAS)
13218 return false;
13219
13220 removeAS(Obj.getType()->getPointerAddressSpace());
13221 return true;
13222 };
13223
13224 const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
13225 getIRPosition(), this, DepClassTy::REQUIRED);
13226 if (!AUO->forallUnderlyingObjects(CheckAddressSpace))
13227 return indicatePessimisticFixpoint();
13228
13229 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13230 : ChangeStatus::CHANGED;
13231 }
13232
13233 ChangeStatus manifest(Attributor &A) override {
13234 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13235
13236 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13237 if (AS != FlatAS || Map.empty())
13238 return ChangeStatus::UNCHANGED;
13239
13240 LLVMContext &Ctx = getAssociatedValue().getContext();
13241 MDNode *NoAliasASNode = nullptr;
13242 MDBuilder MDB(Ctx);
13243 // Has to use iterator to get the range info.
13244 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13245 if (!I.value())
13246 continue;
13247 unsigned Upper = I.stop();
13248 unsigned Lower = I.start();
13249 if (!NoAliasASNode) {
13250 NoAliasASNode = MDB.createRange(APInt(32, Lower), APInt(32, Upper + 1));
13251 continue;
13252 }
13253 MDNode *ASRange = MDB.createRange(APInt(32, Lower), APInt(32, Upper + 1));
13254 NoAliasASNode = MDNode::getMostGenericRange(NoAliasASNode, ASRange);
13255 }
13256
13257 Value *AssociatedValue = &getAssociatedValue();
13258 bool Changed = false;
13259
13260 auto AddNoAliasAttr = [&](const Use &U, bool &) {
13261 if (U.get() != AssociatedValue)
13262 return true;
13263 Instruction *Inst = dyn_cast<Instruction>(U.getUser());
13264 if (!Inst || Inst->hasMetadata(LLVMContext::MD_noalias_addrspace))
13265 return true;
13266 if (!isa<LoadInst>(Inst) && !isa<StoreInst>(Inst) &&
13268 return true;
13269 if (!A.isRunOn(Inst->getFunction()))
13270 return true;
13271 Inst->setMetadata(LLVMContext::MD_noalias_addrspace, NoAliasASNode);
13272 Changed = true;
13273 return true;
13274 };
13275 (void)A.checkForAllUses(AddNoAliasAttr, *this, *AssociatedValue,
13276 /*CheckBBLivenessOnly=*/true);
13277 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13278 }
13279
13280 const std::string getAsStr(Attributor *A) const override {
13281 if (!isValidState())
13282 return "<invalid>";
13283 std::string Str;
13284 raw_string_ostream OS(Str);
13285 OS << "CanNotBeAddrSpace(";
13286 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13287 unsigned Upper = I.stop();
13288 unsigned Lower = I.start();
13289 OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
13290 }
13291 OS << " )";
13292 return OS.str();
13293 }
13294
13295private:
13296 void removeAS(unsigned AS) {
13297 RangeMap::iterator I = Map.find(AS);
13298
13299 if (I != Map.end()) {
13300 unsigned Upper = I.stop();
13301 unsigned Lower = I.start();
13302 I.erase();
13303 if (Upper == Lower)
13304 return;
13305 if (AS != ~((unsigned)0) && AS + 1 <= Upper)
13306 Map.insert(AS + 1, Upper, /*what ever this variable name is=*/true);
13307 if (AS != 0 && Lower <= AS - 1)
13308 Map.insert(Lower, AS - 1, true);
13309 }
13310 }
13311
13312 void resetASRanges(Attributor &A) {
13313 Map.clear();
13314 Map.insert(0, A.getInfoCache().getMaxAddrSpace(), true);
13315 }
13316};
13317
13318struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13319 AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
13320 : AANoAliasAddrSpaceImpl(IRP, A) {}
13321
13322 void trackStatistics() const override {
13323 STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
13324 }
13325};
13326
13327struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13328 AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
13329 : AANoAliasAddrSpaceImpl(IRP, A) {}
13330
13331 void trackStatistics() const override {
13332 STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
13333 }
13334};
13335
13336struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13337 AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13338 : AANoAliasAddrSpaceImpl(IRP, A) {}
13339
13340 void trackStatistics() const override {
13341 STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
13342 }
13343};
13344
13345struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13346 AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
13347 : AANoAliasAddrSpaceImpl(IRP, A) {}
13348
13349 void trackStatistics() const override {
13350 STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
13351 }
13352};
13353
13354struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13355 AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13356 : AANoAliasAddrSpaceImpl(IRP, A) {}
13357
13358 void trackStatistics() const override {
13359 STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
13360 }
13361};
13362} // namespace
13363/// ----------- Allocation Info ----------
13364namespace {
13365struct AAAllocationInfoImpl : public AAAllocationInfo {
13366 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
13367 : AAAllocationInfo(IRP, A) {}
13368
13369 std::optional<TypeSize> getAllocatedSize() const override {
13370 assert(isValidState() && "the AA is invalid");
13371 return AssumedAllocatedSize;
13372 }
13373
13374 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
13375 const DataLayout &DL) {
13376
13377 // TODO: implement case for malloc like instructions
13378 switch (I->getOpcode()) {
13379 case Instruction::Alloca: {
13380 AllocaInst *AI = cast<AllocaInst>(I);
13381 return AI->getAllocationSize(DL);
13382 }
13383 default:
13384 return std::nullopt;
13385 }
13386 }
13387
13388 ChangeStatus updateImpl(Attributor &A) override {
13389
13390 const IRPosition &IRP = getIRPosition();
13391 Instruction *I = IRP.getCtxI();
13392
13393 // TODO: update check for malloc like calls
13394 if (!isa<AllocaInst>(I))
13395 return indicatePessimisticFixpoint();
13396
13397 bool IsKnownNoCapture;
13399 A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
13400 return indicatePessimisticFixpoint();
13401
13402 const AAPointerInfo *PI =
13403 A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
13404
13405 if (!PI)
13406 return indicatePessimisticFixpoint();
13407
13408 if (!PI->getState().isValidState() || PI->reachesReturn())
13409 return indicatePessimisticFixpoint();
13410
13411 const DataLayout &DL = A.getDataLayout();
13412 const auto AllocationSize = findInitialAllocationSize(I, DL);
13413
13414 // If allocation size is nullopt, we give up.
13415 if (!AllocationSize)
13416 return indicatePessimisticFixpoint();
13417
13418 // For zero sized allocations, we give up.
13419 // Since we can't reduce further
13420 if (*AllocationSize == 0)
13421 return indicatePessimisticFixpoint();
13422
13423 int64_t BinSize = PI->numOffsetBins();
13424
13425 // TODO: implement for multiple bins
13426 if (BinSize > 1)
13427 return indicatePessimisticFixpoint();
13428
13429 if (BinSize == 0) {
13430 auto NewAllocationSize = std::make_optional<TypeSize>(0, false);
13431 if (!changeAllocationSize(NewAllocationSize))
13432 return ChangeStatus::UNCHANGED;
13433 return ChangeStatus::CHANGED;
13434 }
13435
13436 // TODO: refactor this to be part of multiple bin case
13437 const auto &It = PI->begin();
13438
13439 // TODO: handle if Offset is not zero
13440 if (It->first.Offset != 0)
13441 return indicatePessimisticFixpoint();
13442
13443 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13444
13445 if (SizeOfBin >= *AllocationSize)
13446 return indicatePessimisticFixpoint();
13447
13448 auto NewAllocationSize = std::make_optional<TypeSize>(SizeOfBin * 8, false);
13449
13450 if (!changeAllocationSize(NewAllocationSize))
13451 return ChangeStatus::UNCHANGED;
13452
13453 return ChangeStatus::CHANGED;
13454 }
13455
13456 ChangeStatus manifest(Attributor &A) override {
13457
13458 assert(isValidState() &&
13459 "Manifest should only be called if the state is valid.");
13460
13461 Instruction *I = getIRPosition().getCtxI();
13462
13463 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13464
13465 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13466
13467 switch (I->getOpcode()) {
13468 // TODO: add case for malloc like calls
13469 case Instruction::Alloca: {
13470
13471 AllocaInst *AI = cast<AllocaInst>(I);
13472
13473 Type *CharType = Type::getInt8Ty(I->getContext());
13474
13475 auto *NumBytesToValue =
13476 ConstantInt::get(I->getContext(), APInt(32, NumBytesToAllocate));
13477
13478 BasicBlock::iterator insertPt = AI->getIterator();
13479 insertPt = std::next(insertPt);
13480 AllocaInst *NewAllocaInst =
13481 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
13482 AI->getAlign(), AI->getName(), insertPt);
13483
13484 if (A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst))
13485 return ChangeStatus::CHANGED;
13486
13487 break;
13488 }
13489 default:
13490 break;
13491 }
13492
13493 return ChangeStatus::UNCHANGED;
13494 }
13495
13496 const std::string getAsStr(Attributor *A) const override {
13497 if (!isValidState())
13498 return "allocationinfo(<invalid>)";
13499 return "allocationinfo(" +
13500 (AssumedAllocatedSize == HasNoAllocationSize
13501 ? "none"
13502 : std::to_string(AssumedAllocatedSize->getFixedValue())) +
13503 ")";
13504 }
13505
13506private:
13507 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13508
13509 // Maintain the computed allocation size of the object.
13510 // Returns (bool) weather the size of the allocation was modified or not.
13511 bool changeAllocationSize(std::optional<TypeSize> Size) {
13512 if (AssumedAllocatedSize == HasNoAllocationSize ||
13513 AssumedAllocatedSize != Size) {
13514 AssumedAllocatedSize = Size;
13515 return true;
13516 }
13517 return false;
13518 }
13519};
13520
13521struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13522 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
13523 : AAAllocationInfoImpl(IRP, A) {}
13524
13525 void trackStatistics() const override {
13526 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
13527 }
13528};
13529
13530struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13531 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
13532 : AAAllocationInfoImpl(IRP, A) {}
13533
13534 void initialize(Attributor &A) override {
13535 // TODO: we don't rewrite function argument for now because it will need to
13536 // rewrite the function signature and all call sites
13537 (void)indicatePessimisticFixpoint();
13538 }
13539
13540 void trackStatistics() const override {
13541 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
13542 }
13543};
13544
13545struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13546 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
13547 : AAAllocationInfoImpl(IRP, A) {}
13548
13549 void trackStatistics() const override {
13550 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
13551 }
13552};
13553
13554struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13555 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
13556 : AAAllocationInfoImpl(IRP, A) {}
13557
13558 void trackStatistics() const override {
13559 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
13560 }
13561};
13562
13563struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13564 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
13565 : AAAllocationInfoImpl(IRP, A) {}
13566
13567 void initialize(Attributor &A) override {
13568
13569 (void)indicatePessimisticFixpoint();
13570 }
13571
13572 void trackStatistics() const override {
13573 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
13574 }
13575};
13576} // namespace
13577
13578const char AANoUnwind::ID = 0;
13579const char AANoSync::ID = 0;
13580const char AANoFree::ID = 0;
13581const char AANonNull::ID = 0;
13582const char AAMustProgress::ID = 0;
13583const char AANoRecurse::ID = 0;
13584const char AANonConvergent::ID = 0;
13585const char AAWillReturn::ID = 0;
13586const char AAUndefinedBehavior::ID = 0;
13587const char AANoAlias::ID = 0;
13588const char AAIntraFnReachability::ID = 0;
13589const char AANoReturn::ID = 0;
13590const char AAIsDead::ID = 0;
13591const char AADereferenceable::ID = 0;
13592const char AAAlign::ID = 0;
13593const char AAInstanceInfo::ID = 0;
13594const char AANoCapture::ID = 0;
13595const char AAValueSimplify::ID = 0;
13596const char AAHeapToStack::ID = 0;
13597const char AAPrivatizablePtr::ID = 0;
13598const char AAMemoryBehavior::ID = 0;
13599const char AAMemoryLocation::ID = 0;
13600const char AAValueConstantRange::ID = 0;
13601const char AAPotentialConstantValues::ID = 0;
13602const char AAPotentialValues::ID = 0;
13603const char AANoUndef::ID = 0;
13604const char AANoFPClass::ID = 0;
13605const char AACallEdges::ID = 0;
13606const char AAInterFnReachability::ID = 0;
13607const char AAPointerInfo::ID = 0;
13608const char AAAssumptionInfo::ID = 0;
13609const char AAUnderlyingObjects::ID = 0;
13610const char AAInvariantLoadPointer::ID = 0;
13611const char AAAddressSpace::ID = 0;
13612const char AANoAliasAddrSpace::ID = 0;
13613const char AAAllocationInfo::ID = 0;
13614const char AAIndirectCallInfo::ID = 0;
13615const char AAGlobalValueInfo::ID = 0;
13616const char AADenormalFPMath::ID = 0;
13617
13618// Macro magic to create the static generator function for attributes that
13619// follow the naming scheme.
13620
13621#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13622 case IRPosition::PK: \
13623 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13624
13625#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13626 case IRPosition::PK: \
13627 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13628 ++NumAAs; \
13629 break;
13630
13631#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13632 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13633 CLASS *AA = nullptr; \
13634 switch (IRP.getPositionKind()) { \
13635 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13636 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13637 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13638 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13639 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13640 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13641 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13642 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13643 } \
13644 return *AA; \
13645 }
13646
13647#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13648 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13649 CLASS *AA = nullptr; \
13650 switch (IRP.getPositionKind()) { \
13651 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13652 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13653 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13654 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13655 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13656 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13657 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13658 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13659 } \
13660 return *AA; \
13661 }
13662
13663#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13664 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13665 CLASS *AA = nullptr; \
13666 switch (IRP.getPositionKind()) { \
13667 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13668 default: \
13669 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13670 " position!"); \
13671 } \
13672 return *AA; \
13673 }
13674
13675#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13676 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13677 CLASS *AA = nullptr; \
13678 switch (IRP.getPositionKind()) { \
13679 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13680 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13681 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13682 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13683 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13684 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13685 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13686 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13687 } \
13688 return *AA; \
13689 }
13690
13691#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13692 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13693 CLASS *AA = nullptr; \
13694 switch (IRP.getPositionKind()) { \
13695 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13696 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13697 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13698 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13699 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13700 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13701 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13702 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13703 } \
13704 return *AA; \
13705 }
13706
13707#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13708 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13709 CLASS *AA = nullptr; \
13710 switch (IRP.getPositionKind()) { \
13711 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13712 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13713 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13714 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13715 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13716 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13717 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13718 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13719 } \
13720 return *AA; \
13721 }
13722
13732
13750
13755
13760
13767
13769
13770#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13771#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13772#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13773#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13774#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13775#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13776#undef SWITCH_PK_CREATE
13777#undef SWITCH_PK_INV
#define Success
const TargetLowering & TLI
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
uint64_t uint64_t
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
ReachingDefInfo InstSet & ToRemove
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
#define STATS_DECLTRACK(NAME, TYPE, MSG)
static std::optional< Constant * > askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA, const IRPosition &IRP, Type &Ty)
static bool collectTailCalls(Attributor &A, const AbstractAttribute &QueryingAA, SmallVectorImpl< CallInst * > &TailCalls)
Collect the non-musttail tail calls in the function of QueryingAA.
static void clampReturnedValueStates(Attributor &A, const AAType &QueryingAA, StateType &S, const IRPosition::CallBaseContext *CBContext=nullptr)
Clamp the information known for all returned values of a function (identified by QueryingAA) into S.
#define STATS_DECLTRACK_FN_ATTR(NAME)
#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define STATS_DECLTRACK_CS_ATTR(NAME)
#define PIPE_OPERATOR(CLASS)
#define STATS_DECLTRACK_ARG_ATTR(NAME)
static const Value * stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Val, const DataLayout &DL, APInt &Offset, bool GetMinOffset, bool AllowNonInbounds, bool UseAssumed=false)
#define STATS_DECLTRACK_CSRET_ATTR(NAME)
static Value * constructPointer(Value *Ptr, int64_t Offset, IRBuilder< NoFolder > &IRB)
Helper function to create a pointer based on Ptr, and advanced by Offset bytes.
#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define BUILD_STAT_NAME(NAME, TYPE)
static bool isDenselyPacked(Type *Ty, const DataLayout &DL)
Checks if a type could have padding bytes.
#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static const Value * getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Ptr, int64_t &BytesOffset, const DataLayout &DL, bool AllowNonInbounds=false)
static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I, bool HeaderOnly, CycleRef *CPtr=nullptr)
#define STATS_DECLTRACK_FNRET_ATTR(NAME)
#define STATS_DECLTRACK_CSARG_ATTR(NAME)
#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS)
#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define STATS_DECLTRACK_FLOATING_ATTR(NAME)
#define STATS_DECL(NAME, TYPE, MSG)
bpf unwind BPF Unwind Lower
basic Basic Alias true
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool isReachableImpl(SmallVectorImpl< BasicBlock * > &Worklist, const StopSetT &StopSet, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet, const DominatorTree *DT, const LoopInfo *LI, const CycleInfo *CI)
Definition CFG.cpp:145
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file declares an analysis pass that computes CycleInfo for LLVM IR, specialized from GenericCycl...
static uint64_t align(uint64_t Size)
DXIL Forward Handle Accesses
DXIL Resource Access
dxil translate DXIL Translate Metadata
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
#define Check(C,...)
static Value * getCondition(Instruction *I)
Hexagon Common GEP
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
#define T
#define T1
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
if(PassOpts->AAPipeline)
static StringRef getName(Value *V)
RISCVCC::CondCode CC
Basic Register Allocator
dot regions Print regions of function to dot true view regions View regions of function(with no function bodies)"
Remove Loads Into Fake Uses
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
bool IsDead
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
std::pair< BasicBlock *, BasicBlock * > Edge
This file contains some templates that are useful if you are working with the STL at all.
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
This file defines generic set operations that may be used on set's of different types,...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
This pass exposes codegen information to IR-level passes.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Value * RHS
Value * LHS
static unsigned getSize(unsigned Kind)
LLVM_ABI AACallGraphNode * operator*() const
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
A trivial helper function to check to see if the specified pointers are no-alias.
Class for arbitrary precision integers.
Definition APInt.h:78
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
CallBase * getInstruction() const
Return the underlying instruction.
bool isCallbackCall() const
Return true if this ACS represents a callback call.
bool isDirectCall() const
Return true if this ACS represents a direct call.
static LLVM_ABI void getCallbackUses(const CallBase &CB, SmallVectorImpl< const Use * > &CallbackUses)
Add operand uses of CB that represent callback uses into CallbackUses.
int getCallArgOperandNo(Argument &Arg) const
Return the operand index of the underlying instruction associated with Arg.
This class represents a conversion between pointers from one address space to another.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
unsigned getAddressSpace() const
Return the address space for the allocation.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
LLVM_ABI bool hasNoAliasAttr() const
Return true if this argument has the noalias attribute.
Definition Function.cpp:266
LLVM_ABI bool onlyReadsMemory() const
Return true if this argument has the readonly or readnone attribute.
Definition Function.cpp:302
LLVM_ABI bool hasPointeeInMemoryValueAttr() const
Return true if this argument has the byval, sret, inalloca, preallocated, or byref attribute.
Definition Function.cpp:169
LLVM_ABI bool hasReturnedAttr() const
Return true if this argument has the returned attribute.
Definition Function.cpp:290
LLVM_ABI bool hasByValAttr() const
Return true if this argument has the byval attribute.
Definition Function.cpp:126
const Function * getParent() const
Definition Argument.h:44
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition Argument.h:50
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
bool contains(Attribute::AttrKind A) const
Return true if the builder has the specified attribute.
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:448
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:107
LLVM_ABI const ConstantRange & getRange() const
Returns the value of the range attribute.
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
LLVM_ABI FPClassTest getNoFPClass() const
Return the FPClassTest for nofpclass.
LLVM_ABI Attribute::AttrKind getKindAsEnum() const
Return the attribute's kind as an enum (Attribute::AttrKind).
LLVM_ABI MemoryEffects getMemoryEffects() const
Returns memory effects.
static LLVM_ABI Attribute getWithCaptureInfo(LLVMContext &Context, CaptureInfo CI)
static bool isEnumAttrKind(AttrKind Kind)
Definition Attributes.h:140
bool isValid() const
Return true if the attribute is any kind of attribute.
Definition Attributes.h:263
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition Attributes.h:126
LLVM_ABI MaybeAlign getAlignment() const
Returns the alignment field of an attribute as a byte alignment value.
LLVM_ABI CaptureInfo getCaptureInfo() const
Returns information from captures attribute.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition BasicBlock.h:206
const Instruction & front() const
Definition BasicBlock.h:463
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
BinaryOps getOpcode() const
Definition InstrTypes.h:409
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI MemoryEffects getMemoryEffects() const
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
Value * getCalledOperand() const
const Use & getCalledOperandUse() const
Attribute getFnAttr(StringRef Kind) const
Get the attribute of a given kind for the function.
const Use & getArgOperandUse(unsigned i) const
Wrappers for getting the Use of a call argument.
LLVM_ABI bool isVolatile() const
Return true if this is a call to an intrinsic with a volatile flag that is set, e....
OperandBundleUse getOperandBundleForOperand(unsigned OpIdx) const
Return the operand bundle for the operand at index OpIdx.
LLVM_ABI std::optional< ConstantRange > getRange() const
If this return value has a range attribute, return the value range of the argument.
Value * getArgOperand(unsigned i) const
bool isBundleOperand(unsigned Idx) const
Return true if the operand at index Idx is a bundle operand.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
unsigned arg_size() const
bool isArgOperand(const Use *U) const
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static CaptureInfo none()
Create CaptureInfo that does not capture any components of the pointer.
Definition ModRef.h:427
This is the base class for all instructions that perform data casts.
Definition InstrTypes.h:512
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition InstrTypes.h:674
LLVM_ABI bool isIntegerCast() const
There are several places where we need to know if a cast instruction only deals with integer source a...
Type * getDestTy() const
Return the destination type, as a convenience.
Definition InstrTypes.h:681
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition InstrTypes.h:978
bool isFalseWhenEqual() const
This is just a convenience.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_NE
not equal
Definition InstrTypes.h:762
bool isTrueWhenEqual() const
This is just a convenience.
Predicate getPredicate() const
Return the predicate for this instruction.
Definition InstrTypes.h:828
Conditional Branch instruction.
Value * getCondition() const
BasicBlock * getSuccessor(unsigned i) const
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:549
static LLVM_ABI Constant * getExtractElement(Constant *Vec, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
This class represents a range of values.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI bool isEmptySet() const
Return true if this set contains no members.
bool isSingleElement() const
Return true if this set contains exactly one member.
static LLVM_ABI ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the smallest range such that all values that may satisfy the given predicate with any value c...
const APInt & getUpper() const
Return the upper value for this range.
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
Opaque handle to a cycle within a GenericCycleInfo that wraps the cycle's preorder index.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
unsigned getProgramAddressSpace() const
Definition DataLayout.h:272
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:757
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:762
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
const BasicBlock & getEntryBlock() const
Definition Function.h:803
Argument * arg_iterator
Definition Function.h:73
iterator_range< arg_iterator > args()
Definition Function.h:886
const Function & getFunction() const
Definition Function.h:167
size_t arg_size() const
Definition Function.h:895
Argument * getArg(unsigned i) const
Definition Function.h:880
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
BlockT * getHeader(CycleRef C) const
CycleRef getCycle(const BlockT *Block) const
Find the innermost cycle containing Block.
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition Globals.cpp:408
bool hasLocalLinkage() const
PointerType * getType() const
Global values are always pointers.
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Value * CreatePtrAddWithSignedOffset(Value *Ptr, int64_t Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Create a ptradd with a signed constant offset.
Definition IRBuilder.h:2123
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
LLVM_ABI bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI void insertBefore(InstListType::iterator InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified position.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
bool isTerminator() const
LLVM_ABI bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
LLVM_ABI bool mayReadOrWriteMemory() const LLVM_READONLY
Return true if this instruction may read or write memory.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
FPClassTest getKnownFPClasses() const
Floating-point classes the value could be one of.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI ConstantRange getConstantRange(Value *V, Instruction *CtxI, bool UndefAllowed)
Return the ConstantRange constraint that is known to hold for the specified value at the specified in...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1588
static LLVM_ABI MDNode * getMostGenericRange(MDNode *A, MDNode *B)
bool empty() const
Definition MapVector.h:80
static MemoryEffectsBase readOnly()
Definition ModRef.h:133
bool doesNotAccessMemory() const
Whether this function accesses no memory.
Definition ModRef.h:246
static MemoryEffectsBase argMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Definition ModRef.h:143
static MemoryEffectsBase inaccessibleMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Definition ModRef.h:149
bool onlyAccessesInaccessibleMem() const
Whether this function only (at most) accesses inaccessible memory.
Definition ModRef.h:265
ModRefInfo getModRef(Location Loc) const
Get ModRefInfo for the given Location.
Definition ModRef.h:219
bool onlyAccessesArgPointees() const
Whether this function only (at most) accesses argument memory.
Definition ModRef.h:255
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
Definition ModRef.h:249
static MemoryEffectsBase writeOnly()
Definition ModRef.h:138
static MemoryEffectsBase inaccessibleOrArgMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Definition ModRef.h:166
static MemoryEffectsBase none()
Definition ModRef.h:128
bool onlyAccessesInaccessibleOrArgMem() const
Whether this function only (at most) accesses argument and inaccessible memory.
Definition ModRef.h:305
static MemoryEffectsBase unknown()
Definition ModRef.h:123
static LLVM_ABI std::optional< MemoryLocation > getOrNone(const Instruction *Inst)
static SizeOffsetValue unknown()
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Class to represent pointers.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
LLVM_ABI SCEVUse getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI unsigned getSmallConstantMaxTripCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
Returns the upper bound of the loop trip count as a normal unsigned value.
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
A vector that has set insertion semantics.
Definition SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition DataLayout.h:746
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:777
TypeSize getSizeInBits() const
Definition DataLayout.h:757
TypeSize getElementOffsetInBits(unsigned Idx) const
Definition DataLayout.h:782
Class to represent struct types.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Analysis pass providing the TargetTransformInfo.
LLVM_ABI bool areTypesABICompatible(const Function *Caller, const Function *Callee, ArrayRef< Type * > Types) const
LLVM_ABI unsigned getAssumedAddrSpace(const Value *V) const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI unsigned getIntegerBitWidth() const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
static UncondBrInst * Create(BasicBlock *Target, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i=0) const
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
User * getUser() const
Returns the User that contains this Use.
Definition Use.h:61
Value * get() const
Definition Use.h:55
const Use & getOperandUse(unsigned i) const
Definition User.h:220
LLVM_ABI bool isDroppable() const
A droppable user is a user for which uses can be dropped without affecting correctness and should be ...
Definition User.cpp:113
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:25
Value * getOperand(unsigned i) const
Definition User.h:207
unsigned getNumOperands() const
Definition User.h:229
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition ValueMap.h:167
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
static constexpr uint64_t MaximumAlignment
Definition Value.h:801
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const
Accumulate the constant offset this value has compared to a base pointer.
bool use_empty() const
Definition Value.h:348
static constexpr unsigned MaxAlignmentExponent
The maximum alignment for instructions.
Definition Value.h:800
iterator_range< use_iterator > uses()
Definition Value.h:382
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
Definition Attributor.h:166
LLVM_ABI bool isAssumedReadNone(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readnone.
LLVM_ABI bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readonly.
raw_ostream & operator<<(raw_ostream &OS, const RangeTy &R)
Definition Attributor.h:328
LLVM_ABI std::optional< Value * > combineOptionalValuesInAAValueLatice(const std::optional< Value * > &A, const std::optional< Value * > &B, Type *Ty)
Return the combination of A and B such that the result is a possible value of both.
LLVM_ABI bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache)
Return true if the value of VAC is a valid at the position of VAC, that is a constant,...
LLVM_ABI bool isAssumedThreadLocalObject(Attributor &A, Value &Obj, const AbstractAttribute &QueryingAA)
Return true if Obj is assumed to be a thread local object.
LLVM_ABI bool isGPUConstantAddressSpace(const Module &M, unsigned AS)
Check if the given address space AS corresponds to a GPU constant address space for the target triple...
LLVM_ABI bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA, const Value &V, bool ForAnalysisOnly=true)
Return true if V is dynamically unique, that is, there are no two "instances" of V at runtime with di...
LLVM_ABI bool getPotentialCopiesOfStoredValue(Attributor &A, StoreInst &SI, SmallSetVector< Value *, 4 > &PotentialCopies, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values of the one stored by SI into PotentialCopies.
LLVM_ABI bool isGPUSharedAddressSpace(const Module &M, unsigned AS)
Check if the given address space AS corresponds to a GPU shared address space for the target triple i...
LLVM_ABI bool isGPULocalAddressSpace(const Module &M, unsigned AS)
Check if the given address space AS corresponds to a GPU local/private address space for the target t...
SmallPtrSet< Instruction *, 4 > InstExclusionSetTy
Definition Attributor.h:167
LLVM_ABI bool isGPU(const Module &M)
Return true iff M target a GPU (and we can use GPU AS reasoning).
ValueScope
Flags to distinguish intra-procedural queries from potentially inter-procedural queries.
Definition Attributor.h:187
@ Intraprocedural
Definition Attributor.h:188
@ Interprocedural
Definition Attributor.h:189
LLVM_ABI bool isValidInScope(const Value &V, const Function *Scope)
Return true if V is a valid value in Scope, that is a constant or an instruction/argument of Scope.
LLVM_ABI bool isPotentiallyReachable(Attributor &A, const Instruction &FromI, const Instruction &ToI, const AbstractAttribute &QueryingAA, const AA::InstExclusionSetTy *ExclusionSet=nullptr, std::function< bool(const Function &F)> GoBackwardsCB=nullptr)
Return true if ToI is potentially reachable from FromI without running into any instruction in Exclus...
LLVM_ABI bool isNoSyncInst(Attributor &A, const Instruction &I, const AbstractAttribute &QueryingAA)
Return true if I is a nosync instruction.
bool hasAssumedIRAttr(Attributor &A, const AbstractAttribute *QueryingAA, const IRPosition &IRP, DepClassTy DepClass, bool &IsKnown, bool IgnoreSubsumingPositions=false, const AAType **AAPtr=nullptr)
Helper to avoid creating an AA for IR Attributes that might already be set.
LLVM_ABI bool getPotentiallyLoadedValues(Attributor &A, LoadInst &LI, SmallSetVector< Value *, 4 > &PotentialValues, SmallSetVector< Instruction *, 4 > &PotentialValueOrigins, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values LI could read into PotentialValues.
LLVM_ABI Value * getWithType(Value &V, Type &Ty)
Try to convert V to type Ty without introducing new instructions.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
@ Unsupported
This operation is completely unsupported on the target.
Offsets
Offsets in bytes from the start of the input buffer.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition LLVMContext.h:63
@ Valid
The data is already valid.
unsigned combineHashValue(unsigned a, unsigned b)
Simplistic combination of 32-bit hash values into 32-bit hash values.
ElementType
The element type of an SRV or UAV resource.
Definition DXILABI.h:68
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:721
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:680
@ User
could "use" a pointer
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
iterator end() const
Definition BasicBlock.h:89
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
LLVM_ABI iterator begin() const
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
bool operator<(int64_t V1, const APSInt &V2)
Definition APSInt.h:360
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
LLVM_ABI Constant * getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty)
If this is a call to an allocation function that initializes memory to a fixed value,...
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
BundleAttr getBundleAttrFromOBU(OperandBundleUse OBU)
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
auto pred_end(const MachineBasicBlock *BB)
unsigned getPointerAddressSpace(const Type *T)
Definition SPIRVUtils.h:385
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
auto successors(const MachineBasicBlock *BB)
LLVM_ABI bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition APFloat.h:1736
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
LLVM_ABI raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
LLVM_ABI Value * getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI)
Gets the alignment argument for an aligned_alloc-like function, using either built-in knowledge based...
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_ABI Value * simplifyInstructionWithOperands(Instruction *I, ArrayRef< Value * > NewOps, const SimplifyQuery &Q)
Like simplifyInstruction but the operands of I are replaced with NewOps.
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
LLVM_ABI bool isNoAliasCall(const Value *V)
Return true if this pointer is returned by a noalias function.
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
Definition ModRef.h:356
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
Definition STLExtras.h:366
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
PotentialValuesState< std::pair< AA::ValueAndContext, AA::ValueScope > > PotentialLLVMValuesState
bool isModSet(const ModRefInfo MRI)
Definition ModRef.h:49
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool isPointerTy(const Type *T)
Definition SPIRVUtils.h:373
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
Definition Local.cpp:415
bool set_union(S1Ty &S1, const S2Ty &S2)
set_union(A, B) - Compute A := A u B, return whether A changed.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI CallBase & promoteCall(CallBase &CB, Function *Callee, CastInst **RetBitCast=nullptr)
Promote the given indirect call site to unconditionally call Callee.
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI bool hasAssumption(const Function &F, const KnownAssumptionString &AssumptionStr)
Return true if F has the assumption AssumptionStr attached.
@ Success
The lock was released successfully.
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ModRefInfo
Flags indicating whether a memory access modifies or references memory.
Definition ModRef.h:28
@ Other
Any other memory.
Definition ModRef.h:68
PotentialValuesState< APInt > PotentialConstantIntValuesState
TargetTransformInfo TTI
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
InterleavedRange< Range > interleaved_array(const Range &R, StringRef Separator=", ")
Output range R as an array of interleaved elements.
ChangeStatus clampStateAndIndicateChange< DerefState >(DerefState &S, const DerefState &R)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataPredicate *IdentityMD=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM.
LLVM_ABI bool isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(const CallBase *Call, bool MustPreserveOffset, bool MustPreserveProvenance=false)
launder.invariant.group and similar intrinsics return a pointer that aliases their argument,...
DWARFExpression::Operation Op
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
ChangeStatus clampStateAndIndicateChange(StateType &S, const StateType &R)
Helper function to clamp a state S of type StateType with the information in R and indicate/return if...
constexpr unsigned BitWidth
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
auto pred_begin(const MachineBasicBlock *BB)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
Definition iterator.h:368
ChangeStatus
{
Definition Attributor.h:474
LLVM_ABI std::optional< APInt > getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, function_ref< const Value *(const Value *)> Mapper=[](const Value *V) { return V;})
Return the size of the requested allocation.
LLVM_ABI DenseSet< StringRef > getAssumptions(const Function &F)
Return the set of all assumptions for the function F.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
Definition Alignment.h:105
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
@ OPTIONAL
The target may be valid if the source is not.
Definition Attributor.h:486
@ NONE
Do not track a dependence between source and target.
Definition Attributor.h:487
@ REQUIRED
The target cannot be valid if the source is not.
Definition Attributor.h:485
LLVM_ABI UseCaptureInfo DetermineUseCaptureKind(const Use &U, const Value *Base)
Determine what kind of capture behaviour U may exhibit.
LLVM_ABI Value * simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a CmpInst, fold the result or return null.
LLVM_ABI bool mayContainIrreducibleControl(const Function &F, const LoopInfo *LI)
BumpPtrAllocatorImpl<> BumpPtrAllocator
The standard BumpPtrAllocator which just uses the default template parameters.
Definition Allocator.h:391
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
bool capturesNothing(CaptureComponents CC)
Definition ModRef.h:375
LLVM_ABI AssumeDereferenceableInfo getAssumeDereferenceableInfo(OperandBundleUse)
LLVM_ABI bool isIdentifiedObject(const Value *V)
Return true if this pointer refers to a distinct and identifiable object.
bool capturesAnyProvenance(CaptureComponents CC)
Definition ModRef.h:400
constexpr StringRef AssumptionAttrKey
The key we use for assumption attributes.
Definition Assumptions.h:29
constexpr bool isCallableCC(CallingConv::ID CC)
bool isRefSet(const ModRefInfo MRI)
Definition ModRef.h:52
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
A type to track pointer/struct usage and accesses for AAPointerInfo.
bool forallInterferingAccesses(AA::RangeTy Range, F CB) const
See AAPointerInfo::forallInterferingAccesses.
AAPointerInfo::const_bin_iterator end() const
ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I, std::optional< Value * > Content, AAPointerInfo::AccessKind Kind, Type *Ty, Instruction *RemoteI=nullptr)
Add a new Access to the state at offset Offset and with size Size.
DenseMap< const Instruction *, SmallVector< unsigned > > RemoteIMap
AAPointerInfo::const_bin_iterator begin() const
AAPointerInfo::OffsetInfo ReturnedOffsets
Flag to determine if the underlying pointer is reaching a return statement in the associated function...
State(State &&SIS)=default
const AAPointerInfo::Access & getAccess(unsigned Index) const
SmallVector< AAPointerInfo::Access > AccessList
bool isAtFixpoint() const override
Return if this abstract state is fixed, thus does not need to be updated if information changes as it...
bool forallInterferingAccesses(Instruction &I, F CB, AA::RangeTy &Range) const
See AAPointerInfo::forallInterferingAccesses.
static State getWorstState(const State &SIS)
Return the worst possible representable state.
AAPointerInfo::OffsetBinsTy OffsetBins
ChangeStatus indicateOptimisticFixpoint() override
Indicate that the abstract state should converge to the optimistic state.
ChangeStatus indicatePessimisticFixpoint() override
Indicate that the abstract state should converge to the pessimistic state.
static State getBestState(const State &SIS)
Return the best possible representable state.
bool isValidState() const override
Return if this abstract state is in a valid state.
----------------—AAIntraFnReachability Attribute-----------------------—
ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
unsigned Hash
Precomputed hash for this RQI.
const Instruction * From
Start here,.
Reachable Result
and remember if it worked:
ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To, const AA::InstExclusionSetTy *ES, bool MakeUnique)
Constructor replacement to ensure unique and stable sets are used for the cache.
const ToTy * To
reach this place,
const AA::InstExclusionSetTy * ExclusionSet
without going through any of these instructions,
An abstract interface for address space information.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all align attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
Align getKnownAlign() const
Return known alignment.
static LLVM_ABI const char ID
An abstract attribute for getting assumption information.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract state for querying live call edges.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract Attribute for specializing "dynamic" components of denormal_fpenv to a known denormal mod...
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all dereferenceable attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for llvm::GlobalValue information interference.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for indirect call information interference.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface to track if a value leaves it's defining function instance.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract Attribute for computing reachability between functions.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
bool canReach(Attributor &A, const Function &Fn) const
If the function represented by this possition can reach Fn.
virtual bool instructionCanReach(Attributor &A, const Instruction &Inst, const Function &Fn, const AA::InstExclusionSetTy *ExclusionSet=nullptr) const =0
Can Inst reach Fn.
An abstract interface to determine reachability of point A to B.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for identifying pointers from which loads can be marked invariant.
static LLVM_ABI const char ID
Unique ID (due to the unique address).
An abstract interface for liveness abstract attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for memory access kind related attributes (readnone/readonly/writeonly).
bool isAssumedReadOnly() const
Return true if we assume that the underlying value is not accessed (=written) in its respective scope...
bool isKnownReadNone() const
Return true if we know that the underlying value is not read or accessed in its respective scope.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
bool isAssumedReadNone() const
Return true if we assume that the underlying value is not read or accessed in its respective scope.
An abstract interface for all memory location attributes (readnone/argmemonly/inaccessiblememonly/ina...
static LLVM_ABI std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK)
Return the locations encoded by MLK as a readable string.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
StateType::base_t MemoryLocationsKind
An abstract interface for all nonnull attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for potential address space information.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all noalias attributes.
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
---------------------— NoAlias Argument Attribute ---------------------—
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all nocapture attributes.
@ NO_CAPTURE_MAYBE_RETURNED
If we do not capture the value in memory or through integers we can only communicate it back as a der...
@ NO_CAPTURE
If we do not capture the value in memory, through integers, or as a derived pointer we know it is not...
static LLVM_ABI const char ID
Unique ID (due to the unique address)
bool isAssumedNoCaptureMaybeReturned() const
Return true if we assume that the underlying value is not captured in its respective scope but we all...
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
--------------------— Variable Capturing ------------------------------—
static LLVM_ABI void determineFunctionCaptureCapabilities(const IRPosition &IRP, const Function &F, BitIntegerState &State)
Update State according to the capture capabilities of F for position IRP.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An AbstractAttribute for nofree.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for norecurse.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An AbstractAttribute for noreturn.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned)
Helper function to determine if CB is an aligned (GPU) barrier.
static LLVM_ABI bool isNonRelaxedAtomic(const Instruction *I)
Helper function used to determine whether an instruction is non-relaxed atomic.
An abstract interface for all noundef attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
---------------------— NoUndef Attribute ------------------------------—
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract Attribute for determining the necessity of the convergent attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all nonnull attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
---------------------— NonNull Argument Attribute ---------------------—
An access description.
A helper containing a list of offsets computed for a Use.
A container for a list of ranges.
static void set_difference(const RangeList &L, const RangeList &R, RangeList &D)
Copy ranges from L that are not in R, into D.
An abstract interface for struct information.
virtual bool reachesReturn() const =0
OffsetBinsTy::const_iterator const_bin_iterator
virtual const_bin_iterator begin() const =0
DenseMap< AA::RangeTy, SmallSet< unsigned, 4 > > OffsetBinsTy
static LLVM_ABI const char ID
Unique ID (due to the unique address)
virtual int64_t numOffsetBins() const =0
An abstract interface for potential values analysis.
StateWrapper< PotentialConstantIntValuesState, AbstractAttribute, bool, unsigned > Base
static LLVM_ABI const char ID
Unique ID (due to the unique address)
LLVM_ABI AAPotentialConstantValues(const IRPosition &IRP, Attributor &A)
---------------— Potential Values Attribute ----------------------—
friend struct Attributor
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI Value * getSingleValue(Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP, SmallVectorImpl< AA::ValueAndContext > &Values)
Extract the single value in Values if any.
An abstract interface for privatizability.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for undefined behavior.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for getting all assumption underlying objects.
virtual bool forallUnderlyingObjects(function_ref< bool(Value &)> Pred, AA::ValueScope Scope=AA::Interprocedural) const =0
Check Pred on all underlying objects in Scope collected so far.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for range value analysis.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for value simplify abstract attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for willreturn.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
Helper to represent an access offset and size, with logic to deal with uncertainty and check for over...
Definition Attributor.h:246
static constexpr int64_t Unknown
Definition Attributor.h:325
static RangeTy getUnknown()
Definition Attributor.h:252
Value * getValue() const
Definition Attributor.h:199
const Instruction * getCtxI() const
Definition Attributor.h:200
Base struct for all "concrete attribute" deductions.
void print(raw_ostream &OS) const
Helper functions, for debug purposes only.
virtual StateType & getState()=0
Return the internal abstract state for inspection.
virtual void initialize(Attributor &A)
Initialize the state with the information in the Attributor A.
AbstractState StateType
An interface to query the internal state of an abstract attribute.
virtual bool isAtFixpoint() const =0
Return if this abstract state is fixed, thus does not need to be updated if information changes as it...
virtual bool isValidState() const =0
Return if this abstract state is in a valid state.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
static constexpr Align fromLog2(unsigned Log2Value)
Returns an alignment of 1 << Log2Value bytes.
Definition Alignment.h:73
std::function< void( const ArgumentReplacementInfo &, Function &, Function::arg_iterator)> CalleeRepairCBTy
Callee repair callback type.
const Argument & getReplacedArg() const
std::function< void(const ArgumentReplacementInfo &, AbstractCallSite, SmallVectorImpl< Value * > &)> ACSRepairCBTy
Abstract call site (ACS) repair callback type.
The fixpoint analysis framework that orchestrates the attribute deduction.
std::function< std::optional< Value * >( const IRPosition &, const AbstractAttribute *, bool &)> SimplifictionCallbackTy
Register CB as a simplification callback.
Specialization of the integer state for a bit-wise encoding.
BitIntegerState & addKnownBits(base_t Bits)
Add the bits in BitsEncoding to the "known bits".
Simple wrapper for a single bit (boolean) state.
static constexpr DenormalFPEnv getDefault()
static unsigned getHashValue(const Access &A)
static bool isEqual(const Access &LHS, const Access &RHS)
static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B)
static unsigned getHashValue(const AA::RangeTy &Range)
DenseMapInfo< std::pair< const Instruction *, const ToTy * > > PairDMI
static bool isEqual(const ReachabilityQueryInfo< ToTy > *LHS, const ReachabilityQueryInfo< ToTy > *RHS)
DenseMapInfo< const AA::InstExclusionSetTy * > InstSetDMI
static unsigned getHashValue(const ReachabilityQueryInfo< ToTy > *RQI)
An information struct used to provide DenseMap with the various necessary components for a given valu...
State for dereferenceable attribute.
IncIntegerState DerefBytesState
State representing for dereferenceable bytes.
ChangeStatus manifest(Attributor &A) override
Helper to describe and deal with positions in the LLVM-IR.
Definition Attributor.h:570
Function * getAssociatedFunction() const
Return the associated function, if any.
Definition Attributor.h:701
static const IRPosition callsite_returned(const CallBase &CB)
Create a position describing the returned value of CB.
Definition Attributor.h:638
static const IRPosition returned(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the returned value of F.
Definition Attributor.h:620
LLVM_ABI Argument * getAssociatedArgument() const
Return the associated argument, if any.
static const IRPosition value(const Value &V, const CallBaseContext *CBContext=nullptr)
Create a position describing the value of V.
Definition Attributor.h:594
CallBase CallBaseContext
Definition Attributor.h:573
int getCalleeArgNo() const
Return the callee argument number of the associated value if it is an argument or call site argument,...
Definition Attributor.h:787
static const IRPosition inst(const Instruction &I, const CallBaseContext *CBContext=nullptr)
Create a position describing the instruction I.
Definition Attributor.h:606
static const IRPosition callsite_argument(const CallBase &CB, unsigned ArgNo)
Create a position describing the argument of CB at position ArgNo.
Definition Attributor.h:643
Kind
The positions we distinguish in the IR.
Definition Attributor.h:576
@ IRP_ARGUMENT
An attribute for a function argument.
Definition Attributor.h:584
@ IRP_RETURNED
An attribute for the function return value.
Definition Attributor.h:580
@ IRP_CALL_SITE
An attribute for a call site (function scope).
Definition Attributor.h:583
@ IRP_CALL_SITE_RETURNED
An attribute for a call site return value.
Definition Attributor.h:581
@ IRP_FUNCTION
An attribute for a function (scope).
Definition Attributor.h:582
@ IRP_CALL_SITE_ARGUMENT
An attribute for a call site argument.
Definition Attributor.h:585
@ IRP_INVALID
An invalid position.
Definition Attributor.h:577
Instruction * getCtxI() const
Return the context instruction, if any.
Definition Attributor.h:754
static const IRPosition argument(const Argument &Arg, const CallBaseContext *CBContext=nullptr)
Create a position describing the argument Arg.
Definition Attributor.h:627
Type * getAssociatedType() const
Return the type this abstract attribute is associated with.
Definition Attributor.h:776
static const IRPosition function(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the function scope of F.
Definition Attributor.h:613
const CallBaseContext * getCallBaseContext() const
Get the call base context from the position.
Definition Attributor.h:893
Value & getAssociatedValue() const
Return the value this abstract attribute is associated with.
Definition Attributor.h:768
Value & getAnchorValue() const
Return the value this abstract attribute is anchored with.
Definition Attributor.h:687
int getCallSiteArgNo() const
Return the call site argument number of the associated value if it is an argument or call site argume...
Definition Attributor.h:796
static const IRPosition function_scope(const IRPosition &IRP, const CallBaseContext *CBContext=nullptr)
Create a position with function scope matching the "context" of IRP.
Definition Attributor.h:666
Kind getPositionKind() const
Return the associated position kind.
Definition Attributor.h:843
bool isArgumentPosition() const
Return true if the position is an argument or call site argument.
Definition Attributor.h:875
static const IRPosition callsite_function(const CallBase &CB)
Create a position describing the function scope of CB.
Definition Attributor.h:633
Function * getAnchorScope() const
Return the Function surrounding the anchor value.
Definition Attributor.h:742
Data structure to hold cached (LLVM-IR) information.
bool isOnlyUsedByAssume(const Instruction &I)
TargetLibraryInfo * getTargetLibraryInfoForFunction(const Function &F)
Return TargetLibraryInfo for function F.
AP::Result * getAnalysisResultForFunction(const Function &F, bool CachedOnly=false)
Return the analysis result from a pass AP for function F.
ConstantRange getKnown() const
Return the known state encoding.
ConstantRange getAssumed() const
Return the assumed state encoding.
base_t getAssumed() const
Return the assumed state encoding.
Helper that allows to insert a new assumption string in the known assumption set by creating a (stati...
Definition Assumptions.h:37
A "must be executed context" for a given program point PP is the set of instructions,...
LLVM_ABI bool checkForAllTerminatorsInContext(const Instruction *PP, function_ref< bool(const Instruction *)> Pred)
Check Pred on all terminators in the context of PP and return true if it holds for all of them.
LLVM_ABI bool isInContextOf(const Instruction *I, const Instruction *PP)
}
Helper to tie a abstract state implementation to an abstract attribute.
StateType & getState() override
CaptureComponents ResultCC
Components captured by the return value of the user of this Use.
LLVM_ABI bool unionAssumed(std::optional< Value * > Other)
Merge Other into the currently assumed simplified value.
std::optional< Value * > SimplifiedAssociatedValue
An assumed simplified value.
Type * Ty
The type of the original value.