LLVM 20.0.0git
AttributorAttributes.cpp
Go to the documentation of this file.
1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/Statistic.h"
40#include "llvm/IR/Argument.h"
41#include "llvm/IR/Assumptions.h"
42#include "llvm/IR/Attributes.h"
43#include "llvm/IR/BasicBlock.h"
44#include "llvm/IR/Constant.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DataLayout.h"
48#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
50#include "llvm/IR/InlineAsm.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
55#include "llvm/IR/IntrinsicsAMDGPU.h"
56#include "llvm/IR/IntrinsicsNVPTX.h"
57#include "llvm/IR/LLVMContext.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/NoFolder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueHandle.h"
74#include <cassert>
75#include <numeric>
76#include <optional>
77#include <string>
78
79using namespace llvm;
80
81#define DEBUG_TYPE "attributor"
82
84 "attributor-manifest-internal", cl::Hidden,
85 cl::desc("Manifest Attributor internal string attributes."),
86 cl::init(false));
87
88static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128),
90
91template <>
93
95
97 "attributor-max-potential-values", cl::Hidden,
98 cl::desc("Maximum number of potential values to be "
99 "tracked for each position."),
101 cl::init(7));
102
104 "attributor-max-potential-values-iterations", cl::Hidden,
105 cl::desc(
106 "Maximum number of iterations we keep dismantling potential values."),
107 cl::init(64));
108
109STATISTIC(NumAAs, "Number of abstract attributes created");
110
111// Some helper macros to deal with statistics tracking.
112//
113// Usage:
114// For simple IR attribute tracking overload trackStatistics in the abstract
115// attribute and choose the right STATS_DECLTRACK_********* macro,
116// e.g.,:
117// void trackStatistics() const override {
118// STATS_DECLTRACK_ARG_ATTR(returned)
119// }
120// If there is a single "increment" side one can use the macro
121// STATS_DECLTRACK with a custom message. If there are multiple increment
122// sides, STATS_DECL and STATS_TRACK can also be used separately.
123//
124#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
125 ("Number of " #TYPE " marked '" #NAME "'")
126#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
127#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
128#define STATS_DECL(NAME, TYPE, MSG) \
129 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
130#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
131#define STATS_DECLTRACK(NAME, TYPE, MSG) \
132 { \
133 STATS_DECL(NAME, TYPE, MSG) \
134 STATS_TRACK(NAME, TYPE) \
135 }
136#define STATS_DECLTRACK_ARG_ATTR(NAME) \
137 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
138#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
139 STATS_DECLTRACK(NAME, CSArguments, \
140 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
141#define STATS_DECLTRACK_FN_ATTR(NAME) \
142 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
143#define STATS_DECLTRACK_CS_ATTR(NAME) \
144 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
145#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
146 STATS_DECLTRACK(NAME, FunctionReturn, \
147 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
148#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
149 STATS_DECLTRACK(NAME, CSReturn, \
150 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
151#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
152 STATS_DECLTRACK(NAME, Floating, \
153 ("Number of floating values known to be '" #NAME "'"))
154
155// Specialization of the operator<< for abstract attributes subclasses. This
156// disambiguates situations where multiple operators are applicable.
157namespace llvm {
158#define PIPE_OPERATOR(CLASS) \
159 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
160 return OS << static_cast<const AbstractAttribute &>(AA); \
161 }
162
200
201#undef PIPE_OPERATOR
202
203template <>
205 const DerefState &R) {
206 ChangeStatus CS0 =
207 clampStateAndIndicateChange(S.DerefBytesState, R.DerefBytesState);
208 ChangeStatus CS1 = clampStateAndIndicateChange(S.GlobalState, R.GlobalState);
209 return CS0 | CS1;
210}
211
212} // namespace llvm
213
214static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
215 bool HeaderOnly, Cycle **CPtr = nullptr) {
216 if (!CI)
217 return true;
218 auto *BB = I->getParent();
219 auto *C = CI->getCycle(BB);
220 if (!C)
221 return false;
222 if (CPtr)
223 *CPtr = C;
224 return !HeaderOnly || BB == C->getHeader();
225}
226
227/// Checks if a type could have padding bytes.
228static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
229 // There is no size information, so be conservative.
230 if (!Ty->isSized())
231 return false;
232
233 // If the alloc size is not equal to the storage size, then there are padding
234 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
235 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
236 return false;
237
238 // FIXME: This isn't the right way to check for padding in vectors with
239 // non-byte-size elements.
240 if (VectorType *SeqTy = dyn_cast<VectorType>(Ty))
241 return isDenselyPacked(SeqTy->getElementType(), DL);
242
243 // For array types, check for padding within members.
244 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty))
245 return isDenselyPacked(SeqTy->getElementType(), DL);
246
247 if (!isa<StructType>(Ty))
248 return true;
249
250 // Check for padding within and between elements of a struct.
251 StructType *StructTy = cast<StructType>(Ty);
252 const StructLayout *Layout = DL.getStructLayout(StructTy);
253 uint64_t StartPos = 0;
254 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
255 Type *ElTy = StructTy->getElementType(I);
256 if (!isDenselyPacked(ElTy, DL))
257 return false;
258 if (StartPos != Layout->getElementOffsetInBits(I))
259 return false;
260 StartPos += DL.getTypeAllocSizeInBits(ElTy);
261 }
262
263 return true;
264}
265
266/// Get pointer operand of memory accessing instruction. If \p I is
267/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
268/// is set to false and the instruction is volatile, return nullptr.
270 bool AllowVolatile) {
271 if (!AllowVolatile && I->isVolatile())
272 return nullptr;
273
274 if (auto *LI = dyn_cast<LoadInst>(I)) {
275 return LI->getPointerOperand();
276 }
277
278 if (auto *SI = dyn_cast<StoreInst>(I)) {
279 return SI->getPointerOperand();
280 }
281
282 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I)) {
283 return CXI->getPointerOperand();
284 }
285
286 if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
287 return RMWI->getPointerOperand();
288 }
289
290 return nullptr;
291}
292
293/// Helper function to create a pointer based on \p Ptr, and advanced by \p
294/// Offset bytes.
296 IRBuilder<NoFolder> &IRB) {
297 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
298 << "-bytes\n");
299
300 if (Offset)
301 Ptr = IRB.CreatePtrAdd(Ptr, IRB.getInt64(Offset),
302 Ptr->getName() + ".b" + Twine(Offset));
303 return Ptr;
304}
305
306static const Value *
308 const Value *Val, const DataLayout &DL, APInt &Offset,
309 bool GetMinOffset, bool AllowNonInbounds,
310 bool UseAssumed = false) {
311
312 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
313 const IRPosition &Pos = IRPosition::value(V);
314 // Only track dependence if we are going to use the assumed info.
315 const AAValueConstantRange *ValueConstantRangeAA =
316 A.getAAFor<AAValueConstantRange>(QueryingAA, Pos,
317 UseAssumed ? DepClassTy::OPTIONAL
318 : DepClassTy::NONE);
319 if (!ValueConstantRangeAA)
320 return false;
321 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
322 : ValueConstantRangeAA->getKnown();
323 if (Range.isFullSet())
324 return false;
325
326 // We can only use the lower part of the range because the upper part can
327 // be higher than what the value can really be.
328 if (GetMinOffset)
329 ROffset = Range.getSignedMin();
330 else
331 ROffset = Range.getSignedMax();
332 return true;
333 };
334
335 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
336 /* AllowInvariant */ true,
337 AttributorAnalysis);
338}
339
340static const Value *
342 const Value *Ptr, int64_t &BytesOffset,
343 const DataLayout &DL, bool AllowNonInbounds = false) {
344 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
345 const Value *Base =
346 stripAndAccumulateOffsets(A, QueryingAA, Ptr, DL, OffsetAPInt,
347 /* GetMinOffset */ true, AllowNonInbounds);
348
349 BytesOffset = OffsetAPInt.getSExtValue();
350 return Base;
351}
352
353/// Clamp the information known for all returned values of a function
354/// (identified by \p QueryingAA) into \p S.
355template <typename AAType, typename StateType = typename AAType::StateType,
356 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
357 bool RecurseForSelectAndPHI = true>
359 Attributor &A, const AAType &QueryingAA, StateType &S,
360 const IRPosition::CallBaseContext *CBContext = nullptr) {
361 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
362 << QueryingAA << " into " << S << "\n");
363
364 assert((QueryingAA.getIRPosition().getPositionKind() ==
366 QueryingAA.getIRPosition().getPositionKind() ==
368 "Can only clamp returned value states for a function returned or call "
369 "site returned position!");
370
371 // Use an optional state as there might not be any return values and we want
372 // to join (IntegerState::operator&) the state of all there are.
373 std::optional<StateType> T;
374
375 // Callback for each possibly returned value.
376 auto CheckReturnValue = [&](Value &RV) -> bool {
377 const IRPosition &RVPos = IRPosition::value(RV, CBContext);
378 // If possible, use the hasAssumedIRAttr interface.
379 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
380 bool IsKnown;
381 return AA::hasAssumedIRAttr<IRAttributeKind>(
382 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
383 }
384
385 const AAType *AA =
386 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
387 if (!AA)
388 return false;
389 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
390 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
391 const StateType &AAS = AA->getState();
392 if (!T)
393 T = StateType::getBestState(AAS);
394 *T &= AAS;
395 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
396 << "\n");
397 return T->isValidState();
398 };
399
400 if (!A.checkForAllReturnedValues(CheckReturnValue, QueryingAA,
401 AA::ValueScope::Intraprocedural,
402 RecurseForSelectAndPHI))
403 S.indicatePessimisticFixpoint();
404 else if (T)
405 S ^= *T;
406}
407
408namespace {
409/// Helper class for generic deduction: return value -> returned position.
410template <typename AAType, typename BaseType,
411 typename StateType = typename BaseType::StateType,
412 bool PropagateCallBaseContext = false,
413 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
414 bool RecurseForSelectAndPHI = true>
415struct AAReturnedFromReturnedValues : public BaseType {
416 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
417 : BaseType(IRP, A) {}
418
419 /// See AbstractAttribute::updateImpl(...).
420 ChangeStatus updateImpl(Attributor &A) override {
421 StateType S(StateType::getBestState(this->getState()));
422 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
423 RecurseForSelectAndPHI>(
424 A, *this, S,
425 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
426 // TODO: If we know we visited all returned values, thus no are assumed
427 // dead, we can take the known information from the state T.
428 return clampStateAndIndicateChange<StateType>(this->getState(), S);
429 }
430};
431
432/// Clamp the information known at all call sites for a given argument
433/// (identified by \p QueryingAA) into \p S.
434template <typename AAType, typename StateType = typename AAType::StateType,
435 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
436static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
437 StateType &S) {
438 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
439 << QueryingAA << " into " << S << "\n");
440
441 assert(QueryingAA.getIRPosition().getPositionKind() ==
443 "Can only clamp call site argument states for an argument position!");
444
445 // Use an optional state as there might not be any return values and we want
446 // to join (IntegerState::operator&) the state of all there are.
447 std::optional<StateType> T;
448
449 // The argument number which is also the call site argument number.
450 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
451
452 auto CallSiteCheck = [&](AbstractCallSite ACS) {
453 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
454 // Check if a coresponding argument was found or if it is on not associated
455 // (which can happen for callback calls).
456 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
457 return false;
458
459 // If possible, use the hasAssumedIRAttr interface.
460 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
461 bool IsKnown;
462 return AA::hasAssumedIRAttr<IRAttributeKind>(
463 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
464 }
465
466 const AAType *AA =
467 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
468 if (!AA)
469 return false;
470 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
471 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
472 << "\n");
473 const StateType &AAS = AA->getState();
474 if (!T)
475 T = StateType::getBestState(AAS);
476 *T &= AAS;
477 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
478 << "\n");
479 return T->isValidState();
480 };
481
482 bool UsedAssumedInformation = false;
483 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
484 UsedAssumedInformation))
485 S.indicatePessimisticFixpoint();
486 else if (T)
487 S ^= *T;
488}
489
490/// This function is the bridge between argument position and the call base
491/// context.
492template <typename AAType, typename BaseType,
493 typename StateType = typename AAType::StateType,
494 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
495bool getArgumentStateFromCallBaseContext(Attributor &A,
496 BaseType &QueryingAttribute,
497 IRPosition &Pos, StateType &State) {
499 "Expected an 'argument' position !");
500 const CallBase *CBContext = Pos.getCallBaseContext();
501 if (!CBContext)
502 return false;
503
504 int ArgNo = Pos.getCallSiteArgNo();
505 assert(ArgNo >= 0 && "Invalid Arg No!");
506 const IRPosition CBArgPos = IRPosition::callsite_argument(*CBContext, ArgNo);
507
508 // If possible, use the hasAssumedIRAttr interface.
509 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
510 bool IsKnown;
511 return AA::hasAssumedIRAttr<IRAttributeKind>(
512 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
513 }
514
515 const auto *AA =
516 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
517 if (!AA)
518 return false;
519 const StateType &CBArgumentState =
520 static_cast<const StateType &>(AA->getState());
521
522 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
523 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
524 << "\n");
525
526 // NOTE: If we want to do call site grouping it should happen here.
527 State ^= CBArgumentState;
528 return true;
529}
530
531/// Helper class for generic deduction: call site argument -> argument position.
532template <typename AAType, typename BaseType,
533 typename StateType = typename AAType::StateType,
534 bool BridgeCallBaseContext = false,
535 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
536struct AAArgumentFromCallSiteArguments : public BaseType {
537 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
538 : BaseType(IRP, A) {}
539
540 /// See AbstractAttribute::updateImpl(...).
541 ChangeStatus updateImpl(Attributor &A) override {
542 StateType S = StateType::getBestState(this->getState());
543
544 if (BridgeCallBaseContext) {
545 bool Success =
546 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
547 IRAttributeKind>(
548 A, *this, this->getIRPosition(), S);
549 if (Success)
550 return clampStateAndIndicateChange<StateType>(this->getState(), S);
551 }
552 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
553 S);
554
555 // TODO: If we know we visited all incoming values, thus no are assumed
556 // dead, we can take the known information from the state T.
557 return clampStateAndIndicateChange<StateType>(this->getState(), S);
558 }
559};
560
561/// Helper class for generic replication: function returned -> cs returned.
562template <typename AAType, typename BaseType,
563 typename StateType = typename BaseType::StateType,
564 bool IntroduceCallBaseContext = false,
565 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
566struct AACalleeToCallSite : public BaseType {
567 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
568
569 /// See AbstractAttribute::updateImpl(...).
570 ChangeStatus updateImpl(Attributor &A) override {
571 auto IRPKind = this->getIRPosition().getPositionKind();
573 IRPKind == IRPosition::IRP_CALL_SITE) &&
574 "Can only wrap function returned positions for call site "
575 "returned positions!");
576 auto &S = this->getState();
577
578 CallBase &CB = cast<CallBase>(this->getAnchorValue());
579 if (IntroduceCallBaseContext)
580 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
581 << "\n");
582
583 ChangeStatus Changed = ChangeStatus::UNCHANGED;
584 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
585 for (const Function *Callee : Callees) {
586 IRPosition FnPos =
588 ? IRPosition::returned(*Callee,
589 IntroduceCallBaseContext ? &CB : nullptr)
591 *Callee, IntroduceCallBaseContext ? &CB : nullptr);
592 // If possible, use the hasAssumedIRAttr interface.
593 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
594 bool IsKnown;
595 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
596 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
597 return false;
598 continue;
599 }
600
601 const AAType *AA =
602 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
603 if (!AA)
604 return false;
605 Changed |= clampStateAndIndicateChange(S, AA->getState());
606 if (S.isAtFixpoint())
607 return S.isValidState();
608 }
609 return true;
610 };
611 if (!A.checkForAllCallees(CalleePred, *this, CB))
612 return S.indicatePessimisticFixpoint();
613 return Changed;
614 }
615};
616
617/// Helper function to accumulate uses.
618template <class AAType, typename StateType = typename AAType::StateType>
619static void followUsesInContext(AAType &AA, Attributor &A,
621 const Instruction *CtxI,
623 StateType &State) {
624 auto EIt = Explorer.begin(CtxI), EEnd = Explorer.end(CtxI);
625 for (unsigned u = 0; u < Uses.size(); ++u) {
626 const Use *U = Uses[u];
627 if (const Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {
628 bool Found = Explorer.findInContextOf(UserI, EIt, EEnd);
629 if (Found && AA.followUseInMBEC(A, U, UserI, State))
630 for (const Use &Us : UserI->uses())
631 Uses.insert(&Us);
632 }
633 }
634}
635
636/// Use the must-be-executed-context around \p I to add information into \p S.
637/// The AAType class is required to have `followUseInMBEC` method with the
638/// following signature and behaviour:
639///
640/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
641/// U - Underlying use.
642/// I - The user of the \p U.
643/// Returns true if the value should be tracked transitively.
644///
645template <class AAType, typename StateType = typename AAType::StateType>
646static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
647 Instruction &CtxI) {
649 A.getInfoCache().getMustBeExecutedContextExplorer();
650 if (!Explorer)
651 return;
652
653 // Container for (transitive) uses of the associated value.
655 for (const Use &U : AA.getIRPosition().getAssociatedValue().uses())
656 Uses.insert(&U);
657
658 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
659
660 if (S.isAtFixpoint())
661 return;
662
664 auto Pred = [&](const Instruction *I) {
665 if (const BranchInst *Br = dyn_cast<BranchInst>(I))
666 if (Br->isConditional())
667 BrInsts.push_back(Br);
668 return true;
669 };
670
671 // Here, accumulate conditional branch instructions in the context. We
672 // explore the child paths and collect the known states. The disjunction of
673 // those states can be merged to its own state. Let ParentState_i be a state
674 // to indicate the known information for an i-th branch instruction in the
675 // context. ChildStates are created for its successors respectively.
676 //
677 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
678 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
679 // ...
680 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
681 //
682 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
683 //
684 // FIXME: Currently, recursive branches are not handled. For example, we
685 // can't deduce that ptr must be dereferenced in below function.
686 //
687 // void f(int a, int c, int *ptr) {
688 // if(a)
689 // if (b) {
690 // *ptr = 0;
691 // } else {
692 // *ptr = 1;
693 // }
694 // else {
695 // if (b) {
696 // *ptr = 0;
697 // } else {
698 // *ptr = 1;
699 // }
700 // }
701 // }
702
703 Explorer->checkForAllContext(&CtxI, Pred);
704 for (const BranchInst *Br : BrInsts) {
705 StateType ParentState;
706
707 // The known state of the parent state is a conjunction of children's
708 // known states so it is initialized with a best state.
709 ParentState.indicateOptimisticFixpoint();
710
711 for (const BasicBlock *BB : Br->successors()) {
712 StateType ChildState;
713
714 size_t BeforeSize = Uses.size();
715 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
716
717 // Erase uses which only appear in the child.
718 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
719 It = Uses.erase(It);
720
721 ParentState &= ChildState;
722 }
723
724 // Use only known state.
725 S += ParentState;
726 }
727}
728} // namespace
729
730/// ------------------------ PointerInfo ---------------------------------------
731
732namespace llvm {
733namespace AA {
734namespace PointerInfo {
735
736struct State;
737
738} // namespace PointerInfo
739} // namespace AA
740
741/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
742template <>
745 static inline Access getEmptyKey();
746 static inline Access getTombstoneKey();
747 static unsigned getHashValue(const Access &A);
748 static bool isEqual(const Access &LHS, const Access &RHS);
749};
750
751/// Helper that allows RangeTy as a key in a DenseMap.
752template <> struct DenseMapInfo<AA::RangeTy> {
753 static inline AA::RangeTy getEmptyKey() {
754 auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();
755 return AA::RangeTy{EmptyKey, EmptyKey};
756 }
757
758 static inline AA::RangeTy getTombstoneKey() {
759 auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();
760 return AA::RangeTy{TombstoneKey, TombstoneKey};
761 }
762
763 static unsigned getHashValue(const AA::RangeTy &Range) {
767 }
768
769 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
770 return A == B;
771 }
772};
773
774/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign
775/// but the instruction
776struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {
779 static inline Access getEmptyKey();
780 static inline Access getTombstoneKey();
781 static unsigned getHashValue(const Access &A);
782 static bool isEqual(const Access &LHS, const Access &RHS);
783};
784
785} // namespace llvm
786
787/// A type to track pointer/struct usage and accesses for AAPointerInfo.
789 /// Return the best possible representable state.
790 static State getBestState(const State &SIS) { return State(); }
791
792 /// Return the worst possible representable state.
793 static State getWorstState(const State &SIS) {
794 State R;
795 R.indicatePessimisticFixpoint();
796 return R;
797 }
798
799 State() = default;
800 State(State &&SIS) = default;
801
802 const State &getAssumed() const { return *this; }
803
804 /// See AbstractState::isValidState().
805 bool isValidState() const override { return BS.isValidState(); }
806
807 /// See AbstractState::isAtFixpoint().
808 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
809
810 /// See AbstractState::indicateOptimisticFixpoint().
814 }
815
816 /// See AbstractState::indicatePessimisticFixpoint().
820 }
821
822 State &operator=(const State &R) {
823 if (this == &R)
824 return *this;
825 BS = R.BS;
826 AccessList = R.AccessList;
827 OffsetBins = R.OffsetBins;
828 RemoteIMap = R.RemoteIMap;
829 return *this;
830 }
831
833 if (this == &R)
834 return *this;
835 std::swap(BS, R.BS);
836 std::swap(AccessList, R.AccessList);
837 std::swap(OffsetBins, R.OffsetBins);
838 std::swap(RemoteIMap, R.RemoteIMap);
839 return *this;
840 }
841
842 /// Add a new Access to the state at offset \p Offset and with size \p Size.
843 /// The access is associated with \p I, writes \p Content (if anything), and
844 /// is of kind \p Kind. If an Access already exists for the same \p I and same
845 /// \p RemoteI, the two are combined, potentially losing information about
846 /// offset and size. The resulting access must now be moved from its original
847 /// OffsetBin to the bin for its new offset.
848 ///
849 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
851 Instruction &I, std::optional<Value *> Content,
853 Instruction *RemoteI = nullptr);
854
857 int64_t numOffsetBins() const { return OffsetBins.size(); }
858
859 const AAPointerInfo::Access &getAccess(unsigned Index) const {
860 return AccessList[Index];
861 }
862
863protected:
864 // Every memory instruction results in an Access object. We maintain a list of
865 // all Access objects that we own, along with the following maps:
866 //
867 // - OffsetBins: RangeTy -> { Access }
868 // - RemoteIMap: RemoteI x LocalI -> Access
869 //
870 // A RemoteI is any instruction that accesses memory. RemoteI is different
871 // from LocalI if and only if LocalI is a call; then RemoteI is some
872 // instruction in the callgraph starting from LocalI. Multiple paths in the
873 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
874 // are all combined into a single Access object. This may result in loss of
875 // information in RangeTy in the Access object.
879
880 /// See AAPointerInfo::forallInterferingAccesses.
883 function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const {
884 if (!isValidState())
885 return false;
886
887 for (const auto &It : OffsetBins) {
888 AA::RangeTy ItRange = It.getFirst();
889 if (!Range.mayOverlap(ItRange))
890 continue;
891 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
892 for (auto Index : It.getSecond()) {
893 auto &Access = AccessList[Index];
894 if (!CB(Access, IsExact))
895 return false;
896 }
897 }
898 return true;
899 }
900
901 /// See AAPointerInfo::forallInterferingAccesses.
903 Instruction &I,
904 function_ref<bool(const AAPointerInfo::Access &, bool)> CB,
905 AA::RangeTy &Range) const {
906 if (!isValidState())
907 return false;
908
909 auto LocalList = RemoteIMap.find(&I);
910 if (LocalList == RemoteIMap.end()) {
911 return true;
912 }
913
914 for (unsigned Index : LocalList->getSecond()) {
915 for (auto &R : AccessList[Index]) {
916 Range &= R;
917 if (Range.offsetAndSizeAreUnknown())
918 break;
919 }
920 }
922 }
923
924private:
925 /// State to track fixpoint and validity.
926 BooleanState BS;
927};
928
931 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
932 Instruction *RemoteI) {
933 RemoteI = RemoteI ? RemoteI : &I;
934
935 // Check if we have an access for this instruction, if not, simply add it.
936 auto &LocalList = RemoteIMap[RemoteI];
937 bool AccExists = false;
938 unsigned AccIndex = AccessList.size();
939 for (auto Index : LocalList) {
940 auto &A = AccessList[Index];
941 if (A.getLocalInst() == &I) {
942 AccExists = true;
943 AccIndex = Index;
944 break;
945 }
946 }
947
948 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
949 LLVM_DEBUG(if (ToAdd.size()) dbgs()
950 << "[AAPointerInfo] Inserting access in new offset bins\n";);
951
952 for (auto Key : ToAdd) {
953 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
954 OffsetBins[Key].insert(AccIndex);
955 }
956 };
957
958 if (!AccExists) {
959 AccessList.emplace_back(&I, RemoteI, Ranges, Content, Kind, Ty);
960 assert((AccessList.size() == AccIndex + 1) &&
961 "New Access should have been at AccIndex");
962 LocalList.push_back(AccIndex);
963 AddToBins(AccessList[AccIndex].getRanges());
965 }
966
967 // Combine the new Access with the existing Access, and then update the
968 // mapping in the offset bins.
969 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
970 auto &Current = AccessList[AccIndex];
971 auto Before = Current;
972 Current &= Acc;
973 if (Current == Before)
975
976 auto &ExistingRanges = Before.getRanges();
977 auto &NewRanges = Current.getRanges();
978
979 // Ranges that are in the old access but not the new access need to be removed
980 // from the offset bins.
982 AAPointerInfo::RangeList::set_difference(ExistingRanges, NewRanges, ToRemove);
983 LLVM_DEBUG(if (ToRemove.size()) dbgs()
984 << "[AAPointerInfo] Removing access from old offset bins\n";);
985
986 for (auto Key : ToRemove) {
987 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
988 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
989 auto &Bin = OffsetBins[Key];
990 assert(Bin.count(AccIndex) &&
991 "Expected bin to actually contain the Access.");
992 Bin.erase(AccIndex);
993 }
994
995 // Ranges that are in the new access but not the old access need to be added
996 // to the offset bins.
998 AAPointerInfo::RangeList::set_difference(NewRanges, ExistingRanges, ToAdd);
999 AddToBins(ToAdd);
1000 return ChangeStatus::CHANGED;
1001}
1002
1003namespace {
1004
1005/// A helper containing a list of offsets computed for a Use. Ideally this
1006/// list should be strictly ascending, but we ensure that only when we
1007/// actually translate the list of offsets to a RangeList.
1008struct OffsetInfo {
1009 using VecTy = SmallVector<int64_t>;
1010 using const_iterator = VecTy::const_iterator;
1011 VecTy Offsets;
1012
1013 const_iterator begin() const { return Offsets.begin(); }
1014 const_iterator end() const { return Offsets.end(); }
1015
1016 bool operator==(const OffsetInfo &RHS) const {
1017 return Offsets == RHS.Offsets;
1018 }
1019
1020 bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); }
1021
1022 void insert(int64_t Offset) { Offsets.push_back(Offset); }
1023 bool isUnassigned() const { return Offsets.size() == 0; }
1024
1025 bool isUnknown() const {
1026 if (isUnassigned())
1027 return false;
1028 if (Offsets.size() == 1)
1029 return Offsets.front() == AA::RangeTy::Unknown;
1030 return false;
1031 }
1032
1033 void setUnknown() {
1034 Offsets.clear();
1035 Offsets.push_back(AA::RangeTy::Unknown);
1036 }
1037
1038 void addToAll(int64_t Inc) {
1039 for (auto &Offset : Offsets) {
1040 Offset += Inc;
1041 }
1042 }
1043
1044 /// Copy offsets from \p R into the current list.
1045 ///
1046 /// Ideally all lists should be strictly ascending, but we defer that to the
1047 /// actual use of the list. So we just blindly append here.
1048 void merge(const OffsetInfo &R) { Offsets.append(R.Offsets); }
1049};
1050
1051#ifndef NDEBUG
1052static raw_ostream &operator<<(raw_ostream &OS, const OffsetInfo &OI) {
1053 ListSeparator LS;
1054 OS << "[";
1055 for (auto Offset : OI) {
1056 OS << LS << Offset;
1057 }
1058 OS << "]";
1059 return OS;
1060}
1061#endif // NDEBUG
1062
1063struct AAPointerInfoImpl
1064 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1066 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1067
1068 /// See AbstractAttribute::getAsStr().
1069 const std::string getAsStr(Attributor *A) const override {
1070 return std::string("PointerInfo ") +
1071 (isValidState() ? (std::string("#") +
1072 std::to_string(OffsetBins.size()) + " bins")
1073 : "<invalid>");
1074 }
1075
1076 /// See AbstractAttribute::manifest(...).
1077 ChangeStatus manifest(Attributor &A) override {
1078 return AAPointerInfo::manifest(A);
1079 }
1080
1081 virtual const_bin_iterator begin() const override { return State::begin(); }
1082 virtual const_bin_iterator end() const override { return State::end(); }
1083 virtual int64_t numOffsetBins() const override {
1084 return State::numOffsetBins();
1085 }
1086
1087 bool forallInterferingAccesses(
1089 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1090 const override {
1091 return State::forallInterferingAccesses(Range, CB);
1092 }
1093
1094 bool forallInterferingAccesses(
1095 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1096 bool FindInterferingWrites, bool FindInterferingReads,
1097 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1099 function_ref<bool(const Access &)> SkipCB) const override {
1100 HasBeenWrittenTo = false;
1101
1102 SmallPtrSet<const Access *, 8> DominatingWrites;
1103 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1104
1105 Function &Scope = *I.getFunction();
1106 bool IsKnownNoSync;
1107 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1108 A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1109 IsKnownNoSync);
1110 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1111 IRPosition::function(Scope), &QueryingAA, DepClassTy::NONE);
1112 bool AllInSameNoSyncFn = IsAssumedNoSync;
1113 bool InstIsExecutedByInitialThreadOnly =
1114 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1115
1116 // If the function is not ending in aligned barriers, we need the stores to
1117 // be in aligned barriers. The load being in one is not sufficient since the
1118 // store might be executed by a thread that disappears after, causing the
1119 // aligned barrier guarding the load to unblock and the load to read a value
1120 // that has no CFG path to the load.
1121 bool InstIsExecutedInAlignedRegion =
1122 FindInterferingReads && ExecDomainAA &&
1123 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1124
1125 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1126 A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1127
1128 InformationCache &InfoCache = A.getInfoCache();
1129 bool IsThreadLocalObj =
1130 AA::isAssumedThreadLocalObject(A, getAssociatedValue(), *this);
1131
1132 // Helper to determine if we need to consider threading, which we cannot
1133 // right now. However, if the function is (assumed) nosync or the thread
1134 // executing all instructions is the main thread only we can ignore
1135 // threading. Also, thread-local objects do not require threading reasoning.
1136 // Finally, we can ignore threading if either access is executed in an
1137 // aligned region.
1138 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1139 if (IsThreadLocalObj || AllInSameNoSyncFn)
1140 return true;
1141 const auto *FnExecDomainAA =
1142 I.getFunction() == &Scope
1143 ? ExecDomainAA
1144 : A.lookupAAFor<AAExecutionDomain>(
1145 IRPosition::function(*I.getFunction()), &QueryingAA,
1146 DepClassTy::NONE);
1147 if (!FnExecDomainAA)
1148 return false;
1149 if (InstIsExecutedInAlignedRegion ||
1150 (FindInterferingWrites &&
1151 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1152 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1153 return true;
1154 }
1155 if (InstIsExecutedByInitialThreadOnly &&
1156 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1157 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1158 return true;
1159 }
1160 return false;
1161 };
1162
1163 // Helper to determine if the access is executed by the same thread as the
1164 // given instruction, for now it is sufficient to avoid any potential
1165 // threading effects as we cannot deal with them anyway.
1166 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1167 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1168 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1169 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1170 };
1171
1172 // TODO: Use inter-procedural reachability and dominance.
1173 bool IsKnownNoRecurse;
1174 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1175 A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,
1176 IsKnownNoRecurse);
1177
1178 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1179 // AAExecutionDomain) such that we allow scopes other than kernels as long
1180 // as the reaching kernels are disjoint.
1181 bool InstInKernel = Scope.hasFnAttribute("kernel");
1182 bool ObjHasKernelLifetime = false;
1183 const bool UseDominanceReasoning =
1184 FindInterferingWrites && IsKnownNoRecurse;
1185 const DominatorTree *DT =
1187
1188 // Helper to check if a value has "kernel lifetime", that is it will not
1189 // outlive a GPU kernel. This is true for shared, constant, and local
1190 // globals on AMD and NVIDIA GPUs.
1191 auto HasKernelLifetime = [&](Value *V, Module &M) {
1192 if (!AA::isGPU(M))
1193 return false;
1194 switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {
1195 case AA::GPUAddressSpace::Shared:
1196 case AA::GPUAddressSpace::Constant:
1197 case AA::GPUAddressSpace::Local:
1198 return true;
1199 default:
1200 return false;
1201 };
1202 };
1203
1204 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1205 // to determine if we should look at reachability from the callee. For
1206 // certain pointers we know the lifetime and we do not have to step into the
1207 // callee to determine reachability as the pointer would be dead in the
1208 // callee. See the conditional initialization below.
1209 std::function<bool(const Function &)> IsLiveInCalleeCB;
1210
1211 if (auto *AI = dyn_cast<AllocaInst>(&getAssociatedValue())) {
1212 // If the alloca containing function is not recursive the alloca
1213 // must be dead in the callee.
1214 const Function *AIFn = AI->getFunction();
1215 ObjHasKernelLifetime = AIFn->hasFnAttribute("kernel");
1216 bool IsKnownNoRecurse;
1217 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1218 A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,
1219 IsKnownNoRecurse)) {
1220 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1221 }
1222 } else if (auto *GV = dyn_cast<GlobalValue>(&getAssociatedValue())) {
1223 // If the global has kernel lifetime we can stop if we reach a kernel
1224 // as it is "dead" in the (unknown) callees.
1225 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1226 if (ObjHasKernelLifetime)
1227 IsLiveInCalleeCB = [](const Function &Fn) {
1228 return !Fn.hasFnAttribute("kernel");
1229 };
1230 }
1231
1232 // Set of accesses/instructions that will overwrite the result and are
1233 // therefore blockers in the reachability traversal.
1234 AA::InstExclusionSetTy ExclusionSet;
1235
1236 auto AccessCB = [&](const Access &Acc, bool Exact) {
1237 Function *AccScope = Acc.getRemoteInst()->getFunction();
1238 bool AccInSameScope = AccScope == &Scope;
1239
1240 // If the object has kernel lifetime we can ignore accesses only reachable
1241 // by other kernels. For now we only skip accesses *in* other kernels.
1242 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1243 AccScope->hasFnAttribute("kernel"))
1244 return true;
1245
1246 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1247 if (Acc.isWrite() || (isa<LoadInst>(I) && Acc.isWriteOrAssumption()))
1248 ExclusionSet.insert(Acc.getRemoteInst());
1249 }
1250
1251 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1252 (!FindInterferingReads || !Acc.isRead()))
1253 return true;
1254
1255 bool Dominates = FindInterferingWrites && DT && Exact &&
1256 Acc.isMustAccess() && AccInSameScope &&
1257 DT->dominates(Acc.getRemoteInst(), &I);
1258 if (Dominates)
1259 DominatingWrites.insert(&Acc);
1260
1261 // Track if all interesting accesses are in the same `nosync` function as
1262 // the given instruction.
1263 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1264
1265 InterferingAccesses.push_back({&Acc, Exact});
1266 return true;
1267 };
1268 if (!State::forallInterferingAccesses(I, AccessCB, Range))
1269 return false;
1270
1271 HasBeenWrittenTo = !DominatingWrites.empty();
1272
1273 // Dominating writes form a chain, find the least/lowest member.
1274 Instruction *LeastDominatingWriteInst = nullptr;
1275 for (const Access *Acc : DominatingWrites) {
1276 if (!LeastDominatingWriteInst) {
1277 LeastDominatingWriteInst = Acc->getRemoteInst();
1278 } else if (DT->dominates(LeastDominatingWriteInst,
1279 Acc->getRemoteInst())) {
1280 LeastDominatingWriteInst = Acc->getRemoteInst();
1281 }
1282 }
1283
1284 // Helper to determine if we can skip a specific write access.
1285 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1286 if (SkipCB && SkipCB(Acc))
1287 return true;
1288 if (!CanIgnoreThreading(Acc))
1289 return false;
1290
1291 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1292 // If we successfully excluded all effects we are interested in, the
1293 // access can be skipped.
1294 bool ReadChecked = !FindInterferingReads;
1295 bool WriteChecked = !FindInterferingWrites;
1296
1297 // If the instruction cannot reach the access, the former does not
1298 // interfere with what the access reads.
1299 if (!ReadChecked) {
1300 if (!AA::isPotentiallyReachable(A, I, *Acc.getRemoteInst(), QueryingAA,
1301 &ExclusionSet, IsLiveInCalleeCB))
1302 ReadChecked = true;
1303 }
1304 // If the instruction cannot be reach from the access, the latter does not
1305 // interfere with what the instruction reads.
1306 if (!WriteChecked) {
1307 if (!AA::isPotentiallyReachable(A, *Acc.getRemoteInst(), I, QueryingAA,
1308 &ExclusionSet, IsLiveInCalleeCB))
1309 WriteChecked = true;
1310 }
1311
1312 // If we still might be affected by the write of the access but there are
1313 // dominating writes in the function of the instruction
1314 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1315 // by them. This would have happend above if they are all in the same
1316 // function, so we only check the inter-procedural case. Effectively, we
1317 // want to show that there is no call after the dominting write that might
1318 // reach the access, and when it returns reach the instruction with the
1319 // updated value. To this end, we iterate all call sites, check if they
1320 // might reach the instruction without going through another access
1321 // (ExclusionSet) and at the same time might reach the access. However,
1322 // that is all part of AAInterFnReachability.
1323 if (!WriteChecked && HasBeenWrittenTo &&
1324 Acc.getRemoteInst()->getFunction() != &Scope) {
1325
1326 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1327 QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL);
1328 if (FnReachabilityAA) {
1329 // Without going backwards in the call tree, can we reach the access
1330 // from the least dominating write. Do not allow to pass the
1331 // instruction itself either.
1332 bool Inserted = ExclusionSet.insert(&I).second;
1333
1334 if (!FnReachabilityAA->instructionCanReach(
1335 A, *LeastDominatingWriteInst,
1336 *Acc.getRemoteInst()->getFunction(), &ExclusionSet))
1337 WriteChecked = true;
1338
1339 if (Inserted)
1340 ExclusionSet.erase(&I);
1341 }
1342 }
1343
1344 if (ReadChecked && WriteChecked)
1345 return true;
1346
1347 if (!DT || !UseDominanceReasoning)
1348 return false;
1349 if (!DominatingWrites.count(&Acc))
1350 return false;
1351 return LeastDominatingWriteInst != Acc.getRemoteInst();
1352 };
1353
1354 // Run the user callback on all accesses we cannot skip and return if
1355 // that succeeded for all or not.
1356 for (auto &It : InterferingAccesses) {
1357 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1358 !CanSkipAccess(*It.first, It.second)) {
1359 if (!UserCB(*It.first, It.second))
1360 return false;
1361 }
1362 }
1363 return true;
1364 }
1365
1366 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1367 const AAPointerInfo &OtherAA,
1368 CallBase &CB) {
1369 using namespace AA::PointerInfo;
1370 if (!OtherAA.getState().isValidState() || !isValidState())
1371 return indicatePessimisticFixpoint();
1372
1373 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1374 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1375
1376 // Combine the accesses bin by bin.
1377 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1378 const auto &State = OtherAAImpl.getState();
1379 for (const auto &It : State) {
1380 for (auto Index : It.getSecond()) {
1381 const auto &RAcc = State.getAccess(Index);
1382 if (IsByval && !RAcc.isRead())
1383 continue;
1384 bool UsedAssumedInformation = false;
1385 AccessKind AK = RAcc.getKind();
1386 auto Content = A.translateArgumentToCallSiteContent(
1387 RAcc.getContent(), CB, *this, UsedAssumedInformation);
1388 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1389 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1390
1391 Changed |= addAccess(A, RAcc.getRanges(), CB, Content, AK,
1392 RAcc.getType(), RAcc.getRemoteInst());
1393 }
1394 }
1395 return Changed;
1396 }
1397
1398 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1399 const OffsetInfo &Offsets, CallBase &CB) {
1400 using namespace AA::PointerInfo;
1401 if (!OtherAA.getState().isValidState() || !isValidState())
1402 return indicatePessimisticFixpoint();
1403
1404 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1405
1406 // Combine the accesses bin by bin.
1407 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1408 const auto &State = OtherAAImpl.getState();
1409 for (const auto &It : State) {
1410 for (auto Index : It.getSecond()) {
1411 const auto &RAcc = State.getAccess(Index);
1412 for (auto Offset : Offsets) {
1413 auto NewRanges = Offset == AA::RangeTy::Unknown
1415 : RAcc.getRanges();
1416 if (!NewRanges.isUnknown()) {
1417 NewRanges.addToAllOffsets(Offset);
1418 }
1419 Changed |=
1420 addAccess(A, NewRanges, CB, RAcc.getContent(), RAcc.getKind(),
1421 RAcc.getType(), RAcc.getRemoteInst());
1422 }
1423 }
1424 }
1425 return Changed;
1426 }
1427
1428 /// Statistic tracking for all AAPointerInfo implementations.
1429 /// See AbstractAttribute::trackStatistics().
1430 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1431
1432 /// Dump the state into \p O.
1433 void dumpState(raw_ostream &O) {
1434 for (auto &It : OffsetBins) {
1435 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1436 << "] : " << It.getSecond().size() << "\n";
1437 for (auto AccIndex : It.getSecond()) {
1438 auto &Acc = AccessList[AccIndex];
1439 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1440 if (Acc.getLocalInst() != Acc.getRemoteInst())
1441 O << " --> " << *Acc.getRemoteInst()
1442 << "\n";
1443 if (!Acc.isWrittenValueYetUndetermined()) {
1444 if (isa_and_nonnull<Function>(Acc.getWrittenValue()))
1445 O << " - c: func " << Acc.getWrittenValue()->getName()
1446 << "\n";
1447 else if (Acc.getWrittenValue())
1448 O << " - c: " << *Acc.getWrittenValue() << "\n";
1449 else
1450 O << " - c: <unknown>\n";
1451 }
1452 }
1453 }
1454 }
1455};
1456
1457struct AAPointerInfoFloating : public AAPointerInfoImpl {
1459 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1460 : AAPointerInfoImpl(IRP, A) {}
1461
1462 /// Deal with an access and signal if it was handled successfully.
1463 bool handleAccess(Attributor &A, Instruction &I,
1464 std::optional<Value *> Content, AccessKind Kind,
1465 SmallVectorImpl<int64_t> &Offsets, ChangeStatus &Changed,
1466 Type &Ty) {
1467 using namespace AA::PointerInfo;
1469 const DataLayout &DL = A.getDataLayout();
1470 TypeSize AccessSize = DL.getTypeStoreSize(&Ty);
1471 if (!AccessSize.isScalable())
1472 Size = AccessSize.getFixedValue();
1473
1474 // Make a strictly ascending list of offsets as required by addAccess()
1475 llvm::sort(Offsets);
1476 auto *Last = llvm::unique(Offsets);
1477 Offsets.erase(Last, Offsets.end());
1478
1479 VectorType *VT = dyn_cast<VectorType>(&Ty);
1480 if (!VT || VT->getElementCount().isScalable() ||
1481 !Content.value_or(nullptr) || !isa<Constant>(*Content) ||
1482 (*Content)->getType() != VT ||
1483 DL.getTypeStoreSize(VT->getElementType()).isScalable()) {
1484 Changed = Changed | addAccess(A, {Offsets, Size}, I, Content, Kind, &Ty);
1485 } else {
1486 // Handle vector stores with constant content element-wise.
1487 // TODO: We could look for the elements or create instructions
1488 // representing them.
1489 // TODO: We need to push the Content into the range abstraction
1490 // (AA::RangeTy) to allow different content values for different
1491 // ranges. ranges. Hence, support vectors storing different values.
1492 Type *ElementType = VT->getElementType();
1493 int64_t ElementSize = DL.getTypeStoreSize(ElementType).getFixedValue();
1494 auto *ConstContent = cast<Constant>(*Content);
1495 Type *Int32Ty = Type::getInt32Ty(ElementType->getContext());
1496 SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1497
1498 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1499 Value *ElementContent = ConstantExpr::getExtractElement(
1500 ConstContent, ConstantInt::get(Int32Ty, i));
1501
1502 // Add the element access.
1503 Changed = Changed | addAccess(A, {ElementOffsets, ElementSize}, I,
1504 ElementContent, Kind, ElementType);
1505
1506 // Advance the offsets for the next element.
1507 for (auto &ElementOffset : ElementOffsets)
1508 ElementOffset += ElementSize;
1509 }
1510 }
1511 return true;
1512 };
1513
1514 /// See AbstractAttribute::updateImpl(...).
1515 ChangeStatus updateImpl(Attributor &A) override;
1516
1517 /// If the indices to \p GEP can be traced to constants, incorporate all
1518 /// of these into \p UsrOI.
1519 ///
1520 /// \return true iff \p UsrOI is updated.
1521 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1522 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1523 const GEPOperator *GEP);
1524
1525 /// See AbstractAttribute::trackStatistics()
1526 void trackStatistics() const override {
1527 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1528 }
1529};
1530
1531bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1532 const DataLayout &DL,
1533 OffsetInfo &UsrOI,
1534 const OffsetInfo &PtrOI,
1535 const GEPOperator *GEP) {
1536 unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());
1537 MapVector<Value *, APInt> VariableOffsets;
1538 APInt ConstantOffset(BitWidth, 0);
1539
1540 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1541 "Don't look for constant values if the offset has already been "
1542 "determined to be unknown.");
1543
1544 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1545 UsrOI.setUnknown();
1546 return true;
1547 }
1548
1549 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1550 << (VariableOffsets.empty() ? "" : "not") << " constant "
1551 << *GEP << "\n");
1552
1553 auto Union = PtrOI;
1554 Union.addToAll(ConstantOffset.getSExtValue());
1555
1556 // Each VI in VariableOffsets has a set of potential constant values. Every
1557 // combination of elements, picked one each from these sets, is separately
1558 // added to the original set of offsets, thus resulting in more offsets.
1559 for (const auto &VI : VariableOffsets) {
1560 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1561 *this, IRPosition::value(*VI.first), DepClassTy::OPTIONAL);
1562 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1563 UsrOI.setUnknown();
1564 return true;
1565 }
1566
1567 // UndefValue is treated as a zero, which leaves Union as is.
1568 if (PotentialConstantsAA->undefIsContained())
1569 continue;
1570
1571 // We need at least one constant in every set to compute an actual offset.
1572 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1573 // don't actually exist. In other words, the absence of constant values
1574 // implies that the operation can be assumed dead for now.
1575 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1576 if (AssumedSet.empty())
1577 return false;
1578
1579 OffsetInfo Product;
1580 for (const auto &ConstOffset : AssumedSet) {
1581 auto CopyPerOffset = Union;
1582 CopyPerOffset.addToAll(ConstOffset.getSExtValue() *
1583 VI.second.getZExtValue());
1584 Product.merge(CopyPerOffset);
1585 }
1586 Union = Product;
1587 }
1588
1589 UsrOI = std::move(Union);
1590 return true;
1591}
1592
1593ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1594 using namespace AA::PointerInfo;
1596 const DataLayout &DL = A.getDataLayout();
1597 Value &AssociatedValue = getAssociatedValue();
1598
1599 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1600 OffsetInfoMap[&AssociatedValue].insert(0);
1601
1602 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1603 // One does not simply walk into a map and assign a reference to a possibly
1604 // new location. That can cause an invalidation before the assignment
1605 // happens, like so:
1606 //
1607 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1608 //
1609 // The RHS is a reference that may be invalidated by an insertion caused by
1610 // the LHS. So we ensure that the side-effect of the LHS happens first.
1611
1612 assert(OffsetInfoMap.contains(CurPtr) &&
1613 "CurPtr does not exist in the map!");
1614
1615 auto &UsrOI = OffsetInfoMap[Usr];
1616 auto &PtrOI = OffsetInfoMap[CurPtr];
1617 assert(!PtrOI.isUnassigned() &&
1618 "Cannot pass through if the input Ptr was not visited!");
1619 UsrOI.merge(PtrOI);
1620 Follow = true;
1621 return true;
1622 };
1623
1624 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1625 Value *CurPtr = U.get();
1626 User *Usr = U.getUser();
1627 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1628 << "\n");
1629 assert(OffsetInfoMap.count(CurPtr) &&
1630 "The current pointer offset should have been seeded!");
1631 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1632 "Current pointer should be assigned");
1633
1634 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
1635 if (CE->isCast())
1636 return HandlePassthroughUser(Usr, CurPtr, Follow);
1637 if (!isa<GEPOperator>(CE)) {
1638 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
1639 << "\n");
1640 return false;
1641 }
1642 }
1643 if (auto *GEP = dyn_cast<GEPOperator>(Usr)) {
1644 // Note the order here, the Usr access might change the map, CurPtr is
1645 // already in it though.
1646 auto &UsrOI = OffsetInfoMap[Usr];
1647 auto &PtrOI = OffsetInfoMap[CurPtr];
1648
1649 if (UsrOI.isUnknown())
1650 return true;
1651
1652 if (PtrOI.isUnknown()) {
1653 Follow = true;
1654 UsrOI.setUnknown();
1655 return true;
1656 }
1657
1658 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1659 return true;
1660 }
1661 if (isa<PtrToIntInst>(Usr))
1662 return false;
1663 if (isa<CastInst>(Usr) || isa<SelectInst>(Usr) || isa<ReturnInst>(Usr))
1664 return HandlePassthroughUser(Usr, CurPtr, Follow);
1665
1666 // For PHIs we need to take care of the recurrence explicitly as the value
1667 // might change while we iterate through a loop. For now, we give up if
1668 // the PHI is not invariant.
1669 if (auto *PHI = dyn_cast<PHINode>(Usr)) {
1670 // Note the order here, the Usr access might change the map, CurPtr is
1671 // already in it though.
1672 bool IsFirstPHIUser = !OffsetInfoMap.count(PHI);
1673 auto &UsrOI = OffsetInfoMap[PHI];
1674 auto &PtrOI = OffsetInfoMap[CurPtr];
1675
1676 // Check if the PHI operand has already an unknown offset as we can't
1677 // improve on that anymore.
1678 if (PtrOI.isUnknown()) {
1679 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1680 << *CurPtr << " in " << *PHI << "\n");
1681 Follow = !UsrOI.isUnknown();
1682 UsrOI.setUnknown();
1683 return true;
1684 }
1685
1686 // Check if the PHI is invariant (so far).
1687 if (UsrOI == PtrOI) {
1688 assert(!PtrOI.isUnassigned() &&
1689 "Cannot assign if the current Ptr was not visited!");
1690 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1691 return true;
1692 }
1693
1694 // Check if the PHI operand can be traced back to AssociatedValue.
1695 APInt Offset(
1696 DL.getIndexSizeInBits(CurPtr->getType()->getPointerAddressSpace()),
1697 0);
1698 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1699 DL, Offset, /* AllowNonInbounds */ true);
1700 auto It = OffsetInfoMap.find(CurPtrBase);
1701 if (It == OffsetInfoMap.end()) {
1702 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1703 << *CurPtr << " in " << *PHI
1704 << " (base: " << *CurPtrBase << ")\n");
1705 UsrOI.setUnknown();
1706 Follow = true;
1707 return true;
1708 }
1709
1710 // Check if the PHI operand is not dependent on the PHI itself. Every
1711 // recurrence is a cyclic net of PHIs in the data flow, and has an
1712 // equivalent Cycle in the control flow. One of those PHIs must be in the
1713 // header of that control flow Cycle. This is independent of the choice of
1714 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1715 // every Cycle header; if such a node is marked unknown, this will
1716 // eventually propagate through the whole net of PHIs in the recurrence.
1717 const auto *CI =
1718 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1719 *PHI->getFunction());
1720 if (mayBeInCycle(CI, cast<Instruction>(Usr), /* HeaderOnly */ true)) {
1721 auto BaseOI = It->getSecond();
1722 BaseOI.addToAll(Offset.getZExtValue());
1723 if (IsFirstPHIUser || BaseOI == UsrOI) {
1724 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1725 << " in " << *Usr << "\n");
1726 return HandlePassthroughUser(Usr, CurPtr, Follow);
1727 }
1728
1729 LLVM_DEBUG(
1730 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1731 << *CurPtr << " in " << *PHI << "\n");
1732 UsrOI.setUnknown();
1733 Follow = true;
1734 return true;
1735 }
1736
1737 UsrOI.merge(PtrOI);
1738 Follow = true;
1739 return true;
1740 }
1741
1742 if (auto *LoadI = dyn_cast<LoadInst>(Usr)) {
1743 // If the access is to a pointer that may or may not be the associated
1744 // value, e.g. due to a PHI, we cannot assume it will be read.
1745 AccessKind AK = AccessKind::AK_R;
1746 if (getUnderlyingObject(CurPtr) == &AssociatedValue)
1747 AK = AccessKind(AK | AccessKind::AK_MUST);
1748 else
1749 AK = AccessKind(AK | AccessKind::AK_MAY);
1750 if (!handleAccess(A, *LoadI, /* Content */ nullptr, AK,
1751 OffsetInfoMap[CurPtr].Offsets, Changed,
1752 *LoadI->getType()))
1753 return false;
1754
1755 auto IsAssumption = [](Instruction &I) {
1756 if (auto *II = dyn_cast<IntrinsicInst>(&I))
1757 return II->isAssumeLikeIntrinsic();
1758 return false;
1759 };
1760
1761 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1762 // Check if the assumption and the load are executed together without
1763 // memory modification.
1764 do {
1765 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1766 return true;
1767 FromI = FromI->getNextNonDebugInstruction();
1768 } while (FromI && FromI != ToI);
1769 return false;
1770 };
1771
1772 BasicBlock *BB = LoadI->getParent();
1773 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1774 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1775 return false;
1776 BasicBlock *IntrBB = IntrI.getParent();
1777 if (IntrI.getParent() == BB) {
1778 if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(), &IntrI))
1779 return false;
1780 } else {
1781 auto PredIt = pred_begin(IntrBB);
1782 if (PredIt == pred_end(IntrBB))
1783 return false;
1784 if ((*PredIt) != BB)
1785 return false;
1786 if (++PredIt != pred_end(IntrBB))
1787 return false;
1788 for (auto *SuccBB : successors(BB)) {
1789 if (SuccBB == IntrBB)
1790 continue;
1791 if (isa<UnreachableInst>(SuccBB->getTerminator()))
1792 continue;
1793 return false;
1794 }
1795 if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(),
1796 BB->getTerminator()))
1797 return false;
1798 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1799 return false;
1800 }
1801 return true;
1802 };
1803
1804 std::pair<Value *, IntrinsicInst *> Assumption;
1805 for (const Use &LoadU : LoadI->uses()) {
1806 if (auto *CmpI = dyn_cast<CmpInst>(LoadU.getUser())) {
1807 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1808 continue;
1809 for (const Use &CmpU : CmpI->uses()) {
1810 if (auto *IntrI = dyn_cast<IntrinsicInst>(CmpU.getUser())) {
1811 if (!IsValidAssume(*IntrI))
1812 continue;
1813 int Idx = CmpI->getOperandUse(0) == LoadU;
1814 Assumption = {CmpI->getOperand(Idx), IntrI};
1815 break;
1816 }
1817 }
1818 }
1819 if (Assumption.first)
1820 break;
1821 }
1822
1823 // Check if we found an assumption associated with this load.
1824 if (!Assumption.first || !Assumption.second)
1825 return true;
1826
1827 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1828 << *Assumption.second << ": " << *LoadI
1829 << " == " << *Assumption.first << "\n");
1830 bool UsedAssumedInformation = false;
1831 std::optional<Value *> Content = nullptr;
1832 if (Assumption.first)
1833 Content =
1834 A.getAssumedSimplified(*Assumption.first, *this,
1835 UsedAssumedInformation, AA::Interprocedural);
1836 return handleAccess(
1837 A, *Assumption.second, Content, AccessKind::AK_ASSUMPTION,
1838 OffsetInfoMap[CurPtr].Offsets, Changed, *LoadI->getType());
1839 }
1840
1841 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1842 ArrayRef<Value *> OtherOps, AccessKind AK) {
1843 for (auto *OtherOp : OtherOps) {
1844 if (OtherOp == CurPtr) {
1845 LLVM_DEBUG(
1846 dbgs()
1847 << "[AAPointerInfo] Escaping use in store like instruction " << I
1848 << "\n");
1849 return false;
1850 }
1851 }
1852
1853 // If the access is to a pointer that may or may not be the associated
1854 // value, e.g. due to a PHI, we cannot assume it will be written.
1855 if (getUnderlyingObject(CurPtr) == &AssociatedValue)
1856 AK = AccessKind(AK | AccessKind::AK_MUST);
1857 else
1858 AK = AccessKind(AK | AccessKind::AK_MAY);
1859 bool UsedAssumedInformation = false;
1860 std::optional<Value *> Content = nullptr;
1861 if (ValueOp)
1862 Content = A.getAssumedSimplified(
1863 *ValueOp, *this, UsedAssumedInformation, AA::Interprocedural);
1864 return handleAccess(A, I, Content, AK, OffsetInfoMap[CurPtr].Offsets,
1865 Changed, ValueTy);
1866 };
1867
1868 if (auto *StoreI = dyn_cast<StoreInst>(Usr))
1869 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1870 *StoreI->getValueOperand()->getType(),
1871 {StoreI->getValueOperand()}, AccessKind::AK_W);
1872 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Usr))
1873 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1874 {RMWI->getValOperand()}, AccessKind::AK_RW);
1875 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Usr))
1876 return HandleStoreLike(
1877 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1878 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1879 AccessKind::AK_RW);
1880
1881 if (auto *CB = dyn_cast<CallBase>(Usr)) {
1882 if (CB->isLifetimeStartOrEnd())
1883 return true;
1884 const auto *TLI =
1885 A.getInfoCache().getTargetLibraryInfoForFunction(*CB->getFunction());
1886 if (getFreedOperand(CB, TLI) == U)
1887 return true;
1888 if (CB->isArgOperand(&U)) {
1889 unsigned ArgNo = CB->getArgOperandNo(&U);
1890 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1891 *this, IRPosition::callsite_argument(*CB, ArgNo),
1893 if (!CSArgPI)
1894 return false;
1895 Changed =
1896 translateAndAddState(A, *CSArgPI, OffsetInfoMap[CurPtr], *CB) |
1897 Changed;
1898 return isValidState();
1899 }
1900 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1901 << "\n");
1902 // TODO: Allow some call uses
1903 return false;
1904 }
1905
1906 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1907 return false;
1908 };
1909 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1910 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1911 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1912 if (OffsetInfoMap.count(NewU)) {
1913 LLVM_DEBUG({
1914 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1915 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1916 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1917 << "\n";
1918 }
1919 });
1920 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1921 }
1922 bool Unused;
1923 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1924 };
1925 if (!A.checkForAllUses(UsePred, *this, AssociatedValue,
1926 /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,
1927 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1928 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1929 return indicatePessimisticFixpoint();
1930 }
1931
1932 LLVM_DEBUG({
1933 dbgs() << "Accesses by bin after update:\n";
1934 dumpState(dbgs());
1935 });
1936
1937 return Changed;
1938}
1939
1940struct AAPointerInfoReturned final : AAPointerInfoImpl {
1941 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1942 : AAPointerInfoImpl(IRP, A) {}
1943
1944 /// See AbstractAttribute::updateImpl(...).
1945 ChangeStatus updateImpl(Attributor &A) override {
1946 return indicatePessimisticFixpoint();
1947 }
1948
1949 /// See AbstractAttribute::trackStatistics()
1950 void trackStatistics() const override {
1951 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1952 }
1953};
1954
1955struct AAPointerInfoArgument final : AAPointerInfoFloating {
1956 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1957 : AAPointerInfoFloating(IRP, A) {}
1958
1959 /// See AbstractAttribute::trackStatistics()
1960 void trackStatistics() const override {
1961 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1962 }
1963};
1964
1965struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1966 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1967 : AAPointerInfoFloating(IRP, A) {}
1968
1969 /// See AbstractAttribute::updateImpl(...).
1970 ChangeStatus updateImpl(Attributor &A) override {
1971 using namespace AA::PointerInfo;
1972 // We handle memory intrinsics explicitly, at least the first (=
1973 // destination) and second (=source) arguments as we know how they are
1974 // accessed.
1975 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(getCtxI())) {
1976 ConstantInt *Length = dyn_cast<ConstantInt>(MI->getLength());
1977 int64_t LengthVal = AA::RangeTy::Unknown;
1978 if (Length)
1979 LengthVal = Length->getSExtValue();
1980 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1981 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1982 if (ArgNo > 1) {
1983 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1984 << *MI << "\n");
1985 return indicatePessimisticFixpoint();
1986 } else {
1987 auto Kind =
1988 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1989 Changed =
1990 Changed | addAccess(A, {0, LengthVal}, *MI, nullptr, Kind, nullptr);
1991 }
1992 LLVM_DEBUG({
1993 dbgs() << "Accesses by bin after update:\n";
1994 dumpState(dbgs());
1995 });
1996
1997 return Changed;
1998 }
1999
2000 // TODO: Once we have call site specific value information we can provide
2001 // call site specific liveness information and then it makes
2002 // sense to specialize attributes for call sites arguments instead of
2003 // redirecting requests to the callee argument.
2004 Argument *Arg = getAssociatedArgument();
2005 if (Arg) {
2006 const IRPosition &ArgPos = IRPosition::argument(*Arg);
2007 auto *ArgAA =
2008 A.getAAFor<AAPointerInfo>(*this, ArgPos, DepClassTy::REQUIRED);
2009 if (ArgAA && ArgAA->getState().isValidState())
2010 return translateAndAddStateFromCallee(A, *ArgAA,
2011 *cast<CallBase>(getCtxI()));
2012 if (!Arg->getParent()->isDeclaration())
2013 return indicatePessimisticFixpoint();
2014 }
2015
2016 bool IsKnownNoCapture;
2017 if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
2018 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
2019 return indicatePessimisticFixpoint();
2020
2021 bool IsKnown = false;
2022 if (AA::isAssumedReadNone(A, getIRPosition(), *this, IsKnown))
2023 return ChangeStatus::UNCHANGED;
2024 bool ReadOnly = AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown);
2025 auto Kind =
2026 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2027 return addAccess(A, AA::RangeTy::getUnknown(), *getCtxI(), nullptr, Kind,
2028 nullptr);
2029 }
2030
2031 /// See AbstractAttribute::trackStatistics()
2032 void trackStatistics() const override {
2033 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2034 }
2035};
2036
2037struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2038 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2039 : AAPointerInfoFloating(IRP, A) {}
2040
2041 /// See AbstractAttribute::trackStatistics()
2042 void trackStatistics() const override {
2043 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2044 }
2045};
2046} // namespace
2047
2048/// -----------------------NoUnwind Function Attribute--------------------------
2049
2050namespace {
2051struct AANoUnwindImpl : AANoUnwind {
2052 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2053
2054 /// See AbstractAttribute::initialize(...).
2055 void initialize(Attributor &A) override {
2056 bool IsKnown;
2057 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2058 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2059 (void)IsKnown;
2060 }
2061
2062 const std::string getAsStr(Attributor *A) const override {
2063 return getAssumed() ? "nounwind" : "may-unwind";
2064 }
2065
2066 /// See AbstractAttribute::updateImpl(...).
2067 ChangeStatus updateImpl(Attributor &A) override {
2068 auto Opcodes = {
2069 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2070 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2071 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2072
2073 auto CheckForNoUnwind = [&](Instruction &I) {
2074 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2075 return true;
2076
2077 if (const auto *CB = dyn_cast<CallBase>(&I)) {
2078 bool IsKnownNoUnwind;
2079 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2080 A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,
2081 IsKnownNoUnwind);
2082 }
2083 return false;
2084 };
2085
2086 bool UsedAssumedInformation = false;
2087 if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes,
2088 UsedAssumedInformation))
2089 return indicatePessimisticFixpoint();
2090
2091 return ChangeStatus::UNCHANGED;
2092 }
2093};
2094
2095struct AANoUnwindFunction final : public AANoUnwindImpl {
2096 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2097 : AANoUnwindImpl(IRP, A) {}
2098
2099 /// See AbstractAttribute::trackStatistics()
2100 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2101};
2102
2103/// NoUnwind attribute deduction for a call sites.
2104struct AANoUnwindCallSite final
2105 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2106 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2107 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2108
2109 /// See AbstractAttribute::trackStatistics()
2110 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2111};
2112} // namespace
2113
2114/// ------------------------ NoSync Function Attribute -------------------------
2115
2116bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2117 switch (CB.getIntrinsicID()) {
2118 case Intrinsic::nvvm_barrier0:
2119 case Intrinsic::nvvm_barrier0_and:
2120 case Intrinsic::nvvm_barrier0_or:
2121 case Intrinsic::nvvm_barrier0_popc:
2122 return true;
2123 case Intrinsic::amdgcn_s_barrier:
2124 if (ExecutedAligned)
2125 return true;
2126 break;
2127 default:
2128 break;
2129 }
2130 return hasAssumption(CB, KnownAssumptionString("ompx_aligned_barrier"));
2131}
2132
2134 if (!I->isAtomic())
2135 return false;
2136
2137 if (auto *FI = dyn_cast<FenceInst>(I))
2138 // All legal orderings for fence are stronger than monotonic.
2139 return FI->getSyncScopeID() != SyncScope::SingleThread;
2140 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
2141 // Unordered is not a legal ordering for cmpxchg.
2142 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2143 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2144 }
2145
2146 AtomicOrdering Ordering;
2147 switch (I->getOpcode()) {
2148 case Instruction::AtomicRMW:
2149 Ordering = cast<AtomicRMWInst>(I)->getOrdering();
2150 break;
2151 case Instruction::Store:
2152 Ordering = cast<StoreInst>(I)->getOrdering();
2153 break;
2154 case Instruction::Load:
2155 Ordering = cast<LoadInst>(I)->getOrdering();
2156 break;
2157 default:
2159 "New atomic operations need to be known in the attributor.");
2160 }
2161
2162 return (Ordering != AtomicOrdering::Unordered &&
2163 Ordering != AtomicOrdering::Monotonic);
2164}
2165
2166/// Return true if this intrinsic is nosync. This is only used for intrinsics
2167/// which would be nosync except that they have a volatile flag. All other
2168/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.
2170 if (auto *MI = dyn_cast<MemIntrinsic>(I))
2171 return !MI->isVolatile();
2172 return false;
2173}
2174
2175namespace {
2176struct AANoSyncImpl : AANoSync {
2177 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2178
2179 /// See AbstractAttribute::initialize(...).
2180 void initialize(Attributor &A) override {
2181 bool IsKnown;
2182 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2183 DepClassTy::NONE, IsKnown));
2184 (void)IsKnown;
2185 }
2186
2187 const std::string getAsStr(Attributor *A) const override {
2188 return getAssumed() ? "nosync" : "may-sync";
2189 }
2190
2191 /// See AbstractAttribute::updateImpl(...).
2192 ChangeStatus updateImpl(Attributor &A) override;
2193};
2194
2195ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2196
2197 auto CheckRWInstForNoSync = [&](Instruction &I) {
2198 return AA::isNoSyncInst(A, I, *this);
2199 };
2200
2201 auto CheckForNoSync = [&](Instruction &I) {
2202 // At this point we handled all read/write effects and they are all
2203 // nosync, so they can be skipped.
2204 if (I.mayReadOrWriteMemory())
2205 return true;
2206
2207 bool IsKnown;
2208 CallBase &CB = cast<CallBase>(I);
2209 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2211 IsKnown))
2212 return true;
2213
2214 // non-convergent and readnone imply nosync.
2215 return !CB.isConvergent();
2216 };
2217
2218 bool UsedAssumedInformation = false;
2219 if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this,
2220 UsedAssumedInformation) ||
2221 !A.checkForAllCallLikeInstructions(CheckForNoSync, *this,
2222 UsedAssumedInformation))
2223 return indicatePessimisticFixpoint();
2224
2226}
2227
2228struct AANoSyncFunction final : public AANoSyncImpl {
2229 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2230 : AANoSyncImpl(IRP, A) {}
2231
2232 /// See AbstractAttribute::trackStatistics()
2233 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2234};
2235
2236/// NoSync attribute deduction for a call sites.
2237struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2238 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2239 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2240
2241 /// See AbstractAttribute::trackStatistics()
2242 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2243};
2244} // namespace
2245
2246/// ------------------------ No-Free Attributes ----------------------------
2247
2248namespace {
2249struct AANoFreeImpl : public AANoFree {
2250 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2251
2252 /// See AbstractAttribute::initialize(...).
2253 void initialize(Attributor &A) override {
2254 bool IsKnown;
2255 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2256 DepClassTy::NONE, IsKnown));
2257 (void)IsKnown;
2258 }
2259
2260 /// See AbstractAttribute::updateImpl(...).
2261 ChangeStatus updateImpl(Attributor &A) override {
2262 auto CheckForNoFree = [&](Instruction &I) {
2263 bool IsKnown;
2264 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2265 A, this, IRPosition::callsite_function(cast<CallBase>(I)),
2266 DepClassTy::REQUIRED, IsKnown);
2267 };
2268
2269 bool UsedAssumedInformation = false;
2270 if (!A.checkForAllCallLikeInstructions(CheckForNoFree, *this,
2271 UsedAssumedInformation))
2272 return indicatePessimisticFixpoint();
2273 return ChangeStatus::UNCHANGED;
2274 }
2275
2276 /// See AbstractAttribute::getAsStr().
2277 const std::string getAsStr(Attributor *A) const override {
2278 return getAssumed() ? "nofree" : "may-free";
2279 }
2280};
2281
2282struct AANoFreeFunction final : public AANoFreeImpl {
2283 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2284 : AANoFreeImpl(IRP, A) {}
2285
2286 /// See AbstractAttribute::trackStatistics()
2287 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2288};
2289
2290/// NoFree attribute deduction for a call sites.
2291struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2292 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2293 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2294
2295 /// See AbstractAttribute::trackStatistics()
2296 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2297};
2298
2299/// NoFree attribute for floating values.
2300struct AANoFreeFloating : AANoFreeImpl {
2301 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2302 : AANoFreeImpl(IRP, A) {}
2303
2304 /// See AbstractAttribute::trackStatistics()
2305 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2306
2307 /// See Abstract Attribute::updateImpl(...).
2308 ChangeStatus updateImpl(Attributor &A) override {
2309 const IRPosition &IRP = getIRPosition();
2310
2311 bool IsKnown;
2312 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this,
2314 DepClassTy::OPTIONAL, IsKnown))
2315 return ChangeStatus::UNCHANGED;
2316
2317 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2318 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2319 Instruction *UserI = cast<Instruction>(U.getUser());
2320 if (auto *CB = dyn_cast<CallBase>(UserI)) {
2321 if (CB->isBundleOperand(&U))
2322 return false;
2323 if (!CB->isArgOperand(&U))
2324 return true;
2325 unsigned ArgNo = CB->getArgOperandNo(&U);
2326
2327 bool IsKnown;
2328 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2329 A, this, IRPosition::callsite_argument(*CB, ArgNo),
2330 DepClassTy::REQUIRED, IsKnown);
2331 }
2332
2333 if (isa<GetElementPtrInst>(UserI) || isa<PHINode>(UserI) ||
2334 isa<SelectInst>(UserI)) {
2335 Follow = true;
2336 return true;
2337 }
2338 if (isa<StoreInst>(UserI) || isa<LoadInst>(UserI) ||
2339 isa<ReturnInst>(UserI))
2340 return true;
2341
2342 // Unknown user.
2343 return false;
2344 };
2345 if (!A.checkForAllUses(Pred, *this, AssociatedValue))
2346 return indicatePessimisticFixpoint();
2347
2348 return ChangeStatus::UNCHANGED;
2349 }
2350};
2351
2352/// NoFree attribute for a call site argument.
2353struct AANoFreeArgument final : AANoFreeFloating {
2354 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2355 : AANoFreeFloating(IRP, A) {}
2356
2357 /// See AbstractAttribute::trackStatistics()
2358 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2359};
2360
2361/// NoFree attribute for call site arguments.
2362struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2363 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2364 : AANoFreeFloating(IRP, A) {}
2365
2366 /// See AbstractAttribute::updateImpl(...).
2367 ChangeStatus updateImpl(Attributor &A) override {
2368 // TODO: Once we have call site specific value information we can provide
2369 // call site specific liveness information and then it makes
2370 // sense to specialize attributes for call sites arguments instead of
2371 // redirecting requests to the callee argument.
2372 Argument *Arg = getAssociatedArgument();
2373 if (!Arg)
2374 return indicatePessimisticFixpoint();
2375 const IRPosition &ArgPos = IRPosition::argument(*Arg);
2376 bool IsKnown;
2377 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, ArgPos,
2378 DepClassTy::REQUIRED, IsKnown))
2379 return ChangeStatus::UNCHANGED;
2380 return indicatePessimisticFixpoint();
2381 }
2382
2383 /// See AbstractAttribute::trackStatistics()
2384 void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nofree)};
2385};
2386
2387/// NoFree attribute for function return value.
2388struct AANoFreeReturned final : AANoFreeFloating {
2389 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2390 : AANoFreeFloating(IRP, A) {
2391 llvm_unreachable("NoFree is not applicable to function returns!");
2392 }
2393
2394 /// See AbstractAttribute::initialize(...).
2395 void initialize(Attributor &A) override {
2396 llvm_unreachable("NoFree is not applicable to function returns!");
2397 }
2398
2399 /// See AbstractAttribute::updateImpl(...).
2400 ChangeStatus updateImpl(Attributor &A) override {
2401 llvm_unreachable("NoFree is not applicable to function returns!");
2402 }
2403
2404 /// See AbstractAttribute::trackStatistics()
2405 void trackStatistics() const override {}
2406};
2407
2408/// NoFree attribute deduction for a call site return value.
2409struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2410 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2411 : AANoFreeFloating(IRP, A) {}
2412
2413 ChangeStatus manifest(Attributor &A) override {
2414 return ChangeStatus::UNCHANGED;
2415 }
2416 /// See AbstractAttribute::trackStatistics()
2417 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2418};
2419} // namespace
2420
2421/// ------------------------ NonNull Argument Attribute ------------------------
2422
2424 Attribute::AttrKind ImpliedAttributeKind,
2425 bool IgnoreSubsumingPositions) {
2427 AttrKinds.push_back(Attribute::NonNull);
2430 AttrKinds.push_back(Attribute::Dereferenceable);
2431 if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))
2432 return true;
2433
2434 DominatorTree *DT = nullptr;
2435 AssumptionCache *AC = nullptr;
2436 InformationCache &InfoCache = A.getInfoCache();
2437 if (const Function *Fn = IRP.getAnchorScope()) {
2438 if (!Fn->isDeclaration()) {
2441 }
2442 }
2443
2445 if (IRP.getPositionKind() != IRP_RETURNED) {
2446 Worklist.push_back({IRP.getAssociatedValue(), IRP.getCtxI()});
2447 } else {
2448 bool UsedAssumedInformation = false;
2449 if (!A.checkForAllInstructions(
2450 [&](Instruction &I) {
2451 Worklist.push_back({*cast<ReturnInst>(I).getReturnValue(), &I});
2452 return true;
2453 },
2454 IRP.getAssociatedFunction(), nullptr, {Instruction::Ret},
2455 UsedAssumedInformation, false, /*CheckPotentiallyDead=*/true))
2456 return false;
2457 }
2458
2459 if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {
2460 return !isKnownNonZero(
2461 VAC.getValue(),
2462 SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2463 }))
2464 return false;
2465
2466 A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(),
2467 Attribute::NonNull)});
2468 return true;
2469}
2470
2471namespace {
2472static int64_t getKnownNonNullAndDerefBytesForUse(
2473 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2474 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2475 TrackUse = false;
2476
2477 const Value *UseV = U->get();
2478 if (!UseV->getType()->isPointerTy())
2479 return 0;
2480
2481 // We need to follow common pointer manipulation uses to the accesses they
2482 // feed into. We can try to be smart to avoid looking through things we do not
2483 // like for now, e.g., non-inbounds GEPs.
2484 if (isa<CastInst>(I)) {
2485 TrackUse = true;
2486 return 0;
2487 }
2488
2489 if (isa<GetElementPtrInst>(I)) {
2490 TrackUse = true;
2491 return 0;
2492 }
2493
2494 Type *PtrTy = UseV->getType();
2495 const Function *F = I->getFunction();
2498 const DataLayout &DL = A.getInfoCache().getDL();
2499 if (const auto *CB = dyn_cast<CallBase>(I)) {
2500 if (CB->isBundleOperand(U)) {
2502 U, {Attribute::NonNull, Attribute::Dereferenceable})) {
2503 IsNonNull |=
2504 (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2505 return RK.ArgValue;
2506 }
2507 return 0;
2508 }
2509
2510 if (CB->isCallee(U)) {
2511 IsNonNull |= !NullPointerIsDefined;
2512 return 0;
2513 }
2514
2515 unsigned ArgNo = CB->getArgOperandNo(U);
2516 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
2517 // As long as we only use known information there is no need to track
2518 // dependences here.
2519 bool IsKnownNonNull;
2520 AA::hasAssumedIRAttr<Attribute::NonNull>(A, &QueryingAA, IRP,
2521 DepClassTy::NONE, IsKnownNonNull);
2522 IsNonNull |= IsKnownNonNull;
2523 auto *DerefAA =
2524 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClassTy::NONE);
2525 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2526 }
2527
2528 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
2529 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2530 Loc->Size.isScalable() || I->isVolatile())
2531 return 0;
2532
2533 int64_t Offset;
2534 const Value *Base =
2535 getMinimalBaseOfPointer(A, QueryingAA, Loc->Ptr, Offset, DL);
2536 if (Base && Base == &AssociatedValue) {
2537 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2538 IsNonNull |= !NullPointerIsDefined;
2539 return std::max(int64_t(0), DerefBytes);
2540 }
2541
2542 /// Corner case when an offset is 0.
2544 /*AllowNonInbounds*/ true);
2545 if (Base && Base == &AssociatedValue && Offset == 0) {
2546 int64_t DerefBytes = Loc->Size.getValue();
2547 IsNonNull |= !NullPointerIsDefined;
2548 return std::max(int64_t(0), DerefBytes);
2549 }
2550
2551 return 0;
2552}
2553
2554struct AANonNullImpl : AANonNull {
2555 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2556
2557 /// See AbstractAttribute::initialize(...).
2558 void initialize(Attributor &A) override {
2559 Value &V = *getAssociatedValue().stripPointerCasts();
2560 if (isa<ConstantPointerNull>(V)) {
2561 indicatePessimisticFixpoint();
2562 return;
2563 }
2564
2565 if (Instruction *CtxI = getCtxI())
2566 followUsesInMBEC(*this, A, getState(), *CtxI);
2567 }
2568
2569 /// See followUsesInMBEC
2570 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2571 AANonNull::StateType &State) {
2572 bool IsNonNull = false;
2573 bool TrackUse = false;
2574 getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I,
2575 IsNonNull, TrackUse);
2576 State.setKnown(IsNonNull);
2577 return TrackUse;
2578 }
2579
2580 /// See AbstractAttribute::getAsStr().
2581 const std::string getAsStr(Attributor *A) const override {
2582 return getAssumed() ? "nonnull" : "may-null";
2583 }
2584};
2585
2586/// NonNull attribute for a floating value.
2587struct AANonNullFloating : public AANonNullImpl {
2588 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2589 : AANonNullImpl(IRP, A) {}
2590
2591 /// See AbstractAttribute::updateImpl(...).
2592 ChangeStatus updateImpl(Attributor &A) override {
2593 auto CheckIRP = [&](const IRPosition &IRP) {
2594 bool IsKnownNonNull;
2595 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2596 A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);
2597 };
2598
2599 bool Stripped;
2600 bool UsedAssumedInformation = false;
2601 Value *AssociatedValue = &getAssociatedValue();
2603 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
2604 AA::AnyScope, UsedAssumedInformation))
2605 Stripped = false;
2606 else
2607 Stripped =
2608 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2609
2610 if (!Stripped) {
2611 bool IsKnown;
2612 if (auto *PHI = dyn_cast<PHINode>(AssociatedValue))
2613 if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) {
2614 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2615 A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,
2616 IsKnown);
2617 }))
2618 return ChangeStatus::UNCHANGED;
2619 if (auto *Select = dyn_cast<SelectInst>(AssociatedValue))
2620 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2621 A, this, IRPosition::value(*Select->getFalseValue()),
2622 DepClassTy::OPTIONAL, IsKnown) &&
2623 AA::hasAssumedIRAttr<Attribute::NonNull>(
2624 A, this, IRPosition::value(*Select->getTrueValue()),
2625 DepClassTy::OPTIONAL, IsKnown))
2626 return ChangeStatus::UNCHANGED;
2627
2628 // If we haven't stripped anything we might still be able to use a
2629 // different AA, but only if the IRP changes. Effectively when we
2630 // interpret this not as a call site value but as a floating/argument
2631 // value.
2632 const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
2633 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2634 return indicatePessimisticFixpoint();
2635 return ChangeStatus::UNCHANGED;
2636 }
2637
2638 for (const auto &VAC : Values)
2639 if (!CheckIRP(IRPosition::value(*VAC.getValue())))
2640 return indicatePessimisticFixpoint();
2641
2642 return ChangeStatus::UNCHANGED;
2643 }
2644
2645 /// See AbstractAttribute::trackStatistics()
2646 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2647};
2648
2649/// NonNull attribute for function return value.
2650struct AANonNullReturned final
2651 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2652 false, AANonNull::IRAttributeKind, false> {
2653 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2654 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2655 false, Attribute::NonNull, false>(IRP, A) {
2656 }
2657
2658 /// See AbstractAttribute::getAsStr().
2659 const std::string getAsStr(Attributor *A) const override {
2660 return getAssumed() ? "nonnull" : "may-null";
2661 }
2662
2663 /// See AbstractAttribute::trackStatistics()
2664 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2665};
2666
2667/// NonNull attribute for function argument.
2668struct AANonNullArgument final
2669 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2670 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2671 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2672
2673 /// See AbstractAttribute::trackStatistics()
2674 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2675};
2676
2677struct AANonNullCallSiteArgument final : AANonNullFloating {
2678 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2679 : AANonNullFloating(IRP, A) {}
2680
2681 /// See AbstractAttribute::trackStatistics()
2682 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2683};
2684
2685/// NonNull attribute for a call site return position.
2686struct AANonNullCallSiteReturned final
2687 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2688 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2689 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2690
2691 /// See AbstractAttribute::trackStatistics()
2692 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2693};
2694} // namespace
2695
2696/// ------------------------ Must-Progress Attributes --------------------------
2697namespace {
2698struct AAMustProgressImpl : public AAMustProgress {
2699 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2700 : AAMustProgress(IRP, A) {}
2701
2702 /// See AbstractAttribute::initialize(...).
2703 void initialize(Attributor &A) override {
2704 bool IsKnown;
2705 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2706 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2707 (void)IsKnown;
2708 }
2709
2710 /// See AbstractAttribute::getAsStr()
2711 const std::string getAsStr(Attributor *A) const override {
2712 return getAssumed() ? "mustprogress" : "may-not-progress";
2713 }
2714};
2715
2716struct AAMustProgressFunction final : AAMustProgressImpl {
2717 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2718 : AAMustProgressImpl(IRP, A) {}
2719
2720 /// See AbstractAttribute::updateImpl(...).
2721 ChangeStatus updateImpl(Attributor &A) override {
2722 bool IsKnown;
2723 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2724 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {
2725 if (IsKnown)
2726 return indicateOptimisticFixpoint();
2727 return ChangeStatus::UNCHANGED;
2728 }
2729
2730 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2731 IRPosition IPos = IRPosition::callsite_function(*ACS.getInstruction());
2732 bool IsKnownMustProgress;
2733 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2734 A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,
2735 /* IgnoreSubsumingPositions */ true);
2736 };
2737
2738 bool AllCallSitesKnown = true;
2739 if (!A.checkForAllCallSites(CheckForMustProgress, *this,
2740 /* RequireAllCallSites */ true,
2741 AllCallSitesKnown))
2742 return indicatePessimisticFixpoint();
2743
2744 return ChangeStatus::UNCHANGED;
2745 }
2746
2747 /// See AbstractAttribute::trackStatistics()
2748 void trackStatistics() const override {
2749 STATS_DECLTRACK_FN_ATTR(mustprogress)
2750 }
2751};
2752
2753/// MustProgress attribute deduction for a call sites.
2754struct AAMustProgressCallSite final : AAMustProgressImpl {
2755 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2756 : AAMustProgressImpl(IRP, A) {}
2757
2758 /// See AbstractAttribute::updateImpl(...).
2759 ChangeStatus updateImpl(Attributor &A) override {
2760 // TODO: Once we have call site specific value information we can provide
2761 // call site specific liveness information and then it makes
2762 // sense to specialize attributes for call sites arguments instead of
2763 // redirecting requests to the callee argument.
2764 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
2765 bool IsKnownMustProgress;
2766 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2767 A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))
2768 return indicatePessimisticFixpoint();
2769 return ChangeStatus::UNCHANGED;
2770 }
2771
2772 /// See AbstractAttribute::trackStatistics()
2773 void trackStatistics() const override {
2774 STATS_DECLTRACK_CS_ATTR(mustprogress);
2775 }
2776};
2777} // namespace
2778
2779/// ------------------------ No-Recurse Attributes ----------------------------
2780
2781namespace {
2782struct AANoRecurseImpl : public AANoRecurse {
2783 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2784
2785 /// See AbstractAttribute::initialize(...).
2786 void initialize(Attributor &A) override {
2787 bool IsKnown;
2788 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2789 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2790 (void)IsKnown;
2791 }
2792
2793 /// See AbstractAttribute::getAsStr()
2794 const std::string getAsStr(Attributor *A) const override {
2795 return getAssumed() ? "norecurse" : "may-recurse";
2796 }
2797};
2798
2799struct AANoRecurseFunction final : AANoRecurseImpl {
2800 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2801 : AANoRecurseImpl(IRP, A) {}
2802
2803 /// See AbstractAttribute::updateImpl(...).
2804 ChangeStatus updateImpl(Attributor &A) override {
2805
2806 // If all live call sites are known to be no-recurse, we are as well.
2807 auto CallSitePred = [&](AbstractCallSite ACS) {
2808 bool IsKnownNoRecurse;
2809 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2810 A, this,
2811 IRPosition::function(*ACS.getInstruction()->getFunction()),
2812 DepClassTy::NONE, IsKnownNoRecurse))
2813 return false;
2814 return IsKnownNoRecurse;
2815 };
2816 bool UsedAssumedInformation = false;
2817 if (A.checkForAllCallSites(CallSitePred, *this, true,
2818 UsedAssumedInformation)) {
2819 // If we know all call sites and all are known no-recurse, we are done.
2820 // If all known call sites, which might not be all that exist, are known
2821 // to be no-recurse, we are not done but we can continue to assume
2822 // no-recurse. If one of the call sites we have not visited will become
2823 // live, another update is triggered.
2824 if (!UsedAssumedInformation)
2825 indicateOptimisticFixpoint();
2826 return ChangeStatus::UNCHANGED;
2827 }
2828
2829 const AAInterFnReachability *EdgeReachability =
2830 A.getAAFor<AAInterFnReachability>(*this, getIRPosition(),
2831 DepClassTy::REQUIRED);
2832 if (EdgeReachability && EdgeReachability->canReach(A, *getAnchorScope()))
2833 return indicatePessimisticFixpoint();
2834 return ChangeStatus::UNCHANGED;
2835 }
2836
2837 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2838};
2839
2840/// NoRecurse attribute deduction for a call sites.
2841struct AANoRecurseCallSite final
2842 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2843 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2844 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2845
2846 /// See AbstractAttribute::trackStatistics()
2847 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2848};
2849} // namespace
2850
2851/// ------------------------ No-Convergent Attribute --------------------------
2852
2853namespace {
2854struct AANonConvergentImpl : public AANonConvergent {
2855 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2856 : AANonConvergent(IRP, A) {}
2857
2858 /// See AbstractAttribute::getAsStr()
2859 const std::string getAsStr(Attributor *A) const override {
2860 return getAssumed() ? "non-convergent" : "may-be-convergent";
2861 }
2862};
2863
2864struct AANonConvergentFunction final : AANonConvergentImpl {
2865 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2866 : AANonConvergentImpl(IRP, A) {}
2867
2868 /// See AbstractAttribute::updateImpl(...).
2869 ChangeStatus updateImpl(Attributor &A) override {
2870 // If all function calls are known to not be convergent, we are not
2871 // convergent.
2872 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2873 CallBase &CB = cast<CallBase>(Inst);
2874 auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
2875 if (!Callee || Callee->isIntrinsic()) {
2876 return false;
2877 }
2878 if (Callee->isDeclaration()) {
2879 return !Callee->hasFnAttribute(Attribute::Convergent);
2880 }
2881 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2882 *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);
2883 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2884 };
2885
2886 bool UsedAssumedInformation = false;
2887 if (!A.checkForAllCallLikeInstructions(CalleeIsNotConvergent, *this,
2888 UsedAssumedInformation)) {
2889 return indicatePessimisticFixpoint();
2890 }
2891 return ChangeStatus::UNCHANGED;
2892 }
2893
2894 ChangeStatus manifest(Attributor &A) override {
2895 if (isKnownNotConvergent() &&
2896 A.hasAttr(getIRPosition(), Attribute::Convergent)) {
2897 A.removeAttrs(getIRPosition(), {Attribute::Convergent});
2898 return ChangeStatus::CHANGED;
2899 }
2900 return ChangeStatus::UNCHANGED;
2901 }
2902
2903 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2904};
2905} // namespace
2906
2907/// -------------------- Undefined-Behavior Attributes ------------------------
2908
2909namespace {
2910struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2911 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2912 : AAUndefinedBehavior(IRP, A) {}
2913
2914 /// See AbstractAttribute::updateImpl(...).
2915 // through a pointer (i.e. also branches etc.)
2916 ChangeStatus updateImpl(Attributor &A) override {
2917 const size_t UBPrevSize = KnownUBInsts.size();
2918 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2919
2920 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2921 // Lang ref now states volatile store is not UB, let's skip them.
2922 if (I.isVolatile() && I.mayWriteToMemory())
2923 return true;
2924
2925 // Skip instructions that are already saved.
2926 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2927 return true;
2928
2929 // If we reach here, we know we have an instruction
2930 // that accesses memory through a pointer operand,
2931 // for which getPointerOperand() should give it to us.
2932 Value *PtrOp =
2933 const_cast<Value *>(getPointerOperand(&I, /* AllowVolatile */ true));
2934 assert(PtrOp &&
2935 "Expected pointer operand of memory accessing instruction");
2936
2937 // Either we stopped and the appropriate action was taken,
2938 // or we got back a simplified value to continue.
2939 std::optional<Value *> SimplifiedPtrOp =
2940 stopOnUndefOrAssumed(A, PtrOp, &I);
2941 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2942 return true;
2943 const Value *PtrOpVal = *SimplifiedPtrOp;
2944
2945 // A memory access through a pointer is considered UB
2946 // only if the pointer has constant null value.
2947 // TODO: Expand it to not only check constant values.
2948 if (!isa<ConstantPointerNull>(PtrOpVal)) {
2949 AssumedNoUBInsts.insert(&I);
2950 return true;
2951 }
2952 const Type *PtrTy = PtrOpVal->getType();
2953
2954 // Because we only consider instructions inside functions,
2955 // assume that a parent function exists.
2956 const Function *F = I.getFunction();
2957
2958 // A memory access using constant null pointer is only considered UB
2959 // if null pointer is _not_ defined for the target platform.
2961 AssumedNoUBInsts.insert(&I);
2962 else
2963 KnownUBInsts.insert(&I);
2964 return true;
2965 };
2966
2967 auto InspectBrInstForUB = [&](Instruction &I) {
2968 // A conditional branch instruction is considered UB if it has `undef`
2969 // condition.
2970
2971 // Skip instructions that are already saved.
2972 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2973 return true;
2974
2975 // We know we have a branch instruction.
2976 auto *BrInst = cast<BranchInst>(&I);
2977
2978 // Unconditional branches are never considered UB.
2979 if (BrInst->isUnconditional())
2980 return true;
2981
2982 // Either we stopped and the appropriate action was taken,
2983 // or we got back a simplified value to continue.
2984 std::optional<Value *> SimplifiedCond =
2985 stopOnUndefOrAssumed(A, BrInst->getCondition(), BrInst);
2986 if (!SimplifiedCond || !*SimplifiedCond)
2987 return true;
2988 AssumedNoUBInsts.insert(&I);
2989 return true;
2990 };
2991
2992 auto InspectCallSiteForUB = [&](Instruction &I) {
2993 // Check whether a callsite always cause UB or not
2994
2995 // Skip instructions that are already saved.
2996 if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
2997 return true;
2998
2999 // Check nonnull and noundef argument attribute violation for each
3000 // callsite.
3001 CallBase &CB = cast<CallBase>(I);
3002 auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
3003 if (!Callee)
3004 return true;
3005 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3006 // If current argument is known to be simplified to null pointer and the
3007 // corresponding argument position is known to have nonnull attribute,
3008 // the argument is poison. Furthermore, if the argument is poison and
3009 // the position is known to have noundef attriubte, this callsite is
3010 // considered UB.
3011 if (idx >= Callee->arg_size())
3012 break;
3013 Value *ArgVal = CB.getArgOperand(idx);
3014 if (!ArgVal)
3015 continue;
3016 // Here, we handle three cases.
3017 // (1) Not having a value means it is dead. (we can replace the value
3018 // with undef)
3019 // (2) Simplified to undef. The argument violate noundef attriubte.
3020 // (3) Simplified to null pointer where known to be nonnull.
3021 // The argument is a poison value and violate noundef attribute.
3022 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, idx);
3023 bool IsKnownNoUndef;
3024 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3025 A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);
3026 if (!IsKnownNoUndef)
3027 continue;
3028 bool UsedAssumedInformation = false;
3029 std::optional<Value *> SimplifiedVal =
3030 A.getAssumedSimplified(IRPosition::value(*ArgVal), *this,
3031 UsedAssumedInformation, AA::Interprocedural);
3032 if (UsedAssumedInformation)
3033 continue;
3034 if (SimplifiedVal && !*SimplifiedVal)
3035 return true;
3036 if (!SimplifiedVal || isa<UndefValue>(**SimplifiedVal)) {
3037 KnownUBInsts.insert(&I);
3038 continue;
3039 }
3040 if (!ArgVal->getType()->isPointerTy() ||
3041 !isa<ConstantPointerNull>(**SimplifiedVal))
3042 continue;
3043 bool IsKnownNonNull;
3044 AA::hasAssumedIRAttr<Attribute::NonNull>(
3045 A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);
3046 if (IsKnownNonNull)
3047 KnownUBInsts.insert(&I);
3048 }
3049 return true;
3050 };
3051
3052 auto InspectReturnInstForUB = [&](Instruction &I) {
3053 auto &RI = cast<ReturnInst>(I);
3054 // Either we stopped and the appropriate action was taken,
3055 // or we got back a simplified return value to continue.
3056 std::optional<Value *> SimplifiedRetValue =
3057 stopOnUndefOrAssumed(A, RI.getReturnValue(), &I);
3058 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3059 return true;
3060
3061 // Check if a return instruction always cause UB or not
3062 // Note: It is guaranteed that the returned position of the anchor
3063 // scope has noundef attribute when this is called.
3064 // We also ensure the return position is not "assumed dead"
3065 // because the returned value was then potentially simplified to
3066 // `undef` in AAReturnedValues without removing the `noundef`
3067 // attribute yet.
3068
3069 // When the returned position has noundef attriubte, UB occurs in the
3070 // following cases.
3071 // (1) Returned value is known to be undef.
3072 // (2) The value is known to be a null pointer and the returned
3073 // position has nonnull attribute (because the returned value is
3074 // poison).
3075 if (isa<ConstantPointerNull>(*SimplifiedRetValue)) {
3076 bool IsKnownNonNull;
3077 AA::hasAssumedIRAttr<Attribute::NonNull>(
3078 A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE,
3079 IsKnownNonNull);
3080 if (IsKnownNonNull)
3081 KnownUBInsts.insert(&I);
3082 }
3083
3084 return true;
3085 };
3086
3087 bool UsedAssumedInformation = false;
3088 A.checkForAllInstructions(InspectMemAccessInstForUB, *this,
3089 {Instruction::Load, Instruction::Store,
3090 Instruction::AtomicCmpXchg,
3091 Instruction::AtomicRMW},
3092 UsedAssumedInformation,
3093 /* CheckBBLivenessOnly */ true);
3094 A.checkForAllInstructions(InspectBrInstForUB, *this, {Instruction::Br},
3095 UsedAssumedInformation,
3096 /* CheckBBLivenessOnly */ true);
3097 A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *this,
3098 UsedAssumedInformation);
3099
3100 // If the returned position of the anchor scope has noundef attriubte, check
3101 // all returned instructions.
3102 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3103 const IRPosition &ReturnIRP = IRPosition::returned(*getAnchorScope());
3104 if (!A.isAssumedDead(ReturnIRP, this, nullptr, UsedAssumedInformation)) {
3105 bool IsKnownNoUndef;
3106 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3107 A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);
3108 if (IsKnownNoUndef)
3109 A.checkForAllInstructions(InspectReturnInstForUB, *this,
3110 {Instruction::Ret}, UsedAssumedInformation,
3111 /* CheckBBLivenessOnly */ true);
3112 }
3113 }
3114
3115 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3116 UBPrevSize != KnownUBInsts.size())
3117 return ChangeStatus::CHANGED;
3118 return ChangeStatus::UNCHANGED;
3119 }
3120
3121 bool isKnownToCauseUB(Instruction *I) const override {
3122 return KnownUBInsts.count(I);
3123 }
3124
3125 bool isAssumedToCauseUB(Instruction *I) const override {
3126 // In simple words, if an instruction is not in the assumed to _not_
3127 // cause UB, then it is assumed UB (that includes those
3128 // in the KnownUBInsts set). The rest is boilerplate
3129 // is to ensure that it is one of the instructions we test
3130 // for UB.
3131
3132 switch (I->getOpcode()) {
3133 case Instruction::Load:
3134 case Instruction::Store:
3135 case Instruction::AtomicCmpXchg:
3136 case Instruction::AtomicRMW:
3137 return !AssumedNoUBInsts.count(I);
3138 case Instruction::Br: {
3139 auto *BrInst = cast<BranchInst>(I);
3140 if (BrInst->isUnconditional())
3141 return false;
3142 return !AssumedNoUBInsts.count(I);
3143 } break;
3144 default:
3145 return false;
3146 }
3147 return false;
3148 }
3149
3150 ChangeStatus manifest(Attributor &A) override {
3151 if (KnownUBInsts.empty())
3152 return ChangeStatus::UNCHANGED;
3153 for (Instruction *I : KnownUBInsts)
3154 A.changeToUnreachableAfterManifest(I);
3155 return ChangeStatus::CHANGED;
3156 }
3157
3158 /// See AbstractAttribute::getAsStr()
3159 const std::string getAsStr(Attributor *A) const override {
3160 return getAssumed() ? "undefined-behavior" : "no-ub";
3161 }
3162
3163 /// Note: The correctness of this analysis depends on the fact that the
3164 /// following 2 sets will stop changing after some point.
3165 /// "Change" here means that their size changes.
3166 /// The size of each set is monotonically increasing
3167 /// (we only add items to them) and it is upper bounded by the number of
3168 /// instructions in the processed function (we can never save more
3169 /// elements in either set than this number). Hence, at some point,
3170 /// they will stop increasing.
3171 /// Consequently, at some point, both sets will have stopped
3172 /// changing, effectively making the analysis reach a fixpoint.
3173
3174 /// Note: These 2 sets are disjoint and an instruction can be considered
3175 /// one of 3 things:
3176 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3177 /// the KnownUBInsts set.
3178 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3179 /// has a reason to assume it).
3180 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3181 /// could not find a reason to assume or prove that it can cause UB,
3182 /// hence it assumes it doesn't. We have a set for these instructions
3183 /// so that we don't reprocess them in every update.
3184 /// Note however that instructions in this set may cause UB.
3185
3186protected:
3187 /// A set of all live instructions _known_ to cause UB.
3188 SmallPtrSet<Instruction *, 8> KnownUBInsts;
3189
3190private:
3191 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3192 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3193
3194 // Should be called on updates in which if we're processing an instruction
3195 // \p I that depends on a value \p V, one of the following has to happen:
3196 // - If the value is assumed, then stop.
3197 // - If the value is known but undef, then consider it UB.
3198 // - Otherwise, do specific processing with the simplified value.
3199 // We return std::nullopt in the first 2 cases to signify that an appropriate
3200 // action was taken and the caller should stop.
3201 // Otherwise, we return the simplified value that the caller should
3202 // use for specific processing.
3203 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3204 Instruction *I) {
3205 bool UsedAssumedInformation = false;
3206 std::optional<Value *> SimplifiedV =
3207 A.getAssumedSimplified(IRPosition::value(*V), *this,
3208 UsedAssumedInformation, AA::Interprocedural);
3209 if (!UsedAssumedInformation) {
3210 // Don't depend on assumed values.
3211 if (!SimplifiedV) {
3212 // If it is known (which we tested above) but it doesn't have a value,
3213 // then we can assume `undef` and hence the instruction is UB.
3214 KnownUBInsts.insert(I);
3215 return std::nullopt;
3216 }
3217 if (!*SimplifiedV)
3218 return nullptr;
3219 V = *SimplifiedV;
3220 }
3221 if (isa<UndefValue>(V)) {
3222 KnownUBInsts.insert(I);
3223 return std::nullopt;
3224 }
3225 return V;
3226 }
3227};
3228
3229struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3230 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3231 : AAUndefinedBehaviorImpl(IRP, A) {}
3232
3233 /// See AbstractAttribute::trackStatistics()
3234 void trackStatistics() const override {
3235 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3236 "Number of instructions known to have UB");
3237 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3238 KnownUBInsts.size();
3239 }
3240};
3241} // namespace
3242
3243/// ------------------------ Will-Return Attributes ----------------------------
3244
3245namespace {
3246// Helper function that checks whether a function has any cycle which we don't
3247// know if it is bounded or not.
3248// Loops with maximum trip count are considered bounded, any other cycle not.
3249static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3250 ScalarEvolution *SE =
3251 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3252 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3253 // If either SCEV or LoopInfo is not available for the function then we assume
3254 // any cycle to be unbounded cycle.
3255 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3256 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3257 if (!SE || !LI) {
3258 for (scc_iterator<Function *> SCCI = scc_begin(&F); !SCCI.isAtEnd(); ++SCCI)
3259 if (SCCI.hasCycle())
3260 return true;
3261 return false;
3262 }
3263
3264 // If there's irreducible control, the function may contain non-loop cycles.
3266 return true;
3267
3268 // Any loop that does not have a max trip count is considered unbounded cycle.
3269 for (auto *L : LI->getLoopsInPreorder()) {
3270 if (!SE->getSmallConstantMaxTripCount(L))
3271 return true;
3272 }
3273 return false;
3274}
3275
3276struct AAWillReturnImpl : public AAWillReturn {
3277 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3278 : AAWillReturn(IRP, A) {}
3279
3280 /// See AbstractAttribute::initialize(...).
3281 void initialize(Attributor &A) override {
3282 bool IsKnown;
3283 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3284 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3285 (void)IsKnown;
3286 }
3287
3288 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3289 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3290 if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress}))
3291 return false;
3292
3293 bool IsKnown;
3294 if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
3295 return IsKnown || !KnownOnly;
3296 return false;
3297 }
3298
3299 /// See AbstractAttribute::updateImpl(...).
3300 ChangeStatus updateImpl(Attributor &A) override {
3301 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3302 return ChangeStatus::UNCHANGED;
3303
3304 auto CheckForWillReturn = [&](Instruction &I) {
3305 IRPosition IPos = IRPosition::callsite_function(cast<CallBase>(I));
3306 bool IsKnown;
3307 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3308 A, this, IPos, DepClassTy::REQUIRED, IsKnown)) {
3309 if (IsKnown)
3310 return true;
3311 } else {
3312 return false;
3313 }
3314 bool IsKnownNoRecurse;
3315 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3316 A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);
3317 };
3318
3319 bool UsedAssumedInformation = false;
3320 if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this,
3321 UsedAssumedInformation))
3322 return indicatePessimisticFixpoint();
3323
3324 return ChangeStatus::UNCHANGED;
3325 }
3326
3327 /// See AbstractAttribute::getAsStr()
3328 const std::string getAsStr(Attributor *A) const override {
3329 return getAssumed() ? "willreturn" : "may-noreturn";
3330 }
3331};
3332
3333struct AAWillReturnFunction final : AAWillReturnImpl {
3334 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3335 : AAWillReturnImpl(IRP, A) {}
3336
3337 /// See AbstractAttribute::initialize(...).
3338 void initialize(Attributor &A) override {
3339 AAWillReturnImpl::initialize(A);
3340
3341 Function *F = getAnchorScope();
3342 assert(F && "Did expect an anchor function");
3343 if (F->isDeclaration() || mayContainUnboundedCycle(*F, A))
3344 indicatePessimisticFixpoint();
3345 }
3346
3347 /// See AbstractAttribute::trackStatistics()
3348 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3349};
3350
3351/// WillReturn attribute deduction for a call sites.
3352struct AAWillReturnCallSite final
3353 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3354 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3355 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3356
3357 /// See AbstractAttribute::updateImpl(...).
3358 ChangeStatus updateImpl(Attributor &A) override {
3359 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3360 return ChangeStatus::UNCHANGED;
3361
3362 return AACalleeToCallSite::updateImpl(A);
3363 }
3364
3365 /// See AbstractAttribute::trackStatistics()
3366 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3367};
3368} // namespace
3369
3370/// -------------------AAIntraFnReachability Attribute--------------------------
3371
3372/// All information associated with a reachability query. This boilerplate code
3373/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3374/// different \p ToTy values.
3375template <typename ToTy> struct ReachabilityQueryInfo {
3376 enum class Reachable {
3377 No,
3378 Yes,
3379 };
3380
3381 /// Start here,
3382 const Instruction *From = nullptr;
3383 /// reach this place,
3384 const ToTy *To = nullptr;
3385 /// without going through any of these instructions,
3386 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3387 /// and remember if it worked:
3388 Reachable Result = Reachable::No;
3389
3390 /// Precomputed hash for this RQI.
3391 unsigned Hash = 0;
3392
3393 unsigned computeHashValue() const {
3394 assert(Hash == 0 && "Computed hash twice!");
3397 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3398 detail::combineHashValue(PairDMI ::getHashValue({From, To}),
3399 InstSetDMI::getHashValue(ExclusionSet));
3400 }
3401
3403 : From(From), To(To) {}
3404
3405 /// Constructor replacement to ensure unique and stable sets are used for the
3406 /// cache.
3408 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3409 : From(&From), To(&To), ExclusionSet(ES) {
3410
3411 if (!ES || ES->empty()) {
3412 ExclusionSet = nullptr;
3413 } else if (MakeUnique) {
3414 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(ES);
3415 }
3416 }
3417
3419 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3420};
3421
3422namespace llvm {
3423template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3426
3429
3430 static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }
3432 return &TombstoneKey;
3433 }
3434 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3435 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3436 }
3439 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3440 return false;
3441 return InstSetDMI::isEqual(LHS->ExclusionSet, RHS->ExclusionSet);
3442 }
3443};
3444
3445#define DefineKeys(ToTy) \
3446 template <> \
3447 ReachabilityQueryInfo<ToTy> \
3448 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \
3449 ReachabilityQueryInfo<ToTy>( \
3450 DenseMapInfo<const Instruction *>::getEmptyKey(), \
3451 DenseMapInfo<const ToTy *>::getEmptyKey()); \
3452 template <> \
3453 ReachabilityQueryInfo<ToTy> \
3454 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \
3455 ReachabilityQueryInfo<ToTy>( \
3456 DenseMapInfo<const Instruction *>::getTombstoneKey(), \
3457 DenseMapInfo<const ToTy *>::getTombstoneKey());
3458
3460#undef DefineKeys
3461
3462} // namespace llvm
3463
3464namespace {
3465
3466template <typename BaseTy, typename ToTy>
3467struct CachedReachabilityAA : public BaseTy {
3468 using RQITy = ReachabilityQueryInfo<ToTy>;
3469
3470 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3471
3472 /// See AbstractAttribute::isQueryAA.
3473 bool isQueryAA() const override { return true; }
3474
3475 /// See AbstractAttribute::updateImpl(...).
3476 ChangeStatus updateImpl(Attributor &A) override {
3477 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3478 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3479 RQITy *RQI = QueryVector[u];
3480 if (RQI->Result == RQITy::Reachable::No &&
3481 isReachableImpl(A, *RQI, /*IsTemporaryRQI=*/false))
3482 Changed = ChangeStatus::CHANGED;
3483 }
3484 return Changed;
3485 }
3486
3487 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3488 bool IsTemporaryRQI) = 0;
3489
3490 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3491 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3492 RQI.Result = Result;
3493
3494 // Remove the temporary RQI from the cache.
3495 if (IsTemporaryRQI)
3496 QueryCache.erase(&RQI);
3497
3498 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3499 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3500 // this query. 2) We did not use the exclusion set, potentially because
3501 // there is none.
3502 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3503 RQITy PlainRQI(RQI.From, RQI.To);
3504 if (!QueryCache.count(&PlainRQI)) {
3505 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3506 RQIPtr->Result = Result;
3507 QueryVector.push_back(RQIPtr);
3508 QueryCache.insert(RQIPtr);
3509 }
3510 }
3511
3512 // Check if we need to insert a new permanent RQI with the exclusion set.
3513 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3514 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3515 "Did not expect empty set!");
3516 RQITy *RQIPtr = new (A.Allocator)
3517 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3518 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3519 RQIPtr->Result = Result;
3520 assert(!QueryCache.count(RQIPtr));
3521 QueryVector.push_back(RQIPtr);
3522 QueryCache.insert(RQIPtr);
3523 }
3524
3525 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3526 A.registerForUpdate(*this);
3527 return Result == RQITy::Reachable::Yes;
3528 }
3529
3530 const std::string getAsStr(Attributor *A) const override {
3531 // TODO: Return the number of reachable queries.
3532 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3533 }
3534
3535 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3536 typename RQITy::Reachable &Result) {
3537 if (!this->getState().isValidState()) {
3538 Result = RQITy::Reachable::Yes;
3539 return true;
3540 }
3541
3542 // If we have an exclusion set we might be able to find our answer by
3543 // ignoring it first.
3544 if (StackRQI.ExclusionSet) {
3545 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3546 auto It = QueryCache.find(&PlainRQI);
3547 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3548 Result = RQITy::Reachable::No;
3549 return true;
3550 }
3551 }
3552
3553 auto It = QueryCache.find(&StackRQI);
3554 if (It != QueryCache.end()) {
3555 Result = (*It)->Result;
3556 return true;
3557 }
3558
3559 // Insert a temporary for recursive queries. We will replace it with a
3560 // permanent entry later.
3561 QueryCache.insert(&StackRQI);
3562 return false;
3563 }
3564
3565private:
3566 SmallVector<RQITy *> QueryVector;
3567 DenseSet<RQITy *> QueryCache;
3568};
3569
3570struct AAIntraFnReachabilityFunction final
3571 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3572 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3573 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3574 : Base(IRP, A) {
3575 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3576 *IRP.getAssociatedFunction());
3577 }
3578
3579 bool isAssumedReachable(
3580 Attributor &A, const Instruction &From, const Instruction &To,
3581 const AA::InstExclusionSetTy *ExclusionSet) const override {
3582 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3583 if (&From == &To)
3584 return true;
3585
3586 RQITy StackRQI(A, From, To, ExclusionSet, false);
3587 typename RQITy::Reachable Result;
3588 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3589 return NonConstThis->isReachableImpl(A, StackRQI,
3590 /*IsTemporaryRQI=*/true);
3591 return Result == RQITy::Reachable::Yes;
3592 }
3593
3594 ChangeStatus updateImpl(Attributor &A) override {
3595 // We only depend on liveness. DeadEdges is all we care about, check if any
3596 // of them changed.
3597 auto *LivenessAA =
3598 A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
3599 if (LivenessAA &&
3600 llvm::all_of(DeadEdges,
3601 [&](const auto &DeadEdge) {
3602 return LivenessAA->isEdgeDead(DeadEdge.first,
3603 DeadEdge.second);
3604 }) &&
3605 llvm::all_of(DeadBlocks, [&](const BasicBlock *BB) {
3606 return LivenessAA->isAssumedDead(BB);
3607 })) {
3608 return ChangeStatus::UNCHANGED;
3609 }
3610 DeadEdges.clear();
3611 DeadBlocks.clear();
3612 return Base::updateImpl(A);
3613 }
3614
3615 bool isReachableImpl(Attributor &A, RQITy &RQI,
3616 bool IsTemporaryRQI) override {
3617 const Instruction *Origin = RQI.From;
3618 bool UsedExclusionSet = false;
3619
3620 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3621 const AA::InstExclusionSetTy *ExclusionSet) {
3622 const Instruction *IP = &From;
3623 while (IP && IP != &To) {
3624 if (ExclusionSet && IP != Origin && ExclusionSet->count(IP)) {
3625 UsedExclusionSet = true;
3626 break;
3627 }
3628 IP = IP->getNextNode();
3629 }
3630 return IP == &To;
3631 };
3632
3633 const BasicBlock *FromBB = RQI.From->getParent();
3634 const BasicBlock *ToBB = RQI.To->getParent();
3635 assert(FromBB->getParent() == ToBB->getParent() &&
3636 "Not an intra-procedural query!");
3637
3638 // Check intra-block reachability, however, other reaching paths are still
3639 // possible.
3640 if (FromBB == ToBB &&
3641 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3642 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3643 IsTemporaryRQI);
3644
3645 // Check if reaching the ToBB block is sufficient or if even that would not
3646 // ensure reaching the target. In the latter case we are done.
3647 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3648 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3649 IsTemporaryRQI);
3650
3651 const Function *Fn = FromBB->getParent();
3653 if (RQI.ExclusionSet)
3654 for (auto *I : *RQI.ExclusionSet)
3655 if (I->getFunction() == Fn)
3656 ExclusionBlocks.insert(I->getParent());
3657
3658 // Check if we make it out of the FromBB block at all.
3659 if (ExclusionBlocks.count(FromBB) &&
3660 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3661 RQI.ExclusionSet))
3662 return rememberResult(A, RQITy::Reachable::No, RQI, true, IsTemporaryRQI);
3663
3664 auto *LivenessAA =
3665 A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
3666 if (LivenessAA && LivenessAA->isAssumedDead(ToBB)) {
3667 DeadBlocks.insert(ToBB);
3668 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3669 IsTemporaryRQI);
3670 }
3671
3674 Worklist.push_back(FromBB);
3675
3677 while (!Worklist.empty()) {
3678 const BasicBlock *BB = Worklist.pop_back_val();
3679 if (!Visited.insert(BB).second)
3680 continue;
3681 for (const BasicBlock *SuccBB : successors(BB)) {
3682 if (LivenessAA && LivenessAA->isEdgeDead(BB, SuccBB)) {
3683 LocalDeadEdges.insert({BB, SuccBB});
3684 continue;
3685 }
3686 // We checked before if we just need to reach the ToBB block.
3687 if (SuccBB == ToBB)
3688 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3689 IsTemporaryRQI);
3690 if (DT && ExclusionBlocks.empty() && DT->dominates(BB, ToBB))
3691 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3692 IsTemporaryRQI);
3693
3694 if (ExclusionBlocks.count(SuccBB)) {
3695 UsedExclusionSet = true;
3696 continue;
3697 }
3698 Worklist.push_back(SuccBB);
3699 }
3700 }
3701
3702 DeadEdges.insert(LocalDeadEdges.begin(), LocalDeadEdges.end());
3703 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3704 IsTemporaryRQI);
3705 }
3706
3707 /// See AbstractAttribute::trackStatistics()
3708 void trackStatistics() const override {}
3709
3710private:
3711 // Set of assumed dead blocks we used in the last query. If any changes we
3712 // update the state.
3714
3715 // Set of assumed dead edges we used in the last query. If any changes we
3716 // update the state.
3718
3719 /// The dominator tree of the function to short-circuit reasoning.
3720 const DominatorTree *DT = nullptr;
3721};
3722} // namespace
3723
3724/// ------------------------ NoAlias Argument Attribute ------------------------
3725
3727 Attribute::AttrKind ImpliedAttributeKind,
3728 bool IgnoreSubsumingPositions) {
3729 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3730 "Unexpected attribute kind");
3731 Value *Val = &IRP.getAssociatedValue();
3732 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3733 if (isa<AllocaInst>(Val))
3734 return true;
3735 } else {
3736 IgnoreSubsumingPositions = true;
3737 }
3738
3739 if (isa<UndefValue>(Val))
3740 return true;
3741
3742 if (isa<ConstantPointerNull>(Val) &&
3745 return true;
3746
3747 if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},
3748 IgnoreSubsumingPositions, Attribute::NoAlias))
3749 return true;
3750
3751 return false;
3752}
3753
3754namespace {
3755struct AANoAliasImpl : AANoAlias {
3756 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3757 assert(getAssociatedType()->isPointerTy() &&
3758 "Noalias is a pointer attribute");
3759 }
3760
3761 const std::string getAsStr(Attributor *A) const override {
3762 return getAssumed() ? "noalias" : "may-alias";
3763 }
3764};
3765
3766/// NoAlias attribute for a floating value.
3767struct AANoAliasFloating final : AANoAliasImpl {
3768 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3769 : AANoAliasImpl(IRP, A) {}
3770
3771 /// See AbstractAttribute::updateImpl(...).
3772 ChangeStatus updateImpl(Attributor &A) override {
3773 // TODO: Implement this.
3774 return indicatePessimisticFixpoint();
3775 }
3776
3777 /// See AbstractAttribute::trackStatistics()
3778 void trackStatistics() const override {
3780 }
3781};
3782
3783/// NoAlias attribute for an argument.
3784struct AANoAliasArgument final
3785 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3786 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3787 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3788
3789 /// See AbstractAttribute::update(...).
3790 ChangeStatus updateImpl(Attributor &A) override {
3791 // We have to make sure no-alias on the argument does not break
3792 // synchronization when this is a callback argument, see also [1] below.
3793 // If synchronization cannot be affected, we delegate to the base updateImpl
3794 // function, otherwise we give up for now.
3795
3796 // If the function is no-sync, no-alias cannot break synchronization.
3797 bool IsKnownNoSycn;
3798 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3799 A, this, IRPosition::function_scope(getIRPosition()),
3800 DepClassTy::OPTIONAL, IsKnownNoSycn))
3801 return Base::updateImpl(A);
3802
3803 // If the argument is read-only, no-alias cannot break synchronization.
3804 bool IsKnown;
3805 if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
3806 return Base::updateImpl(A);
3807
3808 // If the argument is never passed through callbacks, no-alias cannot break
3809 // synchronization.
3810 bool UsedAssumedInformation = false;
3811 if (A.checkForAllCallSites(
3812 [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this,
3813 true, UsedAssumedInformation))
3814 return Base::updateImpl(A);
3815
3816 // TODO: add no-alias but make sure it doesn't break synchronization by
3817 // introducing fake uses. See:
3818 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3819 // International Workshop on OpenMP 2018,
3820 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3821
3822 return indicatePessimisticFixpoint();
3823 }
3824
3825 /// See AbstractAttribute::trackStatistics()
3826 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3827};
3828
3829struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3830 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3831 : AANoAliasImpl(IRP, A) {}
3832
3833 /// Determine if the underlying value may alias with the call site argument
3834 /// \p OtherArgNo of \p ICS (= the underlying call site).
3835 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3836 const AAMemoryBehavior &MemBehaviorAA,
3837 const CallBase &CB, unsigned OtherArgNo) {
3838 // We do not need to worry about aliasing with the underlying IRP.
3839 if (this->getCalleeArgNo() == (int)OtherArgNo)
3840 return false;
3841
3842 // If it is not a pointer or pointer vector we do not alias.
3843 const Value *ArgOp = CB.getArgOperand(OtherArgNo);
3844 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3845 return false;
3846
3847 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3848 *this, IRPosition::callsite_argument(CB, OtherArgNo), DepClassTy::NONE);
3849
3850 // If the argument is readnone, there is no read-write aliasing.
3851 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3852 A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
3853 return false;
3854 }
3855
3856 // If the argument is readonly and the underlying value is readonly, there
3857 // is no read-write aliasing.
3858 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3859 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3860 IsReadOnly) {
3861 A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL);
3862 A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
3863 return false;
3864 }
3865
3866 // We have to utilize actual alias analysis queries so we need the object.
3867 if (!AAR)
3868 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3869 *getAnchorScope());
3870
3871 // Try to rule it out at the call site.
3872 bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp);
3873 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3874 "callsite arguments: "
3875 << getAssociatedValue() << " " << *ArgOp << " => "
3876 << (IsAliasing ? "" : "no-") << "alias \n");
3877
3878 return IsAliasing;
3879 }
3880
3881 bool isKnownNoAliasDueToNoAliasPreservation(
3882 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3883 // We can deduce "noalias" if the following conditions hold.
3884 // (i) Associated value is assumed to be noalias in the definition.
3885 // (ii) Associated value is assumed to be no-capture in all the uses
3886 // possibly executed before this callsite.
3887 // (iii) There is no other pointer argument which could alias with the
3888 // value.
3889
3890 auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
3891 const auto *DerefAA = A.getAAFor<AADereferenceable>(
3892 *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
3893 return DerefAA ? DerefAA->getAssumedDereferenceableBytes() : 0;
3894 };
3895
3896 const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
3897 const Function *ScopeFn = VIRP.getAnchorScope();
3898 // Check whether the value is captured in the scope using AANoCapture.
3899 // Look at CFG and check only uses possibly executed before this
3900 // callsite.
3901 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3902 Instruction *UserI = cast<Instruction>(U.getUser());
3903
3904 // If UserI is the curr instruction and there is a single potential use of
3905 // the value in UserI we allow the use.
3906 // TODO: We should inspect the operands and allow those that cannot alias
3907 // with the value.
3908 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3909 return true;
3910
3911 if (ScopeFn) {
3912 if (auto *CB = dyn_cast<CallBase>(UserI)) {
3913 if (CB->isArgOperand(&U)) {
3914
3915 unsigned ArgNo = CB->getArgOperandNo(&U);
3916
3917 bool IsKnownNoCapture;
3918 if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
3919 A, this, IRPosition::callsite_argument(*CB, ArgNo),
3920 DepClassTy::OPTIONAL, IsKnownNoCapture))
3921 return true;
3922 }
3923 }
3924
3926 A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr,
3927 [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3928 return true;
3929 }
3930
3931 // TODO: We should track the capturing uses in AANoCapture but the problem
3932 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3933 // a value in the module slice.
3934 switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
3935 case UseCaptureKind::NO_CAPTURE:
3936 return true;
3937 case UseCaptureKind::MAY_CAPTURE:
3938 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI
3939 << "\n");
3940 return false;
3941 case UseCaptureKind::PASSTHROUGH:
3942 Follow = true;
3943 return true;
3944 }
3945 llvm_unreachable("unknown UseCaptureKind");
3946 };
3947
3948 bool IsKnownNoCapture;
3949 const AANoCapture *NoCaptureAA = nullptr;
3950 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
3951 A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);
3952 if (!IsAssumedNoCapture &&
3953 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
3954 if (!A.checkForAllUses(UsePred, *this, getAssociatedValue())) {
3955 LLVM_DEBUG(
3956 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
3957 << " cannot be noalias as it is potentially captured\n");
3958 return false;
3959 }
3960 }
3961 if (NoCaptureAA)
3962 A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL);
3963
3964 // Check there is no other pointer argument which could alias with the
3965 // value passed at this call site.
3966 // TODO: AbstractCallSite
3967 const auto &CB = cast<CallBase>(getAnchorValue());
3968 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
3969 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
3970 return false;
3971
3972 return true;
3973 }
3974
3975 /// See AbstractAttribute::updateImpl(...).
3976 ChangeStatus updateImpl(Attributor &A) override {
3977 // If the argument is readnone we are done as there are no accesses via the
3978 // argument.
3979 auto *MemBehaviorAA =
3980 A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
3981 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
3982 A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
3983 return ChangeStatus::UNCHANGED;
3984 }
3985
3986 bool IsKnownNoAlias;
3987 const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
3988 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
3989 A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
3990 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
3991 << " is not no-alias at the definition\n");
3992 return indicatePessimisticFixpoint();
3993 }
3994
3995 AAResults *AAR = nullptr;
3996 if (MemBehaviorAA &&
3997 isKnownNoAliasDueToNoAliasPreservation(A, AAR, *MemBehaviorAA)) {
3998 LLVM_DEBUG(
3999 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
4000 return ChangeStatus::UNCHANGED;
4001 }
4002
4003 return indicatePessimisticFixpoint();
4004 }
4005
4006 /// See AbstractAttribute::trackStatistics()
4007 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4008};
4009
4010/// NoAlias attribute for function return value.
4011struct AANoAliasReturned final : AANoAliasImpl {
4012 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4013 : AANoAliasImpl(IRP, A) {}
4014
4015 /// See AbstractAttribute::updateImpl(...).
4016 ChangeStatus updateImpl(Attributor &A) override {
4017
4018 auto CheckReturnValue = [&](Value &RV) -> bool {
4019 if (Constant *C = dyn_cast<Constant>(&RV))
4020 if (C->isNullValue() || isa<UndefValue>(C))
4021 return true;
4022
4023 /// For now, we can only deduce noalias if we have call sites.
4024 /// FIXME: add more support.
4025 if (!isa<CallBase>(&RV))
4026 return false;
4027
4028 const IRPosition &RVPos = IRPosition::value(RV);
4029 bool IsKnownNoAlias;
4030 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4031 A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))
4032 return false;
4033
4034 bool IsKnownNoCapture;
4035 const AANoCapture *NoCaptureAA = nullptr;
4036 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
4037 A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
4038 &NoCaptureAA);
4039 return IsAssumedNoCapture ||
4040 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4041 };
4042
4043 if (!A.checkForAllReturnedValues(CheckReturnValue, *this))
4044 return indicatePessimisticFixpoint();
4045
4046 return ChangeStatus::UNCHANGED;
4047 }
4048
4049 /// See AbstractAttribute::trackStatistics()
4050 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4051};
4052
4053/// NoAlias attribute deduction for a call site return value.
4054struct AANoAliasCallSiteReturned final
4055 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4056 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4057 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4058
4059 /// See AbstractAttribute::trackStatistics()
4060 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4061};
4062} // namespace
4063
4064/// -------------------AAIsDead Function Attribute-----------------------
4065
4066namespace {
4067struct AAIsDeadValueImpl : public AAIsDead {
4068 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4069
4070 /// See AAIsDead::isAssumedDead().
4071 bool isAssumedDead() const override { return isAssumed(IS_DEAD); }
4072
4073 /// See AAIsDead::isKnownDead().
4074 bool isKnownDead() const override { return isKnown(IS_DEAD); }
4075
4076 /// See AAIsDead::isAssumedDead(BasicBlock *).
4077 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4078
4079 /// See AAIsDead::isKnownDead(BasicBlock *).
4080 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4081
4082 /// See AAIsDead::isAssumedDead(Instruction *I).
4083 bool isAssumedDead(const Instruction *I) const override {
4084 return I == getCtxI() && isAssumedDead();
4085 }
4086
4087 /// See AAIsDead::isKnownDead(Instruction *I).
4088 bool isKnownDead(const Instruction *I) const override {
4089 return isAssumedDead(I) && isKnownDead();
4090 }
4091
4092 /// See AbstractAttribute::getAsStr().
4093 const std::string getAsStr(Attributor *A) const override {
4094 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4095 }
4096
4097 /// Check if all uses are assumed dead.
4098 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4099 // Callers might not check the type, void has no uses.
4100 if (V.getType()->isVoidTy() || V.use_empty())
4101 return true;
4102
4103 // If we replace a value with a constant there are no uses left afterwards.
4104 if (!isa<Constant>(V)) {
4105 if (auto *I = dyn_cast<Instruction>(&V))
4106 if (!A.isRunOn(*I->getFunction()))
4107 return false;
4108 bool UsedAssumedInformation = false;
4109 std::optional<Constant *> C =
4110 A.getAssumedConstant(V, *this, UsedAssumedInformation);
4111 if (!C || *C)
4112 return true;
4113 }
4114
4115 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4116 // Explicitly set the dependence class to required because we want a long
4117 // chain of N dependent instructions to be considered live as soon as one is
4118 // without going through N update cycles. This is not required for
4119 // correctness.
4120 return A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ false,
4121 DepClassTy::REQUIRED,
4122 /* IgnoreDroppableUses */ false);
4123 }
4124
4125 /// Determine if \p I is assumed to be side-effect free.
4126 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4128 return true;
4129
4130 auto *CB = dyn_cast<CallBase>(I);
4131 if (!CB || isa<IntrinsicInst>(CB))
4132 return false;
4133
4134 const IRPosition &CallIRP = IRPosition::callsite_function(*CB);
4135
4136 bool IsKnownNoUnwind;
4137 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4138 A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))
4139 return false;
4140
4141 bool IsKnown;
4142 return AA::isAssumedReadOnly(A, CallIRP, *this, IsKnown);
4143 }
4144};
4145
4146struct AAIsDeadFloating : public AAIsDeadValueImpl {
4147 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4148 : AAIsDeadValueImpl(IRP, A) {}
4149
4150 /// See AbstractAttribute::initialize(...).
4151 void initialize(Attributor &A) override {
4152 AAIsDeadValueImpl::initialize(A);
4153
4154 if (isa<UndefValue>(getAssociatedValue())) {
4155 indicatePessimisticFixpoint();
4156 return;
4157 }
4158
4159 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4160 if (!isAssumedSideEffectFree(A, I)) {
4161 if (!isa_and_nonnull<StoreInst>(I) && !isa_and_nonnull<FenceInst>(I))
4162 indicatePessimisticFixpoint();
4163 else
4164 removeAssumedBits(HAS_NO_EFFECT);
4165 }
4166 }
4167
4168 bool isDeadFence(Attributor &A, FenceInst &FI) {
4169 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4170 IRPosition::function(*FI.getFunction()), *this, DepClassTy::NONE);
4171 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4172 return false;
4173 A.recordDependence(*ExecDomainAA, *this, DepClassTy::OPTIONAL);
4174 return true;
4175 }
4176
4177 bool isDeadStore(Attributor &A, StoreInst &SI,
4178 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4179 // Lang ref now states volatile store is not UB/dead, let's skip them.
4180 if (SI.isVolatile())
4181 return false;
4182
4183 // If we are collecting assumes to be deleted we are in the manifest stage.
4184 // It's problematic to collect the potential copies again now so we use the
4185 // cached ones.
4186 bool UsedAssumedInformation = false;
4187 if (!AssumeOnlyInst) {
4188 PotentialCopies.clear();
4189 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, *this,
4190 UsedAssumedInformation)) {
4191 LLVM_DEBUG(
4192 dbgs()
4193 << "[AAIsDead] Could not determine potential copies of store!\n");
4194 return false;
4195 }
4196 }
4197 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4198 << " potential copies.\n");
4199
4200 InformationCache &InfoCache = A.getInfoCache();
4201 return llvm::all_of(PotentialCopies, [&](Value *V) {
4202 if (A.isAssumedDead(IRPosition::value(*V), this, nullptr,
4203 UsedAssumedInformation))
4204 return true;
4205 if (auto *LI = dyn_cast<LoadInst>(V)) {
4206 if (llvm::all_of(LI->uses(), [&](const Use &U) {
4207 auto &UserI = cast<Instruction>(*U.getUser());
4208 if (InfoCache.isOnlyUsedByAssume(UserI)) {
4209 if (AssumeOnlyInst)
4210 AssumeOnlyInst->insert(&UserI);
4211 return true;
4212 }
4213 return A.isAssumedDead(U, this, nullptr, UsedAssumedInformation);
4214 })) {
4215 return true;
4216 }
4217 }
4218 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4219 << " is assumed live!\n");
4220 return false;
4221 });
4222 }
4223
4224 /// See AbstractAttribute::getAsStr().
4225 const std::string getAsStr(Attributor *A) const override {
4226 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4227 if (isa_and_nonnull<StoreInst>(I))
4228 if (isValidState())
4229 return "assumed-dead-store";
4230 if (isa_and_nonnull<FenceInst>(I))
4231 if (isValidState())
4232 return "assumed-dead-fence";
4233 return AAIsDeadValueImpl::getAsStr(A);
4234 }
4235
4236 /// See AbstractAttribute::updateImpl(...).
4237 ChangeStatus updateImpl(Attributor &A) override {
4238 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
4239 if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {
4240 if (!isDeadStore(A, *SI))
4241 return indicatePessimisticFixpoint();
4242 } else if (auto *FI = dyn_cast_or_null<FenceInst>(I)) {
4243 if (!isDeadFence(A, *FI))
4244 return indicatePessimisticFixpoint();
4245 } else {
4246 if (!isAssumedSideEffectFree(A, I))
4247 return indicatePessimisticFixpoint();
4248 if (!areAllUsesAssumedDead(A, getAssociatedValue()))
4249 return indicatePessimisticFixpoint();
4250 }
4252 }
4253
4254 bool isRemovableStore() const override {
4255 return isAssumed(IS_REMOVABLE) && isa<StoreInst>(&getAssociatedValue());
4256 }
4257
4258 /// See AbstractAttribute::manifest(...).
4259 ChangeStatus manifest(Attributor &A) override {
4260 Value &V = getAssociatedValue();
4261 if (auto *I = dyn_cast<Instruction>(&V)) {
4262 // If we get here we basically know the users are all dead. We check if
4263 // isAssumedSideEffectFree returns true here again because it might not be
4264 // the case and only the users are dead but the instruction (=call) is
4265 // still needed.
4266 if (auto *SI = dyn_cast<StoreInst>(I)) {
4267 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4268 bool IsDead = isDeadStore(A, *SI, &AssumeOnlyInst);
4269 (void)IsDead;
4270 assert(IsDead && "Store was assumed to be dead!");
4271 A.deleteAfterManifest(*I);
4272 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4273 Instruction *AOI = AssumeOnlyInst[i];
4274 for (auto *Usr : AOI->users())
4275 AssumeOnlyInst.insert(cast<Instruction>(Usr));
4276 A.deleteAfterManifest(*AOI);
4277 }
4278 return ChangeStatus::CHANGED;
4279 }
4280 if (auto *FI = dyn_cast<FenceInst>(I)) {
4281 assert(isDeadFence(A, *FI));
4282 A.deleteAfterManifest(*FI);
4283 return ChangeStatus::CHANGED;
4284 }
4285 if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(I)) {
4286 A.deleteAfterManifest(*I);
4287 return ChangeStatus::CHANGED;
4288 }
4289 }
4291 }
4292
4293 /// See AbstractAttribute::trackStatistics()
4294 void trackStatistics() const override {
4296 }
4297
4298private:
4299 // The potential copies of a dead store, used for deletion during manifest.
4300 SmallSetVector<Value *, 4> PotentialCopies;
4301};
4302
4303struct AAIsDeadArgument : public AAIsDeadFloating {
4304 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4305 : AAIsDeadFloating(IRP, A) {}
4306
4307 /// See AbstractAttribute::manifest(...).
4308 ChangeStatus manifest(Attributor &A) override {
4309 Argument &Arg = *getAssociatedArgument();
4310 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4311 if (A.registerFunctionSignatureRewrite(
4312 Arg, /* ReplacementTypes */ {},
4315 return ChangeStatus::CHANGED;
4316 }
4317 return ChangeStatus::UNCHANGED;
4318 }
4319
4320 /// See AbstractAttribute::trackStatistics()
4321 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4322};
4323
4324struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4325 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4326 : AAIsDeadValueImpl(IRP, A) {}
4327
4328 /// See AbstractAttribute::initialize(...).
4329 void initialize(Attributor &A) override {
4330 AAIsDeadValueImpl::initialize(A);
4331 if (isa<UndefValue>(getAssociatedValue()))
4332 indicatePessimisticFixpoint();
4333 }
4334
4335 /// See AbstractAttribute::updateImpl(...).
4336 ChangeStatus updateImpl(Attributor &A) override {
4337 // TODO: Once we have call site specific value information we can provide
4338 // call site specific liveness information and then it makes
4339 // sense to specialize attributes for call sites arguments instead of
4340 // redirecting requests to the callee argument.
4341 Argument *Arg = getAssociatedArgument();
4342 if (!Arg)
4343 return indicatePessimisticFixpoint();
4344 const IRPosition &ArgPos = IRPosition::argument(*Arg);
4345 auto *ArgAA = A.getAAFor<AAIsDead>(*this, ArgPos, DepClassTy::REQUIRED);
4346 if (!ArgAA)
4347 return indicatePessimisticFixpoint();
4348 return clampStateAndIndicateChange(getState(), ArgAA->getState());
4349 }
4350
4351 /// See AbstractAttribute::manifest(...).
4352 ChangeStatus manifest(Attributor &A) override {
4353 CallBase &CB = cast<CallBase>(getAnchorValue());
4354 Use &U = CB.getArgOperandUse(getCallSiteArgNo());
4355 assert(!isa<UndefValue>(U.get()) &&
4356 "Expected undef values to be filtered out!");
4357 UndefValue &UV = *UndefValue::get(U->getType());
4358 if (A.changeUseAfterManifest(U, UV))
4359 return ChangeStatus::CHANGED;
4360 return ChangeStatus::UNCHANGED;
4361 }
4362
4363 /// See AbstractAttribute::trackStatistics()
4364 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4365};
4366
4367struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4368 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4369 : AAIsDeadFloating(IRP, A) {}
4370
4371 /// See AAIsDead::isAssumedDead().
4372 bool isAssumedDead() const override {
4373 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4374 }
4375
4376 /// See AbstractAttribute::initialize(...).
4377 void initialize(Attributor &A) override {
4378 AAIsDeadFloating::initialize(A);
4379 if (isa<UndefValue>(getAssociatedValue())) {
4380 indicatePessimisticFixpoint();
4381 return;
4382 }
4383
4384 // We track this separately as a secondary state.
4385 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, getCtxI());
4386 }
4387
4388 /// See AbstractAttribute::updateImpl(...).
4389 ChangeStatus updateImpl(Attributor &A) override {
4390 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4391 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, getCtxI())) {
4392 IsAssumedSideEffectFree = false;
4393 Changed = ChangeStatus::CHANGED;
4394 }
4395 if (!areAllUsesAssumedDead(A, getAssociatedValue()))
4396 return indicatePessimisticFixpoint();
4397 return Changed;
4398 }
4399
4400 /// See AbstractAttribute::trackStatistics()
4401 void trackStatistics() const override {
4402 if (IsAssumedSideEffectFree)
4404 else
4405 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4406 }
4407
4408 /// See AbstractAttribute::getAsStr().
4409 const std::string getAsStr(Attributor *A) const override {
4410 return isAssumedDead()
4411 ? "assumed-dead"
4412 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4413 }
4414
4415private:
4416 bool IsAssumedSideEffectFree = true;
4417};
4418
4419struct AAIsDeadReturned : public AAIsDeadValueImpl {
4420 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4421 : AAIsDeadValueImpl(IRP, A) {}
4422
4423 /// See AbstractAttribute::updateImpl(...).
4424 ChangeStatus updateImpl(Attributor &A) override {
4425
4426 bool UsedAssumedInformation = false;
4427 A.checkForAllInstructions([](Instruction &) { return true; }, *this,
4428 {Instruction::Ret}, UsedAssumedInformation);
4429
4430 auto PredForCallSite = [&](AbstractCallSite ACS) {
4431 if (ACS.isCallbackCall() || !ACS.getInstruction())
4432 return false;
4433 return areAllUsesAssumedDead(A, *ACS.getInstruction());
4434 };
4435
4436 if (!A.checkForAllCallSites(PredForCallSite, *this, true,
4437 UsedAssumedInformation))
4438 return indicatePessimisticFixpoint();
4439
4440 return ChangeStatus::UNCHANGED;
4441 }
4442
4443 /// See AbstractAttribute::manifest(...).
4444 ChangeStatus manifest(Attributor &A) override {
4445 // TODO: Rewrite the signature to return void?
4446 bool AnyChange = false;
4447 UndefValue &UV = *UndefValue::get(getAssociatedFunction()->getReturnType());
4448 auto RetInstPred = [&](Instruction &I) {
4449 ReturnInst &RI = cast<ReturnInst>(I);
4450 if (!isa<UndefValue>(RI.getReturnValue()))
4451 AnyChange |= A.changeUseAfterManifest(RI.getOperandUse(0), UV);
4452 return true;
4453 };
4454 bool UsedAssumedInformation = false;
4455 A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
4456 UsedAssumedInformation);
4457 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4458 }
4459
4460 /// See AbstractAttribute::trackStatistics()
4461 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4462};
4463
4464struct AAIsDeadFunction : public AAIsDead {
4465 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4466
4467 /// See AbstractAttribute::initialize(...).
4468 void initialize(Attributor &A) override {
4469 Function *F = getAnchorScope();
4470 assert(F && "Did expect an anchor function");
4471 if (!isAssumedDeadInternalFunction(A)) {
4472 ToBeExploredFrom.insert(&F->getEntryBlock().front());
4473 assumeLive(A, F->getEntryBlock());
4474 }
4475 }
4476
4477 bool isAssumedDeadInternalFunction(Attributor &A) {
4478 if (!getAnchorScope()->hasLocalLinkage())
4479 return false;
4480 bool UsedAssumedInformation = false;
4481 return A.checkForAllCallSites([](AbstractCallSite) { return false; }, *this,
4482 true, UsedAssumedInformation);
4483 }
4484
4485 /// See AbstractAttribute::getAsStr().
4486 const std::string getAsStr(Attributor *A) const override {
4487 return "Live[#BB " + std::to_string(AssumedLiveBlocks.size()) + "/" +
4488 std::to_string(getAnchorScope()->size()) + "][#TBEP " +
4489 std::to_string(ToBeExploredFrom.size()) + "][#KDE " +
4490 std::to_string(KnownDeadEnds.size()) + "]";
4491 }
4492
4493 /// See AbstractAttribute::manifest(...).
4494 ChangeStatus manifest(Attributor &A) override {
4495 assert(getState().isValidState() &&
4496 "Attempted to manifest an invalid state!");
4497
4498 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4499 Function &F = *getAnchorScope();
4500
4501 if (AssumedLiveBlocks.empty()) {
4502 A.deleteAfterManifest(F);
4503 return ChangeStatus::CHANGED;
4504 }
4505
4506 // Flag to determine if we can change an invoke to a call assuming the
4507 // callee is nounwind. This is not possible if the personality of the
4508 // function allows to catch asynchronous exceptions.
4509 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4510
4511 KnownDeadEnds.set_union(ToBeExploredFrom);
4512 for (const Instruction *DeadEndI : KnownDeadEnds) {
4513 auto *CB = dyn_cast<CallBase>(DeadEndI);
4514 if (!CB)
4515 continue;
4516 bool IsKnownNoReturn;
4517 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4518 A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL,
4519 IsKnownNoReturn);
4520 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(CB)))
4521 continue;
4522
4523 if (auto *II = dyn_cast<InvokeInst>(DeadEndI))
4524 A.registerInvokeWithDeadSuccessor(const_cast<InvokeInst &>(*II));
4525 else
4526 A.changeToUnreachableAfterManifest(
4527 const_cast<Instruction *>(DeadEndI->getNextNode()));
4528 HasChanged = ChangeStatus::CHANGED;
4529 }
4530
4531 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4532 for (BasicBlock &BB : F)
4533 if (!AssumedLiveBlocks.count(&BB)) {
4534 A.deleteAfterManifest(BB);
4536 HasChanged = ChangeStatus::CHANGED;
4537 }
4538
4539 return HasChanged;
4540 }
4541
4542 /// See AbstractAttribute::updateImpl(...).
4543 ChangeStatus updateImpl(Attributor &A) override;
4544
4545 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4546 assert(From->getParent() == getAnchorScope() &&
4547 To->getParent() == getAnchorScope() &&
4548 "Used AAIsDead of the wrong function");
4549 return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To));
4550 }
4551
4552 /// See AbstractAttribute::trackStatistics()
4553 void trackStatistics() const override {}
4554
4555 /// Returns true if the function is assumed dead.
4556 bool isAssumedDead() const override { return false; }
4557
4558 /// See AAIsDead::isKnownDead().
4559 bool isKnownDead() const override { return false; }
4560
4561 /// See AAIsDead::isAssumedDead(BasicBlock *).
4562 bool isAssumedDead(const BasicBlock *BB) const override {
4563 assert(BB->getParent() == getAnchorScope() &&
4564 "BB must be in the same anchor scope function.");
4565
4566 if (!getAssumed())
4567 return false;
4568 return !AssumedLiveBlocks.count(BB);
4569 }
4570
4571 /// See AAIsDead::isKnownDead(BasicBlock *).
4572 bool isKnownDead(const BasicBlock *BB) const override {
4573 return getKnown() && isAssumedDead(BB);
4574 }
4575
4576 /// See AAIsDead::isAssumed(Instruction *I).
4577 bool isAssumedDead(const Instruction *I) const override {
4578 assert(I->getParent()->getParent() == getAnchorScope() &&
4579 "Instruction must be in the same anchor scope function.");
4580
4581 if (!getAssumed())
4582 return false;
4583
4584 // If it is not in AssumedLiveBlocks then it for sure dead.
4585 // Otherwise, it can still be after noreturn call in a live block.
4586 if (!AssumedLiveBlocks.count(I->getParent()))
4587 return true;
4588
4589 // If it is not after a liveness barrier it is live.
4590 const Instruction *PrevI = I->getPrevNode();
4591 while (PrevI) {
4592 if (KnownDeadEnds.count(PrevI) || ToBeExploredFrom.count(PrevI))
4593 return true;
4594 PrevI = PrevI->getPrevNode();
4595 }
4596 return false;
4597 }
4598
4599 /// See AAIsDead::isKnownDead(Instruction *I).
4600 bool isKnownDead(const Instruction *I) const override {
4601 return getKnown() && isAssumedDead(I);
4602 }
4603
4604 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4605 /// that internal function called from \p BB should now be looked at.
4606 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4607 if (!AssumedLiveBlocks.insert(&BB).second)
4608 return false;
4609
4610 // We assume that all of BB is (probably) live now and if there are calls to
4611 // internal functions we will assume that those are now live as well. This
4612 // is a performance optimization for blocks with calls to a lot of internal
4613 // functions. It can however cause dead functions to be treated as live.
4614 for (const Instruction &I : BB)
4615 if (const auto *CB = dyn_cast<CallBase>(&I))
4616 if (auto *F = dyn_cast_if_present<Function>(CB->getCalledOperand()))
4617 if (F->hasLocalLinkage())
4618 A.markLiveInternalFunction(*F);
4619 return true;
4620 }
4621
4622 /// Collection of instructions that need to be explored again, e.g., we
4623 /// did assume they do not transfer control to (one of their) successors.
4625
4626 /// Collection of instructions that are known to not transfer control.
4628
4629 /// Collection of all assumed live edges
4631
4632 /// Collection of all assumed live BasicBlocks.
4633 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4634};
4635
4636static bool
4637identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4639 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4640 const IRPosition &IPos = IRPosition::callsite_function(CB);
4641
4642 bool IsKnownNoReturn;
4643 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4644 A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn))
4645 return !IsKnownNoReturn;
4646 if (CB.isTerminator())
4647 AliveSuccessors.push_back(&CB.getSuccessor(0)->front());
4648 else
4649 AliveSuccessors.push_back(CB.getNextNode());
4650 return false;
4651}
4652
4653static bool
4654identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4656 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4657 bool UsedAssumedInformation =
4658 identifyAliveSuccessors(A, cast<CallBase>(II), AA, AliveSuccessors);
4659
4660 // First, determine if we can change an invoke to a call assuming the
4661 // callee is nounwind. This is not possible if the personality of the
4662 // function allows to catch asynchronous exceptions.
4663 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(*II.getFunction())) {
4664 AliveSuccessors.push_back(&II.getUnwindDest()->front());
4665 } else {
4667
4668 bool IsKnownNoUnwind;
4669 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4670 A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
4671 UsedAssumedInformation |= !IsKnownNoUnwind;
4672 } else {
4673 AliveSuccessors.push_back(&II.getUnwindDest()->front());
4674 }
4675 }
4676 return UsedAssumedInformation;
4677}
4678
4679static bool
4680identifyAliveSuccessors(Attributor &A, const BranchInst &BI,
4682 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4683 bool UsedAssumedInformation = false;
4684 if (BI.getNumSuccessors() == 1) {
4685 AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
4686 } else {
4687 std::optional<Constant *> C =
4688 A.getAssumedConstant(*BI.getCondition(), AA, UsedAssumedInformation);
4689 if (!C || isa_and_nonnull<UndefValue>(*C)) {
4690 // No value yet, assume both edges are dead.
4691 } else if (isa_and_nonnull<ConstantInt>(*C)) {
4692 const BasicBlock *SuccBB =
4693 BI.getSuccessor(1 - cast<ConstantInt>(*C)->getValue().getZExtValue());
4694 AliveSuccessors.push_back(&SuccBB->front());
4695 } else {
4696 AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
4697 AliveSuccessors.push_back(&BI.getSuccessor(1)->front());
4698 UsedAssumedInformation = false;
4699 }
4700 }
4701 return UsedAssumedInformation;
4702}
4703
4704static bool
4705identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4707 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4708 bool UsedAssumedInformation = false;
4710 if (!A.getAssumedSimplifiedValues(IRPosition::value(*SI.getCondition()), &AA,
4711 Values, AA::AnyScope,
4712 UsedAssumedInformation)) {
4713 // Something went wrong, assume all successors are live.
4714 for (const BasicBlock *SuccBB : successors(SI.getParent()))
4715 AliveSuccessors.push_back(&SuccBB->front());
4716 return false;
4717 }
4718
4719 if (Values.empty() ||
4720 (Values.size() == 1 &&
4721 isa_and_nonnull<UndefValue>(Values.front().getValue()))) {
4722 // No valid value yet, assume all edges are dead.
4723 return UsedAssumedInformation;
4724 }
4725
4726 Type &Ty = *SI.getCondition()->getType();
4728 auto CheckForConstantInt = [&](Value *V) {
4729 if (auto *CI = dyn_cast_if_present<ConstantInt>(AA::getWithType(*V, Ty))) {
4730 Constants.insert(CI);
4731 return true;
4732 }
4733 return false;
4734 };
4735
4736 if (!all_of(Values, [&](AA::ValueAndContext &VAC) {
4737 return CheckForConstantInt(VAC.getValue());
4738 })) {
4739 for (const BasicBlock *SuccBB : successors(SI.getParent()))
4740 AliveSuccessors.push_back(&SuccBB->front());
4741 return UsedAssumedInformation;
4742 }
4743
4744 unsigned MatchedCases = 0;
4745 for (const auto &CaseIt : SI.cases()) {
4746 if (Constants.count(CaseIt.getCaseValue())) {
4747 ++MatchedCases;
4748 AliveSuccessors.push_back(&CaseIt.getCaseSuccessor()->front());
4749 }
4750 }
4751
4752 // If all potential values have been matched, we will not visit the default
4753 // case.
4754 if (MatchedCases < Constants.size())
4755 AliveSuccessors.push_back(&SI.getDefaultDest()->front());
4756 return UsedAssumedInformation;
4757}
4758
4759ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4761
4762 if (AssumedLiveBlocks.empty()) {
4763 if (isAssumedDeadInternalFunction(A))
4765
4766 Function *F = getAnchorScope();
4767 ToBeExploredFrom.insert(&F->getEntryBlock().front());
4768 assumeLive(A, F->getEntryBlock());
4769 Change = ChangeStatus::CHANGED;
4770 }
4771
4772 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4773 << getAnchorScope()->size() << "] BBs and "
4774 << ToBeExploredFrom.size() << " exploration points and "
4775 << KnownDeadEnds.size() << " known dead ends\n");
4776
4777 // Copy and clear the list of instructions we need to explore from. It is
4778 // refilled with instructions the next update has to look at.
4779 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4780 ToBeExploredFrom.end());
4781 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4782
4784 while (!Worklist.empty()) {
4785 const Instruction *I = Worklist.pop_back_val();
4786 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4787
4788 // Fast forward for uninteresting instructions. We could look for UB here
4789 // though.
4790 while (!I->isTerminator() && !isa<CallBase>(I))
4791 I = I->getNextNode();
4792
4793 AliveSuccessors.clear();
4794
4795 bool UsedAssumedInformation = false;
4796 switch (I->getOpcode()) {
4797 // TODO: look for (assumed) UB to backwards propagate "deadness".
4798 default:
4799 assert(I->isTerminator() &&
4800 "Expected non-terminators to be handled already!");
4801 for (const BasicBlock *SuccBB : successors(I->getParent()))
4802 AliveSuccessors.push_back(&SuccBB->front());
4803 break;
4804 case Instruction::Call:
4805 UsedAssumedInformation = identifyAliveSuccessors(A, cast<CallInst>(*I),
4806 *this, AliveSuccessors);
4807 break;
4808 case Instruction::Invoke:
4809 UsedAssumedInformation = identifyAliveSuccessors(A, cast<InvokeInst>(*I),
4810 *this, AliveSuccessors);
4811 break;
4812 case Instruction::Br:
4813 UsedAssumedInformation = identifyAliveSuccessors(A, cast<BranchInst>(*I),
4814 *this, AliveSuccessors);
4815 break;
4816 case Instruction::Switch:
4817 UsedAssumedInformation = identifyAliveSuccessors(A, cast<SwitchInst>(*I),
4818 *this, AliveSuccessors);
4819 break;
4820 }
4821
4822 if (UsedAssumedInformation) {
4823 NewToBeExploredFrom.insert(I);
4824 } else if (AliveSuccessors.empty() ||
4825 (I->isTerminator() &&
4826 AliveSuccessors.size() < I->getNumSuccessors())) {
4827 if (KnownDeadEnds.insert(I))
4828 Change = ChangeStatus::CHANGED;
4829 }
4830
4831 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4832 << AliveSuccessors.size() << " UsedAssumedInformation: "
4833 << UsedAssumedInformation << "\n");
4834
4835 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4836 if (!I->isTerminator()) {
4837 assert(AliveSuccessors.size() == 1 &&
4838 "Non-terminator expected to have a single successor!");
4839 Worklist.push_back(AliveSuccessor);
4840 } else {
4841 // record the assumed live edge
4842 auto Edge = std::make_pair(I->getParent(), AliveSuccessor->getParent());
4843 if (AssumedLiveEdges.insert(Edge).second)
4844 Change = ChangeStatus::CHANGED;
4845 if (assumeLive(A, *AliveSuccessor->getParent()))
4846 Worklist.push_back(AliveSuccessor);
4847 }
4848 }
4849 }
4850
4851 // Check if the content of ToBeExploredFrom changed, ignore the order.
4852 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4853 llvm::any_of(NewToBeExploredFrom, [&](const Instruction *I) {
4854 return !ToBeExploredFrom.count(I);
4855 })) {
4856 Change = ChangeStatus::CHANGED;
4857 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4858 }
4859
4860 // If we know everything is live there is no need to query for liveness.
4861 // Instead, indicating a pessimistic fixpoint will cause the state to be
4862 // "invalid" and all queries to be answered conservatively without lookups.
4863 // To be in this state we have to (1) finished the exploration and (3) not
4864 // discovered any non-trivial dead end and (2) not ruled unreachable code
4865 // dead.
4866 if (ToBeExploredFrom.empty() &&
4867 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4868 llvm::all_of(KnownDeadEnds, [](const Instruction *DeadEndI) {
4869 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4870 }))
4871 return indicatePessimisticFixpoint();
4872 return Change;
4873}
4874
4875/// Liveness information for a call sites.
4876struct AAIsDeadCallSite final : AAIsDeadFunction {
4877 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4878 : AAIsDeadFunction(IRP, A) {}
4879
4880 /// See AbstractAttribute::initialize(...).
4881 void initialize(Attributor &A) override {
4882 // TODO: Once we have call site specific value information we can provide
4883 // call site specific liveness information and then it makes
4884 // sense to specialize attributes for call sites instead of
4885 // redirecting requests to the callee.
4886 llvm_unreachable("Abstract attributes for liveness are not "
4887 "supported for call sites yet!");
4888 }
4889
4890 /// See AbstractAttribute::updateImpl(...).
4891 ChangeStatus updateImpl(Attributor &A) override {
4892 return indicatePessimisticFixpoint();
4893 }
4894
4895 /// See AbstractAttribute::trackStatistics()
4896 void trackStatistics() const override {}
4897};
4898} // namespace
4899
4900/// -------------------- Dereferenceable Argument Attribute --------------------
4901
4902namespace {
4903struct AADereferenceableImpl : AADereferenceable {
4904 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4905 : AADereferenceable(IRP, A) {}
4906 using StateType = DerefState;
4907
4908 /// See AbstractAttribute::initialize(...).
4909 void initialize(Attributor &A) override {
4910 Value &V = *getAssociatedValue().stripPointerCasts();
4912 A.getAttrs(getIRPosition(),
4913 {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4914 Attrs, /* IgnoreSubsumingPositions */ false);
4915 for (const Attribute &Attr : Attrs)
4916 takeKnownDerefBytesMaximum(Attr.getValueAsInt());
4917
4918 // Ensure we initialize the non-null AA (if necessary).
4919 bool IsKnownNonNull;
4920 AA::hasAssumedIRAttr<Attribute::NonNull>(
4921 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);
4922
4923 bool CanBeNull, CanBeFreed;
4924 takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes(
4925 A.getDataLayout(), CanBeNull, CanBeFreed));
4926
4927 if (Instruction *CtxI = getCtxI())
4928 followUsesInMBEC(*this, A, getState(), *CtxI);
4929 }
4930
4931 /// See AbstractAttribute::getState()
4932 /// {
4933 StateType &getState() override { return *this; }
4934 const StateType &getState() const override { return *this; }
4935 /// }
4936
4937 /// Helper function for collecting accessed bytes in must-be-executed-context
4938 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
4939 DerefState &State) {
4940 const Value *UseV = U->get();
4941 if (!UseV->getType()->isPointerTy())
4942 return;
4943
4944 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
4945 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
4946 return;
4947
4948 int64_t Offset;
4950 Loc->Ptr, Offset, A.getDataLayout(), /*AllowNonInbounds*/ true);
4951 if (Base && Base == &getAssociatedValue())
4952 State.addAccessedBytes(Offset, Loc->Size.getValue());
4953 }
4954
4955 /// See followUsesInMBEC
4956 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
4958 bool IsNonNull = false;
4959 bool TrackUse = false;
4960 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4961 A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse);
4962 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
4963 << " for instruction " << *I << "\n");
4964
4965 addAccessedBytesForUse(A, U, I, State);
4966 State.takeKnownDerefBytesMaximum(DerefBytes);
4967 return TrackUse;
4968 }
4969
4970 /// See AbstractAttribute::manifest(...).
4971 ChangeStatus manifest(Attributor &A) override {
4972 ChangeStatus Change = AADereferenceable::manifest(A);
4973 bool IsKnownNonNull;
4974 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4975 A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4976 if (IsAssumedNonNull &&
4977 A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {
4978 A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});
4979 return ChangeStatus::CHANGED;
4980 }
4981 return Change;
4982 }
4983
4984 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
4985 SmallVectorImpl<Attribute> &Attrs) const override {
4986 // TODO: Add *_globally support
4987 bool IsKnownNonNull;
4988 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4989 A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4990 if (IsAssumedNonNull)
4992 Ctx, getAssumedDereferenceableBytes()));
4993 else
4995 Ctx, getAssumedDereferenceableBytes()));
4996 }
4997
4998 /// See AbstractAttribute::getAsStr().
4999 const std::string getAsStr(Attributor *A) const override {
5000 if (!getAssumedDereferenceableBytes())
5001 return "unknown-dereferenceable";
5002 bool IsKnownNonNull;
5003 bool IsAssumedNonNull = false;
5004 if (A)
5005 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5006 *A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
5007 return std::string("dereferenceable") +
5008 (IsAssumedNonNull ? "" : "_or_null") +
5009 (isAssumedGlobal() ? "_globally" : "") + "<" +
5010 std::to_string(getKnownDereferenceableBytes()) + "-" +
5011 std::to_string(getAssumedDereferenceableBytes()) + ">" +
5012 (!A ? " [non-null is unknown]" : "");
5013 }
5014};
5015
5016/// Dereferenceable attribute for a floating value.
5017struct AADereferenceableFloating : AADereferenceableImpl {
5018 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5019 : AADereferenceableImpl(IRP, A) {}
5020
5021 /// See AbstractAttribute::updateImpl(...).
5022 ChangeStatus updateImpl(Attributor &A) override {
5023 bool Stripped;
5024 bool UsedAssumedInformation = false;
5026 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
5027 AA::AnyScope, UsedAssumedInformation)) {
5028 Values.push_back({getAssociatedValue(), getCtxI()});
5029 Stripped = false;
5030 } else {
5031 Stripped = Values.size() != 1 ||
5032 Values.front().getValue() != &getAssociatedValue();
5033 }
5034
5035 const DataLayout &DL = A.getDataLayout();
5036 DerefState T;
5037
5038 auto VisitValueCB = [&](const Value &V) -> bool {
5039 unsigned IdxWidth =
5040 DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace());
5041 APInt Offset(IdxWidth, 0);
5043 A, *this, &V, DL, Offset, /* GetMinOffset */ false,
5044 /* AllowNonInbounds */ true);
5045
5046 const auto *AA = A.getAAFor<AADereferenceable>(
5047 *this, IRPosition::value(*Base), DepClassTy::REQUIRED);
5048 int64_t DerefBytes = 0;
5049 if (!AA || (!Stripped && this == AA)) {
5050 // Use IR information if we did not strip anything.
5051 // TODO: track globally.
5052 bool CanBeNull, CanBeFreed;
5053 DerefBytes =
5054 Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
5055 T.GlobalState.indicatePessimisticFixpoint();
5056 } else {
5057 const DerefState &DS = AA->getState();
5058 DerefBytes = DS.DerefBytesState.getAssumed();
5059 T.GlobalState &= DS.GlobalState;
5060 }
5061
5062 // For now we do not try to "increase" dereferenceability due to negative
5063 // indices as we first have to come up with code to deal with loops and
5064 // for overflows of the dereferenceable bytes.
5065 int64_t OffsetSExt = Offset.getSExtValue();
5066 if (OffsetSExt < 0)
5067 OffsetSExt = 0;
5068
5069 T.takeAssumedDerefBytesMinimum(
5070 std::max(int64_t(0), DerefBytes - OffsetSExt));
5071
5072 if (this == AA) {
5073 if (!Stripped) {
5074 // If nothing was stripped IR information is all we got.
5075 T.takeKnownDerefBytesMaximum(
5076 std::max(int64_t(0), DerefBytes - OffsetSExt));
5077 T.indicatePessimisticFixpoint();
5078 } else if (OffsetSExt > 0) {
5079 // If something was stripped but there is circular reasoning we look
5080 // for the offset. If it is positive we basically decrease the
5081 // dereferenceable bytes in a circular loop now, which will simply
5082 // drive them down to the known value in a very slow way which we
5083 // can accelerate.
5084 T.indicatePessimisticFixpoint();
5085 }
5086 }
5087
5088 return T.isValidState();
5089 };
5090
5091 for (const auto &VAC : Values)
5092 if (!VisitValueCB(*VAC.getValue()))
5093 return indicatePessimisticFixpoint();
5094
5095 return clampStateAndIndicateChange(getState(), T);
5096 }
5097
5098 /// See AbstractAttribute::trackStatistics()
5099 void trackStatistics() const override {
5100 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5101 }
5102};
5103
5104/// Dereferenceable attribute for a return value.
5105struct AADereferenceableReturned final
5106 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5107 using Base =
5108 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5109 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5110 : Base(IRP, A) {}
5111
5112 /// See AbstractAttribute::trackStatistics()
5113 void trackStatistics() const override {
5114 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5115 }
5116};
5117
5118/// Dereferenceable attribute for an argument
5119struct AADereferenceableArgument final
5120 : AAArgumentFromCallSiteArguments<AADereferenceable,
5121 AADereferenceableImpl> {
5122 using Base =
5123 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5124 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5125 : Base(IRP, A) {}
5126
5127 /// See AbstractAttribute::trackStatistics()
5128 void trackStatistics() const override {
5129 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5130 }
5131};
5132
5133/// Dereferenceable attribute for a call site argument.
5134struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5135 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5136 : AADereferenceableFloating(IRP, A) {}
5137
5138 /// See AbstractAttribute::trackStatistics()
5139 void trackStatistics() const override {
5140 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5141 }
5142};
5143
5144/// Dereferenceable attribute deduction for a call site return value.
5145struct AADereferenceableCallSiteReturned final
5146 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5147 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5148 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5149 : Base(IRP, A) {}
5150
5151 /// See AbstractAttribute::trackStatistics()
5152 void trackStatistics() const override {
5153 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5154 }
5155};
5156} // namespace
5157
5158// ------------------------ Align Argument Attribute ------------------------
5159
5160namespace {
5161static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5162 Value &AssociatedValue, const Use *U,
5163 const Instruction *I, bool &TrackUse) {
5164 // We need to follow common pointer manipulation uses to the accesses they
5165 // feed into.
5166 if (isa<CastInst>(I)) {
5167 // Follow all but ptr2int casts.
5168 TrackUse = !isa<PtrToIntInst>(I);
5169 return 0;
5170 }
5171 if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
5172 if (GEP->hasAllConstantIndices())
5173 TrackUse = true;
5174 return 0;
5175 }
5176
5177 MaybeAlign MA;
5178 if (const auto *CB = dyn_cast<CallBase>(I)) {
5179 if (CB->isBundleOperand(U) || CB->isCallee(U))
5180 return 0;
5181
5182 unsigned ArgNo = CB->getArgOperandNo(U);
5183 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
5184 // As long as we only use known information there is no need to track
5185 // dependences here.
5186 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE);
5187 if (AlignAA)
5188 MA = MaybeAlign(AlignAA->getKnownAlign());
5189 }
5190
5191 const DataLayout &DL = A.getDataLayout();
5192 const Value *UseV = U->get();
5193 if (auto *SI = dyn_cast<StoreInst>(I)) {
5194 if (SI->getPointerOperand() == UseV)
5195 MA = SI->getAlign();
5196 } else if (auto *LI = dyn_cast<LoadInst>(I)) {
5197 if (LI->getPointerOperand() == UseV)
5198 MA = LI->getAlign();
5199 } else if (auto *AI = dyn_cast<AtomicRMWInst>(I)) {
5200 if (AI->getPointerOperand() == UseV)
5201 MA = AI->getAlign();
5202 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
5203 if (AI->getPointerOperand() == UseV)
5204 MA = AI->getAlign();
5205 }
5206
5207 if (!MA || *MA <= QueryingAA.getKnownAlign())
5208 return 0;
5209
5210 unsigned Alignment = MA->value();
5211 int64_t Offset;
5212
5213 if (const Value *Base = GetPointerBaseWithConstantOffset(UseV, Offset, DL)) {
5214 if (Base == &AssociatedValue) {
5215 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5216 // So we can say that the maximum power of two which is a divisor of
5217 // gcd(Offset, Alignment) is an alignment.
5218
5219 uint32_t gcd = std::gcd(uint32_t(abs((int32_t)Offset)), Alignment);
5220 Alignment = llvm::bit_floor(gcd);
5221 }
5222 }
5223
5224 return Alignment;
5225}
5226
5227struct AAAlignImpl : AAAlign {
5228 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5229
5230 /// See AbstractAttribute::initialize(...).
5231 void initialize(Attributor &A) override {
5233 A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs);
5234 for (const Attribute &Attr : Attrs)
5235 takeKnownMaximum(Attr.getValueAsInt());
5236
5237 Value &V = *getAssociatedValue().stripPointerCasts();
5238 takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value());
5239
5240 if (Instruction *CtxI = getCtxI())
5241 followUsesInMBEC(*this, A, getState(), *CtxI);
5242 }
5243
5244 /// See AbstractAttribute::manifest(...).
5245 ChangeStatus manifest(Attributor &A) override {
5246 ChangeStatus LoadStoreChanged = ChangeStatus::UNCHANGED;
5247
5248 // Check for users that allow alignment annotations.
5249 Value &AssociatedValue = getAssociatedValue();
5250 for (const Use &U : AssociatedValue.uses()) {
5251 if (auto *SI = dyn_cast<StoreInst>(U.getUser())) {
5252 if (SI->getPointerOperand() == &AssociatedValue)
5253 if (SI->getAlign() < getAssumedAlign()) {
5254 STATS_DECLTRACK(AAAlign, Store,
5255 "Number of times alignment added to a store");
5256 SI->setAlignment(getAssumedAlign());
5257 LoadStoreChanged = ChangeStatus::CHANGED;
5258 }
5259 } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {
5260 if (LI->getPointerOperand() == &AssociatedValue)
5261 if (LI->getAlign() < getAssumedAlign()) {
5262 LI->setAlignment(getAssumedAlign());
5264 "Number of times alignment added to a load");
5265 LoadStoreChanged = ChangeStatus::CHANGED;
5266 }
5267 }
5268 }
5269
5270 ChangeStatus Changed = AAAlign::manifest(A);
5271
5272 Align InheritAlign =
5273 getAssociatedValue().getPointerAlignment(A.getDataLayout());
5274 if (InheritAlign >= getAssumedAlign())
5275 return LoadStoreChanged;
5276 return Changed | LoadStoreChanged;
5277 }
5278
5279 // TODO: Provide a helper to determine the implied ABI alignment and check in
5280 // the existing manifest method and a new one for AAAlignImpl that value
5281 // to avoid making the alignment explicit if it did not improve.
5282
5283 /// See AbstractAttribute::getDeducedAttributes
5284 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5285 SmallVectorImpl<Attribute> &Attrs) const override {
5286 if (getAssumedAlign() > 1)
5287 Attrs.emplace_back(
5288 Attribute::getWithAlignment(Ctx, Align(getAssumedAlign())));
5289 }
5290
5291 /// See followUsesInMBEC
5292 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5293 AAAlign::StateType &State) {
5294 bool TrackUse = false;
5295
5296 unsigned int KnownAlign =
5297 getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse);
5298 State.takeKnownMaximum(KnownAlign);
5299
5300 return TrackUse;
5301 }
5302
5303 /// See AbstractAttribute::getAsStr().
5304 const std::string getAsStr(Attributor *A) const override {
5305 return "align<" + std::to_string(getKnownAlign().value()) + "-" +
5306 std::to_string(getAssumedAlign().value()) + ">";
5307 }
5308};
5309
5310/// Align attribute for a floating value.
5311struct AAAlignFloating : AAAlignImpl {
5312 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5313
5314 /// See AbstractAttribute::updateImpl(...).
5315 ChangeStatus updateImpl(Attributor &A) override {
5316 const DataLayout &DL = A.getDataLayout();
5317
5318 bool Stripped;
5319 bool UsedAssumedInformation = false;
5321 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
5322 AA::AnyScope, UsedAssumedInformation)) {
5323 Values.push_back({getAssociatedValue(), getCtxI()});
5324 Stripped = false;
5325 } else {
5326 Stripped = Values.size() != 1 ||
5327 Values.front().getValue() != &getAssociatedValue();
5328 }
5329
5330 StateType T;
5331 auto VisitValueCB = [&](Value &V) -> bool {
5332 if (isa<UndefValue>(V) || isa<ConstantPointerNull>(V))
5333 return true;
5334 const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V),
5335 DepClassTy::REQUIRED);
5336 if (!AA || (!Stripped && this == AA)) {
5337 int64_t Offset;
5338 unsigned Alignment = 1;
5339 if (const Value *Base =
5341 // TODO: Use AAAlign for the base too.
5342 Align PA = Base->getPointerAlignment(DL);
5343 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5344 // So we can say that the maximum power of two which is a divisor of
5345 // gcd(Offset, Alignment) is an alignment.
5346
5347 uint32_t gcd =
5348 std::gcd(uint32_t(abs((int32_t)Offset)), uint32_t(PA.value()));
5349 Alignment = llvm::bit_floor(gcd);
5350 } else {
5351 Alignment = V.getPointerAlignment(DL).value();
5352 }
5353 // Use only IR information if we did not strip anything.
5354 T.takeKnownMaximum(Alignment);
5355 T.indicatePessimisticFixpoint();
5356 } else {
5357 // Use abstract attribute information.
5358 const AAAlign::StateType &DS = AA->getState();
5359 T ^= DS;
5360 }
5361 return T.isValidState();
5362 };
5363
5364 for (const auto &VAC : Values) {
5365 if (!VisitValueCB(*VAC.getValue()))
5366 return indicatePessimisticFixpoint();
5367 }
5368
5369 // TODO: If we know we visited all incoming values, thus no are assumed
5370 // dead, we can take the known information from the state T.
5371 return clampStateAndIndicateChange(getState(), T);
5372 }
5373
5374 /// See AbstractAttribute::trackStatistics()
5375 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5376};
5377
5378/// Align attribute for function return value.
5379struct AAAlignReturned final
5380 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5381 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5382 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5383
5384 /// See AbstractAttribute::trackStatistics()
5385 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5386};
5387
5388/// Align attribute for function argument.
5389struct AAAlignArgument final
5390 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5391 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5392 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5393
5394 /// See AbstractAttribute::manifest(...).
5395 ChangeStatus manifest(Attributor &A) override {
5396 // If the associated argument is involved in a must-tail call we give up
5397 // because we would need to keep the argument alignments of caller and
5398 // callee in-sync. Just does not seem worth the trouble right now.
5399 if (A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))
5400 return ChangeStatus::UNCHANGED;
5401 return Base::manifest(A);
5402 }
5403
5404 /// See AbstractAttribute::trackStatistics()
5405 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5406};
5407
5408struct AAAlignCallSiteArgument final : AAAlignFloating {
5409 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5410 : AAAlignFloating(IRP, A) {}
5411
5412 /// See AbstractAttribute::manifest(...).
5413 ChangeStatus manifest(Attributor &A) override {
5414 // If the associated argument is involved in a must-tail call we give up
5415 // because we would need to keep the argument alignments of caller and
5416 // callee in-sync. Just does not seem worth the trouble right now.
5417 if (Argument *Arg = getAssociatedArgument())
5418 if (A.getInfoCache().isInvolvedInMustTailCall(*Arg))
5419 return ChangeStatus::UNCHANGED;
5420 ChangeStatus Changed = AAAlignImpl::manifest(A);
5421 Align InheritAlign =
5422 getAssociatedValue().getPointerAlignment(A.getDataLayout());
5423 if (InheritAlign >= getAssumedAlign())
5424 Changed = ChangeStatus::UNCHANGED;
5425 return Changed;
5426 }
5427
5428 /// See AbstractAttribute::updateImpl(Attributor &A).
5429 ChangeStatus updateImpl(Attributor &A) override {
5430 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5431 if (Argument *Arg = getAssociatedArgument()) {
5432 // We only take known information from the argument
5433 // so we do not need to track a dependence.
5434 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5435 *this, IRPosition::argument(*Arg), DepClassTy::NONE);
5436 if (ArgAlignAA)
5437 takeKnownMaximum(ArgAlignAA->getKnownAlign().value());
5438 }
5439 return Changed;
5440 }
5441
5442 /// See AbstractAttribute::trackStatistics()
5443 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5444};
5445
5446/// Align attribute deduction for a call site return value.
5447struct AAAlignCallSiteReturned final
5448 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5449 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5450 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5451 : Base(IRP, A) {}
5452
5453 /// See AbstractAttribute::trackStatistics()
5454 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5455};
5456} // namespace
5457
5458/// ------------------ Function No-Return Attribute ----------------------------
5459namespace {
5460struct AANoReturnImpl : public AANoReturn {
5461 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5462
5463 /// See AbstractAttribute::initialize(...).
5464 void initialize(Attributor &A) override {
5465 bool IsKnown;
5466 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5467 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5468 (void)IsKnown;
5469 }
5470
5471 /// See AbstractAttribute::getAsStr().
5472 const std::string getAsStr(Attributor *A) const override {
5473 return getAssumed() ? "noreturn" : "may-return";
5474 }
5475
5476 /// See AbstractAttribute::updateImpl(Attributor &A).
5477 ChangeStatus updateImpl(Attributor &A) override {
5478 auto CheckForNoReturn = [](Instruction &) { return false; };
5479 bool UsedAssumedInformation = false;
5480 if (!A.checkForAllInstructions(CheckForNoReturn, *this,
5481 {(unsigned)Instruction::Ret},
5482 UsedAssumedInformation))
5483 return indicatePessimisticFixpoint();
5484 return ChangeStatus::UNCHANGED;
5485 }
5486};
5487
5488struct AANoReturnFunction final : AANoReturnImpl {
5489 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5490 : AANoReturnImpl(IRP, A) {}
5491
5492 /// See AbstractAttribute::trackStatistics()
5493 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5494};
5495
5496/// NoReturn attribute deduction for a call sites.
5497struct AANoReturnCallSite final
5498 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5499 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5500 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5501
5502 /// See AbstractAttribute::trackStatistics()
5503 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5504};
5505} // namespace
5506
5507/// ----------------------- Instance Info ---------------------------------
5508
5509namespace {
5510/// A class to hold the state of for no-capture attributes.
5511struct AAInstanceInfoImpl : public AAInstanceInfo {
5512 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5513 : AAInstanceInfo(IRP, A) {}
5514
5515 /// See AbstractAttribute::initialize(...).
5516 void initialize(Attributor &A) override {
5517 Value &V = getAssociatedValue();
5518 if (auto *C = dyn_cast<Constant>(&V)) {
5519 if (C->isThreadDependent())
5520 indicatePessimisticFixpoint();
5521 else
5522 indicateOptimisticFixpoint();
5523 return;
5524 }
5525 if (auto *CB = dyn_cast<CallBase>(&V))
5526 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5527 !CB->mayReadFromMemory()) {
5528 indicateOptimisticFixpoint();
5529 return;
5530 }
5531 if (auto *I = dyn_cast<Instruction>(&V)) {
5532 const auto *CI =
5533 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5534 *I->getFunction());
5535 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5536 indicatePessimisticFixpoint();
5537 return;
5538 }
5539 }
5540 }
5541
5542 /// See AbstractAttribute::updateImpl(...).
5543 ChangeStatus updateImpl(Attributor &A) override {
5544 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5545
5546 Value &V = getAssociatedValue();
5547 const Function *Scope = nullptr;
5548 if (auto *I = dyn_cast<Instruction>(&V))
5549 Scope = I->getFunction();
5550 if (auto *A = dyn_cast<Argument>(&V)) {
5551 Scope = A->getParent();
5552 if (!Scope->hasLocalLinkage())
5553 return Changed;
5554 }
5555 if (!Scope)
5556 return indicateOptimisticFixpoint();
5557
5558 bool IsKnownNoRecurse;
5559 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5560 A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL,
5561 IsKnownNoRecurse))
5562 return Changed;
5563
5564 auto UsePred = [&](const Use &U, bool &Follow) {
5565 const Instruction *UserI = dyn_cast<Instruction>(U.getUser());
5566 if (!UserI || isa<GetElementPtrInst>(UserI) || isa<CastInst>(UserI) ||
5567 isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
5568 Follow = true;
5569 return true;
5570 }
5571 if (isa<LoadInst>(UserI) || isa<CmpInst>(UserI) ||
5572 (isa<StoreInst>(UserI) &&
5573 cast<StoreInst>(UserI)->getValueOperand() != U.get()))
5574 return true;
5575 if (auto *CB = dyn_cast<CallBase>(UserI)) {
5576 // This check is not guaranteeing uniqueness but for now that we cannot
5577 // end up with two versions of \p U thinking it was one.
5578 auto *Callee = dyn_cast_if_present<Function>(CB->getCalledOperand());
5579 if (!Callee || !Callee->hasLocalLinkage())
5580 return true;
5581 if (!CB->isArgOperand(&U))
5582 return false;
5583 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5585 DepClassTy::OPTIONAL);
5586 if (!ArgInstanceInfoAA ||
5587 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5588 return false;
5589 // If this call base might reach the scope again we might forward the
5590 // argument back here. This is very conservative.
5592 A, *CB, *Scope, *this, /* ExclusionSet */ nullptr,
5593 [Scope](const Function &Fn) { return &Fn != Scope; }))
5594 return false;
5595 return true;
5596 }
5597 return false;
5598 };
5599
5600 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5601 if (auto *SI = dyn_cast<StoreInst>(OldU.getUser())) {
5602 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5603 if ((isa<AllocaInst>(Ptr) || isNoAliasCall(Ptr)) &&
5604 AA::isDynamicallyUnique(A, *this, *Ptr))
5605 return true;
5606 }
5607 return false;
5608 };
5609
5610 if (!A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ true,
5611 DepClassTy::OPTIONAL,
5612 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5613 return indicatePessimisticFixpoint();
5614
5615 return Changed;
5616 }
5617
5618 /// See AbstractState::getAsStr().
5619 const std::string getAsStr(Attributor *A) const override {
5620 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5621 }
5622
5623 /// See AbstractAttribute::trackStatistics()
5624 void trackStatistics() const override {}
5625};
5626
5627/// InstanceInfo attribute for floating values.
5628struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5629 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5630 : AAInstanceInfoImpl(IRP, A) {}
5631};
5632
5633/// NoCapture attribute for function arguments.
5634struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5635 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5636 : AAInstanceInfoFloating(IRP, A) {}
5637};
5638
5639/// InstanceInfo attribute for call site arguments.
5640struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5641 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5642 : AAInstanceInfoImpl(IRP, A) {}
5643
5644 /// See AbstractAttribute::updateImpl(...).
5645 ChangeStatus updateImpl(Attributor &A) override {
5646 // TODO: Once we have call site specific value information we can provide
5647 // call site specific liveness information and then it makes
5648 // sense to specialize attributes for call sites arguments instead of
5649 // redirecting requests to the callee argument.
5650 Argument *Arg = getAssociatedArgument();
5651 if (!Arg)
5652 return indicatePessimisticFixpoint();
5653 const IRPosition &ArgPos = IRPosition::argument(*Arg);
5654 auto *ArgAA =
5655 A.getAAFor<AAInstanceInfo>(*this, ArgPos, DepClassTy::REQUIRED);
5656 if (!ArgAA)
5657 return indicatePessimisticFixpoint();
5658 return clampStateAndIndicateChange(getState(), ArgAA->getState());
5659 }
5660};
5661
5662/// InstanceInfo attribute for function return value.
5663struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5664 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5665 : AAInstanceInfoImpl(IRP, A) {
5666 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5667 }
5668
5669 /// See AbstractAttribute::initialize(...).
5670 void initialize(Attributor &A) override {
5671 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5672 }
5673
5674 /// See AbstractAttribute::updateImpl(...).
5675 ChangeStatus updateImpl(Attributor &A) override {
5676 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5677 }
5678};
5679
5680/// InstanceInfo attribute deduction for a call site return value.
5681struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5682 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5683 : AAInstanceInfoFloating(IRP, A) {}
5684};
5685} // namespace
5686
5687/// ----------------------- Variable Capturing ---------------------------------
5689 Attribute::AttrKind ImpliedAttributeKind,
5690 bool IgnoreSubsumingPositions) {
5691 assert(ImpliedAttributeKind == Attribute::NoCapture &&
5692 "Unexpected attribute kind");
5693 Value &V = IRP.getAssociatedValue();
5694 if (!IRP.isArgumentPosition())
5695 return V.use_empty();
5696
5697 // You cannot "capture" null in the default address space.
5698 //
5699 // FIXME: This should use NullPointerIsDefined to account for the function
5700 // attribute.
5701 if (isa<UndefValue>(V) || (isa<ConstantPointerNull>(V) &&
5702 V.getType()->getPointerAddressSpace() == 0)) {
5703 return true;
5704 }
5705
5706 if (A.hasAttr(IRP, {Attribute::NoCapture},
5707 /* IgnoreSubsumingPositions */ true, Attribute::NoCapture))
5708 return true;
5709
5710 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5711 if (Argument *Arg = IRP.getAssociatedArgument())
5712 if (A.hasAttr(IRPosition::argument(*Arg),
5713 {Attribute::NoCapture, Attribute::ByVal},
5714 /* IgnoreSubsumingPositions */ true)) {
5715 A.manifestAttrs(IRP,
5716 Attribute::get(V.getContext(), Attribute::NoCapture));
5717 return true;
5718 }
5719
5720 if (const Function *F = IRP.getAssociatedFunction()) {
5721 // Check what state the associated function can actually capture.
5723 determineFunctionCaptureCapabilities(IRP, *F, State);
5724 if (State.isKnown(NO_CAPTURE)) {
5725 A.manifestAttrs(IRP,
5726 Attribute::get(V.getContext(), Attribute::NoCapture));
5727 return true;
5728 }
5729 }
5730
5731 return false;
5732}
5733
5734/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5735/// depending on the ability of the function associated with \p IRP to capture
5736/// state in memory and through "returning/throwing", respectively.
5738 const Function &F,
5739 BitIntegerState &State) {
5740 // TODO: Once we have memory behavior attributes we should use them here.
5741
5742 // If we know we cannot communicate or write to memory, we do not care about
5743 // ptr2int anymore.
5744 bool ReadOnly = F.onlyReadsMemory();
5745 bool NoThrow = F.doesNotThrow();
5746 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5747 if (ReadOnly && NoThrow && IsVoidReturn) {
5748 State.addKnownBits(NO_CAPTURE);
5749 return;
5750 }
5751
5752 // A function cannot capture state in memory if it only reads memory, it can
5753 // however return/throw state and the state might be influenced by the
5754 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5755 if (ReadOnly)
5756 State.addKnownBits(NOT_CAPTURED_IN_MEM);
5757
5758 // A function cannot communicate state back if it does not through
5759 // exceptions and doesn not return values.
5760 if (NoThrow && IsVoidReturn)
5761 State.addKnownBits(NOT_CAPTURED_IN_RET);
5762
5763 // Check existing "returned" attributes.
5764 int ArgNo = IRP.getCalleeArgNo();
5765 if (!NoThrow || ArgNo < 0 ||
5766 !F.getAttributes().hasAttrSomewhere(Attribute::Returned))
5767 return;
5768
5769 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5770 if (F.hasParamAttribute(U, Attribute::Returned)) {
5771 if (U == unsigned(ArgNo))
5772 State.removeAssumedBits(NOT_CAPTURED_IN_RET);
5773 else if (ReadOnly)
5774 State.addKnownBits(NO_CAPTURE);
5775 else
5776 State.addKnownBits(NOT_CAPTURED_IN_RET);
5777 break;
5778 }
5779}
5780
5781namespace {
5782/// A class to hold the state of for no-capture attributes.
5783struct AANoCaptureImpl : public AANoCapture {
5784 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5785
5786 /// See AbstractAttribute::initialize(...).
5787 void initialize(Attributor &A) override {
5788 bool IsKnown;
5789 assert(!AA::hasAssumedIRAttr<Attribute::NoCapture>(
5790 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5791 (void)IsKnown;
5792 }
5793
5794 /// See AbstractAttribute::updateImpl(...).
5795 ChangeStatus updateImpl(Attributor &A) override;
5796
5797 /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5798 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5799 SmallVectorImpl<Attribute> &Attrs) const override {
5800 if (!isAssumedNoCaptureMaybeReturned())
5801 return;
5802
5803 if (isArgumentPosition()) {
5804 if (isAssumedNoCapture())
5805 Attrs.emplace_back(Attribute::get(Ctx, Attribute::NoCapture));
5806 else if (ManifestInternal)
5807 Attrs.emplace_back(Attribute::get(Ctx, "no-capture-maybe-returned"));
5808 }
5809 }
5810
5811 /// See AbstractState::getAsStr().
5812 const std::string getAsStr(Attributor *A) const override {
5813 if (isKnownNoCapture())
5814 return "known not-captured";
5815 if (isAssumedNoCapture())
5816 return "assumed not-captured";
5817 if (isKnownNoCaptureMaybeReturned())
5818 return "known not-captured-maybe-returned";
5819 if (isAssumedNoCaptureMaybeReturned())
5820 return "assumed not-captured-maybe-returned";
5821 return "assumed-captured";
5822 }
5823
5824 /// Check the use \p U and update \p State accordingly. Return true if we
5825 /// should continue to update the state.
5826 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
5827 bool &Follow) {
5828 Instruction *UInst = cast<Instruction>(U.getUser());
5829 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
5830 << *UInst << "\n");
5831
5832 // Deal with ptr2int by following uses.
5833 if (isa<PtrToIntInst>(UInst)) {
5834 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
5835 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5836 /* Return */ true);
5837 }
5838
5839 // For stores we already checked if we can follow them, if they make it
5840 // here we give up.
5841 if (isa<StoreInst>(UInst))
5842 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5843 /* Return */ true);
5844
5845 // Explicitly catch return instructions.
5846 if (isa<ReturnInst>(UInst)) {
5847 if (UInst->getFunction() == getAnchorScope())
5848 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5849 /* Return */ true);
5850 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5851 /* Return */ true);
5852 }
5853
5854 // For now we only use special logic for call sites. However, the tracker
5855 // itself knows about a lot of other non-capturing cases already.
5856 auto *CB = dyn_cast<CallBase>(UInst);
5857 if (!CB || !CB->isArgOperand(&U))
5858 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5859 /* Return */ true);
5860
5861 unsigned ArgNo = CB->getArgOperandNo(&U);
5862 const IRPosition &CSArgPos = IRPosition::callsite_argument(*CB, ArgNo);
5863 // If we have a abstract no-capture attribute for the argument we can use
5864 // it to justify a non-capture attribute here. This allows recursion!
5865 bool IsKnownNoCapture;
5866 const AANoCapture *ArgNoCaptureAA = nullptr;
5867 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
5868 A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
5869 &ArgNoCaptureAA);
5870 if (IsAssumedNoCapture)
5871 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5872 /* Return */ false);
5873 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
5874 Follow = true;
5875 return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
5876 /* Return */ false);
5877 }
5878
5879 // Lastly, we could not find a reason no-capture can be assumed so we don't.
5880 return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
5881 /* Return */ true);
5882 }
5883
5884 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
5885 /// \p CapturedInRet, then return true if we should continue updating the
5886 /// state.
5887 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
5888 bool CapturedInInt, bool CapturedInRet) {
5889 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
5890 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
5891 if (CapturedInMem)
5892 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM);
5893 if (CapturedInInt)
5894 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT);
5895 if (CapturedInRet)
5896 State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET);
5897 return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
5898 }
5899};
5900
5901ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
5902 const IRPosition &IRP = getIRPosition();
5903 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
5904 : &IRP.getAssociatedValue();
5905 if (!V)
5906 return indicatePessimisticFixpoint();
5907
5908 const Function *F =
5909 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
5910
5911 // TODO: Is the checkForAllUses below useful for constants?
5912 if (!F)
5913 return indicatePessimisticFixpoint();
5914
5916 const IRPosition &FnPos = IRPosition::function(*F);
5917
5918 // Readonly means we cannot capture through memory.
5919 bool IsKnown;
5920 if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) {
5921 T.addKnownBits(NOT_CAPTURED_IN_MEM);
5922 if (IsKnown)
5923 addKnownBits(NOT_CAPTURED_IN_MEM);
5924 }
5925
5926 // Make sure all returned values are different than the underlying value.
5927 // TODO: we could do this in a more sophisticated way inside
5928 // AAReturnedValues, e.g., track all values that escape through returns
5929 // directly somehow.
5930 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
5932 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*F), this, Values,
5934 UsedAssumedInformation))
5935 return false;
5936 bool SeenConstant = false;
5937 for (const AA::ValueAndContext &VAC : Values) {
5938 if (isa<Constant>(VAC.getValue())) {
5939 if (SeenConstant)
5940 return false;
5941 SeenConstant = true;
5942 } else if (!isa<Argument>(VAC.getValue()) ||
5943 VAC.getValue() == getAssociatedArgument())
5944 return false;
5945 }
5946 return true;
5947 };
5948
5949 bool IsKnownNoUnwind;
5950 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
5951 A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
5952 bool IsVoidTy = F->getReturnType()->isVoidTy();
5953 bool UsedAssumedInformation = false;
5954 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
5955 T.addKnownBits(NOT_CAPTURED_IN_RET);
5956 if (T.isKnown(NOT_CAPTURED_IN_MEM))
5958 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
5959 addKnownBits(NOT_CAPTURED_IN_RET);
5960 if (isKnown(NOT_CAPTURED_IN_MEM))
5961 return indicateOptimisticFixpoint();
5962 }
5963 }
5964 }
5965
5966 auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
5967 const auto *DerefAA = A.getAAFor<AADereferenceable>(
5969 return DerefAA && DerefAA->getAssumedDereferenceableBytes();
5970 };
5971
5972 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
5973 switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
5975 return true;
5977 return checkUse(A, T, U, Follow);
5979 Follow = true;
5980 return true;
5981 }
5982 llvm_unreachable("Unexpected use capture kind!");
5983 };
5984
5985 if (!A.checkForAllUses(UseCheck, *this, *V))
5986 return indicatePessimisticFixpoint();
5987
5988 AANoCapture::StateType &S = getState();
5989 auto Assumed = S.getAssumed();
5990 S.intersectAssumedBits(T.getAssumed());
5991 if (!isAssumedNoCaptureMaybeReturned())
5992 return indicatePessimisticFixpoint();
5993 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
5995}
5996
5997/// NoCapture attribute for function arguments.
5998struct AANoCaptureArgument final : AANoCaptureImpl {
5999 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6000 : AANoCaptureImpl(IRP, A) {}
6001
6002 /// See AbstractAttribute::trackStatistics()
6003 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6004};
6005
6006/// NoCapture attribute for call site arguments.
6007struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6008 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6009 : AANoCaptureImpl(IRP, A) {}
6010
6011 /// See AbstractAttribute::updateImpl(...).
6012 ChangeStatus updateImpl(Attributor &A) override {
6013 // TODO: Once we have call site specific value information we can provide
6014 // call site specific liveness information and then it makes
6015 // sense to specialize attributes for call sites arguments instead of
6016 // redirecting requests to the callee argument.
6017 Argument *Arg = getAssociatedArgument();
6018 if (!Arg)
6019 return indicatePessimisticFixpoint();
6020 const IRPosition &ArgPos = IRPosition::argument(*Arg);
6021 bool IsKnownNoCapture;
6022 const AANoCapture *ArgAA = nullptr;
6023 if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
6024 A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
6025 &ArgAA))
6026 return ChangeStatus::UNCHANGED;
6027 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6028 return indicatePessimisticFixpoint();
6029 return clampStateAndIndicateChange(getState(), ArgAA->getState());
6030 }
6031
6032 /// See AbstractAttribute::trackStatistics()
6033 void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nocapture)};
6034};
6035
6036/// NoCapture attribute for floating values.
6037struct AANoCaptureFloating final : AANoCaptureImpl {
6038 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6039 : AANoCaptureImpl(IRP, A) {}
6040
6041 /// See AbstractAttribute::trackStatistics()
6042 void trackStatistics() const override {
6044 }
6045};
6046
6047/// NoCapture attribute for function return value.
6048struct AANoCaptureReturned final : AANoCaptureImpl {
6049 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6050 : AANoCaptureImpl(IRP, A) {
6051 llvm_unreachable("NoCapture is not applicable to function returns!");
6052 }
6053
6054 /// See AbstractAttribute::initialize(...).
6055 void initialize(Attributor &A) override {
6056 llvm_unreachable("NoCapture is not applicable to function returns!");
6057 }
6058
6059 /// See AbstractAttribute::updateImpl(...).
6060 ChangeStatus updateImpl(Attributor &A) override {
6061 llvm_unreachable("NoCapture is not applicable to function returns!");
6062 }
6063
6064 /// See AbstractAttribute::trackStatistics()
6065 void trackStatistics() const override {}
6066};
6067
6068/// NoCapture attribute deduction for a call site return value.
6069struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6070 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6071 : AANoCaptureImpl(IRP, A) {}
6072
6073 /// See AbstractAttribute::initialize(...).
6074 void initialize(Attributor &A) override {
6075 const Function *F = getAnchorScope();
6076 // Check what state the associated function can actually capture.
6077 determineFunctionCaptureCapabilities(getIRPosition(), *F, *this);
6078 }
6079
6080 /// See AbstractAttribute::trackStatistics()
6081 void trackStatistics() const override {
6083 }
6084};
6085} // namespace
6086
6087/// ------------------ Value Simplify Attribute ----------------------------
6088
6089bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6090 // FIXME: Add a typecast support.
6091 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6092 SimplifiedAssociatedValue, Other, Ty);
6093 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6094 return false;
6095
6096 LLVM_DEBUG({
6097 if (SimplifiedAssociatedValue)
6098 dbgs() << "[ValueSimplify] is assumed to be "
6099 << **SimplifiedAssociatedValue << "\n";
6100 else
6101 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6102 });
6103 return true;
6104}
6105
6106namespace {
6107struct AAValueSimplifyImpl : AAValueSimplify {
6108 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6109 : AAValueSimplify(IRP, A) {}
6110
6111 /// See AbstractAttribute::initialize(...).
6112 void initialize(Attributor &A) override {
6113 if (getAssociatedValue().getType()->isVoidTy())
6114 indicatePessimisticFixpoint();
6115 if (A.hasSimplificationCallback(getIRPosition()))
6116 indicatePessimisticFixpoint();
6117 }
6118
6119 /// See AbstractAttribute::getAsStr().
6120 const std::string getAsStr(Attributor *A) const override {
6121 LLVM_DEBUG({
6122 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6123 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6124 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6125 });
6126 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6127 : "not-simple";
6128 }
6129
6130 /// See AbstractAttribute::trackStatistics()
6131 void trackStatistics() const override {}
6132
6133 /// See AAValueSimplify::getAssumedSimplifiedValue()
6134 std::optional<Value *>
6135 getAssumedSimplifiedValue(Attributor &A) const override {
6136 return SimplifiedAssociatedValue;
6137 }
6138
6139 /// Ensure the return value is \p V with type \p Ty, if not possible return
6140 /// nullptr. If \p Check is true we will only verify such an operation would
6141 /// suceed and return a non-nullptr value if that is the case. No IR is
6142 /// generated or modified.
6143 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6144 bool Check) {
6145 if (auto *TypedV = AA::getWithType(V, Ty))
6146 return TypedV;
6147 if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty))
6148 return Check ? &V
6149 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6150 &V, &Ty, "", CtxI->getIterator());
6151 return nullptr;
6152 }
6153
6154 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6155 /// If \p Check is true we will only verify such an operation would suceed and
6156 /// return a non-nullptr value if that is the case. No IR is generated or
6157 /// modified.
6158 static Value *reproduceInst(Attributor &A,
6159 const AbstractAttribute &QueryingAA,
6160 Instruction &I, Type &Ty, Instruction *CtxI,
6161 bool Check, ValueToValueMapTy &VMap) {
6162 assert(CtxI && "Cannot reproduce an instruction without context!");
6163 if (Check && (I.mayReadFromMemory() ||
6164 !isSafeToSpeculativelyExecute(&I, CtxI, /* DT */ nullptr,
6165 /* TLI */ nullptr)))
6166 return nullptr;
6167 for (Value *Op : I.operands()) {
6168 Value *NewOp = reproduceValue(A, QueryingAA, *Op, Ty, CtxI, Check, VMap);
6169 if (!NewOp) {
6170 assert(Check && "Manifest of new value unexpectedly failed!");
6171 return nullptr;
6172 }
6173 if (!Check)
6174 VMap[Op] = NewOp;
6175 }
6176 if (Check)
6177 return &I;
6178
6179 Instruction *CloneI = I.clone();
6180 // TODO: Try to salvage debug information here.
6181 CloneI->setDebugLoc(DebugLoc());
6182 VMap[&I] = CloneI;
6183 CloneI->insertBefore(CtxI);
6184 RemapInstruction(CloneI, VMap);
6185 return CloneI;
6186 }
6187
6188 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6189 /// If \p Check is true we will only verify such an operation would suceed and
6190 /// return a non-nullptr value if that is the case. No IR is generated or
6191 /// modified.
6192 static Value *reproduceValue(Attributor &A,
6193 const AbstractAttribute &QueryingAA, Value &V,
6194 Type &Ty, Instruction *CtxI, bool Check,
6195 ValueToValueMapTy &VMap) {
6196 if (const auto &NewV = VMap.lookup(&V))
6197 return NewV;
6198 bool UsedAssumedInformation = false;
6199 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6200 V, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
6201 if (!SimpleV.has_value())
6202 return PoisonValue::get(&Ty);
6203 Value *EffectiveV = &V;
6204 if (*SimpleV)
6205 EffectiveV = *SimpleV;
6206 if (auto *C = dyn_cast<Constant>(EffectiveV))
6207 return C;
6208 if (CtxI && AA::isValidAtPosition(AA::ValueAndContext(*EffectiveV, *CtxI),
6209 A.getInfoCache()))
6210 return ensureType(A, *EffectiveV, Ty, CtxI, Check);
6211 if (auto *I = dyn_cast<Instruction>(EffectiveV))
6212 if (Value *NewV = reproduceInst(A, QueryingAA, *I, Ty, CtxI, Check, VMap))
6213 return ensureType(A, *NewV, Ty, CtxI, Check);
6214 return nullptr;
6215 }
6216
6217 /// Return a value we can use as replacement for the associated one, or
6218 /// nullptr if we don't have one that makes sense.
6219 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6220 Value *NewV = SimplifiedAssociatedValue
6221 ? *SimplifiedAssociatedValue
6222 : UndefValue::get(getAssociatedType());
6223 if (NewV && NewV != &getAssociatedValue()) {
6224 ValueToValueMapTy VMap;
6225 // First verify we can reprduce the value with the required type at the
6226 // context location before we actually start modifying the IR.
6227 if (reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
6228 /* CheckOnly */ true, VMap))
6229 return reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
6230 /* CheckOnly */ false, VMap);
6231 }
6232 return nullptr;
6233 }
6234
6235 /// Helper function for querying AAValueSimplify and updating candidate.
6236 /// \param IRP The value position we are trying to unify with SimplifiedValue
6237 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6238 const IRPosition &IRP, bool Simplify = true) {
6239 bool UsedAssumedInformation = false;
6240 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6241 if (Simplify)
6242 QueryingValueSimplified = A.getAssumedSimplified(
6243 IRP, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
6244 return unionAssumed(QueryingValueSimplified);
6245 }
6246
6247 /// Returns a candidate is found or not
6248 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6249 if (!getAssociatedValue().getType()->isIntegerTy())
6250 return false;
6251
6252 // This will also pass the call base context.
6253 const auto *AA =
6254 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6255 if (!AA)
6256 return false;
6257
6258 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6259
6260 if (!COpt) {
6261 SimplifiedAssociatedValue = std::nullopt;
6262 A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
6263 return true;
6264 }
6265 if (auto *C = *COpt) {
6266 SimplifiedAssociatedValue = C;
6267 A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
6268 return true;
6269 }
6270 return false;
6271 }
6272
6273 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6274 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6275 return true;
6276 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6277 return true;
6278 return false;
6279 }
6280
6281 /// See AbstractAttribute::manifest(...).
6282 ChangeStatus manifest(Attributor &A) override {
6283 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6284 for (auto &U : getAssociatedValue().uses()) {
6285 // Check if we need to adjust the insertion point to make sure the IR is
6286 // valid.
6287 Instruction *IP = dyn_cast<Instruction>(U.getUser());
6288 if (auto *PHI = dyn_cast_or_null<PHINode>(IP))
6289 IP = PHI->getIncomingBlock(U)->getTerminator();
6290 if (auto *NewV = manifestReplacementValue(A, IP)) {
6291 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6292 << " -> " << *NewV << " :: " << *this << "\n");
6293 if (A.changeUseAfterManifest(U, *NewV))
6294 Changed = ChangeStatus::CHANGED;
6295 }
6296 }
6297
6298 return Changed | AAValueSimplify::manifest(A);
6299 }
6300
6301 /// See AbstractState::indicatePessimisticFixpoint(...).
6302 ChangeStatus indicatePessimisticFixpoint() override {
6303 SimplifiedAssociatedValue = &getAssociatedValue();
6304 return AAValueSimplify::indicatePessimisticFixpoint();
6305 }
6306};
6307
6308struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6309 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6310 : AAValueSimplifyImpl(IRP, A) {}
6311
6312 void initialize(Attributor &A) override {
6313 AAValueSimplifyImpl::initialize(A);
6314 if (A.hasAttr(getIRPosition(),
6315 {Attribute::InAlloca, Attribute::Preallocated,
6316 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6317 /* IgnoreSubsumingPositions */ true))
6318 indicatePessimisticFixpoint();
6319 }
6320
6321 /// See AbstractAttribute::updateImpl(...).
6322 ChangeStatus updateImpl(Attributor &A) override {
6323 // Byval is only replacable if it is readonly otherwise we would write into
6324 // the replaced value and not the copy that byval creates implicitly.
6325 Argument *Arg = getAssociatedArgument();
6326 if (Arg->hasByValAttr()) {
6327 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6328 // there is no race by not copying a constant byval.
6329 bool IsKnown;
6330 if (!AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
6331 return indicatePessimisticFixpoint();
6332 }
6333
6334 auto Before = SimplifiedAssociatedValue;
6335
6336 auto PredForCallSite = [&](AbstractCallSite ACS) {
6337 const IRPosition &ACSArgPos =
6338 IRPosition::callsite_argument(ACS, getCallSiteArgNo());
6339 // Check if a coresponding argument was found or if it is on not
6340 // associated (which can happen for callback calls).
6341 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6342 return false;
6343
6344 // Simplify the argument operand explicitly and check if the result is
6345 // valid in the current scope. This avoids refering to simplified values
6346 // in other functions, e.g., we don't want to say a an argument in a
6347 // static function is actually an argument in a different function.
6348 bool UsedAssumedInformation = false;
6349 std::optional<Constant *> SimpleArgOp =
6350 A.getAssumedConstant(ACSArgPos, *this, UsedAssumedInformation);
6351 if (!SimpleArgOp)
6352 return true;
6353 if (!*SimpleArgOp)
6354 return false;
6355 if (!AA::isDynamicallyUnique(A, *this, **SimpleArgOp))
6356 return false;
6357 return unionAssumed(*SimpleArgOp);
6358 };
6359
6360 // Generate a answer specific to a call site context.
6361 bool Success;
6362 bool UsedAssumedInformation = false;
6363 if (hasCallBaseContext() &&
6364 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6365 Success = PredForCallSite(
6366 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6367 else
6368 Success = A.checkForAllCallSites(PredForCallSite, *this, true,
6369 UsedAssumedInformation);
6370
6371 if (!Success)
6372 if (!askSimplifiedValueForOtherAAs(A))
6373 return indicatePessimisticFixpoint();
6374
6375 // If a candidate was found in this update, return CHANGED.
6376 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6377 : ChangeStatus ::CHANGED;
6378 }
6379
6380 /// See AbstractAttribute::trackStatistics()
6381 void trackStatistics() const override {
6382 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6383 }
6384};
6385
6386struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6387 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6388 : AAValueSimplifyImpl(IRP, A) {}
6389
6390 /// See AAValueSimplify::getAssumedSimplifiedValue()
6391 std::optional<Value *>
6392 getAssumedSimplifiedValue(Attributor &A) const override {
6393 if (!isValidState())
6394 return nullptr;
6395 return SimplifiedAssociatedValue;
6396 }
6397
6398 /// See AbstractAttribute::updateImpl(...).
6399 ChangeStatus updateImpl(Attributor &A) override {
6400 auto Before = SimplifiedAssociatedValue;
6401
6402 auto ReturnInstCB = [&](Instruction &I) {
6403 auto &RI = cast<ReturnInst>(I);
6404 return checkAndUpdate(
6405 A, *this,
6406 IRPosition::value(*RI.getReturnValue(), getCallBaseContext()));
6407 };
6408
6409 bool UsedAssumedInformation = false;
6410 if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret},
6411 UsedAssumedInformation))
6412 if (!askSimplifiedValueForOtherAAs(A))
6413 return indicatePessimisticFixpoint();
6414
6415 // If a candidate was found in this update, return CHANGED.
6416 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6417 : ChangeStatus ::CHANGED;
6418 }
6419
6420 ChangeStatus manifest(Attributor &A) override {
6421 // We queried AAValueSimplify for the returned values so they will be
6422 // replaced if a simplified form was found. Nothing to do here.
6423 return ChangeStatus::UNCHANGED;
6424 }
6425
6426 /// See AbstractAttribute::trackStatistics()
6427 void trackStatistics() const override {
6428 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6429 }
6430};
6431
6432struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6433 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6434 : AAValueSimplifyImpl(IRP, A) {}
6435
6436 /// See AbstractAttribute::initialize(...).
6437 void initialize(Attributor &A) override {
6438 AAValueSimplifyImpl::initialize(A);
6439 Value &V = getAnchorValue();
6440
6441 // TODO: add other stuffs
6442 if (isa<Constant>(V))
6443 indicatePessimisticFixpoint();
6444 }
6445
6446 /// See AbstractAttribute::updateImpl(...).
6447 ChangeStatus updateImpl(Attributor &A) override {
6448 auto Before = SimplifiedAssociatedValue;
6449 if (!askSimplifiedValueForOtherAAs(A))
6450 return indicatePessimisticFixpoint();
6451
6452 // If a candidate was found in this update, return CHANGED.
6453 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6454 : ChangeStatus ::CHANGED;
6455 }
6456
6457 /// See AbstractAttribute::trackStatistics()
6458 void trackStatistics() const override {
6459 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6460 }
6461};
6462
6463struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6464 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6465 : AAValueSimplifyImpl(IRP, A) {}
6466
6467 /// See AbstractAttribute::initialize(...).
6468 void initialize(Attributor &A) override {
6469 SimplifiedAssociatedValue = nullptr;
6470 indicateOptimisticFixpoint();
6471 }
6472 /// See AbstractAttribute::initialize(...).
6473 ChangeStatus updateImpl(Attributor &A) override {
6475 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6476 }
6477 /// See AbstractAttribute::trackStatistics()
6478 void trackStatistics() const override {
6479 STATS_DECLTRACK_FN_ATTR(value_simplify)
6480 }
6481};
6482
6483struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6484 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6485 : AAValueSimplifyFunction(IRP, A) {}
6486 /// See AbstractAttribute::trackStatistics()
6487 void trackStatistics() const override {
6488 STATS_DECLTRACK_CS_ATTR(value_simplify)
6489 }
6490};
6491
6492struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6493 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6494 : AAValueSimplifyImpl(IRP, A) {}
6495
6496 void initialize(Attributor &A) override {
6497 AAValueSimplifyImpl::initialize(A);
6498 Function *Fn = getAssociatedFunction();
6499 assert(Fn && "Did expect an associted function");
6500 for (Argument &Arg : Fn->args()) {
6501 if (Arg.hasReturnedAttr()) {
6502 auto IRP = IRPosition::callsite_argument(*cast<CallBase>(getCtxI()),
6503 Arg.getArgNo());
6505 checkAndUpdate(A, *this, IRP))
6506 indicateOptimisticFixpoint();
6507 else
6508 indicatePessimisticFixpoint();
6509 return;
6510 }
6511 }
6512 }
6513
6514 /// See AbstractAttribute::updateImpl(...).
6515 ChangeStatus updateImpl(Attributor &A) override {
6516 return indicatePessimisticFixpoint();
6517 }
6518
6519 void trackStatistics() const override {
6520 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6521 }
6522};
6523
6524struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6525 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6526 : AAValueSimplifyFloating(IRP, A) {}
6527
6528 /// See AbstractAttribute::manifest(...).
6529 ChangeStatus manifest(Attributor &A) override {
6530 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6531 // TODO: We should avoid simplification duplication to begin with.
6532 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6533 IRPosition::value(getAssociatedValue()), this, DepClassTy::NONE);
6534 if (FloatAA && FloatAA->getState().isValidState())
6535 return Changed;
6536
6537 if (auto *NewV = manifestReplacementValue(A, getCtxI())) {
6538 Use &U = cast<CallBase>(&getAnchorValue())
6539 ->getArgOperandUse(getCallSiteArgNo());
6540 if (A.changeUseAfterManifest(U, *NewV))
6541 Changed = ChangeStatus::CHANGED;
6542 }
6543
6544 return Changed | AAValueSimplify::manifest(A);
6545 }
6546
6547 void trackStatistics() const override {
6548 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6549 }
6550};
6551} // namespace
6552
6553/// ----------------------- Heap-To-Stack Conversion ---------------------------
6554namespace {
6555struct AAHeapToStackFunction final : public AAHeapToStack {
6556
6557 struct AllocationInfo {
6558 /// The call that allocates the memory.
6559 CallBase *const CB;
6560
6561 /// The library function id for the allocation.
6562 LibFunc LibraryFunctionId = NotLibFunc;
6563
6564 /// The status wrt. a rewrite.
6565 enum {
6566 STACK_DUE_TO_USE,
6567 STACK_DUE_TO_FREE,
6568 INVALID,
6569 } Status = STACK_DUE_TO_USE;
6570
6571 /// Flag to indicate if we encountered a use that might free this allocation
6572 /// but which is not in the deallocation infos.
6573 bool HasPotentiallyFreeingUnknownUses = false;
6574
6575 /// Flag to indicate that we should place the new alloca in the function
6576 /// entry block rather than where the call site (CB) is.
6577 bool MoveAllocaIntoEntry = true;
6578
6579 /// The set of free calls that use this allocation.
6580 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6581 };
6582
6583 struct DeallocationInfo {
6584 /// The call that deallocates the memory.
6585 CallBase *const CB;
6586 /// The value freed by the call.
6587 Value *FreedOp;
6588
6589 /// Flag to indicate if we don't know all objects this deallocation might
6590 /// free.
6591 bool MightFreeUnknownObjects = false;
6592
6593 /// The set of allocation calls that are potentially freed.
6594 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6595 };
6596
6597 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6598 : AAHeapToStack(IRP, A) {}
6599
6600 ~AAHeapToStackFunction() {
6601 // Ensure we call the destructor so we release any memory allocated in the
6602 // sets.
6603 for (auto &It : AllocationInfos)
6604 It.second->~AllocationInfo();
6605 for (auto &It : DeallocationInfos)
6606 It.second->~DeallocationInfo();
6607 }
6608
6609 void initialize(Attributor &A) override {
6610 AAHeapToStack::initialize(A);
6611
6612 const Function *F = getAnchorScope();
6613 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6614
6615 auto AllocationIdentifierCB = [&](Instruction &I) {
6616 CallBase *CB = dyn_cast<CallBase>(&I);
6617 if (!CB)
6618 return true;
6619 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6620 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{CB, FreedOp};
6621 return true;
6622 }
6623 // To do heap to stack, we need to know that the allocation itself is
6624 // removable once uses are rewritten, and that we can initialize the
6625 // alloca to the same pattern as the original allocation result.
6626 if (isRemovableAlloc(CB, TLI)) {
6627 auto *I8Ty = Type::getInt8Ty(CB->getParent()->getContext());
6628 if (nullptr != getInitialValueOfAllocation(CB, TLI, I8Ty)) {
6629 AllocationInfo *AI = new (A.Allocator) AllocationInfo{CB};
6630 AllocationInfos[CB] = AI;
6631 if (TLI)
6632 TLI->getLibFunc(*CB, AI->LibraryFunctionId);
6633 }
6634 }
6635 return true;
6636 };
6637
6638 bool UsedAssumedInformation = false;
6639 bool Success = A.checkForAllCallLikeInstructions(
6640 AllocationIdentifierCB, *this, UsedAssumedInformation,
6641 /* CheckBBLivenessOnly */ false,
6642 /* CheckPotentiallyDead */ true);
6643 (void)Success;
6644 assert(Success && "Did not expect the call base visit callback to fail!");
6645
6647 [](const IRPosition &, const AbstractAttribute *,
6648 bool &) -> std::optional<Value *> { return nullptr; };
6649 for (const auto &It : AllocationInfos)
6650 A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
6651 SCB);
6652 for (const auto &It : DeallocationInfos)
6653 A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
6654 SCB);
6655 }
6656
6657 const std::string getAsStr(Attributor *A) const override {
6658 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6659 for (const auto &It : AllocationInfos) {
6660 if (It.second->Status == AllocationInfo::INVALID)
6661 ++NumInvalidMallocs;
6662 else
6663 ++NumH2SMallocs;
6664 }
6665 return "[H2S] Mallocs Good/Bad: " + std::to_string(NumH2SMallocs) + "/" +
6666 std::to_string(NumInvalidMallocs);
6667 }
6668
6669 /// See AbstractAttribute::trackStatistics().
6670 void trackStatistics() const override {
6671 STATS_DECL(
6672 MallocCalls, Function,
6673 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6674 for (const auto &It : AllocationInfos)
6675 if (It.second->Status != AllocationInfo::INVALID)
6676 ++BUILD_STAT_NAME(MallocCalls, Function);
6677 }
6678
6679 bool isAssumedHeapToStack(const CallBase &CB) const override {
6680 if (isValidState())
6681 if (AllocationInfo *AI =
6682 AllocationInfos.lookup(const_cast<CallBase *>(&CB)))
6683 return AI->Status != AllocationInfo::INVALID;
6684 return false;
6685 }
6686
6687 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6688 if (!isValidState())
6689 return false;
6690
6691 for (const auto &It : AllocationInfos) {
6692 AllocationInfo &AI = *It.second;
6693 if (AI.Status == AllocationInfo::INVALID)
6694 continue;
6695
6696 if (AI.PotentialFreeCalls.count(&CB))
6697 return true;
6698 }
6699
6700 return false;
6701 }
6702
6703 ChangeStatus manifest(Attributor &A) override {
6704 assert(getState().isValidState() &&
6705 "Attempted to manifest an invalid state!");
6706
6707 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6708 Function *F = getAnchorScope();
6709 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6710
6711 for (auto &It : AllocationInfos) {
6712 AllocationInfo &AI = *It.second;
6713 if (AI.Status == AllocationInfo::INVALID)
6714 continue;
6715
6716 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6717 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6718 A.deleteAfterManifest(*FreeCall);
6719 HasChanged = ChangeStatus::CHANGED;
6720 }
6721
6722 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6723 << "\n");
6724
6725 auto Remark = [&](OptimizationRemark OR) {
6726 LibFunc IsAllocShared;
6727 if (TLI->getLibFunc(*AI.CB, IsAllocShared))
6728 if (IsAllocShared == LibFunc___kmpc_alloc_shared)
6729 return OR << "Moving globalized variable to the stack.";
6730 return OR << "Moving memory allocation from the heap to the stack.";
6731 };
6732 if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
6733 A.emitRemark<OptimizationRemark>(AI.CB, "OMP110", Remark);
6734 else
6735 A.emitRemark<OptimizationRemark>(AI.CB, "HeapToStack", Remark);
6736
6737 const DataLayout &DL = A.getInfoCache().getDL();
6738 Value *Size;
6739 std::optional<APInt> SizeAPI = getSize(A, *this, AI);
6740 if (SizeAPI) {
6741 Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);
6742 } else {
6743 LLVMContext &Ctx = AI.CB->getContext();
6744 ObjectSizeOpts Opts;
6745 ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
6746 SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);
6747 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6748 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6749 Size = SizeOffsetPair.Size;
6750 }
6751
6752 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6753 ? F->getEntryBlock().begin()
6754 : AI.CB->getIterator();
6755
6756 Align Alignment(1);
6757 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6758 Alignment = std::max(Alignment, *RetAlign);
6759 if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
6760 std::optional<APInt> AlignmentAPI = getAPInt(A, *this, *Align);
6761 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6762 "Expected an alignment during manifest!");
6763 Alignment =
6764 std::max(Alignment, assumeAligned(AlignmentAPI->getZExtValue()));
6765 }
6766
6767 // TODO: Hoist the alloca towards the function entry.
6768 unsigned AS = DL.getAllocaAddrSpace();
6769 Instruction *Alloca =
6770 new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment,
6771 AI.CB->getName() + ".h2s", IP);
6772
6773 if (Alloca->getType() != AI.CB->getType())
6774 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6775 Alloca, AI.CB->getType(), "malloc_cast", AI.CB->getIterator());
6776
6777 auto *I8Ty = Type::getInt8Ty(F->getContext());
6778 auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty);
6779 assert(InitVal &&
6780 "Must be able to materialize initial memory state of allocation");
6781
6782 A.changeAfterManifest(IRPosition::inst(*AI.CB), *Alloca);
6783
6784 if (auto *II = dyn_cast<InvokeInst>(AI.CB)) {
6785 auto *NBB = II->getNormalDest();
6786 BranchInst::Create(NBB, AI.CB->getParent());
6787 A.deleteAfterManifest(*AI.CB);
6788 } else {
6789 A.deleteAfterManifest(*AI.CB);
6790 }
6791
6792 // Initialize the alloca with the same value as used by the allocation
6793 // function. We can skip undef as the initial value of an alloc is
6794 // undef, and the memset would simply end up being DSEd.
6795 if (!isa<UndefValue>(InitVal)) {
6796 IRBuilder<> Builder(Alloca->getNextNode());
6797 // TODO: Use alignment above if align!=1
6798 Builder.CreateMemSet(Alloca, InitVal, Size, std::nullopt);
6799 }
6800 HasChanged = ChangeStatus::CHANGED;
6801 }
6802
6803 return HasChanged;
6804 }
6805
6806 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6807 Value &V) {
6808 bool UsedAssumedInformation = false;
6809 std::optional<Constant *> SimpleV =
6810 A.getAssumedConstant(V, AA, UsedAssumedInformation);
6811 if (!SimpleV)
6812 return APInt(64, 0);
6813 if (auto *CI = dyn_cast_or_null<ConstantInt>(*SimpleV))
6814 return CI->getValue();
6815 return std::nullopt;
6816 }
6817
6818 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
6819 AllocationInfo &AI) {
6820 auto Mapper = [&](const Value *V) -> const Value * {
6821 bool UsedAssumedInformation = false;
6822 if (std::optional<Constant *> SimpleV =
6823 A.getAssumedConstant(*V, AA, UsedAssumedInformation))
6824 if (*SimpleV)
6825 return *SimpleV;
6826 return V;
6827 };
6828
6829 const Function *F = getAnchorScope();
6830 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6831 return getAllocSize(AI.CB, TLI, Mapper);
6832 }
6833
6834 /// Collection of all malloc-like calls in a function with associated
6835 /// information.
6837
6838 /// Collection of all free-like calls in a function with associated
6839 /// information.
6841
6842 ChangeStatus updateImpl(Attributor &A) override;
6843};
6844
6845ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
6847 const Function *F = getAnchorScope();
6848 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
6849
6850 const auto *LivenessAA =
6851 A.getAAFor<AAIsDead>(*this, IRPosition::function(*F), DepClassTy::NONE);
6852
6854 A.getInfoCache().getMustBeExecutedContextExplorer();
6855
6856 bool StackIsAccessibleByOtherThreads =
6857 A.getInfoCache().stackIsAccessibleByOtherThreads();
6858
6859 LoopInfo *LI =
6860 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F);
6861 std::optional<bool> MayContainIrreducibleControl;
6862 auto IsInLoop = [&](BasicBlock &BB) {
6863 if (&F->getEntryBlock() == &BB)
6864 return false;
6865 if (!MayContainIrreducibleControl.has_value())
6866 MayContainIrreducibleControl = mayContainIrreducibleControl(*F, LI);
6867 if (*MayContainIrreducibleControl)
6868 return true;
6869 if (!LI)
6870 return true;
6871 return LI->getLoopFor(&BB) != nullptr;
6872 };
6873
6874 // Flag to ensure we update our deallocation information at most once per
6875 // updateImpl call and only if we use the free check reasoning.
6876 bool HasUpdatedFrees = false;
6877
6878 auto UpdateFrees = [&]() {
6879 HasUpdatedFrees = true;
6880
6881 for (auto &It : DeallocationInfos) {
6882 DeallocationInfo &DI = *It.second;
6883 // For now we cannot use deallocations that have unknown inputs, skip
6884 // them.
6885 if (DI.MightFreeUnknownObjects)
6886 continue;
6887
6888 // No need to analyze dead calls, ignore them instead.
6889 bool UsedAssumedInformation = false;
6890 if (A.isAssumedDead(*DI.CB, this, LivenessAA, UsedAssumedInformation,
6891 /* CheckBBLivenessOnly */ true))
6892 continue;
6893
6894 // Use the non-optimistic version to get the freed object.
6895 Value *Obj = getUnderlyingObject(DI.FreedOp);
6896 if (!Obj) {
6897 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
6898 DI.MightFreeUnknownObjects = true;
6899 continue;
6900 }
6901
6902 // Free of null and undef can be ignored as no-ops (or UB in the latter
6903 // case).
6904 if (isa<ConstantPointerNull>(Obj) || isa<UndefValue>(Obj))
6905 continue;
6906
6907 CallBase *ObjCB = dyn_cast<CallBase>(Obj);
6908 if (!ObjCB) {
6909 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
6910 << "\n");
6911 DI.MightFreeUnknownObjects = true;
6912 continue;
6913 }
6914
6915 AllocationInfo *AI = AllocationInfos.lookup(ObjCB);
6916 if (!AI) {
6917 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
6918 << "\n");
6919 DI.MightFreeUnknownObjects = true;
6920 continue;
6921 }
6922
6923 DI.PotentialAllocationCalls.insert(ObjCB);
6924 }
6925 };
6926
6927 auto FreeCheck = [&](AllocationInfo &AI) {
6928 // If the stack is not accessible by other threads, the "must-free" logic
6929 // doesn't apply as the pointer could be shared and needs to be places in
6930 // "shareable" memory.
6931 if (!StackIsAccessibleByOtherThreads) {
6932 bool IsKnownNoSycn;
6933 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
6934 A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) {
6935 LLVM_DEBUG(
6936 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
6937 "other threads and function is not nosync:\n");
6938 return false;
6939 }
6940 }
6941 if (!HasUpdatedFrees)
6942 UpdateFrees();
6943
6944 // TODO: Allow multi exit functions that have different free calls.
6945 if (AI.PotentialFreeCalls.size() != 1) {
6946 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
6947 << AI.PotentialFreeCalls.size() << "\n");
6948 return false;
6949 }
6950 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
6951 DeallocationInfo *DI = DeallocationInfos.lookup(UniqueFree);
6952 if (!DI) {
6953 LLVM_DEBUG(
6954 dbgs() << "[H2S] unique free call was not known as deallocation call "
6955 << *UniqueFree << "\n");
6956 return false;
6957 }
6958 if (DI->MightFreeUnknownObjects) {
6959 LLVM_DEBUG(
6960 dbgs() << "[H2S] unique free call might free unknown allocations\n");
6961 return false;
6962 }
6963 if (DI->PotentialAllocationCalls.empty())
6964 return true;
6965 if (DI->PotentialAllocationCalls.size() > 1) {
6966 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
6967 << DI->PotentialAllocationCalls.size()
6968 << " different allocations\n");
6969 return false;
6970 }
6971 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
6972 LLVM_DEBUG(
6973 dbgs()
6974 << "[H2S] unique free call not known to free this allocation but "
6975 << **DI->PotentialAllocationCalls.begin() << "\n");
6976 return false;
6977 }
6978
6979 // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.
6980 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {
6981 Instruction *CtxI = isa<InvokeInst>(AI.CB) ? AI.CB : AI.CB->getNextNode();
6982 if (!Explorer || !Explorer->findInContextOf(UniqueFree, CtxI)) {
6983 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
6984 "with the allocation "
6985 << *UniqueFree << "\n");
6986 return false;
6987 }
6988 }
6989 return true;
6990 };
6991
6992 auto UsesCheck = [&](AllocationInfo &AI) {
6993 bool ValidUsesOnly = true;
6994
6995 auto Pred = [&](const Use &U, bool &Follow) -> bool {
6996 Instruction *UserI = cast<Instruction>(U.getUser());
6997 if (isa<LoadInst>(UserI))
6998 return true;
6999 if (auto *SI = dyn_cast<StoreInst>(UserI)) {
7000 if (SI->getValueOperand() == U.get()) {
7002 << "[H2S] escaping store to memory: " << *UserI << "\n");
7003 ValidUsesOnly = false;
7004 } else {
7005 // A store into the malloc'ed memory is fine.
7006 }
7007 return true;
7008 }
7009 if (auto *CB = dyn_cast<CallBase>(UserI)) {
7010 if (!CB->isArgOperand(&U) || CB->isLifetimeStartOrEnd())
7011 return true;
7012 if (DeallocationInfos.count(CB)) {
7013 AI.PotentialFreeCalls.insert(CB);
7014 return true;
7015 }
7016
7017 unsigned ArgNo = CB->getArgOperandNo(&U);
7018 auto CBIRP = IRPosition::callsite_argument(*CB, ArgNo);
7019
7020 bool IsKnownNoCapture;
7021 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
7022 A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);
7023
7024 // If a call site argument use is nofree, we are fine.
7025 bool IsKnownNoFree;
7026 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7027 A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree);
7028
7029 if (!IsAssumedNoCapture ||
7030 (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7031 !IsAssumedNoFree)) {
7032 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7033
7034 // Emit a missed remark if this is missed OpenMP globalization.
7035 auto Remark = [&](OptimizationRemarkMissed ORM) {
7036 return ORM
7037 << "Could not move globalized variable to the stack. "
7038 "Variable is potentially captured in call. Mark "
7039 "parameter as `__attribute__((noescape))` to override.";
7040 };
7041
7042 if (ValidUsesOnly &&
7043 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
7044 A.emitRemark<OptimizationRemarkMissed>(CB, "OMP113", Remark);
7045
7046 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7047 ValidUsesOnly = false;
7048 }
7049 return true;
7050 }
7051
7052 if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
7053 isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
7054 Follow = true;
7055 return true;
7056 }
7057 // Unknown user for which we can not track uses further (in a way that
7058 // makes sense).
7059 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7060 ValidUsesOnly = false;
7061 return true;
7062 };
7063 if (!A.checkForAllUses(Pred, *this, *AI.CB, /* CheckBBLivenessOnly */ false,
7064 DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7065 [&](const Use &OldU, const Use &NewU) {
7066 auto *SI = dyn_cast<StoreInst>(OldU.getUser());
7067 return !SI || StackIsAccessibleByOtherThreads ||
7068 AA::isAssumedThreadLocalObject(
7069 A, *SI->getPointerOperand(), *this);
7070 }))
7071 return false;
7072 return ValidUsesOnly;
7073 };
7074
7075 // The actual update starts here. We look at all allocations and depending on
7076 // their status perform the appropriate check(s).
7077 for (auto &It : AllocationInfos) {
7078 AllocationInfo &AI = *It.second;
7079 if (AI.Status == AllocationInfo::INVALID)
7080 continue;
7081
7082 if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
7083 std::optional<APInt> APAlign = getAPInt(A, *this, *Align);
7084 if (!APAlign) {
7085 // Can't generate an alloca which respects the required alignment
7086 // on the allocation.
7087 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7088 << "\n");
7089 AI.Status = AllocationInfo::INVALID;
7090 Changed = ChangeStatus::CHANGED;
7091 continue;
7092 }
7093 if (APAlign->ugt(llvm::Value::MaximumAlignment) ||
7094 !APAlign->isPowerOf2()) {
7095 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7096 << "\n");
7097 AI.Status = AllocationInfo::INVALID;
7098 Changed = ChangeStatus::CHANGED;
7099 continue;
7100 }
7101 }
7102
7103 std::optional<APInt> Size = getSize(A, *this, AI);
7104 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7105 MaxHeapToStackSize != -1) {
7106 if (!Size || Size->ugt(MaxHeapToStackSize)) {
7107 LLVM_DEBUG({
7108 if (!Size)
7109 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7110 else
7111 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7112 << MaxHeapToStackSize << "\n";
7113 });
7114
7115 AI.Status = AllocationInfo::INVALID;
7116 Changed = ChangeStatus::CHANGED;
7117 continue;
7118 }
7119 }
7120
7121 switch (AI.Status) {
7122 case AllocationInfo::STACK_DUE_TO_USE:
7123 if (UsesCheck(AI))
7124 break;
7125 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7126 [[fallthrough]];
7127 case AllocationInfo::STACK_DUE_TO_FREE:
7128 if (FreeCheck(AI))
7129 break;
7130 AI.Status = AllocationInfo::INVALID;
7131 Changed = ChangeStatus::CHANGED;
7132 break;
7133 case AllocationInfo::INVALID:
7134 llvm_unreachable("Invalid allocations should never reach this point!");
7135 };
7136
7137 // Check if we still think we can move it into the entry block. If the
7138 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7139 // ignore the potential compilations associated with loops.
7140 bool IsGlobalizedLocal =
7141 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;
7142 if (AI.MoveAllocaIntoEntry &&
7143 (!Size.has_value() ||
7144 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7145 AI.MoveAllocaIntoEntry = false;
7146 }
7147
7148 return Changed;
7149}
7150} // namespace
7151
7152/// ----------------------- Privatizable Pointers ------------------------------
7153namespace {
7154struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7155 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7156 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7157
7158 ChangeStatus indicatePessimisticFixpoint() override {
7159 AAPrivatizablePtr::indicatePessimisticFixpoint();
7160 PrivatizableType = nullptr;
7161 return ChangeStatus::CHANGED;
7162 }
7163
7164 /// Identify the type we can chose for a private copy of the underlying
7165 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7166 /// none.
7167 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7168
7169 /// Return a privatizable type that encloses both T0 and T1.
7170 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7171 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7172 std::optional<Type *> T1) {
7173 if (!T0)
7174 return T1;
7175 if (!T1)
7176 return T0;
7177 if (T0 == T1)
7178 return T0;
7179 return nullptr;
7180 }
7181
7182 std::optional<Type *> getPrivatizableType() const override {
7183 return PrivatizableType;
7184 }
7185
7186 const std::string getAsStr(Attributor *A) const override {
7187 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7188 }
7189
7190protected:
7191 std::optional<Type *> PrivatizableType;
7192};
7193
7194// TODO: Do this for call site arguments (probably also other values) as well.
7195
7196struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7197 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7198 : AAPrivatizablePtrImpl(IRP, A) {}
7199
7200 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7201 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7202 // If this is a byval argument and we know all the call sites (so we can
7203 // rewrite them), there is no need to check them explicitly.
7204 bool UsedAssumedInformation = false;
7206 A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs,
7207 /* IgnoreSubsumingPositions */ true);
7208 if (!Attrs.empty() &&
7209 A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *this,
7210 true, UsedAssumedInformation))
7211 return Attrs[0].getValueAsType();
7212
7213 std::optional<Type *> Ty;
7214 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7215
7216 // Make sure the associated call site argument has the same type at all call
7217 // sites and it is an allocation we know is safe to privatize, for now that
7218 // means we only allow alloca instructions.
7219 // TODO: We can additionally analyze the accesses in the callee to create
7220 // the type from that information instead. That is a little more
7221 // involved and will be done in a follow up patch.
7222 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7223 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7224 // Check if a coresponding argument was found or if it is one not
7225 // associated (which can happen for callback calls).
7226 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7227 return false;
7228
7229 // Check that all call sites agree on a type.
7230 auto *PrivCSArgAA =
7231 A.getAAFor<AAPrivatizablePtr>(*this, ACSArgPos, DepClassTy::REQUIRED);
7232 if (!PrivCSArgAA)
7233 return false;
7234 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7235
7236 LLVM_DEBUG({
7237 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7238 if (CSTy && *CSTy)
7239 (*CSTy)->print(dbgs());
7240 else if (CSTy)
7241 dbgs() << "<nullptr>";
7242 else
7243 dbgs() << "<none>";
7244 });
7245
7246 Ty = combineTypes(Ty, CSTy);
7247
7248 LLVM_DEBUG({
7249 dbgs() << " : New Type: ";
7250 if (Ty && *Ty)
7251 (*Ty)->print(dbgs());
7252 else if (Ty)
7253 dbgs() << "<nullptr>";
7254 else
7255 dbgs() << "<none>";
7256 dbgs() << "\n";
7257 });
7258
7259 return !Ty || *Ty;
7260 };
7261
7262 if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
7263 UsedAssumedInformation))
7264 return nullptr;
7265 return Ty;
7266 }
7267
7268 /// See AbstractAttribute::updateImpl(...).
7269 ChangeStatus updateImpl(Attributor &A) override {
7270 PrivatizableType = identifyPrivatizableType(A);
7271 if (!PrivatizableType)
7272 return ChangeStatus::UNCHANGED;
7273 if (!*PrivatizableType)
7274 return indicatePessimisticFixpoint();
7275
7276 // The dependence is optional so we don't give up once we give up on the
7277 // alignment.
7278 A.getAAFor<AAAlign>(*this, IRPosition::value(getAssociatedValue()),
7279 DepClassTy::OPTIONAL);
7280
7281 // Avoid arguments with padding for now.
7282 if (!A.hasAttr(getIRPosition(), Attribute::ByVal) &&
7283 !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) {
7284 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7285 return indicatePessimisticFixpoint();
7286 }
7287
7288 // Collect the types that will replace the privatizable type in the function
7289 // signature.
7290 SmallVector<Type *, 16> ReplacementTypes;
7291 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7292
7293 // Verify callee and caller agree on how the promoted argument would be
7294 // passed.
7295 Function &Fn = *getIRPosition().getAnchorScope();
7296 const auto *TTI =
7297 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);
7298 if (!TTI) {
7299 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7300 << Fn.getName() << "\n");
7301 return indicatePessimisticFixpoint();
7302 }
7303
7304 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7305 CallBase *CB = ACS.getInstruction();
7306 return TTI->areTypesABICompatible(
7307 CB->getCaller(),
7308 dyn_cast_if_present<Function>(CB->getCalledOperand()),
7309 ReplacementTypes);
7310 };
7311 bool UsedAssumedInformation = false;
7312 if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
7313 UsedAssumedInformation)) {
7314 LLVM_DEBUG(
7315 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7316 << Fn.getName() << "\n");
7317 return indicatePessimisticFixpoint();
7318 }
7319
7320 // Register a rewrite of the argument.
7321 Argument *Arg = getAssociatedArgument();
7322 if (!A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {
7323 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7324 return indicatePessimisticFixpoint();
7325 }
7326
7327 unsigned ArgNo = Arg->getArgNo();
7328
7329 // Helper to check if for the given call site the associated argument is
7330 // passed to a callback where the privatization would be different.
7331 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7332 SmallVector<const Use *, 4> CallbackUses;
7333 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7334 for (const Use *U : CallbackUses) {
7335 AbstractCallSite CBACS(U);
7336 assert(CBACS && CBACS.isCallbackCall());
7337 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7338 int CBArgNo = CBACS.getCallArgOperandNo(CBArg);
7339
7340 LLVM_DEBUG({
7341 dbgs()
7342 << "[AAPrivatizablePtr] Argument " << *Arg
7343 << "check if can be privatized in the context of its parent ("
7344 << Arg->getParent()->getName()
7345 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7346 "callback ("
7347 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7348 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7349 << CBACS.getCallArgOperand(CBArg) << " vs "
7350 << CB.getArgOperand(ArgNo) << "\n"
7351 << "[AAPrivatizablePtr] " << CBArg << " : "
7352 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7353 });
7354
7355 if (CBArgNo != int(ArgNo))
7356 continue;
7357 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7358 *this, IRPosition::argument(CBArg), DepClassTy::REQUIRED);
7359 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7360 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7361 if (!CBArgPrivTy)
7362 continue;
7363 if (*CBArgPrivTy == PrivatizableType)
7364 continue;
7365 }
7366
7367 LLVM_DEBUG({
7368 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7369 << " cannot be privatized in the context of its parent ("
7370 << Arg->getParent()->getName()
7371 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7372 "callback ("
7373 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7374 << ").\n[AAPrivatizablePtr] for which the argument "
7375 "privatization is not compatible.\n";
7376 });
7377 return false;
7378 }
7379 }
7380 return true;
7381 };
7382
7383 // Helper to check if for the given call site the associated argument is
7384 // passed to a direct call where the privatization would be different.
7385 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7386 CallBase *DC = cast<CallBase>(ACS.getInstruction());
7387 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7388 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7389 "Expected a direct call operand for callback call operand");
7390
7391 Function *DCCallee =
7392 dyn_cast_if_present<Function>(DC->getCalledOperand());
7393 LLVM_DEBUG({
7394 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7395 << " check if be privatized in the context of its parent ("
7396 << Arg->getParent()->getName()
7397 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7398 "direct call of ("
7399 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7400 });
7401
7402 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7403 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7404 *this, IRPosition::argument(*DCCallee->getArg(DCArgNo)),
7405 DepClassTy::REQUIRED);
7406 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7407 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7408 if (!DCArgPrivTy)
7409 return true;
7410 if (*DCArgPrivTy == PrivatizableType)
7411 return true;
7412 }
7413 }
7414
7415 LLVM_DEBUG({
7416 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7417 << " cannot be privatized in the context of its parent ("
7418 << Arg->getParent()->getName()
7419 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7420 "direct call of ("
7422 << ").\n[AAPrivatizablePtr] for which the argument "
7423 "privatization is not compatible.\n";
7424 });
7425 return false;
7426 };
7427
7428 // Helper to check if the associated argument is used at the given abstract
7429 // call site in a way that is incompatible with the privatization assumed
7430 // here.
7431 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7432 if (ACS.isDirectCall())
7433 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7434 if (ACS.isCallbackCall())
7435 return IsCompatiblePrivArgOfDirectCS(ACS);
7436 return false;
7437 };
7438
7439 if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true,
7440 UsedAssumedInformation))
7441 return indicatePessimisticFixpoint();
7442
7443 return ChangeStatus::UNCHANGED;
7444 }
7445
7446 /// Given a type to private \p PrivType, collect the constituates (which are
7447 /// used) in \p ReplacementTypes.
7448 static void
7449 identifyReplacementTypes(Type *PrivType,
7450 SmallVectorImpl<Type *> &ReplacementTypes) {
7451 // TODO: For now we expand the privatization type to the fullest which can
7452 // lead to dead arguments that need to be removed later.
7453 assert(PrivType && "Expected privatizable type!");
7454
7455 // Traverse the type, extract constituate types on the outermost level.
7456 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7457 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7458 ReplacementTypes.push_back(PrivStructType->getElementType(u));
7459 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7460 ReplacementTypes.append(PrivArrayType->getNumElements(),
7461 PrivArrayType->getElementType());
7462 } else {
7463 ReplacementTypes.push_back(PrivType);
7464 }
7465 }
7466
7467 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7468 /// The values needed are taken from the arguments of \p F starting at
7469 /// position \p ArgNo.
7470 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7471 unsigned ArgNo, BasicBlock::iterator IP) {
7472 assert(PrivType && "Expected privatizable type!");
7473
7474 IRBuilder<NoFolder> IRB(IP->getParent(), IP);
7475 const DataLayout &DL = F.getDataLayout();
7476
7477 // Traverse the type, build GEPs and stores.
7478 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7479 const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
7480 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7481 Value *Ptr =
7482 constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB);
7483 new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
7484 }
7485 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7486 Type *PointeeTy = PrivArrayType->getElementType();
7487 uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
7488 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7489 Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB);
7490 new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
7491 }
7492 } else {
7493 new StoreInst(F.getArg(ArgNo), &Base, IP);
7494 }
7495 }
7496
7497 /// Extract values from \p Base according to the type \p PrivType at the
7498 /// call position \p ACS. The values are appended to \p ReplacementValues.
7499 void createReplacementValues(Align Alignment, Type *PrivType,
7501 SmallVectorImpl<Value *> &ReplacementValues) {
7502 assert(Base && "Expected base value!");
7503 assert(PrivType && "Expected privatizable type!");
7504 Instruction *IP = ACS.getInstruction();
7505
7506 IRBuilder<NoFolder> IRB(IP);
7507 const DataLayout &DL = IP->getDataLayout();
7508
7509 // Traverse the type, build GEPs and loads.
7510 if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
7511 const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
7512 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7513 Type *PointeeTy = PrivStructType->getElementType(u);
7514 Value *Ptr =
7515 constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB);
7516 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7517 L->setAlignment(Alignment);
7518 ReplacementValues.push_back(L);
7519 }
7520 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
7521 Type *PointeeTy = PrivArrayType->getElementType();
7522 uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
7523 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7524 Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB);
7525 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7526 L->setAlignment(Alignment);
7527 ReplacementValues.push_back(L);
7528 }
7529 } else {
7530 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7531 L->setAlignment(Alignment);
7532 ReplacementValues.push_back(L);
7533 }
7534 }
7535
7536 /// See AbstractAttribute::manifest(...)
7537 ChangeStatus manifest(Attributor &A) override {
7538 if (!PrivatizableType)
7539 return ChangeStatus::UNCHANGED;
7540 assert(*PrivatizableType && "Expected privatizable type!");
7541
7542 // Collect all tail calls in the function as we cannot allow new allocas to
7543 // escape into tail recursion.
7544 // TODO: Be smarter about new allocas escaping into tail calls.
7546 bool UsedAssumedInformation = false;
7547 if (!A.checkForAllInstructions(
7548 [&](Instruction &I) {
7549 CallInst &CI = cast<CallInst>(I);
7550 if (CI.isTailCall())
7551 TailCalls.push_back(&CI);
7552 return true;
7553 },
7554 *this, {Instruction::Call}, UsedAssumedInformation))
7555 return ChangeStatus::UNCHANGED;
7556
7557 Argument *Arg = getAssociatedArgument();
7558 // Query AAAlign attribute for alignment of associated argument to
7559 // determine the best alignment of loads.
7560 const auto *AlignAA =
7561 A.getAAFor<AAAlign>(*this, IRPosition::value(*Arg), DepClassTy::NONE);
7562
7563 // Callback to repair the associated function. A new alloca is placed at the
7564 // beginning and initialized with the values passed through arguments. The
7565 // new alloca replaces the use of the old pointer argument.
7568 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7569 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7571 const DataLayout &DL = IP->getDataLayout();
7572 unsigned AS = DL.getAllocaAddrSpace();
7573 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7574 Arg->getName() + ".priv", IP);
7575 createInitialization(*PrivatizableType, *AI, ReplacementFn,
7576 ArgIt->getArgNo(), IP);
7577
7578 if (AI->getType() != Arg->getType())
7579 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7580 AI, Arg->getType(), "", IP);
7581 Arg->replaceAllUsesWith(AI);
7582
7583 for (CallInst *CI : TailCalls)
7584 CI->setTailCall(false);
7585 };
7586
7587 // Callback to repair a call site of the associated function. The elements
7588 // of the privatizable type are loaded prior to the call and passed to the
7589 // new function version.
7592 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7593 // When no alignment is specified for the load instruction,
7594 // natural alignment is assumed.
7595 createReplacementValues(
7596 AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7597 *PrivatizableType, ACS,
7598 ACS.getCallArgOperand(ARI.getReplacedArg().getArgNo()),
7599 NewArgOperands);
7600 };
7601
7602 // Collect the types that will replace the privatizable type in the function
7603 // signature.
7604 SmallVector<Type *, 16> ReplacementTypes;
7605 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7606
7607 // Register a rewrite of the argument.
7608 if (A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes,
7609 std::move(FnRepairCB),
7610 std::move(ACSRepairCB)))
7611 return ChangeStatus::CHANGED;
7612 return ChangeStatus::UNCHANGED;
7613 }
7614
7615 /// See AbstractAttribute::trackStatistics()
7616 void trackStatistics() const override {
7617 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7618 }
7619};
7620
7621struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7622 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7623 : AAPrivatizablePtrImpl(IRP, A) {}
7624
7625 /// See AbstractAttribute::initialize(...).
7626 void initialize(Attributor &A) override {
7627 // TODO: We can privatize more than arguments.
7628 indicatePessimisticFixpoint();
7629 }
7630
7631 ChangeStatus updateImpl(Attributor &A) override {
7632 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7633 "updateImpl will not be called");
7634 }
7635
7636 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7637 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7638 Value *Obj = getUnderlyingObject(&getAssociatedValue());
7639 if (!Obj) {
7640 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7641 return nullptr;
7642 }
7643
7644 if (auto *AI = dyn_cast<AllocaInst>(Obj))
7645 if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
7646 if (CI->isOne())
7647 return AI->getAllocatedType();
7648 if (auto *Arg = dyn_cast<Argument>(Obj)) {
7649 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7650 *this, IRPosition::argument(*Arg), DepClassTy::REQUIRED);
7651 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7652 return PrivArgAA->getPrivatizableType();
7653 }
7654
7655 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7656 "alloca nor privatizable argument: "
7657 << *Obj << "!\n");
7658 return nullptr;
7659 }
7660
7661 /// See AbstractAttribute::trackStatistics()
7662 void trackStatistics() const override {
7663 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7664 }
7665};
7666
7667struct AAPrivatizablePtrCallSiteArgument final
7668 : public AAPrivatizablePtrFloating {
7669 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7670 : AAPrivatizablePtrFloating(IRP, A) {}
7671
7672 /// See AbstractAttribute::initialize(...).
7673 void initialize(Attributor &A) override {
7674 if (A.hasAttr(getIRPosition(), Attribute::ByVal))
7675 indicateOptimisticFixpoint();
7676 }
7677
7678 /// See AbstractAttribute::updateImpl(...).
7679 ChangeStatus updateImpl(Attributor &A) override {
7680 PrivatizableType = identifyPrivatizableType(A);
7681 if (!PrivatizableType)
7682 return ChangeStatus::UNCHANGED;
7683 if (!*PrivatizableType)
7684 return indicatePessimisticFixpoint();
7685
7686 const IRPosition &IRP = getIRPosition();
7687 bool IsKnownNoCapture;
7688 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
7689 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
7690 if (!IsAssumedNoCapture) {
7691 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7692 return indicatePessimisticFixpoint();
7693 }
7694
7695 bool IsKnownNoAlias;
7696 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7697 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
7698 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7699 return indicatePessimisticFixpoint();
7700 }
7701
7702 bool IsKnown;
7703 if (!AA::isAssumedReadOnly(A, IRP, *this, IsKnown)) {
7704 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7705 return indicatePessimisticFixpoint();
7706 }
7707
7708 return ChangeStatus::UNCHANGED;
7709 }
7710
7711 /// See AbstractAttribute::trackStatistics()
7712 void trackStatistics() const override {
7713 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7714 }
7715};
7716
7717struct AAPrivatizablePtrCallSiteReturned final
7718 : public AAPrivatizablePtrFloating {
7719 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7720 : AAPrivatizablePtrFloating(IRP, A) {}
7721
7722 /// See AbstractAttribute::initialize(...).
7723 void initialize(Attributor &A) override {
7724 // TODO: We can privatize more than arguments.
7725 indicatePessimisticFixpoint();
7726 }
7727
7728 /// See AbstractAttribute::trackStatistics()
7729 void trackStatistics() const override {
7730 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7731 }
7732};
7733
7734struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7735 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7736 : AAPrivatizablePtrFloating(IRP, A) {}
7737
7738 /// See AbstractAttribute::initialize(...).
7739 void initialize(Attributor &A) override {
7740 // TODO: We can privatize more than arguments.
7741 indicatePessimisticFixpoint();
7742 }
7743
7744 /// See AbstractAttribute::trackStatistics()
7745 void trackStatistics() const override {
7746 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7747 }
7748};
7749} // namespace
7750
7751/// -------------------- Memory Behavior Attributes ----------------------------
7752/// Includes read-none, read-only, and write-only.
7753/// ----------------------------------------------------------------------------
7754namespace {
7755struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7756 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7757 : AAMemoryBehavior(IRP, A) {}
7758
7759 /// See AbstractAttribute::initialize(...).
7760 void initialize(Attributor &A) override {
7761 intersectAssumedBits(BEST_STATE);
7762 getKnownStateFromValue(A, getIRPosition(), getState());
7763 AAMemoryBehavior::initialize(A);
7764 }
7765
7766 /// Return the memory behavior information encoded in the IR for \p IRP.
7767 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7768 BitIntegerState &State,
7769 bool IgnoreSubsumingPositions = false) {
7771 A.getAttrs(IRP, AttrKinds, Attrs, IgnoreSubsumingPositions);
7772 for (const Attribute &Attr : Attrs) {
7773 switch (Attr.getKindAsEnum()) {
7774 case Attribute::ReadNone:
7775 State.addKnownBits(NO_ACCESSES);
7776 break;
7777 case Attribute::ReadOnly:
7778 State.addKnownBits(NO_WRITES);
7779 break;
7780 case Attribute::WriteOnly:
7781 State.addKnownBits(NO_READS);
7782 break;
7783 default:
7784 llvm_unreachable("Unexpected attribute!");
7785 }
7786 }
7787
7788 if (auto *I = dyn_cast<Instruction>(&IRP.getAnchorValue())) {
7789 if (!I->mayReadFromMemory())
7790 State.addKnownBits(NO_READS);
7791 if (!I->mayWriteToMemory())
7792 State.addKnownBits(NO_WRITES);
7793 }
7794 }
7795
7796 /// See AbstractAttribute::getDeducedAttributes(...).
7797 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7798 SmallVectorImpl<Attribute> &Attrs) const override {
7799 assert(Attrs.size() == 0);
7800 if (isAssumedReadNone())
7801 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));
7802 else if (isAssumedReadOnly())
7803 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));
7804 else if (isAssumedWriteOnly())
7805 Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));
7806 assert(Attrs.size() <= 1);
7807 }
7808
7809 /// See AbstractAttribute::manifest(...).
7810 ChangeStatus manifest(Attributor &A) override {
7811 const IRPosition &IRP = getIRPosition();
7812
7813 if (A.hasAttr(IRP, Attribute::ReadNone,
7814 /* IgnoreSubsumingPositions */ true))
7815 return ChangeStatus::UNCHANGED;
7816
7817 // Check if we would improve the existing attributes first.
7818 SmallVector<Attribute, 4> DeducedAttrs;
7819 getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
7820 if (llvm::all_of(DeducedAttrs, [&](const Attribute &Attr) {
7821 return A.hasAttr(IRP, Attr.getKindAsEnum(),
7822 /* IgnoreSubsumingPositions */ true);
7823 }))
7824 return ChangeStatus::UNCHANGED;
7825
7826 // Clear existing attributes.
7827 A.removeAttrs(IRP, AttrKinds);
7828 // Clear conflicting writable attribute.
7829 if (isAssumedReadOnly())
7830 A.removeAttrs(IRP, Attribute::Writable);
7831
7832 // Use the generic manifest method.
7833 return IRAttribute::manifest(A);
7834 }
7835
7836 /// See AbstractState::getAsStr().
7837 const std::string getAsStr(Attributor *A) const override {
7838 if (isAssumedReadNone())
7839 return "readnone";
7840 if (isAssumedReadOnly())
7841 return "readonly";
7842 if (isAssumedWriteOnly())
7843 return "writeonly";
7844 return "may-read/write";
7845 }
7846
7847 /// The set of IR attributes AAMemoryBehavior deals with.
7848 static const Attribute::AttrKind AttrKinds[3];
7849};
7850
7851const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
7852 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7853
7854/// Memory behavior attribute for a floating value.
7855struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7856 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
7857 : AAMemoryBehaviorImpl(IRP, A) {}
7858
7859 /// See AbstractAttribute::updateImpl(...).
7860 ChangeStatus updateImpl(Attributor &A) override;
7861
7862 /// See AbstractAttribute::trackStatistics()
7863 void trackStatistics() const override {
7864 if (isAssumedReadNone())
7866 else if (isAssumedReadOnly())
7868 else if (isAssumedWriteOnly())
7870 }
7871
7872private:
7873 /// Return true if users of \p UserI might access the underlying
7874 /// variable/location described by \p U and should therefore be analyzed.
7875 bool followUsersOfUseIn(Attributor &A, const Use &U,
7876 const Instruction *UserI);
7877
7878 /// Update the state according to the effect of use \p U in \p UserI.
7879 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
7880};
7881
7882/// Memory behavior attribute for function argument.
7883struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
7884 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
7885 : AAMemoryBehaviorFloating(IRP, A) {}
7886
7887 /// See AbstractAttribute::initialize(...).
7888 void initialize(Attributor &A) override {
7889 intersectAssumedBits(BEST_STATE);
7890 const IRPosition &IRP = getIRPosition();
7891 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
7892 // can query it when we use has/getAttr. That would allow us to reuse the
7893 // initialize of the base class here.
7894 bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal},
7895 /* IgnoreSubsumingPositions */ true);
7896 getKnownStateFromValue(A, IRP, getState(),
7897 /* IgnoreSubsumingPositions */ HasByVal);
7898 }
7899
7900 ChangeStatus manifest(Attributor &A) override {
7901 // TODO: Pointer arguments are not supported on vectors of pointers yet.
7902 if (!getAssociatedValue().getType()->isPointerTy())
7903 return ChangeStatus::UNCHANGED;
7904
7905 // TODO: From readattrs.ll: "inalloca parameters are always
7906 // considered written"
7907 if (A.hasAttr(getIRPosition(),
7908 {Attribute::InAlloca, Attribute::Preallocated})) {
7909 removeKnownBits(NO_WRITES);
7910 removeAssumedBits(NO_WRITES);
7911 }
7912 A.removeAttrs(getIRPosition(), AttrKinds);
7913 return AAMemoryBehaviorFloating::manifest(A);
7914 }
7915
7916 /// See AbstractAttribute::trackStatistics()
7917 void trackStatistics() const override {
7918 if (isAssumedReadNone())
7919 STATS_DECLTRACK_ARG_ATTR(readnone)
7920 else if (isAssumedReadOnly())
7921 STATS_DECLTRACK_ARG_ATTR(readonly)
7922 else if (isAssumedWriteOnly())
7923 STATS_DECLTRACK_ARG_ATTR(writeonly)
7924 }
7925};
7926
7927struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
7928 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
7929 : AAMemoryBehaviorArgument(IRP, A) {}
7930
7931 /// See AbstractAttribute::initialize(...).
7932 void initialize(Attributor &A) override {
7933 // If we don't have an associated attribute this is either a variadic call
7934 // or an indirect call, either way, nothing to do here.
7935 Argument *Arg = getAssociatedArgument();
7936 if (!Arg) {
7937 indicatePessimisticFixpoint();
7938 return;
7939 }
7940 if (Arg->hasByValAttr()) {
7941 addKnownBits(NO_WRITES);
7942 removeKnownBits(NO_READS);
7943 removeAssumedBits(NO_READS);
7944 }
7945 AAMemoryBehaviorArgument::initialize(A);
7946 if (getAssociatedFunction()->isDeclaration())
7947 indicatePessimisticFixpoint();
7948 }
7949
7950 /// See AbstractAttribute::updateImpl(...).
7951 ChangeStatus updateImpl(Attributor &A) override {
7952 // TODO: Once we have call site specific value information we can provide
7953 // call site specific liveness liveness information and then it makes
7954 // sense to specialize attributes for call sites arguments instead of
7955 // redirecting requests to the callee argument.
7956 Argument *Arg = getAssociatedArgument();
7957 const IRPosition &ArgPos = IRPosition::argument(*Arg);
7958 auto *ArgAA =
7959 A.getAAFor<AAMemoryBehavior>(*this, ArgPos, DepClassTy::REQUIRED);
7960 if (!ArgAA)
7961 return indicatePessimisticFixpoint();
7962 return clampStateAndIndicateChange(getState(), ArgAA->getState());
7963 }
7964
7965 /// See AbstractAttribute::trackStatistics()
7966 void trackStatistics() const override {
7967 if (isAssumedReadNone())
7969 else if (isAssumedReadOnly())
7971 else if (isAssumedWriteOnly())
7973 }
7974};
7975
7976/// Memory behavior attribute for a call site return position.
7977struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
7978 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
7979 : AAMemoryBehaviorFloating(IRP, A) {}
7980
7981 /// See AbstractAttribute::initialize(...).
7982 void initialize(Attributor &A) override {
7983 AAMemoryBehaviorImpl::initialize(A);
7984 }
7985 /// See AbstractAttribute::manifest(...).
7986 ChangeStatus manifest(Attributor &A) override {
7987 // We do not annotate returned values.
7988 return ChangeStatus::UNCHANGED;
7989 }
7990
7991 /// See AbstractAttribute::trackStatistics()
7992 void trackStatistics() const override {}
7993};
7994
7995/// An AA to represent the memory behavior function attributes.
7996struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
7997 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
7998 : AAMemoryBehaviorImpl(IRP, A) {}
7999
8000 /// See AbstractAttribute::updateImpl(Attributor &A).
8001 ChangeStatus updateImpl(Attributor &A) override;
8002
8003 /// See AbstractAttribute::manifest(...).
8004 ChangeStatus manifest(Attributor &A) override {
8005 // TODO: It would be better to merge this with AAMemoryLocation, so that
8006 // we could determine read/write per location. This would also have the
8007 // benefit of only one place trying to manifest the memory attribute.
8008 Function &F = cast<Function>(getAnchorValue());
8010 if (isAssumedReadNone())
8011 ME = MemoryEffects::none();
8012 else if (isAssumedReadOnly())
8014 else if (isAssumedWriteOnly())
8016
8017 A.removeAttrs(getIRPosition(), AttrKinds);
8018 // Clear conflicting writable attribute.
8019 if (ME.onlyReadsMemory())
8020 for (Argument &Arg : F.args())
8021 A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);
8022 return A.manifestAttrs(getIRPosition(),
8023 Attribute::getWithMemoryEffects(F.getContext(), ME));
8024 }
8025
8026 /// See AbstractAttribute::trackStatistics()
8027 void trackStatistics() const override {
8028 if (isAssumedReadNone())
8029 STATS_DECLTRACK_FN_ATTR(readnone)
8030 else if (isAssumedReadOnly())
8031 STATS_DECLTRACK_FN_ATTR(readonly)
8032 else if (isAssumedWriteOnly())
8033 STATS_DECLTRACK_FN_ATTR(writeonly)
8034 }
8035};
8036
8037/// AAMemoryBehavior attribute for call sites.
8038struct AAMemoryBehaviorCallSite final
8039 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8040 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8041 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8042
8043 /// See AbstractAttribute::manifest(...).
8044 ChangeStatus manifest(Attributor &A) override {
8045 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8046 CallBase &CB = cast<CallBase>(getAnchorValue());
8048 if (isAssumedReadNone())
8049 ME = MemoryEffects::none();
8050 else if (isAssumedReadOnly())
8052 else if (isAssumedWriteOnly())
8054
8055 A.removeAttrs(getIRPosition(), AttrKinds);
8056 // Clear conflicting writable attribute.
8057 if (ME.onlyReadsMemory())
8058 for (Use &U : CB.args())
8059 A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()),
8060 Attribute::Writable);
8061 return A.manifestAttrs(
8062 getIRPosition(), Attribute::getWithMemoryEffects(CB.getContext(), ME));
8063 }
8064
8065 /// See AbstractAttribute::trackStatistics()
8066 void trackStatistics() const override {
8067 if (isAssumedReadNone())
8068 STATS_DECLTRACK_CS_ATTR(readnone)
8069 else if (isAssumedReadOnly())
8070 STATS_DECLTRACK_CS_ATTR(readonly)
8071 else if (isAssumedWriteOnly())
8072 STATS_DECLTRACK_CS_ATTR(writeonly)
8073 }
8074};
8075
8076ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8077
8078 // The current assumed state used to determine a change.
8079 auto AssumedState = getAssumed();
8080
8081 auto CheckRWInst = [&](Instruction &I) {
8082 // If the instruction has an own memory behavior state, use it to restrict
8083 // the local state. No further analysis is required as the other memory
8084 // state is as optimistic as it gets.
8085 if (const auto *CB = dyn_cast<CallBase>(&I)) {
8086 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8088 if (MemBehaviorAA) {
8089 intersectAssumedBits(MemBehaviorAA->getAssumed());
8090 return !isAtFixpoint();
8091 }
8092 }
8093
8094 // Remove access kind modifiers if necessary.
8095 if (I.mayReadFromMemory())
8096 removeAssumedBits(NO_READS);
8097 if (I.mayWriteToMemory())
8098 removeAssumedBits(NO_WRITES);
8099 return !isAtFixpoint();
8100 };
8101
8102 bool UsedAssumedInformation = false;
8103 if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
8104 UsedAssumedInformation))
8105 return indicatePessimisticFixpoint();
8106
8107 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8109}
8110
8111ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8112
8113 const IRPosition &IRP = getIRPosition();
8114 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8115 AAMemoryBehavior::StateType &S = getState();
8116
8117 // First, check the function scope. We take the known information and we avoid
8118 // work if the assumed information implies the current assumed information for
8119 // this attribute. This is a valid for all but byval arguments.
8120 Argument *Arg = IRP.getAssociatedArgument();
8121 AAMemoryBehavior::base_t FnMemAssumedState =
8123 if (!Arg || !Arg->hasByValAttr()) {
8124 const auto *FnMemAA =
8125 A.getAAFor<AAMemoryBehavior>(*this, FnPos, DepClassTy::OPTIONAL);
8126 if (FnMemAA) {
8127 FnMemAssumedState = FnMemAA->getAssumed();
8128 S.addKnownBits(FnMemAA->getKnown());
8129 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8131 }
8132 }
8133
8134 // The current assumed state used to determine a change.
8135 auto AssumedState = S.getAssumed();
8136
8137 // Make sure the value is not captured (except through "return"), if
8138 // it is, any information derived would be irrelevant anyway as we cannot
8139 // check the potential aliases introduced by the capture. However, no need
8140 // to fall back to anythign less optimistic than the function state.
8141 bool IsKnownNoCapture;
8142 const AANoCapture *ArgNoCaptureAA = nullptr;
8143 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
8144 A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,
8145 &ArgNoCaptureAA);
8146
8147 if (!IsAssumedNoCapture &&
8148 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8149 S.intersectAssumedBits(FnMemAssumedState);
8150 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8152 }
8153
8154 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8155 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8156 Instruction *UserI = cast<Instruction>(U.getUser());
8157 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8158 << " \n");
8159
8160 // Droppable users, e.g., llvm::assume does not actually perform any action.
8161 if (UserI->isDroppable())
8162 return true;
8163
8164 // Check if the users of UserI should also be visited.
8165 Follow = followUsersOfUseIn(A, U, UserI);
8166
8167 // If UserI might touch memory we analyze the use in detail.
8168 if (UserI->mayReadOrWriteMemory())
8169 analyzeUseIn(A, U, UserI);
8170
8171 return !isAtFixpoint();
8172 };
8173
8174 if (!A.checkForAllUses(UsePred, *this, getAssociatedValue()))
8175 return indicatePessimisticFixpoint();
8176
8177 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8179}
8180
8181bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8182 const Instruction *UserI) {
8183 // The loaded value is unrelated to the pointer argument, no need to
8184 // follow the users of the load.
8185 if (isa<LoadInst>(UserI) || isa<ReturnInst>(UserI))
8186 return false;
8187
8188 // By default we follow all uses assuming UserI might leak information on U,
8189 // we have special handling for call sites operands though.
8190 const auto *CB = dyn_cast<CallBase>(UserI);
8191 if (!CB || !CB->isArgOperand(&U))
8192 return true;
8193
8194 // If the use is a call argument known not to be captured, the users of
8195 // the call do not need to be visited because they have to be unrelated to
8196 // the input. Note that this check is not trivial even though we disallow
8197 // general capturing of the underlying argument. The reason is that the
8198 // call might the argument "through return", which we allow and for which we
8199 // need to check call users.
8200 if (U.get()->getType()->isPointerTy()) {
8201 unsigned ArgNo = CB->getArgOperandNo(&U);
8202 bool IsKnownNoCapture;
8203 return !AA::hasAssumedIRAttr<Attribute::NoCapture>(
8204 A, this, IRPosition::callsite_argument(*CB, ArgNo),
8205 DepClassTy::OPTIONAL, IsKnownNoCapture);
8206 }
8207
8208 return true;
8209}
8210
8211void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8212 const Instruction *UserI) {
8213 assert(UserI->mayReadOrWriteMemory());
8214
8215 switch (UserI->getOpcode()) {
8216 default:
8217 // TODO: Handle all atomics and other side-effect operations we know of.
8218 break;
8219 case Instruction::Load:
8220 // Loads cause the NO_READS property to disappear.
8221 removeAssumedBits(NO_READS);
8222 return;
8223
8224 case Instruction::Store:
8225 // Stores cause the NO_WRITES property to disappear if the use is the
8226 // pointer operand. Note that while capturing was taken care of somewhere
8227 // else we need to deal with stores of the value that is not looked through.
8228 if (cast<StoreInst>(UserI)->getPointerOperand() == U.get())
8229 removeAssumedBits(NO_WRITES);
8230 else
8231 indicatePessimisticFixpoint();
8232 return;
8233
8234 case Instruction::Call:
8235 case Instruction::CallBr:
8236 case Instruction::Invoke: {
8237 // For call sites we look at the argument memory behavior attribute (this
8238 // could be recursive!) in order to restrict our own state.
8239 const auto *CB = cast<CallBase>(UserI);
8240
8241 // Give up on operand bundles.
8242 if (CB->isBundleOperand(&U)) {
8243 indicatePessimisticFixpoint();
8244 return;
8245 }
8246
8247 // Calling a function does read the function pointer, maybe write it if the
8248 // function is self-modifying.
8249 if (CB->isCallee(&U)) {
8250 removeAssumedBits(NO_READS);
8251 break;
8252 }
8253
8254 // Adjust the possible access behavior based on the information on the
8255 // argument.
8256 IRPosition Pos;
8257 if (U.get()->getType()->isPointerTy())
8259 else
8261 const auto *MemBehaviorAA =
8262 A.getAAFor<AAMemoryBehavior>(*this, Pos, DepClassTy::OPTIONAL);
8263 if (!MemBehaviorAA)
8264 break;
8265 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8266 // and at least "known".
8267 intersectAssumedBits(MemBehaviorAA->getAssumed());
8268 return;
8269 }
8270 };
8271
8272 // Generally, look at the "may-properties" and adjust the assumed state if we
8273 // did not trigger special handling before.
8274 if (UserI->mayReadFromMemory())
8275 removeAssumedBits(NO_READS);
8276 if (UserI->mayWriteToMemory())
8277 removeAssumedBits(NO_WRITES);
8278}
8279} // namespace
8280
8281/// -------------------- Memory Locations Attributes ---------------------------
8282/// Includes read-none, argmemonly, inaccessiblememonly,
8283/// inaccessiblememorargmemonly
8284/// ----------------------------------------------------------------------------
8285
8288 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8289 return "all memory";
8291 return "no memory";
8292 std::string S = "memory:";
8293 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8294 S += "stack,";
8295 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8296 S += "constant,";
8298 S += "internal global,";
8300 S += "external global,";
8301 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8302 S += "argument,";
8304 S += "inaccessible,";
8305 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8306 S += "malloced,";
8307 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8308 S += "unknown,";
8309 S.pop_back();
8310 return S;
8311}
8312
8313namespace {
8314struct AAMemoryLocationImpl : public AAMemoryLocation {
8315
8316 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8318 AccessKind2Accesses.fill(nullptr);
8319 }
8320
8321 ~AAMemoryLocationImpl() {
8322 // The AccessSets are allocated via a BumpPtrAllocator, we call
8323 // the destructor manually.
8324 for (AccessSet *AS : AccessKind2Accesses)
8325 if (AS)
8326 AS->~AccessSet();
8327 }
8328
8329 /// See AbstractAttribute::initialize(...).
8330 void initialize(Attributor &A) override {
8331 intersectAssumedBits(BEST_STATE);
8332 getKnownStateFromValue(A, getIRPosition(), getState());
8333 AAMemoryLocation::initialize(A);
8334 }
8335
8336 /// Return the memory behavior information encoded in the IR for \p IRP.
8337 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8338 BitIntegerState &State,
8339 bool IgnoreSubsumingPositions = false) {
8340 // For internal functions we ignore `argmemonly` and
8341 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8342 // constant propagation. It is unclear if this is the best way but it is
8343 // unlikely this will cause real performance problems. If we are deriving
8344 // attributes for the anchor function we even remove the attribute in
8345 // addition to ignoring it.
8346 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8347 // MemoryEffects::Other as a possible location.
8348 bool UseArgMemOnly = true;
8349 Function *AnchorFn = IRP.getAnchorScope();
8350 if (AnchorFn && A.isRunOn(*AnchorFn))
8351 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8352
8354 A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8355 for (const Attribute &Attr : Attrs) {
8356 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8357 MemoryEffects ME = Attr.getMemoryEffects();
8358 if (ME.doesNotAccessMemory()) {
8359 State.addKnownBits(NO_LOCAL_MEM | NO_CONST_MEM);
8360 continue;
8361 }
8362 if (ME.onlyAccessesInaccessibleMem()) {
8363 State.addKnownBits(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
8364 continue;
8365 }
8366 if (ME.onlyAccessesArgPointees()) {
8367 if (UseArgMemOnly)
8368 State.addKnownBits(inverseLocation(NO_ARGUMENT_MEM, true, true));
8369 else {
8370 // Remove location information, only keep read/write info.
8371 ME = MemoryEffects(ME.getModRef());
8372 A.manifestAttrs(IRP,
8374 IRP.getAnchorValue().getContext(), ME),
8375 /*ForceReplace*/ true);
8376 }
8377 continue;
8378 }
8380 if (UseArgMemOnly)
8381 State.addKnownBits(inverseLocation(
8382 NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
8383 else {
8384 // Remove location information, only keep read/write info.
8385 ME = MemoryEffects(ME.getModRef());
8386 A.manifestAttrs(IRP,
8388 IRP.getAnchorValue().getContext(), ME),
8389 /*ForceReplace*/ true);
8390 }
8391 continue;
8392 }
8393 }
8394 }
8395
8396 /// See AbstractAttribute::getDeducedAttributes(...).
8397 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8398 SmallVectorImpl<Attribute> &Attrs) const override {
8399 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8400 assert(Attrs.size() == 0);
8401 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8402 if (isAssumedReadNone())
8403 Attrs.push_back(
8405 else if (isAssumedInaccessibleMemOnly())
8408 else if (isAssumedArgMemOnly())
8409 Attrs.push_back(
8411 else if (isAssumedInaccessibleOrArgMemOnly())
8414 }
8415 assert(Attrs.size() <= 1);
8416 }
8417
8418 /// See AbstractAttribute::manifest(...).
8419 ChangeStatus manifest(Attributor &A) override {
8420 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8421 // provide per-location modref information here.
8422 const IRPosition &IRP = getIRPosition();
8423
8424 SmallVector<Attribute, 1> DeducedAttrs;
8425 getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
8426 if (DeducedAttrs.size() != 1)
8427 return ChangeStatus::UNCHANGED;
8428 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8429
8430 return A.manifestAttrs(IRP, Attribute::getWithMemoryEffects(
8431 IRP.getAnchorValue().getContext(), ME));
8432 }
8433
8434 /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8435 bool checkForAllAccessesToMemoryKind(
8436 function_ref<bool(const Instruction *, const Value *, AccessKind,
8437 MemoryLocationsKind)>
8438 Pred,
8439 MemoryLocationsKind RequestedMLK) const override {
8440 if (!isValidState())
8441 return false;
8442
8443 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8444 if (AssumedMLK == NO_LOCATIONS)
8445 return true;
8446
8447 unsigned Idx = 0;
8448 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8449 CurMLK *= 2, ++Idx) {
8450 if (CurMLK & RequestedMLK)
8451 continue;
8452
8453 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8454 for (const AccessInfo &AI : *Accesses)
8455 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8456 return false;
8457 }
8458
8459 return true;
8460 }
8461
8462 ChangeStatus indicatePessimisticFixpoint() override {
8463 // If we give up and indicate a pessimistic fixpoint this instruction will
8464 // become an access for all potential access kinds:
8465 // TODO: Add pointers for argmemonly and globals to improve the results of
8466 // checkForAllAccessesToMemoryKind.
8467 bool Changed = false;
8468 MemoryLocationsKind KnownMLK = getKnown();
8469 Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
8470 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8471 if (!(CurMLK & KnownMLK))
8472 updateStateAndAccessesMap(getState(), CurMLK, I, nullptr, Changed,
8473 getAccessKindFromInst(I));
8474 return AAMemoryLocation::indicatePessimisticFixpoint();
8475 }
8476
8477protected:
8478 /// Helper struct to tie together an instruction that has a read or write
8479 /// effect with the pointer it accesses (if any).
8480 struct AccessInfo {
8481
8482 /// The instruction that caused the access.
8483 const Instruction *I;
8484
8485 /// The base pointer that is accessed, or null if unknown.
8486 const Value *Ptr;
8487
8488 /// The kind of access (read/write/read+write).
8490
8491 bool operator==(const AccessInfo &RHS) const {
8492 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8493 }
8494 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8495 if (LHS.I != RHS.I)
8496 return LHS.I < RHS.I;
8497 if (LHS.Ptr != RHS.Ptr)
8498 return LHS.Ptr < RHS.Ptr;
8499 if (LHS.Kind != RHS.Kind)
8500 return LHS.Kind < RHS.Kind;
8501 return false;
8502 }
8503 };
8504
8505 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8506 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8507 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8508 std::array<AccessSet *, llvm::CTLog2<VALID_STATE>()> AccessKind2Accesses;
8509
8510 /// Categorize the pointer arguments of CB that might access memory in
8511 /// AccessedLoc and update the state and access map accordingly.
8512 void
8513 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8514 AAMemoryLocation::StateType &AccessedLocs,
8515 bool &Changed);
8516
8517 /// Return the kind(s) of location that may be accessed by \p V.
8519 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8520
8521 /// Return the access kind as determined by \p I.
8522 AccessKind getAccessKindFromInst(const Instruction *I) {
8523 AccessKind AK = READ_WRITE;
8524 if (I) {
8525 AK = I->mayReadFromMemory() ? READ : NONE;
8526 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8527 }
8528 return AK;
8529 }
8530
8531 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8532 /// an access of kind \p AK to a \p MLK memory location with the access
8533 /// pointer \p Ptr.
8534 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8535 MemoryLocationsKind MLK, const Instruction *I,
8536 const Value *Ptr, bool &Changed,
8537 AccessKind AK = READ_WRITE) {
8538
8539 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8540 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(MLK)];
8541 if (!Accesses)
8542 Accesses = new (Allocator) AccessSet();
8543 Changed |= Accesses->insert(AccessInfo{I, Ptr, AK}).second;
8544 if (MLK == NO_UNKOWN_MEM)
8545 MLK = NO_LOCATIONS;
8546 State.removeAssumedBits(MLK);
8547 }
8548
8549 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8550 /// arguments, and update the state and access map accordingly.
8551 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8552 AAMemoryLocation::StateType &State, bool &Changed,
8553 unsigned AccessAS = 0);
8554
8555 /// Used to allocate access sets.
8557};
8558
8559void AAMemoryLocationImpl::categorizePtrValue(
8560 Attributor &A, const Instruction &I, const Value &Ptr,
8561 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8562 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8563 << Ptr << " ["
8564 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8565
8566 auto Pred = [&](Value &Obj) {
8567 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8568 // TODO: recognize the TBAA used for constant accesses.
8569 MemoryLocationsKind MLK = NO_LOCATIONS;
8570
8571 // Filter accesses to constant (GPU) memory if we have an AS at the access
8572 // site or the object is known to actually have the associated AS.
8573 if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||
8574 (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&
8575 isIdentifiedObject(&Obj))) &&
8576 AA::isGPU(*I.getModule()))
8577 return true;
8578
8579 if (isa<UndefValue>(&Obj))
8580 return true;
8581 if (isa<Argument>(&Obj)) {
8582 // TODO: For now we do not treat byval arguments as local copies performed
8583 // on the call edge, though, we should. To make that happen we need to
8584 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8585 // would also allow us to mark functions only accessing byval arguments as
8586 // readnone again, arguably their accesses have no effect outside of the
8587 // function, like accesses to allocas.
8588 MLK = NO_ARGUMENT_MEM;
8589 } else if (auto *GV = dyn_cast<GlobalValue>(&Obj)) {
8590 // Reading constant memory is not treated as a read "effect" by the
8591 // function attr pass so we won't neither. Constants defined by TBAA are
8592 // similar. (We know we do not write it because it is constant.)
8593 if (auto *GVar = dyn_cast<GlobalVariable>(GV))
8594 if (GVar->isConstant())
8595 return true;
8596
8597 if (GV->hasLocalLinkage())
8598 MLK = NO_GLOBAL_INTERNAL_MEM;
8599 else
8600 MLK = NO_GLOBAL_EXTERNAL_MEM;
8601 } else if (isa<ConstantPointerNull>(&Obj) &&
8602 (!NullPointerIsDefined(getAssociatedFunction(), AccessAS) ||
8603 !NullPointerIsDefined(getAssociatedFunction(), ObjectAS))) {
8604 return true;
8605 } else if (isa<AllocaInst>(&Obj)) {
8606 MLK = NO_LOCAL_MEM;
8607 } else if (const auto *CB = dyn_cast<CallBase>(&Obj)) {
8608 bool IsKnownNoAlias;
8609 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8611 IsKnownNoAlias))
8612 MLK = NO_MALLOCED_MEM;
8613 else
8614 MLK = NO_UNKOWN_MEM;
8615 } else {
8616 MLK = NO_UNKOWN_MEM;
8617 }
8618
8619 assert(MLK != NO_LOCATIONS && "No location specified!");
8620 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8621 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8622 updateStateAndAccessesMap(State, MLK, &I, &Obj, Changed,
8623 getAccessKindFromInst(&I));
8624
8625 return true;
8626 };
8627
8628 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8630 if (!AA || !AA->forallUnderlyingObjects(Pred, AA::Intraprocedural)) {
8631 LLVM_DEBUG(
8632 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8633 updateStateAndAccessesMap(State, NO_UNKOWN_MEM, &I, nullptr, Changed,
8634 getAccessKindFromInst(&I));
8635 return;
8636 }
8637
8638 LLVM_DEBUG(
8639 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8640 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8641}
8642
8643void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8644 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8645 bool &Changed) {
8646 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8647
8648 // Skip non-pointer arguments.
8649 const Value *ArgOp = CB.getArgOperand(ArgNo);
8650 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8651 continue;
8652
8653 // Skip readnone arguments.
8654 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8655 const auto *ArgOpMemLocationAA =
8656 A.getAAFor<AAMemoryBehavior>(*this, ArgOpIRP, DepClassTy::OPTIONAL);
8657
8658 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8659 continue;
8660
8661 // Categorize potentially accessed pointer arguments as if there was an
8662 // access instruction with them as pointer.
8663 categorizePtrValue(A, CB, *ArgOp, AccessedLocs, Changed);
8664 }
8665}
8666
8668AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8669 bool &Changed) {
8670 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8671 << I << "\n");
8672
8673 AAMemoryLocation::StateType AccessedLocs;
8674 AccessedLocs.intersectAssumedBits(NO_LOCATIONS);
8675
8676 if (auto *CB = dyn_cast<CallBase>(&I)) {
8677
8678 // First check if we assume any memory is access is visible.
8679 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8681 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8682 << " [" << CBMemLocationAA << "]\n");
8683 if (!CBMemLocationAA) {
8684 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr,
8685 Changed, getAccessKindFromInst(&I));
8686 return NO_UNKOWN_MEM;
8687 }
8688
8689 if (CBMemLocationAA->isAssumedReadNone())
8690 return NO_LOCATIONS;
8691
8692 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8693 updateStateAndAccessesMap(AccessedLocs, NO_INACCESSIBLE_MEM, &I, nullptr,
8694 Changed, getAccessKindFromInst(&I));
8695 return AccessedLocs.getAssumed();
8696 }
8697
8698 uint32_t CBAssumedNotAccessedLocs =
8699 CBMemLocationAA->getAssumedNotAccessedLocation();
8700
8701 // Set the argmemonly and global bit as we handle them separately below.
8702 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8703 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8704
8705 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8706 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8707 continue;
8708 updateStateAndAccessesMap(AccessedLocs, CurMLK, &I, nullptr, Changed,
8709 getAccessKindFromInst(&I));
8710 }
8711
8712 // Now handle global memory if it might be accessed. This is slightly tricky
8713 // as NO_GLOBAL_MEM has multiple bits set.
8714 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8715 if (HasGlobalAccesses) {
8716 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8717 AccessKind Kind, MemoryLocationsKind MLK) {
8718 updateStateAndAccessesMap(AccessedLocs, MLK, &I, Ptr, Changed,
8719 getAccessKindFromInst(&I));
8720 return true;
8721 };
8722 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8723 AccessPred, inverseLocation(NO_GLOBAL_MEM, false, false)))
8724 return AccessedLocs.getWorstState();
8725 }
8726
8727 LLVM_DEBUG(
8728 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8729 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8730
8731 // Now handle argument memory if it might be accessed.
8732 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8733 if (HasArgAccesses)
8734 categorizeArgumentPointerLocations(A, *CB, AccessedLocs, Changed);
8735
8736 LLVM_DEBUG(
8737 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8738 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8739
8740 return AccessedLocs.getAssumed();
8741 }
8742
8743 if (const Value *Ptr = getPointerOperand(&I, /* AllowVolatile */ true)) {
8744 LLVM_DEBUG(
8745 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8746 << I << " [" << *Ptr << "]\n");
8747 categorizePtrValue(A, I, *Ptr, AccessedLocs, Changed,
8748 Ptr->getType()->getPointerAddressSpace());
8749 return AccessedLocs.getAssumed();
8750 }
8751
8752 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8753 << I << "\n");
8754 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr, Changed,
8755 getAccessKindFromInst(&I));
8756 return AccessedLocs.getAssumed();
8757}
8758
8759/// An AA to represent the memory behavior function attributes.
8760struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8761 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8762 : AAMemoryLocationImpl(IRP, A) {}
8763
8764 /// See AbstractAttribute::updateImpl(Attributor &A).
8765 ChangeStatus updateImpl(Attributor &A) override {
8766
8767 const auto *MemBehaviorAA =
8768 A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
8769 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8770 if (MemBehaviorAA->isKnownReadNone())
8771 return indicateOptimisticFixpoint();
8773 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8774 A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
8775 return ChangeStatus::UNCHANGED;
8776 }
8777
8778 // The current assumed state used to determine a change.
8779 auto AssumedState = getAssumed();
8780 bool Changed = false;
8781
8782 auto CheckRWInst = [&](Instruction &I) {
8783 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8784 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8785 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8786 removeAssumedBits(inverseLocation(MLK, false, false));
8787 // Stop once only the valid bit set in the *not assumed location*, thus
8788 // once we don't actually exclude any memory locations in the state.
8789 return getAssumedNotAccessedLocation() != VALID_STATE;
8790 };
8791
8792 bool UsedAssumedInformation = false;
8793 if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
8794 UsedAssumedInformation))
8795 return indicatePessimisticFixpoint();
8796
8797 Changed |= AssumedState != getAssumed();
8798 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8799 }
8800
8801 /// See AbstractAttribute::trackStatistics()
8802 void trackStatistics() const override {
8803 if (isAssumedReadNone())
8804 STATS_DECLTRACK_FN_ATTR(readnone)
8805 else if (isAssumedArgMemOnly())
8806 STATS_DECLTRACK_FN_ATTR(argmemonly)
8807 else if (isAssumedInaccessibleMemOnly())
8808 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8809 else if (isAssumedInaccessibleOrArgMemOnly())
8810 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8811 }
8812};
8813
8814/// AAMemoryLocation attribute for call sites.
8815struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8816 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8817 : AAMemoryLocationImpl(IRP, A) {}
8818
8819 /// See AbstractAttribute::updateImpl(...).
8820 ChangeStatus updateImpl(Attributor &A) override {
8821 // TODO: Once we have call site specific value information we can provide
8822 // call site specific liveness liveness information and then it makes
8823 // sense to specialize attributes for call sites arguments instead of
8824 // redirecting requests to the callee argument.
8825 Function *F = getAssociatedFunction();
8826 const IRPosition &FnPos = IRPosition::function(*F);
8827 auto *FnAA =
8828 A.getAAFor<AAMemoryLocation>(*this, FnPos, DepClassTy::REQUIRED);
8829 if (!FnAA)
8830 return indicatePessimisticFixpoint();
8831 bool Changed = false;
8832 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
8833 AccessKind Kind, MemoryLocationsKind MLK) {
8834 updateStateAndAccessesMap(getState(), MLK, I, Ptr, Changed,
8835 getAccessKindFromInst(I));
8836 return true;
8837 };
8838 if (!FnAA->checkForAllAccessesToMemoryKind(AccessPred, ALL_LOCATIONS))
8839 return indicatePessimisticFixpoint();
8840 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8841 }
8842
8843 /// See AbstractAttribute::trackStatistics()
8844 void trackStatistics() const override {
8845 if (isAssumedReadNone())
8846 STATS_DECLTRACK_CS_ATTR(readnone)
8847 }
8848};
8849} // namespace
8850
8851/// ------------------ denormal-fp-math Attribute -------------------------
8852
8853namespace {
8854struct AADenormalFPMathImpl : public AADenormalFPMath {
8855 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
8856 : AADenormalFPMath(IRP, A) {}
8857
8858 const std::string getAsStr(Attributor *A) const override {
8859 std::string Str("AADenormalFPMath[");
8861
8862 DenormalState Known = getKnown();
8863 if (Known.Mode.isValid())
8864 OS << "denormal-fp-math=" << Known.Mode;
8865 else
8866 OS << "invalid";
8867
8868 if (Known.ModeF32.isValid())
8869 OS << " denormal-fp-math-f32=" << Known.ModeF32;
8870 OS << ']';
8871 return Str;
8872 }
8873};
8874
8875struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
8876 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
8877 : AADenormalFPMathImpl(IRP, A) {}
8878
8879 void initialize(Attributor &A) override {
8880 const Function *F = getAnchorScope();
8881 DenormalMode Mode = F->getDenormalModeRaw();
8882 DenormalMode ModeF32 = F->getDenormalModeF32Raw();
8883
8884 // TODO: Handling this here prevents handling the case where a callee has a
8885 // fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from
8886 // a function with a fully fixed mode.
8887 if (ModeF32 == DenormalMode::getInvalid())
8888 ModeF32 = Mode;
8889 Known = DenormalState{Mode, ModeF32};
8890 if (isModeFixed())
8891 indicateFixpoint();
8892 }
8893
8894 ChangeStatus updateImpl(Attributor &A) override {
8895 ChangeStatus Change = ChangeStatus::UNCHANGED;
8896
8897 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
8898 Function *Caller = CS.getInstruction()->getFunction();
8899 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
8900 << "->" << getAssociatedFunction()->getName() << '\n');
8901
8902 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
8903 *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
8904 if (!CallerInfo)
8905 return false;
8906
8907 Change = Change | clampStateAndIndicateChange(this->getState(),
8908 CallerInfo->getState());
8909 return true;
8910 };
8911
8912 bool AllCallSitesKnown = true;
8913 if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown))
8914 return indicatePessimisticFixpoint();
8915
8916 if (Change == ChangeStatus::CHANGED && isModeFixed())
8917 indicateFixpoint();
8918 return Change;
8919 }
8920
8921 ChangeStatus manifest(Attributor &A) override {
8922 LLVMContext &Ctx = getAssociatedFunction()->getContext();
8923
8924 SmallVector<Attribute, 2> AttrToAdd;
8925 SmallVector<StringRef, 2> AttrToRemove;
8926 if (Known.Mode == DenormalMode::getDefault()) {
8927 AttrToRemove.push_back("denormal-fp-math");
8928 } else {
8929 AttrToAdd.push_back(
8930 Attribute::get(Ctx, "denormal-fp-math", Known.Mode.str()));
8931 }
8932
8933 if (Known.ModeF32 != Known.Mode) {
8934 AttrToAdd.push_back(
8935 Attribute::get(Ctx, "denormal-fp-math-f32", Known.ModeF32.str()));
8936 } else {
8937 AttrToRemove.push_back("denormal-fp-math-f32");
8938 }
8939
8940 auto &IRP = getIRPosition();
8941
8942 // TODO: There should be a combined add and remove API.
8943 return A.removeAttrs(IRP, AttrToRemove) |
8944 A.manifestAttrs(IRP, AttrToAdd, /*ForceReplace=*/true);
8945 }
8946
8947 void trackStatistics() const override {
8948 STATS_DECLTRACK_FN_ATTR(denormal_fp_math)
8949 }
8950};
8951} // namespace
8952
8953/// ------------------ Value Constant Range Attribute -------------------------
8954
8955namespace {
8956struct AAValueConstantRangeImpl : AAValueConstantRange {
8957 using StateType = IntegerRangeState;
8958 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
8959 : AAValueConstantRange(IRP, A) {}
8960
8961 /// See AbstractAttribute::initialize(..).
8962 void initialize(Attributor &A) override {
8963 if (A.hasSimplificationCallback(getIRPosition())) {
8964 indicatePessimisticFixpoint();
8965 return;
8966 }
8967
8968 // Intersect a range given by SCEV.
8969 intersectKnown(getConstantRangeFromSCEV(A, getCtxI()));
8970
8971 // Intersect a range given by LVI.
8972 intersectKnown(getConstantRangeFromLVI(A, getCtxI()));
8973 }
8974
8975 /// See AbstractAttribute::getAsStr().
8976 const std::string getAsStr(Attributor *A) const override {
8977 std::string Str;
8979 OS << "range(" << getBitWidth() << ")<";
8980 getKnown().print(OS);
8981 OS << " / ";
8982 getAssumed().print(OS);
8983 OS << ">";
8984 return Str;
8985 }
8986
8987 /// Helper function to get a SCEV expr for the associated value at program
8988 /// point \p I.
8989 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
8990 if (!getAnchorScope())
8991 return nullptr;
8992
8993 ScalarEvolution *SE =
8994 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
8995 *getAnchorScope());
8996
8997 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
8998 *getAnchorScope());
8999
9000 if (!SE || !LI)
9001 return nullptr;
9002
9003 const SCEV *S = SE->getSCEV(&getAssociatedValue());
9004 if (!I)
9005 return S;
9006
9007 return SE->getSCEVAtScope(S, LI->getLoopFor(I->getParent()));
9008 }
9009
9010 /// Helper function to get a range from SCEV for the associated value at
9011 /// program point \p I.
9012 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9013 const Instruction *I = nullptr) const {
9014 if (!getAnchorScope())
9015 return getWorstState(getBitWidth());
9016
9017 ScalarEvolution *SE =
9018 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9019 *getAnchorScope());
9020
9021 const SCEV *S = getSCEV(A, I);
9022 if (!SE || !S)
9023 return getWorstState(getBitWidth());
9024
9025 return SE->getUnsignedRange(S);
9026 }
9027
9028 /// Helper function to get a range from LVI for the associated value at
9029 /// program point \p I.
9031 getConstantRangeFromLVI(Attributor &A,
9032 const Instruction *CtxI = nullptr) const {
9033 if (!getAnchorScope())
9034 return getWorstState(getBitWidth());
9035
9036 LazyValueInfo *LVI =
9037 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9038 *getAnchorScope());
9039
9040 if (!LVI || !CtxI)
9041 return getWorstState(getBitWidth());
9042 return LVI->getConstantRange(&getAssociatedValue(),
9043 const_cast<Instruction *>(CtxI),
9044 /*UndefAllowed*/ false);
9045 }
9046
9047 /// Return true if \p CtxI is valid for querying outside analyses.
9048 /// This basically makes sure we do not ask intra-procedural analysis
9049 /// about a context in the wrong function or a context that violates
9050 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9051 /// if the original context of this AA is OK or should be considered invalid.
9052 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9053 const Instruction *CtxI,
9054 bool AllowAACtxI) const {
9055 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9056 return false;
9057
9058 // Our context might be in a different function, neither intra-procedural
9059 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9060 if (!AA::isValidInScope(getAssociatedValue(), CtxI->getFunction()))
9061 return false;
9062
9063 // If the context is not dominated by the value there are paths to the
9064 // context that do not define the value. This cannot be handled by
9065 // LazyValueInfo so we need to bail.
9066 if (auto *I = dyn_cast<Instruction>(&getAssociatedValue())) {
9067 InformationCache &InfoCache = A.getInfoCache();
9068 const DominatorTree *DT =
9070 *I->getFunction());
9071 return DT && DT->dominates(I, CtxI);
9072 }
9073
9074 return true;
9075 }
9076
9077 /// See AAValueConstantRange::getKnownConstantRange(..).
9079 getKnownConstantRange(Attributor &A,
9080 const Instruction *CtxI = nullptr) const override {
9081 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9082 /* AllowAACtxI */ false))
9083 return getKnown();
9084
9085 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9086 ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
9087 return getKnown().intersectWith(SCEVR).intersectWith(LVIR);
9088 }
9089
9090 /// See AAValueConstantRange::getAssumedConstantRange(..).
9092 getAssumedConstantRange(Attributor &A,
9093 const Instruction *CtxI = nullptr) const override {
9094 // TODO: Make SCEV use Attributor assumption.
9095 // We may be able to bound a variable range via assumptions in
9096 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9097 // evolve to x^2 + x, then we can say that y is in [2, 12].
9098 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9099 /* AllowAACtxI */ false))
9100 return getAssumed();
9101
9102 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9103 ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
9104 return getAssumed().intersectWith(SCEVR).intersectWith(LVIR);
9105 }
9106
9107 /// Helper function to create MDNode for range metadata.
9108 static MDNode *
9109 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9110 const ConstantRange &AssumedConstantRange) {
9111 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(ConstantInt::get(
9112 Ty, AssumedConstantRange.getLower())),
9113 ConstantAsMetadata::get(ConstantInt::get(
9114 Ty, AssumedConstantRange.getUpper()))};
9115 return MDNode::get(Ctx, LowAndHigh);
9116 }
9117
9118 /// Return true if \p Assumed is included in \p KnownRanges.
9119 static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) {
9120
9121 if (Assumed.isFullSet())
9122 return false;
9123
9124 if (!KnownRanges)
9125 return true;
9126
9127 // If multiple ranges are annotated in IR, we give up to annotate assumed
9128 // range for now.
9129
9130 // TODO: If there exists a known range which containts assumed range, we
9131 // can say assumed range is better.
9132 if (KnownRanges->getNumOperands() > 2)
9133 return false;
9134
9136 mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
9138 mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
9139
9140 ConstantRange Known(Lower->getValue(), Upper->getValue());
9141 return Known.contains(Assumed) && Known != Assumed;
9142 }
9143
9144 /// Helper function to set range metadata.
9145 static bool
9146 setRangeMetadataIfisBetterRange(Instruction *I,
9147 const ConstantRange &AssumedConstantRange) {
9148 auto *OldRangeMD = I->getMetadata(LLVMContext::MD_range);
9149 if (isBetterRange(AssumedConstantRange, OldRangeMD)) {
9150 if (!AssumedConstantRange.isEmptySet()) {
9151 I->setMetadata(LLVMContext::MD_range,
9152 getMDNodeForConstantRange(I->getType(), I->getContext(),
9153 AssumedConstantRange));
9154 return true;
9155 }
9156 }
9157 return false;
9158 }
9159
9160 /// See AbstractAttribute::manifest()
9161 ChangeStatus manifest(Attributor &A) override {
9162 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9163 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9164 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9165
9166 auto &V = getAssociatedValue();
9167 if (!AssumedConstantRange.isEmptySet() &&
9168 !AssumedConstantRange.isSingleElement()) {
9169 if (Instruction *I = dyn_cast<Instruction>(&V)) {
9170 assert(I == getCtxI() && "Should not annotate an instruction which is "
9171 "not the context instruction");
9172 if (isa<CallInst>(I) || isa<LoadInst>(I))
9173 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9174 Changed = ChangeStatus::CHANGED;
9175 }
9176 }
9177
9178 return Changed;
9179 }
9180};
9181
9182struct AAValueConstantRangeArgument final
9183 : AAArgumentFromCallSiteArguments<
9184 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9185 true /* BridgeCallBaseContext */> {
9186 using Base = AAArgumentFromCallSiteArguments<
9187 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9188 true /* BridgeCallBaseContext */>;
9189 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9190 : Base(IRP, A) {}
9191
9192 /// See AbstractAttribute::trackStatistics()
9193 void trackStatistics() const override {
9194 STATS_DECLTRACK_ARG_ATTR(value_range)
9195 }
9196};
9197
9198struct AAValueConstantRangeReturned
9199 : AAReturnedFromReturnedValues<AAValueConstantRange,
9200 AAValueConstantRangeImpl,
9201 AAValueConstantRangeImpl::StateType,
9202 /* PropogateCallBaseContext */ true> {
9203 using Base =
9204 AAReturnedFromReturnedValues<AAValueConstantRange,
9205 AAValueConstantRangeImpl,
9207 /* PropogateCallBaseContext */ true>;
9208 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9209 : Base(IRP, A) {}
9210
9211 /// See AbstractAttribute::initialize(...).
9212 void initialize(Attributor &A) override {
9213 if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
9214 indicatePessimisticFixpoint();
9215 }
9216
9217 /// See AbstractAttribute::trackStatistics()
9218 void trackStatistics() const override {
9219 STATS_DECLTRACK_FNRET_ATTR(value_range)
9220 }
9221};
9222
9223struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9224 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9225 : AAValueConstantRangeImpl(IRP, A) {}
9226
9227 /// See AbstractAttribute::initialize(...).
9228 void initialize(Attributor &A) override {
9229 AAValueConstantRangeImpl::initialize(A);
9230 if (isAtFixpoint())
9231 return;
9232
9233 Value &V = getAssociatedValue();
9234
9235 if (auto *C = dyn_cast<ConstantInt>(&V)) {
9236 unionAssumed(ConstantRange(C->getValue()));
9237 indicateOptimisticFixpoint();
9238 return;
9239 }
9240
9241 if (isa<UndefValue>(&V)) {
9242 // Collapse the undef state to 0.
9243 unionAssumed(ConstantRange(APInt(getBitWidth(), 0)));
9244 indicateOptimisticFixpoint();
9245 return;
9246 }
9247
9248 if (isa<CallBase>(&V))
9249 return;
9250
9251 if (isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<CastInst>(&V))
9252 return;
9253
9254 // If it is a load instruction with range metadata, use it.
9255 if (LoadInst *LI = dyn_cast<LoadInst>(&V))
9256 if (auto *RangeMD = LI->getMetadata(LLVMContext::MD_range)) {
9257 intersectKnown(getConstantRangeFromMetadata(*RangeMD));
9258 return;
9259 }
9260
9261 // We can work with PHI and select instruction as we traverse their operands
9262 // during update.
9263 if (isa<SelectInst>(V) || isa<PHINode>(V))
9264 return;
9265
9266 // Otherwise we give up.
9267 indicatePessimisticFixpoint();
9268
9269 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9270 << getAssociatedValue() << "\n");
9271 }
9272
9273 bool calculateBinaryOperator(
9275 const Instruction *CtxI,
9277 Value *LHS = BinOp->getOperand(0);
9278 Value *RHS = BinOp->getOperand(1);
9279
9280 // Simplify the operands first.
9281 bool UsedAssumedInformation = false;
9282 const auto &SimplifiedLHS = A.getAssumedSimplified(
9283 IRPosition::value(*LHS, getCallBaseContext()), *this,
9284 UsedAssumedInformation, AA::Interprocedural);
9285 if (!SimplifiedLHS.has_value())
9286 return true;
9287 if (!*SimplifiedLHS)
9288 return false;
9289 LHS = *SimplifiedLHS;
9290
9291 const auto &SimplifiedRHS = A.getAssumedSimplified(
9292 IRPosition::value(*RHS, getCallBaseContext()), *this,
9293 UsedAssumedInformation, AA::Interprocedural);
9294 if (!SimplifiedRHS.has_value())
9295 return true;
9296 if (!*SimplifiedRHS)
9297 return false;
9298 RHS = *SimplifiedRHS;
9299
9300 // TODO: Allow non integers as well.
9301 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9302 return false;
9303
9304 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9305 *this, IRPosition::value(*LHS, getCallBaseContext()),
9306 DepClassTy::REQUIRED);
9307 if (!LHSAA)
9308 return false;
9309 QuerriedAAs.push_back(LHSAA);
9310 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9311
9312 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9313 *this, IRPosition::value(*RHS, getCallBaseContext()),
9314 DepClassTy::REQUIRED);
9315 if (!RHSAA)
9316 return false;
9317 QuerriedAAs.push_back(RHSAA);
9318 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9319
9320 auto AssumedRange = LHSAARange.binaryOp(BinOp->getOpcode(), RHSAARange);
9321
9322 T.unionAssumed(AssumedRange);
9323
9324 // TODO: Track a known state too.
9325
9326 return T.isValidState();
9327 }
9328
9329 bool calculateCastInst(
9331 const Instruction *CtxI,
9333 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9334 // TODO: Allow non integers as well.
9335 Value *OpV = CastI->getOperand(0);
9336
9337 // Simplify the operand first.
9338 bool UsedAssumedInformation = false;
9339 const auto &SimplifiedOpV = A.getAssumedSimplified(
9340 IRPosition::value(*OpV, getCallBaseContext()), *this,
9341 UsedAssumedInformation, AA::Interprocedural);
9342 if (!SimplifiedOpV.has_value())
9343 return true;
9344 if (!*SimplifiedOpV)
9345 return false;
9346 OpV = *SimplifiedOpV;
9347
9348 if (!OpV->getType()->isIntegerTy())
9349 return false;
9350
9351 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9352 *this, IRPosition::value(*OpV, getCallBaseContext()),
9353 DepClassTy::REQUIRED);
9354 if (!OpAA)
9355 return false;
9356 QuerriedAAs.push_back(OpAA);
9357 T.unionAssumed(OpAA->getAssumed().castOp(CastI->getOpcode(),
9358 getState().getBitWidth()));
9359 return T.isValidState();
9360 }
9361
9362 bool
9363 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9364 const Instruction *CtxI,
9366 Value *LHS = CmpI->getOperand(0);
9367 Value *RHS = CmpI->getOperand(1);
9368
9369 // Simplify the operands first.
9370 bool UsedAssumedInformation = false;
9371 const auto &SimplifiedLHS = A.getAssumedSimplified(
9372 IRPosition::value(*LHS, getCallBaseContext()), *this,
9373 UsedAssumedInformation, AA::Interprocedural);
9374 if (!SimplifiedLHS.has_value())
9375 return true;
9376 if (!*SimplifiedLHS)
9377 return false;
9378 LHS = *SimplifiedLHS;
9379
9380 const auto &SimplifiedRHS = A.getAssumedSimplified(
9381 IRPosition::value(*RHS, getCallBaseContext()), *this,
9382 UsedAssumedInformation, AA::Interprocedural);
9383 if (!SimplifiedRHS.has_value())
9384 return true;
9385 if (!*SimplifiedRHS)
9386 return false;
9387 RHS = *SimplifiedRHS;
9388
9389 // TODO: Allow non integers as well.
9390 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9391 return false;
9392
9393 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9394 *this, IRPosition::value(*LHS, getCallBaseContext()),
9395 DepClassTy::REQUIRED);
9396 if (!LHSAA)
9397 return false;
9398 QuerriedAAs.push_back(LHSAA);
9399 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9400 *this, IRPosition::value(*RHS, getCallBaseContext()),
9401 DepClassTy::REQUIRED);
9402 if (!RHSAA)
9403 return false;
9404 QuerriedAAs.push_back(RHSAA);
9405 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9406 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9407
9408 // If one of them is empty set, we can't decide.
9409 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9410 return true;
9411
9412 bool MustTrue = false, MustFalse = false;
9413
9414 auto AllowedRegion =
9416
9417 if (AllowedRegion.intersectWith(LHSAARange).isEmptySet())
9418 MustFalse = true;
9419
9420 if (LHSAARange.icmp(CmpI->getPredicate(), RHSAARange))
9421 MustTrue = true;
9422
9423 assert((!MustTrue || !MustFalse) &&
9424 "Either MustTrue or MustFalse should be false!");
9425
9426 if (MustTrue)
9427 T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9428 else if (MustFalse)
9429 T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9430 else
9431 T.unionAssumed(ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9432
9433 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9434 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9435 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9436 << *RHSAA);
9437
9438 // TODO: Track a known state too.
9439 return T.isValidState();
9440 }
9441
9442 /// See AbstractAttribute::updateImpl(...).
9443 ChangeStatus updateImpl(Attributor &A) override {
9444
9446 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9447 Instruction *I = dyn_cast<Instruction>(&V);
9448 if (!I || isa<CallBase>(I)) {
9449
9450 // Simplify the operand first.
9451 bool UsedAssumedInformation = false;
9452 const auto &SimplifiedOpV = A.getAssumedSimplified(
9453 IRPosition::value(V, getCallBaseContext()), *this,
9454 UsedAssumedInformation, AA::Interprocedural);
9455 if (!SimplifiedOpV.has_value())
9456 return true;
9457 if (!*SimplifiedOpV)
9458 return false;
9459 Value *VPtr = *SimplifiedOpV;
9460
9461 // If the value is not instruction, we query AA to Attributor.
9462 const auto *AA = A.getAAFor<AAValueConstantRange>(
9463 *this, IRPosition::value(*VPtr, getCallBaseContext()),
9464 DepClassTy::REQUIRED);
9465
9466 // Clamp operator is not used to utilize a program point CtxI.
9467 if (AA)
9468 T.unionAssumed(AA->getAssumedConstantRange(A, CtxI));
9469 else
9470 return false;
9471
9472 return T.isValidState();
9473 }
9474
9476 if (auto *BinOp = dyn_cast<BinaryOperator>(I)) {
9477 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9478 return false;
9479 } else if (auto *CmpI = dyn_cast<CmpInst>(I)) {
9480 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9481 return false;
9482 } else if (auto *CastI = dyn_cast<CastInst>(I)) {
9483 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9484 return false;
9485 } else {
9486 // Give up with other instructions.
9487 // TODO: Add other instructions
9488
9489 T.indicatePessimisticFixpoint();
9490 return false;
9491 }
9492
9493 // Catch circular reasoning in a pessimistic way for now.
9494 // TODO: Check how the range evolves and if we stripped anything, see also
9495 // AADereferenceable or AAAlign for similar situations.
9496 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9497 if (QueriedAA != this)
9498 continue;
9499 // If we are in a stady state we do not need to worry.
9500 if (T.getAssumed() == getState().getAssumed())
9501 continue;
9502 T.indicatePessimisticFixpoint();
9503 }
9504
9505 return T.isValidState();
9506 };
9507
9508 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9509 return indicatePessimisticFixpoint();
9510
9511 // Ensure that long def-use chains can't cause circular reasoning either by
9512 // introducing a cutoff below.
9513 if (clampStateAndIndicateChange(getState(), T) == ChangeStatus::UNCHANGED)
9514 return ChangeStatus::UNCHANGED;
9515 if (++NumChanges > MaxNumChanges) {
9516 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9517 << " but only " << MaxNumChanges
9518 << " are allowed to avoid cyclic reasoning.");
9519 return indicatePessimisticFixpoint();
9520 }
9521 return ChangeStatus::CHANGED;
9522 }
9523
9524 /// See AbstractAttribute::trackStatistics()
9525 void trackStatistics() const override {
9527 }
9528
9529 /// Tracker to bail after too many widening steps of the constant range.
9530 int NumChanges = 0;
9531
9532 /// Upper bound for the number of allowed changes (=widening steps) for the
9533 /// constant range before we give up.
9534 static constexpr int MaxNumChanges = 5;
9535};
9536
9537struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9538 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9539 : AAValueConstantRangeImpl(IRP, A) {}
9540
9541 /// See AbstractAttribute::initialize(...).
9542 ChangeStatus updateImpl(Attributor &A) override {
9543 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9544 "not be called");
9545 }
9546
9547 /// See AbstractAttribute::trackStatistics()
9548 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9549};
9550
9551struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9552 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9553 : AAValueConstantRangeFunction(IRP, A) {}
9554
9555 /// See AbstractAttribute::trackStatistics()
9556 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9557};
9558
9559struct AAValueConstantRangeCallSiteReturned
9560 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9561 AAValueConstantRangeImpl::StateType,
9562 /* IntroduceCallBaseContext */ true> {
9563 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9564 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9565 AAValueConstantRangeImpl::StateType,
9566 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9567
9568 /// See AbstractAttribute::initialize(...).
9569 void initialize(Attributor &A) override {
9570 // If it is a load instruction with range metadata, use the metadata.
9571 if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue()))
9572 if (auto *RangeMD = CI->getMetadata(LLVMContext::MD_range))
9573 intersectKnown(getConstantRangeFromMetadata(*RangeMD));
9574
9575 AAValueConstantRangeImpl::initialize(A);
9576 }
9577
9578 /// See AbstractAttribute::trackStatistics()
9579 void trackStatistics() const override {
9580 STATS_DECLTRACK_CSRET_ATTR(value_range)
9581 }
9582};
9583struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9584 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9585 : AAValueConstantRangeFloating(IRP, A) {}
9586
9587 /// See AbstractAttribute::manifest()
9588 ChangeStatus manifest(Attributor &A) override {
9589 return ChangeStatus::UNCHANGED;
9590 }
9591
9592 /// See AbstractAttribute::trackStatistics()
9593 void trackStatistics() const override {
9594 STATS_DECLTRACK_CSARG_ATTR(value_range)
9595 }
9596};
9597} // namespace
9598
9599/// ------------------ Potential Values Attribute -------------------------
9600
9601namespace {
9602struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9603 using StateType = PotentialConstantIntValuesState;
9604
9605 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9606 : AAPotentialConstantValues(IRP, A) {}
9607
9608 /// See AbstractAttribute::initialize(..).
9609 void initialize(Attributor &A) override {
9610 if (A.hasSimplificationCallback(getIRPosition()))
9611 indicatePessimisticFixpoint();
9612 else
9613 AAPotentialConstantValues::initialize(A);
9614 }
9615
9616 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9617 bool &ContainsUndef, bool ForSelf) {
9619 bool UsedAssumedInformation = false;
9620 if (!A.getAssumedSimplifiedValues(IRP, *this, Values, AA::Interprocedural,
9621 UsedAssumedInformation)) {
9622 // Avoid recursion when the caller is computing constant values for this
9623 // IRP itself.
9624 if (ForSelf)
9625 return false;
9626 if (!IRP.getAssociatedType()->isIntegerTy())
9627 return false;
9628 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9629 *this, IRP, DepClassTy::REQUIRED);
9630 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9631 return false;
9632 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9633 S = PotentialValuesAA->getState().getAssumedSet();
9634 return true;
9635 }
9636
9637 // Copy all the constant values, except UndefValue. ContainsUndef is true
9638 // iff Values contains only UndefValue instances. If there are other known
9639 // constants, then UndefValue is dropped.
9640 ContainsUndef = false;
9641 for (auto &It : Values) {
9642 if (isa<UndefValue>(It.getValue())) {
9643 ContainsUndef = true;
9644 continue;
9645 }
9646 auto *CI = dyn_cast<ConstantInt>(It.getValue());
9647 if (!CI)
9648 return false;
9649 S.insert(CI->getValue());
9650 }
9651 ContainsUndef &= S.empty();
9652
9653 return true;
9654 }
9655
9656 /// See AbstractAttribute::getAsStr().
9657 const std::string getAsStr(Attributor *A) const override {
9658 std::string Str;
9660 OS << getState();
9661 return Str;
9662 }
9663
9664 /// See AbstractAttribute::updateImpl(...).
9665 ChangeStatus updateImpl(Attributor &A) override {
9666 return indicatePessimisticFixpoint();
9667 }
9668};
9669
9670struct AAPotentialConstantValuesArgument final
9671 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9672 AAPotentialConstantValuesImpl,
9673 PotentialConstantIntValuesState> {
9674 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9675 AAPotentialConstantValuesImpl,
9677 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9678 : Base(IRP, A) {}
9679
9680 /// See AbstractAttribute::trackStatistics()
9681 void trackStatistics() const override {
9682 STATS_DECLTRACK_ARG_ATTR(potential_values)
9683 }
9684};
9685
9686struct AAPotentialConstantValuesReturned
9687 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9688 AAPotentialConstantValuesImpl> {
9689 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9690 AAPotentialConstantValuesImpl>;
9691 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9692 : Base(IRP, A) {}
9693
9694 void initialize(Attributor &A) override {
9695 if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
9696 indicatePessimisticFixpoint();
9697 Base::initialize(A);
9698 }
9699
9700 /// See AbstractAttribute::trackStatistics()
9701 void trackStatistics() const override {
9702 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9703 }
9704};
9705
9706struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9707 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9708 : AAPotentialConstantValuesImpl(IRP, A) {}
9709
9710 /// See AbstractAttribute::initialize(..).
9711 void initialize(Attributor &A) override {
9712 AAPotentialConstantValuesImpl::initialize(A);
9713 if (isAtFixpoint())
9714 return;
9715
9716 Value &V = getAssociatedValue();
9717
9718 if (auto *C = dyn_cast<ConstantInt>(&V)) {
9719 unionAssumed(C->getValue());
9720 indicateOptimisticFixpoint();
9721 return;
9722 }
9723
9724 if (isa<UndefValue>(&V)) {
9725 unionAssumedWithUndef();
9726 indicateOptimisticFixpoint();
9727 return;
9728 }
9729
9730 if (isa<BinaryOperator>(&V) || isa<ICmpInst>(&V) || isa<CastInst>(&V))
9731 return;
9732
9733 if (isa<SelectInst>(V) || isa<PHINode>(V) || isa<LoadInst>(V))
9734 return;
9735
9736 indicatePessimisticFixpoint();
9737
9738 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9739 << getAssociatedValue() << "\n");
9740 }
9741
9742 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9743 const APInt &RHS) {
9744 return ICmpInst::compare(LHS, RHS, ICI->getPredicate());
9745 }
9746
9747 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9748 uint32_t ResultBitWidth) {
9749 Instruction::CastOps CastOp = CI->getOpcode();
9750 switch (CastOp) {
9751 default:
9752 llvm_unreachable("unsupported or not integer cast");
9753 case Instruction::Trunc:
9754 return Src.trunc(ResultBitWidth);
9755 case Instruction::SExt:
9756 return Src.sext(ResultBitWidth);
9757 case Instruction::ZExt:
9758 return Src.zext(ResultBitWidth);
9759 case Instruction::BitCast:
9760 return Src;
9761 }
9762 }
9763
9764 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9765 const APInt &LHS, const APInt &RHS,
9766 bool &SkipOperation, bool &Unsupported) {
9767 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9768 // Unsupported is set to true when the binary operator is not supported.
9769 // SkipOperation is set to true when UB occur with the given operand pair
9770 // (LHS, RHS).
9771 // TODO: we should look at nsw and nuw keywords to handle operations
9772 // that create poison or undef value.
9773 switch (BinOpcode) {
9774 default:
9775 Unsupported = true;
9776 return LHS;
9777 case Instruction::Add:
9778 return LHS + RHS;
9779 case Instruction::Sub:
9780 return LHS - RHS;
9781 case Instruction::Mul:
9782 return LHS * RHS;
9783 case Instruction::UDiv:
9784 if (RHS.isZero()) {
9785 SkipOperation = true;
9786 return LHS;
9787 }
9788 return LHS.udiv(RHS);
9789 case Instruction::SDiv:
9790 if (RHS.isZero()) {
9791 SkipOperation = true;
9792 return LHS;
9793 }
9794 return LHS.sdiv(RHS);
9795 case Instruction::URem:
9796 if (RHS.isZero()) {
9797 SkipOperation = true;
9798 return LHS;
9799 }
9800 return LHS.urem(RHS);
9801 case Instruction::SRem:
9802 if (RHS.isZero()) {
9803 SkipOperation = true;
9804 return LHS;
9805 }
9806 return LHS.srem(RHS);
9807 case Instruction::Shl:
9808 return LHS.shl(RHS);
9809 case Instruction::LShr:
9810 return LHS.lshr(RHS);
9811 case Instruction::AShr:
9812 return LHS.ashr(RHS);
9813 case Instruction::And:
9814 return LHS & RHS;
9815 case Instruction::Or:
9816 return LHS | RHS;
9817 case Instruction::Xor:
9818 return LHS ^ RHS;
9819 }
9820 }
9821
9822 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
9823 const APInt &LHS, const APInt &RHS) {
9824 bool SkipOperation = false;
9825 bool Unsupported = false;
9826 APInt Result =
9827 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
9828 if (Unsupported)
9829 return false;
9830 // If SkipOperation is true, we can ignore this operand pair (L, R).
9831 if (!SkipOperation)
9832 unionAssumed(Result);
9833 return isValidState();
9834 }
9835
9836 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
9837 auto AssumedBefore = getAssumed();
9838 Value *LHS = ICI->getOperand(0);
9839 Value *RHS = ICI->getOperand(1);
9840
9841 bool LHSContainsUndef = false, RHSContainsUndef = false;
9842 SetTy LHSAAPVS, RHSAAPVS;
9843 if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9844 LHSContainsUndef, /* ForSelf */ false) ||
9845 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9846 RHSContainsUndef, /* ForSelf */ false))
9847 return indicatePessimisticFixpoint();
9848
9849 // TODO: make use of undef flag to limit potential values aggressively.
9850 bool MaybeTrue = false, MaybeFalse = false;
9851 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
9852 if (LHSContainsUndef && RHSContainsUndef) {
9853 // The result of any comparison between undefs can be soundly replaced
9854 // with undef.
9855 unionAssumedWithUndef();
9856 } else if (LHSContainsUndef) {
9857 for (const APInt &R : RHSAAPVS) {
9858 bool CmpResult = calculateICmpInst(ICI, Zero, R);
9859 MaybeTrue |= CmpResult;
9860 MaybeFalse |= !CmpResult;
9861 if (MaybeTrue & MaybeFalse)
9862 return indicatePessimisticFixpoint();
9863 }
9864 } else if (RHSContainsUndef) {
9865 for (const APInt &L : LHSAAPVS) {
9866 bool CmpResult = calculateICmpInst(ICI, L, Zero);
9867 MaybeTrue |= CmpResult;
9868 MaybeFalse |= !CmpResult;
9869 if (MaybeTrue & MaybeFalse)
9870 return indicatePessimisticFixpoint();
9871 }
9872 } else {
9873 for (const APInt &L : LHSAAPVS) {
9874 for (const APInt &R : RHSAAPVS) {
9875 bool CmpResult = calculateICmpInst(ICI, L, R);
9876 MaybeTrue |= CmpResult;
9877 MaybeFalse |= !CmpResult;
9878 if (MaybeTrue & MaybeFalse)
9879 return indicatePessimisticFixpoint();
9880 }
9881 }
9882 }
9883 if (MaybeTrue)
9884 unionAssumed(APInt(/* numBits */ 1, /* val */ 1));
9885 if (MaybeFalse)
9886 unionAssumed(APInt(/* numBits */ 1, /* val */ 0));
9887 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9888 : ChangeStatus::CHANGED;
9889 }
9890
9891 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
9892 auto AssumedBefore = getAssumed();
9893 Value *LHS = SI->getTrueValue();
9894 Value *RHS = SI->getFalseValue();
9895
9896 bool UsedAssumedInformation = false;
9897 std::optional<Constant *> C = A.getAssumedConstant(
9898 *SI->getCondition(), *this, UsedAssumedInformation);
9899
9900 // Check if we only need one operand.
9901 bool OnlyLeft = false, OnlyRight = false;
9902 if (C && *C && (*C)->isOneValue())
9903 OnlyLeft = true;
9904 else if (C && *C && (*C)->isZeroValue())
9905 OnlyRight = true;
9906
9907 bool LHSContainsUndef = false, RHSContainsUndef = false;
9908 SetTy LHSAAPVS, RHSAAPVS;
9909 if (!OnlyRight &&
9910 !fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9911 LHSContainsUndef, /* ForSelf */ false))
9912 return indicatePessimisticFixpoint();
9913
9914 if (!OnlyLeft &&
9915 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9916 RHSContainsUndef, /* ForSelf */ false))
9917 return indicatePessimisticFixpoint();
9918
9919 if (OnlyLeft || OnlyRight) {
9920 // select (true/false), lhs, rhs
9921 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
9922 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
9923
9924 if (Undef)
9925 unionAssumedWithUndef();
9926 else {
9927 for (const auto &It : *OpAA)
9928 unionAssumed(It);
9929 }
9930
9931 } else if (LHSContainsUndef && RHSContainsUndef) {
9932 // select i1 *, undef , undef => undef
9933 unionAssumedWithUndef();
9934 } else {
9935 for (const auto &It : LHSAAPVS)
9936 unionAssumed(It);
9937 for (const auto &It : RHSAAPVS)
9938 unionAssumed(It);
9939 }
9940 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9941 : ChangeStatus::CHANGED;
9942 }
9943
9944 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
9945 auto AssumedBefore = getAssumed();
9946 if (!CI->isIntegerCast())
9947 return indicatePessimisticFixpoint();
9948 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
9949 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
9950 Value *Src = CI->getOperand(0);
9951
9952 bool SrcContainsUndef = false;
9953 SetTy SrcPVS;
9954 if (!fillSetWithConstantValues(A, IRPosition::value(*Src), SrcPVS,
9955 SrcContainsUndef, /* ForSelf */ false))
9956 return indicatePessimisticFixpoint();
9957
9958 if (SrcContainsUndef)
9959 unionAssumedWithUndef();
9960 else {
9961 for (const APInt &S : SrcPVS) {
9962 APInt T = calculateCastInst(CI, S, ResultBitWidth);
9963 unionAssumed(T);
9964 }
9965 }
9966 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9967 : ChangeStatus::CHANGED;
9968 }
9969
9970 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
9971 auto AssumedBefore = getAssumed();
9972 Value *LHS = BinOp->getOperand(0);
9973 Value *RHS = BinOp->getOperand(1);
9974
9975 bool LHSContainsUndef = false, RHSContainsUndef = false;
9976 SetTy LHSAAPVS, RHSAAPVS;
9977 if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
9978 LHSContainsUndef, /* ForSelf */ false) ||
9979 !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
9980 RHSContainsUndef, /* ForSelf */ false))
9981 return indicatePessimisticFixpoint();
9982
9983 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
9984
9985 // TODO: make use of undef flag to limit potential values aggressively.
9986 if (LHSContainsUndef && RHSContainsUndef) {
9987 if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, Zero))
9988 return indicatePessimisticFixpoint();
9989 } else if (LHSContainsUndef) {
9990 for (const APInt &R : RHSAAPVS) {
9991 if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, R))
9992 return indicatePessimisticFixpoint();
9993 }
9994 } else if (RHSContainsUndef) {
9995 for (const APInt &L : LHSAAPVS) {
9996 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, Zero))
9997 return indicatePessimisticFixpoint();
9998 }
9999 } else {
10000 for (const APInt &L : LHSAAPVS) {
10001 for (const APInt &R : RHSAAPVS) {
10002 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, R))
10003 return indicatePessimisticFixpoint();
10004 }
10005 }
10006 }
10007 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10008 : ChangeStatus::CHANGED;
10009 }
10010
10011 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10012 auto AssumedBefore = getAssumed();
10013 SetTy Incoming;
10014 bool ContainsUndef;
10015 if (!fillSetWithConstantValues(A, IRPosition::value(*Inst), Incoming,
10016 ContainsUndef, /* ForSelf */ true))
10017 return indicatePessimisticFixpoint();
10018 if (ContainsUndef) {
10019 unionAssumedWithUndef();
10020 } else {
10021 for (const auto &It : Incoming)
10022 unionAssumed(It);
10023 }
10024 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10025 : ChangeStatus::CHANGED;
10026 }
10027
10028 /// See AbstractAttribute::updateImpl(...).
10029 ChangeStatus updateImpl(Attributor &A) override {
10030 Value &V = getAssociatedValue();
10031 Instruction *I = dyn_cast<Instruction>(&V);
10032
10033 if (auto *ICI = dyn_cast<ICmpInst>(I))
10034 return updateWithICmpInst(A, ICI);
10035
10036 if (auto *SI = dyn_cast<SelectInst>(I))
10037 return updateWithSelectInst(A, SI);
10038
10039 if (auto *CI = dyn_cast<CastInst>(I))
10040 return updateWithCastInst(A, CI);
10041
10042 if (auto *BinOp = dyn_cast<BinaryOperator>(I))
10043 return updateWithBinaryOperator(A, BinOp);
10044
10045 if (isa<PHINode>(I) || isa<LoadInst>(I))
10046 return updateWithInstruction(A, I);
10047
10048 return indicatePessimisticFixpoint();
10049 }
10050
10051 /// See AbstractAttribute::trackStatistics()
10052 void trackStatistics() const override {
10053 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10054 }
10055};
10056
10057struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10058 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10059 : AAPotentialConstantValuesImpl(IRP, A) {}
10060
10061 /// See AbstractAttribute::initialize(...).
10062 ChangeStatus updateImpl(Attributor &A) override {
10064 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10065 "not be called");
10066 }
10067
10068 /// See AbstractAttribute::trackStatistics()
10069 void trackStatistics() const override {
10070 STATS_DECLTRACK_FN_ATTR(potential_values)
10071 }
10072};
10073
10074struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10075 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10076 : AAPotentialConstantValuesFunction(IRP, A) {}
10077
10078 /// See AbstractAttribute::trackStatistics()
10079 void trackStatistics() const override {
10080 STATS_DECLTRACK_CS_ATTR(potential_values)
10081 }
10082};
10083
10084struct AAPotentialConstantValuesCallSiteReturned
10085 : AACalleeToCallSite<AAPotentialConstantValues,
10086 AAPotentialConstantValuesImpl> {
10087 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10088 Attributor &A)
10089 : AACalleeToCallSite<AAPotentialConstantValues,
10090 AAPotentialConstantValuesImpl>(IRP, A) {}
10091
10092 /// See AbstractAttribute::trackStatistics()
10093 void trackStatistics() const override {
10094 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10095 }
10096};
10097
10098struct AAPotentialConstantValuesCallSiteArgument
10099 : AAPotentialConstantValuesFloating {
10100 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10101 Attributor &A)
10102 : AAPotentialConstantValuesFloating(IRP, A) {}
10103
10104 /// See AbstractAttribute::initialize(..).
10105 void initialize(Attributor &A) override {
10106 AAPotentialConstantValuesImpl::initialize(A);
10107 if (isAtFixpoint())
10108 return;
10109
10110 Value &V = getAssociatedValue();
10111
10112 if (auto *C = dyn_cast<ConstantInt>(&V)) {
10113 unionAssumed(C->getValue());
10114 indicateOptimisticFixpoint();
10115 return;
10116 }
10117
10118 if (isa<UndefValue>(&V)) {
10119 unionAssumedWithUndef();
10120 indicateOptimisticFixpoint();
10121 return;
10122 }
10123 }
10124
10125 /// See AbstractAttribute::updateImpl(...).
10126 ChangeStatus updateImpl(Attributor &A) override {
10127 Value &V = getAssociatedValue();
10128 auto AssumedBefore = getAssumed();
10129 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10130 *this, IRPosition::value(V), DepClassTy::REQUIRED);
10131 if (!AA)
10132 return indicatePessimisticFixpoint();
10133 const auto &S = AA->getAssumed();
10134 unionAssumed(S);
10135 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10136 : ChangeStatus::CHANGED;
10137 }
10138
10139 /// See AbstractAttribute::trackStatistics()
10140 void trackStatistics() const override {
10141 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10142 }
10143};
10144} // namespace
10145
10146/// ------------------------ NoUndef Attribute ---------------------------------
10148 Attribute::AttrKind ImpliedAttributeKind,
10149 bool IgnoreSubsumingPositions) {
10150 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10151 "Unexpected attribute kind");
10152 if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,
10153 Attribute::NoUndef))
10154 return true;
10155
10156 Value &Val = IRP.getAssociatedValue();
10159 LLVMContext &Ctx = Val.getContext();
10160 A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef));
10161 return true;
10162 }
10163
10164 return false;
10165}
10166
10167namespace {
10168struct AANoUndefImpl : AANoUndef {
10169 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10170
10171 /// See AbstractAttribute::initialize(...).
10172 void initialize(Attributor &A) override {
10173 Value &V = getAssociatedValue();
10174 if (isa<UndefValue>(V))
10175 indicatePessimisticFixpoint();
10176 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10177 }
10178
10179 /// See followUsesInMBEC
10180 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10181 AANoUndef::StateType &State) {
10182 const Value *UseV = U->get();
10183 const DominatorTree *DT = nullptr;
10184 AssumptionCache *AC = nullptr;
10185 InformationCache &InfoCache = A.getInfoCache();
10186 if (Function *F = getAnchorScope()) {
10189 }
10190 State.setKnown(isGuaranteedNotToBeUndefOrPoison(UseV, AC, I, DT));
10191 bool TrackUse = false;
10192 // Track use for instructions which must produce undef or poison bits when
10193 // at least one operand contains such bits.
10194 if (isa<CastInst>(*I) || isa<GetElementPtrInst>(*I))
10195 TrackUse = true;
10196 return TrackUse;
10197 }
10198
10199 /// See AbstractAttribute::getAsStr().
10200 const std::string getAsStr(Attributor *A) const override {
10201 return getAssumed() ? "noundef" : "may-undef-or-poison";
10202 }
10203
10204 ChangeStatus manifest(Attributor &A) override {
10205 // We don't manifest noundef attribute for dead positions because the
10206 // associated values with dead positions would be replaced with undef
10207 // values.
10208 bool UsedAssumedInformation = false;
10209 if (A.isAssumedDead(getIRPosition(), nullptr, nullptr,
10210 UsedAssumedInformation))
10211 return ChangeStatus::UNCHANGED;
10212 // A position whose simplified value does not have any value is
10213 // considered to be dead. We don't manifest noundef in such positions for
10214 // the same reason above.
10215 if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation,
10217 .has_value())
10218 return ChangeStatus::UNCHANGED;
10219 return AANoUndef::manifest(A);
10220 }
10221};
10222
10223struct AANoUndefFloating : public AANoUndefImpl {
10224 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10225 : AANoUndefImpl(IRP, A) {}
10226
10227 /// See AbstractAttribute::initialize(...).
10228 void initialize(Attributor &A) override {
10229 AANoUndefImpl::initialize(A);
10230 if (!getState().isAtFixpoint() && getAnchorScope() &&
10231 !getAnchorScope()->isDeclaration())
10232 if (Instruction *CtxI = getCtxI())
10233 followUsesInMBEC(*this, A, getState(), *CtxI);
10234 }
10235
10236 /// See AbstractAttribute::updateImpl(...).
10237 ChangeStatus updateImpl(Attributor &A) override {
10238 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10239 bool IsKnownNoUndef;
10240 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10241 A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);
10242 };
10243
10244 bool Stripped;
10245 bool UsedAssumedInformation = false;
10246 Value *AssociatedValue = &getAssociatedValue();
10248 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
10249 AA::AnyScope, UsedAssumedInformation))
10250 Stripped = false;
10251 else
10252 Stripped =
10253 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10254
10255 if (!Stripped) {
10256 // If we haven't stripped anything we might still be able to use a
10257 // different AA, but only if the IRP changes. Effectively when we
10258 // interpret this not as a call site value but as a floating/argument
10259 // value.
10260 const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
10261 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10262 return indicatePessimisticFixpoint();
10263 return ChangeStatus::UNCHANGED;
10264 }
10265
10266 for (const auto &VAC : Values)
10267 if (!VisitValueCB(IRPosition::value(*VAC.getValue())))
10268 return indicatePessimisticFixpoint();
10269
10270 return ChangeStatus::UNCHANGED;
10271 }
10272
10273 /// See AbstractAttribute::trackStatistics()
10274 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10275};
10276
10277struct AANoUndefReturned final
10278 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10279 AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10280 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10281
10282 /// See AbstractAttribute::trackStatistics()
10283 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10284};
10285
10286struct AANoUndefArgument final
10287 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10288 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10289 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10290
10291 /// See AbstractAttribute::trackStatistics()
10292 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10293};
10294
10295struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10296 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10297 : AANoUndefFloating(IRP, A) {}
10298
10299 /// See AbstractAttribute::trackStatistics()
10300 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10301};
10302
10303struct AANoUndefCallSiteReturned final
10304 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10305 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10306 : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10307
10308 /// See AbstractAttribute::trackStatistics()
10309 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10310};
10311
10312/// ------------------------ NoFPClass Attribute -------------------------------
10313
10314struct AANoFPClassImpl : AANoFPClass {
10315 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10316
10317 void initialize(Attributor &A) override {
10318 const IRPosition &IRP = getIRPosition();
10319
10320 Value &V = IRP.getAssociatedValue();
10321 if (isa<UndefValue>(V)) {
10322 indicateOptimisticFixpoint();
10323 return;
10324 }
10325
10327 A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false);
10328 for (const auto &Attr : Attrs) {
10329 addKnownBits(Attr.getNoFPClass());
10330 }
10331
10332 const DataLayout &DL = A.getDataLayout();
10333 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10335 addKnownBits(~KnownFPClass.KnownFPClasses);
10336 }
10337
10338 if (Instruction *CtxI = getCtxI())
10339 followUsesInMBEC(*this, A, getState(), *CtxI);
10340 }
10341
10342 /// See followUsesInMBEC
10343 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10344 AANoFPClass::StateType &State) {
10345 // TODO: Determine what instructions can be looked through.
10346 auto *CB = dyn_cast<CallBase>(I);
10347 if (!CB)
10348 return false;
10349
10350 if (!CB->isArgOperand(U))
10351 return false;
10352
10353 unsigned ArgNo = CB->getArgOperandNo(U);
10354 IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
10355 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(*this, IRP, DepClassTy::NONE))
10356 State.addKnownBits(NoFPAA->getState().getKnown());
10357 return false;
10358 }
10359
10360 const std::string getAsStr(Attributor *A) const override {
10361 std::string Result = "nofpclass";
10362 raw_string_ostream OS(Result);
10363 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10364 return Result;
10365 }
10366
10367 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10368 SmallVectorImpl<Attribute> &Attrs) const override {
10369 Attrs.emplace_back(Attribute::getWithNoFPClass(Ctx, getAssumedNoFPClass()));
10370 }
10371};
10372
10373struct AANoFPClassFloating : public AANoFPClassImpl {
10374 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10375 : AANoFPClassImpl(IRP, A) {}
10376
10377 /// See AbstractAttribute::updateImpl(...).
10378 ChangeStatus updateImpl(Attributor &A) override {
10380 bool UsedAssumedInformation = false;
10381 if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
10382 AA::AnyScope, UsedAssumedInformation)) {
10383 Values.push_back({getAssociatedValue(), getCtxI()});
10384 }
10385
10386 StateType T;
10387 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10388 const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V),
10389 DepClassTy::REQUIRED);
10390 if (!AA || this == AA) {
10391 T.indicatePessimisticFixpoint();
10392 } else {
10393 const AANoFPClass::StateType &S =
10394 static_cast<const AANoFPClass::StateType &>(AA->getState());
10395 T ^= S;
10396 }
10397 return T.isValidState();
10398 };
10399
10400 for (const auto &VAC : Values)
10401 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10402 return indicatePessimisticFixpoint();
10403
10404 return clampStateAndIndicateChange(getState(), T);
10405 }
10406
10407 /// See AbstractAttribute::trackStatistics()
10408 void trackStatistics() const override {
10410 }
10411};
10412
10413struct AANoFPClassReturned final
10414 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10415 AANoFPClassImpl::StateType, false,
10416 Attribute::None, false> {
10417 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10418 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10419 AANoFPClassImpl::StateType, false,
10420 Attribute::None, false>(IRP, A) {}
10421
10422 /// See AbstractAttribute::trackStatistics()
10423 void trackStatistics() const override {
10425 }
10426};
10427
10428struct AANoFPClassArgument final
10429 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10430 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10431 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10432
10433 /// See AbstractAttribute::trackStatistics()
10434 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10435};
10436
10437struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10438 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10439 : AANoFPClassFloating(IRP, A) {}
10440
10441 /// See AbstractAttribute::trackStatistics()
10442 void trackStatistics() const override {
10444 }
10445};
10446
10447struct AANoFPClassCallSiteReturned final
10448 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10449 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10450 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10451
10452 /// See AbstractAttribute::trackStatistics()
10453 void trackStatistics() const override {
10455 }
10456};
10457
10458struct AACallEdgesImpl : public AACallEdges {
10459 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10460
10461 const SetVector<Function *> &getOptimisticEdges() const override {
10462 return CalledFunctions;
10463 }
10464
10465 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10466
10467 bool hasNonAsmUnknownCallee() const override {
10468 return HasUnknownCalleeNonAsm;
10469 }
10470
10471 const std::string getAsStr(Attributor *A) const override {
10472 return "CallEdges[" + std::to_string(HasUnknownCallee) + "," +
10473 std::to_string(CalledFunctions.size()) + "]";
10474 }
10475
10476 void trackStatistics() const override {}
10477
10478protected:
10479 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10480 if (CalledFunctions.insert(Fn)) {
10481 Change = ChangeStatus::CHANGED;
10482 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10483 << "\n");
10484 }
10485 }
10486
10487 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10488 if (!HasUnknownCallee)
10489 Change = ChangeStatus::CHANGED;
10490 if (NonAsm && !HasUnknownCalleeNonAsm)
10491 Change = ChangeStatus::CHANGED;
10492 HasUnknownCalleeNonAsm |= NonAsm;
10493 HasUnknownCallee = true;
10494 }
10495
10496private:
10497 /// Optimistic set of functions that might be called by this position.
10498 SetVector<Function *> CalledFunctions;
10499
10500 /// Is there any call with a unknown callee.
10501 bool HasUnknownCallee = false;
10502
10503 /// Is there any call with a unknown callee, excluding any inline asm.
10504 bool HasUnknownCalleeNonAsm = false;
10505};
10506
10507struct AACallEdgesCallSite : public AACallEdgesImpl {
10508 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10509 : AACallEdgesImpl(IRP, A) {}
10510 /// See AbstractAttribute::updateImpl(...).
10511 ChangeStatus updateImpl(Attributor &A) override {
10512 ChangeStatus Change = ChangeStatus::UNCHANGED;
10513
10514 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10515 if (Function *Fn = dyn_cast<Function>(&V)) {
10516 addCalledFunction(Fn, Change);
10517 } else {
10518 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10519 setHasUnknownCallee(true, Change);
10520 }
10521
10522 // Explore all values.
10523 return true;
10524 };
10525
10527 // Process any value that we might call.
10528 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10529 if (isa<Constant>(V)) {
10530 VisitValue(*V, CtxI);
10531 return;
10532 }
10533
10534 bool UsedAssumedInformation = false;
10535 Values.clear();
10536 if (!A.getAssumedSimplifiedValues(IRPosition::value(*V), *this, Values,
10537 AA::AnyScope, UsedAssumedInformation)) {
10538 Values.push_back({*V, CtxI});
10539 }
10540 for (auto &VAC : Values)
10541 VisitValue(*VAC.getValue(), VAC.getCtxI());
10542 };
10543
10544 CallBase *CB = cast<CallBase>(getCtxI());
10545
10546 if (auto *IA = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
10547 if (IA->hasSideEffects() &&
10548 !hasAssumption(*CB->getCaller(), "ompx_no_call_asm") &&
10549 !hasAssumption(*CB, "ompx_no_call_asm")) {
10550 setHasUnknownCallee(false, Change);
10551 }
10552 return Change;
10553 }
10554
10555 if (CB->isIndirectCall())
10556 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10557 *this, getIRPosition(), DepClassTy::OPTIONAL))
10558 if (IndirectCallAA->foreachCallee(
10559 [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10560 return Change;
10561
10562 // The most simple case.
10563 ProcessCalledOperand(CB->getCalledOperand(), CB);
10564
10565 // Process callback functions.
10566 SmallVector<const Use *, 4u> CallbackUses;
10567 AbstractCallSite::getCallbackUses(*CB, CallbackUses);
10568 for (const Use *U : CallbackUses)
10569 ProcessCalledOperand(U->get(), CB);
10570
10571 return Change;
10572 }
10573};
10574
10575struct AACallEdgesFunction : public AACallEdgesImpl {
10576 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10577 : AACallEdgesImpl(IRP, A) {}
10578
10579 /// See AbstractAttribute::updateImpl(...).
10580 ChangeStatus updateImpl(Attributor &A) override {
10581 ChangeStatus Change = ChangeStatus::UNCHANGED;
10582
10583 auto ProcessCallInst = [&](Instruction &Inst) {
10584 CallBase &CB = cast<CallBase>(Inst);
10585
10586 auto *CBEdges = A.getAAFor<AACallEdges>(
10587 *this, IRPosition::callsite_function(CB), DepClassTy::REQUIRED);
10588 if (!CBEdges)
10589 return false;
10590 if (CBEdges->hasNonAsmUnknownCallee())
10591 setHasUnknownCallee(true, Change);
10592 if (CBEdges->hasUnknownCallee())
10593 setHasUnknownCallee(false, Change);
10594
10595 for (Function *F : CBEdges->getOptimisticEdges())
10596 addCalledFunction(F, Change);
10597
10598 return true;
10599 };
10600
10601 // Visit all callable instructions.
10602 bool UsedAssumedInformation = false;
10603 if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this,
10604 UsedAssumedInformation,
10605 /* CheckBBLivenessOnly */ true)) {
10606 // If we haven't looked at all call like instructions, assume that there
10607 // are unknown callees.
10608 setHasUnknownCallee(true, Change);
10609 }
10610
10611 return Change;
10612 }
10613};
10614
10615/// -------------------AAInterFnReachability Attribute--------------------------
10616
10617struct AAInterFnReachabilityFunction
10618 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10619 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10620 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10621 : Base(IRP, A) {}
10622
10623 bool instructionCanReach(
10624 Attributor &A, const Instruction &From, const Function &To,
10625 const AA::InstExclusionSetTy *ExclusionSet) const override {
10626 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10627 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10628
10629 RQITy StackRQI(A, From, To, ExclusionSet, false);
10630 typename RQITy::Reachable Result;
10631 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10632 return NonConstThis->isReachableImpl(A, StackRQI,
10633 /*IsTemporaryRQI=*/true);
10634 return Result == RQITy::Reachable::Yes;
10635 }
10636
10637 bool isReachableImpl(Attributor &A, RQITy &RQI,
10638 bool IsTemporaryRQI) override {
10639 const Instruction *EntryI =
10640 &RQI.From->getFunction()->getEntryBlock().front();
10641 if (EntryI != RQI.From &&
10642 !instructionCanReach(A, *EntryI, *RQI.To, nullptr))
10643 return rememberResult(A, RQITy::Reachable::No, RQI, false,
10644 IsTemporaryRQI);
10645
10646 auto CheckReachableCallBase = [&](CallBase *CB) {
10647 auto *CBEdges = A.getAAFor<AACallEdges>(
10648 *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
10649 if (!CBEdges || !CBEdges->getState().isValidState())
10650 return false;
10651 // TODO Check To backwards in this case.
10652 if (CBEdges->hasUnknownCallee())
10653 return false;
10654
10655 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10656 if (Fn == RQI.To)
10657 return false;
10658
10659 if (Fn->isDeclaration()) {
10660 if (Fn->hasFnAttribute(Attribute::NoCallback))
10661 continue;
10662 // TODO Check To backwards in this case.
10663 return false;
10664 }
10665
10666 if (Fn == getAnchorScope()) {
10667 if (EntryI == RQI.From)
10668 continue;
10669 return false;
10670 }
10671
10672 const AAInterFnReachability *InterFnReachability =
10673 A.getAAFor<AAInterFnReachability>(*this, IRPosition::function(*Fn),
10674 DepClassTy::OPTIONAL);
10675
10676 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10677 if (!InterFnReachability ||
10678 InterFnReachability->instructionCanReach(A, FnFirstInst, *RQI.To,
10679 RQI.ExclusionSet))
10680 return false;
10681 }
10682 return true;
10683 };
10684
10685 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10686 *this, IRPosition::function(*RQI.From->getFunction()),
10687 DepClassTy::OPTIONAL);
10688
10689 // Determine call like instructions that we can reach from the inst.
10690 auto CheckCallBase = [&](Instruction &CBInst) {
10691 // There are usually less nodes in the call graph, check inter function
10692 // reachability first.
10693 if (CheckReachableCallBase(cast<CallBase>(&CBInst)))
10694 return true;
10695 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10696 A, *RQI.From, CBInst, RQI.ExclusionSet);
10697 };
10698
10699 bool UsedExclusionSet = /* conservative */ true;
10700 bool UsedAssumedInformation = false;
10701 if (!A.checkForAllCallLikeInstructions(CheckCallBase, *this,
10702 UsedAssumedInformation,
10703 /* CheckBBLivenessOnly */ true))
10704 return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10705 IsTemporaryRQI);
10706
10707 return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
10708 IsTemporaryRQI);
10709 }
10710
10711 void trackStatistics() const override {}
10712};
10713} // namespace
10714
10715template <typename AAType>
10716static std::optional<Constant *>
10718 const IRPosition &IRP, Type &Ty) {
10719 if (!Ty.isIntegerTy())
10720 return nullptr;
10721
10722 // This will also pass the call base context.
10723 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10724 if (!AA)
10725 return nullptr;
10726
10727 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10728
10729 if (!COpt.has_value()) {
10730 A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
10731 return std::nullopt;
10732 }
10733 if (auto *C = *COpt) {
10734 A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
10735 return C;
10736 }
10737 return nullptr;
10738}
10739
10741 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10743 Type &Ty = *IRP.getAssociatedType();
10744 std::optional<Value *> V;
10745 for (auto &It : Values) {
10746 V = AA::combineOptionalValuesInAAValueLatice(V, It.getValue(), &Ty);
10747 if (V.has_value() && !*V)
10748 break;
10749 }
10750 if (!V.has_value())
10751 return UndefValue::get(&Ty);
10752 return *V;
10753}
10754
10755namespace {
10756struct AAPotentialValuesImpl : AAPotentialValues {
10757 using StateType = PotentialLLVMValuesState;
10758
10759 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10760 : AAPotentialValues(IRP, A) {}
10761
10762 /// See AbstractAttribute::initialize(..).
10763 void initialize(Attributor &A) override {
10764 if (A.hasSimplificationCallback(getIRPosition())) {
10765 indicatePessimisticFixpoint();
10766 return;
10767 }
10768 Value *Stripped = getAssociatedValue().stripPointerCasts();
10769 if (isa<Constant>(Stripped) && !isa<ConstantExpr>(Stripped)) {
10770 addValue(A, getState(), *Stripped, getCtxI(), AA::AnyScope,
10771 getAnchorScope());
10772 indicateOptimisticFixpoint();
10773 return;
10774 }
10775 AAPotentialValues::initialize(A);
10776 }
10777
10778 /// See AbstractAttribute::getAsStr().
10779 const std::string getAsStr(Attributor *A) const override {
10780 std::string Str;
10782 OS << getState();
10783 return Str;
10784 }
10785
10786 template <typename AAType>
10787 static std::optional<Value *> askOtherAA(Attributor &A,
10788 const AbstractAttribute &AA,
10789 const IRPosition &IRP, Type &Ty) {
10790 if (isa<Constant>(IRP.getAssociatedValue()))
10791 return &IRP.getAssociatedValue();
10792 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
10793 if (!C)
10794 return std::nullopt;
10795 if (*C)
10796 if (auto *CC = AA::getWithType(**C, Ty))
10797 return CC;
10798 return nullptr;
10799 }
10800
10801 virtual void addValue(Attributor &A, StateType &State, Value &V,
10802 const Instruction *CtxI, AA::ValueScope S,
10803 Function *AnchorScope) const {
10804
10805 IRPosition ValIRP = IRPosition::value(V);
10806 if (auto *CB = dyn_cast_or_null<CallBase>(CtxI)) {
10807 for (const auto &U : CB->args()) {
10808 if (U.get() != &V)
10809 continue;
10810 ValIRP = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));
10811 break;
10812 }
10813 }
10814
10815 Value *VPtr = &V;
10816 if (ValIRP.getAssociatedType()->isIntegerTy()) {
10817 Type &Ty = *getAssociatedType();
10818 std::optional<Value *> SimpleV =
10819 askOtherAA<AAValueConstantRange>(A, *this, ValIRP, Ty);
10820 if (SimpleV.has_value() && !*SimpleV) {
10821 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
10822 *this, ValIRP, DepClassTy::OPTIONAL);
10823 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
10824 for (const auto &It : PotentialConstantsAA->getAssumedSet())
10825 State.unionAssumed({{*ConstantInt::get(&Ty, It), nullptr}, S});
10826 if (PotentialConstantsAA->undefIsContained())
10827 State.unionAssumed({{*UndefValue::get(&Ty), nullptr}, S});
10828 return;
10829 }
10830 }
10831 if (!SimpleV.has_value())
10832 return;
10833
10834 if (*SimpleV)
10835 VPtr = *SimpleV;
10836 }
10837
10838 if (isa<ConstantInt>(VPtr))
10839 CtxI = nullptr;
10840 if (!AA::isValidInScope(*VPtr, AnchorScope))
10842
10843 State.unionAssumed({{*VPtr, CtxI}, S});
10844 }
10845
10846 /// Helper struct to tie a value+context pair together with the scope for
10847 /// which this is the simplified version.
10848 struct ItemInfo {
10851
10852 bool operator==(const ItemInfo &II) const {
10853 return II.I == I && II.S == S;
10854 };
10855 bool operator<(const ItemInfo &II) const {
10856 if (I == II.I)
10857 return S < II.S;
10858 return I < II.I;
10859 };
10860 };
10861
10862 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
10864 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
10865 if (!(CS & S))
10866 continue;
10867
10868 bool UsedAssumedInformation = false;
10870 if (!A.getAssumedSimplifiedValues(IRP, this, Values, CS,
10871 UsedAssumedInformation))
10872 return false;
10873
10874 for (auto &It : Values)
10875 ValueScopeMap[It] += CS;
10876 }
10877 for (auto &It : ValueScopeMap)
10878 addValue(A, getState(), *It.first.getValue(), It.first.getCtxI(),
10879 AA::ValueScope(It.second), getAnchorScope());
10880
10881 return true;
10882 }
10883
10884 void giveUpOnIntraprocedural(Attributor &A) {
10885 auto NewS = StateType::getBestState(getState());
10886 for (const auto &It : getAssumedSet()) {
10887 if (It.second == AA::Intraprocedural)
10888 continue;
10889 addValue(A, NewS, *It.first.getValue(), It.first.getCtxI(),
10890 AA::Interprocedural, getAnchorScope());
10891 }
10892 assert(!undefIsContained() && "Undef should be an explicit value!");
10893 addValue(A, NewS, getAssociatedValue(), getCtxI(), AA::Intraprocedural,
10894 getAnchorScope());
10895 getState() = NewS;
10896 }
10897
10898 /// See AbstractState::indicatePessimisticFixpoint(...).
10899 ChangeStatus indicatePessimisticFixpoint() override {
10900 getState() = StateType::getBestState(getState());
10901 getState().unionAssumed({{getAssociatedValue(), getCtxI()}, AA::AnyScope});
10903 return ChangeStatus::CHANGED;
10904 }
10905
10906 /// See AbstractAttribute::updateImpl(...).
10907 ChangeStatus updateImpl(Attributor &A) override {
10908 return indicatePessimisticFixpoint();
10909 }
10910
10911 /// See AbstractAttribute::manifest(...).
10912 ChangeStatus manifest(Attributor &A) override {
10915 Values.clear();
10916 if (!getAssumedSimplifiedValues(A, Values, S))
10917 continue;
10918 Value &OldV = getAssociatedValue();
10919 if (isa<UndefValue>(OldV))
10920 continue;
10921 Value *NewV = getSingleValue(A, *this, getIRPosition(), Values);
10922 if (!NewV || NewV == &OldV)
10923 continue;
10924 if (getCtxI() &&
10925 !AA::isValidAtPosition({*NewV, *getCtxI()}, A.getInfoCache()))
10926 continue;
10927 if (A.changeAfterManifest(getIRPosition(), *NewV))
10928 return ChangeStatus::CHANGED;
10929 }
10931 }
10932
10933 bool getAssumedSimplifiedValues(
10935 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
10936 if (!isValidState())
10937 return false;
10938 bool UsedAssumedInformation = false;
10939 for (const auto &It : getAssumedSet())
10940 if (It.second & S) {
10941 if (RecurseForSelectAndPHI && (isa<PHINode>(It.first.getValue()) ||
10942 isa<SelectInst>(It.first.getValue()))) {
10943 if (A.getAssumedSimplifiedValues(
10944 IRPosition::inst(*cast<Instruction>(It.first.getValue())),
10945 this, Values, S, UsedAssumedInformation))
10946 continue;
10947 }
10948 Values.push_back(It.first);
10949 }
10950 assert(!undefIsContained() && "Undef should be an explicit value!");
10951 return true;
10952 }
10953};
10954
10955struct AAPotentialValuesFloating : AAPotentialValuesImpl {
10956 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
10957 : AAPotentialValuesImpl(IRP, A) {}
10958
10959 /// See AbstractAttribute::updateImpl(...).
10960 ChangeStatus updateImpl(Attributor &A) override {
10961 auto AssumedBefore = getAssumed();
10962
10963 genericValueTraversal(A, &getAssociatedValue());
10964
10965 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
10966 : ChangeStatus::CHANGED;
10967 }
10968
10969 /// Helper struct to remember which AAIsDead instances we actually used.
10970 struct LivenessInfo {
10971 const AAIsDead *LivenessAA = nullptr;
10972 bool AnyDead = false;
10973 };
10974
10975 /// Check if \p Cmp is a comparison we can simplify.
10976 ///
10977 /// We handle multiple cases, one in which at least one operand is an
10978 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
10979 /// operand. Return true if successful, in that case Worklist will be updated.
10980 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
10981 CmpInst::Predicate Pred, ItemInfo II,
10982 SmallVectorImpl<ItemInfo> &Worklist) {
10983
10984 // Simplify the operands first.
10985 bool UsedAssumedInformation = false;
10986 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
10987 auto GetSimplifiedValues = [&](Value &V,
10989 if (!A.getAssumedSimplifiedValues(
10990 IRPosition::value(V, getCallBaseContext()), this, Values,
10991 AA::Intraprocedural, UsedAssumedInformation)) {
10992 Values.clear();
10993 Values.push_back(AA::ValueAndContext{V, II.I.getCtxI()});
10994 }
10995 return Values.empty();
10996 };
10997 if (GetSimplifiedValues(*LHS, LHSValues))
10998 return true;
10999 if (GetSimplifiedValues(*RHS, RHSValues))
11000 return true;
11001
11002 LLVMContext &Ctx = LHS->getContext();
11003
11004 InformationCache &InfoCache = A.getInfoCache();
11005 Instruction *CmpI = dyn_cast<Instruction>(&Cmp);
11006 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11007 const auto *DT =
11009 : nullptr;
11010 const auto *TLI =
11011 F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
11012 auto *AC =
11014 : nullptr;
11015
11016 const DataLayout &DL = A.getDataLayout();
11017 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11018
11019 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11020 if (isa<UndefValue>(LHSV) || isa<UndefValue>(RHSV)) {
11021 addValue(A, getState(), *UndefValue::get(Cmp.getType()),
11022 /* CtxI */ nullptr, II.S, getAnchorScope());
11023 return true;
11024 }
11025
11026 // Handle the trivial case first in which we don't even need to think
11027 // about null or non-null.
11028 if (&LHSV == &RHSV &&
11030 Constant *NewV = ConstantInt::get(Type::getInt1Ty(Ctx),
11032 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11033 getAnchorScope());
11034 return true;
11035 }
11036
11037 auto *TypedLHS = AA::getWithType(LHSV, *LHS->getType());
11038 auto *TypedRHS = AA::getWithType(RHSV, *RHS->getType());
11039 if (TypedLHS && TypedRHS) {
11040 Value *NewV = simplifyCmpInst(Pred, TypedLHS, TypedRHS, Q);
11041 if (NewV && NewV != &Cmp) {
11042 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11043 getAnchorScope());
11044 return true;
11045 }
11046 }
11047
11048 // From now on we only handle equalities (==, !=).
11049 if (!CmpInst::isEquality(Pred))
11050 return false;
11051
11052 bool LHSIsNull = isa<ConstantPointerNull>(LHSV);
11053 bool RHSIsNull = isa<ConstantPointerNull>(RHSV);
11054 if (!LHSIsNull && !RHSIsNull)
11055 return false;
11056
11057 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11058 // non-nullptr operand and if we assume it's non-null we can conclude the
11059 // result of the comparison.
11060 assert((LHSIsNull || RHSIsNull) &&
11061 "Expected nullptr versus non-nullptr comparison at this point");
11062
11063 // The index is the operand that we assume is not null.
11064 unsigned PtrIdx = LHSIsNull;
11065 bool IsKnownNonNull;
11066 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11067 A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)),
11068 DepClassTy::REQUIRED, IsKnownNonNull);
11069 if (!IsAssumedNonNull)
11070 return false;
11071
11072 // The new value depends on the predicate, true for != and false for ==.
11073 Constant *NewV =
11074 ConstantInt::get(Type::getInt1Ty(Ctx), Pred == CmpInst::ICMP_NE);
11075 addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
11076 getAnchorScope());
11077 return true;
11078 };
11079
11080 for (auto &LHSValue : LHSValues)
11081 for (auto &RHSValue : RHSValues)
11082 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11083 return false;
11084 return true;
11085 }
11086
11087 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11088 SmallVectorImpl<ItemInfo> &Worklist) {
11089 const Instruction *CtxI = II.I.getCtxI();
11090 bool UsedAssumedInformation = false;
11091
11092 std::optional<Constant *> C =
11093 A.getAssumedConstant(*SI.getCondition(), *this, UsedAssumedInformation);
11094 bool NoValueYet = !C.has_value();
11095 if (NoValueYet || isa_and_nonnull<UndefValue>(*C))
11096 return true;
11097 if (auto *CI = dyn_cast_or_null<ConstantInt>(*C)) {
11098 if (CI->isZero())
11099 Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
11100 else
11101 Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
11102 } else if (&SI == &getAssociatedValue()) {
11103 // We could not simplify the condition, assume both values.
11104 Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
11105 Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
11106 } else {
11107 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11108 IRPosition::inst(SI), *this, UsedAssumedInformation, II.S);
11109 if (!SimpleV.has_value())
11110 return true;
11111 if (*SimpleV) {
11112 addValue(A, getState(), **SimpleV, CtxI, II.S, getAnchorScope());
11113 return true;
11114 }
11115 return false;
11116 }
11117 return true;
11118 }
11119
11120 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11121 SmallVectorImpl<ItemInfo> &Worklist) {
11122 SmallSetVector<Value *, 4> PotentialCopies;
11123 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11124 bool UsedAssumedInformation = false;
11125 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialCopies,
11126 PotentialValueOrigins, *this,
11127 UsedAssumedInformation,
11128 /* OnlyExact */ true)) {
11129 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11130 "loaded values for load instruction "
11131 << LI << "\n");
11132 return false;
11133 }
11134
11135 // Do not simplify loads that are only used in llvm.assume if we cannot also
11136 // remove all stores that may feed into the load. The reason is that the
11137 // assume is probably worth something as long as the stores are around.
11138 InformationCache &InfoCache = A.getInfoCache();
11139 if (InfoCache.isOnlyUsedByAssume(LI)) {
11140 if (!llvm::all_of(PotentialValueOrigins, [&](Instruction *I) {
11141 if (!I || isa<AssumeInst>(I))
11142 return true;
11143 if (auto *SI = dyn_cast<StoreInst>(I))
11144 return A.isAssumedDead(SI->getOperandUse(0), this,
11145 /* LivenessAA */ nullptr,
11146 UsedAssumedInformation,
11147 /* CheckBBLivenessOnly */ false);
11148 return A.isAssumedDead(*I, this, /* LivenessAA */ nullptr,
11149 UsedAssumedInformation,
11150 /* CheckBBLivenessOnly */ false);
11151 })) {
11152 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11153 "and we cannot delete all the stores: "
11154 << LI << "\n");
11155 return false;
11156 }
11157 }
11158
11159 // Values have to be dynamically unique or we loose the fact that a
11160 // single llvm::Value might represent two runtime values (e.g.,
11161 // stack locations in different recursive calls).
11162 const Instruction *CtxI = II.I.getCtxI();
11163 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11164 bool AllLocal = ScopeIsLocal;
11165 bool DynamicallyUnique = llvm::all_of(PotentialCopies, [&](Value *PC) {
11166 AllLocal &= AA::isValidInScope(*PC, getAnchorScope());
11167 return AA::isDynamicallyUnique(A, *this, *PC);
11168 });
11169 if (!DynamicallyUnique) {
11170 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11171 "values are dynamically unique: "
11172 << LI << "\n");
11173 return false;
11174 }
11175
11176 for (auto *PotentialCopy : PotentialCopies) {
11177 if (AllLocal) {
11178 Worklist.push_back({{*PotentialCopy, CtxI}, II.S});
11179 } else {
11180 Worklist.push_back({{*PotentialCopy, CtxI}, AA::Interprocedural});
11181 }
11182 }
11183 if (!AllLocal && ScopeIsLocal)
11184 addValue(A, getState(), LI, CtxI, AA::Intraprocedural, getAnchorScope());
11185 return true;
11186 }
11187
11188 bool handlePHINode(
11189 Attributor &A, PHINode &PHI, ItemInfo II,
11190 SmallVectorImpl<ItemInfo> &Worklist,
11192 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11193 LivenessInfo &LI = LivenessAAs[&F];
11194 if (!LI.LivenessAA)
11195 LI.LivenessAA = A.getAAFor<AAIsDead>(*this, IRPosition::function(F),
11196 DepClassTy::NONE);
11197 return LI;
11198 };
11199
11200 if (&PHI == &getAssociatedValue()) {
11201 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11202 const auto *CI =
11203 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11204 *PHI.getFunction());
11205
11206 Cycle *C = nullptr;
11207 bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);
11208 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11209 BasicBlock *IncomingBB = PHI.getIncomingBlock(u);
11210 if (LI.LivenessAA &&
11211 LI.LivenessAA->isEdgeDead(IncomingBB, PHI.getParent())) {
11212 LI.AnyDead = true;
11213 continue;
11214 }
11215 Value *V = PHI.getIncomingValue(u);
11216 if (V == &PHI)
11217 continue;
11218
11219 // If the incoming value is not the PHI but an instruction in the same
11220 // cycle we might have multiple versions of it flying around.
11221 if (CyclePHI && isa<Instruction>(V) &&
11222 (!C || C->contains(cast<Instruction>(V)->getParent())))
11223 return false;
11224
11225 Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});
11226 }
11227 return true;
11228 }
11229
11230 bool UsedAssumedInformation = false;
11231 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11232 IRPosition::inst(PHI), *this, UsedAssumedInformation, II.S);
11233 if (!SimpleV.has_value())
11234 return true;
11235 if (!(*SimpleV))
11236 return false;
11237 addValue(A, getState(), **SimpleV, &PHI, II.S, getAnchorScope());
11238 return true;
11239 }
11240
11241 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11242 /// simplify any operand of the instruction \p I. Return true if successful,
11243 /// in that case Worklist will be updated.
11244 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11245 SmallVectorImpl<ItemInfo> &Worklist) {
11246 bool SomeSimplified = false;
11247 bool UsedAssumedInformation = false;
11248
11249 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11250 int Idx = 0;
11251 for (Value *Op : I.operands()) {
11252 const auto &SimplifiedOp = A.getAssumedSimplified(
11253 IRPosition::value(*Op, getCallBaseContext()), *this,
11254 UsedAssumedInformation, AA::Intraprocedural);
11255 // If we are not sure about any operand we are not sure about the entire
11256 // instruction, we'll wait.
11257 if (!SimplifiedOp.has_value())
11258 return true;
11259
11260 if (*SimplifiedOp)
11261 NewOps[Idx] = *SimplifiedOp;
11262 else
11263 NewOps[Idx] = Op;
11264
11265 SomeSimplified |= (NewOps[Idx] != Op);
11266 ++Idx;
11267 }
11268
11269 // We won't bother with the InstSimplify interface if we didn't simplify any
11270 // operand ourselves.
11271 if (!SomeSimplified)
11272 return false;
11273
11274 InformationCache &InfoCache = A.getInfoCache();
11275 Function *F = I.getFunction();
11276 const auto *DT =
11278 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
11279 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
11280
11281 const DataLayout &DL = I.getDataLayout();
11282 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11283 Value *NewV = simplifyInstructionWithOperands(&I, NewOps, Q);
11284 if (!NewV || NewV == &I)
11285 return false;
11286
11287 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11288 << *NewV << "\n");
11289 Worklist.push_back({{*NewV, II.I.getCtxI()}, II.S});
11290 return true;
11291 }
11292
11294 Attributor &A, Instruction &I, ItemInfo II,
11295 SmallVectorImpl<ItemInfo> &Worklist,
11297 if (auto *CI = dyn_cast<CmpInst>(&I))
11298 return handleCmp(A, *CI, CI->getOperand(0), CI->getOperand(1),
11299 CI->getPredicate(), II, Worklist);
11300
11301 switch (I.getOpcode()) {
11302 case Instruction::Select:
11303 return handleSelectInst(A, cast<SelectInst>(I), II, Worklist);
11304 case Instruction::PHI:
11305 return handlePHINode(A, cast<PHINode>(I), II, Worklist, LivenessAAs);
11306 case Instruction::Load:
11307 return handleLoadInst(A, cast<LoadInst>(I), II, Worklist);
11308 default:
11309 return handleGenericInst(A, I, II, Worklist);
11310 };
11311 return false;
11312 }
11313
11314 void genericValueTraversal(Attributor &A, Value *InitialV) {
11316
11317 SmallSet<ItemInfo, 16> Visited;
11319 Worklist.push_back({{*InitialV, getCtxI()}, AA::AnyScope});
11320
11321 int Iteration = 0;
11322 do {
11323 ItemInfo II = Worklist.pop_back_val();
11324 Value *V = II.I.getValue();
11325 assert(V);
11326 const Instruction *CtxI = II.I.getCtxI();
11327 AA::ValueScope S = II.S;
11328
11329 // Check if we should process the current value. To prevent endless
11330 // recursion keep a record of the values we followed!
11331 if (!Visited.insert(II).second)
11332 continue;
11333
11334 // Make sure we limit the compile time for complex expressions.
11335 if (Iteration++ >= MaxPotentialValuesIterations) {
11336 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11337 << Iteration << "!\n");
11338 addValue(A, getState(), *V, CtxI, S, getAnchorScope());
11339 continue;
11340 }
11341
11342 // Explicitly look through calls with a "returned" attribute if we do
11343 // not have a pointer as stripPointerCasts only works on them.
11344 Value *NewV = nullptr;
11345 if (V->getType()->isPointerTy()) {
11346 NewV = AA::getWithType(*V->stripPointerCasts(), *V->getType());
11347 } else {
11348 if (auto *CB = dyn_cast<CallBase>(V))
11349 if (auto *Callee =
11350 dyn_cast_if_present<Function>(CB->getCalledOperand())) {
11351 for (Argument &Arg : Callee->args())
11352 if (Arg.hasReturnedAttr()) {
11353 NewV = CB->getArgOperand(Arg.getArgNo());
11354 break;
11355 }
11356 }
11357 }
11358 if (NewV && NewV != V) {
11359 Worklist.push_back({{*NewV, CtxI}, S});
11360 continue;
11361 }
11362
11363 if (auto *I = dyn_cast<Instruction>(V)) {
11364 if (simplifyInstruction(A, *I, II, Worklist, LivenessAAs))
11365 continue;
11366 }
11367
11368 if (V != InitialV || isa<Argument>(V))
11369 if (recurseForValue(A, IRPosition::value(*V), II.S))
11370 continue;
11371
11372 // If we haven't stripped anything we give up.
11373 if (V == InitialV && CtxI == getCtxI()) {
11374 indicatePessimisticFixpoint();
11375 return;
11376 }
11377
11378 addValue(A, getState(), *V, CtxI, S, getAnchorScope());
11379 } while (!Worklist.empty());
11380
11381 // If we actually used liveness information so we have to record a
11382 // dependence.
11383 for (auto &It : LivenessAAs)
11384 if (It.second.AnyDead)
11385 A.recordDependence(*It.second.LivenessAA, *this, DepClassTy::OPTIONAL);
11386 }
11387
11388 /// See AbstractAttribute::trackStatistics()
11389 void trackStatistics() const override {
11390 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11391 }
11392};
11393
11394struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11395 using Base = AAPotentialValuesImpl;
11396 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11397 : Base(IRP, A) {}
11398
11399 /// See AbstractAttribute::initialize(..).
11400 void initialize(Attributor &A) override {
11401 auto &Arg = cast<Argument>(getAssociatedValue());
11403 indicatePessimisticFixpoint();
11404 }
11405
11406 /// See AbstractAttribute::updateImpl(...).
11407 ChangeStatus updateImpl(Attributor &A) override {
11408 auto AssumedBefore = getAssumed();
11409
11410 unsigned ArgNo = getCalleeArgNo();
11411
11412 bool UsedAssumedInformation = false;
11414 auto CallSitePred = [&](AbstractCallSite ACS) {
11415 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11416 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11417 return false;
11418
11419 if (!A.getAssumedSimplifiedValues(CSArgIRP, this, Values,
11421 UsedAssumedInformation))
11422 return false;
11423
11424 return isValidState();
11425 };
11426
11427 if (!A.checkForAllCallSites(CallSitePred, *this,
11428 /* RequireAllCallSites */ true,
11429 UsedAssumedInformation))
11430 return indicatePessimisticFixpoint();
11431
11432 Function *Fn = getAssociatedFunction();
11433 bool AnyNonLocal = false;
11434 for (auto &It : Values) {
11435 if (isa<Constant>(It.getValue())) {
11436 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
11437 getAnchorScope());
11438 continue;
11439 }
11440 if (!AA::isDynamicallyUnique(A, *this, *It.getValue()))
11441 return indicatePessimisticFixpoint();
11442
11443 if (auto *Arg = dyn_cast<Argument>(It.getValue()))
11444 if (Arg->getParent() == Fn) {
11445 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
11446 getAnchorScope());
11447 continue;
11448 }
11449 addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::Interprocedural,
11450 getAnchorScope());
11451 AnyNonLocal = true;
11452 }
11453 assert(!undefIsContained() && "Undef should be an explicit value!");
11454 if (AnyNonLocal)
11455 giveUpOnIntraprocedural(A);
11456
11457 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11458 : ChangeStatus::CHANGED;
11459 }
11460
11461 /// See AbstractAttribute::trackStatistics()
11462 void trackStatistics() const override {
11463 STATS_DECLTRACK_ARG_ATTR(potential_values)
11464 }
11465};
11466
11467struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11468 using Base = AAPotentialValuesFloating;
11469 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11470 : Base(IRP, A) {}
11471
11472 /// See AbstractAttribute::initialize(..).
11473 void initialize(Attributor &A) override {
11474 Function *F = getAssociatedFunction();
11475 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11476 indicatePessimisticFixpoint();
11477 return;
11478 }
11479
11480 for (Argument &Arg : F->args())
11481 if (Arg.hasReturnedAttr()) {
11482 addValue(A, getState(), Arg, nullptr, AA::AnyScope, F);
11483 ReturnedArg = &Arg;
11484 break;
11485 }
11486 if (!A.isFunctionIPOAmendable(*F) ||
11487 A.hasSimplificationCallback(getIRPosition())) {
11488 if (!ReturnedArg)
11489 indicatePessimisticFixpoint();
11490 else
11491 indicateOptimisticFixpoint();
11492 }
11493 }
11494
11495 /// See AbstractAttribute::updateImpl(...).
11496 ChangeStatus updateImpl(Attributor &A) override {
11497 auto AssumedBefore = getAssumed();
11498 bool UsedAssumedInformation = false;
11499
11501 Function *AnchorScope = getAnchorScope();
11502 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11503 bool AddValues) {
11505 Values.clear();
11506 if (!A.getAssumedSimplifiedValues(IRPosition::value(V), this, Values, S,
11507 UsedAssumedInformation,
11508 /* RecurseForSelectAndPHI */ true))
11509 return false;
11510 if (!AddValues)
11511 continue;
11512 for (const AA::ValueAndContext &VAC : Values)
11513 addValue(A, getState(), *VAC.getValue(),
11514 VAC.getCtxI() ? VAC.getCtxI() : CtxI, S, AnchorScope);
11515 }
11516 return true;
11517 };
11518
11519 if (ReturnedArg) {
11520 HandleReturnedValue(*ReturnedArg, nullptr, true);
11521 } else {
11522 auto RetInstPred = [&](Instruction &RetI) {
11523 bool AddValues = true;
11524 if (isa<PHINode>(RetI.getOperand(0)) ||
11525 isa<SelectInst>(RetI.getOperand(0))) {
11526 addValue(A, getState(), *RetI.getOperand(0), &RetI, AA::AnyScope,
11527 AnchorScope);
11528 AddValues = false;
11529 }
11530 return HandleReturnedValue(*RetI.getOperand(0), &RetI, AddValues);
11531 };
11532
11533 if (!A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
11534 UsedAssumedInformation,
11535 /* CheckBBLivenessOnly */ true))
11536 return indicatePessimisticFixpoint();
11537 }
11538
11539 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11540 : ChangeStatus::CHANGED;
11541 }
11542
11543 void addValue(Attributor &A, StateType &State, Value &V,
11544 const Instruction *CtxI, AA::ValueScope S,
11545 Function *AnchorScope) const override {
11546 Function *F = getAssociatedFunction();
11547 if (auto *CB = dyn_cast<CallBase>(&V))
11548 if (CB->getCalledOperand() == F)
11549 return;
11550 Base::addValue(A, State, V, CtxI, S, AnchorScope);
11551 }
11552
11553 ChangeStatus manifest(Attributor &A) override {
11554 if (ReturnedArg)
11555 return ChangeStatus::UNCHANGED;
11557 if (!getAssumedSimplifiedValues(A, Values, AA::ValueScope::Intraprocedural,
11558 /* RecurseForSelectAndPHI */ true))
11559 return ChangeStatus::UNCHANGED;
11560 Value *NewVal = getSingleValue(A, *this, getIRPosition(), Values);
11561 if (!NewVal)
11562 return ChangeStatus::UNCHANGED;
11563
11564 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11565 if (auto *Arg = dyn_cast<Argument>(NewVal)) {
11566 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11567 "Number of function with unique return");
11568 Changed |= A.manifestAttrs(
11570 {Attribute::get(Arg->getContext(), Attribute::Returned)});
11571 STATS_DECLTRACK_ARG_ATTR(returned);
11572 }
11573
11574 auto RetInstPred = [&](Instruction &RetI) {
11575 Value *RetOp = RetI.getOperand(0);
11576 if (isa<UndefValue>(RetOp) || RetOp == NewVal)
11577 return true;
11578 if (AA::isValidAtPosition({*NewVal, RetI}, A.getInfoCache()))
11579 if (A.changeUseAfterManifest(RetI.getOperandUse(0), *NewVal))
11580 Changed = ChangeStatus::CHANGED;
11581 return true;
11582 };
11583 bool UsedAssumedInformation = false;
11584 (void)A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
11585 UsedAssumedInformation,
11586 /* CheckBBLivenessOnly */ true);
11587 return Changed;
11588 }
11589
11590 ChangeStatus indicatePessimisticFixpoint() override {
11592 }
11593
11594 /// See AbstractAttribute::trackStatistics()
11595 void trackStatistics() const override{
11596 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11597
11598 /// The argumented with an existing `returned` attribute.
11599 Argument *ReturnedArg = nullptr;
11600};
11601
11602struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11603 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11604 : AAPotentialValuesImpl(IRP, A) {}
11605
11606 /// See AbstractAttribute::updateImpl(...).
11607 ChangeStatus updateImpl(Attributor &A) override {
11608 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11609 "not be called");
11610 }
11611
11612 /// See AbstractAttribute::trackStatistics()
11613 void trackStatistics() const override {
11614 STATS_DECLTRACK_FN_ATTR(potential_values)
11615 }
11616};
11617
11618struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11619 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11620 : AAPotentialValuesFunction(IRP, A) {}
11621
11622 /// See AbstractAttribute::trackStatistics()
11623 void trackStatistics() const override {
11624 STATS_DECLTRACK_CS_ATTR(potential_values)
11625 }
11626};
11627
11628struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11629 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11630 : AAPotentialValuesImpl(IRP, A) {}
11631
11632 /// See AbstractAttribute::updateImpl(...).
11633 ChangeStatus updateImpl(Attributor &A) override {
11634 auto AssumedBefore = getAssumed();
11635
11636 Function *Callee = getAssociatedFunction();
11637 if (!Callee)
11638 return indicatePessimisticFixpoint();
11639
11640 bool UsedAssumedInformation = false;
11641 auto *CB = cast<CallBase>(getCtxI());
11642 if (CB->isMustTailCall() &&
11643 !A.isAssumedDead(IRPosition::inst(*CB), this, nullptr,
11644 UsedAssumedInformation))
11645 return indicatePessimisticFixpoint();
11646
11648 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
11649 Values, AA::Intraprocedural,
11650 UsedAssumedInformation))
11651 return indicatePessimisticFixpoint();
11652
11653 Function *Caller = CB->getCaller();
11654
11655 bool AnyNonLocal = false;
11656 for (auto &It : Values) {
11657 Value *V = It.getValue();
11658 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11659 V, *CB, *this, UsedAssumedInformation);
11660 if (!CallerV.has_value()) {
11661 // Nothing to do as long as no value was determined.
11662 continue;
11663 }
11664 V = *CallerV ? *CallerV : V;
11665 if (AA::isDynamicallyUnique(A, *this, *V) &&
11666 AA::isValidInScope(*V, Caller)) {
11667 if (*CallerV) {
11669 IRPosition IRP = IRPosition::value(*V);
11670 if (auto *Arg = dyn_cast<Argument>(V))
11671 if (Arg->getParent() == CB->getCalledOperand())
11672 IRP = IRPosition::callsite_argument(*CB, Arg->getArgNo());
11673 if (recurseForValue(A, IRP, AA::AnyScope))
11674 continue;
11675 }
11676 addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
11677 } else {
11678 AnyNonLocal = true;
11679 break;
11680 }
11681 }
11682 if (AnyNonLocal) {
11683 Values.clear();
11684 if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
11685 Values, AA::Interprocedural,
11686 UsedAssumedInformation))
11687 return indicatePessimisticFixpoint();
11688 AnyNonLocal = false;
11690 for (auto &It : Values) {
11691 Value *V = It.getValue();
11692 if (!AA::isDynamicallyUnique(A, *this, *V))
11693 return indicatePessimisticFixpoint();
11694 if (AA::isValidInScope(*V, Caller)) {
11695 addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
11696 } else {
11697 AnyNonLocal = true;
11698 addValue(A, getState(), *V, CB, AA::Interprocedural,
11699 getAnchorScope());
11700 }
11701 }
11702 if (AnyNonLocal)
11703 giveUpOnIntraprocedural(A);
11704 }
11705 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11706 : ChangeStatus::CHANGED;
11707 }
11708
11709 ChangeStatus indicatePessimisticFixpoint() override {
11710 return AAPotentialValues::indicatePessimisticFixpoint();
11711 }
11712
11713 /// See AbstractAttribute::trackStatistics()
11714 void trackStatistics() const override {
11715 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11716 }
11717};
11718
11719struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11720 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11721 : AAPotentialValuesFloating(IRP, A) {}
11722
11723 /// See AbstractAttribute::trackStatistics()
11724 void trackStatistics() const override {
11725 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11726 }
11727};
11728} // namespace
11729
11730/// ---------------------- Assumption Propagation ------------------------------
11731namespace {
11732struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11733 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11734 const DenseSet<StringRef> &Known)
11735 : AAAssumptionInfo(IRP, A, Known) {}
11736
11737 /// See AbstractAttribute::manifest(...).
11738 ChangeStatus manifest(Attributor &A) override {
11739 // Don't manifest a universal set if it somehow made it here.
11740 if (getKnown().isUniversal())
11741 return ChangeStatus::UNCHANGED;
11742
11743 const IRPosition &IRP = getIRPosition();
11744 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11745 getAssumed().getSet().end());
11746 llvm::sort(Set);
11747 return A.manifestAttrs(IRP,
11750 llvm::join(Set, ",")),
11751 /*ForceReplace=*/true);
11752 }
11753
11754 bool hasAssumption(const StringRef Assumption) const override {
11755 return isValidState() && setContains(Assumption);
11756 }
11757
11758 /// See AbstractAttribute::getAsStr()
11759 const std::string getAsStr(Attributor *A) const override {
11760 const SetContents &Known = getKnown();
11761 const SetContents &Assumed = getAssumed();
11762
11763 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11764 llvm::sort(Set);
11765 const std::string KnownStr = llvm::join(Set, ",");
11766
11767 std::string AssumedStr = "Universal";
11768 if (!Assumed.isUniversal()) {
11769 Set.assign(Assumed.getSet().begin(), Assumed.getSet().end());
11770 AssumedStr = llvm::join(Set, ",");
11771 }
11772 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11773 }
11774};
11775
11776/// Propagates assumption information from parent functions to all of their
11777/// successors. An assumption can be propagated if the containing function
11778/// dominates the called function.
11779///
11780/// We start with a "known" set of assumptions already valid for the associated
11781/// function and an "assumed" set that initially contains all possible
11782/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11783/// contents as concrete values are known. The concrete values are seeded by the
11784/// first nodes that are either entries into the call graph, or contains no
11785/// assumptions. Each node is updated as the intersection of the assumed state
11786/// with all of its predecessors.
11787struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11788 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11789 : AAAssumptionInfoImpl(IRP, A,
11790 getAssumptions(*IRP.getAssociatedFunction())) {}
11791
11792 /// See AbstractAttribute::updateImpl(...).
11793 ChangeStatus updateImpl(Attributor &A) override {
11794 bool Changed = false;
11795
11796 auto CallSitePred = [&](AbstractCallSite ACS) {
11797 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11798 *this, IRPosition::callsite_function(*ACS.getInstruction()),
11799 DepClassTy::REQUIRED);
11800 if (!AssumptionAA)
11801 return false;
11802 // Get the set of assumptions shared by all of this function's callers.
11803 Changed |= getIntersection(AssumptionAA->getAssumed());
11804 return !getAssumed().empty() || !getKnown().empty();
11805 };
11806
11807 bool UsedAssumedInformation = false;
11808 // Get the intersection of all assumptions held by this node's predecessors.
11809 // If we don't know all the call sites then this is either an entry into the
11810 // call graph or an empty node. This node is known to only contain its own
11811 // assumptions and can be propagated to its successors.
11812 if (!A.checkForAllCallSites(CallSitePred, *this, true,
11813 UsedAssumedInformation))
11814 return indicatePessimisticFixpoint();
11815
11816 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11817 }
11818
11819 void trackStatistics() const override {}
11820};
11821
11822/// Assumption Info defined for call sites.
11823struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11824
11825 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
11826 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
11827
11828 /// See AbstractAttribute::initialize(...).
11829 void initialize(Attributor &A) override {
11830 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
11831 A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
11832 }
11833
11834 /// See AbstractAttribute::updateImpl(...).
11835 ChangeStatus updateImpl(Attributor &A) override {
11836 const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
11837 auto *AssumptionAA =
11838 A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
11839 if (!AssumptionAA)
11840 return indicatePessimisticFixpoint();
11841 bool Changed = getIntersection(AssumptionAA->getAssumed());
11842 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11843 }
11844
11845 /// See AbstractAttribute::trackStatistics()
11846 void trackStatistics() const override {}
11847
11848private:
11849 /// Helper to initialized the known set as all the assumptions this call and
11850 /// the callee contain.
11851 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
11852 const CallBase &CB = cast<CallBase>(IRP.getAssociatedValue());
11853 auto Assumptions = getAssumptions(CB);
11854 if (const Function *F = CB.getCaller())
11855 set_union(Assumptions, getAssumptions(*F));
11856 if (Function *F = IRP.getAssociatedFunction())
11857 set_union(Assumptions, getAssumptions(*F));
11858 return Assumptions;
11859 }
11860};
11861} // namespace
11862
11864 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
11865 A.getOrCreateAAFor<AACallEdges>(IRPosition::function(**I))));
11866}
11867
11869
11870/// ------------------------ UnderlyingObjects ---------------------------------
11871
11872namespace {
11873struct AAUnderlyingObjectsImpl
11874 : StateWrapper<BooleanState, AAUnderlyingObjects> {
11876 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
11877
11878 /// See AbstractAttribute::getAsStr().
11879 const std::string getAsStr(Attributor *A) const override {
11880 if (!isValidState())
11881 return "<invalid>";
11882 std::string Str;
11884 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
11885 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
11886 << " objects.\n";
11887 if (!InterAssumedUnderlyingObjects.empty()) {
11888 OS << "inter objects:\n";
11889 for (auto *Obj : InterAssumedUnderlyingObjects)
11890 OS << *Obj << '\n';
11891 }
11892 if (!IntraAssumedUnderlyingObjects.empty()) {
11893 OS << "intra objects:\n";
11894 for (auto *Obj : IntraAssumedUnderlyingObjects)
11895 OS << *Obj << '\n';
11896 }
11897 return Str;
11898 }
11899
11900 /// See AbstractAttribute::trackStatistics()
11901 void trackStatistics() const override {}
11902
11903 /// See AbstractAttribute::updateImpl(...).
11904 ChangeStatus updateImpl(Attributor &A) override {
11905 auto &Ptr = getAssociatedValue();
11906
11907 bool UsedAssumedInformation = false;
11908 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
11910 SmallPtrSet<Value *, 8> SeenObjects;
11912
11913 if (!A.getAssumedSimplifiedValues(IRPosition::value(Ptr), *this, Values,
11914 Scope, UsedAssumedInformation))
11915 return UnderlyingObjects.insert(&Ptr);
11916
11917 bool Changed = false;
11918
11919 for (unsigned I = 0; I < Values.size(); ++I) {
11920 auto &VAC = Values[I];
11921 auto *Obj = VAC.getValue();
11922 Value *UO = getUnderlyingObject(Obj);
11923 if (!SeenObjects.insert(UO ? UO : Obj).second)
11924 continue;
11925 if (UO && UO != Obj) {
11926 if (isa<AllocaInst>(UO) || isa<GlobalValue>(UO)) {
11927 Changed |= UnderlyingObjects.insert(UO);
11928 continue;
11929 }
11930
11931 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
11932 *this, IRPosition::value(*UO), DepClassTy::OPTIONAL);
11933 auto Pred = [&](Value &V) {
11934 if (&V == UO)
11935 Changed |= UnderlyingObjects.insert(UO);
11936 else
11937 Values.emplace_back(V, nullptr);
11938 return true;
11939 };
11940
11941 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
11943 "The forall call should not return false at this position");
11944 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
11945 continue;
11946 }
11947
11948 if (isa<SelectInst>(Obj)) {
11949 Changed |= handleIndirect(A, *Obj, UnderlyingObjects, Scope,
11950 UsedAssumedInformation);
11951 continue;
11952 }
11953 if (auto *PHI = dyn_cast<PHINode>(Obj)) {
11954 // Explicitly look through PHIs as we do not care about dynamically
11955 // uniqueness.
11956 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
11957 Changed |=
11958 handleIndirect(A, *PHI->getIncomingValue(u), UnderlyingObjects,
11959 Scope, UsedAssumedInformation);
11960 }
11961 continue;
11962 }
11963
11964 Changed |= UnderlyingObjects.insert(Obj);
11965 }
11966
11967 return Changed;
11968 };
11969
11970 bool Changed = false;
11971 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
11972 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
11973 if (!UsedAssumedInformation)
11974 indicateOptimisticFixpoint();
11975 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11976 }
11977
11978 bool forallUnderlyingObjects(
11979 function_ref<bool(Value &)> Pred,
11980 AA::ValueScope Scope = AA::Interprocedural) const override {
11981 if (!isValidState())
11982 return Pred(getAssociatedValue());
11983
11984 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
11985 ? IntraAssumedUnderlyingObjects
11986 : InterAssumedUnderlyingObjects;
11987 for (Value *Obj : AssumedUnderlyingObjects)
11988 if (!Pred(*Obj))
11989 return false;
11990
11991 return true;
11992 }
11993
11994private:
11995 /// Handle the case where the value is not the actual underlying value, such
11996 /// as a phi node or a select instruction.
11997 bool handleIndirect(Attributor &A, Value &V,
11998 SmallSetVector<Value *, 8> &UnderlyingObjects,
11999 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12000 bool Changed = false;
12001 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12002 *this, IRPosition::value(V), DepClassTy::OPTIONAL);
12003 auto Pred = [&](Value &V) {
12004 Changed |= UnderlyingObjects.insert(&V);
12005 return true;
12006 };
12007 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12009 "The forall call should not return false at this position");
12010 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12011 return Changed;
12012 }
12013
12014 /// All the underlying objects collected so far via intra procedural scope.
12015 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12016 /// All the underlying objects collected so far via inter procedural scope.
12017 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12018};
12019
12020struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12021 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12022 : AAUnderlyingObjectsImpl(IRP, A) {}
12023};
12024
12025struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12026 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12027 : AAUnderlyingObjectsImpl(IRP, A) {}
12028};
12029
12030struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12031 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12032 : AAUnderlyingObjectsImpl(IRP, A) {}
12033};
12034
12035struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12036 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12037 : AAUnderlyingObjectsImpl(IRP, A) {}
12038};
12039
12040struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12041 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12042 : AAUnderlyingObjectsImpl(IRP, A) {}
12043};
12044
12045struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12046 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12047 : AAUnderlyingObjectsImpl(IRP, A) {}
12048};
12049
12050struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12051 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12052 : AAUnderlyingObjectsImpl(IRP, A) {}
12053};
12054} // namespace
12055
12056/// ------------------------ Global Value Info -------------------------------
12057namespace {
12058struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12059 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12060 : AAGlobalValueInfo(IRP, A) {}
12061
12062 /// See AbstractAttribute::initialize(...).
12063 void initialize(Attributor &A) override {}
12064
12065 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12067 Instruction *UInst = dyn_cast<Instruction>(U.getUser());
12068 if (!UInst) {
12069 Follow = true;
12070 return true;
12071 }
12072
12073 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12074 << *UInst << "\n");
12075
12076 if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
12077 int Idx = &Cmp->getOperandUse(0) == &U;
12078 if (isa<Constant>(Cmp->getOperand(Idx)))
12079 return true;
12080 return U == &getAnchorValue();
12081 }
12082
12083 // Explicitly catch return instructions.
12084 if (isa<ReturnInst>(UInst)) {
12085 auto CallSitePred = [&](AbstractCallSite ACS) {
12086 Worklist.push_back(ACS.getInstruction());
12087 return true;
12088 };
12089 bool UsedAssumedInformation = false;
12090 // TODO: We should traverse the uses or add a "non-call-site" CB.
12091 if (!A.checkForAllCallSites(CallSitePred, *UInst->getFunction(),
12092 /*RequireAllCallSites=*/true, this,
12093 UsedAssumedInformation))
12094 return false;
12095 return true;
12096 }
12097
12098 // For now we only use special logic for call sites. However, the tracker
12099 // itself knows about a lot of other non-capturing cases already.
12100 auto *CB = dyn_cast<CallBase>(UInst);
12101 if (!CB)
12102 return false;
12103 // Direct calls are OK uses.
12104 if (CB->isCallee(&U))
12105 return true;
12106 // Non-argument uses are scary.
12107 if (!CB->isArgOperand(&U))
12108 return false;
12109 // TODO: Iterate callees.
12110 auto *Fn = dyn_cast<Function>(CB->getCalledOperand());
12111 if (!Fn || !A.isFunctionIPOAmendable(*Fn))
12112 return false;
12113
12114 unsigned ArgNo = CB->getArgOperandNo(&U);
12115 Worklist.push_back(Fn->getArg(ArgNo));
12116 return true;
12117 }
12118
12119 ChangeStatus updateImpl(Attributor &A) override {
12120 unsigned NumUsesBefore = Uses.size();
12121
12124 Worklist.push_back(&getAnchorValue());
12125
12126 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12127 Uses.insert(&U);
12128 switch (DetermineUseCaptureKind(U, nullptr)) {
12129 case UseCaptureKind::NO_CAPTURE:
12130 return checkUse(A, U, Follow, Worklist);
12131 case UseCaptureKind::MAY_CAPTURE:
12132 return checkUse(A, U, Follow, Worklist);
12133 case UseCaptureKind::PASSTHROUGH:
12134 Follow = true;
12135 return true;
12136 }
12137 return true;
12138 };
12139 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12140 Uses.insert(&OldU);
12141 return true;
12142 };
12143
12144 while (!Worklist.empty()) {
12145 const Value *V = Worklist.pop_back_val();
12146 if (!Visited.insert(V).second)
12147 continue;
12148 if (!A.checkForAllUses(UsePred, *this, *V,
12149 /* CheckBBLivenessOnly */ true,
12150 DepClassTy::OPTIONAL,
12151 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12152 return indicatePessimisticFixpoint();
12153 }
12154 }
12155
12156 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12157 : ChangeStatus::CHANGED;
12158 }
12159
12160 bool isPotentialUse(const Use &U) const override {
12161 return !isValidState() || Uses.contains(&U);
12162 }
12163
12164 /// See AbstractAttribute::manifest(...).
12165 ChangeStatus manifest(Attributor &A) override {
12166 return ChangeStatus::UNCHANGED;
12167 }
12168
12169 /// See AbstractAttribute::getAsStr().
12170 const std::string getAsStr(Attributor *A) const override {
12171 return "[" + std::to_string(Uses.size()) + " uses]";
12172 }
12173
12174 void trackStatistics() const override {
12175 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12176 }
12177
12178private:
12179 /// Set of (transitive) uses of this GlobalValue.
12181};
12182} // namespace
12183
12184/// ------------------------ Indirect Call Info -------------------------------
12185namespace {
12186struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12187 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12188 : AAIndirectCallInfo(IRP, A) {}
12189
12190 /// See AbstractAttribute::initialize(...).
12191 void initialize(Attributor &A) override {
12192 auto *MD = getCtxI()->getMetadata(LLVMContext::MD_callees);
12193 if (!MD && !A.isClosedWorldModule())
12194 return;
12195
12196 if (MD) {
12197 for (const auto &Op : MD->operands())
12198 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(Op))
12199 PotentialCallees.insert(Callee);
12200 } else if (A.isClosedWorldModule()) {
12201 ArrayRef<Function *> IndirectlyCallableFunctions =
12202 A.getInfoCache().getIndirectlyCallableFunctions(A);
12203 PotentialCallees.insert(IndirectlyCallableFunctions.begin(),
12204 IndirectlyCallableFunctions.end());
12205 }
12206
12207 if (PotentialCallees.empty())
12208 indicateOptimisticFixpoint();
12209 }
12210
12211 ChangeStatus updateImpl(Attributor &A) override {
12212 CallBase *CB = cast<CallBase>(getCtxI());
12213 const Use &CalleeUse = CB->getCalledOperandUse();
12214 Value *FP = CB->getCalledOperand();
12215
12216 SmallSetVector<Function *, 4> AssumedCalleesNow;
12217 bool AllCalleesKnownNow = AllCalleesKnown;
12218
12219 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12220 bool &UsedAssumedInformation) {
12221 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12222 *this, IRPosition::value(PotentialCallee), DepClassTy::OPTIONAL);
12223 if (!GIAA || GIAA->isPotentialUse(CalleeUse))
12224 return true;
12225 UsedAssumedInformation = !GIAA->isAtFixpoint();
12226 return false;
12227 };
12228
12229 auto AddPotentialCallees = [&]() {
12230 for (auto *PotentialCallee : PotentialCallees) {
12231 bool UsedAssumedInformation = false;
12232 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12233 AssumedCalleesNow.insert(PotentialCallee);
12234 }
12235 };
12236
12237 // Use simplification to find potential callees, if !callees was present,
12238 // fallback to that set if necessary.
12239 bool UsedAssumedInformation = false;
12241 if (!A.getAssumedSimplifiedValues(IRPosition::value(*FP), this, Values,
12242 AA::ValueScope::AnyScope,
12243 UsedAssumedInformation)) {
12244 if (PotentialCallees.empty())
12245 return indicatePessimisticFixpoint();
12246 AddPotentialCallees();
12247 }
12248
12249 // Try to find a reason for \p Fn not to be a potential callee. If none was
12250 // found, add it to the assumed callees set.
12251 auto CheckPotentialCallee = [&](Function &Fn) {
12252 if (!PotentialCallees.empty() && !PotentialCallees.count(&Fn))
12253 return false;
12254
12255 auto &CachedResult = FilterResults[&Fn];
12256 if (CachedResult.has_value())
12257 return CachedResult.value();
12258
12259 bool UsedAssumedInformation = false;
12260 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12261 if (!UsedAssumedInformation)
12262 CachedResult = false;
12263 return false;
12264 }
12265
12266 int NumFnArgs = Fn.arg_size();
12267 int NumCBArgs = CB->arg_size();
12268
12269 // Check if any excess argument (which we fill up with poison) is known to
12270 // be UB on undef.
12271 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12272 bool IsKnown = false;
12273 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12274 A, this, IRPosition::argument(*Fn.getArg(I)),
12275 DepClassTy::OPTIONAL, IsKnown)) {
12276 if (IsKnown)
12277 CachedResult = false;
12278 return false;
12279 }
12280 }
12281
12282 CachedResult = true;
12283 return true;
12284 };
12285
12286 // Check simplification result, prune known UB callees, also restrict it to
12287 // the !callees set, if present.
12288 for (auto &VAC : Values) {
12289 if (isa<UndefValue>(VAC.getValue()))
12290 continue;
12291 if (isa<ConstantPointerNull>(VAC.getValue()) &&
12292 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12293 continue;
12294 // TODO: Check for known UB, e.g., poison + noundef.
12295 if (auto *VACFn = dyn_cast<Function>(VAC.getValue())) {
12296 if (CheckPotentialCallee(*VACFn))
12297 AssumedCalleesNow.insert(VACFn);
12298 continue;
12299 }
12300 if (!PotentialCallees.empty()) {
12301 AddPotentialCallees();
12302 break;
12303 }
12304 AllCalleesKnownNow = false;
12305 }
12306
12307 if (AssumedCalleesNow == AssumedCallees &&
12308 AllCalleesKnown == AllCalleesKnownNow)
12309 return ChangeStatus::UNCHANGED;
12310
12311 std::swap(AssumedCallees, AssumedCalleesNow);
12312 AllCalleesKnown = AllCalleesKnownNow;
12313 return ChangeStatus::CHANGED;
12314 }
12315
12316 /// See AbstractAttribute::manifest(...).
12317 ChangeStatus manifest(Attributor &A) override {
12318 // If we can't specialize at all, give up now.
12319 if (!AllCalleesKnown && AssumedCallees.empty())
12320 return ChangeStatus::UNCHANGED;
12321
12322 CallBase *CB = cast<CallBase>(getCtxI());
12323 bool UsedAssumedInformation = false;
12324 if (A.isAssumedDead(*CB, this, /*LivenessAA=*/nullptr,
12325 UsedAssumedInformation))
12326 return ChangeStatus::UNCHANGED;
12327
12328 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12329 Value *FP = CB->getCalledOperand();
12330 if (FP->getType()->getPointerAddressSpace())
12331 FP = new AddrSpaceCastInst(FP, PointerType::get(FP->getType(), 0),
12332 FP->getName() + ".as0", CB->getIterator());
12333
12334 bool CBIsVoid = CB->getType()->isVoidTy();
12336 FunctionType *CSFT = CB->getFunctionType();
12337 SmallVector<Value *> CSArgs(CB->arg_begin(), CB->arg_end());
12338
12339 // If we know all callees and there are none, the call site is (effectively)
12340 // dead (or UB).
12341 if (AssumedCallees.empty()) {
12342 assert(AllCalleesKnown &&
12343 "Expected all callees to be known if there are none.");
12344 A.changeToUnreachableAfterManifest(CB);
12345 return ChangeStatus::CHANGED;
12346 }
12347
12348 // Special handling for the single callee case.
12349 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12350 auto *NewCallee = AssumedCallees.front();
12351 if (isLegalToPromote(*CB, NewCallee)) {
12352 promoteCall(*CB, NewCallee, nullptr);
12353 return ChangeStatus::CHANGED;
12354 }
12355 Instruction *NewCall =
12356 CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
12357 CB->getName(), CB->getIterator());
12358 if (!CBIsVoid)
12359 A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewCall);
12360 A.deleteAfterManifest(*CB);
12361 return ChangeStatus::CHANGED;
12362 }
12363
12364 // For each potential value we create a conditional
12365 //
12366 // ```
12367 // if (ptr == value) value(args);
12368 // else ...
12369 // ```
12370 //
12371 bool SpecializedForAnyCallees = false;
12372 bool SpecializedForAllCallees = AllCalleesKnown;
12373 ICmpInst *LastCmp = nullptr;
12374 SmallVector<Function *, 8> SkippedAssumedCallees;
12376 for (Function *NewCallee : AssumedCallees) {
12377 if (!A.shouldSpecializeCallSiteForCallee(*this, *CB, *NewCallee,
12378 AssumedCallees.size())) {
12379 SkippedAssumedCallees.push_back(NewCallee);
12380 SpecializedForAllCallees = false;
12381 continue;
12382 }
12383 SpecializedForAnyCallees = true;
12384
12385 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12386 Instruction *ThenTI =
12387 SplitBlockAndInsertIfThen(LastCmp, IP, /* Unreachable */ false);
12388 BasicBlock *CBBB = CB->getParent();
12389 A.registerManifestAddedBasicBlock(*ThenTI->getParent());
12390 A.registerManifestAddedBasicBlock(*IP->getParent());
12391 auto *SplitTI = cast<BranchInst>(LastCmp->getNextNode());
12392 BasicBlock *ElseBB;
12393 if (&*IP == CB) {
12394 ElseBB = BasicBlock::Create(ThenTI->getContext(), "",
12395 ThenTI->getFunction(), CBBB);
12396 A.registerManifestAddedBasicBlock(*ElseBB);
12397 IP = BranchInst::Create(CBBB, ElseBB)->getIterator();
12398 SplitTI->replaceUsesOfWith(CBBB, ElseBB);
12399 } else {
12400 ElseBB = IP->getParent();
12401 ThenTI->replaceUsesOfWith(ElseBB, CBBB);
12402 }
12403 CastInst *RetBC = nullptr;
12404 CallInst *NewCall = nullptr;
12405 if (isLegalToPromote(*CB, NewCallee)) {
12406 auto *CBClone = cast<CallBase>(CB->clone());
12407 CBClone->insertBefore(ThenTI);
12408 NewCall = &cast<CallInst>(promoteCall(*CBClone, NewCallee, &RetBC));
12409 } else {
12410 NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
12411 CB->getName(), ThenTI->getIterator());
12412 }
12413 NewCalls.push_back({NewCall, RetBC});
12414 }
12415
12416 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12417 if (!AllCalleesKnown)
12418 return ChangeStatus::UNCHANGED;
12419 MDBuilder MDB(IndirectCB.getContext());
12420 MDNode *Callees = MDB.createCallees(SkippedAssumedCallees);
12421 IndirectCB.setMetadata(LLVMContext::MD_callees, Callees);
12422 return ChangeStatus::CHANGED;
12423 };
12424
12425 if (!SpecializedForAnyCallees)
12426 return AttachCalleeMetadata(*CB);
12427
12428 // Check if we need the fallback indirect call still.
12429 if (SpecializedForAllCallees) {
12431 LastCmp->eraseFromParent();
12432 new UnreachableInst(IP->getContext(), IP);
12433 IP->eraseFromParent();
12434 } else {
12435 auto *CBClone = cast<CallInst>(CB->clone());
12436 CBClone->setName(CB->getName());
12437 CBClone->insertBefore(*IP->getParent(), IP);
12438 NewCalls.push_back({CBClone, nullptr});
12439 AttachCalleeMetadata(*CBClone);
12440 }
12441
12442 // Check if we need a PHI to merge the results.
12443 if (!CBIsVoid) {
12444 auto *PHI = PHINode::Create(CB->getType(), NewCalls.size(),
12445 CB->getName() + ".phi",
12446 CB->getParent()->getFirstInsertionPt());
12447 for (auto &It : NewCalls) {
12448 CallBase *NewCall = It.first;
12449 Instruction *CallRet = It.second ? It.second : It.first;
12450 if (CallRet->getType() == CB->getType())
12451 PHI->addIncoming(CallRet, CallRet->getParent());
12452 else if (NewCall->getType()->isVoidTy())
12453 PHI->addIncoming(PoisonValue::get(CB->getType()),
12454 NewCall->getParent());
12455 else
12456 llvm_unreachable("Call return should match or be void!");
12457 }
12458 A.changeAfterManifest(IRPosition::callsite_returned(*CB), *PHI);
12459 }
12460
12461 A.deleteAfterManifest(*CB);
12462 Changed = ChangeStatus::CHANGED;
12463
12464 return Changed;
12465 }
12466
12467 /// See AbstractAttribute::getAsStr().
12468 const std::string getAsStr(Attributor *A) const override {
12469 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12470 " indirect call site with " + std::to_string(AssumedCallees.size()) +
12471 " functions";
12472 }
12473
12474 void trackStatistics() const override {
12475 if (AllCalleesKnown) {
12477 Eliminated, CallSites,
12478 "Number of indirect call sites eliminated via specialization")
12479 } else {
12480 STATS_DECLTRACK(Specialized, CallSites,
12481 "Number of indirect call sites specialized")
12482 }
12483 }
12484
12485 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12486 return isValidState() && AllCalleesKnown && all_of(AssumedCallees, CB);
12487 }
12488
12489private:
12490 /// Map to remember filter results.
12492
12493 /// If the !callee metadata was present, this set will contain all potential
12494 /// callees (superset).
12495 SmallSetVector<Function *, 4> PotentialCallees;
12496
12497 /// This set contains all currently assumed calllees, which might grow over
12498 /// time.
12499 SmallSetVector<Function *, 4> AssumedCallees;
12500
12501 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12502 /// if there could be others.
12503 bool AllCalleesKnown = true;
12504};
12505} // namespace
12506
12507/// ------------------------ Address Space ------------------------------------
12508namespace {
12509struct AAAddressSpaceImpl : public AAAddressSpace {
12510 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
12511 : AAAddressSpace(IRP, A) {}
12512
12513 int32_t getAddressSpace() const override {
12514 assert(isValidState() && "the AA is invalid");
12515 return AssumedAddressSpace;
12516 }
12517
12518 /// See AbstractAttribute::initialize(...).
12519 void initialize(Attributor &A) override {
12520 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
12521 "Associated value is not a pointer");
12522 if (getAssociatedType()->getPointerAddressSpace())
12523 indicateOptimisticFixpoint();
12524 }
12525
12526 ChangeStatus updateImpl(Attributor &A) override {
12527 int32_t OldAddressSpace = AssumedAddressSpace;
12528 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(), this,
12529 DepClassTy::REQUIRED);
12530 auto Pred = [&](Value &Obj) {
12531 if (isa<UndefValue>(&Obj))
12532 return true;
12533 return takeAddressSpace(Obj.getType()->getPointerAddressSpace());
12534 };
12535
12536 if (!AUO->forallUnderlyingObjects(Pred))
12537 return indicatePessimisticFixpoint();
12538
12539 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
12540 : ChangeStatus::CHANGED;
12541 }
12542
12543 /// See AbstractAttribute::manifest(...).
12544 ChangeStatus manifest(Attributor &A) override {
12545 Value *AssociatedValue = &getAssociatedValue();
12546 Value *OriginalValue = peelAddrspacecast(AssociatedValue);
12547 if (getAddressSpace() == NoAddressSpace ||
12548 static_cast<uint32_t>(getAddressSpace()) ==
12549 getAssociatedType()->getPointerAddressSpace())
12550 return ChangeStatus::UNCHANGED;
12551
12552 Type *NewPtrTy = PointerType::get(getAssociatedType()->getContext(),
12553 static_cast<uint32_t>(getAddressSpace()));
12554 bool UseOriginalValue =
12555 OriginalValue->getType()->getPointerAddressSpace() ==
12556 static_cast<uint32_t>(getAddressSpace());
12557
12558 bool Changed = false;
12559
12560 auto MakeChange = [&](Instruction *I, Use &U) {
12561 Changed = true;
12562 if (UseOriginalValue) {
12563 A.changeUseAfterManifest(U, *OriginalValue);
12564 return;
12565 }
12566 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
12567 CastInst->insertBefore(cast<Instruction>(I));
12568 A.changeUseAfterManifest(U, *CastInst);
12569 };
12570
12571 auto Pred = [&](const Use &U, bool &) {
12572 if (U.get() != AssociatedValue)
12573 return true;
12574 auto *Inst = dyn_cast<Instruction>(U.getUser());
12575 if (!Inst)
12576 return true;
12577 // This is a WA to make sure we only change uses from the corresponding
12578 // CGSCC if the AA is run on CGSCC instead of the entire module.
12579 if (!A.isRunOn(Inst->getFunction()))
12580 return true;
12581 if (isa<LoadInst>(Inst))
12582 MakeChange(Inst, const_cast<Use &>(U));
12583 if (isa<StoreInst>(Inst)) {
12584 // We only make changes if the use is the pointer operand.
12585 if (U.getOperandNo() == 1)
12586 MakeChange(Inst, const_cast<Use &>(U));
12587 }
12588 return true;
12589 };
12590
12591 // It doesn't matter if we can't check all uses as we can simply
12592 // conservatively ignore those that can not be visited.
12593 (void)A.checkForAllUses(Pred, *this, getAssociatedValue(),
12594 /* CheckBBLivenessOnly */ true);
12595
12596 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12597 }
12598
12599 /// See AbstractAttribute::getAsStr().
12600 const std::string getAsStr(Attributor *A) const override {
12601 if (!isValidState())
12602 return "addrspace(<invalid>)";
12603 return "addrspace(" +
12604 (AssumedAddressSpace == NoAddressSpace
12605 ? "none"
12606 : std::to_string(AssumedAddressSpace)) +
12607 ")";
12608 }
12609
12610private:
12611 int32_t AssumedAddressSpace = NoAddressSpace;
12612
12613 bool takeAddressSpace(int32_t AS) {
12614 if (AssumedAddressSpace == NoAddressSpace) {
12615 AssumedAddressSpace = AS;
12616 return true;
12617 }
12618 return AssumedAddressSpace == AS;
12619 }
12620
12621 static Value *peelAddrspacecast(Value *V) {
12622 if (auto *I = dyn_cast<AddrSpaceCastInst>(V))
12623 return peelAddrspacecast(I->getPointerOperand());
12624 if (auto *C = dyn_cast<ConstantExpr>(V))
12625 if (C->getOpcode() == Instruction::AddrSpaceCast)
12626 return peelAddrspacecast(C->getOperand(0));
12627 return V;
12628 }
12629};
12630
12631struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
12632 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
12633 : AAAddressSpaceImpl(IRP, A) {}
12634
12635 void trackStatistics() const override {
12637 }
12638};
12639
12640struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
12641 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
12642 : AAAddressSpaceImpl(IRP, A) {}
12643
12644 /// See AbstractAttribute::initialize(...).
12645 void initialize(Attributor &A) override {
12646 // TODO: we don't rewrite function argument for now because it will need to
12647 // rewrite the function signature and all call sites.
12648 (void)indicatePessimisticFixpoint();
12649 }
12650
12651 void trackStatistics() const override {
12652 STATS_DECLTRACK_FNRET_ATTR(addrspace);
12653 }
12654};
12655
12656struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
12657 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
12658 : AAAddressSpaceImpl(IRP, A) {}
12659
12660 void trackStatistics() const override {
12661 STATS_DECLTRACK_CSRET_ATTR(addrspace);
12662 }
12663};
12664
12665struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
12666 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
12667 : AAAddressSpaceImpl(IRP, A) {}
12668
12669 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
12670};
12671
12672struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
12673 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
12674 : AAAddressSpaceImpl(IRP, A) {}
12675
12676 /// See AbstractAttribute::initialize(...).
12677 void initialize(Attributor &A) override {
12678 // TODO: we don't rewrite call site argument for now because it will need to
12679 // rewrite the function signature of the callee.
12680 (void)indicatePessimisticFixpoint();
12681 }
12682
12683 void trackStatistics() const override {
12684 STATS_DECLTRACK_CSARG_ATTR(addrspace);
12685 }
12686};
12687} // namespace
12688
12689/// ----------- Allocation Info ----------
12690namespace {
12691struct AAAllocationInfoImpl : public AAAllocationInfo {
12692 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
12693 : AAAllocationInfo(IRP, A) {}
12694
12695 std::optional<TypeSize> getAllocatedSize() const override {
12696 assert(isValidState() && "the AA is invalid");
12697 return AssumedAllocatedSize;
12698 }
12699
12700 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
12701 const DataLayout &DL) {
12702
12703 // TODO: implement case for malloc like instructions
12704 switch (I->getOpcode()) {
12705 case Instruction::Alloca: {
12706 AllocaInst *AI = cast<AllocaInst>(I);
12707 return AI->getAllocationSize(DL);
12708 }
12709 default:
12710 return std::nullopt;
12711 }
12712 }
12713
12714 ChangeStatus updateImpl(Attributor &A) override {
12715
12716 const IRPosition &IRP = getIRPosition();
12717 Instruction *I = IRP.getCtxI();
12718
12719 // TODO: update check for malloc like calls
12720 if (!isa<AllocaInst>(I))
12721 return indicatePessimisticFixpoint();
12722
12723 bool IsKnownNoCapture;
12724 if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
12725 A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
12726 return indicatePessimisticFixpoint();
12727
12728 const AAPointerInfo *PI =
12729 A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
12730
12731 if (!PI)
12732 return indicatePessimisticFixpoint();
12733
12734 if (!PI->getState().isValidState())
12735 return indicatePessimisticFixpoint();
12736
12737 const DataLayout &DL = A.getDataLayout();
12738 const auto AllocationSize = findInitialAllocationSize(I, DL);
12739
12740 // If allocation size is nullopt, we give up.
12741 if (!AllocationSize)
12742 return indicatePessimisticFixpoint();
12743
12744 // For zero sized allocations, we give up.
12745 // Since we can't reduce further
12746 if (*AllocationSize == 0)
12747 return indicatePessimisticFixpoint();
12748
12749 int64_t BinSize = PI->numOffsetBins();
12750
12751 // TODO: implement for multiple bins
12752 if (BinSize > 1)
12753 return indicatePessimisticFixpoint();
12754
12755 if (BinSize == 0) {
12756 auto NewAllocationSize = std::optional<TypeSize>(TypeSize(0, false));
12757 if (!changeAllocationSize(NewAllocationSize))
12758 return ChangeStatus::UNCHANGED;
12759 return ChangeStatus::CHANGED;
12760 }
12761
12762 // TODO: refactor this to be part of multiple bin case
12763 const auto &It = PI->begin();
12764
12765 // TODO: handle if Offset is not zero
12766 if (It->first.Offset != 0)
12767 return indicatePessimisticFixpoint();
12768
12769 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
12770
12771 if (SizeOfBin >= *AllocationSize)
12772 return indicatePessimisticFixpoint();
12773
12774 auto NewAllocationSize =
12775 std::optional<TypeSize>(TypeSize(SizeOfBin * 8, false));
12776
12777 if (!changeAllocationSize(NewAllocationSize))
12778 return ChangeStatus::UNCHANGED;
12779
12780 return ChangeStatus::CHANGED;
12781 }
12782
12783 /// See AbstractAttribute::manifest(...).
12784 ChangeStatus manifest(Attributor &A) override {
12785
12786 assert(isValidState() &&
12787 "Manifest should only be called if the state is valid.");
12788
12789 Instruction *I = getIRPosition().getCtxI();
12790
12791 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
12792
12793 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
12794
12795 switch (I->getOpcode()) {
12796 // TODO: add case for malloc like calls
12797 case Instruction::Alloca: {
12798
12799 AllocaInst *AI = cast<AllocaInst>(I);
12800
12801 Type *CharType = Type::getInt8Ty(I->getContext());
12802
12803 auto *NumBytesToValue =
12804 ConstantInt::get(I->getContext(), APInt(32, NumBytesToAllocate));
12805
12806 BasicBlock::iterator insertPt = AI->getIterator();
12807 insertPt = std::next(insertPt);
12808 AllocaInst *NewAllocaInst =
12809 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
12810 AI->getAlign(), AI->getName(), insertPt);
12811
12812 if (A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst))
12813 return ChangeStatus::CHANGED;
12814
12815 break;
12816 }
12817 default:
12818 break;
12819 }
12820
12821 return ChangeStatus::UNCHANGED;
12822 }
12823
12824 /// See AbstractAttribute::getAsStr().
12825 const std::string getAsStr(Attributor *A) const override {
12826 if (!isValidState())
12827 return "allocationinfo(<invalid>)";
12828 return "allocationinfo(" +
12829 (AssumedAllocatedSize == HasNoAllocationSize
12830 ? "none"
12831 : std::to_string(AssumedAllocatedSize->getFixedValue())) +
12832 ")";
12833 }
12834
12835private:
12836 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
12837
12838 // Maintain the computed allocation size of the object.
12839 // Returns (bool) weather the size of the allocation was modified or not.
12840 bool changeAllocationSize(std::optional<TypeSize> Size) {
12841 if (AssumedAllocatedSize == HasNoAllocationSize ||
12842 AssumedAllocatedSize != Size) {
12843 AssumedAllocatedSize = Size;
12844 return true;
12845 }
12846 return false;
12847 }
12848};
12849
12850struct AAAllocationInfoFloating : AAAllocationInfoImpl {
12851 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
12852 : AAAllocationInfoImpl(IRP, A) {}
12853
12854 void trackStatistics() const override {
12855 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
12856 }
12857};
12858
12859struct AAAllocationInfoReturned : AAAllocationInfoImpl {
12860 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
12861 : AAAllocationInfoImpl(IRP, A) {}
12862
12863 /// See AbstractAttribute::initialize(...).
12864 void initialize(Attributor &A) override {
12865 // TODO: we don't rewrite function argument for now because it will need to
12866 // rewrite the function signature and all call sites
12867 (void)indicatePessimisticFixpoint();
12868 }
12869
12870 void trackStatistics() const override {
12871 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
12872 }
12873};
12874
12875struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
12876 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
12877 : AAAllocationInfoImpl(IRP, A) {}
12878
12879 void trackStatistics() const override {
12880 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
12881 }
12882};
12883
12884struct AAAllocationInfoArgument : AAAllocationInfoImpl {
12885 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
12886 : AAAllocationInfoImpl(IRP, A) {}
12887
12888 void trackStatistics() const override {
12889 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
12890 }
12891};
12892
12893struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
12894 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
12895 : AAAllocationInfoImpl(IRP, A) {}
12896
12897 /// See AbstractAttribute::initialize(...).
12898 void initialize(Attributor &A) override {
12899
12900 (void)indicatePessimisticFixpoint();
12901 }
12902
12903 void trackStatistics() const override {
12904 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
12905 }
12906};
12907} // namespace
12908
12909const char AANoUnwind::ID = 0;
12910const char AANoSync::ID = 0;
12911const char AANoFree::ID = 0;
12912const char AANonNull::ID = 0;
12913const char AAMustProgress::ID = 0;
12914const char AANoRecurse::ID = 0;
12915const char AANonConvergent::ID = 0;
12916const char AAWillReturn::ID = 0;
12917const char AAUndefinedBehavior::ID = 0;
12918const char AANoAlias::ID = 0;
12919const char AAIntraFnReachability::ID = 0;
12920const char AANoReturn::ID = 0;
12921const char AAIsDead::ID = 0;
12922const char AADereferenceable::ID = 0;
12923const char AAAlign::ID = 0;
12924const char AAInstanceInfo::ID = 0;
12925const char AANoCapture::ID = 0;
12926const char AAValueSimplify::ID = 0;
12927const char AAHeapToStack::ID = 0;
12928const char AAPrivatizablePtr::ID = 0;
12929const char AAMemoryBehavior::ID = 0;
12930const char AAMemoryLocation::ID = 0;
12931const char AAValueConstantRange::ID = 0;
12932const char AAPotentialConstantValues::ID = 0;
12933const char AAPotentialValues::ID = 0;
12934const char AANoUndef::ID = 0;
12935const char AANoFPClass::ID = 0;
12936const char AACallEdges::ID = 0;
12937const char AAInterFnReachability::ID = 0;
12938const char AAPointerInfo::ID = 0;
12939const char AAAssumptionInfo::ID = 0;
12940const char AAUnderlyingObjects::ID = 0;
12941const char AAAddressSpace::ID = 0;
12942const char AAAllocationInfo::ID = 0;
12943const char AAIndirectCallInfo::ID = 0;
12944const char AAGlobalValueInfo::ID = 0;
12945const char AADenormalFPMath::ID = 0;
12946
12947// Macro magic to create the static generator function for attributes that
12948// follow the naming scheme.
12949
12950#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
12951 case IRPosition::PK: \
12952 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
12953
12954#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
12955 case IRPosition::PK: \
12956 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
12957 ++NumAAs; \
12958 break;
12959
12960#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12961 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12962 CLASS *AA = nullptr; \
12963 switch (IRP.getPositionKind()) { \
12964 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12965 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
12966 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
12967 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
12968 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
12969 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
12970 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
12971 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
12972 } \
12973 return *AA; \
12974 }
12975
12976#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
12977 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12978 CLASS *AA = nullptr; \
12979 switch (IRP.getPositionKind()) { \
12980 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
12981 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
12982 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
12983 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
12984 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
12985 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
12986 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
12987 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
12988 } \
12989 return *AA; \
12990 }
12991
12992#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
12993 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
12994 CLASS *AA = nullptr; \
12995 switch (IRP.getPositionKind()) { \
12996 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
12997 default: \
12998 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
12999 " position!"); \
13000 } \
13001 return *AA; \
13002 }
13003
13004#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13005 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13006 CLASS *AA = nullptr; \
13007 switch (IRP.getPositionKind()) { \
13008 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13009 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13010 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13011 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13012 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13013 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13014 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13015 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13016 } \
13017 return *AA; \
13018 }
13019
13020#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13021 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13022 CLASS *AA = nullptr; \
13023 switch (IRP.getPositionKind()) { \
13024 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13025 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13026 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13027 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13028 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13029 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13030 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13031 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13032 } \
13033 return *AA; \
13034 }
13035
13036#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13037 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13038 CLASS *AA = nullptr; \
13039 switch (IRP.getPositionKind()) { \
13040 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13041 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13042 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13043 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13044 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13045 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13046 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13047 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13048 } \
13049 return *AA; \
13050 }
13051
13061
13077
13082
13087
13094
13096
13097#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13098#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13099#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13100#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13101#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13102#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13103#undef SWITCH_PK_CREATE
13104#undef SWITCH_PK_INV
#define Success
amdgpu AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
ReachingDefAnalysis InstSet & ToRemove
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
#define STATS_DECLTRACK(NAME, TYPE, MSG)
static std::optional< Constant * > askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA, const IRPosition &IRP, Type &Ty)
static cl::opt< unsigned, true > MaxPotentialValues("attributor-max-potential-values", cl::Hidden, cl::desc("Maximum number of potential values to be " "tracked for each position."), cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues), cl::init(7))
static const Value * getPointerOperand(const Instruction *I, bool AllowVolatile)
Get pointer operand of memory accessing instruction.
static void clampReturnedValueStates(Attributor &A, const AAType &QueryingAA, StateType &S, const IRPosition::CallBaseContext *CBContext=nullptr)
Clamp the information known for all returned values of a function (identified by QueryingAA) into S.
#define STATS_DECLTRACK_FN_ATTR(NAME)
#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static cl::opt< int > MaxPotentialValuesIterations("attributor-max-potential-values-iterations", cl::Hidden, cl::desc("Maximum number of iterations we keep dismantling potential values."), cl::init(64))
#define STATS_DECLTRACK_CS_ATTR(NAME)
#define PIPE_OPERATOR(CLASS)
#define DefineKeys(ToTy)
static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I, bool HeaderOnly, Cycle **CPtr=nullptr)
#define STATS_DECLTRACK_ARG_ATTR(NAME)
static const Value * stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Val, const DataLayout &DL, APInt &Offset, bool GetMinOffset, bool AllowNonInbounds, bool UseAssumed=false)
#define STATS_DECLTRACK_CSRET_ATTR(NAME)
static cl::opt< bool > ManifestInternal("attributor-manifest-internal", cl::Hidden, cl::desc("Manifest Attributor internal string attributes."), cl::init(false))
static Value * constructPointer(Value *Ptr, int64_t Offset, IRBuilder< NoFolder > &IRB)
Helper function to create a pointer based on Ptr, and advanced by Offset bytes.
#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define BUILD_STAT_NAME(NAME, TYPE)
static bool isDenselyPacked(Type *Ty, const DataLayout &DL)
Checks if a type could have padding bytes.
#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static const Value * getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Ptr, int64_t &BytesOffset, const DataLayout &DL, bool AllowNonInbounds=false)
#define STATS_DECLTRACK_FNRET_ATTR(NAME)
#define STATS_DECLTRACK_CSARG_ATTR(NAME)
#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS)
#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static cl::opt< int > MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128), cl::Hidden)
#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define STATS_DECLTRACK_FLOATING_ATTR(NAME)
#define STATS_DECL(NAME, TYPE, MSG)
basic Basic Alias true
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static bool isReachableImpl(SmallVectorImpl< BasicBlock * > &Worklist, const StopSetT &StopSet, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet, const DominatorTree *DT, const LoopInfo *LI)
Definition: CFG.cpp:134
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file declares an analysis pass that computes CycleInfo for LLVM IR, specialized from GenericCycl...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
Performs the initial survey of the specified function
Given that RA is a live value
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines DenseMapInfo traits for DenseMap.
T Content
uint64_t Size
Rewrite Partial Register Uses
#define Check(C,...)
Hexagon Common GEP
IRTranslator LLVM IR MI
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file implements a map that provides insertion order iteration.
#define T
#define T1
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
if(PassOpts->AAPipeline)
static StringRef getName(Value *V)
Basic Register Allocator
static cl::opt< RegAllocEvictionAdvisorAnalysis::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, "development", "for training")))
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool IsDead
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file defines generic set operations that may be used on set's of different types,...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:166
This file contains some functions that are useful when dealing with strings.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringLiteral > StandardNames)
Initialize the set of available library functions based on the specified target triple.
@ Floating
This pass exposes codegen information to IR-level passes.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Value * RHS
Value * LHS
static unsigned getSize(unsigned Kind)
AACallGraphNode * operator*() const
A manager for alias analyses.
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
A trivial helper function to check to see if the specified pointers are no-alias.
Class for arbitrary precision integers.
Definition: APInt.h:78
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1520
AbstractCallSite.
CallBase * getInstruction() const
Return the underlying instruction.
bool isCallbackCall() const
Return true if this ACS represents a callback call.
bool isDirectCall() const
Return true if this ACS represents a direct call.
static void getCallbackUses(const CallBase &CB, SmallVectorImpl< const Use * > &CallbackUses)
Add operand uses of CB that represent callback uses into CallbackUses.
int getCallArgOperandNo(Argument &Arg) const
Return the operand index of the underlying instruction associated with Arg.
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
Definition: Instructions.h:61
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:122
unsigned getAddressSpace() const
Return the address space for the allocation.
Definition: Instructions.h:102
std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
bool hasPointeeInMemoryValueAttr() const
Return true if this argument has the byval, sret, inalloca, preallocated, or byref attribute.
Definition: Function.cpp:201
bool hasReturnedAttr() const
Return true if this argument has the returned attribute.
Definition: Function.cpp:326
bool hasByValAttr() const
Return true if this argument has the byval attribute.
Definition: Function.cpp:163
const Function * getParent() const
Definition: Argument.h:43
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:49
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
iterator begin() const
Definition: ArrayRef.h:153
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:94
static Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:243
FPClassTest getNoFPClass() const
Return the FPClassTest for nofpclass.
Definition: Attributes.cpp:490
Attribute::AttrKind getKindAsEnum() const
Return the attribute's kind as an enum (Attribute::AttrKind).
Definition: Attributes.cpp:363
MemoryEffects getMemoryEffects() const
Returns memory effects.
Definition: Attributes.cpp:484
static Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:249
static Attribute getWithNoFPClass(LLVMContext &Context, FPClassTest Mask)
Definition: Attributes.cpp:285
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition: Attributes.h:86
static bool isEnumAttrKind(AttrKind Kind)
Definition: Attributes.h:99
static Attribute getWithMemoryEffects(LLVMContext &Context, MemoryEffects ME)
Definition: Attributes.cpp:280
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
Definition: Attributes.cpp:233
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:416
const Instruction & front() const
Definition: BasicBlock.h:471
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:212
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:219
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:177
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:239
BinaryOps getOpcode() const
Definition: InstrTypes.h:442
Conditional or Unconditional Branch instruction.
unsigned getNumSuccessors() const
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
Value * getCondition() const
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:66
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1385
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
Definition: InstrTypes.h:1476
Value * getCalledOperand() const
Definition: InstrTypes.h:1458
const Use & getCalledOperandUse() const
Definition: InstrTypes.h:1460
const Use & getArgOperandUse(unsigned i) const
Wrappers for getting the Use of a call argument.
Definition: InstrTypes.h:1421
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1391
bool isBundleOperand(unsigned Idx) const
Return true if the operand at index Idx is a bundle operand.
Definition: InstrTypes.h:2076
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2027
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1323
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1401
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
Definition: InstrTypes.h:1441
unsigned arg_size() const
Definition: InstrTypes.h:1408
bool isArgOperand(const Use *U) const
Definition: InstrTypes.h:1430
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:530
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:694
bool isIntegerCast() const
There are several places where we need to know if a cast instruction only deals with integer source a...
Type * getDestTy() const
Return the destination type, as a convenience.
Definition: InstrTypes.h:701
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:747
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition: InstrTypes.h:997
bool isFalseWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:1062
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:757
@ ICMP_EQ
equal
Definition: InstrTypes.h:778
@ ICMP_NE
not equal
Definition: InstrTypes.h:779
bool isTrueWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:1056
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:847
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:528
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1097
static Constant * getExtractElement(Constant *Vec, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
Definition: Constants.cpp:2528
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:850
This class represents a range of values.
Definition: ConstantRange.h:47
const APInt & getLower() const
Return the lower value for this range.
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
bool isEmptySet() const
Return true if this set contains no members.
APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
bool isSingleElement() const
Return true if this set contains exactly one member.
static ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the smallest range such that all values that may satisfy the given predicate with any value c...
const APInt & getUpper() const
Return the upper value for this range.
APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
This is an important base class in LLVM.
Definition: Constant.h:42
Analysis pass which computes a CycleInfo.
Definition: CycleAnalysis.h:46
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
unsigned size() const
Definition: DenseMap.h:99
iterator begin()
Definition: DenseMap.h:75
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition: DenseMap.h:146
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:211
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:279
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
An instruction for ordering other memory operations.
Definition: Instructions.h:420
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
const BasicBlock & getEntryBlock() const
Definition: Function.h:807
iterator_range< arg_iterator > args()
Definition: Function.h:890
const Function & getFunction() const
Definition: Function.h:170
size_t arg_size() const
Definition: Function.h:899
Argument * getArg(unsigned i) const
Definition: Function.h:884
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:743
CycleT * getCycle(const BlockT *Block) const
Find the innermost cycle containing a given block.
A possibly irreducible generalization of a Loop.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:290
bool hasLocalLinkage() const
Definition: GlobalValue.h:528
This instruction compares its operands according to the predicate given to the constructor.
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition: IRBuilder.h:1996
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:488
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2686
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
Definition: Instruction.h:754
bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
Definition: Instruction.cpp:97
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:92
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
bool isTerminator() const
Definition: Instruction.h:277
bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
const Instruction * getNextNonDebugInstruction(bool SkipPseudoOp=false) const
Return a pointer to the next non-debug instruction in the same basic block as 'this',...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:274
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:463
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition: Instruction.cpp:74
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:48
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
Analysis to compute lazy value information.
This pass computes, caches, and vends lazy value constraint information.
Definition: LazyValueInfo.h:32
ConstantRange getConstantRange(Value *V, Instruction *CxtI, bool UndefAllowed)
Return the ConstantRange constraint that is known to hold for the specified value at the specified in...
An instruction for reading from memory.
Definition: Instructions.h:174
Analysis pass that exposes the LoopInfo for a function.
Definition: LoopInfo.h:566
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Metadata node.
Definition: Metadata.h:1069
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1430
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1542
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1436
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
bool empty() const
Definition: MapVector.h:79
static MemoryEffectsBase readOnly()
Create MemoryEffectsBase that can read any memory.
Definition: ModRef.h:122
bool doesNotAccessMemory() const
Whether this function accesses no memory.
Definition: ModRef.h:192
static MemoryEffectsBase argMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Create MemoryEffectsBase that can only access argument memory.
Definition: ModRef.h:132
static MemoryEffectsBase inaccessibleMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Create MemoryEffectsBase that can only access inaccessible memory.
Definition: ModRef.h:138
bool onlyAccessesInaccessibleMem() const
Whether this function only (at most) accesses inaccessible memory.
Definition: ModRef.h:211
ModRefInfo getModRef(Location Loc) const
Get ModRefInfo for the given Location.
Definition: ModRef.h:165
bool onlyAccessesArgPointees() const
Whether this function only (at most) accesses argument memory.
Definition: ModRef.h:201
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
Definition: ModRef.h:195
static MemoryEffectsBase writeOnly()
Create MemoryEffectsBase that can write any memory.
Definition: ModRef.h:127
static MemoryEffectsBase inaccessibleOrArgMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
Create MemoryEffectsBase that can only access inaccessible or argument memory.
Definition: ModRef.h:145
static MemoryEffectsBase none()
Create MemoryEffectsBase that cannot read or write any memory.
Definition: ModRef.h:117
bool onlyAccessesInaccessibleOrArgMem() const
Whether this function only (at most) accesses argument and inaccessible memory.
Definition: ModRef.h:217
static MemoryEffectsBase unknown()
Create MemoryEffectsBase that can read and write any memory.
Definition: ModRef.h:112
static std::optional< MemoryLocation > getOrNone(const Instruction *Inst)
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Evaluate the size and offset of an object pointed to by a Value*.
static SizeOffsetValue unknown()
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1852
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents an analyzed expression in the program.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
const SCEV * getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
unsigned getSmallConstantMaxTripCount(const Loop *L)
Returns the upper bound of the loop trip count as a normal unsigned value.
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
bool erase(PtrType Ptr)
Remove pointer from the set.
Definition: SmallPtrSet.h:384
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:435
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:367
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:502
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:290
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:571
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:600
TypeSize getElementOffsetInBits(unsigned Idx) const
Definition: DataLayout.h:605
Class to represent struct types.
Definition: DerivedTypes.h:216
unsigned getNumElements() const
Random access to the elements.
Definition: DerivedTypes.h:341
Type * getElementType(unsigned N) const
Definition: DerivedTypes.h:342
Multiway switch.
Analysis pass providing the TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool areTypesABICompatible(const Function *Caller, const Function *Callee, const ArrayRef< Type * > &Types) const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:251
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:298
static IntegerType * getInt8Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition: Type.h:258
static IntegerType * getInt32Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:224
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
'undef' values are things that do not have specified contents.
Definition: Constants.h:1398
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1833
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:72
Value * get() const
Definition: Use.h:66
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:21
const Use & getOperandUse(unsigned i) const
Definition: User.h:182
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
bool isDroppable() const
A droppable user is a user for which uses can be dropped without affecting correctness and should be ...
Definition: User.cpp:115
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: ValueMap.h:164
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
static constexpr uint64_t MaximumAlignment
Definition: Value.h:807
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition: ilist_node.h:32
self_iterator getIterator()
Definition: ilist_node.h:132
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:353
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
Enumerate the SCCs of a directed graph in reverse topological order of the SCC DAG.
Definition: SCCIterator.h:49
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isAssumedReadNone(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readnone.
Definition: Attributor.cpp:654
bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readonly.
Definition: Attributor.cpp:649
raw_ostream & operator<<(raw_ostream &OS, const RangeTy &R)
Definition: Attributor.h:314
std::optional< Value * > combineOptionalValuesInAAValueLatice(const std::optional< Value * > &A, const std::optional< Value * > &B, Type *Ty)
Return the combination of A and B such that the result is a possible value of both.
Definition: Attributor.cpp:340
bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache)
Return true if the value of VAC is a valid at the position of VAC, that is a constant,...
Definition: Attributor.cpp:291
bool isAssumedThreadLocalObject(Attributor &A, Value &Obj, const AbstractAttribute &QueryingAA)
Return true if Obj is assumed to be a thread local object.
Definition: Attributor.cpp:836
bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA, const Value &V, bool ForAnalysisOnly=true)
Return true if V is dynamically unique, that is, there are no two "instances" of V at runtime with di...
Definition: Attributor.cpp:232
bool getPotentialCopiesOfStoredValue(Attributor &A, StoreInst &SI, SmallSetVector< Value *, 4 > &PotentialCopies, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values of the one stored by SI into PotentialCopies.
Definition: Attributor.cpp:600
bool isGPU(const Module &M)
Return true iff M target a GPU (and we can use GPU AS reasoning).
Definition: Attributor.cpp:201
ValueScope
Flags to distinguish intra-procedural queries from potentially inter-procedural queries.
Definition: Attributor.h:180
@ Intraprocedural
Definition: Attributor.h:181
@ Interprocedural
Definition: Attributor.h:182
bool isValidInScope(const Value &V, const Function *Scope)
Return true if V is a valid value in Scope, that is a constant or an instruction/argument of Scope.
Definition: Attributor.cpp:281
bool isPotentiallyReachable(Attributor &A, const Instruction &FromI, const Instruction &ToI, const AbstractAttribute &QueryingAA, const AA::InstExclusionSetTy *ExclusionSet=nullptr, std::function< bool(const Function &F)> GoBackwardsCB=nullptr)
Return true if ToI is potentially reachable from FromI without running into any instruction in Exclus...
Definition: Attributor.cpp:817
bool isNoSyncInst(Attributor &A, const Instruction &I, const AbstractAttribute &QueryingAA)
Return true if I is a nosync instruction.
Definition: Attributor.cpp:206
bool getPotentiallyLoadedValues(Attributor &A, LoadInst &LI, SmallSetVector< Value *, 4 > &PotentialValues, SmallSetVector< Instruction *, 4 > &PotentialValueOrigins, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values LI could read into PotentialValues.
Definition: Attributor.cpp:590
Value * getWithType(Value &V, Type &Ty)
Try to convert V to type Ty without introducing new instructions.
Definition: Attributor.cpp:317
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
AddressSpace getAddressSpace(T *V)
Definition: AVR.h:65
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Unsupported
This operation is completely unsupported on the target.
@ Undef
Value of the register doesn't matter.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1589
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition: LLVMContext.h:54
@ CE
Windows NT (Windows on ARM)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:463
unsigned combineHashValue(unsigned a, unsigned b)
Simplistic combination of 32-bit hash values into 32-bit hash values.
Definition: DenseMapInfo.h:39
ElementType
The element type of an SRV or UAV resource.
Definition: DXILABI.h:58
constexpr double e
Definition: MathExtras.h:47
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:227
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:236
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
pred_iterator pred_end(BasicBlock *BB)
Definition: CFG.h:114
@ Offset
Definition: DWP.cpp:480
@ Length
Definition: DWP.cpp:480
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:361
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)
Definition: DynamicAPInt.h:390
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1722
bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
Constant * getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty)
If this is a call to an allocation function that initializes memory to a fixed value,...
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1680
unsigned getPointerAddressSpace(const Type *T)
Definition: SPIRVUtils.h:126
auto successors(const MachineBasicBlock *BB)
bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1446
raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
bool operator!=(uint64_t V1, const APInt &V2)
Definition: APInt.h:2060
UseCaptureKind DetermineUseCaptureKind(const Use &U, llvm::function_ref< bool(Value *, const DataLayout &)> IsDereferenceableOrNull)
Determine what kind of capture behaviour U may exhibit.
Value * getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI)
Gets the alignment argument for an aligned_alloc-like function, using either built-in knowledge based...
Value * simplifyInstructionWithOperands(Instruction *I, ArrayRef< Value * > NewOps, const SimplifyQuery &Q)
Like simplifyInstruction but the operands of I are replaced with NewOps.
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
Definition: SCCIterator.h:233
PotentialValuesState< std::pair< AA::ValueAndContext, AA::ValueScope > > PotentialLLVMValuesState
Definition: Attributor.h:5215
bool isNoAliasCall(const Value *V)
Return true if this pointer is returned by a noalias function.
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
Definition: GraphWriter.h:359
auto unique(Range &&R, Predicate P)
Definition: STLExtras.h:2038
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
@ NONE
Definition: Attributor.h:6418
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
pred_iterator pred_begin(BasicBlock *BB)
Definition: CFG.h:110
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:340
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:291
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
Definition: ModRef.h:268
@ None
Definition: CodeGenData.h:101
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
Definition: Function.cpp:2132
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool isPointerTy(const Type *T)
Definition: SPIRVUtils.h:120
bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
Definition: Local.cpp:419
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool set_union(S1Ty &S1, const S2Ty &S2)
set_union(A, B) - Compute A := A u B, return whether A changed.
Definition: SetOperations.h:43
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM.
Definition: ValueMapper.h:263
CallBase & promoteCall(CallBase &CB, Function *Callee, CastInst **RetBitCast=nullptr)
Promote the given indirect call site to unconditionally call Callee.
bool hasAssumption(const Function &F, const KnownAssumptionString &AssumptionStr)
Return true if F has the assumption AssumptionStr attached.
Definition: Assumptions.cpp:70
RetainedKnowledge getKnowledgeFromUse(const Use *U, ArrayRef< Attribute::AttrKind > AttrKinds)
Return a valid Knowledge associated to the Use U if its Attribute kind is in AttrKinds.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Other
Any other memory.
ChangeStatus clampStateAndIndicateChange< DerefState >(DerefState &S, const DerefState &R)
Value * simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a CmpInst, fold the result or return null.
PotentialValuesState< APInt > PotentialConstantIntValuesState
Definition: Attributor.h:5213
DWARFExpression::Operation Op
bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
ChangeStatus clampStateAndIndicateChange(StateType &S, const StateType &R)
Helper function to clamp a state S of type StateType with the information in R and indicate/return if...
Definition: Attributor.h:3456
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
ChangeStatus
{
Definition: Attributor.h:484
std::optional< APInt > getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, function_ref< const Value *(const Value *)> Mapper=[](const Value *V) { return V;})
Return the size of the requested allocation.
DenseSet< StringRef > getAssumptions(const Function &F)
Return the set of all assumptions for the function F.
Definition: Assumptions.cpp:86
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
Definition: Alignment.h:111
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
@ OPTIONAL
The target may be valid if the source is not.
@ NONE
Do not track a dependence between source and target.
@ REQUIRED
The target cannot be valid if the source is not.
bool mayContainIrreducibleControl(const Function &F, const LoopInfo *LI)
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, unsigned Depth, const SimplifyQuery &SQ)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool isIdentifiedObject(const Value *V)
Return true if this pointer refers to a distinct and identifiable object.
constexpr StringRef AssumptionAttrKey
The key we use for assumption attributes.
Definition: Assumptions.h:28
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
A type to track pointer/struct usage and accesses for AAPointerInfo.
AAPointerInfo::const_bin_iterator end() const
ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I, std::optional< Value * > Content, AAPointerInfo::AccessKind Kind, Type *Ty, Instruction *RemoteI=nullptr)
Add a new Access to the state at offset Offset and with size Size.
DenseMap< const Instruction *, SmallVector< unsigned > > RemoteIMap
AAPointerInfo::const_bin_iterator begin() const
bool forallInterferingAccesses(Instruction &I, function_ref< bool(const AAPointerInfo::Access &, bool)> CB, AA::RangeTy &Range) const
See AAPointerInfo::forallInterferingAccesses.
State(State &&SIS)=default
const AAPointerInfo::Access & getAccess(unsigned Index) const
bool forallInterferingAccesses(AA::RangeTy Range, function_ref< bool(const AAPointerInfo::Access &, bool)> CB) const
See AAPointerInfo::forallInterferingAccesses.
SmallVector< AAPointerInfo::Access > AccessList
bool isAtFixpoint() const override
See AbstractState::isAtFixpoint().
static State getWorstState(const State &SIS)
Return the worst possible representable state.
AAPointerInfo::OffsetBinsTy OffsetBins
ChangeStatus indicateOptimisticFixpoint() override
See AbstractState::indicateOptimisticFixpoint().
State & operator=(const State &R)
ChangeStatus indicatePessimisticFixpoint() override
See AbstractState::indicatePessimisticFixpoint().
static State getBestState(const State &SIS)
Return the best possible representable state.
bool isValidState() const override
See AbstractState::isValidState().
----------------—AAIntraFnReachability Attribute-----------------------—
ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
unsigned Hash
Precomputed hash for this RQI.
ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To, const AA::InstExclusionSetTy *ES, bool MakeUnique)
Constructor replacement to ensure unique and stable sets are used for the cache.
An abstract interface for address space information.
Definition: Attributor.h:6229
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6268
An abstract interface for all align attributes.
Definition: Attributor.h:4264
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4295
Align getKnownAlign() const
Return known alignment.
Definition: Attributor.h:4278
static const char ID
Definition: Attributor.h:6303
An abstract attribute for getting assumption information.
Definition: Attributor.h:6155
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6183
An abstract state for querying live call edges.
Definition: Attributor.h:5479
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5522
An abstract Attribute for specializing "dynamic" components of "denormal-fp-math" and "denormal-fp-ma...
Definition: Attributor.h:6389
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6411
An abstract interface for all dereferenceable attribute.
Definition: Attributor.h:4210
uint32_t getKnownDereferenceableBytes() const
Return known dereferenceable bytes.
Definition: Attributor.h:4234
uint32_t getAssumedDereferenceableBytes() const
Return assumed dereferenceable bytes.
Definition: Attributor.h:4229
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4255
An abstract interface for llvm::GlobalValue information interference.
Definition: Attributor.h:6308
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6342
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4557
An abstract interface for indirect call information interference.
Definition: Attributor.h:6347
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6383
An abstract interface to track if a value leaves it's defining function instance.
Definition: Attributor.h:4302
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4335
An abstract Attribute for computing reachability between functions.
Definition: Attributor.h:5675
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5710
bool canReach(Attributor &A, const Function &Fn) const
If the function represented by this possition can reach Fn.
Definition: Attributor.h:5681
virtual bool instructionCanReach(Attributor &A, const Instruction &Inst, const Function &Fn, const AA::InstExclusionSetTy *ExclusionSet=nullptr) const =0
Can Inst reach Fn.
An abstract interface to determine reachability of point A to B.
Definition: Attributor.h:3808
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3836
virtual bool isAssumedReachable(Attributor &A, const Instruction &From, const Instruction &To, const AA::InstExclusionSetTy *ExclusionSet=nullptr) const =0
Returns true if 'From' instruction is assumed to reach, 'To' instruction.
An abstract interface for liveness abstract attribute.
Definition: Attributor.h:3968
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4058
An abstract interface for memory access kind related attributes (readnone/readonly/writeonly).
Definition: Attributor.h:4622
bool isAssumedReadOnly() const
Return true if we assume that the underlying value is not accessed (=written) in its respective scope...
Definition: Attributor.h:4661
bool isKnownReadNone() const
Return true if we know that the underlying value is not read or accessed in its respective scope.
Definition: Attributor.h:4649
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4688
bool isAssumedReadNone() const
Return true if we assume that the underlying value is not read or accessed in its respective scope.
Definition: Attributor.h:4653
An abstract interface for all memory location attributes (readnone/argmemonly/inaccessiblememonly/ina...
Definition: Attributor.h:4697
static std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK)
Return the locations encoded by MLK as a readable string.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4873
StateType::base_t MemoryLocationsKind
Definition: Attributor.h:4698
An abstract interface for all nonnull attributes.
Definition: Attributor.h:3584
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3619
An abstract interface for all noalias attributes.
Definition: Attributor.h:3843
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See IRAttribute::isImpliedByIR.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3882
An abstract interface for all nocapture attributes.
Definition: Attributor.h:4343
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4416
@ NO_CAPTURE_MAYBE_RETURNED
If we do not capture the value in memory or through integers we can only communicate it back as a der...
Definition: Attributor.h:4373
bool isAssumedNoCaptureMaybeReturned() const
Return true if we assume that the underlying value is not captured in its respective scope but we all...
Definition: Attributor.h:4397
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See IRAttribute::isImpliedByIR.
static void determineFunctionCaptureCapabilities(const IRPosition &IRP, const Function &F, BitIntegerState &State)
Update State according to the capture capabilities of F for position IRP.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5433
An AbstractAttribute for nofree.
Definition: Attributor.h:3889
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3932
An abstract attribute for norecurse.
Definition: Attributor.h:3677
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3701
An AbstractAttribute for noreturn.
Definition: Attributor.h:3939
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3963
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3577
static bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned)
Helper function to determine if CB is an aligned (GPU) barrier.
static bool isNonRelaxedAtomic(const Instruction *I)
Helper function used to determine whether an instruction is non-relaxed atomic.
static bool isNoSyncIntrinsic(const Instruction *I)
Helper function specific for intrinsics which are potentially volatile.
An abstract interface for all noundef attributes.
Definition: Attributor.h:5347
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5382
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See IRAttribute::isImpliedByIR.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3494
An abstract Attribute for determining the necessity of the convergent attribute.
Definition: Attributor.h:5715
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5743
bool isAssumedNotConvergent() const
Return true if "non-convergent" is assumed.
Definition: Attributor.h:5725
An abstract interface for all nonnull attributes.
Definition: Attributor.h:3626
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3670
static bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
See AbstractAttribute::isImpliedByIR(...).
An access description.
Definition: Attributor.h:5933
A container for a list of ranges.
Definition: Attributor.h:5782
static void set_difference(const RangeList &L, const RangeList &R, RangeList &D)
Copy ranges from L that are not in R, into D.
Definition: Attributor.h:5818
An abstract interface for struct information.
Definition: Attributor.h:5747
virtual const_bin_iterator begin() const =0
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6147
virtual int64_t numOffsetBins() const =0
An abstract interface for potential values analysis.
Definition: Attributor.h:5237
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5296
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:5333
static Value * getSingleValue(Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP, SmallVectorImpl< AA::ValueAndContext > &Values)
Extract the single value in Values if any.
An abstract interface for privatizability.
Definition: Attributor.h:4571
virtual std::optional< Type * > getPrivatizableType() const =0
Return the type we can choose for a private copy of the underlying value.
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4613
An abstract attribute for undefined behavior.
Definition: Attributor.h:3770
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3803
An abstract attribute for getting all assumption underlying objects.
Definition: Attributor.h:6187
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:6217
An abstract interface for range value analysis.
Definition: Attributor.h:4878
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4941
An abstract interface for value simplify abstract attribute.
Definition: Attributor.h:4495
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:4517
An abstract attribute for willreturn.
Definition: Attributor.h:3708
static const char ID
Unique ID (due to the unique address)
Definition: Attributor.h:3765
Helper to represent an access offset and size, with logic to deal with uncertainty and check for over...
Definition: Attributor.h:237
static constexpr int64_t Unknown
Definition: Attributor.h:311
static RangeTy getUnknown()
Definition: Attributor.h:243
Base struct for all "concrete attribute" deductions.
Definition: Attributor.h:3276
void print(raw_ostream &OS) const
Helper functions, for debug purposes only.
Definition: Attributor.h:3360
virtual StateType & getState()=0
Return the internal abstract state for inspection.
An interface to query the internal state of an abstract attribute.
Definition: Attributor.h:2596
virtual ChangeStatus indicatePessimisticFixpoint()=0
Indicate that the abstract state should converge to the pessimistic state.
virtual bool isAtFixpoint() const =0
Return if this abstract state is fixed, thus does not need to be updated if information changes as it...
virtual bool isValidState() const =0
Return if this abstract state is in a valid state.
virtual ChangeStatus indicateOptimisticFixpoint()=0
Indicate that the abstract state should converge to the optimistic state.
Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign but the instruction.
static unsigned getHashValue(const Access &A)
static Access getTombstoneKey()
static bool isEqual(const Access &LHS, const Access &RHS)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Helper struct used in the communication between an abstract attribute (AA) that wants to change the s...
Definition: Attributor.h:2204
std::function< void(const ArgumentReplacementInfo &, AbstractCallSite, SmallVectorImpl< Value * > &)> ACSRepairCBTy
Abstract call site (ACS) repair callback type.
Definition: Attributor.h:2227
const Argument & getReplacedArg() const
Definition: Attributor.h:2234
std::function< void(const ArgumentReplacementInfo &, Function &, Function::arg_iterator)> CalleeRepairCBTy
Callee repair callback type.
Definition: Attributor.h:2213
The fixpoint analysis framework that orchestrates the attribute deduction.
Definition: Attributor.h:1508
std::function< std::optional< Value * >(const IRPosition &, const AbstractAttribute *, bool &)> SimplifictionCallbackTy
Register CB as a simplification callback.
Definition: Attributor.h:2009
Specialization of the integer state for a bit-wise encoding.
Definition: Attributor.h:2737
BitIntegerState & removeAssumedBits(base_t BitsEncoding)
Remove the bits in BitsEncoding from the "assumed bits" if not known.
Definition: Attributor.h:2762
BitIntegerState & addKnownBits(base_t Bits)
Add the bits in BitsEncoding to the "known bits".
Definition: Attributor.h:2754
Simple wrapper for a single bit (boolean) state.
Definition: Attributor.h:2880
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getDefault()
Return the assumed default mode for a function without denormal-fp-math.
static constexpr DenormalMode getInvalid()
static unsigned getHashValue(const Access &A)
static bool isEqual(const Access &LHS, const Access &RHS)
static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B)
static unsigned getHashValue(const AA::RangeTy &Range)
static ReachabilityQueryInfo< ToTy > * getEmptyKey()
static ReachabilityQueryInfo< ToTy > * getTombstoneKey()
static bool isEqual(const ReachabilityQueryInfo< ToTy > *LHS, const ReachabilityQueryInfo< ToTy > *RHS)
static unsigned getHashValue(const ReachabilityQueryInfo< ToTy > *RQI)
An information struct used to provide DenseMap with the various necessary components for a given valu...
Definition: DenseMapInfo.h:52
State for dereferenceable attribute.
Definition: Attributor.h:4064
IncIntegerState DerefBytesState
State representing for dereferenceable bytes.
Definition: Attributor.h:4080
ChangeStatus manifest(Attributor &A) override
See AbstractAttribute::manifest(...).
Definition: Attributor.h:3213
Helper to describe and deal with positions in the LLVM-IR.
Definition: Attributor.h:581
Function * getAssociatedFunction() const
Return the associated function, if any.
Definition: Attributor.h:712
static const IRPosition callsite_returned(const CallBase &CB)
Create a position describing the returned value of CB.
Definition: Attributor.h:649
static const IRPosition returned(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the returned value of F.
Definition: Attributor.h:631
Argument * getAssociatedArgument() const
Return the associated argument, if any.
Definition: Attributor.cpp:996
static const IRPosition value(const Value &V, const CallBaseContext *CBContext=nullptr)
Create a position describing the value of V.
Definition: Attributor.h:605
int getCalleeArgNo() const
Return the callee argument number of the associated value if it is an argument or call site argument,...
Definition: Attributor.h:799
static const IRPosition inst(const Instruction &I, const CallBaseContext *CBContext=nullptr)
Create a position describing the instruction I.
Definition: Attributor.h:617
static const IRPosition callsite_argument(const CallBase &CB, unsigned ArgNo)
Create a position describing the argument of CB at position ArgNo.
Definition: Attributor.h:654
@ IRP_ARGUMENT
An attribute for a function argument.
Definition: Attributor.h:595
@ IRP_RETURNED
An attribute for the function return value.
Definition: Attributor.h:591
@ IRP_CALL_SITE
An attribute for a call site (function scope).
Definition: Attributor.h:594
@ IRP_CALL_SITE_RETURNED
An attribute for a call site return value.
Definition: Attributor.h:592
@ IRP_FUNCTION
An attribute for a function (scope).
Definition: Attributor.h:593
@ IRP_CALL_SITE_ARGUMENT
An attribute for a call site argument.
Definition: Attributor.h:596
@ IRP_INVALID
An invalid position.
Definition: Attributor.h:588
Instruction * getCtxI() const
Return the context instruction, if any.
Definition: Attributor.h:765
static const IRPosition argument(const Argument &Arg, const CallBaseContext *CBContext=nullptr)
Create a position describing the argument Arg.
Definition: Attributor.h:638
Type * getAssociatedType() const
Return the type this abstract attribute is associated with.
Definition: Attributor.h:788
static const IRPosition function(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the function scope of F.
Definition: Attributor.h:624
const CallBaseContext * getCallBaseContext() const
Get the call base context from the position.
Definition: Attributor.h:927
Value & getAssociatedValue() const
Return the value this abstract attribute is associated with.
Definition: Attributor.h:779
Value & getAnchorValue() const
Return the value this abstract attribute is anchored with.
Definition: Attributor.h:698
int getCallSiteArgNo() const
Return the call site argument number of the associated value if it is an argument or call site argume...
Definition: Attributor.h:808
static const IRPosition function_scope(const IRPosition &IRP, const CallBaseContext *CBContext=nullptr)
Create a position with function scope matching the "context" of IRP.
Definition: Attributor.h:677
Kind getPositionKind() const
Return the associated position kind.
Definition: Attributor.h:877
bool isArgumentPosition() const
Return true if the position is an argument or call site argument.
Definition: Attributor.h:909
static const IRPosition callsite_function(const CallBase &CB)
Create a position describing the function scope of CB.
Definition: Attributor.h:644
Function * getAnchorScope() const
Return the Function surrounding the anchor value.
Definition: Attributor.h:753
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
Data structure to hold cached (LLVM-IR) information.
Definition: Attributor.h:1198
bool isOnlyUsedByAssume(const Instruction &I) const
Definition: Attributor.h:1292
AP::Result * getAnalysisResultForFunction(const Function &F, bool CachedOnly=false)
Return the analysis result from a pass AP for function F.
Definition: Attributor.h:1302
State for an integer range.
Definition: Attributor.h:2922
ConstantRange getKnown() const
Return the known state encoding.
Definition: Attributor.h:2978
ConstantRange getAssumed() const
Return the assumed state encoding.
Definition: Attributor.h:2981
bool isValidState() const override
See AbstractState::isValidState() NOTE: For now we simply pretend that the worst possible state is in...
Definition: Attributor.h:2655
bool isAtFixpoint() const override
See AbstractState::isAtFixpoint()
Definition: Attributor.h:2658
ChangeStatus indicateOptimisticFixpoint() override
See AbstractState::indicateOptimisticFixpoint(...)
Definition: Attributor.h:2661
base_t getAssumed() const
Return the assumed state encoding.
Definition: Attributor.h:2676
static constexpr base_t getWorstState()
Return the worst possible representable state.
Definition: Attributor.h:2648
ChangeStatus indicatePessimisticFixpoint() override
See AbstractState::indicatePessimisticFixpoint(...)
Definition: Attributor.h:2667
Helper that allows to insert a new assumption string in the known assumption set by creating a (stati...
Definition: Assumptions.h:36
FPClassTest KnownFPClasses
Floating-point classes the value could be one of.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
A "must be executed context" for a given program point PP is the set of instructions,...
Definition: MustExecute.h:385
iterator & end()
Return an universal end iterator.
Definition: MustExecute.h:433
bool findInContextOf(const Instruction *I, const Instruction *PP)
Helper to look for I in the context of PP.
Definition: MustExecute.h:469
iterator & begin(const Instruction *PP)
Return an iterator to explore the context around PP.
Definition: MustExecute.h:419
bool checkForAllContext(const Instruction *PP, function_ref< bool(const Instruction *)> Pred)
}
Definition: MustExecute.h:455
Various options to control the behavior of getObjectSize.
A class for a set state.
Definition: Attributor.h:4952
static unsigned MaxPotentialValues
Maximum number of potential values to be tracked.
Definition: Attributor.h:5005
void unionAssumed(const MemberTy &C)
Union assumed set with the passed value.
Definition: Attributor.h:5022
static PotentialValuesState getBestState()
Return empty set as the best state of potential values.
Definition: Attributor.h:5008
const SetTy & getAssumedSet() const
Return this set.
Definition: Attributor.h:4982
Represent one information held inside an operand bundle of an llvm.assume.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
Helper to tie a abstract state implementation to an abstract attribute.
Definition: Attributor.h:3165
StateType & getState() override
See AbstractAttribute::getState(...).
Definition: Attributor.h:3173
bool unionAssumed(std::optional< Value * > Other)
Merge Other into the currently assumed simplified value.