138#define DEBUG_TYPE "infer-address-spaces"
144 std::numeric_limits<unsigned>::max();
155using PredicatedAddrSpaceMapTy =
160 unsigned FlatAddrSpace = 0;
169 InferAddressSpaces(
unsigned AS) : FunctionPass(ID), FlatAddrSpace(AS) {
173 void getAnalysisUsage(AnalysisUsage &AU)
const override {
182class InferAddressSpacesImpl {
185 const DominatorTree *DT =
nullptr;
186 const TargetTransformInfo *TTI =
nullptr;
187 const DataLayout *DL =
nullptr;
191 unsigned FlatAddrSpace = 0;
195 const bool AssumeDefaultIsFlatAddressSpace =
false;
197 DenseMap<const Value *, Value *> PtrIntCastPairs;
202 Value *getIntToPtrPointerOperand(
const Operator *I2P)
const;
207 void collectIntToPtrPointerOperand();
211 bool isSafeToCastIntToPtrAddrSpace(
const Operator *I2P)
const {
212 return PtrIntCastPairs.contains(I2P);
214 bool isAddressExpression(
const Value &
V,
const DataLayout &DL,
215 const TargetTransformInfo *TTI)
const;
216 Value *cloneConstantExprWithNewAddressSpace(
217 ConstantExpr *CE,
unsigned NewAddrSpace,
219 const TargetTransformInfo *TTI)
const;
221 SmallVector<Value *, 2>
222 getPointerOperands(
const Value &
V,
const DataLayout &DL,
223 const TargetTransformInfo *TTI)
const;
227 bool updateAddressSpace(
const Value &
V,
228 ValueToAddrSpaceMapTy &InferredAddrSpace,
229 PredicatedAddrSpaceMapTy &PredicatedAS)
const;
232 void enqueueUsers(
Value &
V,
const ValueToAddrSpaceMapTy &InferredAddrSpace,
233 SetVector<Value *> &Worklist)
const;
236 void runToFixPoint(SetVector<Value *> &Worklist,
237 ValueToAddrSpaceMapTy &InferredAddrSpace,
238 PredicatedAddrSpaceMapTy &PredicatedAS)
const;
243 ValueToAddrSpaceMapTy &InferredAddrSpace,
244 PredicatedAddrSpaceMapTy &PredicatedAS)
const;
246 bool isSafeToCastConstAddrSpace(Constant *
C,
unsigned NewAS)
const;
248 Value *clonePtrMaskWithNewAddressSpace(
249 IntrinsicInst *
I,
unsigned NewAddrSpace,
251 const PredicatedAddrSpaceMapTy &PredicatedAS,
252 SmallVectorImpl<const Use *> *PoisonUsesToFix)
const;
254 Value *cloneInstructionWithNewAddressSpace(
255 Instruction *
I,
unsigned NewAddrSpace,
257 const PredicatedAddrSpaceMapTy &PredicatedAS,
258 SmallVectorImpl<const Use *> *PoisonUsesToFix)
const;
260 void performPointerReplacement(
262 SmallVectorImpl<Instruction *> &DeadInstructions)
const;
267 bool rewriteWithNewAddressSpaces(
269 const ValueToAddrSpaceMapTy &InferredAddrSpace,
270 const PredicatedAddrSpaceMapTy &PredicatedAS)
const;
272 void appendsFlatAddressExpressionToPostorderStack(
273 Value *
V, PostorderStackTy &PostorderStack,
274 DenseSet<Value *> &Visited)
const;
276 bool rewriteIntrinsicOperands(IntrinsicInst *
II,
Value *OldV,
278 void collectRewritableIntrinsicOperands(IntrinsicInst *
II,
279 PostorderStackTy &PostorderStack,
280 DenseSet<Value *> &Visited)
const;
282 std::vector<WeakTrackingVH> collectFlatAddressExpressions(
Function &F)
const;
284 Value *cloneValueWithNewAddressSpace(
285 Value *
V,
unsigned NewAddrSpace,
287 const PredicatedAddrSpaceMapTy &PredicatedAS,
288 SmallVectorImpl<const Use *> *PoisonUsesToFix)
const;
289 unsigned joinAddressSpaces(
unsigned AS1,
unsigned AS2)
const;
291 unsigned getPredicatedAddrSpace(
const Value &PtrV,
292 const Value *UserCtx)
const;
295 InferAddressSpacesImpl(AssumptionCache &AC,
const DominatorTree *DT,
296 const TargetTransformInfo *TTI,
unsigned FlatAddrSpace,
297 bool AssumeDefaultIsFlatAddressSpace)
298 : AC(AC), DT(DT), TTI(TTI), FlatAddrSpace(FlatAddrSpace),
299 AssumeDefaultIsFlatAddressSpace(AssumeDefaultIsFlatAddressSpace) {}
305char InferAddressSpaces::ID = 0;
315 assert(Ty->isPtrOrPtrVectorTy());
317 return Ty->getWithNewType(NPT);
327 if (!P2I || P2I->getOpcode() != Instruction::PtrToInt)
343 unsigned P2IOp0AS = P2I->getOperand(0)->getType()->getPointerAddressSpace();
349 P2I->getOperand(0)->getType(), P2I->getType(),
351 (P2IOp0AS == I2PAS ||
TTI->isNoopAddrSpaceCast(P2IOp0AS, I2PAS));
358bool InferAddressSpacesImpl::isAddressExpression(
363 return Arg->getType()->isPointerTy() &&
370 switch (
Op->getOpcode()) {
371 case Instruction::PHI:
372 assert(
Op->getType()->isPtrOrPtrVectorTy());
374 case Instruction::BitCast:
375 case Instruction::AddrSpaceCast:
376 case Instruction::GetElementPtr:
378 case Instruction::Select:
379 return Op->getType()->isPtrOrPtrVectorTy();
380 case Instruction::Call: {
382 return II &&
II->getIntrinsicID() == Intrinsic::ptrmask;
384 case Instruction::IntToPtr:
386 isSafeToCastIntToPtrAddrSpace(
Op);
396SmallVector<Value *, 2> InferAddressSpacesImpl::getPointerOperands(
397 const Value &
V,
const DataLayout &
DL,
398 const TargetTransformInfo *
TTI)
const {
403 switch (
Op.getOpcode()) {
404 case Instruction::PHI: {
406 return {IncomingValues.begin(), IncomingValues.end()};
408 case Instruction::BitCast:
409 case Instruction::AddrSpaceCast:
410 case Instruction::GetElementPtr:
411 return {
Op.getOperand(0)};
412 case Instruction::Select:
413 return {
Op.getOperand(1),
Op.getOperand(2)};
414 case Instruction::Call: {
416 assert(
II.getIntrinsicID() == Intrinsic::ptrmask &&
417 "unexpected intrinsic call");
418 return {
II.getArgOperand(0)};
420 case Instruction::IntToPtr: {
423 return {P2I->getOperand(0)};
425 assert(isSafeToCastIntToPtrAddrSpace(&
Op));
426 return {getIntToPtrPointerOperand(&
Op)};
440 switch (
Op->getOpcode()) {
441 case Instruction::Xor:
442 case Instruction::Or:
444 case Instruction::And:
452InferAddressSpacesImpl::getIntToPtrPointerOperand(
const Operator *I2P)
const {
459 if (
auto *OldPtr = PtrIntCastPairs.
lookup(I2P))
472 Value *
LHS = LogicalOp->getOperand(0), *
RHS = LogicalOp->getOperand(1);
488 if (PreservedPtrMask.
isZero())
490 APInt ChangedPtrBits =
499 if (ChangedPtrBits.
isSubsetOf(PreservedPtrMask))
505void InferAddressSpacesImpl::collectIntToPtrPointerOperand() {
506 auto Collect = [&](
const Operator *I2P) {
507 if (
auto *OldPtr = getIntToPtrPointerOperand(I2P))
508 PtrIntCastPairs.
insert({I2P, OldPtr});
519 DenseSet<ConstantExpr *> VisitedCE;
521 auto PushConstantExpr = [&](
Value *
V) {
523 if (VisitedCE.
insert(CE).second)
531 PushConstantExpr(
Op);
534 while (!Worklist.
empty()) {
536 if (
CE->getOpcode() == Instruction::IntToPtr)
539 PushConstantExpr(
Op);
543bool InferAddressSpacesImpl::rewriteIntrinsicOperands(IntrinsicInst *
II,
546 Module *
M =
II->getParent()->getParent()->getParent();
549 case Intrinsic::objectsize:
550 case Intrinsic::masked_load: {
551 Type *DestTy =
II->getType();
555 II->setArgOperand(0, NewV);
556 II->setCalledFunction(NewDecl);
559 case Intrinsic::ptrmask:
562 case Intrinsic::masked_gather: {
563 Type *RetTy =
II->getType();
567 II->setArgOperand(0, NewV);
568 II->setCalledFunction(NewDecl);
571 case Intrinsic::masked_store:
572 case Intrinsic::masked_scatter: {
573 Type *ValueTy =
II->getOperand(0)->getType();
576 M,
II->getIntrinsicID(), {ValueTy, NewPtrTy});
577 II->setArgOperand(1, NewV);
578 II->setCalledFunction(NewDecl);
581 case Intrinsic::prefetch:
582 case Intrinsic::is_constant: {
584 M,
II->getIntrinsicID(), {NewV->getType()});
585 II->setArgOperand(0, NewV);
586 II->setCalledFunction(NewDecl);
589 case Intrinsic::fake_use: {
590 II->replaceUsesOfWith(OldV, NewV);
593 case Intrinsic::lifetime_start:
594 case Intrinsic::lifetime_end: {
598 M,
II->getIntrinsicID(), {NewV->getType()});
599 II->setArgOperand(0, NewV);
600 II->setCalledFunction(NewDecl);
608 II->replaceAllUsesWith(Rewrite);
609 II->eraseFromParent();
616void InferAddressSpacesImpl::collectRewritableIntrinsicOperands(
617 IntrinsicInst *
II, PostorderStackTy &PostorderStack,
618 DenseSet<Value *> &Visited)
const {
619 auto IID =
II->getIntrinsicID();
621 case Intrinsic::ptrmask:
622 case Intrinsic::objectsize:
623 appendsFlatAddressExpressionToPostorderStack(
II->getArgOperand(0),
624 PostorderStack, Visited);
626 case Intrinsic::is_constant: {
627 Value *Ptr =
II->getArgOperand(0);
629 appendsFlatAddressExpressionToPostorderStack(Ptr, PostorderStack,
635 case Intrinsic::masked_load:
636 case Intrinsic::masked_gather:
637 case Intrinsic::prefetch:
638 appendsFlatAddressExpressionToPostorderStack(
II->getArgOperand(0),
639 PostorderStack, Visited);
641 case Intrinsic::masked_store:
642 case Intrinsic::masked_scatter:
643 appendsFlatAddressExpressionToPostorderStack(
II->getArgOperand(1),
644 PostorderStack, Visited);
646 case Intrinsic::fake_use: {
648 if (
Op->getType()->isPtrOrPtrVectorTy()) {
649 appendsFlatAddressExpressionToPostorderStack(
Op, PostorderStack,
656 case Intrinsic::lifetime_start:
657 case Intrinsic::lifetime_end: {
658 appendsFlatAddressExpressionToPostorderStack(
II->getArgOperand(0),
659 PostorderStack, Visited);
663 SmallVector<int, 2> OpIndexes;
665 for (
int Idx : OpIndexes) {
666 appendsFlatAddressExpressionToPostorderStack(
II->getArgOperand(Idx),
667 PostorderStack, Visited);
677void InferAddressSpacesImpl::appendsFlatAddressExpressionToPostorderStack(
678 Value *
V, PostorderStackTy &PostorderStack,
679 DenseSet<Value *> &Visited)
const {
680 assert(
V->getType()->isPtrOrPtrVectorTy());
686 if (isAddressExpression(*CE, *
DL,
TTI) && Visited.
insert(CE).second)
687 PostorderStack.emplace_back(CE,
false);
692 if (
V->getType()->getPointerAddressSpace() == FlatAddrSpace &&
693 isAddressExpression(*
V, *
DL,
TTI)) {
694 if (Visited.
insert(
V).second) {
695 PostorderStack.emplace_back(
V,
false);
698 for (
auto &O :
Op->operands())
700 if (isAddressExpression(*CE, *
DL,
TTI) && Visited.
insert(CE).second)
701 PostorderStack.emplace_back(CE,
false);
708std::vector<WeakTrackingVH>
709InferAddressSpacesImpl::collectFlatAddressExpressions(
Function &
F)
const {
712 PostorderStackTy PostorderStack;
714 DenseSet<Value *> Visited;
716 auto PushPtrOperand = [&](
Value *Ptr) {
717 appendsFlatAddressExpressionToPostorderStack(Ptr, PostorderStack, Visited);
722 DenseSet<ConstantExpr *> VisitedCE;
724 auto PushConstantExpr = [&](
Value *
V) {
734 PushPtrOperand(
GEP->getPointerOperand());
736 PushPtrOperand(LI->getPointerOperand());
738 PushPtrOperand(
SI->getPointerOperand());
740 PushPtrOperand(RMW->getPointerOperand());
742 PushPtrOperand(CmpX->getPointerOperand());
745 PushPtrOperand(
MI->getRawDest());
749 PushPtrOperand(MTI->getRawSource());
751 collectRewritableIntrinsicOperands(
II, PostorderStack, Visited);
753 if (
Cmp->getOperand(0)->getType()->isPtrOrPtrVectorTy()) {
754 PushPtrOperand(
Cmp->getOperand(0));
755 PushPtrOperand(
Cmp->getOperand(1));
758 PushPtrOperand(ASC->getPointerOperand());
765 if (
auto *RV = RI->getReturnValue();
766 RV && RV->getType()->isPtrOrPtrVectorTy())
775 PushConstantExpr(
Op);
776 while (!CEWorklist.
empty()) {
779 ASC && ASC->getSrcAddressSpace() == FlatAddrSpace)
780 PushPtrOperand(ASC->getPointerOperand());
782 PushConstantExpr(
Op);
786 std::vector<WeakTrackingVH> Postorder;
787 while (!PostorderStack.empty()) {
788 Value *TopVal = PostorderStack.back().getPointer();
791 if (PostorderStack.back().getInt()) {
793 Postorder.push_back(TopVal);
794 PostorderStack.pop_back();
798 PostorderStack.back().setInt(
true);
801 for (
Value *PtrOperand : getPointerOperands(*TopVal, *
DL,
TTI)) {
802 appendsFlatAddressExpressionToPostorderStack(PtrOperand, PostorderStack,
814 auto InsertBefore = [NewI](
auto It) {
824 auto InsertI =
F->getEntryBlock().getFirstNonPHIIt();
825 return InsertBefore(InsertI);
835 auto InsertI = OpInst->
getParent()->getFirstNonPHIIt();
836 return InsertBefore(InsertI);
849 const Use &OperandUse,
unsigned NewAddrSpace,
851 const PredicatedAddrSpaceMapTy &PredicatedAS,
861 if (
Value *NewOperand = ValueWithNewAddrSpace.
lookup(Operand)) {
862 Operand = NewOperand;
863 }
else if (!PredicatedAS.contains(std::make_pair(Inst, Operand))) {
864 assert(PoisonUsesToFix &&
"missing inferred operand replacement");
869 if (Operand->
getType() == NewPtrTy)
881 if (
Value *NewUser = ValueWithNewAddrSpace.
lookup(Inst))
890Value *InferAddressSpacesImpl::clonePtrMaskWithNewAddressSpace(
891 IntrinsicInst *
I,
unsigned NewAddrSpace,
893 const PredicatedAddrSpaceMapTy &PredicatedAS,
894 SmallVectorImpl<const Use *> *PoisonUsesToFix)
const {
895 const Use &PtrOpUse =
I->getArgOperandUse(0);
897 Value *MaskOp =
I->getArgOperand(1);
900 KnownBits OldPtrBits{
DL->getPointerSizeInBits(OldAddrSpace)};
901 KnownBits NewPtrBits{
DL->getPointerSizeInBits(NewAddrSpace)};
903 std::tie(OldPtrBits, NewPtrBits) =
918 OldPtrBits.
One |= ~OldPtrBits.Zero;
920 KnownBits ClearedBits =
KnownBits::sub(OldPtrBits, OldPtrBits & MaskBits);
925 std::optional<BasicBlock::iterator> InsertPoint =
926 I->getInsertionPointAfterDef();
927 assert(InsertPoint &&
"insertion after ptrmask should be possible");
930 new AddrSpaceCastInst(
I, NewPtrType,
"", *InsertPoint);
932 return AddrSpaceCast;
939 MaskOp =
B.CreateTrunc(MaskOp, MaskTy);
942 PtrOpUse, NewAddrSpace, ValueWithNewAddrSpace, PredicatedAS,
944 return B.CreateIntrinsic(Intrinsic::ptrmask, {NewPtr->
getType(), MaskTy},
957Value *InferAddressSpacesImpl::cloneInstructionWithNewAddressSpace(
958 Instruction *
I,
unsigned NewAddrSpace,
960 const PredicatedAddrSpaceMapTy &PredicatedAS,
961 SmallVectorImpl<const Use *> *PoisonUsesToFix)
const {
964 if (
I->getOpcode() == Instruction::AddrSpaceCast) {
965 Value *Src =
I->getOperand(0);
969 assert(Src->getType()->getPointerAddressSpace() == NewAddrSpace);
976 assert(
II->getIntrinsicID() == Intrinsic::ptrmask);
977 return clonePtrMaskWithNewAddressSpace(
978 II, NewAddrSpace, ValueWithNewAddrSpace, PredicatedAS, PoisonUsesToFix);
986 auto *NewI =
new AddrSpaceCastInst(
I, NewPtrTy);
987 NewI->insertAfter(
I->getIterator());
988 NewI->setDebugLoc(
I->getDebugLoc());
993 SmallVector<Value *, 4> NewPointerOperands;
994 for (
const Use &OperandUse :
I->operands()) {
995 if (!OperandUse.get()->getType()->isPtrOrPtrVectorTy())
999 OperandUse, NewAddrSpace, ValueWithNewAddrSpace, PredicatedAS,
1003 switch (
I->getOpcode()) {
1004 case Instruction::BitCast:
1005 return new BitCastInst(NewPointerOperands[0], NewPtrType);
1006 case Instruction::PHI: {
1007 assert(
I->getType()->isPtrOrPtrVectorTy());
1010 for (
unsigned Index = 0;
Index <
PHI->getNumIncomingValues(); ++
Index) {
1012 NewPHI->
addIncoming(NewPointerOperands[OperandNo],
1013 PHI->getIncomingBlock(Index));
1017 case Instruction::GetElementPtr: {
1020 GEP->getSourceElementType(), NewPointerOperands[0],
1021 SmallVector<Value *, 4>(
GEP->indices()));
1025 case Instruction::Select:
1026 assert(
I->getType()->isPtrOrPtrVectorTy());
1028 NewPointerOperands[2],
"",
nullptr,
I);
1029 case Instruction::IntToPtr: {
1032 if (Src->getType() == NewPtrType)
1038 return new AddrSpaceCastInst(Src, NewPtrType);
1041 AddrSpaceCastInst *AsCast =
new AddrSpaceCastInst(
I, NewPtrType);
1053Value *InferAddressSpacesImpl::cloneConstantExprWithNewAddressSpace(
1054 ConstantExpr *CE,
unsigned NewAddrSpace,
1056 const TargetTransformInfo *
TTI)
const {
1058 CE->getType()->isPtrOrPtrVectorTy()
1062 switch (
CE->getOpcode()) {
1063 case Instruction::AddrSpaceCast:
1067 assert(CE->getOperand(0)->getType()->getPointerAddressSpace() ==
1069 return CE->getOperand(0);
1071 case Instruction::BitCast:
1072 if (Value *NewOperand = ValueWithNewAddrSpace.lookup(CE->getOperand(0)))
1073 return ConstantExpr::getBitCast(cast<Constant>(NewOperand), TargetType);
1074 return ConstantExpr::getAddrSpaceCast(CE, TargetType);
1076 case Instruction::IntToPtr:
1077 if (isNoopPtrIntCastPair(cast<Operator>(CE), *DL, TTI)) {
1078 Constant *Src = cast<ConstantExpr>(CE->getOperand(0))->getOperand(0);
1082 if (Src->getType() == TargetType)
1084 return ConstantExpr::getAddrSpaceCast(Src, TargetType);
1089 case Instruction::GetElementPtr: {
1097 NewBase = cloneConstantExprWithNewAddressSpace(
1100 assert(NewBase &&
"pointer operand of a rewritten GEP was not rewritten");
1102 SmallVector<Constant *, 4> NewOperands;
1104 for (
unsigned Index = 1;
Index <
CE->getNumOperands(); ++
Index)
1108 return CE->getWithOperands(NewOperands, TargetType,
false,
1122Value *InferAddressSpacesImpl::cloneValueWithNewAddressSpace(
1123 Value *
V,
unsigned NewAddrSpace,
1125 const PredicatedAddrSpaceMapTy &PredicatedAS,
1126 SmallVectorImpl<const Use *> *PoisonUsesToFix)
const {
1128 assert(
V->getType()->getPointerAddressSpace() == FlatAddrSpace &&
1129 isAddressExpression(*
V, *
DL,
TTI));
1137 Type *NewPtrTy = PointerType::get(Arg->getContext(), NewAddrSpace);
1138 auto *NewI =
new AddrSpaceCastInst(Arg, NewPtrTy);
1139 NewI->insertBefore(Insert);
1144 Value *NewV = cloneInstructionWithNewAddressSpace(
1145 I, NewAddrSpace, ValueWithNewAddrSpace, PredicatedAS, PoisonUsesToFix);
1147 if (NewI->getParent() ==
nullptr) {
1148 NewI->insertBefore(
I->getIterator());
1150 NewI->setDebugLoc(
I->getDebugLoc());
1156 return cloneConstantExprWithNewAddressSpace(
1162unsigned InferAddressSpacesImpl::joinAddressSpaces(
unsigned AS1,
1163 unsigned AS2)
const {
1167 if (AS1 == FlatAddrSpace || AS2 == FlatAddrSpace)
1178bool InferAddressSpacesImpl::run(
Function &CurFn) {
1180 DL = &
F->getDataLayout();
1181 PtrIntCastPairs.
clear();
1183 if (AssumeDefaultIsFlatAddressSpace)
1192 collectIntToPtrPointerOperand();
1194 std::vector<WeakTrackingVH> Postorder = collectFlatAddressExpressions(*
F);
1198 ValueToAddrSpaceMapTy InferredAddrSpace;
1199 PredicatedAddrSpaceMapTy PredicatedAS;
1200 inferAddressSpaces(Postorder, InferredAddrSpace, PredicatedAS);
1204 return rewriteWithNewAddressSpaces(Postorder, InferredAddrSpace,
1208void InferAddressSpacesImpl::enqueueUsers(
1209 Value &
V,
const ValueToAddrSpaceMapTy &InferredAddrSpace,
1210 SetVector<Value *> &Worklist)
const {
1211 for (
Value *User :
V.users()) {
1213 if (Worklist.
count(User))
1216 ValueToAddrSpaceMapTy::const_iterator Pos = InferredAddrSpace.find(User);
1219 if (Pos == InferredAddrSpace.end())
1225 if (Pos->second == FlatAddrSpace)
1236 for (
auto &[I2P, OldPtr] : PtrIntCastPairs) {
1239 auto Pos = InferredAddrSpace.find(I2P);
1240 if (Pos == InferredAddrSpace.end() || Pos->second == FlatAddrSpace)
1246void InferAddressSpacesImpl::runToFixPoint(
1247 SetVector<Value *> &Worklist, ValueToAddrSpaceMapTy &InferredAddrSpace,
1248 PredicatedAddrSpaceMapTy &PredicatedAS)
const {
1249 while (!Worklist.
empty()) {
1254 if (!updateAddressSpace(*
V, InferredAddrSpace, PredicatedAS))
1257 enqueueUsers(*
V, InferredAddrSpace, Worklist);
1263void InferAddressSpacesImpl::inferAddressSpaces(
1265 ValueToAddrSpaceMapTy &InferredAddrSpace,
1266 PredicatedAddrSpaceMapTy &PredicatedAS)
const {
1269 for (
Value *
V : Postorder)
1272 runToFixPoint(Worklist, InferredAddrSpace, PredicatedAS);
1278 SmallVector<Value *, 4> Lowered;
1279 for (
Value *
V : Postorder) {
1280 ValueToAddrSpaceMapTy::iterator
I = InferredAddrSpace.find(
V);
1288 enqueueUsers(*
V, InferredAddrSpace, Worklist);
1290 runToFixPoint(Worklist, InferredAddrSpace, PredicatedAS);
1294InferAddressSpacesImpl::getPredicatedAddrSpace(
const Value &Ptr,
1295 const Value *UserCtx)
const {
1318bool InferAddressSpacesImpl::updateAddressSpace(
1319 const Value &
V, ValueToAddrSpaceMapTy &InferredAddrSpace,
1320 PredicatedAddrSpaceMapTy &PredicatedAS)
const {
1321 assert(InferredAddrSpace.count(&
V));
1323 LLVM_DEBUG(
dbgs() <<
"Updating the address space of\n " <<
V <<
'\n');
1339 SmallVector<Value *, 2> PtrOps = getPointerOperands(
V, *
DL,
TTI);
1340 for (
Value *PtrOperand : PtrOps) {
1341 auto I = InferredAddrSpace.find(PtrOperand);
1343 if (
I == InferredAddrSpace.end()) {
1344 OperandAS = PtrOperand->getType()->getPointerAddressSpace();
1351 if (OperandAS == FlatAddrSpace) {
1353 unsigned AS = getPredicatedAddrSpace(*PtrOperand, &
V);
1356 <<
" deduce operand AS from the predicate addrspace "
1360 PredicatedAS[std::make_pair(&
V, PtrOperand)] = OperandAS;
1364 OperandAS =
I->second;
1367 NewAS = joinAddressSpaces(NewAS, OperandAS);
1368 if (NewAS == FlatAddrSpace)
1373 if (
any_of(ConstantPtrOps, [=](Constant *
C) {
1374 return !isSafeToCastConstAddrSpace(
C, NewAS);
1376 NewAS = FlatAddrSpace;
1381 PtrOps.size() == ConstantPtrOps.
size())
1385 unsigned OldAS = InferredAddrSpace.lookup(&
V);
1386 assert(OldAS != FlatAddrSpace);
1393 InferredAddrSpace[&
V] = NewAS;
1402 if (U.get() == OldVal) {
1410template <
typename InstrType>
1412 InstrType *MemInstr,
Value *OldV,
1415 if (!MemInstr->isVolatile() ||
TTI.hasVolatileVariant(MemInstr, AddrSpace)) {
1454 B.CreateMemSet(NewV, MSI->getValue(), MSI->getLength(), MSI->getDestAlign(),
1456 MI->getAAMetadata());
1458 Value *Src = MTI->getRawSource();
1459 Value *Dest = MTI->getRawDest();
1469 if (MCI->isForceInlined())
1470 B.CreateMemCpyInline(Dest, MTI->getDestAlign(), Src,
1471 MTI->getSourceAlign(), MTI->getLength(),
1473 MI->getAAMetadata());
1475 B.CreateMemCpy(Dest, MTI->getDestAlign(), Src, MTI->getSourceAlign(),
1478 MI->getAAMetadata());
1481 B.CreateMemMove(Dest, MTI->getDestAlign(), Src, MTI->getSourceAlign(),
1484 MI->getAAMetadata());
1489 MI->eraseFromParent();
1495bool InferAddressSpacesImpl::isSafeToCastConstAddrSpace(Constant *
C,
1496 unsigned NewAS)
const {
1499 unsigned SrcAS =
C->getType()->getPointerAddressSpace();
1504 if (SrcAS != FlatAddrSpace && NewAS != FlatAddrSpace)
1513 if (
Op->getOpcode() == Instruction::AddrSpaceCast)
1517 if (
Op->getOpcode() == Instruction::IntToPtr &&
1518 Op->getType()->getPointerAddressSpace() == FlatAddrSpace)
1527 User *CurUser =
I->getUser();
1530 while (
I !=
End &&
I->getUser() == CurUser)
1536void InferAddressSpacesImpl::performPointerReplacement(
1538 SmallVectorImpl<Instruction *> &DeadInstructions)
const {
1540 User *CurUser =
U.getUser();
1543 if (CurUser == NewV)
1547 if (!CurUserI || CurUserI->getFunction() !=
F)
1560 if (rewriteIntrinsicOperands(
II,
V, NewV))
1572 int SrcIdx =
U.getOperandNo();
1573 int OtherIdx = (SrcIdx == 0) ? 1 : 0;
1574 Value *OtherSrc =
Cmp->getOperand(OtherIdx);
1576 if (
Value *OtherNewV = ValueWithNewAddrSpace.
lookup(OtherSrc)) {
1577 if (OtherNewV->getType()->getPointerAddressSpace() == NewAS) {
1578 Cmp->setOperand(OtherIdx, OtherNewV);
1579 Cmp->setOperand(SrcIdx, NewV);
1586 if (isSafeToCastConstAddrSpace(KOtherSrc, NewAS)) {
1587 Cmp->setOperand(SrcIdx, NewV);
1597 if (ASC->getDestAddressSpace() == NewAS) {
1598 ASC->replaceAllUsesWith(NewV);
1613 InsertPos = std::next(NewVInst->getIterator());
1621 V,
new AddrSpaceCastInst(NewV,
V->getType(),
"", InsertPos));
1623 CurUserI->replaceUsesOfWith(
1628bool InferAddressSpacesImpl::rewriteWithNewAddressSpaces(
1630 const ValueToAddrSpaceMapTy &InferredAddrSpace,
1631 const PredicatedAddrSpaceMapTy &PredicatedAS)
const {
1638 for (
Value *
V : Postorder) {
1639 unsigned NewAddrSpace = InferredAddrSpace.lookup(
V);
1646 if (
V->getType()->getPointerAddressSpace() != NewAddrSpace) {
1648 cloneValueWithNewAddressSpace(
V, NewAddrSpace, ValueWithNewAddrSpace,
1649 PredicatedAS, &PoisonUsesToFix);
1651 ValueWithNewAddrSpace[
V] =
New;
1655 if (ValueWithNewAddrSpace.
empty())
1659 for (
const Use *PoisonUse : PoisonUsesToFix) {
1660 User *
V = PoisonUse->getUser();
1665 unsigned OperandNo = PoisonUse->getOperandNo();
1670 *PoisonUse, NewAS, ValueWithNewAddrSpace, PredicatedAS,
nullptr);
1674 SmallVector<Instruction *, 16> DeadInstructions;
1677 for (
const WeakTrackingVH &WVH : Postorder) {
1678 assert(WVH &&
"value was unexpectedly deleted");
1681 if (NewV ==
nullptr)
1696 if (
I->getFunction() ==
F)
1699 WorkList.
append(
U->user_begin(),
U->user_end());
1702 if (!WorkList.
empty()) {
1708 ValueMapper VMapper(VMap,
1710 DenseSet<User *> Visited{WorkList.
begin(), WorkList.
end()};
1711 while (!WorkList.
empty()) {
1714 if (
I->getFunction() ==
F)
1715 VMapper.remapInstruction(*
I);
1718 for (User *U2 :
U->users())
1719 if (Visited.
insert(U2).second)
1727 Value::use_iterator
I,
E,
Next;
1728 for (
I =
V->use_begin(),
E =
V->use_end();
I !=
E;) {
1735 performPointerReplacement(
V, NewV, U, ValueWithNewAddrSpace,
1739 if (
V->use_empty()) {
1748 auto DeadInstructionHandles =
1755bool InferAddressSpaces::runOnFunction(
Function &
F) {
1756 if (skipFunction(
F))
1759 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
1760 DominatorTree *DT = DTWP ? &DTWP->getDomTree() :
nullptr;
1761 return InferAddressSpacesImpl(
1762 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
F), DT,
1763 &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
F),
1764 FlatAddrSpace,
false)
1773 bool AssumeDefaultIsFlatAddressSpace)
1775 AssumeDefaultIsFlatAddressSpace(AssumeDefaultIsFlatAddressSpace) {}
1777 unsigned AddressSpace,
bool AssumeDefaultIsFlatAddressSpace)
1779 AssumeDefaultIsFlatAddressSpace(AssumeDefaultIsFlatAddressSpace) {}
1787 AssumeDefaultIsFlatAddressSpace)
1799 static_cast<PassInfoMixin<InferAddressSpacesPass> *
>(
this)->
printPipeline(
1800 OS, MapClassName2PassName);
1801 if (AssumeDefaultIsFlatAddressSpace)
1802 OS <<
"<assume-default-is-flat-addrspace>";
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_UNLIKELY(EXPR)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
static bool runOnFunction(Function &F, bool PostInlining)
This header defines various interfaces for pass management in LLVM.
This defines the Use class.
static bool replaceOperandIfSame(Instruction *Inst, unsigned OpIdx, Value *OldVal, Value *NewVal)
Replace operand OpIdx in Inst, if the value is the same as OldVal with NewVal.
static bool isNoopPtrIntCastPair(const Operator *I2P, const DataLayout &DL, const TargetTransformInfo *TTI)
static bool replaceIfSimplePointerUse(const TargetTransformInfo &TTI, User *Inst, Value *OldV, Value *NewV)
If OldV is used as the pointer operand of a compatible memory operation Inst, replaces the pointer op...
static Value * phiNodeOperandWithNewAddressSpace(AddrSpaceCastInst *NewI, Value *Operand)
static bool handleMemIntrinsicPtrUse(MemIntrinsic *MI, Value *OldV, Value *NewV)
Update memory intrinsic uses that require more complex processing than simple memory instructions.
static Value * operandWithNewAddressSpaceOrCreatePoison(const Use &OperandUse, unsigned NewAddrSpace, const ValueToValueMapTy &ValueWithNewAddrSpace, const PredicatedAddrSpaceMapTy &PredicatedAS, SmallVectorImpl< const Use * > *PoisonUsesToFix)
static Value::use_iterator skipToNextUser(Value::use_iterator I, Value::use_iterator End)
Infer address static false Type * getPtrOrVecOfPtrsWithNewAS(Type *Ty, unsigned NewAddrSpace)
static bool replaceSimplePointerUse(const TargetTransformInfo &TTI, InstrType *MemInstr, Value *OldV, Value *NewV)
static APInt computeMaxChangedPtrBits(const Operator *Op, const Value *Mask, const DataLayout &DL, AssumptionCache *AC, const DominatorTree *DT)
static const unsigned UninitializedAddressSpace
Machine Check Debug Module
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
This class represents a conversion between pointers from one address space to another.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addRequired()
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
MutableArrayRef< ResultElem > assumptionsFor(const Value *V)
Access the list of assumptions which affect this value.
InstListType::iterator iterator
Instruction iterators...
Represents analyses that only rely on functions' control flow.
Value * getArgOperand(unsigned i) const
static LLVM_ABI bool isNoopCast(Instruction::CastOps Opcode, Type *SrcTy, Type *DstTy, const DataLayout &DL)
A no-op cast is one that can be effected without changing any bits.
static LLVM_ABI Constant * getAddrSpaceCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
FunctionPass class - This class is used to implement most global optimizations.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
LLVM_ABI void setIsInBounds(bool b=true)
Set or clear the inbounds flag on this GEP instruction.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI InferAddressSpacesPass(bool AssumeDefaultIsFlatAddressSpace=false)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI void insertBefore(InstListType::iterator InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified position.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool isBitwiseLogicOp() const
Return true if this is and/or/xor.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
This is the common base class for memset/memcpy/memmove.
This is a utility class that provides an abstraction for the common functionality between Instruction...
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static unsigned getOperandNumForIncomingValue(unsigned i)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
value_type pop_back_val()
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
const Use & getOperandUse(unsigned i) const
void setOperand(unsigned i, Value *Val)
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Value * getOperand(unsigned i) const
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI const Value * stripInBoundsOffsets(function_ref< void(const Value *)> Func=[](const Value *) {}) const
Strip off pointer casts and inbounds GEPs.
use_iterator_impl< Use > use_iterator
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
std::pair< iterator, bool > insert(const ValueT &V)
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
bool match(Val *V, const Pattern &P)
auto m_Value()
Match an arbitrary value and ignore it.
CastOperator_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
@ CE
Windows NT (Windows on ARM)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
@ User
could "use" a pointer
NodeAddr< UseNode * > Use
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
@ Known
Known to have no common set bits.
LLVM_ABI void initializeInferAddressSpacesPass(PassRegistry &)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr from_range_t from_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
auto cast_or_null(const Y &Val)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI bool isValidAssumeForContext(const Instruction *I, const Instruction *CtxI, const DominatorTree *DT=nullptr, bool AllowEphemerals=false)
Return true if it is valid to use the assumptions provided by an assume intrinsic,...
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
@ RF_IgnoreMissingLocals
If this flag is set, the remapper ignores missing function-local entries (Argument,...
@ RF_NoModuleLevelChanges
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
LLVM_ABI FunctionPass * createInferAddressSpacesPass(unsigned AddressSpace=~0u)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
@ End
Run the machine verifier once, at the end of the pipeline.
SmallVector< Out, Size > to_vector_of(R &&Range)
unsigned getBitWidth() const
Get the bit width of this value.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
static KnownBits sub(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from subtraction of LHS and RHS.