214#include "llvm/IR/IntrinsicsAMDGPU.h"
228#define DEBUG_TYPE "amdgpu-lower-buffer-fat-pointers"
251 void clear() { Map.clear(); }
257class BufferFatPtrToIntTypeMap :
public BufferFatPtrTypeLoweringBase {
258 using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase;
268class BufferFatPtrToStructTypeMap :
public BufferFatPtrTypeLoweringBase {
269 using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase;
278Type *BufferFatPtrTypeLoweringBase::remapTypeImpl(
283 if (
auto *PT = dyn_cast<PointerType>(Ty)) {
285 return *
Entry = remapScalar(PT);
288 if (
auto *VT = dyn_cast<VectorType>(Ty)) {
289 auto *PT = dyn_cast<PointerType>(VT->getElementType());
291 return *
Entry = remapVector(VT);
298 StructType *TyAsStruct = dyn_cast<StructType>(Ty);
299 bool IsUniqued = !TyAsStruct || TyAsStruct->
isLiteral();
306 if (!Seen.
insert(TyAsStruct).second) {
308 return *
Entry = Placeholder;
311 bool Changed =
false;
315 Type *NewElem = remapTypeImpl(OldElem, Seen);
316 ElementTypes[
I] = NewElem;
317 Changed |= (OldElem != NewElem);
324 if (
auto *ArrTy = dyn_cast<ArrayType>(Ty))
325 return *Entry = ArrayType::get(ElementTypes[0], ArrTy->getNumElements());
326 if (
auto *FnTy = dyn_cast<FunctionType>(Ty))
327 return *Entry = FunctionType::get(ElementTypes[0],
330 if (
auto *STy = dyn_cast<StructType>(Ty)) {
339 Type **RecursionEntry = &
Map[Ty];
340 if (*RecursionEntry) {
341 auto *Placeholder = cast<StructType>(*RecursionEntry);
342 Placeholder->setBody(ElementTypes, IsPacked);
343 Placeholder->setName(
Name);
344 return *
Entry = Placeholder;
352Type *BufferFatPtrTypeLoweringBase::remapType(
Type *SrcTy) {
354 return remapTypeImpl(SrcTy, Visited);
381 auto *ST = dyn_cast<StructType>(Ty);
384 if (!ST->isLiteral() || ST->getNumElements() != 2)
387 dyn_cast<PointerType>(ST->getElementType(0)->getScalarType());
389 dyn_cast<IntegerType>(ST->getElementType(1)->getScalarType());
390 return MaybeRsrc && MaybeOff &&
399 return isBufferFatPtrOrVector(U.get()->getType());
412class StoreFatPtrsAsIntsVisitor
413 :
public InstVisitor<StoreFatPtrsAsIntsVisitor, bool> {
414 BufferFatPtrToIntTypeMap *TypeMap;
429 StoreFatPtrsAsIntsVisitor(BufferFatPtrToIntTypeMap *TypeMap,
LLVMContext &Ctx)
430 : TypeMap(TypeMap), IRB(Ctx) {}
446 if (
Find != ConvertedForStore.end())
449 Value *Cast = IRB.CreatePtrToInt(V, To,
Name +
".int");
450 ConvertedForStore[
V] = Cast;
453 if (
From->getNumContainedTypes() == 0)
457 if (
auto *AT = dyn_cast<ArrayType>(
From)) {
459 Type *ToPart = cast<ArrayType>(To)->getElementType();
460 for (
uint64_t I = 0, E = AT->getArrayNumElements();
I < E; ++
I) {
464 Ret = IRB.CreateInsertValue(Ret, NewField,
I);
467 for (
auto [
Idx, FromPart, ToPart] :
472 Ret = IRB.CreateInsertValue(Ret, NewField,
Idx);
475 ConvertedForStore[
V] =
Ret;
484 Value *Cast = IRB.CreateIntToPtr(V, To,
Name +
".ptr");
487 if (
From->getNumContainedTypes() == 0)
491 if (
auto *AT = dyn_cast<ArrayType>(
From)) {
493 Type *ToPart = cast<ArrayType>(To)->getElementType();
494 for (
uint64_t I = 0, E = AT->getArrayNumElements();
I < E; ++
I) {
498 Ret = IRB.CreateInsertValue(Ret, NewField,
I);
501 for (
auto [
Idx, FromPart, ToPart] :
506 Ret = IRB.CreateInsertValue(Ret, NewField,
Idx);
512bool StoreFatPtrsAsIntsVisitor::processFunction(
Function &
F) {
513 bool Changed =
false;
519 ConvertedForStore.clear();
523bool StoreFatPtrsAsIntsVisitor::visitAllocaInst(
AllocaInst &
I) {
524 Type *Ty =
I.getAllocatedType();
525 Type *NewTy = TypeMap->remapType(Ty);
528 I.setAllocatedType(NewTy);
533 Type *Ty =
I.getSourceElementType();
534 Type *NewTy = TypeMap->remapType(Ty);
539 I.setSourceElementType(NewTy);
540 I.setResultElementType(TypeMap->remapType(
I.getResultElementType()));
544bool StoreFatPtrsAsIntsVisitor::visitLoadInst(
LoadInst &LI) {
546 Type *IntTy = TypeMap->remapType(Ty);
550 IRB.SetInsertPoint(&LI);
551 auto *NLI = cast<LoadInst>(LI.
clone());
552 NLI->mutateType(IntTy);
553 NLI = IRB.Insert(NLI);
557 Value *CastBack = intsToFatPtrs(NLI, IntTy, Ty, NLI->getName());
563bool StoreFatPtrsAsIntsVisitor::visitStoreInst(
StoreInst &SI) {
565 Type *Ty =
V->getType();
566 Type *IntTy = TypeMap->remapType(Ty);
570 IRB.SetInsertPoint(&SI);
571 Value *IntV = fatPtrsToInts(V, Ty, IntTy,
V->getName());
575 SI.setOperand(0, IntV);
581static std::pair<Constant *, Constant *>
584 return std::make_pair(
C->getAggregateElement(0u),
C->getAggregateElement(1u));
590 BufferFatPtrToStructTypeMap *TypeMap;
602 FatPtrConstMaterializer(BufferFatPtrToStructTypeMap *TypeMap,
605 InternalMapper(UnderlyingMap,
RF_None, TypeMap, this) {}
606 virtual ~FatPtrConstMaterializer() =
default;
613 Type *SrcTy =
C->getType();
614 auto *NewTy = dyn_cast<StructType>(TypeMap->remapType(SrcTy));
615 if (
C->isNullValue())
616 return ConstantAggregateZero::getNullValue(NewTy);
617 if (isa<PoisonValue>(
C)) {
622 if (isa<UndefValue>(
C)) {
628 if (
auto *VC = dyn_cast<ConstantVector>(
C)) {
630 Constant *NewS = InternalMapper.mapConstant(*S);
634 auto EC =
VC->getType()->getElementCount();
641 auto *NewOp = dyn_cast_or_null<Constant>(InternalMapper.mapValue(*
Op));
653 if (isa<GlobalValue>(
C))
655 "fat pointer) values are not supported");
657 if (isa<ConstantExpr>(
C))
659 "fat pointer) values should have been expanded earlier");
664Value *FatPtrConstMaterializer::materialize(
Value *V) {
672 return materializeBufferFatPtrConst(
C);
680class SplitPtrStructs :
public InstVisitor<SplitPtrStructs, PtrParts> {
723 void processConditionals();
743 :
TM(
TM), ST(
nullptr), IRB(Ctx) {}
771void SplitPtrStructs::copyMetadata(
Value *Dest,
Value *Src) {
772 auto *DestI = dyn_cast<Instruction>(Dest);
773 auto *SrcI = dyn_cast<Instruction>(Src);
778 DestI->copyMetadata(*SrcI);
783 "of something that wasn't rewritten");
784 auto *RsrcEntry = &RsrcParts[
V];
785 auto *OffEntry = &OffParts[
V];
786 if (*RsrcEntry && *OffEntry)
787 return {*RsrcEntry, *OffEntry};
789 if (
auto *
C = dyn_cast<Constant>(V)) {
791 return {*RsrcEntry = Rsrc, *OffEntry =
Off};
795 if (
auto *
I = dyn_cast<Instruction>(V)) {
797 auto [Rsrc,
Off] = visit(*
I);
799 return {*RsrcEntry = Rsrc, *OffEntry =
Off};
802 IRB.SetInsertPoint(*
I->getInsertionPointAfterDef());
803 IRB.SetCurrentDebugLocation(
I->getDebugLoc());
804 }
else if (
auto *
A = dyn_cast<Argument>(V)) {
805 IRB.SetInsertPointPastAllocas(
A->getParent());
806 IRB.SetCurrentDebugLocation(
DebugLoc());
808 Value *Rsrc = IRB.CreateExtractValue(V, 0,
V->getName() +
".rsrc");
809 Value *
Off = IRB.CreateExtractValue(V, 1,
V->getName() +
".off");
810 return {*RsrcEntry = Rsrc, *OffEntry =
Off};
822 while (
auto *
GEP = dyn_cast<GEPOperator>(V))
823 V =
GEP->getPointerOperand();
824 while (
auto *ASC = dyn_cast<AddrSpaceCastOperator>(V))
825 V = ASC->getPointerOperand();
829void SplitPtrStructs::getPossibleRsrcRoots(
Instruction *
I,
832 if (
auto *
PHI = dyn_cast<PHINode>(
I)) {
835 for (
Value *In :
PHI->incoming_values()) {
838 if (isa<PHINode, SelectInst>(In))
839 getPossibleRsrcRoots(cast<Instruction>(In), Roots, Seen);
841 }
else if (
auto *SI = dyn_cast<SelectInst>(
I)) {
842 if (!Seen.
insert(SI).second)
848 if (isa<PHINode, SelectInst>(TrueVal))
849 getPossibleRsrcRoots(cast<Instruction>(TrueVal), Roots, Seen);
850 if (isa<PHINode, SelectInst>(FalseVal))
851 getPossibleRsrcRoots(cast<Instruction>(FalseVal), Roots, Seen);
857void SplitPtrStructs::processConditionals() {
863 Value *Rsrc = RsrcParts[
I];
865 assert(Rsrc && Off &&
"must have visited conditionals by now");
867 std::optional<Value *> MaybeRsrc;
868 auto MaybeFoundRsrc = FoundRsrcs.
find(
I);
869 if (MaybeFoundRsrc != FoundRsrcs.
end()) {
870 MaybeRsrc = MaybeFoundRsrc->second;
875 getPossibleRsrcRoots(
I, Roots, Seen);
878 for (
Value *V : Roots)
880 for (
Value *V : Seen)
892 if (Diff.size() == 1) {
893 Value *RootVal = *Diff.begin();
897 MaybeRsrc = std::get<0>(getPtrParts(RootVal));
904 if (
auto *
PHI = dyn_cast<PHINode>(
I)) {
907 IRB.SetInsertPoint(*
PHI->getInsertionPointAfterDef());
908 IRB.SetCurrentDebugLocation(
PHI->getDebugLoc());
910 NewRsrc = *MaybeRsrc;
913 auto *RsrcPHI = IRB.CreatePHI(RsrcTy,
PHI->getNumIncomingValues());
914 RsrcPHI->takeName(Rsrc);
915 for (
auto [V, BB] :
llvm::zip(
PHI->incoming_values(),
PHI->blocks())) {
916 Value *VRsrc = std::get<0>(getPtrParts(V));
917 RsrcPHI->addIncoming(VRsrc, BB);
919 copyMetadata(RsrcPHI,
PHI);
924 auto *NewOff = IRB.CreatePHI(OffTy,
PHI->getNumIncomingValues());
925 NewOff->takeName(Off);
926 for (
auto [V, BB] :
llvm::zip(
PHI->incoming_values(),
PHI->blocks())) {
927 assert(OffParts.count(V) &&
"An offset part had to be created by now");
928 Value *VOff = std::get<1>(getPtrParts(V));
929 NewOff->addIncoming(VOff, BB);
931 copyMetadata(NewOff,
PHI);
937 ConditionalTemps.push_back(cast<Instruction>(Rsrc));
938 ConditionalTemps.push_back(cast<Instruction>(Off));
940 Off->replaceAllUsesWith(NewOff);
944 for (
Value *V : Seen)
945 FoundRsrcs[cast<Instruction>(V)] = NewRsrc;
946 }
else if (isa<SelectInst>(
I)) {
948 ConditionalTemps.push_back(cast<Instruction>(Rsrc));
950 for (
Value *V : Seen)
951 FoundRsrcs[cast<Instruction>(V)] = *MaybeRsrc;
959void SplitPtrStructs::killAndReplaceSplitInstructions(
962 I->eraseFromParent();
965 if (!SplitUsers.contains(
I))
970 for (
auto *Dbg : Dbgs) {
971 IRB.SetInsertPoint(Dbg);
972 auto &
DL =
I->getDataLayout();
974 "We should've RAUW'd away loads, stores, etc. at this point");
975 auto *OffDbg = cast<DbgValueInst>(
Dbg->clone());
976 copyMetadata(OffDbg, Dbg);
977 auto [Rsrc,
Off] = getPtrParts(
I);
979 int64_t RsrcSz =
DL.getTypeSizeInBits(Rsrc->
getType());
980 int64_t OffSz =
DL.getTypeSizeInBits(
Off->getType());
982 std::optional<DIExpression *> RsrcExpr =
985 std::optional<DIExpression *> OffExpr =
989 OffDbg->setExpression(*OffExpr);
990 OffDbg->replaceVariableLocationOp(
I, Off);
993 OffDbg->deleteValue();
996 Dbg->setExpression(*RsrcExpr);
997 Dbg->replaceVariableLocationOp(
I, Rsrc);
1004 I->replaceUsesWithIf(Poison, [&](
const Use &U) ->
bool {
1005 if (
const auto *UI = dyn_cast<Instruction>(
U.getUser()))
1006 return SplitUsers.contains(UI);
1010 if (
I->use_empty()) {
1011 I->eraseFromParent();
1014 IRB.SetInsertPoint(*
I->getInsertionPointAfterDef());
1015 IRB.SetCurrentDebugLocation(
I->getDebugLoc());
1016 auto [Rsrc,
Off] = getPtrParts(
I);
1022 I->replaceAllUsesWith(
Struct);
1023 I->eraseFromParent();
1035 case AtomicOrdering::Release:
1036 case AtomicOrdering::AcquireRelease:
1037 case AtomicOrdering::SequentiallyConsistent:
1038 IRB.CreateFence(AtomicOrdering::Release, SSID);
1048 case AtomicOrdering::Acquire:
1049 case AtomicOrdering::AcquireRelease:
1050 case AtomicOrdering::SequentiallyConsistent:
1051 IRB.CreateFence(AtomicOrdering::Acquire, SSID);
1062 IRB.SetInsertPoint(
I);
1064 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1067 Args.push_back(Arg);
1068 Args.push_back(Rsrc);
1069 Args.push_back(Off);
1070 insertPreMemOpFence(Order, SSID);
1074 Args.push_back(IRB.getInt32(0));
1078 (isa<LoadInst>(
I) &&
I->getMetadata(LLVMContext::MD_invariant_load));
1079 bool IsNonTemporal =
I->getMetadata(LLVMContext::MD_nontemporal);
1081 bool IsOneWayAtomic =
1082 !isa<AtomicRMWInst>(
I) && Order != AtomicOrdering::NotAtomic;
1085 if (IsNonTemporal && !IsInvariant)
1091 Args.push_back(IRB.getInt32(Aux));
1094 if (isa<LoadInst>(
I))
1096 IID = Intrinsic::amdgcn_raw_ptr_buffer_load;
1097 else if (isa<StoreInst>(
I))
1098 IID = Intrinsic::amdgcn_raw_ptr_buffer_store;
1099 else if (
auto *RMW = dyn_cast<AtomicRMWInst>(
I)) {
1100 switch (RMW->getOperation()) {
1102 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap;
1105 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_add;
1108 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub;
1111 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_and;
1114 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_or;
1117 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor;
1120 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax;
1123 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin;
1126 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax;
1129 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin;
1132 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd;
1135 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax;
1138 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin;
1142 "buffer resources and should've been expanded away");
1147 "should've been expanded away");
1152 "buffer resources and should've ben expanded away");
1159 auto *
Call = IRB.CreateIntrinsic(IID, Ty, Args);
1160 copyMetadata(Call,
I);
1161 setAlign(Call, Alignment, Arg ? 1 : 0);
1164 insertPostMemOpFence(Order, SSID);
1167 SplitUsers.insert(
I);
1168 I->replaceAllUsesWith(Call);
1173 return {
nullptr,
nullptr};
1178 return {
nullptr,
nullptr};
1182 return {
nullptr,
nullptr};
1187 return {
nullptr,
nullptr};
1188 Value *Arg =
SI.getValueOperand();
1189 handleMemoryInst(&SI, Arg,
SI.getPointerOperand(), Arg->
getType(),
1190 SI.getAlign(),
SI.getOrdering(),
SI.isVolatile(),
1191 SI.getSyncScopeID());
1192 return {
nullptr,
nullptr};
1197 return {
nullptr,
nullptr};
1202 return {
nullptr,
nullptr};
1210 return {
nullptr,
nullptr};
1211 IRB.SetInsertPoint(&AI);
1216 bool IsNonTemporal = AI.
getMetadata(LLVMContext::MD_nontemporal);
1218 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1219 insertPreMemOpFence(Order, SSID);
1227 IRB.CreateIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap, Ty,
1229 Off, IRB.getInt32(0), IRB.getInt32(Aux)});
1230 copyMetadata(Call, &AI);
1232 Call->takeName(&AI);
1233 insertPostMemOpFence(Order, SSID);
1236 Res = IRB.CreateInsertValue(Res, Call, 0);
1239 Res = IRB.CreateInsertValue(Res, Succeeded, 1);
1241 SplitUsers.insert(&AI);
1243 return {
nullptr,
nullptr};
1250 return {
nullptr,
nullptr};
1251 IRB.SetInsertPoint(&
GEP);
1253 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1255 bool InBounds =
GEP.isInBounds();
1260 if (
auto *VT = dyn_cast<VectorType>(
Off->getType()))
1261 FatPtrTy = VectorType::get(FatPtrTy, VT->getElementCount());
1262 GEP.mutateType(FatPtrTy);
1264 GEP.mutateType(
Ptr->getType());
1266 SplitUsers.insert(&
GEP);
1270 bool HasNonNegativeOff =
false;
1271 if (
auto *CI = dyn_cast<ConstantInt>(OffAccum)) {
1272 HasNonNegativeOff = !CI->isNegative();
1278 NewOff = IRB.CreateAdd(Off, OffAccum,
"",
1279 InBounds && HasNonNegativeOff,
1282 copyMetadata(NewOff, &
GEP);
1284 SplitUsers.insert(&
GEP);
1285 return {Rsrc, NewOff};
1291 return {
nullptr,
nullptr};
1292 IRB.SetInsertPoint(&PI);
1297 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1303 Res = IRB.CreateIntCast(Off, ResTy,
false,
1306 Value *RsrcInt = IRB.CreatePtrToInt(Rsrc, ResTy, PI.
getName() +
".rsrc");
1307 Value *Shl = IRB.CreateShl(
1310 "", Width >= FatPtrWidth, Width > FatPtrWidth);
1311 Value *OffCast = IRB.CreateIntCast(Off, ResTy,
false,
1313 Res = IRB.CreateOr(Shl, OffCast);
1316 copyMetadata(Res, &PI);
1318 SplitUsers.insert(&PI);
1320 return {
nullptr,
nullptr};
1325 return {
nullptr,
nullptr};
1326 IRB.SetInsertPoint(&IP);
1335 Type *RsrcTy =
RetTy->getElementType(0);
1337 Value *RsrcPart = IRB.CreateLShr(
1340 Value *RsrcInt = IRB.CreateIntCast(RsrcPart, RsrcIntTy,
false);
1341 Value *Rsrc = IRB.CreateIntToPtr(RsrcInt, RsrcTy, IP.
getName() +
".rsrc");
1343 IRB.CreateIntCast(
Int, OffTy,
false, IP.
getName() +
".off");
1345 copyMetadata(Rsrc, &IP);
1346 SplitUsers.insert(&IP);
1352 return {
nullptr,
nullptr};
1353 IRB.SetInsertPoint(&
I);
1356 if (
In->getType() ==
I.getType()) {
1357 auto [Rsrc,
Off] = getPtrParts(In);
1358 SplitUsers.insert(&
I);
1363 "buffer fat pointers (addrspace 7)");
1364 Type *OffTy = cast<StructType>(
I.getType())->getElementType(1);
1366 SplitUsers.insert(&
I);
1367 return {
In, ZeroOff};
1373 return {
nullptr,
nullptr};
1375 IRB.SetInsertPoint(&Cmp);
1378 assert((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
1379 "Pointer comparison is only equal or unequal");
1380 auto [LhsRsrc, LhsOff] = getPtrParts(Lhs);
1381 auto [RhsRsrc, RhsOff] = getPtrParts(Rhs);
1383 IRB.CreateICmp(Pred, LhsRsrc, RhsRsrc,
Cmp.getName() +
".rsrc");
1384 copyMetadata(RsrcCmp, &Cmp);
1385 Value *OffCmp = IRB.CreateICmp(Pred, LhsOff, RhsOff,
Cmp.getName() +
".off");
1386 copyMetadata(OffCmp, &Cmp);
1388 Value *Res =
nullptr;
1389 if (Pred == ICmpInst::ICMP_EQ)
1390 Res = IRB.CreateAnd(RsrcCmp, OffCmp);
1391 else if (Pred == ICmpInst::ICMP_NE)
1392 Res = IRB.CreateOr(RsrcCmp, OffCmp);
1393 copyMetadata(Res, &Cmp);
1395 SplitUsers.insert(&Cmp);
1396 Cmp.replaceAllUsesWith(Res);
1397 return {
nullptr,
nullptr};
1402 return {
nullptr,
nullptr};
1403 IRB.SetInsertPoint(&
I);
1404 auto [Rsrc,
Off] = getPtrParts(
I.getOperand(0));
1406 Value *RsrcRes = IRB.CreateFreeze(Rsrc,
I.getName() +
".rsrc");
1407 copyMetadata(RsrcRes, &
I);
1408 Value *OffRes = IRB.CreateFreeze(Off,
I.getName() +
".off");
1409 copyMetadata(OffRes, &
I);
1410 SplitUsers.insert(&
I);
1411 return {RsrcRes, OffRes};
1416 return {
nullptr,
nullptr};
1417 IRB.SetInsertPoint(&
I);
1418 Value *Vec =
I.getVectorOperand();
1420 auto [Rsrc,
Off] = getPtrParts(Vec);
1422 Value *RsrcRes = IRB.CreateExtractElement(Rsrc,
Idx,
I.getName() +
".rsrc");
1423 copyMetadata(RsrcRes, &
I);
1424 Value *OffRes = IRB.CreateExtractElement(Off,
Idx,
I.getName() +
".off");
1425 copyMetadata(OffRes, &
I);
1426 SplitUsers.insert(&
I);
1427 return {RsrcRes, OffRes};
1434 return {
nullptr,
nullptr};
1435 IRB.SetInsertPoint(&
I);
1436 Value *Vec =
I.getOperand(0);
1437 Value *Elem =
I.getOperand(1);
1439 auto [VecRsrc, VecOff] = getPtrParts(Vec);
1440 auto [ElemRsrc, ElemOff] = getPtrParts(Elem);
1443 IRB.CreateInsertElement(VecRsrc, ElemRsrc,
Idx,
I.getName() +
".rsrc");
1444 copyMetadata(RsrcRes, &
I);
1446 IRB.CreateInsertElement(VecOff, ElemOff,
Idx,
I.getName() +
".off");
1447 copyMetadata(OffRes, &
I);
1448 SplitUsers.insert(&
I);
1449 return {RsrcRes, OffRes};
1455 return {
nullptr,
nullptr};
1456 IRB.SetInsertPoint(&
I);
1458 Value *V1 =
I.getOperand(0);
1461 auto [V1Rsrc, V1Off] = getPtrParts(V1);
1462 auto [V2Rsrc, V2Off] = getPtrParts(V2);
1465 IRB.CreateShuffleVector(V1Rsrc, V2Rsrc, Mask,
I.getName() +
".rsrc");
1466 copyMetadata(RsrcRes, &
I);
1468 IRB.CreateShuffleVector(V1Off, V2Off, Mask,
I.getName() +
".off");
1469 copyMetadata(OffRes, &
I);
1470 SplitUsers.insert(&
I);
1471 return {RsrcRes, OffRes};
1476 return {
nullptr,
nullptr};
1477 IRB.SetInsertPoint(*
PHI.getInsertionPointAfterDef());
1483 Value *TmpRsrc = IRB.CreateExtractValue(&
PHI, 0,
PHI.getName() +
".rsrc");
1484 Value *TmpOff = IRB.CreateExtractValue(&
PHI, 1,
PHI.getName() +
".off");
1485 Conditionals.push_back(&
PHI);
1486 SplitUsers.insert(&
PHI);
1487 return {TmpRsrc, TmpOff};
1492 return {
nullptr,
nullptr};
1493 IRB.SetInsertPoint(&SI);
1496 Value *True =
SI.getTrueValue();
1497 Value *False =
SI.getFalseValue();
1498 auto [TrueRsrc, TrueOff] = getPtrParts(True);
1499 auto [FalseRsrc, FalseOff] = getPtrParts(False);
1502 IRB.CreateSelect(
Cond, TrueRsrc, FalseRsrc,
SI.getName() +
".rsrc", &SI);
1503 copyMetadata(RsrcRes, &SI);
1504 Conditionals.push_back(&SI);
1506 IRB.CreateSelect(
Cond, TrueOff, FalseOff,
SI.getName() +
".off", &SI);
1507 copyMetadata(OffRes, &SI);
1508 SplitUsers.insert(&SI);
1509 return {RsrcRes, OffRes};
1520 case Intrinsic::ptrmask:
1521 case Intrinsic::invariant_start:
1522 case Intrinsic::invariant_end:
1523 case Intrinsic::launder_invariant_group:
1524 case Intrinsic::strip_invariant_group:
1534 case Intrinsic::ptrmask: {
1537 return {
nullptr,
nullptr};
1539 IRB.SetInsertPoint(&
I);
1540 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1541 if (
Mask->getType() !=
Off->getType())
1543 "pointer (data layout not set up correctly?)");
1544 Value *OffRes = IRB.CreateAnd(Off, Mask,
I.getName() +
".off");
1545 copyMetadata(OffRes, &
I);
1546 SplitUsers.insert(&
I);
1547 return {Rsrc, OffRes};
1551 case Intrinsic::invariant_start: {
1554 return {
nullptr,
nullptr};
1555 IRB.SetInsertPoint(&
I);
1556 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1558 auto *NewRsrc = IRB.CreateIntrinsic(IID, {NewTy}, {
I.getOperand(0), Rsrc});
1559 copyMetadata(NewRsrc, &
I);
1561 SplitUsers.insert(&
I);
1562 I.replaceAllUsesWith(NewRsrc);
1563 return {
nullptr,
nullptr};
1565 case Intrinsic::invariant_end: {
1566 Value *RealPtr =
I.getArgOperand(2);
1568 return {
nullptr,
nullptr};
1569 IRB.SetInsertPoint(&
I);
1570 Value *RealRsrc = getPtrParts(RealPtr).first;
1571 Value *InvPtr =
I.getArgOperand(0);
1573 Value *NewRsrc = IRB.CreateIntrinsic(IID, {RealRsrc->
getType()},
1574 {InvPtr,
Size, RealRsrc});
1575 copyMetadata(NewRsrc, &
I);
1577 SplitUsers.insert(&
I);
1578 I.replaceAllUsesWith(NewRsrc);
1579 return {
nullptr,
nullptr};
1581 case Intrinsic::launder_invariant_group:
1582 case Intrinsic::strip_invariant_group: {
1585 return {
nullptr,
nullptr};
1586 IRB.SetInsertPoint(&
I);
1587 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1588 Value *NewRsrc = IRB.CreateIntrinsic(IID, {Rsrc->
getType()}, {Rsrc});
1589 copyMetadata(NewRsrc, &
I);
1591 SplitUsers.insert(&
I);
1592 return {NewRsrc,
Off};
1595 return {
nullptr,
nullptr};
1598void SplitPtrStructs::processFunction(
Function &
F) {
1601 LLVM_DEBUG(
dbgs() <<
"Splitting pointer structs in function: " <<
F.getName()
1604 Originals.push_back(&
I);
1606 auto [Rsrc,
Off] = visit(
I);
1607 assert(((Rsrc && Off) || (!Rsrc && !Off)) &&
1608 "Can't have a resource but no offset");
1610 RsrcParts[
I] = Rsrc;
1614 processConditionals();
1615 killAndReplaceSplitInstructions(Originals);
1621 Conditionals.clear();
1622 ConditionalTemps.clear();
1626class AMDGPULowerBufferFatPointers :
public ModulePass {
1646 BufferFatPtrToStructTypeMap *TypeMap) {
1647 bool HasFatPointers =
false;
1650 HasFatPointers |= (
I.getType() != TypeMap->remapType(
I.getType()));
1651 return HasFatPointers;
1655 BufferFatPtrToStructTypeMap *TypeMap) {
1656 Type *Ty =
F.getFunctionType();
1657 return Ty != TypeMap->remapType(Ty);
1674 while (!OldF->
empty()) {
1686 {Attribute::Dereferenceable, Attribute::DereferenceableOrNull,
1687 Attribute::NoAlias, Attribute::NoCapture, Attribute::NoFree,
1688 Attribute::NonNull, Attribute::NullPointerIsValid, Attribute::ReadNone,
1689 Attribute::ReadOnly, Attribute::WriteOnly}) {
1696 CloneMap[&NewArg] = &OldArg;
1697 NewArg.takeName(&OldArg);
1698 Type *OldArgTy = OldArg.getType(), *NewArgTy = NewArg.getType();
1700 NewArg.mutateType(OldArgTy);
1701 OldArg.replaceAllUsesWith(&NewArg);
1702 NewArg.mutateType(NewArgTy);
1706 if (OldArgTy != NewArgTy && !IsIntrinsic)
1722 CloneMap[&BB] = &BB;
1729 bool Changed =
false;
1736 BufferFatPtrToStructTypeMap StructTM(
DL);
1737 BufferFatPtrToIntTypeMap IntTM(
DL);
1741 "space (7) are not supported");
1742 Type *VT = GV.getValueType();
1743 if (VT != StructTM.remapType(VT))
1745 "(address space 7 pointers) are unsupported. Use "
1746 "buffer resource pointers (address space 8) instead.");
1755 if (isa<ConstantExpr>(
Op) || isa<ConstantAggregate>(
Op))
1761 while (!Worklist.
empty()) {
1763 if (!Visited.
insert(
C).second)
1768 if (isa<ConstantExpr>(
Op) || isa<ConstantAggregate>(
Op))
1779 StoreFatPtrsAsIntsVisitor MemOpsRewrite(&IntTM,
M.getContext());
1783 Changed |= MemOpsRewrite.processFunction(
F);
1784 if (InterfaceChange || BodyChanges)
1785 NeedsRemap.
push_back(std::make_pair(&
F, InterfaceChange));
1787 if (NeedsRemap.
empty())
1794 FatPtrConstMaterializer Materializer(&StructTM, CloneMap);
1797 for (
auto [
F, InterfaceChange] : NeedsRemap) {
1799 if (InterfaceChange)
1801 F, cast<FunctionType>(StructTM.remapType(
F->getFunctionType())),
1805 LowerInFuncs.remapFunction(*NewF);
1810 if (InterfaceChange) {
1811 F->replaceAllUsesWith(NewF);
1812 F->eraseFromParent();
1820 SplitPtrStructs Splitter(
M.getContext(), &
TM);
1822 Splitter.processFunction(*
F);
1825 F->eraseFromParent();
1829 F->replaceAllUsesWith(*NewF);
1835bool AMDGPULowerBufferFatPointers::runOnModule(
Module &M) {
1841char AMDGPULowerBufferFatPointers::ID = 0;
1845void AMDGPULowerBufferFatPointers::getAnalysisUsage(
AnalysisUsage &AU)
const {
1849#define PASS_DESC "Lower buffer fat pointer operations to buffer resources"
1858 return new AMDGPULowerBufferFatPointers();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Function * moveFunctionAdaptingType(Function *OldF, FunctionType *NewTy, ValueToValueMapTy &CloneMap)
Move the body of OldF into a new function, returning it.
static void makeCloneInPraceMap(Function *F, ValueToValueMapTy &CloneMap)
static bool isBufferFatPtrOrVector(Type *Ty)
static bool isSplitFatPtr(Type *Ty)
std::pair< Value *, Value * > PtrParts
static bool hasFatPointerInterface(const Function &F, BufferFatPtrToStructTypeMap *TypeMap)
static bool isRemovablePointerIntrinsic(Intrinsic::ID IID)
Returns true if this intrinsic needs to be removed when it is applied to ptr addrspace(7) values.
static bool containsBufferFatPointers(const Function &F, BufferFatPtrToStructTypeMap *TypeMap)
Returns true if there are values that have a buffer fat pointer in them, which means we'll need to pe...
static Value * rsrcPartRoot(Value *V)
Returns the instruction that defines the resource part of the value V.
static constexpr unsigned BufferOffsetWidth
static bool isBufferFatPtrConst(Constant *C)
static std::pair< Constant *, Constant * > splitLoweredFatBufferConst(Constant *C)
Return the ptr addrspace(8) and i32 (resource and offset parts) in a lowered buffer fat pointer const...
The AMDGPU TargetMachine interface definition for hw codegen targets.
Expand Atomic instructions
Atomic ordering constants.
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void clear(coro::Shape &Shape)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
AMD GCN specific subclass of TargetSubtarget.
static const T * Find(StringRef S, ArrayRef< T > A)
Find KV in array using binary search.
uint64_t IntrinsicInst * II
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines generic set operations that may be used on set's of different types,...
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Target-Independent Code Generator Pass Configuration Options pass.
Class for arbitrary precision integers.
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
A container for analyses that lazily runs them and caches their results.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getNewValOperand()
AtomicOrdering getMergedOrdering() const
Returns a single ordering which is at least as strong as both the success and failure orderings for t...
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
Value * getCompareOperand()
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
Value * getPointerOperand()
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
AttributeSet removeAttributes(LLVMContext &C, const AttributeMask &AttrsToRemove) const
Remove the specified attributes from this set.
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
void removeFromParent()
Unlink 'this' from the containing function, but do not delete it.
void insertInto(Function *Parent, BasicBlock *InsertBefore=nullptr)
Insert unlinked basic block into a function.
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
Implements a dense probed hash-table based set.
This class represents a freeze function that returns random concrete value if an operand is either a ...
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & front() const
iterator_range< arg_iterator > args()
bool IsNewDbgInfoFormat
Is this function using intrinsics to record the position of debugging information,...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void updateAfterNameChange()
Update internal caches that depend on the function name (such as the intrinsic ID and libcall cache).
Type * getReturnType() const
Returns the type of the ret val.
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void copyMetadata(const GlobalObject *Src, unsigned Offset)
Copy metadata from Src, adjusting offsets by Offset.
LinkageTypes getLinkage() const
void setDLLStorageClass(DLLStorageClassTypes C)
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
DLLStorageClassTypes getDLLStorageClass() const
This instruction compares its operands according to the predicate given to the constructor.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This instruction inserts a single (scalar) element into a VectorType value.
Base class for instruction visitors.
RetTy visitFreezeInst(FreezeInst &I)
RetTy visitPtrToIntInst(PtrToIntInst &I)
RetTy visitExtractElementInst(ExtractElementInst &I)
RetTy visitIntrinsicInst(IntrinsicInst &I)
RetTy visitShuffleVectorInst(ShuffleVectorInst &I)
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
RetTy visitIntToPtrInst(IntToPtrInst &I)
RetTy visitPHINode(PHINode &I)
RetTy visitStoreInst(StoreInst &I)
RetTy visitInsertElementInst(InsertElementInst &I)
RetTy visitAtomicRMWInst(AtomicRMWInst &I)
RetTy visitAddrSpaceCastInst(AddrSpaceCastInst &I)
RetTy visitAllocaInst(AllocaInst &I)
RetTy visitICmpInst(ICmpInst &I)
RetTy visitSelectInst(SelectInst &I)
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
void visitInstruction(Instruction &I)
RetTy visitLoadInst(LoadInst &I)
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
This class represents a cast from an integer to a pointer.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Type * getPointerOperandType() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Align getAlign() const
Return the alignment of the access that is being performed.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
const FunctionListType & getFunctionList() const
Get the Module's list of functions (constant).
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
This class represents a cast from a pointer to an integer.
Value * getPointerOperand()
Gets the pointer operand.
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
bool isLiteral() const
Return true if this type is uniqued by structural equivalence, false if it is a struct definition.
Type * getElementType(unsigned N) const
Primary interface to the complete machine description for the target machine.
Target-Independent Code Generator Pass Configuration Options.
TMC & getTM() const
Get the right type of TargetMachine for this target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getArrayElementType() const
ArrayRef< Type * > subtypes() const
unsigned getNumContainedTypes() const
Return the number of types in the derived type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
This is a class that can be implemented by clients to remap types when cloning constants and instruct...
virtual Type * remapType(Type *SrcTy)=0
The client should implement this method if they want to remap types while mapping values.
Context for (re-)mapping values (and metadata).
This is a class that can be implemented by clients to materialize Values on demand.
virtual Value * materialize(Value *V)=0
This method can be implemented to generate a mapped Value on demand.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
self_iterator getIterator()
iterator insertAfter(iterator where, pointer New)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
std::optional< Function * > remangleIntrinsicFunction(Function *F)
bool match(Val *V, const Pattern &P)
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
AssignmentMarkerRange getAssignmentMarkers(DIAssignID *ID)
Return a range of dbg.assign intrinsics which use \ID as an operand.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
ModulePass * createAMDGPULowerBufferFatPointersPass()
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source)
Copy the metadata from the source instruction to the destination (the replacement for the source inst...
bool set_is_subset(const S1Ty &S1, const S2Ty &S2)
set_is_subset(A, B) - Return true iff A in B
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
void findDbgValues(SmallVectorImpl< DbgValueInst * > &DbgValues, Value *V, SmallVectorImpl< DbgVariableRecord * > *DbgVariableRecords=nullptr)
Finds the llvm.dbg.value intrinsics describing a value.
bool convertUsersOfConstantsToInstructions(ArrayRef< Constant * > Consts, Function *RestrictToFunc=nullptr, bool RemoveDeadConstants=true, bool IncludeSelf=false)
Replace constant expressions users of the given constants with instructions.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Value * emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions=false)
Given a getelementptr instruction/constantexpr, emit the code necessary to compute the offset from th...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
char & AMDGPULowerBufferFatPointersID
AtomicOrdering
Atomic ordering for LLVM's memory model.
S1Ty set_difference(const S1Ty &S1, const S2Ty &S2)
set_difference(A, B) - Return A - B
void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
This struct is a compact representation of a valid (non-zero power of two) alignment.