213#include "llvm/IR/IntrinsicsAMDGPU.h"
226#define DEBUG_TYPE "amdgpu-lower-buffer-fat-pointers"
249 void clear() { Map.clear(); }
255class BufferFatPtrToIntTypeMap :
public BufferFatPtrTypeLoweringBase {
256 using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase;
266class BufferFatPtrToStructTypeMap :
public BufferFatPtrTypeLoweringBase {
267 using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase;
276Type *BufferFatPtrTypeLoweringBase::remapTypeImpl(
281 if (
auto *PT = dyn_cast<PointerType>(Ty)) {
283 return *Entry = remapScalar(PT);
286 if (
auto *VT = dyn_cast<VectorType>(Ty)) {
287 auto *PT = dyn_cast<PointerType>(VT->getElementType());
289 return *Entry = remapVector(VT);
296 StructType *TyAsStruct = dyn_cast<StructType>(Ty);
297 bool IsUniqued = !TyAsStruct || TyAsStruct->
isLiteral();
304 if (!Seen.
insert(TyAsStruct).second) {
306 return *Entry = Placeholder;
309 bool Changed =
false;
313 Type *NewElem = remapTypeImpl(OldElem, Seen);
314 ElementTypes[
I] = NewElem;
315 Changed |= (OldElem != NewElem);
322 if (
auto *ArrTy = dyn_cast<ArrayType>(Ty))
323 return *Entry = ArrayType::get(ElementTypes[0], ArrTy->getNumElements());
324 if (
auto *FnTy = dyn_cast<FunctionType>(Ty))
325 return *Entry = FunctionType::get(ElementTypes[0],
328 if (
auto *STy = dyn_cast<StructType>(Ty)) {
337 Type **RecursionEntry = &
Map[Ty];
338 if (*RecursionEntry) {
339 auto *Placeholder = cast<StructType>(*RecursionEntry);
340 Placeholder->setBody(ElementTypes, IsPacked);
341 Placeholder->setName(
Name);
342 return *Entry = Placeholder;
350Type *BufferFatPtrTypeLoweringBase::remapType(
Type *SrcTy) {
352 return remapTypeImpl(SrcTy, Visited);
379 auto *ST = dyn_cast<StructType>(Ty);
382 if (!ST->isLiteral() || ST->getNumElements() != 2)
385 dyn_cast<PointerType>(ST->getElementType(0)->getScalarType());
387 dyn_cast<IntegerType>(ST->getElementType(1)->getScalarType());
388 return MaybeRsrc && MaybeOff &&
397 return isBufferFatPtrOrVector(U.get()->getType());
410class StoreFatPtrsAsIntsVisitor
411 :
public InstVisitor<StoreFatPtrsAsIntsVisitor, bool> {
412 BufferFatPtrToIntTypeMap *TypeMap;
427 StoreFatPtrsAsIntsVisitor(BufferFatPtrToIntTypeMap *TypeMap,
LLVMContext &Ctx)
428 : TypeMap(TypeMap), IRB(Ctx) {}
444 if (
Find != ConvertedForStore.end())
447 Value *Cast = IRB.CreatePtrToInt(V, To,
Name +
".int");
448 ConvertedForStore[
V] = Cast;
451 if (
From->getNumContainedTypes() == 0)
455 if (
auto *AT = dyn_cast<ArrayType>(
From)) {
457 Type *ToPart = cast<ArrayType>(To)->getElementType();
458 for (
uint64_t I = 0, E = AT->getArrayNumElements();
I < E; ++
I) {
462 Ret = IRB.CreateInsertValue(Ret, NewField,
I);
465 for (
auto [
Idx, FromPart, ToPart] :
470 Ret = IRB.CreateInsertValue(Ret, NewField,
Idx);
473 ConvertedForStore[
V] =
Ret;
482 Value *Cast = IRB.CreateIntToPtr(V, To,
Name +
".ptr");
485 if (
From->getNumContainedTypes() == 0)
489 if (
auto *AT = dyn_cast<ArrayType>(
From)) {
491 Type *ToPart = cast<ArrayType>(To)->getElementType();
492 for (
uint64_t I = 0, E = AT->getArrayNumElements();
I < E; ++
I) {
496 Ret = IRB.CreateInsertValue(Ret, NewField,
I);
499 for (
auto [
Idx, FromPart, ToPart] :
504 Ret = IRB.CreateInsertValue(Ret, NewField,
Idx);
510bool StoreFatPtrsAsIntsVisitor::processFunction(
Function &
F) {
511 bool Changed =
false;
517 ConvertedForStore.clear();
521bool StoreFatPtrsAsIntsVisitor::visitAllocaInst(
AllocaInst &
I) {
522 Type *Ty =
I.getAllocatedType();
523 Type *NewTy = TypeMap->remapType(Ty);
526 I.setAllocatedType(NewTy);
531 Type *Ty =
I.getSourceElementType();
532 Type *NewTy = TypeMap->remapType(Ty);
537 I.setSourceElementType(NewTy);
538 I.setResultElementType(TypeMap->remapType(
I.getResultElementType()));
542bool StoreFatPtrsAsIntsVisitor::visitLoadInst(
LoadInst &LI) {
544 Type *IntTy = TypeMap->remapType(Ty);
548 IRB.SetInsertPoint(&LI);
549 auto *NLI = cast<LoadInst>(LI.
clone());
550 NLI->mutateType(IntTy);
551 NLI = IRB.Insert(NLI);
555 Value *CastBack = intsToFatPtrs(NLI, IntTy, Ty, NLI->getName());
561bool StoreFatPtrsAsIntsVisitor::visitStoreInst(
StoreInst &SI) {
563 Type *Ty =
V->getType();
564 Type *IntTy = TypeMap->remapType(Ty);
568 IRB.SetInsertPoint(&SI);
569 Value *IntV = fatPtrsToInts(V, Ty, IntTy,
V->getName());
573 SI.setOperand(0, IntV);
579static std::pair<Constant *, Constant *>
581 if (
auto *AZ = dyn_cast<ConstantAggregateZero>(
C))
582 return std::make_pair(AZ->getStructElement(0), AZ->getStructElement(1));
583 if (
auto *SC = dyn_cast<ConstantStruct>(
C))
584 return std::make_pair(SC->getOperand(0), SC->getOperand(1));
591 BufferFatPtrToStructTypeMap *TypeMap;
592 BufferFatPtrToIntTypeMap *IntTypeMap;
606 FatPtrConstMaterializer(BufferFatPtrToStructTypeMap *TypeMap,
608 BufferFatPtrToIntTypeMap *IntTypeMap,
610 : TypeMap(TypeMap), IntTypeMap(IntTypeMap),
611 InternalMapper(UnderlyingMap,
RF_None, TypeMap, this),
DL(
DL) {}
612 virtual ~FatPtrConstMaterializer() =
default;
619 Type *SrcTy =
C->getType();
620 auto *NewTy = dyn_cast<StructType>(TypeMap->remapType(SrcTy));
621 if (
C->isNullValue())
622 return ConstantAggregateZero::getNullValue(NewTy);
623 if (isa<PoisonValue>(
C)) {
628 if (isa<UndefValue>(
C)) {
634 if (isa<GlobalValue>(
C))
636 "fat pointer) values are not supported");
638 if (
auto *VC = dyn_cast<ConstantVector>(
C)) {
640 Constant *NewS = InternalMapper.mapConstant(*S);
644 auto EC =
VC->getType()->getElementCount();
651 auto *NewOp = dyn_cast_or_null<Constant>(InternalMapper.mapValue(*
Op));
665 auto *
CE = dyn_cast<ConstantExpr>(
C);
668 if (
auto *GEPO = dyn_cast<GEPOperator>(
C)) {
670 InternalMapper.mapConstant(*cast<Constant>(GEPO->getPointerOperand()));
673 bool InBounds = GEPO->isInBounds();
680 "Scalable vector or unsized struct in fat pointer GEP");
684 for (
auto [Arg, Multiple] : VariableOffs) {
685 Constant *NewArg = InternalMapper.mapConstant(*cast<Constant>(Arg));
687 if (!Multiple.isOne()) {
688 if (Multiple.isPowerOf2()) {
707 Constant *NewConstOff =
CE->getIntegerValue(OffTy, NewConstOffVal);
712 OffAccum = NewConstOff;
713 bool HasNonNegativeOff =
false;
714 if (
auto *CI = dyn_cast<ConstantInt>(OffAccum)) {
715 HasNonNegativeOff = !CI->isNegative();
718 Off, OffAccum, InBounds && HasNonNegativeOff,
723 if (
auto *PI = dyn_cast<PtrToIntOperator>(CE)) {
725 InternalMapper.mapConstant(*cast<Constant>(PI->getPointerOperand()));
729 unsigned FatPtrWidth =
731 Constant *RsrcInt =
CE->getPtrToInt(Rsrc, SrcTy);
738 RsrcInt, Shift, Width >= FatPtrWidth, Width > FatPtrWidth);
744 if (
CE->getOpcode() == Instruction::IntToPtr) {
745 auto *Arg = cast<Constant>(
CE->getOperand(0));
746 unsigned FatPtrWidth =
749 auto *WantedTy = Arg->getType()->getWithNewBitWidth(FatPtrWidth);
755 Type *RsrcTy = NewTy->getElementType(0);
760 Constant *Rsrc =
CE->getIntToPtr(RsrcInt, RsrcTy);
766 if (
auto *AC = dyn_cast<AddrSpaceCastOperator>(CE)) {
767 unsigned SrcAS = AC->getSrcAddressSpace();
768 unsigned DstAS = AC->getDestAddressSpace();
769 auto *Arg = cast<Constant>(AC->getPointerOperand());
770 auto *NewArg = InternalMapper.mapConstant(*Arg);
778 auto *NullOff =
CE->getNullValue(NewTy->getElementType(1));
782 "Unsupported address space cast for a buffer fat pointer");
787Value *FatPtrConstMaterializer::materialize(
Value *V) {
791 if (
auto *GEPO = dyn_cast<GEPOperator>(
C)) {
795 Type *SrcTy = GEPO->getSourceElementType();
796 Type *NewSrcTy = IntTypeMap->remapType(SrcTy);
797 if (SrcTy != NewSrcTy) {
799 Ops.
reserve(GEPO->getNumOperands());
800 for (
const Use &U : GEPO->operands())
804 GEPO->isInBounds(), GEPO->getInRange());
805 LLVM_DEBUG(
dbgs() <<
"p7-getting GEP: " << *GEPO <<
" becomes " << *NewGEP
807 Value *FurtherMap = materialize(NewGEP);
808 return FurtherMap ? FurtherMap : NewGEP;
815 return materializeBufferFatPtrConst(
C);
823class SplitPtrStructs :
public InstVisitor<SplitPtrStructs, PtrParts> {
866 void processConditionals();
886 :
TM(
TM), ST(
nullptr), IRB(Ctx) {}
914void SplitPtrStructs::copyMetadata(
Value *Dest,
Value *Src) {
915 auto *DestI = dyn_cast<Instruction>(Dest);
916 auto *SrcI = dyn_cast<Instruction>(Src);
921 DestI->copyMetadata(*SrcI);
926 "of something that wasn't rewritten");
927 auto *RsrcEntry = &RsrcParts[
V];
928 auto *OffEntry = &OffParts[
V];
929 if (*RsrcEntry && *OffEntry)
930 return {*RsrcEntry, *OffEntry};
932 if (
auto *
C = dyn_cast<Constant>(V)) {
934 return {*RsrcEntry = Rsrc, *OffEntry =
Off};
938 if (
auto *
I = dyn_cast<Instruction>(V)) {
940 auto [Rsrc,
Off] = visit(*
I);
942 return {*RsrcEntry = Rsrc, *OffEntry =
Off};
945 IRB.SetInsertPoint(*
I->getInsertionPointAfterDef());
946 IRB.SetCurrentDebugLocation(
I->getDebugLoc());
947 }
else if (
auto *
A = dyn_cast<Argument>(V)) {
948 IRB.SetInsertPointPastAllocas(
A->getParent());
949 IRB.SetCurrentDebugLocation(
DebugLoc());
951 Value *Rsrc = IRB.CreateExtractValue(V, 0,
V->getName() +
".rsrc");
952 Value *
Off = IRB.CreateExtractValue(V, 1,
V->getName() +
".off");
953 return {*RsrcEntry = Rsrc, *OffEntry =
Off};
965 while (
auto *
GEP = dyn_cast<GEPOperator>(V))
966 V =
GEP->getPointerOperand();
967 while (
auto *ASC = dyn_cast<AddrSpaceCastOperator>(V))
968 V = ASC->getPointerOperand();
972void SplitPtrStructs::getPossibleRsrcRoots(
Instruction *
I,
975 if (
auto *
PHI = dyn_cast<PHINode>(
I)) {
978 for (
Value *In :
PHI->incoming_values()) {
981 if (isa<PHINode, SelectInst>(In))
982 getPossibleRsrcRoots(cast<Instruction>(In), Roots, Seen);
984 }
else if (
auto *SI = dyn_cast<SelectInst>(
I)) {
985 if (!Seen.
insert(SI).second)
991 if (isa<PHINode, SelectInst>(TrueVal))
992 getPossibleRsrcRoots(cast<Instruction>(TrueVal), Roots, Seen);
993 if (isa<PHINode, SelectInst>(FalseVal))
994 getPossibleRsrcRoots(cast<Instruction>(FalseVal), Roots, Seen);
1000void SplitPtrStructs::processConditionals() {
1006 Value *Rsrc = RsrcParts[
I];
1008 assert(Rsrc && Off &&
"must have visited conditionals by now");
1010 std::optional<Value *> MaybeRsrc;
1011 auto MaybeFoundRsrc = FoundRsrcs.
find(
I);
1012 if (MaybeFoundRsrc != FoundRsrcs.
end()) {
1013 MaybeRsrc = MaybeFoundRsrc->second;
1018 getPossibleRsrcRoots(
I, Roots, Seen);
1021 for (
Value *V : Roots)
1023 for (
Value *V : Seen)
1035 if (Diff.size() == 1) {
1036 Value *RootVal = *Diff.begin();
1040 MaybeRsrc = std::get<0>(getPtrParts(RootVal));
1042 MaybeRsrc = RootVal;
1047 if (
auto *
PHI = dyn_cast<PHINode>(
I)) {
1050 IRB.SetInsertPoint(*
PHI->getInsertionPointAfterDef());
1051 IRB.SetCurrentDebugLocation(
PHI->getDebugLoc());
1053 NewRsrc = *MaybeRsrc;
1056 auto *RsrcPHI = IRB.CreatePHI(RsrcTy,
PHI->getNumIncomingValues());
1057 RsrcPHI->takeName(Rsrc);
1058 for (
auto [V, BB] :
llvm::zip(
PHI->incoming_values(),
PHI->blocks())) {
1059 Value *VRsrc = std::get<0>(getPtrParts(V));
1060 RsrcPHI->addIncoming(VRsrc, BB);
1062 copyMetadata(RsrcPHI,
PHI);
1067 auto *NewOff = IRB.CreatePHI(OffTy,
PHI->getNumIncomingValues());
1069 for (
auto [V, BB] :
llvm::zip(
PHI->incoming_values(),
PHI->blocks())) {
1070 assert(OffParts.count(V) &&
"An offset part had to be created by now");
1071 Value *VOff = std::get<1>(getPtrParts(V));
1072 NewOff->addIncoming(VOff, BB);
1074 copyMetadata(NewOff,
PHI);
1080 ConditionalTemps.push_back(cast<Instruction>(Rsrc));
1081 ConditionalTemps.push_back(cast<Instruction>(Off));
1083 Off->replaceAllUsesWith(NewOff);
1087 for (
Value *V : Seen)
1088 FoundRsrcs[cast<Instruction>(V)] = NewRsrc;
1089 }
else if (isa<SelectInst>(
I)) {
1091 ConditionalTemps.push_back(cast<Instruction>(Rsrc));
1093 for (
Value *V : Seen)
1094 FoundRsrcs[cast<Instruction>(V)] = *MaybeRsrc;
1102void SplitPtrStructs::killAndReplaceSplitInstructions(
1105 I->eraseFromParent();
1108 if (!SplitUsers.contains(
I))
1113 for (
auto *Dbg : Dbgs) {
1114 IRB.SetInsertPoint(Dbg);
1115 auto &
DL =
I->getModule()->getDataLayout();
1117 "We should've RAUW'd away loads, stores, etc. at this point");
1118 auto *OffDbg = cast<DbgValueInst>(
Dbg->clone());
1119 copyMetadata(OffDbg, Dbg);
1120 auto [Rsrc,
Off] = getPtrParts(
I);
1122 int64_t RsrcSz =
DL.getTypeSizeInBits(Rsrc->
getType());
1123 int64_t OffSz =
DL.getTypeSizeInBits(
Off->getType());
1125 std::optional<DIExpression *> RsrcExpr =
1128 std::optional<DIExpression *> OffExpr =
1132 OffDbg->setExpression(*OffExpr);
1133 OffDbg->replaceVariableLocationOp(
I, Off);
1136 OffDbg->deleteValue();
1139 Dbg->setExpression(*RsrcExpr);
1140 Dbg->replaceVariableLocationOp(
I, Rsrc);
1147 I->replaceUsesWithIf(Poison, [&](
const Use &U) ->
bool {
1148 if (
const auto *UI = dyn_cast<Instruction>(
U.getUser()))
1149 return SplitUsers.contains(UI);
1153 if (
I->use_empty()) {
1154 I->eraseFromParent();
1157 IRB.SetInsertPoint(*
I->getInsertionPointAfterDef());
1158 IRB.SetCurrentDebugLocation(
I->getDebugLoc());
1159 auto [Rsrc,
Off] = getPtrParts(
I);
1165 I->replaceAllUsesWith(
Struct);
1166 I->eraseFromParent();
1178 case AtomicOrdering::Release:
1179 case AtomicOrdering::AcquireRelease:
1180 case AtomicOrdering::SequentiallyConsistent:
1181 IRB.CreateFence(AtomicOrdering::Release, SSID);
1191 case AtomicOrdering::Acquire:
1192 case AtomicOrdering::AcquireRelease:
1193 case AtomicOrdering::SequentiallyConsistent:
1194 IRB.CreateFence(AtomicOrdering::Acquire, SSID);
1205 IRB.SetInsertPoint(
I);
1207 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1210 Args.push_back(Arg);
1211 Args.push_back(Rsrc);
1212 Args.push_back(Off);
1213 insertPreMemOpFence(Order, SSID);
1217 Args.push_back(IRB.getInt32(0));
1221 (isa<LoadInst>(
I) &&
I->getMetadata(LLVMContext::MD_invariant_load));
1222 bool IsNonTemporal =
I->getMetadata(LLVMContext::MD_nontemporal);
1224 bool IsOneWayAtomic =
1225 !isa<AtomicRMWInst>(
I) && Order != AtomicOrdering::NotAtomic;
1228 if (IsNonTemporal && !IsInvariant)
1234 Args.push_back(IRB.getInt32(Aux));
1237 if (isa<LoadInst>(
I))
1239 IID = Intrinsic::amdgcn_raw_ptr_buffer_load;
1240 else if (isa<StoreInst>(
I))
1241 IID = Intrinsic::amdgcn_raw_ptr_buffer_store;
1242 else if (
auto *RMW = dyn_cast<AtomicRMWInst>(
I)) {
1243 switch (RMW->getOperation()) {
1245 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap;
1248 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_add;
1251 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub;
1254 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_and;
1257 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_or;
1260 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor;
1263 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax;
1266 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin;
1269 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax;
1272 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin;
1275 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd;
1278 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax;
1281 IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin;
1285 "buffer resources and should've been expanded away");
1290 "should've been expanded away");
1295 "buffer resources and should've ben expanded away");
1302 auto *
Call = IRB.CreateIntrinsic(IID, Ty, Args);
1303 copyMetadata(Call,
I);
1304 setAlign(Call, Alignment, Arg ? 1 : 0);
1307 insertPostMemOpFence(Order, SSID);
1310 SplitUsers.insert(
I);
1311 I->replaceAllUsesWith(Call);
1316 return {
nullptr,
nullptr};
1321 return {
nullptr,
nullptr};
1325 return {
nullptr,
nullptr};
1330 return {
nullptr,
nullptr};
1331 Value *Arg =
SI.getValueOperand();
1332 handleMemoryInst(&SI, Arg,
SI.getPointerOperand(), Arg->
getType(),
1333 SI.getAlign(),
SI.getOrdering(),
SI.isVolatile(),
1334 SI.getSyncScopeID());
1335 return {
nullptr,
nullptr};
1340 return {
nullptr,
nullptr};
1345 return {
nullptr,
nullptr};
1353 return {
nullptr,
nullptr};
1354 IRB.SetInsertPoint(&AI);
1359 bool IsNonTemporal = AI.
getMetadata(LLVMContext::MD_nontemporal);
1361 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1362 insertPreMemOpFence(Order, SSID);
1370 IRB.CreateIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap, Ty,
1372 Off, IRB.getInt32(0), IRB.getInt32(Aux)});
1373 copyMetadata(Call, &AI);
1375 Call->takeName(&AI);
1376 insertPostMemOpFence(Order, SSID);
1379 Res = IRB.CreateInsertValue(Res, Call, 0);
1382 Res = IRB.CreateInsertValue(Res, Succeeded, 1);
1384 SplitUsers.insert(&AI);
1386 return {
nullptr,
nullptr};
1392 return {
nullptr,
nullptr};
1393 IRB.SetInsertPoint(&
GEP);
1395 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1398 bool InBounds =
GEP.isInBounds();
1402 GEP.setOperand(
GEP.getPointerOperandIndex(),
1408 GEP.setOperand(
GEP.getPointerOperandIndex(),
Ptr);
1409 Value *OffAccum =
nullptr;
1412 for (
auto [Arg, Multiple] : VariableOffs) {
1413 if (
auto *OffVecTy = dyn_cast<VectorType>(OffTy))
1415 Arg = IRB.CreateVectorSplat(OffVecTy->getElementCount(), Arg);
1416 Arg = IRB.CreateIntCast(Arg, OffTy,
true);
1417 if (!Multiple.isOne()) {
1418 if (Multiple.isPowerOf2())
1419 Arg = IRB.CreateShl(Arg, Multiple.logBase2(),
"", InBounds,
1422 Arg = IRB.CreateMul(Arg, ConstantExpr::getIntegerValue(OffTy, Multiple),
1423 "", InBounds, InBounds);
1426 OffAccum = IRB.CreateAdd(OffAccum, Arg,
"", InBounds,
1431 if (!ConstOffVal.
isZero()) {
1432 Constant *ConstOff = ConstantExpr::getIntegerValue(OffTy, ConstOffVal);
1434 OffAccum = IRB.CreateAdd(OffAccum, ConstOff,
"", InBounds,
1437 OffAccum = ConstOff;
1441 SplitUsers.insert(&
GEP);
1445 bool HasNonNegativeOff =
false;
1446 if (
auto *CI = dyn_cast<ConstantInt>(OffAccum)) {
1447 HasNonNegativeOff = !CI->isNegative();
1453 NewOff = IRB.CreateAdd(Off, OffAccum,
"",
1454 InBounds && HasNonNegativeOff,
1457 copyMetadata(NewOff, &
GEP);
1459 SplitUsers.insert(&
GEP);
1460 return {Rsrc, NewOff};
1466 return {
nullptr,
nullptr};
1467 IRB.SetInsertPoint(&PI);
1472 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1478 RsrcInt = ConstantExpr::getIntegerValue(ResTy,
APInt::getZero(Width));
1480 RsrcInt = IRB.CreatePtrToInt(Rsrc, ResTy, PI.
getName() +
".rsrc");
1481 copyMetadata(RsrcInt, &PI);
1483 Value *Shl = IRB.CreateShl(
1486 Width >= FatPtrWidth, Width > FatPtrWidth);
1488 IRB.CreateIntCast(Off, ResTy,
false, PI.
getName() +
".off");
1489 Value *Res = IRB.CreateOr(Shl, OffCast);
1491 SplitUsers.insert(&PI);
1493 return {
nullptr,
nullptr};
1498 return {
nullptr,
nullptr};
1499 IRB.SetInsertPoint(&IP);
1508 Type *RsrcTy =
RetTy->getElementType(0);
1510 Value *RsrcPart = IRB.CreateLShr(
1513 Value *RsrcInt = IRB.CreateIntCast(RsrcPart, RsrcIntTy,
false);
1514 Value *Rsrc = IRB.CreateIntToPtr(RsrcInt, RsrcTy, IP.
getName() +
".rsrc");
1516 IRB.CreateIntCast(
Int, OffTy,
false, IP.
getName() +
".off");
1518 copyMetadata(Rsrc, &IP);
1519 SplitUsers.insert(&IP);
1525 return {
nullptr,
nullptr};
1526 IRB.SetInsertPoint(&
I);
1529 if (
In->getType() ==
I.getType()) {
1530 auto [Rsrc,
Off] = getPtrParts(In);
1531 SplitUsers.insert(&
I);
1536 "buffer fat pointers (addrspace 7)");
1537 Type *OffTy = cast<StructType>(
I.getType())->getElementType(1);
1539 SplitUsers.insert(&
I);
1540 return {
In, ZeroOff};
1546 return {
nullptr,
nullptr};
1548 IRB.SetInsertPoint(&Cmp);
1551 assert((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
1552 "Pointer comparison is only equal or unequal");
1553 auto [LhsRsrc, LhsOff] = getPtrParts(Lhs);
1554 auto [RhsRsrc, RhsOff] = getPtrParts(Rhs);
1556 IRB.CreateICmp(Pred, LhsRsrc, RhsRsrc,
Cmp.getName() +
".rsrc");
1557 copyMetadata(RsrcCmp, &Cmp);
1558 Value *OffCmp = IRB.CreateICmp(Pred, LhsOff, RhsOff,
Cmp.getName() +
".off");
1559 copyMetadata(OffCmp, &Cmp);
1561 Value *Res =
nullptr;
1562 if (Pred == ICmpInst::ICMP_EQ)
1563 Res = IRB.CreateAnd(RsrcCmp, OffCmp);
1564 else if (Pred == ICmpInst::ICMP_NE)
1565 Res = IRB.CreateOr(RsrcCmp, OffCmp);
1566 copyMetadata(Res, &Cmp);
1568 SplitUsers.insert(&Cmp);
1569 Cmp.replaceAllUsesWith(Res);
1570 return {
nullptr,
nullptr};
1575 return {
nullptr,
nullptr};
1576 IRB.SetInsertPoint(&
I);
1577 auto [Rsrc,
Off] = getPtrParts(
I.getOperand(0));
1579 Value *RsrcRes = IRB.CreateFreeze(Rsrc,
I.getName() +
".rsrc");
1580 copyMetadata(RsrcRes, &
I);
1581 Value *OffRes = IRB.CreateFreeze(Off,
I.getName() +
".off");
1582 copyMetadata(OffRes, &
I);
1583 SplitUsers.insert(&
I);
1584 return {RsrcRes, OffRes};
1589 return {
nullptr,
nullptr};
1590 IRB.SetInsertPoint(&
I);
1591 Value *Vec =
I.getVectorOperand();
1593 auto [Rsrc,
Off] = getPtrParts(Vec);
1595 Value *RsrcRes = IRB.CreateExtractElement(Rsrc,
Idx,
I.getName() +
".rsrc");
1596 copyMetadata(RsrcRes, &
I);
1597 Value *OffRes = IRB.CreateExtractElement(Off,
Idx,
I.getName() +
".off");
1598 copyMetadata(OffRes, &
I);
1599 SplitUsers.insert(&
I);
1600 return {RsrcRes, OffRes};
1607 return {
nullptr,
nullptr};
1608 IRB.SetInsertPoint(&
I);
1609 Value *Vec =
I.getOperand(0);
1610 Value *Elem =
I.getOperand(1);
1612 auto [VecRsrc, VecOff] = getPtrParts(Vec);
1613 auto [ElemRsrc, ElemOff] = getPtrParts(Elem);
1616 IRB.CreateInsertElement(VecRsrc, ElemRsrc,
Idx,
I.getName() +
".rsrc");
1617 copyMetadata(RsrcRes, &
I);
1619 IRB.CreateInsertElement(VecOff, ElemOff,
Idx,
I.getName() +
".off");
1620 copyMetadata(OffRes, &
I);
1621 SplitUsers.insert(&
I);
1622 return {RsrcRes, OffRes};
1628 return {
nullptr,
nullptr};
1629 IRB.SetInsertPoint(&
I);
1631 Value *V1 =
I.getOperand(0);
1634 auto [V1Rsrc, V1Off] = getPtrParts(V1);
1635 auto [V2Rsrc, V2Off] = getPtrParts(V2);
1638 IRB.CreateShuffleVector(V1Rsrc, V2Rsrc, Mask,
I.getName() +
".rsrc");
1639 copyMetadata(RsrcRes, &
I);
1641 IRB.CreateShuffleVector(V1Off, V2Off, Mask,
I.getName() +
".off");
1642 copyMetadata(OffRes, &
I);
1643 SplitUsers.insert(&
I);
1644 return {RsrcRes, OffRes};
1649 return {
nullptr,
nullptr};
1650 IRB.SetInsertPoint(*
PHI.getInsertionPointAfterDef());
1656 Value *TmpRsrc = IRB.CreateExtractValue(&
PHI, 0,
PHI.getName() +
".rsrc");
1657 Value *TmpOff = IRB.CreateExtractValue(&
PHI, 1,
PHI.getName() +
".off");
1658 Conditionals.push_back(&
PHI);
1659 SplitUsers.insert(&
PHI);
1660 return {TmpRsrc, TmpOff};
1665 return {
nullptr,
nullptr};
1666 IRB.SetInsertPoint(&SI);
1669 Value *True =
SI.getTrueValue();
1670 Value *False =
SI.getFalseValue();
1671 auto [TrueRsrc, TrueOff] = getPtrParts(True);
1672 auto [FalseRsrc, FalseOff] = getPtrParts(False);
1675 IRB.CreateSelect(
Cond, TrueRsrc, FalseRsrc,
SI.getName() +
".rsrc", &SI);
1676 copyMetadata(RsrcRes, &SI);
1677 Conditionals.push_back(&SI);
1679 IRB.CreateSelect(
Cond, TrueOff, FalseOff,
SI.getName() +
".off", &SI);
1680 copyMetadata(OffRes, &SI);
1681 SplitUsers.insert(&SI);
1682 return {RsrcRes, OffRes};
1693 case Intrinsic::ptrmask:
1694 case Intrinsic::invariant_start:
1695 case Intrinsic::invariant_end:
1696 case Intrinsic::launder_invariant_group:
1697 case Intrinsic::strip_invariant_group:
1707 case Intrinsic::ptrmask: {
1710 return {
nullptr,
nullptr};
1712 IRB.SetInsertPoint(&
I);
1713 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1714 if (
Mask->getType() !=
Off->getType())
1716 "pointer (data layout not set up correctly?)");
1717 Value *OffRes = IRB.CreateAnd(Off, Mask,
I.getName() +
".off");
1718 copyMetadata(OffRes, &
I);
1719 SplitUsers.insert(&
I);
1720 return {Rsrc, OffRes};
1724 case Intrinsic::invariant_start: {
1727 return {
nullptr,
nullptr};
1728 IRB.SetInsertPoint(&
I);
1729 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1731 auto *NewRsrc = IRB.CreateIntrinsic(IID, {NewTy}, {
I.getOperand(0), Rsrc});
1732 copyMetadata(NewRsrc, &
I);
1734 SplitUsers.insert(&
I);
1735 I.replaceAllUsesWith(NewRsrc);
1736 return {
nullptr,
nullptr};
1738 case Intrinsic::invariant_end: {
1739 Value *RealPtr =
I.getArgOperand(2);
1741 return {
nullptr,
nullptr};
1742 IRB.SetInsertPoint(&
I);
1743 Value *RealRsrc = getPtrParts(RealPtr).first;
1744 Value *InvPtr =
I.getArgOperand(0);
1746 Value *NewRsrc = IRB.CreateIntrinsic(IID, {RealRsrc->
getType()},
1747 {InvPtr,
Size, RealRsrc});
1748 copyMetadata(NewRsrc, &
I);
1750 SplitUsers.insert(&
I);
1751 I.replaceAllUsesWith(NewRsrc);
1752 return {
nullptr,
nullptr};
1754 case Intrinsic::launder_invariant_group:
1755 case Intrinsic::strip_invariant_group: {
1758 return {
nullptr,
nullptr};
1759 IRB.SetInsertPoint(&
I);
1760 auto [Rsrc,
Off] = getPtrParts(
Ptr);
1761 Value *NewRsrc = IRB.CreateIntrinsic(IID, {Rsrc->
getType()}, {Rsrc});
1762 copyMetadata(NewRsrc, &
I);
1764 SplitUsers.insert(&
I);
1765 return {NewRsrc,
Off};
1768 return {
nullptr,
nullptr};
1771void SplitPtrStructs::processFunction(
Function &
F) {
1774 LLVM_DEBUG(
dbgs() <<
"Splitting pointer structs in function: " <<
F.getName()
1777 Originals.push_back(&
I);
1779 auto [Rsrc,
Off] = visit(
I);
1780 assert(((Rsrc && Off) || (!Rsrc && !Off)) &&
1781 "Can't have a resource but no offset");
1783 RsrcParts[
I] = Rsrc;
1787 processConditionals();
1788 killAndReplaceSplitInstructions(Originals);
1794 Conditionals.clear();
1795 ConditionalTemps.clear();
1799class AMDGPULowerBufferFatPointers :
public ModulePass {
1819 BufferFatPtrToStructTypeMap *TypeMap) {
1820 bool HasFatPointers =
false;
1823 HasFatPointers |= (
I.getType() != TypeMap->remapType(
I.getType()));
1824 for (
const Use &U :
I.operands())
1825 if (
auto *
C = dyn_cast<Constant>(U.get()))
1829 return HasFatPointers;
1833 BufferFatPtrToStructTypeMap *TypeMap) {
1834 Type *Ty =
F.getFunctionType();
1835 return Ty != TypeMap->remapType(Ty);
1852 while (!OldF->
empty()) {
1864 {Attribute::Dereferenceable, Attribute::DereferenceableOrNull,
1865 Attribute::NoAlias, Attribute::NoCapture, Attribute::NoFree,
1866 Attribute::NonNull, Attribute::NullPointerIsValid, Attribute::ReadNone,
1867 Attribute::ReadOnly, Attribute::WriteOnly}) {
1874 CloneMap[&NewArg] = &OldArg;
1876 Type *OldArgTy = OldArg.getType(), *NewArgTy = NewArg.
getType();
1879 OldArg.replaceAllUsesWith(&NewArg);
1884 if (OldArgTy != NewArgTy && !IsIntrinsic)
1900 CloneMap[&BB] = &BB;
1907 bool Changed =
false;
1914 BufferFatPtrToStructTypeMap StructTM(
DL);
1915 BufferFatPtrToIntTypeMap IntTM(
DL);
1919 "space (7) are not supported");
1920 Type *VT = GV.getValueType();
1921 if (VT != StructTM.remapType(VT))
1923 "(address space 7 pointers) are unsupported. Use "
1924 "buffer resource pointers (address space 8) instead.");
1927 StoreFatPtrsAsIntsVisitor MemOpsRewrite(&IntTM,
M.getContext());
1931 Changed |= MemOpsRewrite.processFunction(
F);
1932 if (InterfaceChange || BodyChanges)
1933 NeedsRemap.
push_back(std::make_pair(&
F, InterfaceChange));
1935 if (NeedsRemap.
empty())
1942 FatPtrConstMaterializer Materializer(&StructTM, CloneMap, &IntTM,
DL);
1945 for (
auto [
F, InterfaceChange] : NeedsRemap) {
1947 if (InterfaceChange)
1949 F, cast<FunctionType>(StructTM.remapType(
F->getFunctionType())),
1953 LowerInFuncs.remapFunction(*NewF);
1958 if (InterfaceChange) {
1959 F->replaceAllUsesWith(NewF);
1960 F->eraseFromParent();
1968 SplitPtrStructs Splitter(
M.getContext(), &
TM);
1970 Splitter.processFunction(*
F);
1973 F->eraseFromParent();
1977 F->replaceAllUsesWith(*NewF);
1983bool AMDGPULowerBufferFatPointers::runOnModule(
Module &M) {
1989char AMDGPULowerBufferFatPointers::ID = 0;
1993void AMDGPULowerBufferFatPointers::getAnalysisUsage(
AnalysisUsage &AU)
const {
1997#define PASS_DESC "Lower buffer fat pointer operations to buffer resources"
2006 return new AMDGPULowerBufferFatPointers();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Function * moveFunctionAdaptingType(Function *OldF, FunctionType *NewTy, ValueToValueMapTy &CloneMap)
Move the body of OldF into a new function, returning it.
static void makeCloneInPraceMap(Function *F, ValueToValueMapTy &CloneMap)
static bool isBufferFatPtrOrVector(Type *Ty)
static bool isSplitFatPtr(Type *Ty)
std::pair< Value *, Value * > PtrParts
static bool hasFatPointerInterface(const Function &F, BufferFatPtrToStructTypeMap *TypeMap)
static bool isRemovablePointerIntrinsic(Intrinsic::ID IID)
Returns true if this intrinsic needs to be removed when it is applied to ptr addrspace(7) values.
static bool containsBufferFatPointers(const Function &F, BufferFatPtrToStructTypeMap *TypeMap)
Returns true if there are values that have a buffer fat pointer in them, which means we'll need to pe...
static Value * rsrcPartRoot(Value *V)
Returns the instruction that defines the resource part of the value V.
static constexpr unsigned BufferOffsetWidth
static bool isBufferFatPtrConst(Constant *C)
static std::pair< Constant *, Constant * > splitLoweredFatBufferConst(Constant *C)
Return the ptr addrspace(8) and i32 (resource and offset parts) in a lowered buffer fat pointer const...
The AMDGPU TargetMachine interface definition for hw codegen targets.
Expand Atomic instructions
Atomic ordering constants.
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void clear(coro::Shape &Shape)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
AMD GCN specific subclass of TargetSubtarget.
static const T * Find(StringRef S, ArrayRef< T > A)
Find KV in array using binary search.
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines generic set operations that may be used on set's of different types,...
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Target-Independent Code Generator Pass Configuration Options pass.
Class for arbitrary precision integers.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
A container for analyses that lazily runs them and caches their results.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getNewValOperand()
AtomicOrdering getMergedOrdering() const
Returns a single ordering which is at least as strong as both the success and failure orderings for t...
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
Value * getCompareOperand()
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
Value * getPointerOperand()
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
AttributeSet removeAttributes(LLVMContext &C, const AttributeMask &AttrsToRemove) const
Remove the specified attributes from this set.
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
void removeFromParent()
Unlink 'this' from the containing function, but do not delete it.
void insertInto(Function *Parent, BasicBlock *InsertBefore=nullptr)
Insert unlinked basic block into a function.
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static Constant * getMul(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getShl(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
Implements a dense probed hash-table based set.
This class represents a freeze function that returns random concrete value if an operand is either a ...
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & front() const
iterator_range< arg_iterator > args()
bool IsNewDbgInfoFormat
Is this function using intrinsics to record the position of debugging information,...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void updateAfterNameChange()
Update internal caches that depend on the function name (such as the intrinsic ID and libcall cache).
Type * getReturnType() const
Returns the type of the ret val.
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void copyMetadata(const GlobalObject *Src, unsigned Offset)
Copy metadata from Src, adjusting offsets by Offset.
LinkageTypes getLinkage() const
void setDLLStorageClass(DLLStorageClassTypes C)
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
DLLStorageClassTypes getDLLStorageClass() const
This instruction compares its operands according to the predicate given to the constructor.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This instruction inserts a single (scalar) element into a VectorType value.
Base class for instruction visitors.
RetTy visitFreezeInst(FreezeInst &I)
RetTy visitPtrToIntInst(PtrToIntInst &I)
RetTy visitExtractElementInst(ExtractElementInst &I)
RetTy visitIntrinsicInst(IntrinsicInst &I)
RetTy visitShuffleVectorInst(ShuffleVectorInst &I)
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
RetTy visitIntToPtrInst(IntToPtrInst &I)
RetTy visitPHINode(PHINode &I)
RetTy visitStoreInst(StoreInst &I)
RetTy visitInsertElementInst(InsertElementInst &I)
RetTy visitAtomicRMWInst(AtomicRMWInst &I)
RetTy visitAddrSpaceCastInst(AddrSpaceCastInst &I)
RetTy visitAllocaInst(AllocaInst &I)
RetTy visitICmpInst(ICmpInst &I)
RetTy visitSelectInst(SelectInst &I)
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
void visitInstruction(Instruction &I)
RetTy visitLoadInst(LoadInst &I)
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
This class represents a cast from an integer to a pointer.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Type * getPointerOperandType() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Align getAlign() const
Return the alignment of the access that is being performed.
This class implements a map that also provides access to all stored values in a deterministic order.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
const FunctionListType & getFunctionList() const
Get the Module's list of functions (constant).
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
This class represents a cast from a pointer to an integer.
Value * getPointerOperand()
Gets the pointer operand.
This class represents the LLVM 'select' instruction.
This instruction constructs a fixed permutation of two input vectors.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
bool isLiteral() const
Return true if this type is uniqued by structural equivalence, false if it is a struct definition.
Type * getElementType(unsigned N) const
Primary interface to the complete machine description for the target machine.
Target-Independent Code Generator Pass Configuration Options.
TMC & getTM() const
Get the right type of TargetMachine for this target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
Type * getArrayElementType() const
ArrayRef< Type * > subtypes() const
unsigned getNumContainedTypes() const
Return the number of types in the derived type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
This is a class that can be implemented by clients to remap types when cloning constants and instruct...
virtual Type * remapType(Type *SrcTy)=0
The client should implement this method if they want to remap types while mapping values.
Context for (re-)mapping values (and metadata).
This is a class that can be implemented by clients to materialize Values on demand.
virtual Value * materialize(Value *V)=0
This method can be implemented to generate a mapped Value on demand.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
void mutateType(Type *Ty)
Mutate the type of this Value to be of the specified type.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
self_iterator getIterator()
iterator insertAfter(iterator where, pointer New)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
std::optional< Function * > remangleIntrinsicFunction(Function *F)
bool match(Val *V, const Pattern &P)
@ CE
Windows NT (Windows on ARM)
AssignmentMarkerRange getAssignmentMarkers(DIAssignID *ID)
Return a range of dbg.assign intrinsics which use \ID as an operand.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
ModulePass * createAMDGPULowerBufferFatPointersPass()
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source)
Copy the metadata from the source instruction to the destination (the replacement for the source inst...
bool set_is_subset(const S1Ty &S1, const S2Ty &S2)
set_is_subset(A, B) - Return true iff A in B
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
void findDbgValues(SmallVectorImpl< DbgValueInst * > &DbgValues, Value *V, SmallVectorImpl< DbgVariableRecord * > *DbgVariableRecords=nullptr)
Finds the llvm.dbg.value intrinsics describing a value.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
char & AMDGPULowerBufferFatPointersID
Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
AtomicOrdering
Atomic ordering for LLVM's memory model.
S1Ty set_difference(const S1Ty &S1, const S2Ty &S2)
set_difference(A, B) - Return A - B
void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &)
Constant * ConstantFoldIntegerCast(Constant *C, Type *DestTy, bool IsSigned, const DataLayout &DL)
Constant fold a zext, sext or trunc, depending on IsSigned and whether the DestTy is wider or narrowe...
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
This struct is a compact representation of a valid (non-zero power of two) alignment.