110#define DEBUG_TYPE "amdgpu-sw-lower-lds"
111#define COV5_HIDDEN_DYN_LDS_SIZE_ARG 15
120struct LDSAccessTypeInfo {
128struct KernelLDSParameters {
132 LDSAccessTypeInfo DirectAccess;
133 LDSAccessTypeInfo IndirectAccess;
135 LDSToReplacementIndicesMap;
143struct NonKernelLDSParameters {
150struct AsanInstrumentInfo {
156struct FunctionsAndLDSAccess {
164class AMDGPUSwLowerLDS {
166 AMDGPUSwLowerLDS(
Module &
Mod, DomTreeCallback Callback)
167 : M(
Mod), IRB(M), DTCallback(Callback) {}
169 void getUsesOfLDSByNonKernels();
170 void getNonKernelsWithLDSInstructions(
const CallGraph &CG);
175 void buildSwLDSGlobal(
Function *Func);
176 void buildSwDynLDSGlobal(
Function *Func);
177 void populateSwMetadataGlobal(
Function *Func);
178 void populateSwLDSAttributeAndMetadata(
Function *Func);
179 void populateLDSToReplacementIndicesMap(
Function *Func);
180 void getLDSMemoryInstructions(
Function *Func,
182 void replaceKernelLDSAccesses(
Function *Func);
183 Value *getTranslatedGlobalMemoryPtrOfLDS(
Value *LoadMallocPtr,
Value *LDSPtr);
186 void translateLDSMemoryOperationsToGlobalMemory(
191 void buildNonKernelLDSOffsetTable(NonKernelLDSParameters &NKLDSParams);
192 void buildNonKernelLDSBaseTable(NonKernelLDSParameters &NKLDSParams);
194 getAddressesOfVariablesInKernel(
Function *Func,
196 void lowerNonKernelLDSAccesses(
Function *Func,
198 NonKernelLDSParameters &NKLDSParams);
200 updateMallocSizeForDynamicLDS(
Function *Func,
Value **CurrMallocSize,
201 Value *HiddenDynLDSSize,
208 DomTreeCallback DTCallback;
209 FunctionsAndLDSAccess FuncLDSAccessInfo;
210 AsanInstrumentInfo AsanInfo;
213template <
typename T>
SetVector<T> sortByName(std::vector<T> &&
V) {
216 sort(
V, [](
const auto *L,
const auto *R) {
217 return L->getName() < R->getName();
226 std::vector<GlobalVariable *>(Variables.
begin(), Variables.
end()));
234 if (Kernels.size() > UINT32_MAX) {
238 sortByName(std::vector<Function *>(Kernels.begin(), Kernels.end()));
239 for (
size_t i = 0; i < Kernels.size(); i++) {
244 Func->setMetadata(
"llvm.amdgcn.lds.kernel.id",
247 return OrderedKernels;
251static bool isLDSMemoryInstruction(
const Instruction &Inst) {
261 unsigned SrcAS = ASC->getSrcAddressSpace();
262 unsigned DstAS = ASC->getDestAddressSpace();
277void AMDGPUSwLowerLDS::getNonKernelsWithLDSInstructions(
const CallGraph &CG) {
282 auto HasLDSInstructions = [&](
Function *
F) {
283 auto [It, Inserted] = HasLDSInstructionsCache.
try_emplace(
F,
false);
287 FuncLDSAccessInfo.NonKernelsWithLDSInstructions.
insert(
F);
296 F.hasAddressTaken(
nullptr,
false,
305 bool ReachesLDSInstructions =
false;
306 bool SeenUnknownCall =
false;
309 auto Visit = [&](
Function *Callee) {
310 if (Callee->isDeclaration() ||
isKernel(*Callee) ||
311 !Visited.
insert(Callee).second)
313 ReachesLDSInstructions |= HasLDSInstructions(Callee);
317 while (!WorkList.
empty()) {
320 if (!CallRecord.second)
322 if (
Function *Callee = CallRecord.second->getFunction()) {
328 SeenUnknownCall =
true;
329 for (
Function *PotentialCallee : AddressTakenFuncs)
330 Visit(PotentialCallee);
334 if (ReachesLDSInstructions)
335 FuncLDSAccessInfo.KernelsWithIndirectLDSAccess.
insert(&Kernel);
339void AMDGPUSwLowerLDS::getUsesOfLDSByNonKernels() {
340 for (
GlobalVariable *GV : FuncLDSAccessInfo.AllNonKernelLDSAccess) {
348 FuncLDSAccessInfo.NonKernelToLDSAccessMap[
F].
insert(GV);
362 ConstantInt::get(IntTy, Address + 1));
363 GV->
setMetadata(LLVMContext::MD_absolute_symbol, MetadataNode);
374 Func->addFnAttr(
"amdgpu-lds-size", Buffer);
383 Intrinsic::donothing, {});
385 Value *UseInstance[] = {SGV};
387 Builder.CreateCall(Decl, {},
391void AMDGPUSwLowerLDS::buildSwLDSGlobal(
Function *Func) {
394 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
402 LDSParams.SwLDS->setSanitizerMetadata(MD);
405void AMDGPUSwLowerLDS::buildSwDynLDSGlobal(
Function *Func) {
407 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
408 if (LDSParams.DirectAccess.DynamicLDSGlobals.empty() &&
409 LDSParams.IndirectAccess.DynamicLDSGlobals.empty())
415 "llvm.amdgcn." + Func->getName() +
".dynlds",
nullptr,
417 markUsedByKernel(Func, LDSParams.SwDynLDS);
420 LDSParams.SwDynLDS->setSanitizerMetadata(MD);
423void AMDGPUSwLowerLDS::populateSwLDSAttributeAndMetadata(
Function *Func) {
424 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
425 bool IsDynLDSUsed = LDSParams.SwDynLDS;
427 recordLDSAbsoluteAddress(M, LDSParams.SwLDS, 0);
428 addLDSSizeAttribute(Func,
Offset, IsDynLDSUsed);
429 if (LDSParams.SwDynLDS)
430 recordLDSAbsoluteAddress(M, LDSParams.SwDynLDS,
Offset);
433void AMDGPUSwLowerLDS::populateSwMetadataGlobal(
Function *Func) {
436 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
437 auto &Ctx = M.getContext();
438 auto &
DL = M.getDataLayout();
439 std::vector<Type *> Items;
441 std::vector<Constant *> Initializers;
442 Align MaxAlignment(1);
445 MaxAlignment = std::max(MaxAlignment, GVAlign);
448 for (
GlobalVariable *GV : LDSParams.DirectAccess.StaticLDSGlobals)
449 UpdateMaxAlignment(GV);
451 for (
GlobalVariable *GV : LDSParams.DirectAccess.DynamicLDSGlobals)
452 UpdateMaxAlignment(GV);
454 for (
GlobalVariable *GV : LDSParams.IndirectAccess.StaticLDSGlobals)
455 UpdateMaxAlignment(GV);
457 for (
GlobalVariable *GV : LDSParams.IndirectAccess.DynamicLDSGlobals)
458 UpdateMaxAlignment(GV);
463 MDItemOS <<
"llvm.amdgcn.sw.lds." << Func->getName() <<
".md.item";
467 uint32_t &MallocSize = LDSParams.MallocSize;
469 int AsanScale = AsanInfo.Scale;
470 auto buildInitializerForSwLDSMD =
472 for (
auto &GV : LDSGlobals) {
475 UniqueLDSGlobals.
insert(GV);
478 const uint64_t SizeInBytes =
DL.getTypeAllocSize(Ty);
479 Items.push_back(LDSItemTy);
480 Constant *ItemStartOffset = ConstantInt::get(Int32Ty, MallocSize);
481 Constant *SizeInBytesConst = ConstantInt::get(Int32Ty, SizeInBytes);
486 MallocSize += SizeInBytes;
488 LDSParams.RedzoneOffsetAndSizeVector.emplace_back(MallocSize,
490 MallocSize += RightRedzoneSize;
493 alignTo(SizeInBytes + RightRedzoneSize, MaxAlignment);
495 ConstantInt::get(Int32Ty, AlignedSize);
497 MallocSize =
alignTo(MallocSize, MaxAlignment);
500 AlignedSizeInBytesConst});
501 Initializers.push_back(InitItem);
505 SwLDSVector.
insert(LDSParams.SwLDS);
506 buildInitializerForSwLDSMD(SwLDSVector);
507 buildInitializerForSwLDSMD(LDSParams.DirectAccess.StaticLDSGlobals);
508 buildInitializerForSwLDSMD(LDSParams.IndirectAccess.StaticLDSGlobals);
509 buildInitializerForSwLDSMD(LDSParams.DirectAccess.DynamicLDSGlobals);
510 buildInitializerForSwLDSMD(LDSParams.IndirectAccess.DynamicLDSGlobals);
513 Type *Ty = LDSParams.SwLDS->getValueType();
514 const uint64_t SizeInBytes =
DL.getTypeAllocSize(Ty);
516 LDSParams.LDSSize = AlignedSize;
519 MDTypeOS <<
"llvm.amdgcn.sw.lds." << Func->getName() <<
".md.type";
524 MDOS <<
"llvm.amdgcn.sw.lds." << Func->getName() <<
".md";
530 LDSParams.SwLDSMetadata->setInitializer(
data);
533 LDSParams.SwLDS->setAlignment(MaxAlignment);
534 if (LDSParams.SwDynLDS)
535 LDSParams.SwDynLDS->setAlignment(MaxAlignment);
538 LDSParams.SwLDSMetadata->setSanitizerMetadata(MD);
541void AMDGPUSwLowerLDS::populateLDSToReplacementIndicesMap(
Function *Func) {
544 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
548 for (
auto &GV : LDSGlobals) {
551 UniqueLDSGlobals.
insert(GV);
552 LDSParams.LDSToReplacementIndicesMap[GV] = {0,
Idx, 0};
558 SwLDSVector.
insert(LDSParams.SwLDS);
559 PopulateIndices(SwLDSVector,
Idx);
560 PopulateIndices(LDSParams.DirectAccess.StaticLDSGlobals,
Idx);
561 PopulateIndices(LDSParams.IndirectAccess.StaticLDSGlobals,
Idx);
562 PopulateIndices(LDSParams.DirectAccess.DynamicLDSGlobals,
Idx);
563 PopulateIndices(LDSParams.IndirectAccess.DynamicLDSGlobals,
Idx);
567 Value *Replacement) {
569 auto ReplaceUsesLambda = [Func](
const Use &U) ->
bool {
570 auto *
V = U.getUser();
581void AMDGPUSwLowerLDS::replaceKernelLDSAccesses(
Function *Func) {
582 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
590 auto &IndirectAccess = LDSParams.IndirectAccess;
591 auto &DirectAccess = LDSParams.DirectAccess;
595 for (
auto &GV : LDSGlobals) {
598 if ((IndirectAccess.StaticLDSGlobals.contains(GV) ||
599 IndirectAccess.DynamicLDSGlobals.contains(GV)) &&
600 (!DirectAccess.StaticLDSGlobals.contains(GV) &&
601 !DirectAccess.DynamicLDSGlobals.contains(GV)))
605 UniqueLDSGlobals.
insert(GV);
606 auto &Indices = LDSParams.LDSToReplacementIndicesMap[GV];
607 assert(Indices.size() == 3);
608 Constant *GEPIdx[] = {ConstantInt::get(Int32Ty, Indices[0]),
609 ConstantInt::get(Int32Ty, Indices[1]),
610 ConstantInt::get(Int32Ty, Indices[2])};
612 Func->getDataLayout(), SwLDSMetadataStructType, SwLDSMetadata, GEPIdx,
615 Value *BasePlusOffset =
619 replacesUsesOfGlobalInFunction(Func, GV, BasePlusOffset);
622 ReplaceLDSGlobalUses(DirectAccess.StaticLDSGlobals);
623 ReplaceLDSGlobalUses(IndirectAccess.StaticLDSGlobals);
624 ReplaceLDSGlobalUses(DirectAccess.DynamicLDSGlobals);
625 ReplaceLDSGlobalUses(IndirectAccess.DynamicLDSGlobals);
628void AMDGPUSwLowerLDS::updateMallocSizeForDynamicLDS(
631 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
636 assert(SwLDS && SwLDSMetadata);
641 Value *MaxAlignValueMinusOne = IRB.
getInt32(MaxAlignment - 1);
644 auto &Indices = LDSParams.LDSToReplacementIndicesMap[DynGV];
646 Constant *Index0 = ConstantInt::get(Int32Ty, 0);
647 Constant *Index1 = ConstantInt::get(Int32Ty, Indices[1]);
649 Constant *Index2Offset = ConstantInt::get(Int32Ty, 0);
651 MetadataStructType, SwLDSMetadata, {Index0, Index1, Index2Offset});
655 Constant *Index2Size = ConstantInt::get(Int32Ty, 1);
657 {Index0, Index1, Index2Size});
661 Constant *Index2AlignedSize = ConstantInt::get(Int32Ty, 2);
663 MetadataStructType, SwLDSMetadata, {Index0, Index1, Index2AlignedSize});
665 Value *AlignedDynLDSSize =
666 IRB.
CreateAdd(CurrDynLDSSize, MaxAlignValueMinusOne);
667 AlignedDynLDSSize = IRB.
CreateUDiv(AlignedDynLDSSize, MaxAlignValue);
668 AlignedDynLDSSize = IRB.
CreateMul(AlignedDynLDSSize, MaxAlignValue);
669 IRB.
CreateStore(AlignedDynLDSSize, GEPForAlignedSize);
672 *CurrMallocSize = IRB.
CreateAdd(*CurrMallocSize, AlignedDynLDSSize);
686void AMDGPUSwLowerLDS::getLDSMemoryInstructions(
689 if (isLDSMemoryInstruction(Inst))
690 LDSInstructions.
insert(&Inst);
693Value *AMDGPUSwLowerLDS::getTranslatedGlobalMemoryPtrOfLDS(
Value *LoadMallocPtr,
695 assert(LDSPtr &&
"Invalid LDS pointer operand");
712Value *AMDGPUSwLowerLDS::getTranslatedLDSPtrOfFlat(
Value *LoadMallocPtr,
729void AMDGPUSwLowerLDS::translateLDSMemoryOperationsToGlobalMemory(
732 LLVM_DEBUG(
dbgs() <<
"Translating LDS memory operations to global memory : "
737 Value *LIOperand = LI->getPointerOperand();
739 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LIOperand);
741 IRB.
CreateLoad(LI->getType(), Replacement, LI->getProperties());
742 AsanInfo.Instructions.
insert(NewLI);
743 LI->replaceAllUsesWith(NewLI);
744 LI->eraseFromParent();
746 Value *SIOperand =
SI->getPointerOperand();
748 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, SIOperand);
750 SI->getProperties());
751 AsanInfo.Instructions.
insert(NewSI);
752 SI->replaceAllUsesWith(NewSI);
753 SI->eraseFromParent();
755 Value *RMWPtrOperand = RMW->getPointerOperand();
756 Value *RMWValOperand = RMW->getValOperand();
758 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, RMWPtrOperand);
760 RMW->getOperation(), Replacement, RMWValOperand, RMW->getAlign(),
761 RMW->getOrdering(), RMW->getSyncScopeID());
763 AsanInfo.Instructions.
insert(NewRMW);
764 RMW->replaceAllUsesWith(NewRMW);
765 RMW->eraseFromParent();
767 Value *XCHGPtrOperand = XCHG->getPointerOperand();
769 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, XCHGPtrOperand);
771 Replacement, XCHG->getCompareOperand(), XCHG->getNewValOperand(),
772 XCHG->getAlign(), XCHG->getSuccessOrdering(),
773 XCHG->getFailureOrdering(), XCHG->getSyncScopeID());
775 AsanInfo.Instructions.
insert(NewXCHG);
776 XCHG->replaceAllUsesWith(NewXCHG);
777 XCHG->eraseFromParent();
779 Value *NewDest =
MI->getRawDest();
781 NewDest = getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, NewDest);
784 if (
MI->isAtomic()) {
786 NewDest, MSI->getValue(), MSI->getLength(),
787 MSI->getDestAlign().valueOrOne(), MSI->getElementSizeInBytes());
789 NewMI = IRB.
CreateMemSet(NewDest, MSI->getValue(), MSI->getLength(),
794 Value *NewSrc = MTI->getRawSource();
796 NewSrc = getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, NewSrc);
797 if (
MI->isAtomic()) {
798 if (
MI->getIntrinsicID() ==
799 Intrinsic::memmove_element_unordered_atomic) {
801 NewDest, MTI->getDestAlign().valueOrOne(), NewSrc,
802 MTI->getSourceAlign().valueOrOne(), MTI->getLength(),
803 MTI->getElementSizeInBytes());
806 NewDest, MTI->getDestAlign().valueOrOne(), NewSrc,
807 MTI->getSourceAlign().valueOrOne(), MTI->getLength(),
808 MTI->getElementSizeInBytes());
812 MI->getIntrinsicID(), NewDest, MTI->getDestAlign(), NewSrc,
813 MTI->getSourceAlign(), MTI->getLength(),
818 AsanInfo.Instructions.
insert(NewMI);
819 MI->replaceAllUsesWith(NewMI);
820 MI->eraseFromParent();
822 Value *AIOperand = ASC->getPointerOperand();
823 Value *NewAI =
nullptr;
826 getTranslatedLDSPtrOfFlat(LoadMallocPtr, AIOperand, ASC->
getType());
829 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, AIOperand);
836 ASC->eraseFromParent();
842void AMDGPUSwLowerLDS::poisonRedzones(
Function *Func,
Value *MallocPtr) {
843 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
847 "__asan_poison_region",
850 auto RedzonesVec = LDSParams.RedzoneOffsetAndSizeVector;
851 size_t VecSize = RedzonesVec.size();
852 for (
unsigned i = 0; i < VecSize; i++) {
853 auto &RedzonePair = RedzonesVec[i];
854 uint64_t RedzoneOffset = RedzonePair.first;
855 uint64_t RedzoneSize = RedzonePair.second;
857 IRB.
getInt8Ty(), MallocPtr, {IRB.getInt64(RedzoneOffset)});
860 {RedzoneAddress, IRB.
getInt64(RedzoneSize)});
864void AMDGPUSwLowerLDS::lowerKernelLDSAccesses(
Function *Func,
866 LLVM_DEBUG(
dbgs() <<
"Sw Lowering Kernel LDS for : " << Func->getName());
867 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
868 auto &Ctx = M.getContext();
869 auto *PrevEntryBlock = &Func->getEntryBlock();
871 getLDSMemoryInstructions(Func, LDSInstructions);
885 auto SplitIt = PrevEntryBlock->getFirstNonPHIOrAlloca();
886 WIdBlock->splice(WIdBlock->end(), PrevEntryBlock, PrevEntryBlock->begin(),
891 AI->moveBefore(*WIdBlock, WIdBlock->end());
906 IRB.
CreateCondBr(WIdzCond, MallocBlock, PrevEntryBlock);
916 assert(SwLDS && SwLDSMetadata);
924 for (
auto &GV : LDSGlobals) {
927 UniqueLDSGlobals.
insert(GV);
931 GetUniqueLDSGlobals(LDSParams.DirectAccess.StaticLDSGlobals);
932 GetUniqueLDSGlobals(LDSParams.IndirectAccess.StaticLDSGlobals);
935 unsigned LastStaticLDSIdx = UniqueLDSGlobals.
size();
936 UniqueLDSGlobals.
clear();
938 auto *GEPForEndStaticLDSOffset =
940 {ConstantInt::get(Int32Ty, 0),
941 ConstantInt::get(Int32Ty, LastStaticLDSIdx),
942 ConstantInt::get(Int32Ty, 0)});
944 auto *GEPForEndStaticLDSSize =
946 {ConstantInt::get(Int32Ty, 0),
947 ConstantInt::get(Int32Ty, LastStaticLDSIdx),
948 ConstantInt::get(Int32Ty, 2)});
950 Value *EndStaticLDSOffset = IRB.
CreateLoad(Int32Ty, GEPForEndStaticLDSOffset);
951 Value *EndStaticLDSSize = IRB.
CreateLoad(Int32Ty, GEPForEndStaticLDSSize);
952 Value *CurrMallocSize = IRB.
CreateAdd(EndStaticLDSOffset, EndStaticLDSSize);
954 if (LDSParams.SwDynLDS) {
957 "Dynamic LDS size query is only supported for CO V5 and later.");
963 {ConstantInt::get(Int64Ty, COV5_HIDDEN_DYN_LDS_SIZE_ARG)});
964 UniqueLDSGlobals.
clear();
965 GetUniqueLDSGlobals(LDSParams.DirectAccess.DynamicLDSGlobals);
966 GetUniqueLDSGlobals(LDSParams.IndirectAccess.DynamicLDSGlobals);
967 updateMallocSizeForDynamicLDS(Func, &CurrMallocSize, HiddenDynLDSSize,
971 CurrMallocSize = IRB.
CreateZExt(CurrMallocSize, Int64Ty);
976 Intrinsic::returnaddress, IRB.
getPtrTy(
DL.getProgramAddressSpace()),
982 Value *MallocCall = IRB.
CreateCall(MallocFunc, {CurrMallocSize, RAPtrToInt});
991 poisonRedzones(Func, MallocPtr);
999 auto *XYZCondPhi = IRB.
CreatePHI(Int1Ty, 2,
"xyzCond");
1001 XYZCondPhi->addIncoming(IRB.
getInt1(1), MallocBlock);
1006 Value *LoadMallocPtr =
1010 replaceKernelLDSAccesses(Func);
1014 translateLDSMemoryOperationsToGlobalMemory(Func, LoadMallocPtr,
1023 RI->eraseFromParent();
1043 Intrinsic::returnaddress, IRB.
getPtrTy(
DL.getProgramAddressSpace()),
1047 IRB.
CreateCall(AsanFreeFunc, {MallocPtrToInt, RAPToInt});
1061Constant *AMDGPUSwLowerLDS::getAddressesOfVariablesInKernel(
1068 auto KernelIt = FuncLDSAccessInfo.KernelToLDSParametersMap.find(Func);
1069 if (KernelIt == FuncLDSAccessInfo.KernelToLDSParametersMap.end() ||
1070 !KernelIt->second.SwLDSMetadata)
1073 auto &LDSParams = KernelIt->second;
1075 auto *SwLDSMetadataStructType =
1079 for (
auto *GV : Variables) {
1080 auto It = LDSParams.LDSToReplacementIndicesMap.find(GV);
1081 if (It == LDSParams.LDSToReplacementIndicesMap.end()) {
1086 auto &Indices = It->second;
1087 Constant *GEPIdx[] = {ConstantInt::get(Int32Ty, Indices[0]),
1088 ConstantInt::get(Int32Ty, Indices[1]),
1089 ConstantInt::get(Int32Ty, Indices[2])};
1091 Func->getDataLayout(), SwLDSMetadataStructType, SwLDSMetadata, GEPIdx,
1093 Elements.push_back(
GEP);
1098void AMDGPUSwLowerLDS::buildNonKernelLDSBaseTable(
1099 NonKernelLDSParameters &NKLDSParams) {
1103 auto &Kernels = NKLDSParams.OrderedKernels;
1104 if (Kernels.empty())
1106 const size_t NumberKernels = Kernels.size();
1109 std::vector<Constant *> OverallConstantExprElts(NumberKernels);
1110 for (
size_t i = 0; i < NumberKernels; i++) {
1114 auto It = FuncLDSAccessInfo.KernelToLDSParametersMap.find(Kernels[i]);
1115 if (It != FuncLDSAccessInfo.KernelToLDSParametersMap.end() &&
1117 SwLDS = It->second.SwLDS;
1118 OverallConstantExprElts[i] = SwLDS;
1131void AMDGPUSwLowerLDS::buildNonKernelLDSOffsetTable(
1132 NonKernelLDSParameters &NKLDSParams) {
1140 auto &Variables = NKLDSParams.OrdereLDSGlobals;
1141 auto &Kernels = NKLDSParams.OrderedKernels;
1142 if (Variables.
empty() || Kernels.empty())
1144 const size_t NumberVariables = Variables.
size();
1145 const size_t NumberKernels = Kernels.size();
1152 std::vector<Constant *> overallConstantExprElts(NumberKernels);
1153 for (
size_t i = 0; i < NumberKernels; i++) {
1155 overallConstantExprElts[i] =
1156 getAddressesOfVariablesInKernel(Func, Variables);
1169void AMDGPUSwLowerLDS::lowerNonKernelLDSAccesses(
1171 NonKernelLDSParameters &NKLDSParams) {
1174 LLVM_DEBUG(
dbgs() <<
"Sw LDS lowering, lower non-kernel access for : "
1175 << Func->getName());
1176 auto InsertAt = Func->getEntryBlock().getFirstNonPHIOrAlloca();
1181 getLDSMemoryInstructions(Func, LDSInstructions);
1183 auto *KernelId = IRB.
CreateIntrinsic(Intrinsic::amdgcn_lds_kernel_id, {});
1186 auto &OrdereLDSGlobals = NKLDSParams.OrdereLDSGlobals;
1188 LDSBaseTable->
getValueType(), LDSBaseTable, {IRB.getInt32(0), KernelId});
1191 Value *LoadMallocPtr =
1195 const auto *GVIt =
llvm::find(OrdereLDSGlobals, GV);
1196 assert(GVIt != OrdereLDSGlobals.end());
1197 uint32_t GVOffset = std::distance(OrdereLDSGlobals.begin(), GVIt);
1201 {IRB.getInt32(0), KernelId, IRB.getInt32(GVOffset)});
1205 Value *BasePlusOffset =
1207 LLVM_DEBUG(
dbgs() <<
"Sw LDS Lowering, Replace non-kernel LDS for "
1209 replacesUsesOfGlobalInFunction(Func, GV, BasePlusOffset);
1211 translateLDSMemoryOperationsToGlobalMemory(Func, LoadMallocPtr,
1215static void reorderStaticDynamicIndirectLDSSet(KernelLDSParameters &LDSParams) {
1218 auto &DirectAccess = LDSParams.DirectAccess;
1219 auto &IndirectAccess = LDSParams.IndirectAccess;
1220 LDSParams.DirectAccess.StaticLDSGlobals = sortByName(
1221 std::vector<GlobalVariable *>(DirectAccess.StaticLDSGlobals.begin(),
1222 DirectAccess.StaticLDSGlobals.end()));
1223 LDSParams.DirectAccess.DynamicLDSGlobals = sortByName(
1224 std::vector<GlobalVariable *>(DirectAccess.DynamicLDSGlobals.begin(),
1225 DirectAccess.DynamicLDSGlobals.end()));
1226 LDSParams.IndirectAccess.StaticLDSGlobals = sortByName(
1227 std::vector<GlobalVariable *>(IndirectAccess.StaticLDSGlobals.begin(),
1228 IndirectAccess.StaticLDSGlobals.end()));
1229 LDSParams.IndirectAccess.DynamicLDSGlobals = sortByName(
1230 std::vector<GlobalVariable *>(IndirectAccess.DynamicLDSGlobals.begin(),
1231 IndirectAccess.DynamicLDSGlobals.end()));
1234void AMDGPUSwLowerLDS::initAsanInfo() {
1240 bool OrShadowOffset;
1242 false, &
Offset, &Scale, &OrShadowOffset);
1243 AsanInfo.Scale = Scale;
1244 AsanInfo.Offset =
Offset;
1248 for (
auto &
K : LDSAccesses) {
1252 if (
F->hasFnAttribute(Attribute::SanitizeAddress))
1258bool AMDGPUSwLowerLDS::run() {
1271 bool LowerAllLDS = hasFnWithSanitizeAddressAttr(LDSUsesInfo.
DirectAccess) ||
1279 bool DirectAccess) {
1280 for (
auto &
K : LDSAccesses) {
1282 if (!
F ||
K.second.empty())
1288 FuncLDSAccessInfo.KernelToLDSParametersMap.insert(
1289 {
F, KernelLDSParameters()});
1291 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[
F];
1293 FuncLDSAccessInfo.KernelsWithIndirectLDSAccess.
insert(
F);
1295 if (!DirectAccess) {
1297 LDSParams.IndirectAccess.DynamicLDSGlobals.insert(GV);
1299 LDSParams.IndirectAccess.StaticLDSGlobals.insert(GV);
1300 FuncLDSAccessInfo.AllNonKernelLDSAccess.insert(GV);
1303 LDSParams.DirectAccess.DynamicLDSGlobals.insert(GV);
1305 LDSParams.DirectAccess.StaticLDSGlobals.insert(GV);
1311 PopulateKernelStaticDynamicLDS(LDSUsesInfo.
DirectAccess,
true);
1312 PopulateKernelStaticDynamicLDS(LDSUsesInfo.
IndirectAccess,
false);
1317 for (
auto &
K : FuncLDSAccessInfo.KernelToLDSParametersMap) {
1319 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
1320 if (LDSParams.DirectAccess.StaticLDSGlobals.empty() &&
1321 LDSParams.DirectAccess.DynamicLDSGlobals.empty() &&
1322 LDSParams.IndirectAccess.StaticLDSGlobals.empty() &&
1323 LDSParams.IndirectAccess.DynamicLDSGlobals.empty())
1328 {
"amdgpu-no-workitem-id-x",
"amdgpu-no-workitem-id-y",
1329 "amdgpu-no-workitem-id-z",
"amdgpu-no-heap-ptr"});
1330 reorderStaticDynamicIndirectLDSSet(LDSParams);
1331 buildSwLDSGlobal(Func);
1332 buildSwDynLDSGlobal(Func);
1333 populateSwMetadataGlobal(Func);
1334 populateSwLDSAttributeAndMetadata(Func);
1335 populateLDSToReplacementIndicesMap(Func);
1336 DomTreeUpdater DTU(DTCallback(*Func), DomTreeUpdater::UpdateStrategy::Lazy);
1337 lowerKernelLDSAccesses(Func, DTU);
1342 getUsesOfLDSByNonKernels();
1345 getNonKernelsWithLDSInstructions(CG);
1349 for (
Function *Func : FuncLDSAccessInfo.KernelsWithIndirectLDSAccess)
1353 if (!FuncLDSAccessInfo.NonKernelToLDSAccessMap.
empty() ||
1354 !FuncLDSAccessInfo.NonKernelsWithLDSInstructions.
empty()) {
1355 NonKernelLDSParameters NKLDSParams;
1356 NKLDSParams.OrderedKernels = getOrderedIndirectLDSAccessingKernels(
1357 FuncLDSAccessInfo.KernelsWithIndirectLDSAccess);
1358 NKLDSParams.OrdereLDSGlobals = getOrderedNonKernelAllLDSGlobals(
1359 FuncLDSAccessInfo.AllNonKernelLDSAccess);
1360 buildNonKernelLDSBaseTable(NKLDSParams);
1361 buildNonKernelLDSOffsetTable(NKLDSParams);
1362 for (
auto &
K : FuncLDSAccessInfo.NonKernelToLDSAccessMap) {
1366 std::vector<GlobalVariable *>(LDSGlobals.
begin(), LDSGlobals.
end()));
1367 lowerNonKernelLDSAccesses(Func, OrderedLDSGlobals, NKLDSParams);
1369 for (
Function *Func : FuncLDSAccessInfo.NonKernelsWithLDSInstructions) {
1370 if (FuncLDSAccessInfo.NonKernelToLDSAccessMap.
contains(Func))
1373 lowerNonKernelLDSAccesses(Func, NoLDSGlobals, NKLDSParams);
1391 .getLibraryOptions()
1392 .get<AMDGPUOptions>()
1393 .asan_instrument_lds) {
1400 for (
auto &Operand : OperandsToInstrument) {
1401 Value *Addr = Operand.getPtr();
1403 Operand.Alignment.valueOrOne(), Operand.TypeStoreSize,
1404 Operand.IsWrite,
nullptr,
false,
false, AsanInfo.Scale,
1413class AMDGPUSwLowerLDSLegacy :
public ModulePass {
1417 bool runOnModule(
Module &M)
override;
1424char AMDGPUSwLowerLDSLegacy::ID = 0;
1428 "AMDGPU Software lowering of LDS",
false,
false)
1433bool AMDGPUSwLowerLDSLegacy::runOnModule(
Module &M) {
1436 if (!M.getModuleFlag(
"nosanitize_address"))
1439 getAnalysisIfAvailable<DominatorTreeWrapperPass>();
1441 return DTW ? &DTW->getDomTree() : nullptr;
1444 AMDGPUSwLowerLDS SwLowerLDSImpl(M, DTCallback);
1445 bool IsChanged = SwLowerLDSImpl.run();
1450 return new AMDGPUSwLowerLDSLegacy();
1457 if (!M.getModuleFlag(
"nosanitize_address"))
1463 AMDGPUSwLowerLDS SwLowerLDSImpl(M, DTCallback);
1464 bool IsChanged = SwLowerLDSImpl.run();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static DebugLoc getOrCreateDebugLoc(const Instruction *InsertBefore, DISubprogram *SP)
This class represents a conversion between pointers from one address space to another.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents any memset intrinsic.
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
void setVolatile(bool V)
Specify whether this is a volatile cmpxchg.
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
std::pair< std::optional< WeakTrackingVH >, CallGraphNode * > CallRecord
A pair of the calling instruction (a call or invoke) and the call graph node being called.
The basic data container for the call graph of a Module of IR.
This class represents a function call, abstracting a target machine's calling convention.
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static LLVM_SUPPRESS_DEPRECATED_DECLARATIONS_PUSH Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static LLVM_ABI Constant * getAddrSpaceCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
LLVM_ABI void removeDeadConstantUsers() const
If there are any dead constant users dangling off of this constant, remove them.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Subprogram description. Uses SubclassData1.
A parsed version of the target data layout string in and methods for querying it.
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
static constexpr UpdateKind Insert
Legacy analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static GEPNoWrapFlags inBounds()
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LLVM_ABI void setSanitizerMetadata(SanitizerMetadata Meta)
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
Type * getValueType() const
MaybeAlign getAlign() const
Returns the alignment of the given variable.
LLVM_ABI void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNull=false)
AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID=SyncScope::System)
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
CondBrInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreatePtrToAddr(Value *V, const Twine &Name="")
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
void SetCurrentDebugLocation(const DebugLoc &L)
Set location information used by debugging information.
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
LLVM_ABI CallInst * CreateElementUnorderedAtomicMemMove(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, const AAMDNodes &AAInfo=AAMDNodes())
Create and insert an element unordered-atomic memmove between the specified pointers.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
UncondBrInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
CallInst * CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, uint64_t Size, Align Alignment, uint32_t ElementSize, const AAMDNodes &AAInfo=AAMDNodes())
Create and insert an element unordered-atomic memset of the region of memory starting at the given po...
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, const AAMDNodes &AAInfo=AAMDNodes())
Create and insert a memset to the specified pointer and the specified value.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Type * getVoidTy()
Fetch the type representing void.
LLVM_ABI CallInst * CreateElementUnorderedAtomicMemCpy(Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, const AAMDNodes &AAInfo=AAMDNodes())
Create and insert an element unordered-atomic memcpy between the specified pointers.
LLVM_ABI CallInst * CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool isVolatile=false, const AAMDNodes &AAInfo=AAMDNodes())
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, bool Elementwise=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
This class implements a map that also provides access to all stored values in a deterministic order.
bool contains(const KeyT &Key) const
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
A Module instance is used to store all the information related to an LLVM module.
A container for an operand bundle being viewed as a set of values rather than a set of uses.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Return a value (possibly void), from a function.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
iterator end()
Get an iterator to the end of the SetVector.
void clear()
Completely clear the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
iterator begin()
Get an iterator to the beginning of the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
Class to represent struct types.
static LLVM_ABI StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Target-Independent Code Generator Pass Configuration Options.
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
LLVM_ABI bool replaceUsesWithIf(Value *New, llvm::function_ref< bool(Use &U)> ShouldReplace)
Go through the uses list for this definition and make each use point to "V" if the callback ShouldRep...
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to an std::string.
A raw_ostream that writes to an SmallVector or SmallString.
StringRef str() const
Return a StringRef for the vector contents.
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
GVUsesInfoTy getTransitiveUsesOfLDSForLowering(const CallGraph &CG, Module &M)
Collects all uses of LDS Global Variables in M using getUsesOfGVByFunction, with isLDSVariableToLower...
void getInterestingMemoryOperands(Module &M, Instruction *I, SmallVectorImpl< InterestingMemoryOperand > &Interesting)
Get all the memory operands from the instruction that needs to be instrumented.
bool isDynamicLDS(const GlobalVariable &GV)
unsigned getAMDHSACodeObjectVersion(const Module &M)
void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot, ArrayRef< StringRef > FnAttrs)
Strip FnAttr attribute from any functions where we may have introduced its use.
bool eliminateGVConstantExprUsesFromAllInstructions(Module &M, function_ref< bool(const GlobalVariable &)> Filter)
Iterates over all GlobalVariables in M, and whenever Filter returns true, replace all constant users ...
LLVM_READNONE constexpr bool isKernel(CallingConv::ID CC)
MapVector< Function *, DenseSet< GlobalVariable * > > FunctionVariableMap
bool isLDSVariableToLower(const GlobalVariable &GV)
Align getAlign(const DataLayout &DL, const GlobalVariable *GV)
void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, Align Alignment, TypeSize TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls, bool Recover, int AsanScale, int AsanOffset)
Instrument the memory operand Addr.
uint64_t getRedzoneSizeForGlobal(int AsanScale, uint64_t SizeInBytes)
Given SizeInBytes of the Value to be instrunmented, Returns the redzone size corresponding to it.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
This is an optimization pass for GlobalISel generic memory operations.
ModulePass * createAMDGPUSwLowerLDSLegacyPass()
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr from_range_t from_range
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
char & AMDGPUSwLowerLDSLegacyPassID
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI void getAddressSanitizerParams(const LLVMContext &Ctx, const Triple &TargetTriple, int LongSize, bool IsKasan, uint64_t *ShadowBase, int *MappingScale, bool *OrShadowOffset)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
FunctionVariableMap DirectAccess
FunctionVariableMap IndirectAccess
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.