62#define DEBUG_TYPE "atomic-expand"
66class AtomicExpandImpl {
86 Ctx.
emitError(DiagnosticInst ? DiagnosticInst : &FailedInst,
Msg);
88 if (!FailedInst.getType()->isVoidTy())
90 FailedInst.eraseFromParent();
93 template <
typename Inst>
94 void handleUnsupportedAtomicSize(Inst *
I,
const Twine &AtomicOpName,
98 bool tryInsertTrailingSeqCstFence(
Instruction *AtomicI);
99 template <
typename AtomicInst>
100 bool tryInsertFencesForAtomic(AtomicInst *AtomicI,
bool OrderingRequiresFence,
104 bool tryExpandAtomicLoad(
LoadInst *LI);
105 bool expandAtomicLoadToLL(
LoadInst *LI);
106 bool expandAtomicLoadToCmpXchg(
LoadInst *LI);
118 void expandAtomicOpToLLSC(
122 void expandPartwordAtomicRMW(
130 Value *insertRMWCmpXchgLoop(
134 CreateCmpXchgInstFun CreateCmpXchg,
Instruction *MetadataSrc);
146 void expandAtomicLoadToLibcall(
LoadInst *LI);
147 void expandAtomicStoreToLibcall(
StoreInst *LI);
150 const Twine &AtomicOpName =
"cmpxchg",
154 CreateCmpXchgInstFun CreateCmpXchg);
179struct ReplacementIRBuilder
180 :
IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> {
182 MDNode *PCSectionsMD =
nullptr;
190 if (
I->getFunction()->hasFnAttribute(Attribute::StrictFP))
191 this->setIsFPConstrained(
true);
193 MMRAMD =
I->getMetadata(LLVMContext::MD_mmra);
194 PCSectionsMD =
I->getMetadata(LLVMContext::MD_pcsections);
199 I->setMetadata(LLVMContext::MD_mmra, MMRAMD);
200 I->setMetadata(LLVMContext::MD_pcsections, PCSectionsMD);
206char AtomicExpandLegacy::ID = 0;
211 "Expand Atomic instructions",
false,
false)
220 return DL.getTypeStoreSize(LI->getType());
225 return DL.getTypeStoreSize(
SI->getValueOperand()->getType());
242 Source.getAllMetadata(MD);
246 for (
auto [ID,
N] : MD) {
248 case LLVMContext::MD_dbg:
249 case LLVMContext::MD_tbaa:
250 case LLVMContext::MD_tbaa_struct:
251 case LLVMContext::MD_alias_scope:
252 case LLVMContext::MD_mem_cache_hint:
253 case LLVMContext::MD_noalias:
254 case LLVMContext::MD_noalias_addrspace:
255 case LLVMContext::MD_access_group:
256 case LLVMContext::MD_mmra:
257 case LLVMContext::MD_atomic_ignore_denormal_mode:
261 if (ID == Ctx.getMDKindID(
"amdgpu.no.remote.memory"))
263 else if (ID == Ctx.getMDKindID(
"amdgpu.no.fine.grained.memory"))
265 else if (ID == Ctx.getMDKindID(
"amdgpu.cfs"))
272template <
typename Inst>
275 Align Alignment =
I->getAlign();
276 unsigned MaxSize =
TLI->getMaxAtomicSizeInBitsSupported() / 8;
277 return Size <= MaxSize &&
TLI->isAtomicAlignmentSupported(Alignment,
Size);
280template <
typename Inst>
284 Align Alignment =
I->getAlign();
285 bool NeedSeparator =
false;
287 if (!
TLI->isAtomicAlignmentSupported(Alignment,
Size)) {
288 OS <<
"instruction alignment " << Alignment.value()
289 <<
" is smaller than the required " <<
Size
290 <<
"-byte alignment for this atomic operation";
291 NeedSeparator =
true;
294 unsigned MaxSize =
TLI->getMaxAtomicSizeInBitsSupported() / 8;
295 if (
Size > MaxSize) {
298 OS <<
"target supports atomics up to " << MaxSize
299 <<
" bytes, but this atomic accesses " <<
Size <<
" bytes";
303template <
typename Inst>
304void AtomicExpandImpl::handleUnsupportedAtomicSize(
307 SmallString<128> FailureReason;
308 raw_svector_ostream OS(FailureReason);
310 handleFailure(*
I, Twine(
"unsupported ") + AtomicOpName +
": " + FailureReason,
314bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
320 Builder, AtomicI, AtomicOrdering::SequentiallyConsistent)) {
321 TrailingFence->moveAfter(AtomicI);
327template <
typename AtomicInst>
328bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
329 bool OrderingRequiresFence,
332 if (OrderingRequiresFence && ShouldInsertFences) {
334 AtomicI->setOrdering(NewOrdering);
335 return bracketInstWithFences(AtomicI, FenceOrdering);
337 if (!ShouldInsertFences)
338 return tryInsertTrailingSeqCstFence(AtomicI);
345bool AtomicExpandImpl::lowerToNonAtomic(Instruction *
I) {
347 FI->eraseFromParent();
358 if (LI->isAtomic()) {
359 LI->setAtomic(AtomicOrdering::NotAtomic);
360 LI->setElementwise(
false);
368 if (
SI->isAtomic()) {
369 SI->setAtomic(AtomicOrdering::NotAtomic);
370 SI->setElementwise(
false);
380bool AtomicExpandImpl::processAtomicInstr(Instruction *
I) {
382 return lowerToNonAtomic(
I);
389 expandAtomicLoadToLibcall(LI);
393 bool MadeChange =
false;
395 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
396 LI = convertAtomicLoadToIntegerType(LI);
404 MadeChange |= tryExpandAtomicLoad(LI);
413 expandAtomicStoreToLibcall(SI);
417 bool MadeChange =
false;
419 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
420 SI = convertAtomicStoreToIntegerType(SI);
428 MadeChange |= tryExpandAtomicStore(SI);
433 bool IsElementwiseExpand =
434 RMWI->isElementwise() &&
436 TargetLoweringBase::AtomicExpansionKind::Expand;
441 if (!IsElementwiseExpand) {
442 expandAtomicRMWToLibcall(RMWI);
447 bool MadeChange =
false;
448 if (!IsElementwiseExpand &&
450 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
451 RMWI = convertAtomicXchgToIntegerType(RMWI);
455 MadeChange |= tryInsertFencesForAtomic(
463 MadeChange |= (
isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
464 tryExpandAtomicRMW(RMWI);
470 expandAtomicCASToLibcall(CASI);
476 bool MadeChange =
false;
477 if (CASI->getCompareOperand()->getType()->isPointerTy()) {
480 CASI = convertCmpXchgToIntegerType(CASI);
486 if (CmpXchgExpansion == TargetLoweringBase::AtomicExpansionKind::None &&
497 CASI->setSuccessOrdering(CASOrdering);
498 CASI->setFailureOrdering(CASOrdering);
499 MadeChange |= bracketInstWithFences(CASI, FenceOrdering);
501 }
else if (CmpXchgExpansion !=
502 TargetLoweringBase::AtomicExpansionKind::LLSC) {
504 MadeChange |= tryInsertTrailingSeqCstFence(CASI);
507 MadeChange |= tryExpandAtomicCmpXchg(CASI);
515 const ModuleLibcallLoweringInfo &LibcallResult,
516 const TargetMachine *TM) {
517 SingleThreaded =
F.getParent()->getThreadModel() == ThreadModel::Single;
525 DL = &
F.getDataLayout();
527 bool MadeChange =
false;
539 if (processAtomicInstr(&Inst)) {
551bool AtomicExpandLegacy::runOnFunction(
Function &
F) {
553 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
556 auto *TM = &TPC->getTM<TargetMachine>();
558 const ModuleLibcallLoweringInfo &LibcallResult =
559 getAnalysis<LibcallLoweringInfoWrapper>().getResult(*
F.getParent());
561 return AE.run(
F, LibcallResult, TM);
565 return new AtomicExpandLegacy();
575 if (!LibcallResult) {
577 "' analysis required");
583 bool Changed = AE.run(
F, *LibcallResult, TM);
590bool AtomicExpandImpl::bracketInstWithFences(
Instruction *
I,
592 ReplacementIRBuilder Builder(
I, *
DL);
594 auto LeadingFence =
TLI->emitLeadingFence(Builder,
I, Order);
596 auto TrailingFence =
TLI->emitTrailingFence(Builder,
I, Order);
600 TrailingFence->moveAfter(
I);
602 return (LeadingFence || TrailingFence);
617LoadInst *AtomicExpandImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {
619 Type *NewTy = getCorrespondingIntegerType(LI->
getType(),
M->getDataLayout());
621 ReplacementIRBuilder Builder(LI, *
DL);
625 auto *NewLI = Builder.CreateLoad(NewTy, Addr, LI->
getProperties());
626 LLVM_DEBUG(
dbgs() <<
"Replaced " << *LI <<
" with " << *NewLI <<
"\n");
629 ? Builder.CreateIntToPtr(NewLI, LI->
getType())
630 : Builder.CreateBitCast(NewLI, LI->
getType());
637AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
642 getCorrespondingIntegerType(RMWI->
getType(),
M->getDataLayout());
644 ReplacementIRBuilder Builder(RMWI, *
DL);
648 Value *NewVal = Builder.CreateBitPreservingCastChain(*
DL, Val, NewTy);
655 LLVM_DEBUG(
dbgs() <<
"Replaced " << *RMWI <<
" with " << *NewRMWI <<
"\n");
659 Builder.CreateBitPreservingCastChain(*
DL, NewRMWI, RMWI->
getType());
682void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
685 Type *LaneTy = VecTy->getElementType();
686 const unsigned NumLanes = VecTy->getNumElements();
690 ReplacementIRBuilder Builder(AI, *
DL);
691 Builder.disableGEPCanonicalization();
696 bool Elementwise) -> AtomicRMWInst * {
698 Builder.CreateAtomicRMW(AI->
getOperation(), HalfPtr, HalfVal,
A,
701 NewAI->setElementwise(Elementwise);
706 Constant *IdxZero = ConstantInt::get(IdxTy, 0);
707 Constant *IdxOne = ConstantInt::get(IdxTy, 1);
712 Value *LaneVal = Builder.CreateExtractElement(Val, IdxZero,
"lane.val");
713 AtomicRMWInst *LaneRMW = CreateRMWInstruction(Ptr, LaneVal, AI->
getAlign(),
716 LaneRMW, IdxZero,
"lane.old");
719 processAtomicInstr(LaneRMW);
726 "elementwise atomicrmw vector length must be a power of two");
727 const unsigned HalfLanes = NumLanes / 2;
729 const uint64_t HalfBytes =
DL->getTypeStoreSize(HalfVecTy).getFixedValue();
731 Value *LoVal = Builder.CreateShuffleVector(
733 Value *HiVal = Builder.CreateShuffleVector(
736 Value *HiPtr = Builder.CreateInBoundsGEP(HalfVecTy, Ptr, IdxOne,
"hi.ptr");
740 AtomicRMWInst *LoRMW =
741 CreateRMWInstruction(Ptr, LoVal, LoAlign,
true);
742 AtomicRMWInst *HiRMW =
743 CreateRMWInstruction(HiPtr, HiVal, HiAlign,
true);
750 processAtomicInstr(LoRMW);
751 processAtomicInstr(HiRMW);
754bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
756 case TargetLoweringBase::AtomicExpansionKind::None:
758 case TargetLoweringBase::AtomicExpansionKind::LLSC:
759 expandAtomicOpToLLSC(
762 [](IRBuilderBase &Builder,
Value *Loaded) { return Loaded; });
764 case TargetLoweringBase::AtomicExpansionKind::LLOnly:
765 return expandAtomicLoadToLL(LI);
766 case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
767 return expandAtomicLoadToCmpXchg(LI);
768 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
769 LI->
setAtomic(AtomicOrdering::NotAtomic);
771 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
779bool AtomicExpandImpl::tryExpandAtomicStore(StoreInst *SI) {
781 case TargetLoweringBase::AtomicExpansionKind::None:
783 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
786 case TargetLoweringBase::AtomicExpansionKind::Expand:
787 expandAtomicStoreToXChg(SI);
789 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
790 SI->setAtomic(AtomicOrdering::NotAtomic);
797bool AtomicExpandImpl::expandAtomicLoadToLL(LoadInst *LI) {
798 ReplacementIRBuilder Builder(LI, *
DL);
813bool AtomicExpandImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {
814 ReplacementIRBuilder Builder(LI, *
DL);
816 if (Order == AtomicOrdering::Unordered)
817 Order = AtomicOrdering::Monotonic;
826 Type *CmpXchgTy = Ty;
828 CmpXchgTy = getCorrespondingIntegerType(Ty, *
DL);
831 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
832 Addr, DummyVal, DummyVal, LI->
getAlign(), Order,
836 Value *
Loaded = Builder.CreateExtractValue(Pair, 0,
"loaded");
838 Loaded = Builder.CreateBitPreservingCastChain(*
DL, Loaded, Ty);
854StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
855 ReplacementIRBuilder Builder(SI, *
DL);
856 auto *
M =
SI->getModule();
857 Type *NewTy = getCorrespondingIntegerType(
SI->getValueOperand()->getType(),
859 Value *NewVal =
SI->getValueOperand()->getType()->isPtrOrPtrVectorTy()
860 ? Builder.CreatePtrToInt(
SI->getValueOperand(), NewTy)
861 : Builder.CreateBitCast(
SI->getValueOperand(), NewTy);
863 Value *Addr =
SI->getPointerOperand();
865 StoreInst *NewSI = Builder.CreateStore(NewVal, Addr,
SI->getProperties());
867 LLVM_DEBUG(
dbgs() <<
"Replaced " << *SI <<
" with " << *NewSI <<
"\n");
868 SI->eraseFromParent();
872void AtomicExpandImpl::expandAtomicStoreToXChg(StoreInst *SI) {
879 ReplacementIRBuilder Builder(SI, *
DL);
881 assert(Ordering != AtomicOrdering::NotAtomic);
883 ? AtomicOrdering::Monotonic
885 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
887 SI->getAlign(), RMWOrdering,
SI->getSyncScopeID());
889 SI->eraseFromParent();
892 TargetLoweringBase::AtomicExpansionKind::CastToInteger)
893 AI = convertAtomicXchgToIntegerType(AI);
896 tryExpandAtomicRMW(AI);
911 NewVal = Builder.CreateBitCast(NewVal, IntTy);
912 Loaded = Builder.CreateBitCast(Loaded, IntTy);
916 Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
922 Success = Builder.CreateExtractValue(Pair, 1,
"success");
923 NewLoaded = Builder.CreateExtractValue(Pair, 0,
"newloaded");
926 NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
929void AtomicExpandImpl::expandAtomicSubToAdd(AtomicRMWInst *AI) {
930 ReplacementIRBuilder Builder(AI, *
DL);
951bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
955 case TargetLoweringBase::AtomicExpansionKind::None:
957 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
960 if (ValueSize < MinCASSize) {
961 expandPartwordAtomicRMW(AI,
962 TargetLoweringBase::AtomicExpansionKind::LLSC);
964 auto PerformOp = [&](IRBuilderBase &Builder,
Value *
Loaded) {
973 case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
976 if (ValueSize < MinCASSize) {
977 expandPartwordAtomicRMW(AI,
978 TargetLoweringBase::AtomicExpansionKind::CmpXChg);
987 return OptimizationRemark(
DEBUG_TYPE,
"Passed", AI)
988 <<
"A compare and swap loop was generated for an atomic "
996 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
999 if (ValueSize < MinCASSize) {
1004 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI));
1008 expandAtomicRMWToMaskedIntrinsic(AI);
1011 case TargetLoweringBase::AtomicExpansionKind::BitTestIntrinsic: {
1015 case TargetLoweringBase::AtomicExpansionKind::CmpArithIntrinsic: {
1019 case TargetLoweringBase::AtomicExpansionKind::Expand:
1021 expandElementwiseAtomicRMW(AI);
1024 expandAtomicSubToAdd(AI);
1026 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
1028 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
1038struct PartwordMaskValues {
1040 Type *WordType =
nullptr;
1042 Type *IntValueType =
nullptr;
1043 Value *AlignedAddr =
nullptr;
1044 Align AlignedAddrAlignment;
1046 Value *ShiftAmt =
nullptr;
1047 Value *Mask =
nullptr;
1048 Value *Inv_Mask =
nullptr;
1052raw_ostream &
operator<<(raw_ostream &O,
const PartwordMaskValues &PMV) {
1053 auto PrintObj = [&
O](
auto *
V) {
1060 O <<
"PartwordMaskValues {\n";
1062 PrintObj(PMV.WordType);
1063 O <<
" ValueType: ";
1064 PrintObj(PMV.ValueType);
1065 O <<
" AlignedAddr: ";
1066 PrintObj(PMV.AlignedAddr);
1067 O <<
" AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.
value() <<
'\n';
1069 PrintObj(PMV.ShiftAmt);
1073 PrintObj(PMV.Inv_Mask);
1099 unsigned MinWordSize) {
1100 PartwordMaskValues PMV;
1105 unsigned ValueSize =
DL.getTypeStoreSize(
ValueType);
1107 PMV.ValueType = PMV.IntValueType =
ValueType;
1112 PMV.WordType = MinWordSize > ValueSize ?
Type::getIntNTy(Ctx, MinWordSize * 8)
1114 if (PMV.ValueType == PMV.WordType) {
1115 PMV.AlignedAddr = Addr;
1116 PMV.AlignedAddrAlignment = AddrAlign;
1117 PMV.ShiftAmt = ConstantInt::get(PMV.ValueType, 0);
1118 PMV.Mask = ConstantInt::get(PMV.ValueType, ~0,
true);
1122 PMV.AlignedAddrAlignment =
Align(MinWordSize);
1124 assert(ValueSize < MinWordSize);
1127 IntegerType *IntTy =
DL.getIndexType(Ctx, PtrTy->getAddressSpace());
1130 if (AddrAlign < MinWordSize) {
1131 PMV.AlignedAddr = Builder.CreateIntrinsic(
1132 Intrinsic::ptrmask, {PtrTy, IntTy},
1134 nullptr,
"AlignedAddr");
1136 Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
1137 PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1,
"PtrLSB");
1140 PMV.AlignedAddr = Addr;
1144 if (
DL.isLittleEndian()) {
1146 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3);
1149 PMV.ShiftAmt = Builder.CreateShl(
1150 Builder.CreateXor(PtrLSB, MinWordSize - ValueSize), 3);
1153 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType,
"ShiftAmt");
1154 PMV.Mask = Builder.CreateShl(
1155 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt,
1158 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask,
"Inv_Mask");
1164 const PartwordMaskValues &PMV) {
1165 assert(WideWord->
getType() == PMV.WordType &&
"Widened type mismatch");
1166 if (PMV.WordType == PMV.ValueType)
1169 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt,
"shifted");
1170 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType,
"extracted");
1171 return Builder.CreateBitCast(Trunc, PMV.ValueType);
1175 Value *Updated,
const PartwordMaskValues &PMV) {
1176 assert(WideWord->
getType() == PMV.WordType &&
"Widened type mismatch");
1177 assert(Updated->
getType() == PMV.ValueType &&
"Value type mismatch");
1178 if (PMV.WordType == PMV.ValueType)
1181 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType);
1183 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType,
"extended");
1185 Builder.CreateShl(ZExt, PMV.ShiftAmt,
"shifted",
true);
1186 Value *
And = Builder.CreateAnd(WideWord, PMV.Inv_Mask,
"unmasked");
1187 Value *
Or = Builder.CreateOr(
And, Shift,
"inserted");
1197 const PartwordMaskValues &PMV) {
1204 "Or/Xor/And handled by widenPartwordAtomicRMW");
1209 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
1212 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, ValOperand_Shifted);
1234 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
1235 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
1236 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked);
1243 assert(!ValOperand_Shifted);
1257void AtomicExpandImpl::expandPartwordAtomicRMW(
1263 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI));
1269 ReplacementIRBuilder Builder(AI, *
DL);
1271 PartwordMaskValues PMV =
1275 Value *ValOperand_Shifted =
nullptr;
1276 bool NeedsShiftedOperand =
1281 if (NeedsShiftedOperand) {
1283 ValOperand_Shifted =
1284 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
1285 "ValOperand_Shifted");
1288 auto PerformPartwordOp = [&](IRBuilderBase &Builder,
Value *
Loaded) {
1294 if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
1295 OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr,
1296 PMV.AlignedAddrAlignment, MemOpOrder, SSID,
1300 assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
1301 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
1302 PMV.AlignedAddrAlignment, MemOpOrder,
1312AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
1313 ReplacementIRBuilder Builder(AI, *
DL);
1318 "Unable to widen operation");
1320 PartwordMaskValues PMV =
1330 Value *ValOperand_Shifted =
1332 "ValOperand_Shifted");
1338 Builder.
CreateOr(ValOperand_Shifted, PMV.Inv_Mask,
"AndOperand");
1340 NewOperand = ValOperand_Shifted;
1343 Op, PMV.AlignedAddr, NewOperand, PMV.AlignedAddrAlignment,
1355bool AtomicExpandImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
1397 ReplacementIRBuilder Builder(CI, *
DL);
1408 std::prev(BB->
end())->eraseFromParent();
1411 PartwordMaskValues PMV =
1416 Value *NewVal_Shifted =
1418 Value *Cmp_Shifted =
1423 LoadInst *InitLoaded = Builder.
CreateLoad(PMV.WordType, PMV.AlignedAddr);
1424 Value *InitLoaded_MaskOut = Builder.
CreateAnd(InitLoaded, PMV.Inv_Mask);
1429 PHINode *Loaded_MaskOut = Builder.
CreatePHI(PMV.WordType, 2);
1430 Loaded_MaskOut->
addIncoming(InitLoaded_MaskOut, BB);
1443 processAtomicInstr(InitLoaded);
1447 Value *FullWord_NewVal = Builder.
CreateOr(Loaded_MaskOut, NewVal_Shifted);
1448 Value *FullWord_Cmp = Builder.
CreateOr(Loaded_MaskOut, Cmp_Shifted);
1450 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment,
1478 Loaded_MaskOut->
addIncoming(OldVal_MaskOut, FailureBB);
1493void AtomicExpandImpl::expandAtomicOpToLLSC(
1494 Instruction *
I,
Type *ResultType,
Value *Addr, Align AddrAlign,
1496 function_ref<
Value *(IRBuilderBase &,
Value *)> PerformOp) {
1497 ReplacementIRBuilder Builder(
I, *
DL);
1498 Value *
Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
1499 MemOpOrder, PerformOp);
1501 I->replaceAllUsesWith(Loaded);
1502 I->eraseFromParent();
1505void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
1506 ReplacementIRBuilder Builder(AI, *
DL);
1508 PartwordMaskValues PMV =
1518 CastOp = Instruction::SExt;
1522 PMV.ShiftAmt,
"ValOperand_Shifted");
1524 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,
1531void AtomicExpandImpl::expandAtomicCmpXchgToMaskedIntrinsic(
1532 AtomicCmpXchgInst *CI) {
1533 ReplacementIRBuilder Builder(CI, *
DL);
1546 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask,
1552 CmpVal_Shifted, Builder.
CreateAnd(OldVal, PMV.Mask),
"Success");
1559Value *AtomicExpandImpl::insertRMWLLSCLoop(
1560 IRBuilderBase &Builder,
Type *ResultTy,
Value *Addr, Align AddrAlign,
1562 function_ref<
Value *(IRBuilderBase &,
Value *)> PerformOp) {
1567 assert(AddrAlign >=
F->getDataLayout().getTypeStoreSize(ResultTy) &&
1568 "Expected at least natural alignment at this point.");
1588 std::prev(BB->
end())->eraseFromParent();
1596 Value *NewVal = PerformOp(Builder, Loaded);
1598 Value *StoreSuccess =
1620AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
1623 M->getDataLayout());
1625 ReplacementIRBuilder Builder(CI, *
DL);
1637 LLVM_DEBUG(
dbgs() <<
"Replaced " << *CI <<
" with " << *NewCI <<
"\n");
1653bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1659 LLVMContext &Ctx =
F->getContext();
1666 ? AtomicOrdering::Monotonic
1678 bool HasReleasedLoadBB = !CI->
isWeak() && ShouldInsertFencesForAtomic &&
1679 SuccessOrder != AtomicOrdering::Monotonic &&
1680 SuccessOrder != AtomicOrdering::Acquire &&
1687 bool UseUnconditionalReleaseBarrier =
1688 (
F->hasMinSize() && !CI->
isWeak()) ||
1743 auto ReleasedLoadBB =
1747 auto ReleasingStoreBB =
1751 ReplacementIRBuilder Builder(CI, *
DL);
1756 std::prev(BB->
end())->eraseFromParent();
1758 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
1761 PartwordMaskValues PMV =
1768 Value *UnreleasedLoad =
1770 Value *UnreleasedLoadExtract =
1777 Builder.
CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB,
1778 MDBuilder(
F->getContext()).createLikelyBranchWeights());
1781 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
1786 PHINode *LoadedTryStore =
1787 Builder.
CreatePHI(PMV.WordType, 2,
"loaded.trystore");
1788 LoadedTryStore->
addIncoming(UnreleasedLoad, ReleasingStoreBB);
1789 Value *NewValueInsert =
1792 PMV.AlignedAddr, MemOpOrder);
1794 StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0),
"success");
1795 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
1797 CI->
isWeak() ? FailureBB : RetryBB,
1798 MDBuilder(
F->getContext()).createLikelyBranchWeights());
1802 if (HasReleasedLoadBB) {
1812 ShouldStore, TryStoreBB, NoStoreBB,
1813 MDBuilder(
F->getContext()).createLikelyBranchWeights());
1815 LoadedTryStore->
addIncoming(SecondLoad, ReleasedLoadBB);
1822 if (ShouldInsertFencesForAtomic ||
1828 PHINode *LoadedNoStore =
1830 LoadedNoStore->
addIncoming(UnreleasedLoad, StartBB);
1831 if (HasReleasedLoadBB)
1832 LoadedNoStore->
addIncoming(SecondLoad, ReleasedLoadBB);
1841 PHINode *LoadedFailure =
1843 LoadedFailure->
addIncoming(LoadedNoStore, NoStoreBB);
1845 LoadedFailure->
addIncoming(LoadedTryStore, TryStoreBB);
1846 if (ShouldInsertFencesForAtomic)
1855 PHINode *LoadedExit =
1857 LoadedExit->
addIncoming(LoadedTryStore, SuccessBB);
1858 LoadedExit->
addIncoming(LoadedFailure, FailureBB);
1865 Value *LoadedFull = LoadedExit;
1873 for (
auto *User : CI->
users()) {
1879 "weird extraction from { iN, i1 }");
1890 for (
auto *EV : PrunedInsts)
1907bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
1922 return C->isMinusOne();
1924 return C->isMaxValue(
true);
1926 return C->isMinValue(
true);
1928 return C->isMaxValue(
false);
1930 return C->isMinValue(
false);
1936bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
1938 tryExpandAtomicLoad(ResultingLoad);
1944Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
1945 IRBuilderBase &Builder,
Type *ResultTy,
Value *Addr, Align AddrAlign,
1947 function_ref<
Value *(IRBuilderBase &,
Value *)> PerformOp,
1948 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc) {
1975 std::prev(BB->
end())->eraseFromParent();
1983 Loaded->addIncoming(InitLoaded, BB);
1992 InitLoaded->
setAtomic(AtomicOrdering::Monotonic, SSID);
1996 processAtomicInstr(InitLoaded);
1999 Value *NewVal = PerformOp(Builder, Loaded);
2001 Value *NewLoaded =
nullptr;
2004 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
2005 MemOpOrder == AtomicOrdering::Unordered
2006 ? AtomicOrdering::Monotonic
2008 SSID, IsVolatile,
Success, NewLoaded, MetadataSrc);
2011 Loaded->addIncoming(NewLoaded, LoopBB);
2024bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
2031 case TargetLoweringBase::AtomicExpansionKind::None:
2032 if (ValueSize < MinCASSize)
2033 return expandPartwordCmpXchg(CI);
2035 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
2036 return expandAtomicCmpXchg(CI);
2038 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic:
2039 expandAtomicCmpXchgToMaskedIntrinsic(CI);
2041 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
2043 case TargetLoweringBase::AtomicExpansionKind::CustomExpand: {
2050bool AtomicExpandImpl::expandAtomicRMWToCmpXchg(
2051 AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) {
2058 Value *
Loaded = AtomicExpandImpl::insertRMWCmpXchgLoop(
2061 [&](IRBuilderBase &Builder,
Value *Loaded) {
2062 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
2063 AI->getValOperand());
2086 unsigned LargestSize =
DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;
2087 return Alignment >=
Size &&
2089 Size <= LargestSize;
2092void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *
I) {
2093 static const RTLIB::Libcall Libcalls[6] = {
2094 RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
2095 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
2098 bool Expanded = expandAtomicOpToLibcall(
2099 I,
Size,
I->getAlign(),
I->getPointerOperand(),
nullptr,
nullptr,
2100 I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
2102 handleUnsupportedAtomicSize(
I,
"atomic load");
2105void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *
I) {
2106 static const RTLIB::Libcall Libcalls[6] = {
2107 RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
2108 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
2111 bool Expanded = expandAtomicOpToLibcall(
2112 I,
Size,
I->getAlign(),
I->getPointerOperand(),
I->getValueOperand(),
2113 nullptr,
I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
2115 handleUnsupportedAtomicSize(
I,
"atomic store");
2118void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *
I,
2119 const Twine &AtomicOpName,
2120 Instruction *DiagnosticInst) {
2121 static const RTLIB::Libcall Libcalls[6] = {
2122 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
2123 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
2124 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
2127 bool Expanded = expandAtomicOpToLibcall(
2128 I,
Size,
I->getAlign(),
I->getPointerOperand(),
I->getNewValOperand(),
2129 I->getCompareOperand(),
I->getSuccessOrdering(),
I->getFailureOrdering(),
2132 handleUnsupportedAtomicSize(
I, AtomicOpName, DiagnosticInst);
2136 static const RTLIB::Libcall LibcallsXchg[6] = {
2137 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
2138 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,
2139 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};
2140 static const RTLIB::Libcall LibcallsAdd[6] = {
2141 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,
2142 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,
2143 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};
2144 static const RTLIB::Libcall LibcallsSub[6] = {
2145 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,
2146 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,
2147 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};
2148 static const RTLIB::Libcall LibcallsAnd[6] = {
2149 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,
2150 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,
2151 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};
2152 static const RTLIB::Libcall LibcallsOr[6] = {
2153 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,
2154 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,
2155 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};
2156 static const RTLIB::Libcall LibcallsXor[6] = {
2157 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,
2158 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,
2159 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};
2160 static const RTLIB::Libcall LibcallsNand[6] = {
2161 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,
2162 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,
2163 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};
2204void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *
I) {
2210 if (!Libcalls.
empty())
2211 Success = expandAtomicOpToLibcall(
2212 I,
Size,
I->getAlign(),
I->getPointerOperand(),
I->getValOperand(),
2213 nullptr,
I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
2220 expandAtomicRMWToCmpXchg(
2221 I, [
this,
I](IRBuilderBase &Builder,
Value *Addr,
Value *Loaded,
2224 Value *&NewLoaded, Instruction *MetadataSrc) {
2227 Addr, Loaded, NewVal, Alignment, MemOpOrder,
2237 expandAtomicCASToLibcall(
2251bool AtomicExpandImpl::expandAtomicOpToLibcall(
2252 Instruction *
I,
unsigned Size, Align Alignment,
Value *PointerOperand,
2257 LLVMContext &Ctx =
I->getContext();
2259 const DataLayout &
DL =
M->getDataLayout();
2261 IRBuilder<> AllocaBuilder(&
I->getFunction()->getEntryBlock().front());
2264 const Align AllocaAlignment =
2265 DL.getPrefTypeAlign(Type::getIntNTy(Ctx,
Size * 8));
2267 assert(Ordering != AtomicOrdering::NotAtomic &&
"expect atomic MO");
2268 assert((!CASExpected || Ordering2 != AtomicOrdering::NotAtomic) &&
2269 "expect atomic MO");
2270 bool HasResult =
I->getType() != Type::getVoidTy(Ctx);
2272 RTLIB::Libcall RTLibType;
2273 if (UseSizedLibcall) {
2276 RTLibType = Libcalls[1];
2279 RTLibType = Libcalls[2];
2282 RTLibType = Libcalls[3];
2285 RTLibType = Libcalls[4];
2288 RTLibType = Libcalls[5];
2291 }
else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {
2292 RTLibType = Libcalls[0];
2299 RTLIB::LibcallImpl LibcallImpl = LibcallLowering->
getLibcallImpl(RTLibType);
2300 if (LibcallImpl == RTLIB::Unsupported) {
2306 Ctx,
M->getTargetTriple(),
DL, LibcallImpl);
2307 assert(FnType &&
"missing signature for atomic libcall");
2335 AllocaInst *AllocaCASExpected =
nullptr;
2336 AllocaInst *AllocaValue =
nullptr;
2337 AllocaInst *AllocaResult =
nullptr;
2342 if (!UseSizedLibcall)
2343 Args.push_back(ConstantInt::get(FnType->getParamType(0),
Size));
2350 Value *PtrVal = PointerOperand;
2352 Args.push_back(PtrVal);
2356 AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->
getType());
2360 Args.push_back(AllocaCASExpected);
2365 if (UseSizedLibcall) {
2367 DL, ValueOperand, FnType->getParamType(
Args.size()));
2368 Args.push_back(IntValue);
2370 AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->
getType());
2374 Args.push_back(AllocaValue);
2379 if (!CASExpected && HasResult && !UseSizedLibcall) {
2380 AllocaResult = AllocaBuilder.CreateAlloca(
I->getType());
2383 Args.push_back(AllocaResult);
2387 Args.push_back(ConstantInt::get(FnType->getParamType(
Args.size()),
2392 Args.push_back(ConstantInt::get(FnType->getParamType(
Args.size()),
2393 (
int)
toCABI(Ordering2)));
2396 assert(
Args.size() == FnType->getNumParams() &&
2397 "atomic libcall signature mismatch");
2399 FunctionCallee LibcallFn =
M->getOrInsertFunction(
2407 if (ValueOperand && !UseSizedLibcall)
2413 Type *FinalResultTy =
I->getType();
2416 CASExpected->
getType(), AllocaCASExpected, AllocaAlignment);
2421 }
else if (HasResult) {
2423 if (UseSizedLibcall) {
2427 if (VTy && PtrTy && !
Result->getType()->isVectorTy()) {
2428 unsigned AS = PtrTy->getAddressSpace();
2430 Result, VTy->getWithNewType(
DL.getIntPtrType(Ctx, AS)));
2439 I->replaceAllUsesWith(
V);
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Value * performMaskedAtomicOp(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *ValOperand_Shifted, Value *Inc, const PartwordMaskValues &PMV)
Emit IR to implement a masked version of a given atomicrmw operation.
static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder, Instruction *I, Type *ValueType, Value *Addr, Align AddrAlign, unsigned MinWordSize)
This is a helper function which builds instructions to provide values necessary for partword atomic o...
static bool canUseSizedAtomicCall(unsigned Size, Align Alignment, const DataLayout &DL)
static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr, Value *Loaded, Value *NewVal, Align AddrAlign, AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile, Value *&Success, Value *&NewLoaded, Instruction *MetadataSrc)
static Value * extractMaskedValue(IRBuilderBase &Builder, Value *WideWord, const PartwordMaskValues &PMV)
Expand Atomic static false unsigned getAtomicOpSize(LoadInst *LI)
static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I, raw_ostream &OS)
static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I)
static Value * insertMaskedValue(IRBuilderBase &Builder, Value *WideWord, Value *Updated, const PartwordMaskValues &PMV)
static void copyMetadataForAtomic(Instruction &Dest, const Instruction &Source)
Copy metadata that's safe to preserve when widening atomics.
static ArrayRef< RTLIB::Libcall > GetRMWLibcall(AtomicRMWInst::BinOp Op)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool runOnFunction(Function &F, bool PostInlining)
Module.h This file contains the declarations for the Module class.
static bool isIdempotentRMW(AtomicRMWInst &RMWI)
Return true if and only if the given instruction does not modify the memory location referenced.
Machine Check Debug Module
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file contains the declarations for profiling metadata utility functions.
This file defines the SmallString class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
const AArch64TargetLowering * getTargetLowering() const override
void setAlignment(Align Align)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getNewValOperand()
AtomicOrdering getMergedOrdering() const
Returns a single ordering which is at least as strong as both the success and failure orderings for t...
void setWeak(bool IsWeak)
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
Value * getCompareOperand()
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
Value * getPointerOperand()
static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)
Returns the strongest permitted ordering on failure, given the desired ordering on success.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
void setVolatile(bool V)
Specify whether this is a volatile cmpxchg.
AtomicOrdering getSuccessOrdering() const
Returns the success ordering constraint of this cmpxchg instruction.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
an instruction that atomically reads a memory location, combines it with another value,...
bool isElementwise() const
Return true if this RMW has elementwise vector semantics.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ FMinimum
*p = minimum(old, v) minimum matches the behavior of llvm.minimum.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ FMaximum
*p = maximum(old, v) maximum matches the behavior of llvm.maximum.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMaximumNum
*p = maximumnum(old, v) maximumnum matches the behavior of llvm.maximumnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ FMinimumNum
*p = minimumnum(old, v) minimumnum matches the behavior of llvm.minimumnum.
Value * getPointerOperand()
void setOperation(BinOp Operation)
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
static LLVM_ABI StringRef getOperationName(BinOp Op)
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
iterator begin()
Instruction iterator methods.
LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
reverse_iterator rbegin()
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
InstListType::reverse_iterator reverse_iterator
void setAttributes(AttributeList A)
Set the attributes for this call.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionPass class - This class is used to implement most global optimizations.
BasicBlockListType::iterator iterator
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Common base class shared among various IRBuilders.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNull=false)
AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID=SyncScope::System)
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
LLVM_ABI CallInst * CreateLifetimeStart(Value *Ptr)
Create a lifetime.start intrinsic.
LLVM_ABI CallInst * CreateLifetimeEnd(Value *Ptr)
Create a lifetime.end intrinsic.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
CondBrInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
UnreachableInst * CreateUnreachable()
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
BasicBlock::iterator GetInsertPoint() const
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
BasicBlock * GetInsertBlock() const
LLVM_ABI Value * CreateBitPreservingCastChain(const DataLayout &DL, Value *V, Type *NewTy)
Create a chain of casts to convert V to NewTy, preserving the bit pattern of V.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
UncondBrInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
void setIsFPConstrained(bool IsCon)
Enable/Disable use of constrained floating point math.
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, bool Elementwise=false)
Provides an 'InsertHelper' that calls a user-provided callback after performing the default insertion...
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
LLVM_ABI void getSyncScopeNames(SmallVectorImpl< StringRef > &SSNs) const
getSyncScopeNames - Populates client supplied SmallVector with synchronization scope names registered...
Tracks which library functions to use for a particular subtarget or function.
const RTLIB::RuntimeLibcallsInfo & getRuntimeLibcallsInfo() const
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
An instruction for reading from memory.
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
void setVolatile(bool V)
Specify whether this is a volatile load or not.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
LoadStoreInstProperties getProperties() const
Returns the properties of this load instruction.
Align getAlign() const
Return the alignment of the access that is being performed.
Records a mapping from an opaque lowering context to its LibcallLoweringInfo.
A Module instance is used to store all the information related to an LLVM module.
LLVMContext & getContext() const
Get the global data context.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
virtual Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const
Perform a store-conditional operation to Addr.
virtual void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a bit test atomicrmw using a target-specific intrinsic.
virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
virtual bool shouldInsertFencesForAtomic(const Instruction *I) const
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
virtual AtomicOrdering atomicOperationOrderAfterFenceSplit(const Instruction *I) const
virtual void emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const
Perform a cmpxchg expansion using a target-specific method.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
virtual Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const
Perform a masked atomicrmw using a target-specific intrinsic.
virtual AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
virtual Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
virtual void emitExpandAtomicRMW(AtomicRMWInst *AI) const
Perform a atomicrmw expansion using a target-specific way.
virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const
virtual void emitExpandAtomicStore(StoreInst *SI) const
Perform a atomic store using a target-specific way.
virtual AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const
Returns how the given atomic atomicrmw should be cast by the IR-level AtomicExpand pass.
virtual bool shouldInsertTrailingSeqCstFenceForAtomicStore(const Instruction *I) const
Whether AtomicExpandPass should automatically insert a seq_cst trailing fence without reducing the or...
virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
virtual Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const
Perform a masked cmpxchg using a target-specific intrinsic.
virtual bool shouldIssueAtomicLoadForAtomicEmulationLoop(void) const
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual void emitExpandAtomicLoad(LoadInst *LI) const
Perform a atomic load using a target-specific way.
virtual AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
virtual bool fenceClearsLoadLinkedReservation() const
Whether a fence between the load-linked and the store-conditional can clear the reservation.
virtual void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a atomicrmw which the result is only used by comparison, using a target-specific intrinsic.
virtual AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
virtual Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
virtual AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
virtual Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
Inserts in the IR a target-specific intrinsic specifying a fence.
virtual LoadInst * lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *RMWI) const
On some platforms, an AtomicRMW that never actually modifies the value (such as fetch_add of 0) can b...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
Target-Independent Code Generator Pass Configuration Options.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
void setOperand(unsigned i, Value *Val)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ BasicBlock
Various leaf nodes.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool canInstructionHaveMMRAs(const Instruction &I)
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool isStrongerThanMonotonic(AtomicOrdering AO)
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
bool isReleaseOrStronger(AtomicOrdering AO)
AtomicOrderingCABI toCABI(AtomicOrdering AO)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI const LibcallLoweringInfo & getLibcallLowering(const ModuleLibcallLoweringInfo &ModuleInfo, const TargetSubtargetInfo &Subtarget)
Resolve the LibcallLoweringInfo for Subtarget from the module-level ModuleInfo, applying the subtarge...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI Value * buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Val)
Emit IR to implement the given atomicrmw operation on values in registers, returning the new value.
AtomicOrdering
Atomic ordering for LLVM's memory model.
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
LLVM_ABI bool lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI)
Convert the given Cmpxchg into primitive load and compare.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool lowerAtomicRMWInst(AtomicRMWInst *RMWI)
Convert the given RMWI into primitive load and stores, assuming that doing so is legal.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI FunctionPass * createAtomicExpandLegacyPass()
AtomicExpandPass - At IR level this pass replace atomic instructions with __atomic_* library calls,...
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
LLVM_ABI char & AtomicExpandID
AtomicExpandID – Lowers atomic operations in terms of either cmpxchg load-linked/store-conditional lo...
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
LLVM_ABI std::pair< FunctionType *, AttributeList > getFunctionTy(LLVMContext &Ctx, const Triple &TT, const DataLayout &DL, RTLIB::LibcallImpl LibcallImpl) const
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.