54 #define DEBUG_TYPE "atomic-expand" 74 bool tryExpandAtomicLoad(
LoadInst *LI);
75 bool expandAtomicLoadToLL(
LoadInst *LI);
76 bool expandAtomicLoadToCmpXchg(
LoadInst *LI);
84 void expandAtomicOpToLLSC(
87 void expandPartwordAtomicRMW(
96 static Value *insertRMWCmpXchgLoop(
112 void expandAtomicLoadToLibcall(
LoadInst *LI);
113 void expandAtomicStoreToLibcall(
StoreInst *LI);
160 assert(Align != 0 &&
"An atomic LoadInst always has an explicit alignment");
169 assert(Align != 0 &&
"An atomic StoreInst always has an explicit alignment");
190 template <
typename Inst>
198 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
203 if (!
TM.getSubtargetImpl(F)->enableAtomicExpand())
205 TLI =
TM.getSubtargetImpl(F)->getTargetLowering();
217 bool MadeChange =
false;
218 for (
auto I : AtomicInsts) {
223 assert((LI || SI || RMWI || CASI) &&
"Unknown atomic instruction");
228 expandAtomicLoadToLibcall(LI);
234 expandAtomicStoreToLibcall(SI);
240 expandAtomicRMWToLibcall(RMWI);
246 expandAtomicCASToLibcall(CASI);
252 if (TLI->shouldInsertFencesForAtomic(
I)) {
255 FenceOrdering = LI->getOrdering();
258 FenceOrdering = SI->getOrdering();
262 FenceOrdering = RMWI->getOrdering();
265 TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
273 FenceOrdering = CASI->getSuccessOrdering();
279 MadeChange |= bracketInstWithFences(
I, FenceOrdering);
284 if (LI->getType()->isFloatingPointTy()) {
287 LI = convertAtomicLoadToIntegerType(LI);
288 assert(LI->getType()->isIntegerTy() &&
"invariant broken");
292 MadeChange |= tryExpandAtomicLoad(LI);
294 if (SI->getValueOperand()->getType()->isFloatingPointTy()) {
297 SI = convertAtomicStoreToIntegerType(SI);
298 assert(SI->getValueOperand()->getType()->isIntegerTy() &&
303 if (TLI->shouldExpandAtomicStoreInIR(SI))
304 MadeChange |= expandAtomicStore(SI);
311 if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
314 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
317 if (ValueSize < MinCASSize &&
320 RMWI = widenPartwordAtomicRMW(RMWI);
324 MadeChange |= tryExpandAtomicRMW(RMWI);
329 assert(!CASI->getCompareOperand()->getType()->isFloatingPointTy() &&
330 "unimplemented - floating point not legal at IR level");
331 if (CASI->getCompareOperand()->getType()->isPointerTy() ) {
334 CASI = convertCmpXchgToIntegerType(CASI);
335 assert(CASI->getCompareOperand()->getType()->isIntegerTy() &&
340 MadeChange |= tryExpandAtomicCmpXchg(CASI);
349 auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);
351 auto TrailingFence = TLI->emitTrailingFence(Builder, I, Order);
355 TrailingFence->moveAfter(I);
357 return (LeadingFence || TrailingFence);
363 EVT VT = TLI->getMemValueType(DL, T);
374 Type *NewTy = getCorrespondingIntegerType(LI->
getType(),
382 Value *NewAddr = Builder.CreateBitCast(Addr, PT);
384 auto *NewLI = Builder.CreateLoad(NewTy, NewAddr);
388 LLVM_DEBUG(
dbgs() <<
"Replaced " << *LI <<
" with " << *NewLI <<
"\n");
390 Value *NewVal = Builder.CreateBitCast(NewLI, LI->
getType());
396 bool AtomicExpand::tryExpandAtomicLoad(
LoadInst *LI) {
397 switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
401 expandAtomicOpToLLSC(
406 return expandAtomicLoadToLL(LI);
408 return expandAtomicLoadToCmpXchg(LI);
414 bool AtomicExpand::expandAtomicLoadToLL(
LoadInst *LI) {
422 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
430 bool AtomicExpand::expandAtomicLoadToCmpXchg(
LoadInst *LI) {
437 Type *Ty = cast<PointerType>(Addr->
getType())->getElementType();
441 Addr, DummyVal, DummyVal, Order,
475 LLVM_DEBUG(
dbgs() <<
"Replaced " << *SI <<
" with " << *NewSI <<
"\n");
480 bool AtomicExpand::expandAtomicStore(
StoreInst *SI) {
494 return tryExpandAtomicRMW(AI);
514 Addr, Loaded, NewVal, MemOpOrder,
532 return Builder.
CreateAdd(Loaded, Inc,
"new");
534 return Builder.
CreateSub(Loaded, Inc,
"new");
536 return Builder.
CreateAnd(Loaded, Inc,
"new");
540 return Builder.
CreateOr(Loaded, Inc,
"new");
542 return Builder.
CreateXor(Loaded, Inc,
"new");
545 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
548 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
551 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
554 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
556 return Builder.
CreateFAdd(Loaded, Inc,
"new");
558 return Builder.
CreateFSub(Loaded, Inc,
"new");
565 switch (TLI->shouldExpandAtomicRMWInIR(AI)) {
569 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
571 if (ValueSize < MinCASSize) {
573 "MinCmpXchgSizeInBits not yet supported for LL/SC architectures.");
585 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
587 if (ValueSize < MinCASSize) {
592 expandPartwordAtomicRMW(AI,
600 expandAtomicRMWToMaskedIntrinsic(AI);
611 struct PartwordMaskValues {
641 PartwordMaskValues
Ret;
652 assert(ValueSize < WordSize);
662 Builder.
CreateAnd(AddrInt, ~(uint64_t)(WordSize - 1)), WordPtrType,
665 Value *PtrLSB = Builder.
CreateAnd(AddrInt, WordSize - 1,
"PtrLSB");
668 Ret.ShiftAmt = Builder.
CreateShl(PtrLSB, 3);
675 Ret.ShiftAmt = Builder.
CreateTrunc(Ret.ShiftAmt, Ret.WordType,
"ShiftAmt");
679 Ret.Inv_Mask = Builder.
CreateNot(Ret.Mask,
"Inv_Mask");
690 const PartwordMaskValues &PMV) {
697 Value *FinalVal = Builder.
CreateOr(Loaded_MaskOut, Shifted_Inc);
711 Value *FinalVal = Builder.
CreateOr(Loaded_MaskOut, NewVal_Masked);
722 Builder.
CreateLShr(Loaded, PMV.ShiftAmt), PMV.ValueType);
725 Builder.
CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
727 Value *FinalVal = Builder.
CreateOr(Loaded_MaskOut, NewVal_Shiftup);
742 void AtomicExpand::expandPartwordAtomicRMW(
750 PartwordMaskValues PMV =
752 TLI->getMinCmpXchgSizeInBits() / 8);
754 Value *ValOperand_Shifted =
756 PMV.ShiftAmt,
"ValOperand_Shifted");
766 insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder,
769 Builder.
CreateLShr(OldResult, PMV.ShiftAmt), PMV.ValueType);
781 "Unable to widen operation");
783 PartwordMaskValues PMV =
785 TLI->getMinCmpXchgSizeInBits() / 8);
787 Value *ValOperand_Shifted =
789 PMV.ShiftAmt,
"ValOperand_Shifted");
795 Builder.
CreateOr(PMV.Inv_Mask, ValOperand_Shifted,
"AndOperand");
797 NewOperand = ValOperand_Shifted;
803 Builder.
CreateLShr(NewAI, PMV.ShiftAmt), PMV.ValueType);
854 const int WordSize = TLI->getMinCmpXchgSizeInBits() / 8;
864 std::prev(BB->
end())->eraseFromParent();
871 Value *NewVal_Shifted =
880 Value *InitLoaded_MaskOut = Builder.
CreateAnd(InitLoaded, PMV.Inv_Mask);
886 Loaded_MaskOut->
addIncoming(InitLoaded_MaskOut, BB);
889 Value *FullWord_NewVal = Builder.
CreateOr(Loaded_MaskOut, NewVal_Shifted);
890 Value *FullWord_Cmp = Builder.
CreateOr(Loaded_MaskOut, Cmp_Shifted);
920 Loaded_MaskOut->
addIncoming(OldVal_MaskOut, FailureBB);
926 Builder.
CreateLShr(OldVal, PMV.ShiftAmt), PMV.ValueType);
935 void AtomicExpand::expandAtomicOpToLLSC(
940 insertRMWLLSCLoop(Builder, ResultType, Addr, MemOpOrder, PerformOp);
946 void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(
AtomicRMWInst *AI) {
949 PartwordMaskValues PMV =
951 TLI->getMinCmpXchgSizeInBits() / 8);
959 CastOp = Instruction::SExt;
963 PMV.ShiftAmt,
"ValOperand_Shifted");
964 Value *OldResult = TLI->emitMaskedAtomicRMWIntrinsic(
965 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,
968 Builder.
CreateLShr(OldResult, PMV.ShiftAmt), PMV.ValueType);
978 TLI->getMinCmpXchgSizeInBits() / 8);
986 Value *OldVal = TLI->emitMaskedAtomicCmpXchgIntrinsic(
987 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask,
990 Builder.
CreateLShr(OldVal, PMV.ShiftAmt), PMV.ValueType);
995 CmpVal_Shifted, Builder.
CreateAnd(OldVal, PMV.Mask),
"Success");
1002 Value *AtomicExpand::insertRMWLLSCLoop(
1028 std::prev(BB->
end())->eraseFromParent();
1034 Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
1036 Value *NewVal = PerformOp(Builder, Loaded);
1038 Value *StoreSuccess =
1039 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
1056 M->getDataLayout());
1063 Value *NewAddr = Builder.CreateBitCast(Addr, PT);
1069 auto *NewCI = Builder.CreateAtomicCmpXchg(NewAddr, NewCmp, NewNewVal,
1074 NewCI->setWeak(CI->
isWeak());
1075 LLVM_DEBUG(
dbgs() <<
"Replaced " << *CI <<
" with " << *NewCI <<
"\n");
1077 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
1078 Value *Succ = Builder.CreateExtractValue(NewCI, 1);
1083 Res = Builder.CreateInsertValue(Res, OldVal, 0);
1084 Res = Builder.CreateInsertValue(Res, Succ, 1);
1102 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(CI);
1115 bool HasReleasedLoadBB = !CI->
isWeak() && ShouldInsertFencesForAtomic &&
1171 auto ReleasedLoadBB =
1175 auto ReleasingStoreBB =
1185 std::prev(BB->
end())->eraseFromParent();
1187 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
1188 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1193 Value *UnreleasedLoad = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
1199 Builder.
CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB);
1202 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
1203 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1207 Value *StoreSuccess = TLI->emitStoreConditional(
1211 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
1213 CI->
isWeak() ? FailureBB : RetryBB);
1217 if (HasReleasedLoadBB) {
1218 SecondLoad = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
1224 Builder.
CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB);
1231 if (ShouldInsertFencesForAtomic)
1232 TLI->emitTrailingFence(Builder, CI, SuccessOrder);
1239 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1243 if (ShouldInsertFencesForAtomic)
1244 TLI->emitTrailingFence(Builder, CI, FailureOrder);
1258 if (!HasReleasedLoadBB)
1259 Loaded = UnreleasedLoad;
1263 TryStoreLoaded->
addIncoming(UnreleasedLoad, ReleasingStoreBB);
1264 TryStoreLoaded->
addIncoming(SecondLoad, ReleasedLoadBB);
1268 NoStoreLoaded->
addIncoming(UnreleasedLoad, StartBB);
1269 NoStoreLoaded->
addIncoming(SecondLoad, ReleasedLoadBB);
1273 ExitLoaded->
addIncoming(TryStoreLoaded, SuccessBB);
1274 ExitLoaded->
addIncoming(NoStoreLoaded, FailureBB);
1276 Loaded = ExitLoaded;
1288 "weird extraction from { iN, i1 }");
1299 for (
auto EV : PrunedInsts)
1300 EV->eraseFromParent();
1329 return C->isMinusOne();
1336 bool AtomicExpand::simplifyIdempotentRMW(
AtomicRMWInst* RMWI) {
1337 if (
auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
1338 tryExpandAtomicLoad(ResultingLoad);
1344 Value *AtomicExpand::insertRMWCmpXchgLoop(
1375 std::prev(BB->
end())->eraseFromParent();
1387 Value *NewVal = PerformOp(Builder, Loaded);
1389 Value *NewLoaded =
nullptr;
1392 CreateCmpXchg(Builder, Addr, Loaded, NewVal,
1396 Success, NewLoaded);
1397 assert(Success && NewLoaded);
1408 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
1411 switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) {
1415 if (ValueSize < MinCASSize)
1416 expandPartwordCmpXchg(CI);
1419 assert(ValueSize >= MinCASSize &&
1420 "MinCmpXchgSizeInBits not yet supported for LL/SC expansions.");
1421 return expandAtomicCmpXchg(CI);
1424 expandAtomicCmpXchgToMaskedIntrinsic(CI);
1433 Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
1461 return Align >= Size &&
1462 (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) &&
1463 Size <= LargestSize;
1466 void AtomicExpand::expandAtomicLoadToLibcall(
LoadInst *I) {
1469 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
1473 bool expanded = expandAtomicOpToLibcall(
1477 assert(expanded &&
"expandAtomicOpToLibcall shouldn't fail tor Load");
1480 void AtomicExpand::expandAtomicStoreToLibcall(
StoreInst *I) {
1483 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
1487 bool expanded = expandAtomicOpToLibcall(
1491 assert(expanded &&
"expandAtomicOpToLibcall shouldn't fail tor Store");
1496 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
1497 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
1498 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
1502 bool expanded = expandAtomicOpToLibcall(
1507 assert(expanded &&
"expandAtomicOpToLibcall shouldn't fail tor CAS");
1512 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
1513 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,
1514 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};
1516 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,
1517 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,
1518 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};
1520 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,
1521 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,
1522 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};
1524 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,
1525 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,
1526 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};
1528 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,
1529 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,
1530 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};
1532 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,
1533 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,
1534 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};
1536 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,
1537 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,
1538 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};
1569 void AtomicExpand::expandAtomicRMWToLibcall(
AtomicRMWInst *I) {
1576 if (!Libcalls.
empty())
1577 Success = expandAtomicOpToLibcall(
1592 Addr, Loaded, NewVal, MemOpOrder,
1598 expandAtomicCASToLibcall(Pair);
1609 bool AtomicExpand::expandAtomicOpToLibcall(
1641 if (UseSizedLibcall) {
1643 case 1: RTLibType = Libcalls[1];
break;
1644 case 2: RTLibType = Libcalls[2];
break;
1645 case 4: RTLibType = Libcalls[3];
break;
1646 case 8: RTLibType = Libcalls[4];
break;
1647 case 16: RTLibType = Libcalls[5];
break;
1649 }
else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {
1650 RTLibType = Libcalls[0];
1684 Value *AllocaCASExpected_i8 =
nullptr;
1686 Value *AllocaValue_i8 =
nullptr;
1688 Value *AllocaResult_i8 =
nullptr;
1695 if (!UseSizedLibcall) {
1713 AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->
getType());
1717 AllocaCASExpected_i8 =
1727 if (UseSizedLibcall) {
1732 AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->
getType());
1743 if (!CASExpected && HasResult && !UseSizedLibcall) {
1744 AllocaResult = AllocaBuilder.CreateAlloca(I->
getType());
1764 }
else if (HasResult && UseSizedLibcall)
1765 ResultTy = SizedIntTy;
1778 Value *Result = Call;
1781 if (ValueOperand && !UseSizedLibcall)
1790 CASExpected->
getType(), AllocaCASExpected, AllocaAlignment);
1795 }
else if (HasResult) {
1797 if (UseSizedLibcall)
Value * getValueOperand()
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
A parsed version of the target data layout string in and methods for querying it. ...
static ConstantInt * getFalse(LLVMContext &Context)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
static IntegerType * getInt1Ty(LLVMContext &C)
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
*p = old <signed v ? old : v
LLVMContext & getContext() const
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Atomic ordering constants.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
This class represents lattice values for constants.
bool isAtomic() const
Return true if this instruction has an AtomicOrdering of unordered or higher.
static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I)
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
A Module instance is used to store all the information related to an LLVM module. ...
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, const char *Name)
Provided to resolve 'CreateAlignedLoad(Ptr, Align, "...")' correctly, instead of converting the strin...
An instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
void push_back(const T &Elt)
This class represents a function call, abstracting a target machine's calling convention.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space...
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this store instruction.
static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I, Type *ValueType, Value *Addr, unsigned WordSize)
This is a helper function which builds instructions to provide values necessary for partword atomic o...
*p = old <unsigned v ? old : v
An efficient, type-erasing, non-owning reference to a callable.
void setVolatile(bool V)
Specify whether this is a volatile cmpxchg.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
*p = old >unsigned v ? old : v
LLVMContext & getContext() const
All values hold a context through their type.
Value * getNewValOperand()
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
char & AtomicExpandID
AtomicExpandID – Lowers atomic operations in terms of either cmpxchg load-linked/store-conditional l...
An instruction for reading from memory.
static IntegerType * getInt64Ty(LLVMContext &C)
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Value * CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name="")
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this store instruction.
void setAlignment(MaybeAlign Align)
*p = old >signed v ? old : v
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, unsigned Align, bool isVolatile=false)
iterator begin()
Instruction iterator methods.
Value * CreateNot(Value *V, const Twine &Name="")
bool isVolatile() const
Return true if this is a load from a volatile memory location.
inst_iterator inst_begin(Function *F)
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
PointerType * getType() const
Overload to return most specific pointer type.
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
BinOp getOperation() const
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This file contains the simple types necessary to represent the attributes associated with functions a...
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
AtomicOrdering
Atomic ordering for LLVM's memory model.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg)
Expand an atomic RMW instruction into a loop utilizing cmpxchg.
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * getPointerOperand()
Class to represent function types.
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Type * getType() const
All values are typed, get the type of this value.
static bool canUseSizedAtomicCall(unsigned Size, unsigned Align, const DataLayout &DL)
Value * CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name="")
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
BinOp
This enumeration lists the possible modifications atomicrmw can make.
AtomicOrdering getSuccessOrdering() const
Returns the success ordering constraint of this cmpxchg instruction.
static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr, Value *Loaded, Value *NewVal, AtomicOrdering MemOpOrder, Value *&Success, Value *&NewLoaded)
BasicBlock * GetInsertBlock() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
bool isLittleEndian() const
Layout endianness...
AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID=SyncScope::System)
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
An instruction for storing to memory.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
const BasicBlock & getEntryBlock() const
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
static bool runOnFunction(Function &F, bool PostInlining)
LLVM Basic Block Representation.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
UnreachableInst * CreateUnreachable()
size_t size() const
size - Get the array size.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Value * getCompareOperand()
void setAlignment(MaybeAlign Align)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
const Instruction & front() const
bool isAcquireOrStronger(AtomicOrdering ao)
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
static Type * getVoidTy(LLVMContext &C)
unsigned getLargestLegalIntTypeSizeInBits() const
Returns the size of largest legal integer type size, or 0 if none are set.
static unsigned getAtomicOpSize(LoadInst *LI)
void initializeAtomicExpandPass(PassRegistry &)
FunctionPass class - This class is used to implement most global optimizations.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Value * getPointerOperand()
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
self_iterator getIterator()
Class to represent integer types.
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
const Function * getFunction() const
Return the function this instruction belongs to.
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
INITIALIZE_PASS(AtomicExpand, DEBUG_TYPE, "Expand Atomic instructions", false, false) FunctionPass *llvm
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
unsigned getMaxAtomicSizeInBitsSupported() const
Returns the maximum atomic operation size (in bits) supported by the backend.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static Value * performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, Value *Loaded, Value *Inc)
Emit IR to implement the given atomicrmw operation on values in registers, returning the new value...
static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)
Returns the strongest permitted ordering on failure, given the desired ordering on success...
bool isReleaseOrStronger(AtomicOrdering ao)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
CallInst * CreateLifetimeEnd(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.end intrinsic.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
unsigned getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
This is the shared class of boolean and integer constants.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
static ConstantInt * getTrue(LLVMContext &Context)
void setAlignment(MaybeAlign Align)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void setVolatile(bool V)
Specify whether this is a volatile store or not.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
void setWeak(bool IsWeak)
iterator_range< user_iterator > users()
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
LLVM_NODISCARD AttributeList addAttribute(LLVMContext &C, unsigned Index, Attribute::AttrKind Kind) const
Add an attribute to the attribute set at the given index.
bool isVolatile() const
Return true if this is a store to a volatile memory location.
unsigned getAlignment() const
Return the alignment of the access that is being performed.
static IntegerType * getInt32Ty(LLVMContext &C)
AtomicOrdering getOrdering() const
Returns the ordering constraint of this store instruction.
Value * getPointerOperand()
const Function * getParent() const
Return the enclosing method, or null if none.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Value * CreateFSub(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
static unsigned getAtomicOpAlign(LoadInst *LI)
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
AtomicOrderingCABI toCABI(AtomicOrdering ao)
unsigned getAlignment() const
Return the alignment of the access that is being performed.
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
static ArrayRef< RTLIB::Libcall > GetRMWLibcall(AtomicRMWInst::BinOp Op)
BranchInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
BasicBlock::iterator GetInsertPoint() const
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Primary interface to the complete machine description for the target machine.
static Value * performMaskedAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, Value *Loaded, Value *Shifted_Inc, Value *Inc, const PartwordMaskValues &PMV)
Emit IR to implement a masked version of a given atomicrmw operation.
CallInst * CreateLifetimeStart(Value *Ptr, ConstantInt *Size=nullptr)
Create a lifetime.start intrinsic.
inst_iterator inst_end(Function *F)
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
FunctionPass * createAtomicExpandPass()
void setVolatile(bool V)
Specify whether this is a volatile load or not.
Value * CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name="")
Value * getPointerOperand()
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
bool empty() const
empty - Check if the array is empty.
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
an instruction to allocate memory on the stack
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...