29 #define DEBUG_TYPE "atomic-expand"
46 bool IsStore,
bool IsLoad);
48 bool expandAtomicLoadToLL(
LoadInst *LI);
49 bool expandAtomicLoadToCmpXchg(
LoadInst *LI);
63 "Expand Atomic calls in terms of either load-linked & store-conditional or cmpxchg",
67 return new AtomicExpand(TM);
70 bool AtomicExpand::runOnFunction(
Function &
F) {
71 if (!
TM || !
TM->getSubtargetImpl(F)->enableAtomicExpand())
73 TLI =
TM->getSubtargetImpl(F)->getTargetLowering();
84 bool MadeChange =
false;
85 for (
auto I : AtomicInsts) {
90 assert((LI || SI || RMWI || CASI || isa<FenceInst>(
I)) &&
91 "Unknown atomic instruction");
95 if (TLI->getInsertFencesForAtomic()) {
97 FenceOrdering = LI->getOrdering();
102 FenceOrdering = SI->getOrdering();
108 FenceOrdering = RMWI->getOrdering();
110 IsStore = IsLoad =
true;
111 }
else if (CASI && !TLI->hasLoadLinkedStoreConditional() &&
118 FenceOrdering = CASI->getSuccessOrdering();
121 IsStore = IsLoad =
true;
125 MadeChange |= bracketInstWithFences(
I, FenceOrdering, IsStore, IsLoad);
129 if (LI && TLI->shouldExpandAtomicLoadInIR(LI)) {
130 MadeChange |= expandAtomicLoad(LI);
131 }
else if (SI && TLI->shouldExpandAtomicStoreInIR(SI)) {
132 MadeChange |= expandAtomicStore(SI);
139 if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
142 MadeChange |= tryExpandAtomicRMW(RMWI);
144 }
else if (CASI && TLI->hasLoadLinkedStoreConditional()) {
145 MadeChange |= expandAtomicCmpXchg(CASI);
152 bool IsStore,
bool IsLoad) {
155 auto LeadingFence = TLI->emitLeadingFence(Builder, Order, IsStore, IsLoad);
157 auto TrailingFence = TLI->emitTrailingFence(Builder, Order, IsStore, IsLoad);
165 TrailingFence->removeFromParent();
166 TrailingFence->insertAfter(I);
169 return (LeadingFence || TrailingFence);
172 bool AtomicExpand::expandAtomicLoad(
LoadInst *LI) {
173 if (TLI->hasLoadLinkedStoreConditional())
174 return expandAtomicLoadToLL(LI);
176 return expandAtomicLoadToCmpXchg(LI);
179 bool AtomicExpand::expandAtomicLoadToLL(
LoadInst *LI) {
194 bool AtomicExpand::expandAtomicLoadToCmpXchg(
LoadInst *LI) {
198 Type *Ty = cast<PointerType>(Addr->
getType())->getElementType();
201 Value *Pair = Builder.CreateAtomicCmpXchg(
202 Addr, DummyVal, DummyVal, Order,
204 Value *Loaded = Builder.CreateExtractValue(Pair, 0,
"loaded");
212 bool AtomicExpand::expandAtomicStore(
StoreInst *SI) {
226 return tryExpandAtomicRMW(AI);
230 switch (TLI->shouldExpandAtomicRMWInIR(AI)) {
234 assert(TLI->hasLoadLinkedStoreConditional() &&
235 "TargetLowering requested we expand AtomicRMW instruction into "
236 "load-linked/store-conditional combos, but such instructions aren't "
239 return expandAtomicRMWToLLSC(AI);
242 return expandAtomicRMWToCmpXchg(AI);
257 return Builder.
CreateAdd(Loaded, Inc,
"new");
259 return Builder.
CreateSub(Loaded, Inc,
"new");
261 return Builder.
CreateAnd(Loaded, Inc,
"new");
265 return Builder.
CreateOr(Loaded, Inc,
"new");
267 return Builder.
CreateXor(Loaded, Inc,
"new");
270 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
273 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
276 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
279 return Builder.
CreateSelect(NewVal, Loaded, Inc,
"new");
285 bool AtomicExpand::expandAtomicRMWToLLSC(
AtomicRMWInst *AI) {
315 std::prev(BB->
end())->eraseFromParent();
316 Builder.SetInsertPoint(BB);
317 Builder.CreateBr(LoopBB);
320 Builder.SetInsertPoint(LoopBB);
321 Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
326 Value *StoreSuccess =
327 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
328 Value *TryAgain = Builder.CreateICmpNE(
330 Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);
332 Builder.SetInsertPoint(ExitBB, ExitBB->
begin());
340 bool AtomicExpand::expandAtomicRMWToCmpXchg(
AtomicRMWInst *AI) {
372 std::prev(BB->
end())->eraseFromParent();
373 Builder.SetInsertPoint(BB);
374 LoadInst *InitLoaded = Builder.CreateLoad(Addr);
377 Builder.CreateBr(LoopBB);
380 Builder.SetInsertPoint(LoopBB);
387 Value *Pair = Builder.CreateAtomicCmpXchg(
388 Addr, Loaded, NewVal, MemOpOrder,
390 Value *NewLoaded = Builder.CreateExtractValue(Pair, 0,
"newloaded");
391 Loaded->addIncoming(NewLoaded, LoopBB);
393 Value *
Success = Builder.CreateExtractValue(Pair, 1,
"success");
394 Builder.CreateCondBr(Success, ExitBB, LoopBB);
396 Builder.SetInsertPoint(ExitBB, ExitBB->
begin());
416 TLI->getInsertFencesForAtomic() ?
Monotonic : SuccessOrder;
455 std::prev(BB->
end())->eraseFromParent();
456 Builder.SetInsertPoint(BB);
457 TLI->emitLeadingFence(Builder, SuccessOrder,
true,
459 Builder.CreateBr(LoopBB);
462 Builder.SetInsertPoint(LoopBB);
463 Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
469 Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB);
471 Builder.SetInsertPoint(TryStoreBB);
472 Value *StoreSuccess = TLI->emitStoreConditional(
474 StoreSuccess = Builder.CreateICmpEQ(
476 Builder.CreateCondBr(StoreSuccess, SuccessBB,
477 CI->
isWeak() ? FailureBB : LoopBB);
480 Builder.SetInsertPoint(SuccessBB);
481 TLI->emitTrailingFence(Builder, SuccessOrder,
true,
483 Builder.CreateBr(ExitBB);
485 Builder.SetInsertPoint(FailureBB);
486 TLI->emitTrailingFence(Builder, FailureOrder,
true,
488 Builder.CreateBr(ExitBB);
495 Builder.SetInsertPoint(ExitBB, ExitBB->
begin());
509 "weird extraction from { iN, i1 }");
520 for (
auto EV : PrunedInsts)
528 Res = Builder.CreateInsertValue(Res, Success, 1);
550 return C->isMinusOne();
557 bool AtomicExpand::simplifyIdempotentRMW(
AtomicRMWInst* RMWI) {
558 if (
auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
559 if (TLI->shouldExpandAtomicLoadInIR(ResultingLoad))
560 expandAtomicLoad(ResultingLoad);
Value * getValueOperand()
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
void push_back(const T &Elt)
AtomicOrdering getFailureOrdering() const
Returns the ordering constraint on this cmpxchg.
static ConstantInt * getFalse(LLVMContext &Context)
static IntegerType * getInt1Ty(LLVMContext &C)
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
*p = old <signed v ? old : v
Value * CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name="")
AtomicOrdering getSuccessOrdering() const
Returns the ordering constraint on this cmpxchg.
bool isAtLeastAcquire(AtomicOrdering Ord)
Returns true if the ordering is at least as strong as acquire (i.e.
AtomicCmpXchgInst - an instruction that atomically checks whether a specified value is in a memory lo...
*p = old <unsigned v ? old : v
*p = old >unsigned v ? old : v
const Function * getParent() const
Return the enclosing method, or null if none.
Value * getNewValOperand()
char & AtomicExpandID
AtomicExpandID – Lowers atomic operations in terms of either cmpxchg load-linked/store-conditional lo...
LoadInst - an instruction for reading from memory.
AtomicRMWInst - an instruction that atomically reads a memory location, combines it with another valu...
*p = old >signed v ? old : v
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
static Constant * getNullValue(Type *Ty)
iterator begin()
Instruction iterator methods.
Value * CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
inst_iterator inst_begin(Function *F)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
FunctionPass * createAtomicExpandPass(const TargetMachine *TM)
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Value * getPointerOperand()
AtomicOrdering getOrdering() const
Returns the ordering constraint on this RMW.
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
BinOp
This enumeration lists the possible modifications atomicrmw can make.
StoreInst - an instruction for storing to memory.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This is an important base class in LLVM.
Value * getCompareOperand()
void initializeAtomicExpandPass(PassRegistry &)
FunctionPass class - This class is used to implement most global optimizations.
Value * getPointerOperand()
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
void setAlignment(unsigned Align)
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
static Value * performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, Value *Loaded, Value *Inc)
Emit IR to implement the given atomicrmw operation on values in registers, returning the new value...
static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)
Returns the strongest permitted ordering on failure, given the desired ordering on success...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
AtomicOrdering getOrdering() const
Returns the ordering effect of this store.
This is the shared class of boolean and integer constants.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
AtomicOrdering getOrdering() const
Returns the ordering effect of this fence.
static ConstantInt * getTrue(LLVMContext &Context)
BinOp getOperation() const
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="")
iterator_range< user_iterator > users()
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static IntegerType * getInt32Ty(LLVMContext &C)
Value * getPointerOperand()
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
bool isAtLeastRelease(AtomicOrdering Ord)
Returns true if the ordering is at least as strong as release (i.e.
INITIALIZE_TM_PASS(AtomicExpand,"atomic-expand","Expand Atomic calls in terms of either load-linked & store-conditional or cmpxchg", false, false) FunctionPass *llvm
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
getPrimitiveSizeInBits - Return the basic size of this type if it is a primitive type.
LLVM Value Representation.
Primary interface to the complete machine description for the target machine.
C - The default llvm calling convention, compatible with C.
inst_iterator inst_end(Function *F)
Value * CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name="")
Value * getPointerOperand()
const BasicBlock * getParent() const
This file describes how to lower LLVM code to machine code.