LLVM 24.0.0git
AtomicExpandPass.cpp
Go to the documentation of this file.
1//===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass (at IR level) to replace atomic instructions with
10// __atomic_* library calls, or target specific instruction which implement the
11// same semantics in a way which better fits the target backend. This can
12// include the use of (intrinsic-based) load-linked/store-conditional loops,
13// AtomicCmpXchg, or type coercions.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/ArrayRef.h"
29#include "llvm/IR/Attributes.h"
30#include "llvm/IR/BasicBlock.h"
31#include "llvm/IR/Constant.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/IRBuilder.h"
37#include "llvm/IR/Instruction.h"
39#include "llvm/IR/MDBuilder.h"
41#include "llvm/IR/Module.h"
43#include "llvm/IR/Type.h"
44#include "llvm/IR/User.h"
45#include "llvm/IR/Value.h"
47#include "llvm/Pass.h"
51#include "llvm/Support/Debug.h"
56#include <cassert>
57#include <cstdint>
58#include <iterator>
59
60using namespace llvm;
61
62#define DEBUG_TYPE "atomic-expand"
63
64namespace {
65
66class AtomicExpandImpl {
67 const TargetLowering *TLI = nullptr;
68 const LibcallLoweringInfo *LibcallLowering = nullptr;
69 const DataLayout *DL = nullptr;
70 bool SingleThreaded = false;
71
72private:
73 /// Callback type for emitting a cmpxchg instruction during RMW expansion.
74 /// Parameters: (Builder, Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
75 /// SSID, IsVolatile, /* OUT */ Success, /* OUT */ NewLoaded,
76 /// MetadataSrc)
77 using CreateCmpXchgInstFun = function_ref<void(
79 SyncScope::ID, bool, Value *&, Value *&, Instruction *)>;
80
81 void handleFailure(Instruction &FailedInst, const Twine &Msg,
82 Instruction *DiagnosticInst = nullptr) const {
83 LLVMContext &Ctx = FailedInst.getContext();
84
85 // TODO: Do not use generic error type.
86 Ctx.emitError(DiagnosticInst ? DiagnosticInst : &FailedInst, Msg);
87
88 if (!FailedInst.getType()->isVoidTy())
89 FailedInst.replaceAllUsesWith(PoisonValue::get(FailedInst.getType()));
90 FailedInst.eraseFromParent();
91 }
92
93 template <typename Inst>
94 void handleUnsupportedAtomicSize(Inst *I, const Twine &AtomicOpName,
95 Instruction *DiagnosticInst = nullptr) const;
96
97 bool bracketInstWithFences(Instruction *I, AtomicOrdering Order);
98 bool tryInsertTrailingSeqCstFence(Instruction *AtomicI);
99 template <typename AtomicInst>
100 bool tryInsertFencesForAtomic(AtomicInst *AtomicI, bool OrderingRequiresFence,
101 AtomicOrdering NewOrdering);
102 IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
103 LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI);
104 bool tryExpandAtomicLoad(LoadInst *LI);
105 bool expandAtomicLoadToLL(LoadInst *LI);
106 bool expandAtomicLoadToCmpXchg(LoadInst *LI);
107 StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI);
108 bool tryExpandAtomicStore(StoreInst *SI);
109 void expandAtomicStoreToXChg(StoreInst *SI);
110 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
111 void expandElementwiseAtomicRMW(AtomicRMWInst *AI);
112 void expandAtomicSubToAdd(AtomicRMWInst *AI);
113 AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
114 Value *
115 insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
116 Align AddrAlign, AtomicOrdering MemOpOrder,
117 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
118 void expandAtomicOpToLLSC(
119 Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
120 AtomicOrdering MemOpOrder,
121 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
122 void expandPartwordAtomicRMW(
124 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
125 bool expandPartwordCmpXchg(AtomicCmpXchgInst *I);
126 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
127 void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
128
129 AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
130 Value *insertRMWCmpXchgLoop(
131 IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
132 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
133 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
134 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc);
135 bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
136
137 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
138 bool isIdempotentRMW(AtomicRMWInst *RMWI);
139 bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
140
141 bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment,
142 Value *PointerOperand, Value *ValueOperand,
143 Value *CASExpected, AtomicOrdering Ordering,
144 AtomicOrdering Ordering2,
145 ArrayRef<RTLIB::Libcall> Libcalls);
146 void expandAtomicLoadToLibcall(LoadInst *LI);
147 void expandAtomicStoreToLibcall(StoreInst *LI);
148 void expandAtomicRMWToLibcall(AtomicRMWInst *I);
149 void expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
150 const Twine &AtomicOpName = "cmpxchg",
151 Instruction *DiagnosticInst = nullptr);
152
153 bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
154 CreateCmpXchgInstFun CreateCmpXchg);
155
156 bool lowerToNonAtomic(Instruction *I);
157 bool processAtomicInstr(Instruction *I);
158
159public:
160 bool run(Function &F, const ModuleLibcallLoweringInfo &LibcallResult,
161 const TargetMachine *TM);
162};
163
164class AtomicExpandLegacy : public FunctionPass {
165public:
166 static char ID; // Pass identification, replacement for typeid
167
168 AtomicExpandLegacy() : FunctionPass(ID) {}
169
170 void getAnalysisUsage(AnalysisUsage &AU) const override {
173 }
174
175 bool runOnFunction(Function &F) override;
176};
177
178// IRBuilder to be used for replacement atomic instructions.
179struct ReplacementIRBuilder
180 : IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> {
181 MDNode *MMRAMD = nullptr;
182 MDNode *PCSectionsMD = nullptr;
183
184 // Preserves the DebugLoc from I, and preserves still valid metadata.
185 // Enable StrictFP builder mode when appropriate.
186 explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL)
187 : IRBuilder(
188 I->getIterator(), InstSimplifyFolder(DL),
189 IRBuilderCallbackInserter([this](Instruction *I) { addMD(I); })) {
190 if (I->getFunction()->hasFnAttribute(Attribute::StrictFP))
191 this->setIsFPConstrained(true);
192
193 MMRAMD = I->getMetadata(LLVMContext::MD_mmra);
194 PCSectionsMD = I->getMetadata(LLVMContext::MD_pcsections);
195 }
196
197 void addMD(Instruction *I) {
199 I->setMetadata(LLVMContext::MD_mmra, MMRAMD);
200 I->setMetadata(LLVMContext::MD_pcsections, PCSectionsMD);
201 }
202};
203
204} // end anonymous namespace
205
206char AtomicExpandLegacy::ID = 0;
207
208char &llvm::AtomicExpandID = AtomicExpandLegacy::ID;
209
211 "Expand Atomic instructions", false, false)
214INITIALIZE_PASS_END(AtomicExpandLegacy, DEBUG_TYPE,
215 "Expand Atomic instructions", false, false)
216
217// Helper functions to retrieve the size of atomic instructions.
218static unsigned getAtomicOpSize(LoadInst *LI) {
219 const DataLayout &DL = LI->getDataLayout();
220 return DL.getTypeStoreSize(LI->getType());
221}
222
223static unsigned getAtomicOpSize(StoreInst *SI) {
224 const DataLayout &DL = SI->getDataLayout();
225 return DL.getTypeStoreSize(SI->getValueOperand()->getType());
226}
227
228static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {
229 const DataLayout &DL = RMWI->getDataLayout();
230 return DL.getTypeStoreSize(RMWI->getValOperand()->getType());
231}
232
233static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
234 const DataLayout &DL = CASI->getDataLayout();
235 return DL.getTypeStoreSize(CASI->getCompareOperand()->getType());
236}
237
238/// Copy metadata that's safe to preserve when widening atomics.
240 const Instruction &Source) {
242 Source.getAllMetadata(MD);
243 LLVMContext &Ctx = Dest.getContext();
244 MDBuilder MDB(Ctx);
245
246 for (auto [ID, N] : MD) {
247 switch (ID) {
248 case LLVMContext::MD_dbg:
249 case LLVMContext::MD_tbaa:
250 case LLVMContext::MD_tbaa_struct:
251 case LLVMContext::MD_alias_scope:
252 case LLVMContext::MD_mem_cache_hint:
253 case LLVMContext::MD_noalias:
254 case LLVMContext::MD_noalias_addrspace:
255 case LLVMContext::MD_access_group:
256 case LLVMContext::MD_mmra:
257 case LLVMContext::MD_atomic_ignore_denormal_mode:
258 Dest.setMetadata(ID, N);
259 break;
260 default:
261 if (ID == Ctx.getMDKindID("amdgpu.no.remote.memory"))
262 Dest.setMetadata(ID, N);
263 else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))
264 Dest.setMetadata(ID, N);
265 else if (ID == Ctx.getMDKindID("amdgpu.cfs"))
266 Dest.setMetadata(ID, N);
267 break;
268 }
269 }
270}
271
272template <typename Inst>
273static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
274 unsigned Size = getAtomicOpSize(I);
275 Align Alignment = I->getAlign();
276 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
277 return Size <= MaxSize && TLI->isAtomicAlignmentSupported(Alignment, Size);
278}
279
280template <typename Inst>
282 raw_ostream &OS) {
283 unsigned Size = getAtomicOpSize(I);
284 Align Alignment = I->getAlign();
285 bool NeedSeparator = false;
286
287 if (!TLI->isAtomicAlignmentSupported(Alignment, Size)) {
288 OS << "instruction alignment " << Alignment.value()
289 << " is smaller than the required " << Size
290 << "-byte alignment for this atomic operation";
291 NeedSeparator = true;
292 }
293
294 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
295 if (Size > MaxSize) {
296 if (NeedSeparator)
297 OS << "; ";
298 OS << "target supports atomics up to " << MaxSize
299 << " bytes, but this atomic accesses " << Size << " bytes";
300 }
301}
302
303template <typename Inst>
304void AtomicExpandImpl::handleUnsupportedAtomicSize(
305 Inst *I, const Twine &AtomicOpName, Instruction *DiagnosticInst) const {
306 assert(!atomicSizeSupported(TLI, I) && "expected unsupported atomic size");
307 SmallString<128> FailureReason;
308 raw_svector_ostream OS(FailureReason);
310 handleFailure(*I, Twine("unsupported ") + AtomicOpName + ": " + FailureReason,
311 DiagnosticInst);
312}
313
314bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
316 return false;
317
318 IRBuilder Builder(AtomicI);
319 if (auto *TrailingFence = TLI->emitTrailingFence(
320 Builder, AtomicI, AtomicOrdering::SequentiallyConsistent)) {
321 TrailingFence->moveAfter(AtomicI);
322 return true;
323 }
324 return false;
325}
326
327template <typename AtomicInst>
328bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
329 bool OrderingRequiresFence,
330 AtomicOrdering NewOrdering) {
331 bool ShouldInsertFences = TLI->shouldInsertFencesForAtomic(AtomicI);
332 if (OrderingRequiresFence && ShouldInsertFences) {
333 AtomicOrdering FenceOrdering = AtomicI->getOrdering();
334 AtomicI->setOrdering(NewOrdering);
335 return bracketInstWithFences(AtomicI, FenceOrdering);
336 }
337 if (!ShouldInsertFences)
338 return tryInsertTrailingSeqCstFence(AtomicI);
339 return false;
340}
341
342/// In a single-threaded environment, atomic operations can be lowered to their
343/// non-atomic equivalents: fences are removed, and atomic loads, stores, RMW,
344/// and cmpxchg become plain memory operations.
345bool AtomicExpandImpl::lowerToNonAtomic(Instruction *I) {
346 if (auto *FI = dyn_cast<FenceInst>(I)) {
347 FI->eraseFromParent();
348 return true;
349 }
350
351 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I))
352 return lowerAtomicCmpXchgInst(CXI);
353
354 if (auto *RMWI = dyn_cast<AtomicRMWInst>(I))
355 return lowerAtomicRMWInst(RMWI);
356
357 if (auto *LI = dyn_cast<LoadInst>(I)) {
358 if (LI->isAtomic()) {
359 LI->setAtomic(AtomicOrdering::NotAtomic);
360 LI->setElementwise(false);
361 return true;
362 }
363
364 return false;
365 }
366
367 if (auto *SI = dyn_cast<StoreInst>(I)) {
368 if (SI->isAtomic()) {
369 SI->setAtomic(AtomicOrdering::NotAtomic);
370 SI->setElementwise(false);
371 return true;
372 }
373
374 return false;
375 }
376
377 return false;
378}
379
380bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
381 if (SingleThreaded)
382 return lowerToNonAtomic(I);
383
384 if (auto *LI = dyn_cast<LoadInst>(I)) {
385 if (!LI->isAtomic())
386 return false;
387
388 if (!atomicSizeSupported(TLI, LI)) {
389 expandAtomicLoadToLibcall(LI);
390 return true;
391 }
392
393 bool MadeChange = false;
394 if (TLI->shouldCastAtomicLoadInIR(LI) ==
395 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
396 LI = convertAtomicLoadToIntegerType(LI);
397 MadeChange = true;
398 }
399
400 MadeChange |=
401 tryInsertFencesForAtomic(LI, isAcquireOrStronger(LI->getOrdering()),
403
404 MadeChange |= tryExpandAtomicLoad(LI);
405 return MadeChange;
406 }
407
408 if (auto *SI = dyn_cast<StoreInst>(I)) {
409 if (!SI->isAtomic())
410 return false;
411
412 if (!atomicSizeSupported(TLI, SI)) {
413 expandAtomicStoreToLibcall(SI);
414 return true;
415 }
416
417 bool MadeChange = false;
419 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
420 SI = convertAtomicStoreToIntegerType(SI);
421 MadeChange = true;
422 }
423
424 MadeChange |=
425 tryInsertFencesForAtomic(SI, isReleaseOrStronger(SI->getOrdering()),
427
428 MadeChange |= tryExpandAtomicStore(SI);
429 return MadeChange;
430 }
431
432 if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
433 bool IsElementwiseExpand =
434 RMWI->isElementwise() &&
436 TargetLoweringBase::AtomicExpansionKind::Expand;
437
438 if (!atomicSizeSupported(TLI, RMWI)) {
439 // Elementwise expansion may split an atomicrmw into smaller, supported
440 // atomic sizes.
441 if (!IsElementwiseExpand) {
442 expandAtomicRMWToLibcall(RMWI);
443 return true;
444 }
445 }
446
447 bool MadeChange = false;
448 if (!IsElementwiseExpand &&
450 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
451 RMWI = convertAtomicXchgToIntegerType(RMWI);
452 MadeChange = true;
453 }
454
455 MadeChange |= tryInsertFencesForAtomic(
456 RMWI, isStrongerThanMonotonic(RMWI->getOrdering()),
458
459 // There are two different ways of expanding RMW instructions:
460 // - into a load if it is idempotent
461 // - into a Cmpxchg/LL-SC loop otherwise
462 // we try them in that order.
463 MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
464 tryExpandAtomicRMW(RMWI);
465 return MadeChange;
466 }
467
468 if (auto *CASI = dyn_cast<AtomicCmpXchgInst>(I)) {
469 if (!atomicSizeSupported(TLI, CASI)) {
470 expandAtomicCASToLibcall(CASI);
471 return true;
472 }
473
474 // TODO: when we're ready to make the change at the IR level, we can
475 // extend convertCmpXchgToInteger for floating point too.
476 bool MadeChange = false;
477 if (CASI->getCompareOperand()->getType()->isPointerTy()) {
478 // TODO: add a TLI hook to control this so that each target can
479 // convert to lowering the original type one at a time.
480 CASI = convertCmpXchgToIntegerType(CASI);
481 MadeChange = true;
482 }
483
484 auto CmpXchgExpansion = TLI->shouldExpandAtomicCmpXchgInIR(CASI);
485 if (TLI->shouldInsertFencesForAtomic(CASI)) {
486 if (CmpXchgExpansion == TargetLoweringBase::AtomicExpansionKind::None &&
487 (isReleaseOrStronger(CASI->getSuccessOrdering()) ||
488 isAcquireOrStronger(CASI->getSuccessOrdering()) ||
489 isAcquireOrStronger(CASI->getFailureOrdering()))) {
490 // If a compare and swap is lowered to LL/SC, we can do smarter fence
491 // insertion, with a stronger one on the success path than on the
492 // failure path. As a result, fence insertion is directly done by
493 // expandAtomicCmpXchg in that case.
494 AtomicOrdering FenceOrdering = CASI->getMergedOrdering();
495 AtomicOrdering CASOrdering =
497 CASI->setSuccessOrdering(CASOrdering);
498 CASI->setFailureOrdering(CASOrdering);
499 MadeChange |= bracketInstWithFences(CASI, FenceOrdering);
500 }
501 } else if (CmpXchgExpansion !=
502 TargetLoweringBase::AtomicExpansionKind::LLSC) {
503 // CmpXchg LLSC is handled in expandAtomicCmpXchg().
504 MadeChange |= tryInsertTrailingSeqCstFence(CASI);
505 }
506
507 MadeChange |= tryExpandAtomicCmpXchg(CASI);
508 return MadeChange;
509 }
510
511 return false;
512}
513
514bool AtomicExpandImpl::run(Function &F,
515 const ModuleLibcallLoweringInfo &LibcallResult,
516 const TargetMachine *TM) {
517 SingleThreaded = F.getParent()->getThreadModel() == ThreadModel::Single;
518
519 const auto *Subtarget = TM->getSubtargetImpl(F);
520 // In a single-threaded environment atomics are lowered to non-atomic form
521 if (!SingleThreaded && !Subtarget->enableAtomicExpand())
522 return false;
524 LibcallLowering = &getLibcallLowering(LibcallResult, *Subtarget);
525 DL = &F.getDataLayout();
526
527 bool MadeChange = false;
528
529 for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
530 BasicBlock *BB = &*BBI;
531
533
534 for (BasicBlock::reverse_iterator I = BB->rbegin(), E = BB->rend(); I != E;
535 I = Next) {
536 Instruction &Inst = *I;
537 Next = std::next(I);
538
539 if (processAtomicInstr(&Inst)) {
540 MadeChange = true;
541
542 // New blocks may have been inserted.
543 BBE = F.end();
544 }
545 }
546 }
547
548 return MadeChange;
549}
550
551bool AtomicExpandLegacy::runOnFunction(Function &F) {
552
553 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
554 if (!TPC)
555 return false;
556 auto *TM = &TPC->getTM<TargetMachine>();
557
558 const ModuleLibcallLoweringInfo &LibcallResult =
559 getAnalysis<LibcallLoweringInfoWrapper>().getResult(*F.getParent());
560 AtomicExpandImpl AE;
561 return AE.run(F, LibcallResult, TM);
562}
563
565 return new AtomicExpandLegacy();
566}
567
570 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
571
572 const ModuleLibcallLoweringInfo *LibcallResult =
573 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(*F.getParent());
574
575 if (!LibcallResult) {
576 F.getContext().emitError("'" + LibcallLoweringModuleAnalysis::name() +
577 "' analysis required");
578 return PreservedAnalyses::all();
579 }
580
581 AtomicExpandImpl AE;
582
583 bool Changed = AE.run(F, *LibcallResult, TM);
584 if (!Changed)
585 return PreservedAnalyses::all();
586
588}
589
590bool AtomicExpandImpl::bracketInstWithFences(Instruction *I,
591 AtomicOrdering Order) {
592 ReplacementIRBuilder Builder(I, *DL);
593
594 auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);
595
596 auto TrailingFence = TLI->emitTrailingFence(Builder, I, Order);
597 // We have a guard here because not every atomic operation generates a
598 // trailing fence.
599 if (TrailingFence)
600 TrailingFence->moveAfter(I);
601
602 return (LeadingFence || TrailingFence);
603}
604
605/// Get the iX type with the same bitwidth as T.
607AtomicExpandImpl::getCorrespondingIntegerType(Type *T, const DataLayout &DL) {
608 EVT VT = TLI->getMemValueType(DL, T);
609 unsigned BitWidth = VT.getStoreSizeInBits();
610 assert(BitWidth == VT.getSizeInBits() && "must be a power of two");
611 return IntegerType::get(T->getContext(), BitWidth);
612}
613
614/// Convert an atomic load of a non-integral type to an integer load of the
615/// equivalent bitwidth. See the function comment on
616/// convertAtomicStoreToIntegerType for background.
617LoadInst *AtomicExpandImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {
618 auto *M = LI->getModule();
619 Type *NewTy = getCorrespondingIntegerType(LI->getType(), M->getDataLayout());
620
621 ReplacementIRBuilder Builder(LI, *DL);
622
623 Value *Addr = LI->getPointerOperand();
624
625 auto *NewLI = Builder.CreateLoad(NewTy, Addr, LI->getProperties());
626 LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");
627
628 Value *NewVal = LI->getType()->isPtrOrPtrVectorTy()
629 ? Builder.CreateIntToPtr(NewLI, LI->getType())
630 : Builder.CreateBitCast(NewLI, LI->getType());
631 LI->replaceAllUsesWith(NewVal);
632 LI->eraseFromParent();
633 return NewLI;
634}
635
636AtomicRMWInst *
637AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
639
640 auto *M = RMWI->getModule();
641 Type *NewTy =
642 getCorrespondingIntegerType(RMWI->getType(), M->getDataLayout());
643
644 ReplacementIRBuilder Builder(RMWI, *DL);
645
646 Value *Addr = RMWI->getPointerOperand();
647 Value *Val = RMWI->getValOperand();
648 Value *NewVal = Builder.CreateBitPreservingCastChain(*DL, Val, NewTy);
649
650 auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,
651 RMWI->getAlign(), RMWI->getOrdering(),
652 RMWI->getSyncScopeID());
653 NewRMWI->setVolatile(RMWI->isVolatile());
654 copyMetadataForAtomic(*NewRMWI, *RMWI);
655 LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
656
657 if (!RMWI->use_empty()) {
658 Value *NewRVal =
659 Builder.CreateBitPreservingCastChain(*DL, NewRMWI, RMWI->getType());
660 RMWI->replaceAllUsesWith(NewRVal);
661 }
662
663 RMWI->eraseFromParent();
664 return NewRMWI;
665}
666
667/// Generic Expand handling for elementwise vector atomicrmw instructions.
668/// Halve an `atomicrmw elementwise <N x T>` and feed each half back through the
669/// normal atomic expansion pipeline. The target's shouldExpandAtomicRMWInIR()
670/// is re-queried at the halved width and may return any expansion kind there
671/// (e.g. None to preserve as a native vector atomic, Expand to keep halving,
672/// CmpXChg to emit one wide cmpxchg loop at the halved width). Two cases:
673/// N == 1: collapse directly to one scalar atomicrmw T
674/// N > 1: split into two atomicrmw elementwise <N/2 x T> (low at Ptr, high
675/// at gep inbounds <N/2 x T>, Ptr, 1); reassemble via a
676/// shufflevector.
677/// Note that halving works because the vectors must always be a power of two.
678///
679/// Each new atomicrmw inherits the original ordering/syncscope/volatility and
680/// metadata, and is fed back through processAtomicInstr() so further expansion
681/// fires per-half.
682void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
683 assert(AI->isElementwise());
684 auto *VecTy = cast<FixedVectorType>(AI->getType());
685 Type *LaneTy = VecTy->getElementType();
686 const unsigned NumLanes = VecTy->getNumElements();
687 Value *Ptr = AI->getPointerOperand();
688 Value *Val = AI->getValOperand();
689
690 ReplacementIRBuilder Builder(AI, *DL);
691 Builder.disableGEPCanonicalization();
692 IntegerType *IdxTy =
693 DL->getIndexType(AI->getContext(), AI->getPointerAddressSpace());
694
695 auto CreateRMWInstruction = [&](Value *HalfPtr, Value *HalfVal, Align A,
696 bool Elementwise) -> AtomicRMWInst * {
697 auto *NewAI =
698 Builder.CreateAtomicRMW(AI->getOperation(), HalfPtr, HalfVal, A,
699 AI->getOrdering(), AI->getSyncScopeID());
700 NewAI->setVolatile(AI->isVolatile());
701 NewAI->setElementwise(Elementwise);
702 copyMetadataForAtomic(*NewAI, *AI);
703 return NewAI;
704 };
705
706 Constant *IdxZero = ConstantInt::get(IdxTy, 0);
707 Constant *IdxOne = ConstantInt::get(IdxTy, 1);
708
709 // N == 1: collapse to a single scalar atomicrmw T and wrap the result back
710 // into a <1 x T> vector.
711 if (NumLanes == 1) {
712 Value *LaneVal = Builder.CreateExtractElement(Val, IdxZero, "lane.val");
713 AtomicRMWInst *LaneRMW = CreateRMWInstruction(Ptr, LaneVal, AI->getAlign(),
714 /*Elementwise=*/false);
715 Value *Result = Builder.CreateInsertElement(PoisonValue::get(VecTy),
716 LaneRMW, IdxZero, "lane.old");
717 AI->replaceAllUsesWith(Result);
718 AI->eraseFromParent();
719 processAtomicInstr(LaneRMW);
720 return;
721 }
722
723 // N > 1: split into two <N/2 x T> elementwise atomicrmws and recurse via
724 // processAtomicInstr().
725 assert(isPowerOf2_32(NumLanes) &&
726 "elementwise atomicrmw vector length must be a power of two");
727 const unsigned HalfLanes = NumLanes / 2;
728 auto *HalfVecTy = FixedVectorType::get(LaneTy, HalfLanes);
729 const uint64_t HalfBytes = DL->getTypeStoreSize(HalfVecTy).getFixedValue();
730
731 Value *LoVal = Builder.CreateShuffleVector(
732 Val, createSequentialMask(0, HalfLanes, 0), "lo.val");
733 Value *HiVal = Builder.CreateShuffleVector(
734 Val, createSequentialMask(HalfLanes, HalfLanes, 0), "hi.val");
735
736 Value *HiPtr = Builder.CreateInBoundsGEP(HalfVecTy, Ptr, IdxOne, "hi.ptr");
737 Align LoAlign = AI->getAlign();
738 Align HiAlign = commonAlignment(LoAlign, HalfBytes);
739
740 AtomicRMWInst *LoRMW =
741 CreateRMWInstruction(Ptr, LoVal, LoAlign, /*Elementwise=*/true);
742 AtomicRMWInst *HiRMW =
743 CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/true);
744
745 Value *Result = concatenateVectors(Builder, {LoRMW, HiRMW});
746 Result->setName("old");
747
748 AI->replaceAllUsesWith(Result);
749 AI->eraseFromParent();
750 processAtomicInstr(LoRMW);
751 processAtomicInstr(HiRMW);
752}
753
754bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
755 switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
756 case TargetLoweringBase::AtomicExpansionKind::None:
757 return false;
758 case TargetLoweringBase::AtomicExpansionKind::LLSC:
759 expandAtomicOpToLLSC(
760 LI, LI->getType(), LI->getPointerOperand(), LI->getAlign(),
761 LI->getOrdering(),
762 [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
763 return true;
764 case TargetLoweringBase::AtomicExpansionKind::LLOnly:
765 return expandAtomicLoadToLL(LI);
766 case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
767 return expandAtomicLoadToCmpXchg(LI);
768 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
769 LI->setAtomic(AtomicOrdering::NotAtomic);
770 return true;
771 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
773 return true;
774 default:
775 llvm_unreachable("Unhandled case in tryExpandAtomicLoad");
776 }
777}
778
779bool AtomicExpandImpl::tryExpandAtomicStore(StoreInst *SI) {
780 switch (TLI->shouldExpandAtomicStoreInIR(SI)) {
781 case TargetLoweringBase::AtomicExpansionKind::None:
782 return false;
783 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
785 return true;
786 case TargetLoweringBase::AtomicExpansionKind::Expand:
787 expandAtomicStoreToXChg(SI);
788 return true;
789 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
790 SI->setAtomic(AtomicOrdering::NotAtomic);
791 return true;
792 default:
793 llvm_unreachable("Unhandled case in tryExpandAtomicStore");
794 }
795}
796
797bool AtomicExpandImpl::expandAtomicLoadToLL(LoadInst *LI) {
798 ReplacementIRBuilder Builder(LI, *DL);
799
800 // On some architectures, load-linked instructions are atomic for larger
801 // sizes than normal loads. For example, the only 64-bit load guaranteed
802 // to be single-copy atomic by ARM is an ldrexd (A3.5.3).
803 Value *Val = TLI->emitLoadLinked(Builder, LI->getType(),
804 LI->getPointerOperand(), LI->getOrdering());
806
807 LI->replaceAllUsesWith(Val);
808 LI->eraseFromParent();
809
810 return true;
811}
812
813bool AtomicExpandImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {
814 ReplacementIRBuilder Builder(LI, *DL);
815 AtomicOrdering Order = LI->getOrdering();
816 if (Order == AtomicOrdering::Unordered)
817 Order = AtomicOrdering::Monotonic;
818
819 Value *Addr = LI->getPointerOperand();
820 Type *Ty = LI->getType();
821
822 // cmpxchg supports only integer and pointer operands. If the load type is
823 // FP or vector, run the cmpxchg on the same-sized integer and bitcast the
824 // result back; mirrors createCmpXchgInstFun.
825 bool NeedBitcast = Ty->isFloatingPointTy() || Ty->isVectorTy();
826 Type *CmpXchgTy = Ty;
827 if (NeedBitcast)
828 CmpXchgTy = getCorrespondingIntegerType(Ty, *DL);
829 Constant *DummyVal = Constant::getNullValue(CmpXchgTy);
830
831 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
832 Addr, DummyVal, DummyVal, LI->getAlign(), Order,
834 LI->getSyncScopeID());
835 Pair->setVolatile(LI->isVolatile());
836 Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");
837 if (NeedBitcast)
838 Loaded = Builder.CreateBitPreservingCastChain(*DL, Loaded, Ty);
839
840 LI->replaceAllUsesWith(Loaded);
841 LI->eraseFromParent();
842
843 return true;
844}
845
846/// Convert an atomic store of a non-integral type to an integer store of the
847/// equivalent bitwidth. We used to not support floating point or vector
848/// atomics in the IR at all. The backends learned to deal with the bitcast
849/// idiom because that was the only way of expressing the notion of a atomic
850/// float or vector store. The long term plan is to teach each backend to
851/// instruction select from the original atomic store, but as a migration
852/// mechanism, we convert back to the old format which the backends understand.
853/// Each backend will need individual work to recognize the new format.
854StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
855 ReplacementIRBuilder Builder(SI, *DL);
856 auto *M = SI->getModule();
857 Type *NewTy = getCorrespondingIntegerType(SI->getValueOperand()->getType(),
858 M->getDataLayout());
859 Value *NewVal = SI->getValueOperand()->getType()->isPtrOrPtrVectorTy()
860 ? Builder.CreatePtrToInt(SI->getValueOperand(), NewTy)
861 : Builder.CreateBitCast(SI->getValueOperand(), NewTy);
862
863 Value *Addr = SI->getPointerOperand();
864
865 StoreInst *NewSI = Builder.CreateStore(NewVal, Addr, SI->getProperties());
866 copyMetadataForAtomic(*NewSI, *SI);
867 LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
868 SI->eraseFromParent();
869 return NewSI;
870}
871
872void AtomicExpandImpl::expandAtomicStoreToXChg(StoreInst *SI) {
873 // This function is only called on atomic stores that are too large to be
874 // atomic if implemented as a native store. So we replace them by an
875 // atomic swap, that can be implemented for example as a ldrex/strex on ARM
876 // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
877 // It is the responsibility of the target to only signal expansion via
878 // shouldExpandAtomicRMW in cases where this is required and possible.
879 ReplacementIRBuilder Builder(SI, *DL);
880 AtomicOrdering Ordering = SI->getOrdering();
881 assert(Ordering != AtomicOrdering::NotAtomic);
882 AtomicOrdering RMWOrdering = Ordering == AtomicOrdering::Unordered
883 ? AtomicOrdering::Monotonic
884 : Ordering;
885 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
886 AtomicRMWInst::Xchg, SI->getPointerOperand(), SI->getValueOperand(),
887 SI->getAlign(), RMWOrdering, SI->getSyncScopeID());
888 AI->setVolatile(SI->isVolatile());
889 SI->eraseFromParent();
890
891 if (TLI->shouldCastAtomicRMWIInIR(AI) ==
892 TargetLoweringBase::AtomicExpansionKind::CastToInteger)
893 AI = convertAtomicXchgToIntegerType(AI);
894
895 // Now we have an appropriate swap instruction, lower it as usual.
896 tryExpandAtomicRMW(AI);
897}
898
899static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
900 Value *Loaded, Value *NewVal, Align AddrAlign,
901 AtomicOrdering MemOpOrder, SyncScope::ID SSID,
902 bool IsVolatile, Value *&Success,
903 Value *&NewLoaded, Instruction *MetadataSrc) {
904 Type *OrigTy = NewVal->getType();
905
906 // This code can go away when cmpxchg supports FP and vector types.
907 assert(!OrigTy->isPointerTy());
908 bool NeedBitcast = OrigTy->isFloatingPointTy() || OrigTy->isVectorTy();
909 if (NeedBitcast) {
910 IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits());
911 NewVal = Builder.CreateBitCast(NewVal, IntTy);
912 Loaded = Builder.CreateBitCast(Loaded, IntTy);
913 }
914
915 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
916 Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
918 Pair->setVolatile(IsVolatile);
919 if (MetadataSrc)
920 copyMetadataForAtomic(*Pair, *MetadataSrc);
921
922 Success = Builder.CreateExtractValue(Pair, 1, "success");
923 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
924
925 if (NeedBitcast)
926 NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
927}
928
929void AtomicExpandImpl::expandAtomicSubToAdd(AtomicRMWInst *AI) {
930 ReplacementIRBuilder Builder(AI, *DL);
932 Value *NewVal;
933
934 switch (AI->getOperation()) {
936 NewOp = AtomicRMWInst::Add;
937 NewVal = Builder.CreateNeg(AI->getValOperand(), "neg");
938 break;
940 NewOp = AtomicRMWInst::FAdd;
941 NewVal = Builder.CreateFNeg(AI->getValOperand(), "fneg");
942 break;
943 default:
944 llvm_unreachable("unsupported atomicrmw expansion");
945 }
946
947 AI->setOperation(NewOp);
948 AI->setOperand(1, NewVal);
949}
950
951bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
952 LLVMContext &Ctx = AI->getModule()->getContext();
953 TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(AI);
954 switch (Kind) {
955 case TargetLoweringBase::AtomicExpansionKind::None:
956 return false;
957 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
958 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
959 unsigned ValueSize = getAtomicOpSize(AI);
960 if (ValueSize < MinCASSize) {
961 expandPartwordAtomicRMW(AI,
962 TargetLoweringBase::AtomicExpansionKind::LLSC);
963 } else {
964 auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
965 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
966 AI->getValOperand());
967 };
968 expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(),
969 AI->getAlign(), AI->getOrdering(), PerformOp);
970 }
971 return true;
972 }
973 case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
974 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
975 unsigned ValueSize = getAtomicOpSize(AI);
976 if (ValueSize < MinCASSize) {
977 expandPartwordAtomicRMW(AI,
978 TargetLoweringBase::AtomicExpansionKind::CmpXChg);
979 } else {
981 Ctx.getSyncScopeNames(SSNs);
982 auto MemScope = SSNs[AI->getSyncScopeID()].empty()
983 ? "system"
984 : SSNs[AI->getSyncScopeID()];
985 OptimizationRemarkEmitter ORE(AI->getFunction());
986 ORE.emit([&]() {
987 return OptimizationRemark(DEBUG_TYPE, "Passed", AI)
988 << "A compare and swap loop was generated for an atomic "
989 << AI->getOperationName(AI->getOperation()) << " operation at "
990 << MemScope << " memory scope";
991 });
992 expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun);
993 }
994 return true;
995 }
996 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
997 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
998 unsigned ValueSize = getAtomicOpSize(AI);
999 if (ValueSize < MinCASSize) {
1001 // Widen And/Or/Xor and give the target another chance at expanding it.
1004 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI));
1005 return true;
1006 }
1007 }
1008 expandAtomicRMWToMaskedIntrinsic(AI);
1009 return true;
1010 }
1011 case TargetLoweringBase::AtomicExpansionKind::BitTestIntrinsic: {
1013 return true;
1014 }
1015 case TargetLoweringBase::AtomicExpansionKind::CmpArithIntrinsic: {
1017 return true;
1018 }
1019 case TargetLoweringBase::AtomicExpansionKind::Expand:
1020 if (AI->isElementwise()) {
1021 expandElementwiseAtomicRMW(AI);
1022 return true;
1023 }
1024 expandAtomicSubToAdd(AI);
1025 return true;
1026 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
1027 return lowerAtomicRMWInst(AI);
1028 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
1030 return true;
1031 default:
1032 llvm_unreachable("Unhandled case in tryExpandAtomicRMW");
1033 }
1034}
1035
1036namespace {
1037
1038struct PartwordMaskValues {
1039 // These three fields are guaranteed to be set by createMaskInstrs.
1040 Type *WordType = nullptr;
1041 Type *ValueType = nullptr;
1042 Type *IntValueType = nullptr;
1043 Value *AlignedAddr = nullptr;
1044 Align AlignedAddrAlignment;
1045 // The remaining fields can be null.
1046 Value *ShiftAmt = nullptr;
1047 Value *Mask = nullptr;
1048 Value *Inv_Mask = nullptr;
1049};
1050
1051[[maybe_unused]]
1052raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
1053 auto PrintObj = [&O](auto *V) {
1054 if (V)
1055 O << *V;
1056 else
1057 O << "nullptr";
1058 O << '\n';
1059 };
1060 O << "PartwordMaskValues {\n";
1061 O << " WordType: ";
1062 PrintObj(PMV.WordType);
1063 O << " ValueType: ";
1064 PrintObj(PMV.ValueType);
1065 O << " AlignedAddr: ";
1066 PrintObj(PMV.AlignedAddr);
1067 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
1068 O << " ShiftAmt: ";
1069 PrintObj(PMV.ShiftAmt);
1070 O << " Mask: ";
1071 PrintObj(PMV.Mask);
1072 O << " Inv_Mask: ";
1073 PrintObj(PMV.Inv_Mask);
1074 O << "}\n";
1075 return O;
1076}
1077
1078} // end anonymous namespace
1079
1080/// This is a helper function which builds instructions to provide
1081/// values necessary for partword atomic operations. It takes an
1082/// incoming address, Addr, and ValueType, and constructs the address,
1083/// shift-amounts and masks needed to work with a larger value of size
1084/// WordSize.
1085///
1086/// AlignedAddr: Addr rounded down to a multiple of WordSize
1087///
1088/// ShiftAmt: Number of bits to right-shift a WordSize value loaded
1089/// from AlignAddr for it to have the same value as if
1090/// ValueType was loaded from Addr.
1091///
1092/// Mask: Value to mask with the value loaded from AlignAddr to
1093/// include only the part that would've been loaded from Addr.
1094///
1095/// Inv_Mask: The inverse of Mask.
1096static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
1098 Value *Addr, Align AddrAlign,
1099 unsigned MinWordSize) {
1100 PartwordMaskValues PMV;
1101
1102 Module *M = I->getModule();
1103 LLVMContext &Ctx = M->getContext();
1104 const DataLayout &DL = M->getDataLayout();
1105 unsigned ValueSize = DL.getTypeStoreSize(ValueType);
1106
1107 PMV.ValueType = PMV.IntValueType = ValueType;
1108 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy())
1109 PMV.IntValueType =
1110 Type::getIntNTy(Ctx, ValueType->getPrimitiveSizeInBits());
1111
1112 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(Ctx, MinWordSize * 8)
1113 : ValueType;
1114 if (PMV.ValueType == PMV.WordType) {
1115 PMV.AlignedAddr = Addr;
1116 PMV.AlignedAddrAlignment = AddrAlign;
1117 PMV.ShiftAmt = ConstantInt::get(PMV.ValueType, 0);
1118 PMV.Mask = ConstantInt::get(PMV.ValueType, ~0, /*isSigned*/ true);
1119 return PMV;
1120 }
1121
1122 PMV.AlignedAddrAlignment = Align(MinWordSize);
1123
1124 assert(ValueSize < MinWordSize);
1125
1126 PointerType *PtrTy = cast<PointerType>(Addr->getType());
1127 IntegerType *IntTy = DL.getIndexType(Ctx, PtrTy->getAddressSpace());
1128 Value *PtrLSB;
1129
1130 if (AddrAlign < MinWordSize) {
1131 PMV.AlignedAddr = Builder.CreateIntrinsic(
1132 Intrinsic::ptrmask, {PtrTy, IntTy},
1133 {Addr, ConstantInt::getSigned(IntTy, ~(uint64_t)(MinWordSize - 1))},
1134 nullptr, "AlignedAddr");
1135
1136 Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
1137 PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");
1138 } else {
1139 // If the alignment is high enough, the LSB are known 0.
1140 PMV.AlignedAddr = Addr;
1141 PtrLSB = ConstantInt::getNullValue(IntTy);
1142 }
1143
1144 if (DL.isLittleEndian()) {
1145 // turn bytes into bits
1146 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3);
1147 } else {
1148 // turn bytes into bits, and count from the other side.
1149 PMV.ShiftAmt = Builder.CreateShl(
1150 Builder.CreateXor(PtrLSB, MinWordSize - ValueSize), 3);
1151 }
1152
1153 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt");
1154 PMV.Mask = Builder.CreateShl(
1155 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt,
1156 "Mask");
1157
1158 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask");
1159
1160 return PMV;
1161}
1162
1163static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1164 const PartwordMaskValues &PMV) {
1165 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1166 if (PMV.WordType == PMV.ValueType)
1167 return WideWord;
1168
1169 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted");
1170 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType, "extracted");
1171 return Builder.CreateBitCast(Trunc, PMV.ValueType);
1172}
1173
1174static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1175 Value *Updated, const PartwordMaskValues &PMV) {
1176 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1177 assert(Updated->getType() == PMV.ValueType && "Value type mismatch");
1178 if (PMV.WordType == PMV.ValueType)
1179 return Updated;
1180
1181 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType);
1182
1183 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended");
1184 Value *Shift =
1185 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true);
1186 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked");
1187 Value *Or = Builder.CreateOr(And, Shift, "inserted");
1188 return Or;
1189}
1190
1191/// Emit IR to implement a masked version of a given atomicrmw
1192/// operation. (That is, only the bits under the Mask should be
1193/// affected by the operation)
1195 IRBuilderBase &Builder, Value *Loaded,
1196 Value *ValOperand_Shifted, Value *Inc,
1197 const PartwordMaskValues &PMV) {
1198 // TODO: update to use
1199 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge in order
1200 // to merge bits from two values without requiring PMV.Inv_Mask.
1201
1204 "Or/Xor/And handled by widenPartwordAtomicRMW");
1205
1206 if (Op == AtomicRMWInst::Xchg) {
1207 // Clear all the bits we are exchanging out. These are the bits under the
1208 // mask. We can clear them with an `and` of the inverse mask.
1209 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
1210 // Now that the prevous bits are cleared, we can swap in the new value with
1211 // an `or`.
1212 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, ValOperand_Shifted);
1213 return FinalVal;
1214 }
1215
1216 if (Op == AtomicRMWInst::Nand ||
1217 (!PMV.ValueType->isVectorTy() &&
1219 // For `Nand` and non-vector `Add` and `Sub`, we can perform the operation
1220 // on the entire word because the extra bits in the unmasked region don't
1221 // affect the computation in the masked region. The operation might still
1222 // overwrite the unmasked region (e.g. from integer overflow or underflow),
1223 // so we have to reapply the unmasked region afterwards.
1224 //
1225 // This trick doesn't work for vector `Add` and `Sub` because we use a
1226 // scalar operation on the entire word. Scalarizing vector `Add` and `Sub`
1227 // isn't legal because the vector versions may have element-wise overflows.
1228 // TODO: For these, can we use a wider vector op with additional lanes?
1229
1230 // Atomic operation across the entire word.
1231 Value *NewVal =
1232 buildAtomicRMWValue(Op, Builder, Loaded, ValOperand_Shifted);
1233 // Reapply the bits in the unmasked region.
1234 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
1235 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
1236 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked);
1237 return FinalVal;
1238 }
1239
1240 // All other ops operate on the sub-word size. Truncate down to the
1241 // original size, and expand out again after doing the operation. Bitcasts
1242 // will be inserted for FP values.
1243 assert(!ValOperand_Shifted);
1244 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV);
1245 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded_Extract, Inc);
1246 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV);
1247 return FinalVal;
1248}
1249
1250/// Expand a sub-word atomicrmw operation into an appropriate
1251/// word-sized operation.
1252///
1253/// It will create an LL/SC or cmpxchg loop, as appropriate, the same
1254/// way as a typical atomicrmw expansion. The only difference here is
1255/// that the operation inside of the loop may operate upon only a
1256/// part of the value.
1257void AtomicExpandImpl::expandPartwordAtomicRMW(
1258 AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
1259 // Widen And/Or/Xor and give the target another chance at expanding it.
1263 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI));
1264 return;
1265 }
1266 AtomicOrdering MemOpOrder = AI->getOrdering();
1267 SyncScope::ID SSID = AI->getSyncScopeID();
1268
1269 ReplacementIRBuilder Builder(AI, *DL);
1270
1271 PartwordMaskValues PMV =
1272 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
1273 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
1274
1275 Value *ValOperand_Shifted = nullptr;
1276 bool NeedsShiftedOperand =
1278 (!PMV.ValueType->isVectorTy() &&
1280
1281 if (NeedsShiftedOperand) {
1282 Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
1283 ValOperand_Shifted =
1284 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
1285 "ValOperand_Shifted");
1286 }
1287
1288 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
1289 return performMaskedAtomicOp(Op, Builder, Loaded, ValOperand_Shifted,
1290 AI->getValOperand(), PMV);
1291 };
1292
1293 Value *OldResult;
1294 if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
1295 OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr,
1296 PMV.AlignedAddrAlignment, MemOpOrder, SSID,
1297 AI->isVolatile(), PerformPartwordOp,
1299 } else {
1300 assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
1301 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
1302 PMV.AlignedAddrAlignment, MemOpOrder,
1303 PerformPartwordOp);
1304 }
1305
1306 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
1307 AI->replaceAllUsesWith(FinalOldResult);
1308 AI->eraseFromParent();
1309}
1310
1311// Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
1312AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
1313 ReplacementIRBuilder Builder(AI, *DL);
1315
1317 Op == AtomicRMWInst::And) &&
1318 "Unable to widen operation");
1319
1320 PartwordMaskValues PMV =
1321 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
1322 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
1323
1324 Value *ValOp = AI->getValOperand();
1325 if (ValOp->getType()->isVectorTy())
1326 // For vectors, bitcast to the integer type before extending. Note that
1327 // or/xor/and on vectors are equivalent to the same operation on an integer
1328 // that spans the vector, so we can use the integer type for the operation.
1329 ValOp = Builder.CreateBitCast(ValOp, PMV.IntValueType);
1330 Value *ValOperand_Shifted =
1331 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
1332 "ValOperand_Shifted");
1333
1334 Value *NewOperand;
1335
1336 if (Op == AtomicRMWInst::And)
1337 NewOperand =
1338 Builder.CreateOr(ValOperand_Shifted, PMV.Inv_Mask, "AndOperand");
1339 else
1340 NewOperand = ValOperand_Shifted;
1341
1342 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
1343 Op, PMV.AlignedAddr, NewOperand, PMV.AlignedAddrAlignment,
1344 AI->getOrdering(), AI->getSyncScopeID());
1345
1346 NewAI->setVolatile(AI->isVolatile());
1347 copyMetadataForAtomic(*NewAI, *AI);
1348
1349 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV);
1350 AI->replaceAllUsesWith(FinalOldResult);
1351 AI->eraseFromParent();
1352 return NewAI;
1353}
1354
1355bool AtomicExpandImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
1356 // The basic idea here is that we're expanding a cmpxchg of a
1357 // smaller memory size up to a word-sized cmpxchg. To do this, we
1358 // need to add a retry-loop for strong cmpxchg, so that
1359 // modifications to other parts of the word don't cause a spurious
1360 // failure.
1361
1362 // This generates code like the following:
1363 // [[Setup mask values PMV.*]]
1364 // %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt
1365 // %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt
1366 // %InitLoaded = load i32* %addr
1367 // %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask
1368 // br partword.cmpxchg.loop
1369 // partword.cmpxchg.loop:
1370 // %Loaded_MaskOut = phi i32 [ %InitLoaded_MaskOut, %entry ],
1371 // [ %OldVal_MaskOut, %partword.cmpxchg.failure ]
1372 // %FullWord_NewVal = or i32 %Loaded_MaskOut, %NewVal_Shifted
1373 // %FullWord_Cmp = or i32 %Loaded_MaskOut, %Cmp_Shifted
1374 // %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp,
1375 // i32 %FullWord_NewVal success_ordering failure_ordering
1376 // %OldVal = extractvalue { i32, i1 } %NewCI, 0
1377 // %Success = extractvalue { i32, i1 } %NewCI, 1
1378 // br i1 %Success, label %partword.cmpxchg.end,
1379 // label %partword.cmpxchg.failure
1380 // partword.cmpxchg.failure:
1381 // %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask
1382 // %ShouldContinue = icmp ne i32 %Loaded_MaskOut, %OldVal_MaskOut
1383 // br i1 %ShouldContinue, label %partword.cmpxchg.loop,
1384 // label %partword.cmpxchg.end
1385 // partword.cmpxchg.end:
1386 // %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt
1387 // %FinalOldVal = trunc i32 %tmp1 to i8
1388 // %tmp2 = insertvalue { i8, i1 } undef, i8 %FinalOldVal, 0
1389 // %Res = insertvalue { i8, i1 } %25, i1 %Success, 1
1390
1391 Value *Addr = CI->getPointerOperand();
1392 Value *Cmp = CI->getCompareOperand();
1393 Value *NewVal = CI->getNewValOperand();
1394
1395 BasicBlock *BB = CI->getParent();
1396 Function *F = BB->getParent();
1397 ReplacementIRBuilder Builder(CI, *DL);
1398 LLVMContext &Ctx = Builder.getContext();
1399
1400 BasicBlock *EndBB =
1401 BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end");
1402 auto FailureBB =
1403 BasicBlock::Create(Ctx, "partword.cmpxchg.failure", F, EndBB);
1404 auto LoopBB = BasicBlock::Create(Ctx, "partword.cmpxchg.loop", F, FailureBB);
1405
1406 // The split call above "helpfully" added a branch at the end of BB
1407 // (to the wrong place).
1408 std::prev(BB->end())->eraseFromParent();
1409 Builder.SetInsertPoint(BB);
1410
1411 PartwordMaskValues PMV =
1412 createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
1413 CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
1414
1415 // Shift the incoming values over, into the right location in the word.
1416 Value *NewVal_Shifted =
1417 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
1418 Value *Cmp_Shifted =
1419 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt);
1420
1421 // Load the entire current word, and mask into place the expected and new
1422 // values
1423 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr);
1424 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask);
1425 Builder.CreateBr(LoopBB);
1426
1427 // partword.cmpxchg.loop:
1428 Builder.SetInsertPoint(LoopBB);
1429 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2);
1430 Loaded_MaskOut->addIncoming(InitLoaded_MaskOut, BB);
1431
1432 // The initial load must be atomic with the same synchronization scope
1433 // to avoid a data race with concurrent stores. If the instruction being
1434 // emulated is volatile, issue a volatile load.
1435 // addIncoming is done first so that any replaceAllUsesWith calls during
1436 // normalization correctly update the PHI incoming value.
1437 InitLoaded->setVolatile(CI->isVolatile());
1439 InitLoaded->setAtomic(AtomicOrdering::Monotonic, CI->getSyncScopeID());
1440 // The newly created load might need to be lowered further. Because it is
1441 // created in the same block as the atomicrmw, the AtomicExpand loop will
1442 // not process it again.
1443 processAtomicInstr(InitLoaded);
1444 }
1445
1446 // Mask/Or the expected and new values into place in the loaded word.
1447 Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);
1448 Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);
1449 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
1450 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment,
1452 NewCI->setVolatile(CI->isVolatile());
1453 // When we're building a strong cmpxchg, we need a loop, so you
1454 // might think we could use a weak cmpxchg inside. But, using strong
1455 // allows the below comparison for ShouldContinue, and we're
1456 // expecting the underlying cmpxchg to be a machine instruction,
1457 // which is strong anyways.
1458 NewCI->setWeak(CI->isWeak());
1459
1460 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
1461 Value *Success = Builder.CreateExtractValue(NewCI, 1);
1462
1463 if (CI->isWeak())
1464 Builder.CreateBr(EndBB);
1465 else
1466 Builder.CreateCondBr(Success, EndBB, FailureBB);
1467
1468 // partword.cmpxchg.failure:
1469 Builder.SetInsertPoint(FailureBB);
1470 // Upon failure, verify that the masked-out part of the loaded value
1471 // has been modified. If it didn't, abort the cmpxchg, since the
1472 // masked-in part must've.
1473 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask);
1474 Value *ShouldContinue = Builder.CreateICmpNE(Loaded_MaskOut, OldVal_MaskOut);
1475 Builder.CreateCondBr(ShouldContinue, LoopBB, EndBB);
1476
1477 // Add the second value to the phi from above
1478 Loaded_MaskOut->addIncoming(OldVal_MaskOut, FailureBB);
1479
1480 // partword.cmpxchg.end:
1481 Builder.SetInsertPoint(CI);
1482
1483 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1484 Value *Res = PoisonValue::get(CI->getType());
1485 Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
1486 Res = Builder.CreateInsertValue(Res, Success, 1);
1487
1488 CI->replaceAllUsesWith(Res);
1489 CI->eraseFromParent();
1490 return true;
1491}
1492
1493void AtomicExpandImpl::expandAtomicOpToLLSC(
1494 Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
1495 AtomicOrdering MemOpOrder,
1496 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1497 ReplacementIRBuilder Builder(I, *DL);
1498 Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
1499 MemOpOrder, PerformOp);
1500
1501 I->replaceAllUsesWith(Loaded);
1502 I->eraseFromParent();
1503}
1504
1505void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
1506 ReplacementIRBuilder Builder(AI, *DL);
1507
1508 PartwordMaskValues PMV =
1509 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
1510 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
1511
1512 // The value operand must be sign-extended for signed min/max so that the
1513 // target's signed comparison instructions can be used. Otherwise, just
1514 // zero-ext.
1515 Instruction::CastOps CastOp = Instruction::ZExt;
1516 AtomicRMWInst::BinOp RMWOp = AI->getOperation();
1517 if (RMWOp == AtomicRMWInst::Max || RMWOp == AtomicRMWInst::Min)
1518 CastOp = Instruction::SExt;
1519
1520 Value *ValOperand_Shifted = Builder.CreateShl(
1521 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType),
1522 PMV.ShiftAmt, "ValOperand_Shifted");
1524 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,
1525 AI->getOrdering());
1526 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
1527 AI->replaceAllUsesWith(FinalOldResult);
1528 AI->eraseFromParent();
1529}
1530
1531void AtomicExpandImpl::expandAtomicCmpXchgToMaskedIntrinsic(
1532 AtomicCmpXchgInst *CI) {
1533 ReplacementIRBuilder Builder(CI, *DL);
1534
1535 PartwordMaskValues PMV = createMaskInstrs(
1536 Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
1537 CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
1538
1539 Value *CmpVal_Shifted = Builder.CreateShl(
1540 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
1541 "CmpVal_Shifted");
1542 Value *NewVal_Shifted = Builder.CreateShl(
1543 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt,
1544 "NewVal_Shifted");
1546 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask,
1547 CI->getMergedOrdering());
1548 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1549 Value *Res = PoisonValue::get(CI->getType());
1550 Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
1551 Value *Success = Builder.CreateICmpEQ(
1552 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success");
1553 Res = Builder.CreateInsertValue(Res, Success, 1);
1554
1555 CI->replaceAllUsesWith(Res);
1556 CI->eraseFromParent();
1557}
1558
1559Value *AtomicExpandImpl::insertRMWLLSCLoop(
1560 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1561 AtomicOrdering MemOpOrder,
1562 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1563 LLVMContext &Ctx = Builder.getContext();
1564 BasicBlock *BB = Builder.GetInsertBlock();
1565 Function *F = BB->getParent();
1566
1567 assert(AddrAlign >= F->getDataLayout().getTypeStoreSize(ResultTy) &&
1568 "Expected at least natural alignment at this point.");
1569
1570 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1571 //
1572 // The standard expansion we produce is:
1573 // [...]
1574 // atomicrmw.start:
1575 // %loaded = @load.linked(%addr)
1576 // %new = some_op iN %loaded, %incr
1577 // %stored = @store_conditional(%new, %addr)
1578 // %try_again = icmp i32 ne %stored, 0
1579 // br i1 %try_again, label %loop, label %atomicrmw.end
1580 // atomicrmw.end:
1581 // [...]
1582 BasicBlock *ExitBB =
1583 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
1584 BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB);
1585
1586 // The split call above "helpfully" added a branch at the end of BB (to the
1587 // wrong place).
1588 std::prev(BB->end())->eraseFromParent();
1589 Builder.SetInsertPoint(BB);
1590 Builder.CreateBr(LoopBB);
1591
1592 // Start the main loop block now that we've taken care of the preliminaries.
1593 Builder.SetInsertPoint(LoopBB);
1594 Value *Loaded = TLI->emitLoadLinked(Builder, ResultTy, Addr, MemOpOrder);
1595
1596 Value *NewVal = PerformOp(Builder, Loaded);
1597
1598 Value *StoreSuccess =
1599 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
1600 Value *TryAgain = Builder.CreateICmpNE(
1601 StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain");
1602
1603 Instruction *CondBr = Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);
1604
1605 // Atomic RMW expands to a Load-linked / Store-Conditional loop, because it is
1606 // hard to predict precise branch weigths we mark the branch as "unknown"
1607 // (50/50) to prevent misleading optimizations.
1609
1610 Builder.SetInsertPoint(ExitBB->begin());
1611 return Loaded;
1612}
1613
1614/// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of
1615/// the equivalent bitwidth. We used to not support pointer cmpxchg in the
1616/// IR. As a migration step, we convert back to what use to be the standard
1617/// way to represent a pointer cmpxchg so that we can update backends one by
1618/// one.
1619AtomicCmpXchgInst *
1620AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
1621 auto *M = CI->getModule();
1622 Type *NewTy = getCorrespondingIntegerType(CI->getCompareOperand()->getType(),
1623 M->getDataLayout());
1624
1625 ReplacementIRBuilder Builder(CI, *DL);
1626
1627 Value *Addr = CI->getPointerOperand();
1628
1629 Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
1630 Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
1631
1632 auto *NewCI = Builder.CreateAtomicCmpXchg(
1633 Addr, NewCmp, NewNewVal, CI->getAlign(), CI->getSuccessOrdering(),
1634 CI->getFailureOrdering(), CI->getSyncScopeID());
1635 NewCI->setVolatile(CI->isVolatile());
1636 NewCI->setWeak(CI->isWeak());
1637 LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");
1638
1639 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
1640 Value *Succ = Builder.CreateExtractValue(NewCI, 1);
1641
1642 OldVal = Builder.CreateIntToPtr(OldVal, CI->getCompareOperand()->getType());
1643
1644 Value *Res = PoisonValue::get(CI->getType());
1645 Res = Builder.CreateInsertValue(Res, OldVal, 0);
1646 Res = Builder.CreateInsertValue(Res, Succ, 1);
1647
1648 CI->replaceAllUsesWith(Res);
1649 CI->eraseFromParent();
1650 return NewCI;
1651}
1652
1653bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1654 AtomicOrdering SuccessOrder = CI->getSuccessOrdering();
1655 AtomicOrdering FailureOrder = CI->getFailureOrdering();
1656 Value *Addr = CI->getPointerOperand();
1657 BasicBlock *BB = CI->getParent();
1658 Function *F = BB->getParent();
1659 LLVMContext &Ctx = F->getContext();
1660 // If shouldInsertFencesForAtomic() returns true, then the target does not
1661 // want to deal with memory orders, and emitLeading/TrailingFence should take
1662 // care of everything. Otherwise, emitLeading/TrailingFence are no-op and we
1663 // should preserve the ordering.
1664 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(CI);
1665 AtomicOrdering MemOpOrder = ShouldInsertFencesForAtomic
1666 ? AtomicOrdering::Monotonic
1667 : CI->getMergedOrdering();
1668
1669 // In implementations which use a barrier to achieve release semantics, we can
1670 // delay emitting this barrier until we know a store is actually going to be
1671 // attempted. The cost of this delay is that we need 2 copies of the block
1672 // emitting the load-linked, affecting code size.
1673 //
1674 // Ideally, this logic would be unconditional except for the minsize check
1675 // since in other cases the extra blocks naturally collapse down to the
1676 // minimal loop. Unfortunately, this puts too much stress on later
1677 // optimisations so we avoid emitting the extra logic in those cases too.
1678 bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&
1679 SuccessOrder != AtomicOrdering::Monotonic &&
1680 SuccessOrder != AtomicOrdering::Acquire &&
1681 !F->hasMinSize();
1682
1683 // There's no overhead for sinking the release barrier in a weak cmpxchg, so
1684 // do it even on minsize.
1685 // A fence between the LL and SC may clear the reservation on some targets.
1686 // Strong cmpxchg retries, but weak cmpxchg cannot recover.
1687 bool UseUnconditionalReleaseBarrier =
1688 (F->hasMinSize() && !CI->isWeak()) ||
1690
1691 // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord
1692 //
1693 // The full expansion we produce is:
1694 // [...]
1695 // %aligned.addr = ...
1696 // cmpxchg.start:
1697 // %unreleasedload = @load.linked(%aligned.addr)
1698 // %unreleasedload.extract = extract value from %unreleasedload
1699 // %should_store = icmp eq %unreleasedload.extract, %desired
1700 // br i1 %should_store, label %cmpxchg.releasingstore,
1701 // label %cmpxchg.nostore
1702 // cmpxchg.releasingstore:
1703 // fence?
1704 // br label cmpxchg.trystore
1705 // cmpxchg.trystore:
1706 // %loaded.trystore = phi [%unreleasedload, %cmpxchg.releasingstore],
1707 // [%releasedload, %cmpxchg.releasedload]
1708 // %updated.new = insert %new into %loaded.trystore
1709 // %stored = @store_conditional(%updated.new, %aligned.addr)
1710 // %success = icmp eq i32 %stored, 0
1711 // br i1 %success, label %cmpxchg.success,
1712 // label %cmpxchg.releasedload/%cmpxchg.failure
1713 // cmpxchg.releasedload:
1714 // %releasedload = @load.linked(%aligned.addr)
1715 // %releasedload.extract = extract value from %releasedload
1716 // %should_store = icmp eq %releasedload.extract, %desired
1717 // br i1 %should_store, label %cmpxchg.trystore,
1718 // label %cmpxchg.failure
1719 // cmpxchg.success:
1720 // fence?
1721 // br label %cmpxchg.end
1722 // cmpxchg.nostore:
1723 // %loaded.nostore = phi [%unreleasedload, %cmpxchg.start],
1724 // [%releasedload,
1725 // %cmpxchg.releasedload/%cmpxchg.trystore]
1726 // @load_linked_fail_balance()?
1727 // br label %cmpxchg.failure
1728 // cmpxchg.failure:
1729 // fence?
1730 // br label %cmpxchg.end
1731 // cmpxchg.end:
1732 // %loaded.exit = phi [%loaded.nostore, %cmpxchg.failure],
1733 // [%loaded.trystore, %cmpxchg.trystore]
1734 // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure]
1735 // %loaded = extract value from %loaded.exit
1736 // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0
1737 // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1
1738 // [...]
1739 BasicBlock *ExitBB = BB->splitBasicBlock(CI->getIterator(), "cmpxchg.end");
1740 auto FailureBB = BasicBlock::Create(Ctx, "cmpxchg.failure", F, ExitBB);
1741 auto NoStoreBB = BasicBlock::Create(Ctx, "cmpxchg.nostore", F, FailureBB);
1742 auto SuccessBB = BasicBlock::Create(Ctx, "cmpxchg.success", F, NoStoreBB);
1743 auto ReleasedLoadBB =
1744 BasicBlock::Create(Ctx, "cmpxchg.releasedload", F, SuccessBB);
1745 auto TryStoreBB =
1746 BasicBlock::Create(Ctx, "cmpxchg.trystore", F, ReleasedLoadBB);
1747 auto ReleasingStoreBB =
1748 BasicBlock::Create(Ctx, "cmpxchg.fencedstore", F, TryStoreBB);
1749 auto StartBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, ReleasingStoreBB);
1750
1751 ReplacementIRBuilder Builder(CI, *DL);
1752
1753 // The split call above "helpfully" added a branch at the end of BB (to the
1754 // wrong place), but we might want a fence too. It's easiest to just remove
1755 // the branch entirely.
1756 std::prev(BB->end())->eraseFromParent();
1757 Builder.SetInsertPoint(BB);
1758 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
1759 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1760
1761 PartwordMaskValues PMV =
1762 createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
1763 CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
1764 Builder.CreateBr(StartBB);
1765
1766 // Start the main loop block now that we've taken care of the preliminaries.
1767 Builder.SetInsertPoint(StartBB);
1768 Value *UnreleasedLoad =
1769 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1770 Value *UnreleasedLoadExtract =
1771 extractMaskedValue(Builder, UnreleasedLoad, PMV);
1772 Value *ShouldStore = Builder.CreateICmpEQ(
1773 UnreleasedLoadExtract, CI->getCompareOperand(), "should_store");
1774
1775 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1776 // jump straight past that fence instruction (if it exists).
1777 Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB,
1778 MDBuilder(F->getContext()).createLikelyBranchWeights());
1779
1780 Builder.SetInsertPoint(ReleasingStoreBB);
1781 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
1782 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1783 Builder.CreateBr(TryStoreBB);
1784
1785 Builder.SetInsertPoint(TryStoreBB);
1786 PHINode *LoadedTryStore =
1787 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore");
1788 LoadedTryStore->addIncoming(UnreleasedLoad, ReleasingStoreBB);
1789 Value *NewValueInsert =
1790 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV);
1791 Value *StoreSuccess = TLI->emitStoreConditional(Builder, NewValueInsert,
1792 PMV.AlignedAddr, MemOpOrder);
1793 StoreSuccess = Builder.CreateICmpEQ(
1794 StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success");
1795 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
1796 Builder.CreateCondBr(StoreSuccess, SuccessBB,
1797 CI->isWeak() ? FailureBB : RetryBB,
1798 MDBuilder(F->getContext()).createLikelyBranchWeights());
1799
1800 Builder.SetInsertPoint(ReleasedLoadBB);
1801 Value *SecondLoad;
1802 if (HasReleasedLoadBB) {
1803 SecondLoad =
1804 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1805 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV);
1806 ShouldStore = Builder.CreateICmpEQ(SecondLoadExtract,
1807 CI->getCompareOperand(), "should_store");
1808
1809 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1810 // jump straight past that fence instruction (if it exists).
1811 Builder.CreateCondBr(
1812 ShouldStore, TryStoreBB, NoStoreBB,
1813 MDBuilder(F->getContext()).createLikelyBranchWeights());
1814 // Update PHI node in TryStoreBB.
1815 LoadedTryStore->addIncoming(SecondLoad, ReleasedLoadBB);
1816 } else
1817 Builder.CreateUnreachable();
1818
1819 // Make sure later instructions don't get reordered with a fence if
1820 // necessary.
1821 Builder.SetInsertPoint(SuccessBB);
1822 if (ShouldInsertFencesForAtomic ||
1824 TLI->emitTrailingFence(Builder, CI, SuccessOrder);
1825 Builder.CreateBr(ExitBB);
1826
1827 Builder.SetInsertPoint(NoStoreBB);
1828 PHINode *LoadedNoStore =
1829 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.nostore");
1830 LoadedNoStore->addIncoming(UnreleasedLoad, StartBB);
1831 if (HasReleasedLoadBB)
1832 LoadedNoStore->addIncoming(SecondLoad, ReleasedLoadBB);
1833
1834 // In the failing case, where we don't execute the store-conditional, the
1835 // target might want to balance out the load-linked with a dedicated
1836 // instruction (e.g., on ARM, clearing the exclusive monitor).
1838 Builder.CreateBr(FailureBB);
1839
1840 Builder.SetInsertPoint(FailureBB);
1841 PHINode *LoadedFailure =
1842 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.failure");
1843 LoadedFailure->addIncoming(LoadedNoStore, NoStoreBB);
1844 if (CI->isWeak())
1845 LoadedFailure->addIncoming(LoadedTryStore, TryStoreBB);
1846 if (ShouldInsertFencesForAtomic)
1847 TLI->emitTrailingFence(Builder, CI, FailureOrder);
1848 Builder.CreateBr(ExitBB);
1849
1850 // Finally, we have control-flow based knowledge of whether the cmpxchg
1851 // succeeded or not. We expose this to later passes by converting any
1852 // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate
1853 // PHI.
1854 Builder.SetInsertPoint(ExitBB->begin());
1855 PHINode *LoadedExit =
1856 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.exit");
1857 LoadedExit->addIncoming(LoadedTryStore, SuccessBB);
1858 LoadedExit->addIncoming(LoadedFailure, FailureBB);
1859 PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2, "success");
1860 Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB);
1861 Success->addIncoming(ConstantInt::getFalse(Ctx), FailureBB);
1862
1863 // This is the "exit value" from the cmpxchg expansion. It may be of
1864 // a type wider than the one in the cmpxchg instruction.
1865 Value *LoadedFull = LoadedExit;
1866
1867 Builder.SetInsertPoint(std::next(Success->getIterator()));
1868 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV);
1869
1870 // Look for any users of the cmpxchg that are just comparing the loaded value
1871 // against the desired one, and replace them with the CFG-derived version.
1873 for (auto *User : CI->users()) {
1874 ExtractValueInst *EV = dyn_cast<ExtractValueInst>(User);
1875 if (!EV)
1876 continue;
1877
1878 assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 &&
1879 "weird extraction from { iN, i1 }");
1880
1881 if (EV->getIndices()[0] == 0)
1882 EV->replaceAllUsesWith(Loaded);
1883 else
1885
1886 PrunedInsts.push_back(EV);
1887 }
1888
1889 // We can remove the instructions now we're no longer iterating through them.
1890 for (auto *EV : PrunedInsts)
1891 EV->eraseFromParent();
1892
1893 if (!CI->use_empty()) {
1894 // Some use of the full struct return that we don't understand has happened,
1895 // so we've got to reconstruct it properly.
1896 Value *Res;
1897 Res = Builder.CreateInsertValue(PoisonValue::get(CI->getType()), Loaded, 0);
1898 Res = Builder.CreateInsertValue(Res, Success, 1);
1899
1900 CI->replaceAllUsesWith(Res);
1901 }
1902
1903 CI->eraseFromParent();
1904 return true;
1905}
1906
1907bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
1908 if (RMWI->isVolatile())
1909 return false;
1910 // TODO: Add floating point support.
1911 auto C = dyn_cast<ConstantInt>(RMWI->getValOperand());
1912 if (!C)
1913 return false;
1914
1915 switch (RMWI->getOperation()) {
1916 case AtomicRMWInst::Add:
1917 case AtomicRMWInst::Sub:
1918 case AtomicRMWInst::Or:
1919 case AtomicRMWInst::Xor:
1920 return C->isZero();
1921 case AtomicRMWInst::And:
1922 return C->isMinusOne();
1923 case AtomicRMWInst::Min:
1924 return C->isMaxValue(true);
1925 case AtomicRMWInst::Max:
1926 return C->isMinValue(true);
1928 return C->isMaxValue(false);
1930 return C->isMinValue(false);
1931 default:
1932 return false;
1933 }
1934}
1935
1936bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
1937 if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
1938 tryExpandAtomicLoad(ResultingLoad);
1939 return true;
1940 }
1941 return false;
1942}
1943
1944Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
1945 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1946 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
1947 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
1948 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc) {
1949 LLVMContext &Ctx = Builder.getContext();
1950 BasicBlock *BB = Builder.GetInsertBlock();
1951 Function *F = BB->getParent();
1952
1953 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1954 //
1955 // The standard expansion we produce is:
1956 // [...]
1957 // %init_loaded = load atomic iN* %addr
1958 // br label %loop
1959 // loop:
1960 // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ]
1961 // %new = some_op iN %loaded, %incr
1962 // %pair = cmpxchg iN* %addr, iN %loaded, iN %new
1963 // %new_loaded = extractvalue { iN, i1 } %pair, 0
1964 // %success = extractvalue { iN, i1 } %pair, 1
1965 // br i1 %success, label %atomicrmw.end, label %loop
1966 // atomicrmw.end:
1967 // [...]
1968 BasicBlock *ExitBB =
1969 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
1970 BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB);
1971
1972 // The split call above "helpfully" added a branch at the end of BB (to the
1973 // wrong place), but we want a load. It's easiest to just remove
1974 // the branch entirely.
1975 std::prev(BB->end())->eraseFromParent();
1976 Builder.SetInsertPoint(BB);
1977 LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign);
1978 Builder.CreateBr(LoopBB);
1979
1980 // Start the main loop block now that we've taken care of the preliminaries.
1981 Builder.SetInsertPoint(LoopBB);
1982 PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded");
1983 Loaded->addIncoming(InitLoaded, BB);
1984
1985 // The initial load must be atomic with the same synchronization scope
1986 // to avoid a data race with concurrent stores. If the instruction being
1987 // emulated is volatile, issue a volatile load.
1988 // addIncoming is done first so that any replaceAllUsesWith calls during
1989 // normalization correctly update the PHI incoming value.
1990 InitLoaded->setVolatile(IsVolatile);
1992 InitLoaded->setAtomic(AtomicOrdering::Monotonic, SSID);
1993 // The newly created load might need to be lowered further. Because it is
1994 // created in the same block as the atomicrmw, the AtomicExpand loop will
1995 // not process it again.
1996 processAtomicInstr(InitLoaded);
1997 }
1998
1999 Value *NewVal = PerformOp(Builder, Loaded);
2000
2001 Value *NewLoaded = nullptr;
2002 Value *Success = nullptr;
2003
2004 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
2005 MemOpOrder == AtomicOrdering::Unordered
2006 ? AtomicOrdering::Monotonic
2007 : MemOpOrder,
2008 SSID, IsVolatile, Success, NewLoaded, MetadataSrc);
2009 assert(Success && NewLoaded);
2010
2011 Loaded->addIncoming(NewLoaded, LoopBB);
2012
2013 Instruction *CondBr = Builder.CreateCondBr(Success, ExitBB, LoopBB);
2014
2015 // Atomic RMW expands to a cmpxchg loop, Since precise branch weights
2016 // cannot be easily determined here, we mark the branch as "unknown" (50/50)
2017 // to prevent misleading optimizations.
2019
2020 Builder.SetInsertPoint(ExitBB->begin());
2021 return NewLoaded;
2022}
2023
2024bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
2025 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
2026 unsigned ValueSize = getAtomicOpSize(CI);
2027
2028 switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) {
2029 default:
2030 llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");
2031 case TargetLoweringBase::AtomicExpansionKind::None:
2032 if (ValueSize < MinCASSize)
2033 return expandPartwordCmpXchg(CI);
2034 return false;
2035 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
2036 return expandAtomicCmpXchg(CI);
2037 }
2038 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic:
2039 expandAtomicCmpXchgToMaskedIntrinsic(CI);
2040 return true;
2041 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
2042 return lowerAtomicCmpXchgInst(CI);
2043 case TargetLoweringBase::AtomicExpansionKind::CustomExpand: {
2045 return true;
2046 }
2047 }
2048}
2049
2050bool AtomicExpandImpl::expandAtomicRMWToCmpXchg(
2051 AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) {
2052 ReplacementIRBuilder Builder(AI, AI->getDataLayout());
2053 Builder.setIsFPConstrained(
2054 AI->getFunction()->hasFnAttribute(Attribute::StrictFP));
2055
2056 // FIXME: If FP exceptions are observable, we should force them off for the
2057 // loop for the FP atomics.
2058 Value *Loaded = AtomicExpandImpl::insertRMWCmpXchgLoop(
2059 Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
2060 AI->getOrdering(), AI->getSyncScopeID(), AI->isVolatile(),
2061 [&](IRBuilderBase &Builder, Value *Loaded) {
2062 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
2063 AI->getValOperand());
2064 },
2065 CreateCmpXchg, /*MetadataSrc=*/AI);
2066
2067 AI->replaceAllUsesWith(Loaded);
2068 AI->eraseFromParent();
2069 return true;
2070}
2071
2072// In order to use one of the sized library calls such as
2073// __atomic_fetch_add_4, the alignment must be sufficient, the size
2074// must be one of the potentially-specialized sizes, and the value
2075// type must actually exist in C on the target (otherwise, the
2076// function wouldn't actually be defined.)
2077static bool canUseSizedAtomicCall(unsigned Size, Align Alignment,
2078 const DataLayout &DL) {
2079 // TODO: "LargestSize" is an approximation for "largest type that
2080 // you can express in C". It seems to be the case that int128 is
2081 // supported on all 64-bit platforms, otherwise only up to 64-bit
2082 // integers are supported. If we get this wrong, then we'll try to
2083 // call a sized libcall that doesn't actually exist. There should
2084 // really be some more reliable way in LLVM of determining integer
2085 // sizes which are valid in the target's C ABI...
2086 unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;
2087 return Alignment >= Size &&
2088 (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) &&
2089 Size <= LargestSize;
2090}
2091
2092void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {
2093 static const RTLIB::Libcall Libcalls[6] = {
2094 RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
2095 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
2096 unsigned Size = getAtomicOpSize(I);
2097
2098 bool Expanded = expandAtomicOpToLibcall(
2099 I, Size, I->getAlign(), I->getPointerOperand(), nullptr, nullptr,
2100 I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
2101 if (!Expanded)
2102 handleUnsupportedAtomicSize(I, "atomic load");
2103}
2104
2105void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {
2106 static const RTLIB::Libcall Libcalls[6] = {
2107 RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
2108 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
2109 unsigned Size = getAtomicOpSize(I);
2110
2111 bool Expanded = expandAtomicOpToLibcall(
2112 I, Size, I->getAlign(), I->getPointerOperand(), I->getValueOperand(),
2113 nullptr, I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
2114 if (!Expanded)
2115 handleUnsupportedAtomicSize(I, "atomic store");
2116}
2117
2118void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
2119 const Twine &AtomicOpName,
2120 Instruction *DiagnosticInst) {
2121 static const RTLIB::Libcall Libcalls[6] = {
2122 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
2123 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
2124 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
2125 unsigned Size = getAtomicOpSize(I);
2126
2127 bool Expanded = expandAtomicOpToLibcall(
2128 I, Size, I->getAlign(), I->getPointerOperand(), I->getNewValOperand(),
2129 I->getCompareOperand(), I->getSuccessOrdering(), I->getFailureOrdering(),
2130 Libcalls);
2131 if (!Expanded)
2132 handleUnsupportedAtomicSize(I, AtomicOpName, DiagnosticInst);
2133}
2134
2136 static const RTLIB::Libcall LibcallsXchg[6] = {
2137 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
2138 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,
2139 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};
2140 static const RTLIB::Libcall LibcallsAdd[6] = {
2141 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,
2142 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,
2143 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};
2144 static const RTLIB::Libcall LibcallsSub[6] = {
2145 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,
2146 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,
2147 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};
2148 static const RTLIB::Libcall LibcallsAnd[6] = {
2149 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,
2150 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,
2151 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};
2152 static const RTLIB::Libcall LibcallsOr[6] = {
2153 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,
2154 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,
2155 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};
2156 static const RTLIB::Libcall LibcallsXor[6] = {
2157 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,
2158 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,
2159 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};
2160 static const RTLIB::Libcall LibcallsNand[6] = {
2161 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,
2162 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,
2163 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};
2164
2165 switch (Op) {
2167 llvm_unreachable("Should not have BAD_BINOP.");
2169 return ArrayRef(LibcallsXchg);
2170 case AtomicRMWInst::Add:
2171 return ArrayRef(LibcallsAdd);
2172 case AtomicRMWInst::Sub:
2173 return ArrayRef(LibcallsSub);
2174 case AtomicRMWInst::And:
2175 return ArrayRef(LibcallsAnd);
2176 case AtomicRMWInst::Or:
2177 return ArrayRef(LibcallsOr);
2178 case AtomicRMWInst::Xor:
2179 return ArrayRef(LibcallsXor);
2181 return ArrayRef(LibcallsNand);
2182 case AtomicRMWInst::Max:
2183 case AtomicRMWInst::Min:
2198 // No atomic libcalls are available for these.
2199 return {};
2200 }
2201 llvm_unreachable("Unexpected AtomicRMW operation.");
2202}
2203
2204void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
2205 ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(I->getOperation());
2206
2207 unsigned Size = getAtomicOpSize(I);
2208
2209 bool Success = false;
2210 if (!Libcalls.empty())
2211 Success = expandAtomicOpToLibcall(
2212 I, Size, I->getAlign(), I->getPointerOperand(), I->getValOperand(),
2213 nullptr, I->getOrdering(), AtomicOrdering::NotAtomic, Libcalls);
2214
2215 // The expansion failed: either there were no libcalls at all for
2216 // the operation (min/max), or there were only size-specialized
2217 // libcalls (add/sub/etc) and we needed a generic. So, expand to a
2218 // CAS libcall, via a CAS loop, instead.
2219 if (!Success) {
2220 expandAtomicRMWToCmpXchg(
2221 I, [this, I](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
2222 Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
2223 SyncScope::ID SSID, bool IsVolatile, Value *&Success,
2224 Value *&NewLoaded, Instruction *MetadataSrc) {
2225 // Create the CAS instruction normally...
2226 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
2227 Addr, Loaded, NewVal, Alignment, MemOpOrder,
2229 Pair->setVolatile(IsVolatile);
2230 if (MetadataSrc)
2231 copyMetadataForAtomic(*Pair, *MetadataSrc);
2232
2233 Success = Builder.CreateExtractValue(Pair, 1, "success");
2234 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
2235
2236 // ...and then expand the CAS into a libcall.
2237 expandAtomicCASToLibcall(
2238 Pair,
2239 "atomicrmw " + AtomicRMWInst::getOperationName(I->getOperation()),
2240 MetadataSrc);
2241 });
2242 }
2243}
2244
2245// A helper routine for the above expandAtomic*ToLibcall functions.
2246//
2247// 'Libcalls' contains an array of enum values for the particular
2248// ATOMIC libcalls to be emitted. All of the other arguments besides
2249// 'I' are extracted from the Instruction subclass by the
2250// caller. Depending on the particular call, some will be null.
2251bool AtomicExpandImpl::expandAtomicOpToLibcall(
2252 Instruction *I, unsigned Size, Align Alignment, Value *PointerOperand,
2253 Value *ValueOperand, Value *CASExpected, AtomicOrdering Ordering,
2254 AtomicOrdering Ordering2, ArrayRef<RTLIB::Libcall> Libcalls) {
2255 assert(Libcalls.size() == 6);
2256
2257 LLVMContext &Ctx = I->getContext();
2258 Module *M = I->getModule();
2259 const DataLayout &DL = M->getDataLayout();
2260 IRBuilder<> Builder(I);
2261 IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front());
2262
2263 bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
2264 const Align AllocaAlignment =
2265 DL.getPrefTypeAlign(Type::getIntNTy(Ctx, Size * 8));
2266
2267 assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO");
2268 assert((!CASExpected || Ordering2 != AtomicOrdering::NotAtomic) &&
2269 "expect atomic MO");
2270 bool HasResult = I->getType() != Type::getVoidTy(Ctx);
2271
2272 RTLIB::Libcall RTLibType;
2273 if (UseSizedLibcall) {
2274 switch (Size) {
2275 case 1:
2276 RTLibType = Libcalls[1];
2277 break;
2278 case 2:
2279 RTLibType = Libcalls[2];
2280 break;
2281 case 4:
2282 RTLibType = Libcalls[3];
2283 break;
2284 case 8:
2285 RTLibType = Libcalls[4];
2286 break;
2287 case 16:
2288 RTLibType = Libcalls[5];
2289 break;
2290 }
2291 } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {
2292 RTLibType = Libcalls[0];
2293 } else {
2294 // Can't use sized function, and there's no generic for this
2295 // operation, so give up.
2296 return false;
2297 }
2298
2299 RTLIB::LibcallImpl LibcallImpl = LibcallLowering->getLibcallImpl(RTLibType);
2300 if (LibcallImpl == RTLIB::Unsupported) {
2301 // This target does not implement the requested atomic libcall so give up.
2302 return false;
2303 }
2304
2305 auto [FnType, Attr] = LibcallLowering->getRuntimeLibcallsInfo().getFunctionTy(
2306 Ctx, M->getTargetTriple(), DL, LibcallImpl);
2307 assert(FnType && "missing signature for atomic libcall");
2308
2309 // Build up the function call. There's two kinds. First, the sized
2310 // variants. These calls are going to be one of the following (with
2311 // N=1,2,4,8,16):
2312 // iN __atomic_load_N(iN *ptr, int ordering)
2313 // void __atomic_store_N(iN *ptr, iN val, int ordering)
2314 // iN __atomic_{exchange|fetch_*}_N(iN *ptr, iN val, int ordering)
2315 // bool __atomic_compare_exchange_N(iN *ptr, iN *expected, iN desired,
2316 // int success_order, int failure_order)
2317 //
2318 // Note that these functions can be used for non-integer atomic
2319 // operations, the values just need to be bitcast to integers on the
2320 // way in and out.
2321 //
2322 // And, then, the generic variants. They look like the following:
2323 // void __atomic_load(size_t size, void *ptr, void *ret, int ordering)
2324 // void __atomic_store(size_t size, void *ptr, void *val, int ordering)
2325 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret,
2326 // int ordering)
2327 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected,
2328 // void *desired, int success_order,
2329 // int failure_order)
2330 //
2331 // The different signatures are built up depending on the
2332 // 'UseSizedLibcall', 'CASExpected', 'ValueOperand', and 'HasResult'
2333 // variables.
2334
2335 AllocaInst *AllocaCASExpected = nullptr;
2336 AllocaInst *AllocaValue = nullptr;
2337 AllocaInst *AllocaResult = nullptr;
2338
2340
2341 // 'size' argument.
2342 if (!UseSizedLibcall)
2343 Args.push_back(ConstantInt::get(FnType->getParamType(0), Size));
2344
2345 // 'ptr' argument.
2346 // note: This assumes all address spaces share a common libfunc
2347 // implementation and that addresses are convertable. For systems without
2348 // that property, we'd need to extend this mechanism to support AS-specific
2349 // families of atomic intrinsics.
2350 Value *PtrVal = PointerOperand;
2351 PtrVal = Builder.CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));
2352 Args.push_back(PtrVal);
2353
2354 // 'expected' argument, if present.
2355 if (CASExpected) {
2356 AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->getType());
2357 AllocaCASExpected->setAlignment(AllocaAlignment);
2358 Builder.CreateLifetimeStart(AllocaCASExpected);
2359 Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment);
2360 Args.push_back(AllocaCASExpected);
2361 }
2362
2363 // 'val' argument ('desired' for cas), if present.
2364 if (ValueOperand) {
2365 if (UseSizedLibcall) {
2366 Value *IntValue = Builder.CreateBitPreservingCastChain(
2367 DL, ValueOperand, FnType->getParamType(Args.size()));
2368 Args.push_back(IntValue);
2369 } else {
2370 AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->getType());
2371 AllocaValue->setAlignment(AllocaAlignment);
2372 Builder.CreateLifetimeStart(AllocaValue);
2373 Builder.CreateAlignedStore(ValueOperand, AllocaValue, AllocaAlignment);
2374 Args.push_back(AllocaValue);
2375 }
2376 }
2377
2378 // 'ret' argument.
2379 if (!CASExpected && HasResult && !UseSizedLibcall) {
2380 AllocaResult = AllocaBuilder.CreateAlloca(I->getType());
2381 AllocaResult->setAlignment(AllocaAlignment);
2382 Builder.CreateLifetimeStart(AllocaResult);
2383 Args.push_back(AllocaResult);
2384 }
2385
2386 // 'ordering' ('success_order' for cas) argument.
2387 Args.push_back(ConstantInt::get(FnType->getParamType(Args.size()),
2388 (int)toCABI(Ordering)));
2389
2390 // 'failure_order' argument, if present.
2391 if (CASExpected) {
2392 Args.push_back(ConstantInt::get(FnType->getParamType(Args.size()),
2393 (int)toCABI(Ordering2)));
2394 }
2395
2396 assert(Args.size() == FnType->getNumParams() &&
2397 "atomic libcall signature mismatch");
2398
2399 FunctionCallee LibcallFn = M->getOrInsertFunction(
2401 Attr);
2402 CallInst *Call = Builder.CreateCall(LibcallFn, Args);
2403 Call->setAttributes(Attr);
2404 Value *Result = Call;
2405
2406 // And then, extract the results...
2407 if (ValueOperand && !UseSizedLibcall)
2408 Builder.CreateLifetimeEnd(AllocaValue);
2409
2410 if (CASExpected) {
2411 // The final result from the CAS is {load of 'expected' alloca, bool result
2412 // from call}
2413 Type *FinalResultTy = I->getType();
2414 Value *V = PoisonValue::get(FinalResultTy);
2415 Value *ExpectedOut = Builder.CreateAlignedLoad(
2416 CASExpected->getType(), AllocaCASExpected, AllocaAlignment);
2417 Builder.CreateLifetimeEnd(AllocaCASExpected);
2418 V = Builder.CreateInsertValue(V, ExpectedOut, 0);
2419 V = Builder.CreateInsertValue(V, Result, 1);
2421 } else if (HasResult) {
2422 Value *V;
2423 if (UseSizedLibcall) {
2424 // Add bitcasts from Result's scalar type to I's <n x ptr> vector type
2425 auto *PtrTy = dyn_cast<PointerType>(I->getType()->getScalarType());
2426 auto *VTy = dyn_cast<VectorType>(I->getType());
2427 if (VTy && PtrTy && !Result->getType()->isVectorTy()) {
2428 unsigned AS = PtrTy->getAddressSpace();
2429 Value *BC = Builder.CreateBitCast(
2430 Result, VTy->getWithNewType(DL.getIntPtrType(Ctx, AS)));
2431 V = Builder.CreateIntToPtr(BC, I->getType());
2432 } else
2433 V = Builder.CreateBitOrPointerCast(Result, I->getType());
2434 } else {
2435 V = Builder.CreateAlignedLoad(I->getType(), AllocaResult,
2436 AllocaAlignment);
2437 Builder.CreateLifetimeEnd(AllocaResult);
2438 }
2439 I->replaceAllUsesWith(V);
2440 }
2441 I->eraseFromParent();
2442 return true;
2443}
#define Success
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Value * performMaskedAtomicOp(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *ValOperand_Shifted, Value *Inc, const PartwordMaskValues &PMV)
Emit IR to implement a masked version of a given atomicrmw operation.
static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder, Instruction *I, Type *ValueType, Value *Addr, Align AddrAlign, unsigned MinWordSize)
This is a helper function which builds instructions to provide values necessary for partword atomic o...
static bool canUseSizedAtomicCall(unsigned Size, Align Alignment, const DataLayout &DL)
static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr, Value *Loaded, Value *NewVal, Align AddrAlign, AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile, Value *&Success, Value *&NewLoaded, Instruction *MetadataSrc)
static Value * extractMaskedValue(IRBuilderBase &Builder, Value *WideWord, const PartwordMaskValues &PMV)
Expand Atomic static false unsigned getAtomicOpSize(LoadInst *LI)
static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I, raw_ostream &OS)
static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I)
static Value * insertMaskedValue(IRBuilderBase &Builder, Value *WideWord, Value *Updated, const PartwordMaskValues &PMV)
static void copyMetadataForAtomic(Instruction &Dest, const Instruction &Source)
Copy metadata that's safe to preserve when widening atomics.
static ArrayRef< RTLIB::Libcall > GetRMWLibcall(AtomicRMWInst::BinOp Op)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool runOnFunction(Function &F, bool PostInlining)
#define DEBUG_TYPE
Module.h This file contains the declarations for the Module class.
static bool isIdempotentRMW(AtomicRMWInst &RMWI)
Return true if and only if the given instruction does not modify the memory location referenced.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Machine Check Debug Module
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
#define T
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file contains the declarations for profiling metadata utility functions.
const char * Msg
This file defines the SmallString class.
This file defines the SmallVector class.
#define LLVM_DEBUG(...)
Definition Debug.h:119
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
const AArch64TargetLowering * getTargetLowering() const override
void setAlignment(Align Align)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
An instruction that atomically checks whether a specified value is in a memory location,...
AtomicOrdering getMergedOrdering() const
Returns a single ordering which is at least as strong as both the success and failure orderings for t...
void setWeak(bool IsWeak)
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)
Returns the strongest permitted ordering on failure, given the desired ordering on success.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
void setVolatile(bool V)
Specify whether this is a volatile cmpxchg.
AtomicOrdering getSuccessOrdering() const
Returns the success ordering constraint of this cmpxchg instruction.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
an instruction that atomically reads a memory location, combines it with another value,...
bool isElementwise() const
Return true if this RMW has elementwise vector semantics.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Add
*p = old + v
@ FAdd
*p = old + v
@ USubCond
Subtract only if no unsigned overflow.
@ FMinimum
*p = minimum(old, v) minimum matches the behavior of llvm.minimum.
@ Min
*p = old <signed v ? old : v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ FMaximum
*p = maximum(old, v) maximum matches the behavior of llvm.maximum.
@ FSub
*p = old - v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMaximumNum
*p = maximumnum(old, v) maximumnum matches the behavior of llvm.maximumnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ FMinimumNum
*p = minimumnum(old, v) minimumnum matches the behavior of llvm.minimumnum.
@ Nand
*p = ~(old & v)
Value * getPointerOperand()
void setOperation(BinOp Operation)
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
static LLVM_ABI StringRef getOperationName(BinOp Op)
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
iterator end()
Definition BasicBlock.h:453
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
reverse_iterator rbegin()
Definition BasicBlock.h:456
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition BasicBlock.h:206
InstListType::reverse_iterator reverse_iterator
Definition BasicBlock.h:172
reverse_iterator rend()
Definition BasicBlock.h:458
void setAttributes(AttributeList A)
Set the attributes for this call.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
ArrayRef< unsigned > getIndices() const
unsigned getNumIndices() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
BasicBlockListType::iterator iterator
Definition Function.h:70
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNull=false)
Definition IRBuilder.h:2324
AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID=SyncScope::System)
Definition IRBuilder.h:1974
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition IRBuilder.h:2802
LLVM_ABI CallInst * CreateLifetimeStart(Value *Ptr)
Create a lifetime.start intrinsic.
LLVM_ABI CallInst * CreateLifetimeEnd(Value *Ptr)
Create a lifetime.end intrinsic.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Definition IRBuilder.h:1940
CondBrInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Definition IRBuilder.h:1218
UnreachableInst * CreateUnreachable()
Definition IRBuilder.h:1360
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition IRBuilder.h:2795
BasicBlock::iterator GetInsertPoint() const
Definition IRBuilder.h:177
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2314
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
Definition IRBuilder.h:2360
BasicBlock * GetInsertBlock() const
Definition IRBuilder.h:174
LLVM_ABI Value * CreateBitPreservingCastChain(const DataLayout &DL, Value *V, Type *NewTy)
Create a chain of casts to convert V to NewTy, preserving the bit pattern of V.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2462
UncondBrInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
Definition IRBuilder.h:1212
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2408
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition IRBuilder.h:2626
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2458
void setIsFPConstrained(bool IsCon)
Enable/Disable use of constrained floating point math.
Definition IRBuilder.h:295
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2319
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition IRBuilder.h:1913
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1513
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition IRBuilder.h:2197
LLVMContext & getContext() const
Definition IRBuilder.h:178
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:1572
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2309
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition IRBuilder.h:2640
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition IRBuilder.h:197
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Definition IRBuilder.h:1959
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
Definition IRBuilder.h:1594
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, bool Elementwise=false)
Definition IRBuilder.h:1987
Provides an 'InsertHelper' that calls a user-provided callback after performing the default insertion...
Definition IRBuilder.h:74
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
LLVM_ABI void getSyncScopeNames(SmallVectorImpl< StringRef > &SSNs) const
getSyncScopeNames - Populates client supplied SmallVector with synchronization scope names registered...
Tracks which library functions to use for a particular subtarget or function.
const RTLIB::RuntimeLibcallsInfo & getRuntimeLibcallsInfo() const
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
An instruction for reading from memory.
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
void setVolatile(bool V)
Specify whether this is a volatile load or not.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
LoadStoreInstProperties getProperties() const
Returns the properties of this load instruction.
Align getAlign() const
Return the alignment of the access that is being performed.
Metadata node.
Definition Metadata.h:1087
Records a mapping from an opaque lowering context to its LibcallLoweringInfo.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
LLVMContext & getContext() const
Get the global data context.
Definition Module.h:332
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition Pass.cpp:114
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition Analysis.h:115
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
virtual Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const
Perform a store-conditional operation to Addr.
virtual void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a bit test atomicrmw using a target-specific intrinsic.
virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
virtual bool shouldInsertFencesForAtomic(const Instruction *I) const
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
virtual AtomicOrdering atomicOperationOrderAfterFenceSplit(const Instruction *I) const
virtual void emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const
Perform a cmpxchg expansion using a target-specific method.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
virtual Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const
Perform a masked atomicrmw using a target-specific intrinsic.
virtual AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
virtual Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
virtual void emitExpandAtomicRMW(AtomicRMWInst *AI) const
Perform a atomicrmw expansion using a target-specific way.
virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const
virtual void emitExpandAtomicStore(StoreInst *SI) const
Perform a atomic store using a target-specific way.
virtual AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const
Returns how the given atomic atomicrmw should be cast by the IR-level AtomicExpand pass.
virtual bool shouldInsertTrailingSeqCstFenceForAtomicStore(const Instruction *I) const
Whether AtomicExpandPass should automatically insert a seq_cst trailing fence without reducing the or...
virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
virtual Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const
Perform a masked cmpxchg using a target-specific intrinsic.
virtual bool shouldIssueAtomicLoadForAtomicEmulationLoop(void) const
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual void emitExpandAtomicLoad(LoadInst *LI) const
Perform a atomic load using a target-specific way.
virtual AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
virtual bool fenceClearsLoadLinkedReservation() const
Whether a fence between the load-linked and the store-conditional can clear the reservation.
virtual void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a atomicrmw which the result is only used by comparison, using a target-specific intrinsic.
virtual AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
virtual Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
virtual AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
virtual Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
Inserts in the IR a target-specific intrinsic specifying a fence.
virtual LoadInst * lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *RMWI) const
On some platforms, an AtomicRMW that never actually modifies the value (such as fetch_add of 0) can b...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
Target-Independent Code Generator Pass Configuration Options.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
void setOperand(unsigned i, Value *Val)
Definition User.h:212
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
bool use_empty() const
Definition Value.h:348
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool canInstructionHaveMMRAs(const Instruction &I)
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isStrongerThanMonotonic(AtomicOrdering AO)
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
bool isReleaseOrStronger(AtomicOrdering AO)
AtomicOrderingCABI toCABI(AtomicOrdering AO)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI const LibcallLoweringInfo & getLibcallLowering(const ModuleLibcallLoweringInfo &ModuleInfo, const TargetSubtargetInfo &Subtarget)
Resolve the LibcallLoweringInfo for Subtarget from the module-level ModuleInfo, applying the subtarge...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI Value * buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Val)
Emit IR to implement the given atomicrmw operation on values in registers, returning the new value.
AtomicOrdering
Atomic ordering for LLVM's memory model.
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
LLVM_ABI bool lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI)
Convert the given Cmpxchg into primitive load and compare.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool lowerAtomicRMWInst(AtomicRMWInst *RMWI)
Convert the given RMWI into primitive load and stores, assuming that doing so is legal.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI FunctionPass * createAtomicExpandLegacyPass()
AtomicExpandPass - At IR level this pass replace atomic instructions with __atomic_* library calls,...
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
LLVM_ABI char & AtomicExpandID
AtomicExpandID – Lowers atomic operations in terms of either cmpxchg load-linked/store-conditional lo...
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition ValueTypes.h:450
Matching combinators.
LLVM_ABI std::pair< FunctionType *, AttributeList > getFunctionTy(LLVMContext &Ctx, const Triple &TT, const DataLayout &DL, RTLIB::LibcallImpl LibcallImpl) const
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.