LLVM  15.0.0git
InstCombineAtomicRMW.cpp
Go to the documentation of this file.
1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the visit functions for atomic rmw instructions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "InstCombineInternal.h"
14 #include "llvm/IR/Instructions.h"
15 
16 using namespace llvm;
17 
18 namespace {
19 /// Return true if and only if the given instruction does not modify the memory
20 /// location referenced. Note that an idemptent atomicrmw may still have
21 /// ordering effects on nearby instructions, or be volatile.
22 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
23 /// Idemptotent is confusing in this context.
24 bool isIdempotentRMW(AtomicRMWInst& RMWI) {
25  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
26  switch(RMWI.getOperation()) {
27  case AtomicRMWInst::FAdd: // -0.0
28  return CF->isZero() && CF->isNegative();
29  case AtomicRMWInst::FSub: // +0.0
30  return CF->isZero() && !CF->isNegative();
31  default:
32  return false;
33  };
34 
35  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
36  if(!C)
37  return false;
38 
39  switch(RMWI.getOperation()) {
40  case AtomicRMWInst::Add:
41  case AtomicRMWInst::Sub:
42  case AtomicRMWInst::Or:
43  case AtomicRMWInst::Xor:
44  return C->isZero();
45  case AtomicRMWInst::And:
46  return C->isMinusOne();
47  case AtomicRMWInst::Min:
48  return C->isMaxValue(true);
49  case AtomicRMWInst::Max:
50  return C->isMinValue(true);
52  return C->isMaxValue(false);
54  return C->isMinValue(false);
55  default:
56  return false;
57  }
58 }
59 
60 /// Return true if the given instruction always produces a value in memory
61 /// equivalent to its value operand.
62 bool isSaturating(AtomicRMWInst& RMWI) {
63  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
64  switch(RMWI.getOperation()) {
67  return CF->isNaN();
68  default:
69  return false;
70  };
71 
72  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
73  if(!C)
74  return false;
75 
76  switch(RMWI.getOperation()) {
77  default:
78  return false;
80  return true;
81  case AtomicRMWInst::Or:
82  return C->isAllOnesValue();
83  case AtomicRMWInst::And:
84  return C->isZero();
85  case AtomicRMWInst::Min:
86  return C->isMinValue(true);
87  case AtomicRMWInst::Max:
88  return C->isMaxValue(true);
90  return C->isMinValue(false);
92  return C->isMaxValue(false);
93  };
94 }
95 } // namespace
96 
98 
99  // Volatile RMWs perform a load and a store, we cannot replace this by just a
100  // load or just a store. We chose not to canonicalize out of general paranoia
101  // about user expectations around volatile.
102  if (RMWI.isVolatile())
103  return nullptr;
104 
105  // Any atomicrmw op which produces a known result in memory can be
106  // replaced w/an atomicrmw xchg.
107  if (isSaturating(RMWI) &&
108  RMWI.getOperation() != AtomicRMWInst::Xchg) {
110  return &RMWI;
111  }
112 
113  AtomicOrdering Ordering = RMWI.getOrdering();
114  assert(Ordering != AtomicOrdering::NotAtomic &&
115  Ordering != AtomicOrdering::Unordered &&
116  "AtomicRMWs don't make sense with Unordered or NotAtomic");
117 
118  // Any atomicrmw xchg with no uses can be converted to a atomic store if the
119  // ordering is compatible.
120  if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
121  RMWI.use_empty()) {
122  if (Ordering != AtomicOrdering::Release &&
123  Ordering != AtomicOrdering::Monotonic)
124  return nullptr;
125  auto *SI = new StoreInst(RMWI.getValOperand(),
126  RMWI.getPointerOperand(), &RMWI);
127  SI->setAtomic(Ordering, RMWI.getSyncScopeID());
128  SI->setAlignment(DL.getABITypeAlign(RMWI.getType()));
129  return eraseInstFromFunction(RMWI);
130  }
131 
132  if (!isIdempotentRMW(RMWI))
133  return nullptr;
134 
135  // We chose to canonicalize all idempotent operations to an single
136  // operation code and constant. This makes it easier for the rest of the
137  // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are
138  // arbitrary.
139  if (RMWI.getType()->isIntegerTy() &&
140  RMWI.getOperation() != AtomicRMWInst::Or) {
142  return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0));
143  } else if (RMWI.getType()->isFloatingPointTy() &&
144  RMWI.getOperation() != AtomicRMWInst::FAdd) {
146  return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType()));
147  }
148 
149  // Check if the required ordering is compatible with an atomic load.
150  if (Ordering != AtomicOrdering::Acquire &&
151  Ordering != AtomicOrdering::Monotonic)
152  return nullptr;
153 
154  LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand(), "",
155  false, DL.getABITypeAlign(RMWI.getType()),
156  Ordering, RMWI.getSyncScopeID());
157  return Load;
158 }
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::AtomicRMWInst::setOperation
void setOperation(BinOp Operation)
Definition: Instructions.h:806
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:740
llvm::AtomicRMWInst::getOperation
BinOp getOperation() const
Definition: Instructions.h:792
llvm::ConstantFP::getNegativeZero
static Constant * getNegativeZero(Type *Ty)
Definition: Constants.h:293
llvm::Type::isFloatingPointTy
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
Definition: Type.h:163
llvm::AtomicOrdering::Monotonic
@ Monotonic
llvm::InstCombinerImpl::visitAtomicRMWInst
Instruction * visitAtomicRMWInst(AtomicRMWInst &SI)
Definition: InstCombineAtomicRMW.cpp:97
llvm::AtomicRMWInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:853
llvm::InstCombinerImpl::eraseInstFromFunction
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Definition: InstCombineInternal.h:453
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
InstCombineInternal.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::Instruction
Definition: Instruction.h:42
llvm::DataLayout::getABITypeAlign
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:829
llvm::AtomicOrdering::Acquire
@ Acquire
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:924
llvm::AtomicRMWInst::Xchg
@ Xchg
*p = v
Definition: Instructions.h:728
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:730
llvm::Value::use_empty
bool use_empty() const
Definition: Value.h:344
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:191
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition: AtomicOrdering.h:56
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:748
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:297
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:732
llvm::AtomicRMWInst::getOrdering
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:832
llvm::AtomicOrdering::Unordered
@ Unordered
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:744
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:738
llvm::AtomicRMWInst::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
Definition: Instructions.h:844
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:173
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:714
llvm::AtomicOrdering::Release
@ Release
llvm::AtomicRMWInst::isVolatile
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
Definition: Instructions.h:822
llvm::AtomicRMWInst::FSub
@ FSub
*p = old - v
Definition: Instructions.h:754
llvm::InstCombiner::DL
const DataLayout & DL
Definition: InstCombiner.h:73
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:734
llvm::AtomicRMWInst::getValOperand
Value * getValOperand()
Definition: Instructions.h:857
llvm::InstCombinerImpl::replaceOperand
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
Definition: InstCombineInternal.h:426
Instructions.h
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition: Instructions.h:751
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:746
llvm::AtomicOrdering::NotAtomic
@ NotAtomic
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:742