AtomicExpandPass.cpp File Reference

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/CodeGen/AtomicExpand.h"
#include "llvm/CodeGen/AtomicExpandUtils.h"
#include "llvm/CodeGen/RuntimeLibcallUtil.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <cassert>
#include <cstdint>
#include <iterator>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "atomic-expand"

Functions
	INITIALIZE_PASS_BEGIN (AtomicExpandLegacy, DEBUG_TYPE, "Expand Atomic instructions", false, false) INITIALIZE_PASS_END(AtomicExpandLegacy
Expand Atomic static false unsigned	getAtomicOpSize (LoadInst *LI)
static unsigned	getAtomicOpSize (StoreInst *SI)
static unsigned	getAtomicOpSize (AtomicRMWInst *RMWI)
static unsigned	getAtomicOpSize (AtomicCmpXchgInst *CASI)
template<typename Inst >
static bool	atomicSizeSupported (const TargetLowering *TLI, Inst *I)
static void	createCmpXchgInstFun (IRBuilderBase &Builder, Value *Addr, Value *Loaded, Value *NewVal, Align AddrAlign, AtomicOrdering MemOpOrder, SyncScope::ID SSID, Value *&Success, Value *&NewLoaded)
static PartwordMaskValues	createMaskInstrs (IRBuilderBase &Builder, Instruction *I, Type *ValueType, Value *Addr, Align AddrAlign, unsigned MinWordSize)
	This is a helper function which builds instructions to provide values necessary for partword atomic operations.
static Value *	extractMaskedValue (IRBuilderBase &Builder, Value *WideWord, const PartwordMaskValues &PMV)
static Value *	insertMaskedValue (IRBuilderBase &Builder, Value *WideWord, Value *Updated, const PartwordMaskValues &PMV)
static Value *	performMaskedAtomicOp (AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Shifted_Inc, Value *Inc, const PartwordMaskValues &PMV)
	Emit IR to implement a masked version of a given atomicrmw operation.
static void	copyMetadataForAtomic (Instruction &Dest, const Instruction &Source)
	Copy metadata that's safe to preserve when widening atomics.
static bool	canUseSizedAtomicCall (unsigned Size, Align Alignment, const DataLayout &DL)
static ArrayRef< RTLIB::Libcall >	GetRMWLibcall (AtomicRMWInst::BinOp Op)

Variables
	DEBUG_TYPE
Expand Atomic	instructions
Expand Atomic	false

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "atomic-expand"

Definition at line 61 of file AtomicExpandPass.cpp.

Function Documentation

atomicSizeSupported()

template<typename Inst >

static bool atomicSizeSupported ( const TargetLowering *  TLI, Inst *  I  )

static

Definition at line 201 of file AtomicExpandPass.cpp.

References getAtomicOpSize(), llvm::TargetLoweringBase::getMaxAtomicSizeInBitsSupported(), I, and Size.

canUseSizedAtomicCall()

static bool canUseSizedAtomicCall ( unsigned  Size, Align  Alignment, const DataLayout &  DL  )

static

Definition at line 1693 of file AtomicExpandPass.cpp.

References DL, and Size.

copyMetadataForAtomic()

static void copyMetadataForAtomic ( Instruction &  Dest, const Instruction &  Source  )

static

Copy metadata that's safe to preserve when widening atomics.

Definition at line 962 of file AtomicExpandPass.cpp.

References llvm::Value::getContext(), llvm::LLVMContext::getMDKindID(), N, and llvm::Instruction::setMetadata().

createCmpXchgInstFun()

static void createCmpXchgInstFun ( IRBuilderBase &  Builder, Value *  Addr, Value *  Loaded, Value *  NewVal, Align  AddrAlign, AtomicOrdering  MemOpOrder, SyncScope::ID  SSID, Value *&  Success, Value *&  NewLoaded  )

static

Definition at line 594 of file AtomicExpandPass.cpp.

References Addr, assert(), llvm::IRBuilderBase::CreateAtomicCmpXchg(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateExtractValue(), llvm::IRBuilderBase::getIntNTy(), llvm::Type::getPrimitiveSizeInBits(), llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(), llvm::Value::getType(), llvm::Type::isFloatingPointTy(), llvm::Type::isPointerTy(), llvm::Type::isVectorTy(), and Success.

createMaskInstrs()

static PartwordMaskValues createMaskInstrs ( IRBuilderBase &  Builder, Instruction *  I, Type *  ValueType, Value *  Addr, Align  AddrAlign, unsigned  MinWordSize  )

static

This is a helper function which builds instructions to provide values necessary for partword atomic operations.

It takes an incoming address, Addr, and ValueType, and constructs the address, shift-amounts and masks needed to work with a larger value of size WordSize.

AlignedAddr: Addr rounded down to a multiple of WordSize

ShiftAmt: Number of bits to right-shift a WordSize value loaded from AlignAddr for it to have the same value as if ValueType was loaded from Addr.

Mask: Value to mask with the value loaded from AlignAddr to include only the part that would've been loaded from Addr.

Inv_Mask: The inverse of Mask.

Definition at line 757 of file AtomicExpandPass.cpp.

References Addr, assert(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::IRBuilderBase::CreateNot(), llvm::IRBuilderBase::CreatePtrToInt(), llvm::IRBuilderBase::CreateShl(), llvm::IRBuilderBase::CreateTrunc(), llvm::IRBuilderBase::CreateXor(), DL, llvm::Type::getIntNTy(), llvm::Constant::getNullValue(), and I.

extractMaskedValue()

static Value * extractMaskedValue ( IRBuilderBase &  Builder, Value *  WideWord, const PartwordMaskValues &  PMV  )

static

Definition at line 824 of file AtomicExpandPass.cpp.

References assert(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateLShr(), llvm::IRBuilderBase::CreateTrunc(), and llvm::Value::getType().

Referenced by performMaskedAtomicOp().

getAtomicOpSize() [1/4]

static unsigned getAtomicOpSize ( AtomicCmpXchgInst *  CASI )

static

Definition at line 192 of file AtomicExpandPass.cpp.

References DL, llvm::AtomicCmpXchgInst::getCompareOperand(), llvm::Instruction::getDataLayout(), and llvm::Value::getType().

getAtomicOpSize() [2/4]

static unsigned getAtomicOpSize ( AtomicRMWInst *  RMWI )

static

Definition at line 187 of file AtomicExpandPass.cpp.

References DL, llvm::Instruction::getDataLayout(), llvm::Value::getType(), and llvm::AtomicRMWInst::getValOperand().

getAtomicOpSize() [3/4]

Expand Atomic static false unsigned getAtomicOpSize ( LoadInst *  LI )

static

Definition at line 177 of file AtomicExpandPass.cpp.

References DL.

Referenced by atomicSizeSupported().

getAtomicOpSize() [4/4]

static unsigned getAtomicOpSize ( StoreInst *  SI )

static

Definition at line 182 of file AtomicExpandPass.cpp.

References DL.

GetRMWLibcall()

static ArrayRef< RTLIB::Libcall > GetRMWLibcall ( AtomicRMWInst::BinOp  Op )

static

Definition at line 1749 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::AtomicRMWInst::BAD_BINOP, llvm::AtomicRMWInst::FAdd, llvm::AtomicRMWInst::FMax, llvm::AtomicRMWInst::FMin, llvm::AtomicRMWInst::FSub, llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	AtomicExpandLegacy	,
		DEBUG_TYPE	,
		"Expand Atomic instructions"	,
		false	,
		false
	)

insertMaskedValue()

static Value * insertMaskedValue ( IRBuilderBase &  Builder, Value *  WideWord, Value *  Updated, const PartwordMaskValues &  PMV  )

static

Definition at line 835 of file AtomicExpandPass.cpp.

References llvm::And, assert(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateOr(), llvm::IRBuilderBase::CreateShl(), llvm::IRBuilderBase::CreateZExt(), llvm::Value::getType(), and llvm::Or.

Referenced by performMaskedAtomicOp().

performMaskedAtomicOp()

static Value * performMaskedAtomicOp ( AtomicRMWInst::BinOp  Op, IRBuilderBase &  Builder, Value *  Loaded, Value *  Shifted_Inc, Value *  Inc, const PartwordMaskValues &  PMV  )

static

Emit IR to implement a masked version of a given atomicrmw operation.

(That is, only the bits under the Mask should be affected by the operation)

Definition at line 855 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::buildAtomicRMWValue(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateOr(), extractMaskedValue(), llvm::AtomicRMWInst::FAdd, llvm::AtomicRMWInst::FMax, llvm::AtomicRMWInst::FMin, llvm::AtomicRMWInst::FSub, insertMaskedValue(), llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 173 of file AtomicExpandPass.cpp.

false

Expand Atomic false

Definition at line 174 of file AtomicExpandPass.cpp.

instructions

Expand Atomic instructions

Definition at line 174 of file AtomicExpandPass.cpp.

Referenced by addBoundsChecking(), llvm::X86TTIImpl::areInlineCompatible(), bitTrackingDCE(), llvm::coro::buildCoroutineFrame(), llvm::coro::Shape::buildFrom(), llvm::Function::callsFunctionThatReturnsTwice(), canTRE(), checkFunctionMemoryAccess(), collectDbgVariableIntrinsics(), collectPreserveStaticOffsetCalls(), convertAnnotation2Metadata(), createSCCNodeSet(), doRematerializations(), eliminateDeadCode(), eraseDebugIntrinsicsWithNonLocalRefs(), explicifyGuards(), fixupDebugInfoPostExtraction(), functionWillReturn(), getEdgeKind(), llvm::GenericUniformityAnalysisImpl< ContextT >::initialize(), INITIALIZE_PASS(), llvm::MLInlineAdvisor::MLInlineAdvisor(), llvm::InstDeleterIRStrategy::mutate(), llvm::StackSafetyGlobalInfo::print(), llvm::ScalarEvolution::print(), processDbgDeclares(), removeTailCallAttribute(), llvm::replaceCreatedSSACopys(), rescheduleLexographically(), llvm::AliasSetsPrinterPass::run(), llvm::MemDerefPrinterPass::run(), llvm::StackLifetimePrinterPass::run(), llvm::KCFIPass::run(), llvm::PAEvalPass::run(), llvm::AssumeBuilderPass::run(), llvm::AMDGPULowerKernelAttributesPass::run(), llvm::DSEPass::run(), llvm::InlinerPass::run(), llvm::DevirtSCCRepeatedPass::run(), llvm::ModuleInlinerPass::run(), llvm::MustBeExecutedContextPrinterPass::run(), runImpl(), runNVVMIntrRange(), llvm::RewriteStatepointsForGC::runOnFunction(), runPass(), sinkLifetimeStartMarkers(), stripDeadDebugInfoImpl(), stripGCRelocates(), stripNonValidDataFromBody(), stripTBAA(), updateCGAndAnalysisManagerForPass(), and usesMSVCFloatingPoint().