AtomicExpandPass.cpp File Reference

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/CodeGen/AtomicExpand.h"
#include "llvm/CodeGen/AtomicExpandUtils.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <cassert>
#include <cstdint>
#include <iterator>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "atomic-expand"

Functions
	INITIALIZE_PASS_BEGIN (AtomicExpandLegacy, DEBUG_TYPE, "Expand Atomic instructions", false, false) INITIALIZE_PASS_END(AtomicExpandLegacy
Expand Atomic static false unsigned	getAtomicOpSize (LoadInst *LI)
static unsigned	getAtomicOpSize (StoreInst *SI)
static unsigned	getAtomicOpSize (AtomicRMWInst *RMWI)
static unsigned	getAtomicOpSize (AtomicCmpXchgInst *CASI)
static void	copyMetadataForAtomic (Instruction &Dest, const Instruction &Source)
	Copy metadata that's safe to preserve when widening atomics.
template<typename Inst >
static bool	atomicSizeSupported (const TargetLowering *TLI, Inst *I)
static void	createCmpXchgInstFun (IRBuilderBase &Builder, Value *Addr, Value *Loaded, Value *NewVal, Align AddrAlign, AtomicOrdering MemOpOrder, SyncScope::ID SSID, Value *&Success, Value *&NewLoaded, Instruction *MetadataSrc)
static PartwordMaskValues	createMaskInstrs (IRBuilderBase &Builder, Instruction *I, Type *ValueType, Value *Addr, Align AddrAlign, unsigned MinWordSize)
	This is a helper function which builds instructions to provide values necessary for partword atomic operations.
static Value *	extractMaskedValue (IRBuilderBase &Builder, Value *WideWord, const PartwordMaskValues &PMV)
static Value *	insertMaskedValue (IRBuilderBase &Builder, Value *WideWord, Value *Updated, const PartwordMaskValues &PMV)
static Value *	performMaskedAtomicOp (AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Shifted_Inc, Value *Inc, const PartwordMaskValues &PMV)
	Emit IR to implement a masked version of a given atomicrmw operation.
static bool	canUseSizedAtomicCall (unsigned Size, Align Alignment, const DataLayout &DL)
static ArrayRef< RTLIB::Libcall >	GetRMWLibcall (AtomicRMWInst::BinOp Op)

Variables
	DEBUG_TYPE
Expand Atomic	instructions
Expand Atomic	false

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "atomic-expand"

Definition at line 59 of file AtomicExpandPass.cpp.

Function Documentation

atomicSizeSupported()

template<typename Inst >

static bool atomicSizeSupported ( const TargetLowering *  TLI, Inst *  I  )

static

Definition at line 232 of file AtomicExpandPass.cpp.

References getAtomicOpSize(), llvm::TargetLoweringBase::getMaxAtomicSizeInBitsSupported(), I, and Size.

canUseSizedAtomicCall()

static bool canUseSizedAtomicCall ( unsigned  Size, Align  Alignment, const DataLayout &  DL  )

static

Definition at line 1711 of file AtomicExpandPass.cpp.

References DL, and Size.

copyMetadataForAtomic()

static void copyMetadataForAtomic ( Instruction &  Dest, const Instruction &  Source  )

static

Copy metadata that's safe to preserve when widening atomics.

Definition at line 196 of file AtomicExpandPass.cpp.

References llvm::Value::getContext(), llvm::LLVMContext::getMDKindID(), N, and llvm::Instruction::setMetadata().

Referenced by createCmpXchgInstFun().

createCmpXchgInstFun()

static void createCmpXchgInstFun ( IRBuilderBase &  Builder, Value *  Addr, Value *  Loaded, Value *  NewVal, Align  AddrAlign, AtomicOrdering  MemOpOrder, SyncScope::ID  SSID, Value *&  Success, Value *&  NewLoaded, Instruction *  MetadataSrc  )

static

Definition at line 632 of file AtomicExpandPass.cpp.

References Addr, assert(), copyMetadataForAtomic(), llvm::IRBuilderBase::CreateAtomicCmpXchg(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateExtractValue(), llvm::IRBuilderBase::getIntNTy(), llvm::Type::getPrimitiveSizeInBits(), llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(), llvm::Value::getType(), llvm::Type::isFloatingPointTy(), llvm::Type::isPointerTy(), llvm::Type::isVectorTy(), and Success.

createMaskInstrs()

static PartwordMaskValues createMaskInstrs ( IRBuilderBase &  Builder, Instruction *  I, Type *  ValueType, Value *  Addr, Align  AddrAlign, unsigned  MinWordSize  )

static

This is a helper function which builds instructions to provide values necessary for partword atomic operations.

It takes an incoming address, Addr, and ValueType, and constructs the address, shift-amounts and masks needed to work with a larger value of size WordSize.

AlignedAddr: Addr rounded down to a multiple of WordSize

ShiftAmt: Number of bits to right-shift a WordSize value loaded from AlignAddr for it to have the same value as if ValueType was loaded from Addr.

Mask: Value to mask with the value loaded from AlignAddr to include only the part that would've been loaded from Addr.

Inv_Mask: The inverse of Mask.

Definition at line 799 of file AtomicExpandPass.cpp.

References Addr, assert(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::IRBuilderBase::CreateNot(), llvm::IRBuilderBase::CreatePtrToInt(), llvm::IRBuilderBase::CreateShl(), llvm::IRBuilderBase::CreateTrunc(), llvm::IRBuilderBase::CreateXor(), DL, llvm::Type::getIntNTy(), llvm::Constant::getNullValue(), and I.

extractMaskedValue()

static Value * extractMaskedValue ( IRBuilderBase &  Builder, Value *  WideWord, const PartwordMaskValues &  PMV  )

static

Definition at line 866 of file AtomicExpandPass.cpp.

References assert(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateLShr(), llvm::IRBuilderBase::CreateTrunc(), and llvm::Value::getType().

Referenced by performMaskedAtomicOp().

getAtomicOpSize() [1/4]

static unsigned getAtomicOpSize ( AtomicCmpXchgInst *  CASI )

static

Definition at line 190 of file AtomicExpandPass.cpp.

References DL, llvm::AtomicCmpXchgInst::getCompareOperand(), llvm::Instruction::getDataLayout(), and llvm::Value::getType().

getAtomicOpSize() [2/4]

static unsigned getAtomicOpSize ( AtomicRMWInst *  RMWI )

static

Definition at line 185 of file AtomicExpandPass.cpp.

References DL, llvm::Instruction::getDataLayout(), llvm::Value::getType(), and llvm::AtomicRMWInst::getValOperand().

getAtomicOpSize() [3/4]

Expand Atomic static false unsigned getAtomicOpSize ( LoadInst *  LI )

static

Definition at line 175 of file AtomicExpandPass.cpp.

References DL.

Referenced by atomicSizeSupported().

getAtomicOpSize() [4/4]

static unsigned getAtomicOpSize ( StoreInst *  SI )

static

Definition at line 180 of file AtomicExpandPass.cpp.

References DL.

GetRMWLibcall()

static ArrayRef< RTLIB::Libcall > GetRMWLibcall ( AtomicRMWInst::BinOp  Op )

static

Definition at line 1767 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::AtomicRMWInst::BAD_BINOP, llvm::AtomicRMWInst::FAdd, llvm::AtomicRMWInst::FMax, llvm::AtomicRMWInst::FMin, llvm::AtomicRMWInst::FSub, llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::USubCond, llvm::AtomicRMWInst::USubSat, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	AtomicExpandLegacy	,
		DEBUG_TYPE	,
		"Expand Atomic instructions"	,
		false	,
		false
	)

insertMaskedValue()

static Value * insertMaskedValue ( IRBuilderBase &  Builder, Value *  WideWord, Value *  Updated, const PartwordMaskValues &  PMV  )

static

Definition at line 877 of file AtomicExpandPass.cpp.

References llvm::And, assert(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateOr(), llvm::IRBuilderBase::CreateShl(), llvm::IRBuilderBase::CreateZExt(), llvm::Value::getType(), and llvm::Or.

Referenced by performMaskedAtomicOp().

performMaskedAtomicOp()

static Value * performMaskedAtomicOp ( AtomicRMWInst::BinOp  Op, IRBuilderBase &  Builder, Value *  Loaded, Value *  Shifted_Inc, Value *  Inc, const PartwordMaskValues &  PMV  )

static

Emit IR to implement a masked version of a given atomicrmw operation.

(That is, only the bits under the Mask should be affected by the operation)

Definition at line 897 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::buildAtomicRMWValue(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateOr(), extractMaskedValue(), llvm::AtomicRMWInst::FAdd, llvm::AtomicRMWInst::FMax, llvm::AtomicRMWInst::FMin, llvm::AtomicRMWInst::FSub, insertMaskedValue(), llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::USubCond, llvm::AtomicRMWInst::USubSat, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 171 of file AtomicExpandPass.cpp.

false

Expand Atomic false

Definition at line 172 of file AtomicExpandPass.cpp.

instructions

Expand Atomic instructions

Definition at line 172 of file AtomicExpandPass.cpp.

Referenced by addBoundsChecking(), llvm::coro::Shape::analyze(), llvm::X86TTIImpl::areInlineCompatible(), bitTrackingDCE(), canTRE(), checkFunctionMemoryAccess(), collectDbgVariableIntrinsics(), llvm::CollectDebugInfoForCloning(), collectMemAccessInfo(), collectPreserveStaticOffsetCalls(), llvm::coro::collectSpillsAndAllocasFromInsts(), convertAnnotation2Metadata(), createSCCNodeSet(), llvm::coro::doRematerializations(), eliminateDeadCode(), eraseDebugIntrinsicsWithNonLocalRefs(), explicifyGuards(), fixupDebugInfoPostExtraction(), functionWillReturn(), getEdgeKind(), llvm::GenericUniformityAnalysisImpl< ContextT >::initialize(), INITIALIZE_PASS(), insertCallAtAllFunctionExitPoints(), llvm::MLInlineAdvisor::MLInlineAdvisor(), llvm::InstDeleterIRStrategy::mutate(), llvm::StackSafetyGlobalInfo::print(), llvm::ScalarEvolution::print(), processDbgDeclares(), removeTailCallAttribute(), llvm::replaceCreatedSSACopys(), llvm::coro::BaseCloner::replaceEntryBlock(), rescheduleLexographically(), llvm::AliasSetsPrinterPass::run(), llvm::MemDerefPrinterPass::run(), llvm::StackLifetimePrinterPass::run(), llvm::KCFIPass::run(), llvm::PAEvalPass::run(), llvm::AssumeBuilderPass::run(), llvm::AMDGPULowerKernelAttributesPass::run(), llvm::DSEPass::run(), llvm::InlinerPass::run(), llvm::DevirtSCCRepeatedPass::run(), llvm::ModuleInlinerPass::run(), llvm::MustBeExecutedContextPrinterPass::run(), runImpl(), runNVVMIntrRange(), llvm::RewriteStatepointsForGC::runOnFunction(), runPass(), sinkLifetimeStartMarkers(), stripDeadDebugInfoImpl(), stripGCRelocates(), stripNonValidDataFromBody(), stripTBAA(), updateCGAndAnalysisManagerForPass(), and usesMSVCFloatingPoint().