ScalarizeMaskedMemIntrin.cpp File Reference

#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <cassert>
#include <optional>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "scalarize-masked-mem-intrin"

Functions
static bool	optimizeBlock (BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
static bool	optimizeCallInst (CallInst *CI, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
	INITIALIZE_PASS_BEGIN (ScalarizeMaskedMemIntrinLegacyPass, DEBUG_TYPE, "Scalarize unsupported masked memory intrinsics", false, false) INITIALIZE_PASS_END(ScalarizeMaskedMemIntrinLegacyPass
static bool	isConstantIntVector (Value *Mask)
static unsigned	adjustForEndian (const DataLayout &DL, unsigned VectorWidth, unsigned Idx)
static void	copyMemCacheHint (Instruction &Dest, const Instruction &Source, unsigned SourcePtrOperand, unsigned DestPtrOperand)
static void	copyMetadataForMemoryAccess (Instruction &Dest, const Instruction &Source, const DataLayout &DL, unsigned SourcePtrOperand, unsigned DestPtrOperand, Type *AccessType, bool IsWholeAccess, std::optional< size_t > ByteOffset)
static void	copyMetadataForScalarizedLoad (LoadInst &Dest, const Instruction &Source, const DataLayout &DL, unsigned SourcePtrOperand, std::optional< size_t > ByteOffset)
static void	copyMetadataForScalarizedStore (StoreInst &Dest, const Instruction &Source, const DataLayout &DL, unsigned SourcePtrOperand, std::optional< size_t > ByteOffset)
static void	scalarizeMaskedLoad (const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedStore (const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedGather (const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedScatter (const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedExpandLoad (const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedCompressStore (const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedVectorHistogram (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static bool	runImpl (Function &F, const TargetTransformInfo &TTI, DominatorTree *DT)

Variables
	DEBUG_TYPE
Scalarize unsupported masked memory	intrinsics
Scalarize unsupported masked memory	false

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "scalarize-masked-mem-intrin"

Definition at line 45 of file ScalarizeMaskedMemIntrin.cpp.

Function Documentation

adjustForEndian()

unsigned adjustForEndian ( const DataLayout & DL, unsigned VectorWidth, unsigned Idx )

static

Definition at line 110 of file ScalarizeMaskedMemIntrin.cpp.

References DL.

Referenced by scalarizeMaskedCompressStore(), scalarizeMaskedExpandLoad(), scalarizeMaskedGather(), scalarizeMaskedLoad(), scalarizeMaskedScatter(), and scalarizeMaskedStore().

copyMemCacheHint()

void copyMemCacheHint ( Instruction & Dest, const Instruction & Source, unsigned SourcePtrOperand, unsigned DestPtrOperand )

static

Definition at line 115 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::mdconst::extract(), llvm::ConstantAsMetadata::get(), llvm::MDNode::get(), llvm::Value::getContext(), llvm::Type::getInt32Ty(), llvm::MDNode::getNumOperands(), llvm::MDNode::getOperand(), and llvm::Instruction::setMetadata().

Referenced by copyMetadataForMemoryAccess().

copyMetadataForMemoryAccess()

void copyMetadataForMemoryAccess ( Instruction & Dest, const Instruction & Source, const DataLayout & DL, unsigned SourcePtrOperand, unsigned DestPtrOperand, Type * AccessType, bool IsWholeAccess, std::optional< size_t > ByteOffset )

static

Definition at line 134 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::AAMDNodes::adjustForAccess(), copyMemCacheHint(), llvm::Instruction::copyMetadata(), DL, llvm::Instruction::setAAMetadata(), and llvm::AAMDNodes::TBAAStruct.

Referenced by copyMetadataForScalarizedLoad(), and copyMetadataForScalarizedStore().

copyMetadataForScalarizedLoad()

void copyMetadataForScalarizedLoad ( LoadInst & Dest, const Instruction & Source, const DataLayout & DL, unsigned SourcePtrOperand, std::optional< size_t > ByteOffset )

static

Definition at line 161 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::Instruction::copyMetadata(), copyMetadataForMemoryAccess(), DL, llvm::LoadInst::getPointerOperandIndex(), and llvm::Value::getType().

Referenced by scalarizeMaskedExpandLoad(), scalarizeMaskedGather(), scalarizeMaskedLoad(), and scalarizeMaskedVectorHistogram().

copyMetadataForScalarizedStore()

void copyMetadataForScalarizedStore ( StoreInst & Dest, const Instruction & Source, const DataLayout & DL, unsigned SourcePtrOperand, std::optional< size_t > ByteOffset )

static

Definition at line 176 of file ScalarizeMaskedMemIntrin.cpp.

References copyMetadataForMemoryAccess(), DL, llvm::StoreInst::getPointerOperandIndex(), llvm::Value::getType(), and llvm::StoreInst::getValueOperand().

Referenced by scalarizeMaskedCompressStore(), scalarizeMaskedScatter(), scalarizeMaskedStore(), and scalarizeMaskedVectorHistogram().

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	ScalarizeMaskedMemIntrinLegacyPass	,
		DEBUG_TYPE	,
		"Scalarize unsupported masked memory intrinsics"	,
		false	,
		false	)

References DEBUG_TYPE, and INITIALIZE_PASS_DEPENDENCY.

isConstantIntVector()

bool isConstantIntVector ( Value * Mask )

static

Definition at line 95 of file ScalarizeMaskedMemIntrin.cpp.

References C(), llvm::cast(), llvm::dyn_cast(), and llvm::isa().

Referenced by optimizeCallInst(), scalarizeMaskedCompressStore(), scalarizeMaskedExpandLoad(), scalarizeMaskedGather(), scalarizeMaskedLoad(), scalarizeMaskedScatter(), scalarizeMaskedStore(), and scalarizeMaskedVectorHistogram().

optimizeBlock()

bool optimizeBlock ( BasicBlock & BB, bool & ModifiedDT, const TargetTransformInfo & TTI, const DataLayout & DL, bool HasBranchDivergence, DomTreeUpdater * DTU )

static

Definition at line 1247 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::BasicBlock::begin(), DL, llvm::dyn_cast(), llvm::BasicBlock::end(), and optimizeCallInst().

Referenced by runImpl().

optimizeCallInst()

bool optimizeCallInst ( CallInst * CI, bool & ModifiedDT, const TargetTransformInfo & TTI, const DataLayout & DL, bool HasBranchDivergence, DomTreeUpdater * DTU )

static

Definition at line 1264 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::any_of(), llvm::cast(), llvm::TargetTransformInfo::ConstantMask, DL, llvm::dyn_cast(), llvm::CallBase::getArgOperand(), llvm::CallBase::getAttributes(), llvm::CallBase::getParamAlign(), llvm::Value::getType(), II, llvm::isa(), isConstantIntVector(), scalarizeMaskedCompressStore(), scalarizeMaskedExpandLoad(), scalarizeMaskedGather(), scalarizeMaskedLoad(), scalarizeMaskedScatter(), scalarizeMaskedStore(), scalarizeMaskedVectorHistogram(), llvm::MaybeAlign::valueOrOne(), and llvm::TargetTransformInfo::VariableOrConstantMask.

Referenced by optimizeBlock().

runImpl()

bool runImpl ( Function & F, const TargetTransformInfo & TTI, DominatorTree * DT )

static

Definition at line 1200 of file ScalarizeMaskedMemIntrin.cpp.

References DL, F, llvm::make_early_inc_range(), and optimizeBlock().

scalarizeMaskedCompressStore()

void scalarizeMaskedCompressStore ( const DataLayout & DL, bool HasBranchDivergence, CallInst * CI, DomTreeUpdater * DTU, bool & ModifiedDT )

static

Definition at line 974 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::PHINode::addIncoming(), adjustForEndian(), llvm::BasicBlock::begin(), llvm::cast(), llvm::commonAlignment(), copyMetadataForScalarizedStore(), DEBUG_TYPE, DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::getExplicitlyUnknownBranchWeightsIfProfiled(), llvm::Instruction::getFunction(), llvm::Type::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), isConstantIntVector(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), llvm::Store, and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedExpandLoad()

void scalarizeMaskedExpandLoad ( const DataLayout & DL, bool HasBranchDivergence, CallInst * CI, DomTreeUpdater * DTU, bool & ModifiedDT )

static

Definition at line 827 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::PHINode::addIncoming(), adjustForEndian(), llvm::BasicBlock::begin(), llvm::cast(), llvm::commonAlignment(), copyMetadataForScalarizedLoad(), DEBUG_TYPE, DL, llvm::Instruction::eraseFromParent(), llvm::PoisonValue::get(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::getExplicitlyUnknownBranchWeightsIfProfiled(), llvm::Instruction::getFunction(), llvm::Type::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), isConstantIntVector(), llvm::Load, llvm::PoisonMaskElem, llvm::Value::replaceAllUsesWith(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedGather()

void scalarizeMaskedGather ( const DataLayout & DL, bool HasBranchDivergence, CallInst * CI, DomTreeUpdater * DTU, bool & ModifiedDT )

static

Definition at line 582 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), llvm::BasicBlock::begin(), llvm::cast(), copyMetadataForScalarizedLoad(), DEBUG_TYPE, DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::getExplicitlyUnknownBranchWeightsIfProfiled(), llvm::Instruction::getFunction(), llvm::Type::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), isConstantIntVector(), llvm::Load, llvm::Value::replaceAllUsesWith(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedLoad()

void scalarizeMaskedLoad ( const DataLayout & DL, bool HasBranchDivergence, CallInst * CI, DomTreeUpdater * DTU, bool & ModifiedDT )

static

Definition at line 220 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), llvm::BasicBlock::begin(), llvm::cast(), llvm::commonAlignment(), copyMetadataForScalarizedLoad(), DEBUG_TYPE, DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::getExplicitlyUnknownBranchWeightsIfProfiled(), llvm::Instruction::getFunction(), llvm::Type::getIntNTy(), llvm::Value::getName(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::isa(), isConstantIntVector(), llvm::isSplatValue(), llvm::Load, llvm::Value::replaceAllUsesWith(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), llvm::Value::takeName(), uint64_t, and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedScatter()

void scalarizeMaskedScatter ( const DataLayout & DL, bool HasBranchDivergence, CallInst * CI, DomTreeUpdater * DTU, bool & ModifiedDT )

static

Definition at line 724 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), assert(), llvm::BasicBlock::begin(), llvm::cast(), copyMetadataForScalarizedStore(), DEBUG_TYPE, DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::getExplicitlyUnknownBranchWeightsIfProfiled(), llvm::Instruction::getFunction(), llvm::Type::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::isa(), isConstantIntVector(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), llvm::Store, and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedStore()

void scalarizeMaskedStore ( const DataLayout & DL, bool HasBranchDivergence, CallInst * CI, DomTreeUpdater * DTU, bool & ModifiedDT )

static

Definition at line 408 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), llvm::BasicBlock::begin(), llvm::cast(), llvm::commonAlignment(), copyMetadataForScalarizedStore(), DEBUG_TYPE, DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::getExplicitlyUnknownBranchWeightsIfProfiled(), llvm::Instruction::getFunction(), llvm::Type::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::isa(), isConstantIntVector(), llvm::isSplatValue(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), llvm::Store, uint64_t, and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedVectorHistogram()

void scalarizeMaskedVectorHistogram ( const DataLayout & DL, CallInst * CI, DomTreeUpdater * DTU, bool & ModifiedDT )

static

Definition at line 1098 of file ScalarizeMaskedMemIntrin.cpp.

References assert(), llvm::BasicBlock::begin(), llvm::cast(), copyMetadataForScalarizedLoad(), copyMetadataForScalarizedStore(), DEBUG_TYPE, DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::getExplicitlyUnknownBranchWeightsIfProfiled(), llvm::Instruction::getFunction(), llvm::CallBase::getIntrinsicID(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), isConstantIntVector(), llvm::Type::isVoidTy(), llvm_unreachable, llvm::Load, llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), and llvm::Store.

Referenced by optimizeCallInst().

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 87 of file ScalarizeMaskedMemIntrin.cpp.

false

Scalarize unsupported masked memory false

Definition at line 88 of file ScalarizeMaskedMemIntrin.cpp.

intrinsics

Scalarize unsupported masked memory intrinsics

Definition at line 88 of file ScalarizeMaskedMemIntrin.cpp.