ScalarizeMaskedMemIntrin.cpp File Reference

#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <cassert>
#include <optional>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "scalarize-masked-mem-intrin"

Functions
static bool	optimizeBlock (BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, DomTreeUpdater *DTU)
static bool	optimizeCallInst (CallInst *CI, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, DomTreeUpdater *DTU)
	INITIALIZE_PASS_BEGIN (ScalarizeMaskedMemIntrinLegacyPass, DEBUG_TYPE, "Scalarize unsupported masked memory intrinsics", false, false) INITIALIZE_PASS_END(ScalarizeMaskedMemIntrinLegacyPass
static bool	isConstantIntVector (Value *Mask)
static unsigned	adjustForEndian (const DataLayout &DL, unsigned VectorWidth, unsigned Idx)
static void	scalarizeMaskedLoad (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedStore (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedGather (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedScatter (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedExpandLoad (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedCompressStore (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static void	scalarizeMaskedVectorHistogram (const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)
static bool	runImpl (Function &F, const TargetTransformInfo &TTI, DominatorTree *DT)

Variables
	DEBUG_TYPE
Scalarize unsupported masked memory	intrinsics
Scalarize unsupported masked memory	false

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "scalarize-masked-mem-intrin"

Definition at line 43 of file ScalarizeMaskedMemIntrin.cpp.

Function Documentation

adjustForEndian()

static unsigned adjustForEndian ( const DataLayout &  DL, unsigned  VectorWidth, unsigned  Idx  )

static

Definition at line 107 of file ScalarizeMaskedMemIntrin.cpp.

References DL, and Idx.

Referenced by scalarizeMaskedCompressStore(), scalarizeMaskedExpandLoad(), scalarizeMaskedGather(), scalarizeMaskedLoad(), scalarizeMaskedScatter(), and scalarizeMaskedStore().

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	ScalarizeMaskedMemIntrinLegacyPass	,
		DEBUG_TYPE	,
		"Scalarize unsupported masked memory intrinsics"	,
		false	,
		false
	)

isConstantIntVector()

static bool isConstantIntVector ( Value *  Mask )

static

Definition at line 92 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::CallingConv::C.

Referenced by scalarizeMaskedCompressStore(), scalarizeMaskedExpandLoad(), scalarizeMaskedGather(), scalarizeMaskedLoad(), scalarizeMaskedScatter(), scalarizeMaskedStore(), and scalarizeMaskedVectorHistogram().

optimizeBlock()

static bool optimizeBlock ( BasicBlock &  BB, bool &  ModifiedDT, const TargetTransformInfo &  TTI, const DataLayout &  DL, DomTreeUpdater *  DTU  )

static

Definition at line 1033 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::BasicBlock::begin(), DL, llvm::BasicBlock::end(), and optimizeCallInst().

Referenced by runImpl().

optimizeCallInst()

static bool optimizeCallInst ( CallInst *  CI, bool &  ModifiedDT, const TargetTransformInfo &  TTI, const DataLayout &  DL, DomTreeUpdater *  DTU  )

static

Definition at line 1049 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::any_of(), DL, llvm::TargetTransformInfo::forceScalarizeMaskedGather(), llvm::TargetTransformInfo::forceScalarizeMaskedScatter(), llvm::AttributeSet::getAlignment(), llvm::CallBase::getArgOperand(), llvm::CallBase::getAttributes(), llvm::AttributeList::getParamAttrs(), llvm::Type::getScalarType(), llvm::Value::getType(), II, llvm::TargetTransformInfo::isLegalMaskedCompressStore(), llvm::TargetTransformInfo::isLegalMaskedExpandLoad(), llvm::TargetTransformInfo::isLegalMaskedGather(), llvm::TargetTransformInfo::isLegalMaskedLoad(), llvm::TargetTransformInfo::isLegalMaskedScatter(), llvm::TargetTransformInfo::isLegalMaskedStore(), llvm::TargetTransformInfo::isLegalMaskedVectorHistogram(), scalarizeMaskedCompressStore(), scalarizeMaskedExpandLoad(), scalarizeMaskedGather(), scalarizeMaskedLoad(), scalarizeMaskedScatter(), scalarizeMaskedStore(), scalarizeMaskedVectorHistogram(), and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeBlock().

runImpl()

static bool runImpl ( Function &  F, const TargetTransformInfo &  TTI, DominatorTree *  DT  )

static

Definition at line 987 of file ScalarizeMaskedMemIntrin.cpp.

References DL, F, llvm::make_early_inc_range(), and optimizeBlock().

scalarizeMaskedCompressStore()

static void scalarizeMaskedCompressStore ( const DataLayout &  DL, CallInst *  CI, DomTreeUpdater *  DTU, bool &  ModifiedDT  )

static

Definition at line 816 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::PHINode::addIncoming(), adjustForEndian(), llvm::BasicBlock::begin(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateConstInBoundsGEP1_32(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateICmpNE(), llvm::IRBuilderBase::CreatePHI(), DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::IRBuilderBase::getInt(), llvm::IRBuilderBase::getIntN(), llvm::IRBuilderBase::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), Idx, isConstantIntVector(), Ptr, llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedExpandLoad()

static void scalarizeMaskedExpandLoad ( const DataLayout &  DL, CallInst *  CI, DomTreeUpdater *  DTU, bool &  ModifiedDT  )

static

Definition at line 684 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::PHINode::addIncoming(), adjustForEndian(), llvm::BasicBlock::begin(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateConstInBoundsGEP1_32(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateICmpNE(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateShuffleVector(), DL, llvm::Instruction::eraseFromParent(), llvm::PoisonValue::get(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::IRBuilderBase::getInt(), llvm::IRBuilderBase::getIntN(), llvm::IRBuilderBase::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::CallBase::getParamAlign(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), Idx, isConstantIntVector(), llvm::PoisonMaskElem, Ptr, llvm::Value::replaceAllUsesWith(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), and llvm::MaybeAlign::valueOrOne().

Referenced by optimizeCallInst().

scalarizeMaskedGather()

static void scalarizeMaskedGather ( const DataLayout &  DL, CallInst *  CI, DomTreeUpdater *  DTU, bool &  ModifiedDT  )

static

Definition at line 464 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), llvm::BasicBlock::begin(), llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateICmpNE(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreatePHI(), DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::IRBuilderBase::getInt(), llvm::IRBuilderBase::getIntN(), llvm::IRBuilderBase::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), Idx, isConstantIntVector(), Ptr, llvm::Value::replaceAllUsesWith(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), and llvm::SplitBlockAndInsertIfThen().

Referenced by optimizeCallInst().

scalarizeMaskedLoad()

static void scalarizeMaskedLoad ( const DataLayout &  DL, CallInst *  CI, DomTreeUpdater *  DTU, bool &  ModifiedDT  )

static

Definition at line 144 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), llvm::BasicBlock::begin(), llvm::commonAlignment(), llvm::Instruction::copyMetadata(), llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateConstInBoundsGEP1_32(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateICmpNE(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreatePHI(), DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::IRBuilderBase::getInt(), llvm::IRBuilderBase::getIntN(), llvm::IRBuilderBase::getIntNTy(), llvm::Value::getName(), llvm::APInt::getOneBitSet(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), Idx, isConstantIntVector(), llvm::isSplatValue(), Ptr, llvm::Value::replaceAllUsesWith(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), and llvm::Value::takeName().

Referenced by optimizeCallInst().

scalarizeMaskedScatter()

static void scalarizeMaskedScatter ( const DataLayout &  DL, CallInst *  CI, DomTreeUpdater *  DTU, bool &  ModifiedDT  )

static

Definition at line 594 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), assert(), llvm::BasicBlock::begin(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateICmpNE(), DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::IRBuilderBase::getInt(), llvm::IRBuilderBase::getIntN(), llvm::IRBuilderBase::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), Idx, isConstantIntVector(), Ptr, llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), and llvm::SplitBlockAndInsertIfThen().

Referenced by optimizeCallInst().

scalarizeMaskedStore()

static void scalarizeMaskedStore ( const DataLayout &  DL, CallInst *  CI, DomTreeUpdater *  DTU, bool &  ModifiedDT  )

static

Definition at line 315 of file ScalarizeMaskedMemIntrin.cpp.

References adjustForEndian(), llvm::BasicBlock::begin(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateConstInBoundsGEP1_32(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateICmpNE(), DL, llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::IRBuilderBase::getInt(), llvm::IRBuilderBase::getIntN(), llvm::IRBuilderBase::getIntNTy(), llvm::APInt::getOneBitSet(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Type::getPrimitiveSizeInBits(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), Idx, isConstantIntVector(), llvm::isSplatValue(), Ptr, llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), and llvm::SplitBlockAndInsertIfThen().

Referenced by optimizeCallInst().

scalarizeMaskedVectorHistogram()

static void scalarizeMaskedVectorHistogram ( const DataLayout &  DL, CallInst *  CI, DomTreeUpdater *  DTU, bool &  ModifiedDT  )

static

Definition at line 924 of file ScalarizeMaskedMemIntrin.cpp.

References llvm::Add, assert(), llvm::BasicBlock::begin(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateLoad(), llvm::IRBuilderBase::CreateStore(), llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::Instruction::getDebugLoc(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), Idx, isConstantIntVector(), llvm::Type::isVoidTy(), Ptr, llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), and llvm::SplitBlockAndInsertIfThen().

Referenced by optimizeCallInst().

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 84 of file ScalarizeMaskedMemIntrin.cpp.

false

Scalarize unsupported masked memory false

Definition at line 85 of file ScalarizeMaskedMemIntrin.cpp.

intrinsics

Scalarize unsupported masked memory intrinsics

Definition at line 85 of file ScalarizeMaskedMemIntrin.cpp.