LowerMemIntrinsics.cpp File Reference

#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <optional>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "lower-mem-intrinsics"

Functions
static Value *	getRuntimeLoopRemainder (IRBuilderBase &B, Value *Len, Value *OpSize, unsigned OpSizeVal)
static Value *	getRuntimeLoopUnits (IRBuilderBase &B, Value *Len, Value *OpSize, unsigned OpSizeVal, Value *RTLoopRemainder=nullptr)
static LoopExpansionInfo	insertLoopExpansion (Instruction *InsertBefore, Value *Len, unsigned MainLoopStep, unsigned ResidualLoopStep, StringRef BBNamePrefix)
	Insert the control flow and loop counters for a memcpy/memset loop expansion.
static std::pair< Value *, Value * >	tryInsertCastToCommonAddrSpace (IRBuilderBase &B, Value *Addr1, Value *Addr2, const TargetTransformInfo &TTI)
static void	createMemMoveLoopUnknownSize (Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, Value *CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo &TTI)
static void	createMemMoveLoopKnownSize (Instruction *InsertBefore, Value *SrcAddr, Value *DstAddr, ConstantInt *CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo &TTI)
static void	createMemSetLoop (Instruction *InsertBefore, Value *DstAddr, Value *CopyLen, Value *SetValue, Align DstAlign, bool IsVolatile)
template<typename T>
static bool	canOverlap (MemTransferBase< T > *Memcpy, ScalarEvolution *SE)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "lower-mem-intrinsics"

Definition at line 20 of file LowerMemIntrinsics.cpp.

Function Documentation

canOverlap()

template<typename T>

bool canOverlap ( MemTransferBase< T > * Memcpy, ScalarEvolution * SE )

static

Definition at line 972 of file LowerMemIntrinsics.cpp.

References llvm::MemTransferBase< BaseCL >::getRawSource(), llvm::ScalarEvolution::getSCEV(), llvm::CmpInst::ICMP_NE, and llvm::ScalarEvolution::isKnownPredicateAt().

Referenced by llvm::expandMemCpyAsLoop().

createMemMoveLoopKnownSize()

void createMemMoveLoopKnownSize ( Instruction * InsertBefore, Value * SrcAddr, Value * DstAddr, ConstantInt * CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo & TTI )

static

Definition at line 758 of file LowerMemIntrinsics.cpp.

References llvm::PHINode::addIncoming(), llvm::alignDown(), llvm::cast(), llvm::commonAlignment(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateCondBr(), llvm::IRBuilderBase::CreateICmpEQ(), llvm::IRBuilderBase::CreateICmpULT(), llvm::IRBuilderBase::CreateInBoundsGEP(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateSub(), DL, llvm::Instruction::eraseFromParent(), F, llvm::BasicBlock::getContext(), llvm::BasicBlock::getFirstNonPHIIt(), llvm::Type::getInt8Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::ConstantInt::getZExtValue(), llvm::ConstantInt::isZero(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::BasicBlock::splitBasicBlock(), llvm::SplitBlockAndInsertIfThenElse(), and tryInsertCastToCommonAddrSpace().

Referenced by llvm::expandMemMoveAsLoop().

createMemMoveLoopUnknownSize()

void createMemMoveLoopUnknownSize ( Instruction * InsertBefore, Value * SrcAddr, Value * DstAddr, Value * CopyLen, Align SrcAlign, Align DstAlign, bool SrcIsVolatile, bool DstIsVolatile, const TargetTransformInfo & TTI )

static

Definition at line 504 of file LowerMemIntrinsics.cpp.

References llvm::PHINode::addIncoming(), llvm::cast(), llvm::commonAlignment(), llvm::BasicBlock::Create(), llvm::BranchInst::Create(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateAlignedLoad(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateCondBr(), llvm::IRBuilderBase::CreateICmpEQ(), llvm::IRBuilderBase::CreateICmpULT(), llvm::IRBuilderBase::CreateInBoundsGEP(), llvm::IRBuilderBase::CreatePHI(), llvm::IRBuilderBase::CreateSub(), llvm::IRBuilderBase::CreateUnreachable(), DL, llvm::Instruction::eraseFromParent(), F, llvm::BasicBlock::getContext(), llvm::Type::getInt8Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), getRuntimeLoopRemainder(), getRuntimeLoopUnits(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThenElse(), and tryInsertCastToCommonAddrSpace().

Referenced by llvm::expandMemMoveAsLoop().

createMemSetLoop()

void createMemSetLoop ( Instruction * InsertBefore, Value * DstAddr, Value * CopyLen, Value * SetValue, Align DstAlign, bool IsVolatile )

static

Definition at line 932 of file LowerMemIntrinsics.cpp.

References llvm::PHINode::addIncoming(), llvm::commonAlignment(), llvm::BasicBlock::Create(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateAlignedStore(), llvm::IRBuilderBase::CreateCondBr(), llvm::IRBuilderBase::CreateICmpULT(), llvm::IRBuilderBase::CreateInBoundsGEP(), llvm::IRBuilderBase::CreatePHI(), DL, llvm::Instruction::eraseFromParent(), F, llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), SetValue(), and llvm::BasicBlock::splitBasicBlock().

Referenced by llvm::expandMemSetAsLoop(), and llvm::expandMemSetPatternAsLoop().

getRuntimeLoopRemainder()

Value * getRuntimeLoopRemainder ( IRBuilderBase & B, Value * Len, Value * OpSize, unsigned OpSizeVal )

static

Returns

Len urem OpSize, checking for optimization opportunities. OpSizeVal must be the integer value of the ConstantInt OpSize.

Definition at line 26 of file LowerMemIntrinsics.cpp.

References B(), and llvm::isPowerOf2_32().

Referenced by createMemMoveLoopUnknownSize(), getRuntimeLoopUnits(), and insertLoopExpansion().

getRuntimeLoopUnits()

Value * getRuntimeLoopUnits ( IRBuilderBase & B, Value * Len, Value * OpSize, unsigned OpSizeVal, Value * RTLoopRemainder = nullptr )

static

Returns

(Len udiv OpSize) mul OpSize, checking for optimization opportunities. If RTLoopRemainder is provided, it must be the result of getRuntimeLoopRemainder() with the same arguments.

Definition at line 38 of file LowerMemIntrinsics.cpp.

References B(), and getRuntimeLoopRemainder().

Referenced by createMemMoveLoopUnknownSize(), and insertLoopExpansion().

insertLoopExpansion()

LoopExpansionInfo insertLoopExpansion ( Instruction * InsertBefore, Value * Len, unsigned MainLoopStep, unsigned ResidualLoopStep, StringRef BBNamePrefix )

static

Insert the control flow and loop counters for a memcpy/memset loop expansion.

This function inserts IR corresponding to the following C code before InsertBefore:

LoopUnits = (Len / MainLoopStep) * MainLoopStep;
ResidualUnits = Len - LoopUnits;
MainLoopIndex = 0;
if (LoopUnits > 0) {
  do {
    // MainLoopIP
    MainLoopIndex += MainLoopStep;
  } while (MainLoopIndex < LoopUnits);
}
for (size_t i = 0; i < ResidualUnits; i += ResidualLoopStep) {
  ResidualLoopIndex = LoopUnits + i;
  // ResidualLoopIP
}

MainLoopStep and ResidualLoopStep determine by how many "units" the loop index is increased in each iteration of the main and residual loops, respectively. In most cases, the "unit" will be bytes, but larger units are useful for lowering memset.pattern.

The computation of LoopUnits and ResidualUnits is performed at compile time if Len is a ConstantInt. The second (residual) loop is omitted if ResidualLoopStep is 0 or equal to MainLoopStep. The generated MainLoopIP, MainLoopIndex, ResidualLoopIP, and ResidualLoopIndex are returned in a LoopExpansionInfo object.

Definition at line 97 of file LowerMemIntrinsics.cpp.

References llvm::PHINode::addIncoming(), llvm::alignDown(), assert(), llvm::cast(), llvm::BasicBlock::Create(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateBr(), llvm::IRBuilderBase::CreateCondBr(), llvm::IRBuilderBase::CreateICmpNE(), llvm::IRBuilderBase::CreateICmpULT(), llvm::IRBuilderBase::CreatePHI(), llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::BasicBlock::getContext(), llvm::BasicBlock::getParent(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), getRuntimeLoopRemainder(), getRuntimeLoopUnits(), llvm::BasicBlock::getTerminator(), and llvm::BasicBlock::splitBasicBlock().

Referenced by llvm::createMemCpyLoopKnownSize().

tryInsertCastToCommonAddrSpace()

std::pair< Value *, Value * > tryInsertCastToCommonAddrSpace ( IRBuilderBase & B, Value * Addr1, Value * Addr2, const TargetTransformInfo & TTI )

static

Definition at line 459 of file LowerMemIntrinsics.cpp.

References B(), llvm::cast(), llvm::Value::getType(), and llvm_unreachable.

Referenced by createMemMoveLoopKnownSize(), and createMemMoveLoopUnknownSize().