#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include "llvm/Analysis/CtxProfAnalysis.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/TypeMetadataUtils.h"
#include "llvm/IR/AttributeMask.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/ProfileData/PGOCtxProfReader.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"

Macros
#define	DEBUG_TYPE "call-promotion-utils"

Functions
static void	fixupPHINodeForNormalDest (InvokeInst Invoke, BasicBlock OrigBlock, BasicBlock *MergeBlock)
	Fix-up phi nodes in an invoke instruction's normal destination.

static void	fixupPHINodeForUnwindDest (InvokeInst Invoke, BasicBlock OrigBlock, BasicBlock ThenBlock, BasicBlock ElseBlock)
	Fix-up phi nodes in an invoke instruction's unwind destination.

static void	createRetPHINode (Instruction OrigInst, Instruction NewInst, BasicBlock *MergeBlock, IRBuilder<> &Builder)
	Create a phi node for the returned value of a call or invoke instruction.

static void	createRetBitCast (CallBase &CB, Type RetTy, CastInst *RetBitCast)
	Cast a call or invoke instruction to the given type.

static CallBase &	versionCallSiteWithCond (CallBase &CB, Value Cond, MDNode BranchWeights)
	Predicate and clone the given call site.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "call-promotion-utils"

Definition at line 29 of file CallPromotionUtils.cpp.

Function Documentation

◆ createRetBitCast()

static void createRetBitCast	(	CallBase &	CB,
		Type *	RetTy,
		CastInst **	RetBitCast
	)

static

Cast a call or invoke instruction to the given type.

When promoting a call site, the return type of the call site might not match that of the callee. If this is the case, we have to cast the returned value to the correct type. The location of the cast depends on if we have a call or invoke instruction.

For example, if the call instruction below requires a bitcast after promotion:

orig_bb: t0 = call i32 @func() ...

The bitcast is placed after the call instruction:

orig_bb: ; Uses of the original return value are replaced by uses of the bitcast. t0 = call i32 @func() t1 = bitcast i32 t0 to ... ...

A similar transformation is performed for invoke instructions. However, since invokes are terminating, a new block is created for the bitcast. For example, if the invoke instruction below requires a bitcast after promotion:

orig_bb: t0 = invoke i32 @func() to label normal_dst unwind label unwind_dst

The edge between the original block and the invoke's normal destination is split, and the bitcast is placed there:

orig_bb: t0 = invoke i32 @func() to label split_bb unwind label unwind_dst

split_bb: ; Uses of the original return value are replaced by uses of the bitcast. t1 = bitcast i32 t0 to ... br label normal_dst

Definition at line 168 of file CallPromotionUtils.cpp.

References llvm::BasicBlock::begin(), llvm::CastInst::CreateBitOrPointerCast(), llvm::ilist_node_impl< OptionsT >::getIterator(), RetTy, llvm::SplitEdge(), and llvm::Value::users().

Referenced by llvm::promoteCall().

◆ createRetPHINode()

static void createRetPHINode	(	Instruction *	OrigInst,
		Instruction *	NewInst,
		BasicBlock *	MergeBlock,
		IRBuilder<> &	Builder
	)

static

Create a phi node for the returned value of a call or invoke instruction.

After versioning a call or invoke instruction that returns a value, we have to merge the value of the original and new instructions. We do this by creating a phi node and replacing uses of the original instruction with this phi node.

For example, if OrigInst is defined in "else_bb" and NewInst is defined in "then_bb", we create the following phi node:

; Uses of the original instruction are replaced by uses of the phi node. t0 = phi i32 [ orig_inst, else_bb ], [ new_inst, then_bb ],

Definition at line 113 of file CallPromotionUtils.cpp.

References llvm::BasicBlock::begin(), llvm::IRBuilderBase::CreatePHI(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Value::getType(), llvm::Type::isVoidTy(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::use_empty(), and llvm::Value::users().

Referenced by versionCallSiteWithCond().

◆ fixupPHINodeForNormalDest()

static void fixupPHINodeForNormalDest	(	InvokeInst *	Invoke,
		BasicBlock *	OrigBlock,
		BasicBlock *	MergeBlock
	)

static

Fix-up phi nodes in an invoke instruction's normal destination.

After versioning an invoke instruction, values coming from the original block will now be coming from the "merge" block. For example, in the code below:

then_bb: t0 = invoke i32 ptr() to label merge_bb unwind label unwind_dst

else_bb: t1 = invoke i32 ptr() to label merge_bb unwind label unwind_dst

merge_bb: t2 = phi i32 [ t0, then_bb ], [ t1, else_bb ] br normal_dst

normal_dst: t3 = phi i32 [ x, orig_bb ], ...

"orig_bb" is no longer a predecessor of "normal_dst", so the phi nodes in "normal_dst" must be fixed to refer to "merge_bb":

normal_dst: t3 = phi i32 [ x, merge_bb ], ...

Definition at line 56 of file CallPromotionUtils.cpp.

References llvm::InvokeInst::getNormalDest(), Idx, and llvm::BasicBlock::phis().

Referenced by versionCallSiteWithCond().

◆ fixupPHINodeForUnwindDest()

static void fixupPHINodeForUnwindDest	(	InvokeInst *	Invoke,
		BasicBlock *	OrigBlock,
		BasicBlock *	ThenBlock,
		BasicBlock *	ElseBlock
	)

static

Fix-up phi nodes in an invoke instruction's unwind destination.

After versioning an invoke instruction, values coming from the original block will now be coming from either the "then" block or the "else" block. For example, in the code below:

then_bb: t0 = invoke i32 ptr() to label merge_bb unwind label unwind_dst

else_bb: t1 = invoke i32 ptr() to label merge_bb unwind label unwind_dst

unwind_dst: t3 = phi i32 [ x, orig_bb ], ...

"orig_bb" is no longer a predecessor of "unwind_dst", so the phi nodes in "unwind_dst" must be fixed to refer to "then_bb" and "else_bb":

unwind_dst: t3 = phi i32 [ x, then_bb ], [ x, else_bb ], ...

Definition at line 87 of file CallPromotionUtils.cpp.

References llvm::InvokeInst::getUnwindDest(), Idx, and llvm::BasicBlock::phis().

Referenced by versionCallSiteWithCond().

◆ versionCallSiteWithCond()

static CallBase & versionCallSiteWithCond	(	CallBase &	CB,
		Value *	Cond,
		MDNode *	BranchWeights
	)

static

Predicate and clone the given call site.

This function creates an if-then-else structure at the location of the call site. The "if" condition is specified by Cond. The original call site is moved into the "else" block, and a clone of the call site is placed in the "then" block. The cloned instruction is returned.

For example, the call instruction below:

orig_bb: t0 = call i32 ptr() ...

Is replace by the following:

orig_bb: cond = Cond br i1 cond, then_bb, else_bb

then_bb: ; The clone of the original call instruction is placed in the "then" ; block. It is not yet promoted. t1 = call i32 ptr() br merge_bb

else_bb: ; The original call instruction is moved to the "else" block. t0 = call i32 ptr() br merge_bb

merge_bb: ; Uses of the original call instruction are replaced by uses of the phi ; node. t2 = phi i32 [ t0, else_bb ], [ t1, then_bb ] ...

A similar transformation is performed for invoke instructions. However, since invokes are terminating, more work is required. For example, the invoke instruction below:

orig_bb: t0 = invoke ptr() to label normal_dst unwind label unwind_dst

Is replace by the following:

orig_bb: cond = Cond br i1 cond, then_bb, else_bb

then_bb: ; The clone of the original invoke instruction is placed in the "then" ; block, and its normal destination is set to the "merge" block. It is ; not yet promoted. t1 = invoke i32 ptr() to label merge_bb unwind label unwind_dst

else_bb: ; The original invoke instruction is moved into the "else" block, and ; its normal destination is set to the "merge" block. t0 = invoke i32 ptr() to label merge_bb unwind label unwind_dst

merge_bb: ; Uses of the original invoke instruction are replaced by uses of the ; phi node, and the merge block branches to the normal destination. t2 = phi i32 [ t0, else_bb ], [ t1, then_bb ] br normal_dst

An indirect musttail call is processed slightly differently in that:

No merge block needed for the orginal and the cloned callsite, since either one ends the flow. No phi node is needed either.
The return statement following the original call site is duplicated too and placed immediately after the cloned call site per the IR convention.

For example, the musttail call instruction below:

orig_bb: t0 = musttail call i32 ptr() ...

Is replaced by the following:

cond_bb: cond = Cond br i1 cond, then_bb, orig_bb

then_bb: ; The clone of the original call instruction is placed in the "then" ; block. It is not yet promoted. t1 = musttail call i32 ptr() ret t1

orig_bb: ; The original call instruction stays in its original block. t0 = musttail call i32 ptr() ret t0

Definition at line 287 of file CallPromotionUtils.cpp.

References assert(), llvm::Instruction::clone(), Cond, llvm::IRBuilderBase::CreateBr(), createRetPHINode(), llvm::Instruction::eraseFromParent(), fixupPHINodeForNormalDest(), fixupPHINodeForUnwindDest(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getNextNode(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Instruction::insertBefore(), llvm::CallBase::isMustTailCall(), llvm::Instruction::moveBefore(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::SplitBlockAndInsertIfThen(), and llvm::SplitBlockAndInsertIfThenElse().

Referenced by llvm::promoteCallWithVTableCmp(), and llvm::versionCallSite().

Macros

Functions

Macro Definition Documentation

◆ DEBUG_TYPE

Function Documentation

◆ createRetBitCast()

◆ createRetPHINode()

◆ fixupPHINodeForNormalDest()

◆ fixupPHINodeForUnwindDest()

◆ versionCallSiteWithCond()