LLVM 22.0.0git
|
#include "llvm/Transforms/Coroutines/CoroSplit.h"
#include "CoroCloner.h"
#include "CoroInternal.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/PriorityWorklist.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Coroutines/MaterializationUtils.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/CallGraphUpdater.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <iterator>
Go to the source code of this file.
Macros | |
#define | DEBUG_TYPE "coro-split" |
#define DEBUG_TYPE "coro-split" |
Definition at line 80 of file CoroSplit.cpp.
|
static |
Definition at line 860 of file CoroSplit.cpp.
Referenced by llvm::coro::BaseCloner::create().
|
static |
Definition at line 845 of file CoroSplit.cpp.
References Size.
Referenced by llvm::coro::BaseCloner::create().
|
static |
Definition at line 2136 of file CoroSplit.cpp.
References llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by llvm::CoroSplitPass::run().
|
static |
Definition at line 867 of file CoroSplit.cpp.
Referenced by llvm::coro::BaseCloner::create().
|
static |
Coerce the arguments in FnArgs
according to FnTy
in CallArgs
.
Definition at line 1683 of file CoroSplit.cpp.
References assert(), getType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ArrayRef< T >::size().
Referenced by llvm::coro::createMustTailCall().
|
static |
Returns all debug records in F.
Definition at line 627 of file CoroSplit.cpp.
References F, llvm::filterDbgVars(), I, instructions, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by doSplitCoroutine(), and llvm::coro::BaseCloner::salvageDebugInfo().
|
static |
Definition at line 444 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::Function::Create(), getFunctionTypeFromAsyncSuspend(), llvm::Value::getName(), llvm::GlobalValue::getParent(), llvm::coro::Shape::getResumeFunctionType(), and llvm::GlobalValue::InternalLinkage.
Referenced by llvm::coro::SwitchCloner::create(), llvm::coro::AnyRetconABI::splitCoroutine(), and llvm::coro::AsyncABI::splitCoroutine().
|
static |
Definition at line 2145 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroBegin, F, llvm::CoroBeginInst::getCustomABI(), llvm::CoroBeginInst::hasCustomABI(), llvm_unreachable, llvm::coro::Retcon, llvm::coro::RetconOnce, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::coro::Switch.
Referenced by llvm::CoroSplitPass::CoroSplitPass(), llvm::CoroSplitPass::CoroSplitPass(), llvm::CoroSplitPass::CoroSplitPass(), and llvm::CoroSplitPass::CoroSplitPass().
|
static |
Definition at line 1999 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, assert(), collectDbgVariableRecords(), llvm::coro::Shape::CoroBegin, llvm::coro::Shape::CoroSuspends, F, handleNoSuspendCoroutine(), hasSafeElideCaller(), lowerAwaitSuspends(), llvm::coro::normalizeCoroutine(), removeCoroEndsFromRampFunction(), removeCoroIsInRampFromRampFunction(), replaceFrameSizeAndAlignment(), replaceSwiftErrorOps(), llvm::coro::salvageDebugInfo(), llvm::coro::Shape::Shape(), simplifySuspendPoints(), and llvm::coro::Switch.
Referenced by llvm::CoroSplitPass::run().
|
static |
Definition at line 1119 of file CoroSplit.cpp.
References llvm::coro::Shape::CoroSizes, DL, and llvm::coro::Shape::FrameTy.
Referenced by replaceFrameSizeAndAlignment().
|
static |
Definition at line 436 of file CoroSplit.cpp.
References llvm::cast(), llvm::FunctionType::get(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), and llvm::Type::getVoidTy().
Referenced by createCloneDeclaration().
|
static |
Definition at line 1165 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroBegin, llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FrameAlign, llvm::coro::Shape::FrameTy, llvm::PoisonValue::get(), llvm::AnyCoroIdInst::getCoroAlloc(), llvm::coro::Shape::getSwitchCoroId(), llvm::coro::replaceCoroFree(), llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.
Referenced by doSplitCoroutine().
|
static |
Definition at line 1240 of file CoroSplit.cpp.
References llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), hasCallsInBlockBetween(), and hasCallsInBlocksBetween().
Referenced by simplifySuspendPoint().
|
static |
Definition at line 1199 of file CoroSplit.cpp.
References I, and llvm::isa().
Referenced by hasCallsBetween(), and hasCallsInBlocksBetween().
|
static |
Definition at line 1211 of file CoroSplit.cpp.
References llvm::SmallVectorTemplateCommon< T, typename >::empty(), hasCallsInBlockBetween(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::predecessors(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by hasCallsBetween().
Definition at line 1981 of file CoroSplit.cpp.
References llvm::dyn_cast(), and F.
Referenced by doSplitCoroutine().
|
static |
Definition at line 86 of file CoroSplit.cpp.
References Call, llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::CallingConv::Fast, FramePtr, llvm::FunctionType::get(), llvm::CallBase::getAttributes(), llvm::CoroAwaitSuspendInst::getAwaiter(), llvm::CallBase::getCalledFunction(), llvm::Value::getContext(), llvm::CoroAwaitSuspendInst::getFrame(), llvm::Function::getIntrinsicID(), llvm::PointerType::getUnqual(), llvm::Type::getVoidTy(), llvm::CoroAwaitSuspendInst::getWrapperFunction(), llvm_unreachable, llvm::coro::LowererBase::makeSubFnCall(), llvm::Value::replaceAllUsesWith(), llvm::CoroSubFnInst::ResumeIndex, llvm::coro::Shape::SymmetricTransfers, and Wrapper.
Referenced by lowerAwaitSuspends().
|
static |
Definition at line 152 of file CoroSplit.cpp.
References llvm::coro::Shape::CoroAwaitSuspends, F, and lowerAwaitSuspend().
Referenced by doSplitCoroutine().
|
static |
Definition at line 313 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, assert(), llvm::cast(), llvm::coro::Shape::CoroSuspends, FramePtr, llvm::coro::Shape::FrameTy, llvm::ConstantPointerNull::get(), llvm::coro::Shape::getIndex(), llvm::coro::Shape::getSwitchIndexField(), llvm::StructType::getTypeAtIndex(), llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend, llvm::coro::Shape::SwitchLoweringStorage::HasUnwindCoroEnd, llvm::coro::Shape::SwitchFieldIndex::Resume, llvm::coro::Switch, and llvm::coro::Shape::SwitchLowering.
Referenced by replaceUnwindCoroEnd().
|
static |
Definition at line 158 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, assert(), llvm::coro::Shape::emitDealloc(), FramePtr, llvm::coro::Shape::RetconLoweringStorage::IsFrameInlineInStorage, llvm::coro::Retcon, llvm::coro::Shape::RetconLowering, and llvm::coro::RetconOnce.
Referenced by replaceFallthroughCoroEnd(), and replaceUnwindCoroEnd().
|
static |
Definition at line 1151 of file CoroSplit.cpp.
References llvm::errs(), F, llvm::removeUnreachableBlocks(), llvm::report_fatal_error(), and llvm::verifyFunction().
Referenced by updateCallGraphAfterCoroutineSplit().
|
static |
Remove calls to llvm.coro.end in the original function.
Definition at line 1962 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, llvm::coro::Shape::CoroEnds, llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FramePtr, replaceCoroEnd(), and llvm::coro::Switch.
Referenced by doSplitCoroutine().
|
static |
Definition at line 1973 of file CoroSplit.cpp.
References llvm::coro::Shape::CoroIsInRampInsts, llvm::ConstantInt::getTrue(), and II.
Referenced by doSplitCoroutine().
|
static |
Definition at line 2123 of file CoroSplit.cpp.
References llvm::CallingConv::C, llvm::cast(), Changed, llvm::make_early_inc_range(), P, replacePrepare(), and llvm::Value::uses().
Referenced by llvm::CoroSplitPass::run().
|
static |
Definition at line 1668 of file CoroSplit.cpp.
References llvm::coro::Continuation, llvm::PoisonValue::get(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::CoroSuspendAsyncInst::getResumeFunction(), llvm::PointerType::getUnqual(), llvm::CoroSuspendAsyncInst::ResumeFunctionArg, and llvm::User::setOperand().
Referenced by llvm::coro::AsyncABI::splitCoroutine().
|
static |
Definition at line 384 of file CoroSplit.cpp.
References llvm::Instruction::eraseFromParent(), FramePtr, llvm::AnyCoroEndInst::isUnwind(), replaceFallthroughCoroEnd(), and replaceUnwindCoroEnd().
Referenced by removeCoroEndsFromRampFunction(), and llvm::coro::BaseCloner::replaceCoroEnds().
|
static |
Replace an llvm.coro.end.async.
Will inline the must tail call function call if there is one.
Definition at line 171 of file CoroSplit.cpp.
References assert(), llvm::cast(), llvm::dyn_cast(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), and llvm::InlineFunction().
Referenced by replaceFallthroughCoroEnd().
|
static |
Replace a non-unwind call to llvm.coro.end.
Definition at line 214 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::cast(), llvm::dyn_cast(), FramePtr, llvm::ConstantPointerNull::get(), llvm::ConstantTokenNone::get(), llvm::PoisonValue::get(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::coro::Shape::getResumeFunctionType(), llvm::FunctionType::getReturnType(), maybeFreeRetconStorage(), replaceCoroEndAsync(), llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.
Referenced by replaceCoroEnd().
|
static |
Definition at line 1127 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroAligns, llvm::coro::Shape::CoroSizes, llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FrameAlign, getFrameSizeForShape(), llvm::Value::getType(), llvm::Value::replaceAllUsesWith(), updateAsyncFuncPointerContextSize(), and llvm::Align::value().
Referenced by doSplitCoroutine().
|
static |
Replace a call to llvm.coro.prepare.retcon.
Definition at line 2087 of file CoroSplit.cpp.
References llvm::CallingConv::C, llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::make_early_inc_range(), llvm::Value::replaceAllUsesWith(), llvm::Value::stripPointerCasts(), and llvm::Value::uses().
Referenced by replaceAllPrepares().
|
static |
Definition at line 569 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::cast(), llvm::coro::Shape::CoroSuspends, F, llvm::Value::getType(), llvm::Value::replaceAllUsesWith(), and llvm::coro::Shape::SwiftErrorOps.
Referenced by llvm::coro::BaseCloner::create(), doSplitCoroutine(), and llvm::coro::BaseCloner::replaceSwiftErrorOps().
|
static |
Replace an unwind call to llvm.coro.end.
Definition at line 347 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::cast(), FramePtr, llvm::CallBase::getOperandBundle(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), markCoroutineAsDone(), maybeFreeRetconStorage(), llvm::LLVMContext::OB_funclet, llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.
Referenced by replaceCoroEnd().
|
static |
Definition at line 1267 of file CoroSplit.cpp.
References llvm::BranchInst::Create(), llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::CallBase::getCalledOperand(), llvm::CoroSuspendInst::getCoroSave(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getPrevNode(), hasCallsBetween(), I, llvm::Value::replaceAllUsesWith(), and llvm::Value::stripPointerCasts().
Referenced by simplifySuspendPoints().
|
static |
Definition at line 1327 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, assert(), llvm::cast(), llvm::coro::Shape::CoroBegin, llvm::coro::Shape::CoroSuspends, llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend, I, N, simplifySuspendPoint(), std::swap(), llvm::coro::Switch, and llvm::coro::Shape::SwitchLowering.
Referenced by doSplitCoroutine().
|
static |
Definition at line 1104 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::coro::Shape::AsyncLoweringStorage::AsyncFuncPointer, llvm::coro::Shape::AsyncLowering, llvm::cast(), llvm::coro::Shape::AsyncLoweringStorage::ContextSize, llvm::ConstantStruct::get(), llvm::GlobalVariable::getInitializer(), and llvm::GlobalVariable::setInitializer().
Referenced by replaceFrameSizeAndAlignment().
|
static |
Definition at line 2048 of file CoroSplit.cpp.
References llvm::coro::Shape::ABI, llvm::LazyCallGraph::addSplitFunction(), llvm::LazyCallGraph::addSplitRefRecursiveFunctions(), llvm::coro::Async, llvm::CallingConv::C, llvm::SmallVectorTemplateCommon< T, typename >::empty(), FAM, N, postSplitCleanup(), llvm::coro::Retcon, llvm::coro::RetconOnce, llvm::coro::Switch, llvm::updateCGAndAnalysisManagerForCGSCCPass(), and llvm::updateCGAndAnalysisManagerForFunctionPass().
Referenced by llvm::CoroSplitPass::run().
|
static |
Adjust the scope line of the funclet to the first line number after the suspend point.
This avoids a jump in the line table from the function declaration (where prologue instructions are attributed to) to the suspend point. Only adjust the scope line when the files are the same. If no candidate line number is found, fallback to the line of ActiveSuspend.
Definition at line 802 of file CoroSplit.cpp.
References DL, llvm::dyn_cast_or_null(), llvm::BasicBlock::end(), llvm::Instruction::getDebugLoc(), llvm::DIScope::getFile(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getNextNode(), llvm::skipDebugIntrinsics(), and llvm::Successor.
Referenced by llvm::coro::BaseCloner::create().