27 for (
Value *BndlV : Bndl) {
29 Operands.push_back(BndlI->getOperand(OpIdx));
39 "Expect Instructions!");
40 auto &Ctx = Bndl[0]->getContext();
50 case Instruction::Opcode::ZExt:
51 case Instruction::Opcode::SExt:
52 case Instruction::Opcode::FPToUI:
53 case Instruction::Opcode::FPToSI:
54 case Instruction::Opcode::FPExt:
55 case Instruction::Opcode::PtrToAddr:
56 case Instruction::Opcode::PtrToInt:
57 case Instruction::Opcode::IntToPtr:
58 case Instruction::Opcode::SIToFP:
59 case Instruction::Opcode::UIToFP:
60 case Instruction::Opcode::Trunc:
61 case Instruction::Opcode::FPTrunc:
62 case Instruction::Opcode::BitCast: {
67 case Instruction::Opcode::FCmp:
68 case Instruction::Opcode::ICmp: {
74 "Expected same predicate across bundle.");
78 case Instruction::Opcode::Select: {
82 case Instruction::Opcode::FNeg: {
84 auto OpC = UOp0->getOpcode();
88 case Instruction::Opcode::Add:
89 case Instruction::Opcode::FAdd:
90 case Instruction::Opcode::Sub:
91 case Instruction::Opcode::FSub:
92 case Instruction::Opcode::Mul:
93 case Instruction::Opcode::FMul:
94 case Instruction::Opcode::UDiv:
95 case Instruction::Opcode::SDiv:
96 case Instruction::Opcode::FDiv:
97 case Instruction::Opcode::URem:
98 case Instruction::Opcode::SRem:
99 case Instruction::Opcode::FRem:
100 case Instruction::Opcode::Shl:
101 case Instruction::Opcode::LShr:
102 case Instruction::Opcode::AShr:
103 case Instruction::Opcode::And:
104 case Instruction::Opcode::Or:
105 case Instruction::Opcode::Xor: {
110 BinOp0->getOpcode(),
LHS,
RHS, BinOp0, WhereIt, Ctx,
"Vec");
112 case Instruction::Opcode::Load: {
115 Value *Ptr = Ld0->getPointerOperand();
118 case Instruction::Opcode::Store: {
124 case Instruction::Opcode::UncondBr:
125 case Instruction::Opcode::CondBr:
126 case Instruction::Opcode::Ret:
127 case Instruction::Opcode::PHI:
128 case Instruction::Opcode::AddrSpaceCast:
129 case Instruction::Opcode::Call:
130 case Instruction::Opcode::GetElementPtr:
141 auto *NewI = CreateVectorInstr(Bndl,
Operands);
151 VecOp->getContext(),
"VShuf");
165 Context &Ctx = ToPack[0]->getContext();
167 unsigned InsertIdx = 0;
168 for (
Value *Elm : ToPack) {
171 if (Elm->getType()->isVectorTy()) {
174 for (
auto ExtrLane :
seq<int>(0, NumElms)) {
187 LastInsert, ExtrI, InsertLaneC, WhereIt, Ctx,
"VPack");
188 LastInsert = InsertI;
198 WhereIt, Ctx,
"Pack");
200 WhereIt = std::next(NewI->getIterator());
209 bool StopForDebug = DebugBndlCnt++ >= Opts->sbvec_stop_bndl &&
213 const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
214 : Legality.canVectorize(Bndl);
226 Action *Action = ActionPtr.get();
227 IMaps->registerVector(Bndl, Action);
228 Actions.push_back(std::move(ActionPtr));
233 SmallPtrSet<Instruction *, 4> Claimed;
234 for (
const auto &NextUserBndl :
236 vectorizeRec(NextUserBndl, Bndl,
Depth + 1, Legality);
243 std::make_unique<Action>(&LegalityRes, Bndl, UserBndl,
Depth);
245 switch (LegalityRes.getSubclassID()) {
248 switch (
I->getOpcode()) {
249 case Instruction::Opcode::Load:
251 case Instruction::Opcode::Store: {
268 IMaps->registerVector(Bndl, ActionPtr.get());
278 ActionPtr->Operands = std::move(
Operands);
279 auto *Action = ActionPtr.get();
280 Actions.push_back(std::move(ActionPtr));
285void BundleVec::ActionsVector::print(raw_ostream &OS)
const {
286 for (
auto [Idx, Action] :
enumerate(Actions)) {
291void BundleVec::ActionsVector::dump()
const {
print(
dbgs()); }
298 WhereIt = std::next(VecI->getIterator());
305 "A widened Bndl should contain instrs!");
320 auto IsExternal = [
this](
const Use &
U) {
321 return !IMaps->isVectorized(
U.getUser());
324 if (
none_of(Elm->uses(), IsExternal))
327 Elm->replaceUsesWithIf(UnpackV, IsExternal);
331Value *BundleVec::emitVectors() {
332 Value *NewVec =
nullptr;
333 for (
const auto &ActionPtr : Actions) {
336 const LegalityResult &LegalityRes = *ActionPtr->LegalityRes;
337 unsigned Depth = ActionPtr->Depth;
338 auto *UserBB = !UserBndl.empty()
342 switch (LegalityRes.getSubclassID()) {
345 SmallVector<Value *, 2> VecOperands;
347 switch (
I->getOpcode()) {
348 case Instruction::Opcode::Load:
351 case Instruction::Opcode::Store:
352 VecOperands.push_back(ActionPtr->Operands[0]->Vec);
356 for (Action *OpA : ActionPtr->Operands)
357 VecOperands.push_back(OpA->Vec);
361 switch (
I->getOpcode()) {
362 case Instruction::Opcode::Load:
365 case Instruction::Opcode::Store: {
367 if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
368 VecOperands.push_back(OpA->Vec);
370 VecOperands.push_back(createPack(OpBndl, UserBB));
375 for (
unsigned OpIdx = 0; OpIdx <
I->getNumOperands(); ++OpIdx) {
377 if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
378 VecOperands.push_back(OpA->Vec);
380 VecOperands.push_back(createPack(OpBndl, UserBB));
385 NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
388 if (NewVec !=
nullptr)
389 DeadInstrMorgue.collectPotentiallyDeadInstrs(Bndl);
392 emitUnpacksForExternalUses(ActionPtr->Bndl, NewVec);
401 const ShuffleMask &
Mask =
403 NewVec = createShuffle(VecOp, Mask, UserBB);
404 assert(NewVec->getType() == VecOp->getType() &&
405 "Expected same type! Bad mask ?");
414 SmallVector<Value *, 4> DescrInstrs;
415 for (
const auto &ElmDescr : Descr.getDescrs()) {
416 auto *
V = ElmDescr.needsExtract() ? ElmDescr.getValue()->Vec
417 : ElmDescr.getScalar();
419 DescrInstrs.push_back(
I);
425 Context &Ctx = LastV->getContext();
427 for (
const auto &ElmDescr : Descr.getDescrs()) {
428 Value *VecOp =
nullptr;
429 Value *ValueToInsert;
430 if (ElmDescr.needsExtract()) {
431 VecOp = ElmDescr.getValue()->Vec;
435 VecOp, IdxC, WhereIt, VecOp->getContext(),
"VExt");
437 ValueToInsert = ElmDescr.getScalar();
440 if (NumLanesToInsert == 1) {
445 WhereIt, Ctx,
"VIns");
454 for (
unsigned LnCnt = 0; LnCnt != NumLanesToInsert; ++LnCnt) {
457 WhereIt, Ctx,
"VExt");
458 unsigned InsLane = Lane + LnCnt;
464 Lane += NumLanesToInsert;
473 NewVec = createPack(Bndl, UserBB);
477 if (NewVec !=
nullptr) {
479 ActionPtr->Vec = NewVec;
482 if (Opts->sbvec_always_verify) {
497 LegalityAnalysis &Legality) {
505 vectorizeRec(Bndl, {}, 0, Legality);
507 <<
"Vec: Vectorization Actions:\n";
510 DeadInstrMorgue.tryEraseDeadInstrs();
516 const auto &SeedSlice = Rgn.
getAux();
517 if (SeedSlice.size() < 2)
519 Function &
F = *SeedSlice[0]->getParent()->getParent();
520 IMaps = std::make_unique<InstrMaps>();
522 F.getParent()->getDataLayout(),
A.getTTI(),
523 F.getContext(), *IMaps, Dir);
530 return tryVectorize(SeedSliceVals, Legality);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define LLVM_UNLIKELY(EXPR)
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
InstListType::iterator iterator
Instruction iterators...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
LLVM Value Representation.
static LLVM_ABI Value * createWithCopiedFlags(Instruction::Opcode Op, Value *LHS, Value *RHS, Value *CopyFrom, InsertPosition Pos, Context &Ctx, const Twine &Name="")
An ArrayRef of Values or Instructions that we can print/dump for debugging.
bool runOnRegion(Region &Rgn, const Analyses &A) final
\Returns true if it modifies R.
static LLVM_ABI Value * create(Type *DestTy, Opcode Op, Value *Operand, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Value * create(Predicate Pred, Value *S1, Value *S2, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static LLVM_ABI ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static LLVM_ABI Value * create(Value *Vec, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Value * create(Value *Vec, Value *NewElt, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
LLVM_ABI BBIterator getIterator() const
\Returns a BasicBlock::iterator for this Instruction.
Performs the legality analysis and returns a LegalityResult object.
static LLVM_ABI LoadInst * create(Type *Ty, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx, const Twine &Name="")
virtual void print(raw_ostream &OS) const
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Context & getContext() const
const SmallVector< Instruction * > & getAux() const
\Returns the auxiliary vector.
static LLVM_ABI Value * create(Value *Cond, Value *True, Value *False, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Value * create(Value *V1, Value *V2, Value *Mask, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI StoreInst * create(Value *V, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx)
static LLVM_ABI IntegerType * getInt32Ty(Context &Ctx)
static LLVM_ABI Value * createWithCopiedFlags(Instruction::Opcode Op, Value *OpV, Value *CopyFrom, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static Type * getExpectedType(const Value *V)
\Returns the expected type of Value V.
static bool verifyFunction(const Function *F, raw_ostream &OS)
Equivalent to llvm::verifyFunction().
A SandboxIR Value has users. This is the base class.
static Type * getCommonScalarType(ArrayRef< Value * > Bndl)
Similar to tryGetCommonScalarType() but will assert that there is a common type.
static Instruction * getLastPHIOrSelf(Instruction *I)
If I is not a PHI it returns it.
static unsigned getNumLanes(Type *Ty)
\Returns the number of vector lanes of Ty or 1 if not a vector.
static Value * unpack(Value *FromVec, Type *ExtrTy, unsigned Lane, BasicBlock::iterator WhereIt)
Emits the necessary instruction sequence to extract element of type ExtrTy at Lane from FromVec.
static LLVM_DUMP_METHOD void dump(ArrayRef< Value * > Bndl)
Helper dump function for debugging.
static Type * getWideType(Type *ElemTy, unsigned NumElts)
\Returns <NumElts x ElemTy>.
static auto enumerateLanes(const ValueContainerT &Range)
Helper for creating LaneValueEnumerator ranges.
static Type * getElementType(Type *Ty)
Returns Ty if scalar or its element type if vector.
static Align getVecAlignment(AlignRangeT AlignRange, TypeRangeT TypeRange)
Given the alignments of the instructions before vectorization, this returns the alignment of the vect...
static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef< Value * > Vals, BasicBlock *BB)
\Returns the BB iterator after the lowest instruction in Vals (skipping instructions not in BB),...
static LLVM_ABI SmallVector< BundleTy > getNextUserBundles(ArrayRef< Value * > Bndl, const InstrMaps &IMaps, SmallPtrSet< Instruction *, 4 > &Claimed)
For each user of lane 0 in Bndl, try to form a bundle of matching users for all lanes.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
NodeAddr< UseNode * > Use
BndlRef(const T &OneElt) -> BndlRef< T >
LLVM_ABI Function * getParent() const
StringLiteral schedDirectionToStr(SchedDirection Dir)
@ DiamondReuseWithShuffle
BasicBlock(llvm::BasicBlock *BB, Context &SBCtx)
friend class Instruction
Iterator for Instructions in a `BasicBlock.
static BundleTy getOperand(BndlRef< Value * > Bndl, unsigned OpIdx)
SmallVector< Value *, 4 > BundleTy
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
constexpr unsigned long SBVecStopDisabled
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
const VectorizeOptions & getVectorizeOptions(const LLVMContext &C)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.