22static cl::opt<unsigned>
24 cl::desc(
"Override the vector register size in bits, "
25 "which is otherwise found by querying TTI."));
28 cl::desc(
"Allow non-power-of-2 vectorization."));
39 for (
Value *BndlV : Bndl) {
40 auto *BndlI = cast<Instruction>(BndlV);
41 Operands.push_back(BndlI->getOperand(OpIdx));
55 return std::next(BotI->getIterator());
62 assert(
all_of(Bndl, [](
auto *V) {
return isa<Instruction>(V); }) &&
63 "Expect Instructions!");
64 auto &Ctx = Bndl[0]->getContext();
70 Bndl, cast<Instruction>(Bndl[0])->
getParent());
72 auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
74 case Instruction::Opcode::ZExt:
75 case Instruction::Opcode::SExt:
76 case Instruction::Opcode::FPToUI:
77 case Instruction::Opcode::FPToSI:
78 case Instruction::Opcode::FPExt:
79 case Instruction::Opcode::PtrToInt:
80 case Instruction::Opcode::IntToPtr:
81 case Instruction::Opcode::SIToFP:
82 case Instruction::Opcode::UIToFP:
83 case Instruction::Opcode::Trunc:
84 case Instruction::Opcode::FPTrunc:
85 case Instruction::Opcode::BitCast: {
90 case Instruction::Opcode::FCmp:
91 case Instruction::Opcode::ICmp: {
92 auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
95 return cast<CmpInst>(SBV)->getPredicate() == Pred;
97 "Expected same predicate across bundle.");
101 case Instruction::Opcode::Select: {
105 case Instruction::Opcode::FNeg: {
106 auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
107 auto OpC = UOp0->getOpcode();
109 WhereIt, Ctx,
"Vec");
111 case Instruction::Opcode::Add:
112 case Instruction::Opcode::FAdd:
113 case Instruction::Opcode::Sub:
114 case Instruction::Opcode::FSub:
115 case Instruction::Opcode::Mul:
116 case Instruction::Opcode::FMul:
117 case Instruction::Opcode::UDiv:
118 case Instruction::Opcode::SDiv:
119 case Instruction::Opcode::FDiv:
120 case Instruction::Opcode::URem:
121 case Instruction::Opcode::SRem:
122 case Instruction::Opcode::FRem:
123 case Instruction::Opcode::Shl:
124 case Instruction::Opcode::LShr:
125 case Instruction::Opcode::AShr:
126 case Instruction::Opcode::And:
127 case Instruction::Opcode::Or:
128 case Instruction::Opcode::Xor: {
129 auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
133 BinOp0->getOpcode(),
LHS,
RHS, BinOp0, WhereIt, Ctx,
"Vec");
135 case Instruction::Opcode::Load: {
136 auto *Ld0 = cast<LoadInst>(Bndl[0]);
137 Value *
Ptr = Ld0->getPointerOperand();
141 case Instruction::Opcode::Store: {
142 auto Align = cast<StoreInst>(Bndl[0])->getAlign();
147 case Instruction::Opcode::Br:
148 case Instruction::Opcode::Ret:
149 case Instruction::Opcode::PHI:
150 case Instruction::Opcode::AddrSpaceCast:
151 case Instruction::Opcode::Call:
152 case Instruction::Opcode::GetElementPtr:
163 auto *VecI = CreateVectorInstr(Bndl,
Operands);
164 if (VecI !=
nullptr) {
166 IMaps->registerVector(Bndl, VecI);
171void BottomUpVec::tryEraseDeadInstrs() {
173 SmallVector<Instruction *> SortedDeadInstrCandidates(
174 DeadInstrCandidates.begin(), DeadInstrCandidates.end());
175 sort(SortedDeadInstrCandidates,
176 [](Instruction *I1, Instruction *I2) {
return I1->comesBefore(I2); });
177 for (Instruction *
I :
reverse(SortedDeadInstrCandidates)) {
179 I->eraseFromParent();
181 DeadInstrCandidates.clear();
184Value *BottomUpVec::createShuffle(Value *VecOp,
const ShuffleMask &Mask,
185 BasicBlock *UserBB) {
188 VecOp->getContext(),
"VShuf");
191Value *BottomUpVec::createPack(ArrayRef<Value *> ToPack, BasicBlock *UserBB) {
201 Context &Ctx = ToPack[0]->getContext();
203 unsigned InsertIdx = 0;
204 for (Value *Elm : ToPack) {
207 if (Elm->getType()->isVectorTy()) {
209 cast<FixedVectorType>(Elm->getType())->getNumElements();
210 for (
auto ExtrLane : seq<int>(0, NumElms)) {
217 if (!isa<Constant>(ExtrI))
218 WhereIt = std::next(cast<Instruction>(ExtrI)->getIterator());
223 LastInsert, ExtrI, InsertLaneC, WhereIt, Ctx,
"VPack");
224 if (!isa<Constant>(InsertI)) {
225 LastInsert = InsertI;
226 WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
235 WhereIt, Ctx,
"Pack");
236 if (
auto *NewI = dyn_cast<Instruction>(LastInsert))
237 WhereIt = std::next(NewI->getIterator());
243void BottomUpVec::collectPotentiallyDeadInstrs(ArrayRef<Value *> Bndl) {
244 for (Value *V : Bndl)
245 DeadInstrCandidates.insert(cast<Instruction>(V));
247 auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
249 case Instruction::Opcode::Load: {
253 DeadInstrCandidates.insert(
Ptr);
256 case Instruction::Opcode::Store: {
260 DeadInstrCandidates.insert(
Ptr);
268Value *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
269 ArrayRef<Value *> UserBndl,
unsigned Depth) {
270 Value *NewVec =
nullptr;
271 auto *UserBB = !UserBndl.empty()
272 ? cast<Instruction>(UserBndl.front())->getParent()
273 : cast<Instruction>(Bndl[0])->getParent();
274 const auto &LegalityRes = Legality->canVectorize(Bndl);
275 switch (LegalityRes.getSubclassID()) {
277 auto *
I = cast<Instruction>(Bndl[0]);
278 SmallVector<Value *, 2> VecOperands;
279 switch (
I->getOpcode()) {
280 case Instruction::Opcode::Load:
284 case Instruction::Opcode::Store: {
287 VecOperands.push_back(VecOp);
293 for (
auto OpIdx : seq<unsigned>(
I->getNumOperands())) {
295 VecOperands.push_back(VecOp);
299 NewVec = createVectorInstr(Bndl, VecOperands);
303 if (NewVec !=
nullptr)
304 collectPotentiallyDeadInstrs(Bndl);
308 NewVec = cast<DiamondReuse>(LegalityRes).getVector();
312 auto *VecOp = cast<DiamondReuseWithShuffle>(LegalityRes).getVector();
313 const ShuffleMask &
Mask =
314 cast<DiamondReuseWithShuffle>(LegalityRes).getMask();
315 NewVec = createShuffle(VecOp, Mask, UserBB);
320 cast<DiamondReuseMultiInput>(LegalityRes).getCollectDescr();
324 SmallVector<Value *, 4> DescrInstrs;
325 for (
const auto &ElmDescr : Descr.getDescrs()) {
326 if (
auto *
I = dyn_cast<Instruction>(ElmDescr.getValue()))
327 DescrInstrs.push_back(
I);
333 for (
auto [Lane, ElmDescr] :
enumerate(Descr.getDescrs())) {
334 Value *VecOp = ElmDescr.getValue();
335 Context &Ctx = VecOp->getContext();
336 Value *ValueToInsert;
337 if (ElmDescr.needsExtract()) {
341 VecOp->getContext(),
"VExt");
343 ValueToInsert = VecOp;
347 WhereIt, Ctx,
"VIns");
357 NewVec = createPack(Bndl, UserBB);
364bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) {
365 DeadInstrCandidates.clear();
367 vectorizeRec(Bndl, {}, 0);
368 tryEraseDeadInstrs();
373 IMaps = std::make_unique<InstrMaps>(
F.getContext());
374 Legality = std::make_unique<LegalityAnalysis>(
375 A.getAA(),
A.getScalarEvolution(),
F.getParent()->getDataLayout(),
376 F.getContext(), *IMaps);
378 const auto &
DL =
F.getParent()->getDataLayout();
379 unsigned VecRegBits =
389 for (
SeedBundle &Seeds : SC.getStoreSeeds()) {
392 Seeds[Seeds.getFirstUnusedElementIdx()])),
395 auto DivideBy2 = [](
unsigned Num) {
403 for (
unsigned SliceElms = std::min(VecRegBits / ElmBits,
404 Seeds.getNumUnusedBits() / ElmBits);
405 SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
410 for (
unsigned Offset = Seeds.getFirstUnusedElementIdx(),
421 if (SeedSlice.empty())
424 assert(SeedSlice.size() >= 2 &&
"Should have been rejected!");
432 Change |= tryVectorize(SeedSliceVals);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
LLVM Value Representation.
Iterator for Instructions in a `BasicBlock.
Contains a list of sandboxir::Instruction's.
static Value * createWithCopiedFlags(Instruction::Opcode Op, Value *LHS, Value *RHS, Value *CopyFrom, InsertPosition Pos, Context &Ctx, const Twine &Name="")
BottomUpVec(StringRef Pipeline)
bool runOnFunction(Function &F, const Analyses &A) final
\Returns true if it modifies F.
static Value * create(Type *DestTy, Opcode Op, Value *Operand, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static Value * create(Predicate Pred, Value *S1, Value *S2, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static ConstantInt * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Value * create(Value *Vec, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A pass that runs on a sandbox::Function.
static Value * create(Value *Vec, Value *NewElt, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LoadInst * create(Type *Ty, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx, const Twine &Name="")
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of candidate Instructions for vectorizing together.
static Value * create(Value *Cond, Value *True, Value *False, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static Value * create(Value *V1, Value *V2, Value *Mask, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static StoreInst * create(Value *V, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx)
static Type * getInt32Ty(Context &Ctx)
static Value * createWithCopiedFlags(Instruction::Opcode Op, Value *OpV, Value *CopyFrom, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static unsigned getNumBits(Type *Ty, const DataLayout &DL)
\Returns the number of bits of Ty.
static Type * getExpectedType(const Value *V)
\Returns the expected type of Value V.
A SandboxIR Value has users. This is the base class.
static Instruction * getLowest(ArrayRef< Instruction * > Instrs)
\Returns the instruction in Instrs that is lowest in the BB.
static Type * getCommonScalarType(ArrayRef< Value * > Bndl)
Similar to tryGetCommonScalarType() but will assert that there is a common type.
static unsigned getNumLanes(Type *Ty)
\Returns the number of vector lanes of Ty or 1 if not a vector.
static Type * getWideType(Type *ElemTy, unsigned NumElts)
\Returns <NumElts x ElemTy>.
static Type * getElementType(Type *Ty)
Returns Ty if scalar or its element type if vector.
static unsigned getFloorPowerOf2(unsigned Num)
\Returns the first integer power of 2 that is <= Num.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
initializer< Ty > init(const Ty &Val)
Type
MessagePack types as defined in the standard, with the exception of Integer being divided into a sign...
@ DiamondReuse
Vectorize by combining scalars to a vector.
@ DiamondReuseWithShuffle
Don't generate new code, reuse existing vector.
@ Widen
Collect scalar values.
@ DiamondReuseMultiInput
Reuse the existing vector but add a shuffle.
static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef< Value * > Vals, BasicBlock *BB)
\Returns the BB iterator after the lowest instruction in Vals, or the top of BB if no instruction fou...
static SmallVector< Value *, 4 > getOperand(ArrayRef< Value * > Bndl, unsigned OpIdx)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
static cl::opt< unsigned > OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden, cl::desc("Override the vector register size in bits, " "which is otherwise found by querying TTI."))
static cl::opt< bool > AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden, cl::desc("Allow non-power-of-2 vectorization."))