LLVM 24.0.0git
BundleVec.cpp
Go to the documentation of this file.
1//===- BundleVec.cpp - A bundle-forming SLP-style vectorizer pass ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
20
21namespace llvm {
22
23namespace sandboxir {
24
25static BundleTy getOperand(BndlRef<Value *> Bndl, unsigned OpIdx) {
27 for (Value *BndlV : Bndl) {
28 auto *BndlI = cast<Instruction>(BndlV);
29 Operands.push_back(BndlI->getOperand(OpIdx));
30 }
31 return Operands;
32}
33
34Value *BundleVec::createVectorInstr(BndlRef<Value *> Bndl,
36 auto CreateVectorInstr = [](BndlRef<Value *> Bndl,
38 assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
39 "Expect Instructions!");
40 auto &Ctx = Bndl[0]->getContext();
41
42 Type *ScalarTy = VecUtils::getElementType(Utils::getExpectedType(Bndl[0]));
43 auto *VecTy = VecUtils::getWideType(ScalarTy, VecUtils::getNumLanes(Bndl));
44
46 Bndl, cast<Instruction>(Bndl[0])->getParent());
47
48 auto Opcode = cast<Instruction>(Bndl[0])->getOpcode();
49 switch (Opcode) {
50 case Instruction::Opcode::ZExt:
51 case Instruction::Opcode::SExt:
52 case Instruction::Opcode::FPToUI:
53 case Instruction::Opcode::FPToSI:
54 case Instruction::Opcode::FPExt:
55 case Instruction::Opcode::PtrToAddr:
56 case Instruction::Opcode::PtrToInt:
57 case Instruction::Opcode::IntToPtr:
58 case Instruction::Opcode::SIToFP:
59 case Instruction::Opcode::UIToFP:
60 case Instruction::Opcode::Trunc:
61 case Instruction::Opcode::FPTrunc:
62 case Instruction::Opcode::BitCast: {
63 assert(Operands.size() == 1u && "Casts are unary!");
64 return CastInst::create(VecTy, Opcode, Operands[0], WhereIt, Ctx,
65 "VCast");
66 }
67 case Instruction::Opcode::FCmp:
68 case Instruction::Opcode::ICmp: {
69 auto Pred = cast<CmpInst>(Bndl[0])->getPredicate();
71 [Pred](auto *SBV) {
72 return cast<CmpInst>(SBV)->getPredicate() == Pred;
73 }) &&
74 "Expected same predicate across bundle.");
75 return CmpInst::create(Pred, Operands[0], Operands[1], WhereIt, Ctx,
76 "VCmp");
77 }
78 case Instruction::Opcode::Select: {
79 return SelectInst::create(Operands[0], Operands[1], Operands[2], WhereIt,
80 Ctx, "Vec");
81 }
82 case Instruction::Opcode::FNeg: {
83 auto *UOp0 = cast<UnaryOperator>(Bndl[0]);
84 auto OpC = UOp0->getOpcode();
86 WhereIt, Ctx, "Vec");
87 }
88 case Instruction::Opcode::Add:
89 case Instruction::Opcode::FAdd:
90 case Instruction::Opcode::Sub:
91 case Instruction::Opcode::FSub:
92 case Instruction::Opcode::Mul:
93 case Instruction::Opcode::FMul:
94 case Instruction::Opcode::UDiv:
95 case Instruction::Opcode::SDiv:
96 case Instruction::Opcode::FDiv:
97 case Instruction::Opcode::URem:
98 case Instruction::Opcode::SRem:
99 case Instruction::Opcode::FRem:
100 case Instruction::Opcode::Shl:
101 case Instruction::Opcode::LShr:
102 case Instruction::Opcode::AShr:
103 case Instruction::Opcode::And:
104 case Instruction::Opcode::Or:
105 case Instruction::Opcode::Xor: {
106 auto *BinOp0 = cast<BinaryOperator>(Bndl[0]);
107 auto *LHS = Operands[0];
108 auto *RHS = Operands[1];
110 BinOp0->getOpcode(), LHS, RHS, BinOp0, WhereIt, Ctx, "Vec");
111 }
112 case Instruction::Opcode::Load: {
113 auto Align = VecUtils::getVecAlignment(Bndl);
114 auto *Ld0 = cast<LoadInst>(Bndl[0]);
115 Value *Ptr = Ld0->getPointerOperand();
116 return LoadInst::create(VecTy, Ptr, Align, WhereIt, Ctx, "VecL");
117 }
118 case Instruction::Opcode::Store: {
119 auto Align = VecUtils::getVecAlignment(Bndl);
120 Value *Val = Operands[0];
121 Value *Ptr = Operands[1];
122 return StoreInst::create(Val, Ptr, Align, WhereIt, Ctx);
123 }
124 case Instruction::Opcode::UncondBr:
125 case Instruction::Opcode::CondBr:
126 case Instruction::Opcode::Ret:
127 case Instruction::Opcode::PHI:
128 case Instruction::Opcode::AddrSpaceCast:
129 case Instruction::Opcode::Call:
130 case Instruction::Opcode::GetElementPtr:
131 llvm_unreachable("Unimplemented");
132 break;
133 default:
134 llvm_unreachable("Unimplemented");
135 break;
136 }
137 llvm_unreachable("Missing switch case!");
138 // TODO: Propagate debug info.
139 };
140
141 auto *NewI = CreateVectorInstr(Bndl, Operands);
142 LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewI << "\n");
143 return NewI;
144}
145
146Value *BundleVec::createShuffle(Value *VecOp, const ShuffleMask &Mask,
147 BasicBlock *UserBB) {
148 BasicBlock::iterator WhereIt =
150 return ShuffleVectorInst::create(VecOp, VecOp, Mask, WhereIt,
151 VecOp->getContext(), "VShuf");
152}
153
154Value *BundleVec::createPack(BndlRef<Value *> ToPack, BasicBlock *UserBB) {
155 BasicBlock::iterator WhereIt =
157
158 Type *ScalarTy = VecUtils::getCommonScalarType(ToPack);
159 unsigned Lanes = VecUtils::getNumLanes(ToPack);
160 Type *VecTy = VecUtils::getWideType(ScalarTy, Lanes);
161
162 // Create a series of pack instructions.
163 Value *LastInsert = PoisonValue::get(VecTy);
164
165 Context &Ctx = ToPack[0]->getContext();
166
167 unsigned InsertIdx = 0;
168 for (Value *Elm : ToPack) {
169 // An element can be either scalar or vector. We need to generate different
170 // IR for each case.
171 if (Elm->getType()->isVectorTy()) {
172 unsigned NumElms =
173 cast<FixedVectorType>(Elm->getType())->getNumElements();
174 for (auto ExtrLane : seq<int>(0, NumElms)) {
175 // We generate extract-insert pairs, for each lane in `Elm`.
176 Constant *ExtrLaneC =
178 // This may return a Constant if Elm is a Constant.
179 auto *ExtrI =
180 ExtractElementInst::create(Elm, ExtrLaneC, WhereIt, Ctx, "VPack");
181 if (!isa<Constant>(ExtrI))
182 WhereIt = std::next(cast<Instruction>(ExtrI)->getIterator());
183 Constant *InsertLaneC =
184 ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
185 // This may also return a Constant if ExtrI is a Constant.
186 auto *InsertI = InsertElementInst::create(
187 LastInsert, ExtrI, InsertLaneC, WhereIt, Ctx, "VPack");
188 LastInsert = InsertI;
189 if (!isa<Constant>(InsertI))
190 WhereIt = std::next(cast<Instruction>(LastInsert)->getIterator());
191 }
192 } else {
193 Constant *InsertLaneC =
194 ConstantInt::getSigned(Type::getInt32Ty(Ctx), InsertIdx++);
195 // This may be folded into a Constant if LastInsert is a Constant. In
196 // that case we only collect the last constant.
197 LastInsert = InsertElementInst::create(LastInsert, Elm, InsertLaneC,
198 WhereIt, Ctx, "Pack");
199 if (auto *NewI = dyn_cast<Instruction>(LastInsert))
200 WhereIt = std::next(NewI->getIterator());
201 }
202 }
203 return LastInsert;
204}
205
206Action *BundleVec::vectorizeRec(BndlRef<Value *> Bndl,
207 BndlRef<Value *> UserBndl, unsigned Depth,
208 LegalityAnalysis &Legality) {
209 bool StopForDebug = DebugBndlCnt++ >= Opts->sbvec_stop_bndl &&
210 Opts->sbvec_stop_bndl != SBVecStopDisabled;
211 LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
212 VecUtils::dump(Bndl));
213 const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
214 : Legality.canVectorize(Bndl);
215 LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
216
217 if (Dir == SchedDirection::TopDown) {
218 // A non-Widen result means we can't extend the vectorized region into
219 // this bundle, so leave its instructions scalar and don't record an
220 // action for it.
221 if (LegalityRes.getSubclassID() != LegalityResultID::Widen)
222 return nullptr;
223
224 auto ActionPtr =
225 std::make_unique<Action>(&LegalityRes, Bndl, BndlRef<Value *>(), Depth);
226 Action *Action = ActionPtr.get();
227 IMaps->registerVector(Bndl, Action);
228 Actions.push_back(std::move(ActionPtr));
229
230 // Walk down the def-use chain. Each lane in \p Bndl may feed several
231 // users, so we form every compatible user bundle and recurse into each
232 // one.
233 SmallPtrSet<Instruction *, 4> Claimed;
234 for (const auto &NextUserBndl :
235 VecUtils::getNextUserBundles(Bndl, *IMaps, Claimed))
236 vectorizeRec(NextUserBndl, Bndl, Depth + 1, Legality);
237
238 return Action;
239 }
240
241 // Bottom up direction
242 auto ActionPtr =
243 std::make_unique<Action>(&LegalityRes, Bndl, UserBndl, Depth);
245 switch (LegalityRes.getSubclassID()) {
247 auto *I = cast<Instruction>(Bndl[0]);
248 switch (I->getOpcode()) {
249 case Instruction::Opcode::Load:
250 break;
251 case Instruction::Opcode::Store: {
252 // Don't recurse towards the pointer operand.
253 Action *OpA =
254 vectorizeRec(getOperand(Bndl, 0), Bndl, Depth + 1, Legality);
255 Operands.push_back(OpA);
256 break;
257 }
258 default:
259 // Visit all operands.
260 for (auto OpIdx : seq<unsigned>(I->getNumOperands())) {
261 Action *OpA =
262 vectorizeRec(getOperand(Bndl, OpIdx), Bndl, Depth + 1, Legality);
263 Operands.push_back(OpA);
264 }
265 break;
266 }
267 // Update the maps to mark Bndl as "vectorized".
268 IMaps->registerVector(Bndl, ActionPtr.get());
269 break;
270 }
275 break;
276 }
277 // Create actions in post-order.
278 ActionPtr->Operands = std::move(Operands);
279 auto *Action = ActionPtr.get();
280 Actions.push_back(std::move(ActionPtr));
281 return Action;
282}
283
284#ifndef NDEBUG
285void BundleVec::ActionsVector::print(raw_ostream &OS) const {
286 for (auto [Idx, Action] : enumerate(Actions)) {
287 Action->print(OS);
288 OS << "\n";
289 }
290}
291void BundleVec::ActionsVector::dump() const { print(dbgs()); }
292#endif // NDEBUG
293
294void BundleVec::emitUnpacksForExternalUses(BndlRef<Value *> Bndl, Value *Vec) {
295 // Find where we should emit the unpacks.
296 BasicBlock::iterator WhereIt;
297 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
298 WhereIt = std::next(VecI->getIterator());
299 } else {
300 // If Vec is a constant then it should be safe to emit the unpacks at the
301 // top of the block.
302 // Note: Extracts from constants are usually folded to constants.
303 assert(isa<Constant>(Vec) && "Expected constant!");
304 assert(isa<Instruction>(Bndl[0]) &&
305 "A widened Bndl should contain instrs!");
306 BasicBlock *BB = cast<Instruction>(Bndl[0])->getParent();
307 WhereIt =
308 BB->empty()
309 ? BB->begin()
310 : std::next(
311 VecUtils::getLastPHIOrSelf(&*BB->begin())->getIterator());
312 }
313
314 for (auto [Lane, Elm] : VecUtils::enumerateLanes(Bndl)) {
315 // Only redirect the external (non-vectorized) uses to an unpack and leave
316 // the vectorized users untouched. A blanket replaceAllUsesWith() would
317 // also rewrite the operands of users we are going to vectorize but have
318 // not emitted yet (in the top-down direction a user bundle is emitted
319 // after its operand bundle), which would corrupt those operands.
320 auto IsExternal = [this](const Use &U) {
321 return !IMaps->isVectorized(U.getUser());
322 };
323 // Don't emit a dead unpack if all uses are internal to the vector region.
324 if (none_of(Elm->uses(), IsExternal))
325 continue;
326 auto *UnpackV = VecUtils::unpack(Vec, Elm->getType(), Lane, WhereIt);
327 Elm->replaceUsesWithIf(UnpackV, IsExternal);
328 }
329}
330
331Value *BundleVec::emitVectors() {
332 Value *NewVec = nullptr;
333 for (const auto &ActionPtr : Actions) {
334 BndlRef<Value *> Bndl = ActionPtr->Bndl;
335 BndlRef<Value *> UserBndl = ActionPtr->UserBndl;
336 const LegalityResult &LegalityRes = *ActionPtr->LegalityRes;
337 unsigned Depth = ActionPtr->Depth;
338 auto *UserBB = !UserBndl.empty()
339 ? cast<Instruction>(UserBndl.front())->getParent()
340 : cast<Instruction>(Bndl[0])->getParent();
341
342 switch (LegalityRes.getSubclassID()) {
344 auto *I = cast<Instruction>(Bndl[0]);
345 SmallVector<Value *, 2> VecOperands;
346 if (Dir == SchedDirection::BottomUp) {
347 switch (I->getOpcode()) {
348 case Instruction::Opcode::Load:
349 VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
350 break;
351 case Instruction::Opcode::Store:
352 VecOperands.push_back(ActionPtr->Operands[0]->Vec);
353 VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
354 break;
355 default:
356 for (Action *OpA : ActionPtr->Operands)
357 VecOperands.push_back(OpA->Vec);
358 break;
359 }
360 } else {
361 switch (I->getOpcode()) {
362 case Instruction::Opcode::Load:
363 VecOperands.push_back(cast<LoadInst>(I)->getPointerOperand());
364 break;
365 case Instruction::Opcode::Store: {
366 auto OpBndl = getOperand(Bndl, 0);
367 if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
368 VecOperands.push_back(OpA->Vec);
369 else
370 VecOperands.push_back(createPack(OpBndl, UserBB));
371 VecOperands.push_back(cast<StoreInst>(I)->getPointerOperand());
372 break;
373 }
374 default:
375 for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) {
376 BundleTy OpBndl = getOperand(Bndl, OpIdx);
377 if (Action *OpA = IMaps->getVectorForOrig(OpBndl[0]))
378 VecOperands.push_back(OpA->Vec);
379 else
380 VecOperands.push_back(createPack(OpBndl, UserBB));
381 }
382 break;
383 }
384 }
385 NewVec = createVectorInstr(ActionPtr->Bndl, VecOperands);
386 // Collect any potentially dead scalar instructions, including the
387 // original scalars and pointer operands of loads/stores.
388 if (NewVec != nullptr)
389 DeadInstrMorgue.collectPotentiallyDeadInstrs(Bndl);
390
391 // Emit unpacks for all external uses, if any.
392 emitUnpacksForExternalUses(ActionPtr->Bndl, NewVec);
393 break;
394 }
396 NewVec = cast<DiamondReuse>(LegalityRes).getVector()->Vec;
397 break;
398 }
400 auto *VecOp = cast<DiamondReuseWithShuffle>(LegalityRes).getVector()->Vec;
401 const ShuffleMask &Mask =
402 cast<DiamondReuseWithShuffle>(LegalityRes).getMask();
403 NewVec = createShuffle(VecOp, Mask, UserBB);
404 assert(NewVec->getType() == VecOp->getType() &&
405 "Expected same type! Bad mask ?");
406 break;
407 }
409 const auto &Descr =
410 cast<DiamondReuseMultiInput>(LegalityRes).getCollectDescr();
411 Type *ResTy = VecUtils::getWideType(Bndl[0]->getType(), Bndl.size());
412
413 // TODO: Try to get WhereIt without creating a vector.
414 SmallVector<Value *, 4> DescrInstrs;
415 for (const auto &ElmDescr : Descr.getDescrs()) {
416 auto *V = ElmDescr.needsExtract() ? ElmDescr.getValue()->Vec
417 : ElmDescr.getScalar();
418 if (auto *I = dyn_cast<Instruction>(V))
419 DescrInstrs.push_back(I);
420 }
421 BasicBlock::iterator WhereIt =
422 VecUtils::getInsertPointAfterInstrs(DescrInstrs, UserBB);
423
424 Value *LastV = PoisonValue::get(ResTy);
425 Context &Ctx = LastV->getContext();
426 unsigned Lane = 0;
427 for (const auto &ElmDescr : Descr.getDescrs()) {
428 Value *VecOp = nullptr;
429 Value *ValueToInsert;
430 if (ElmDescr.needsExtract()) {
431 VecOp = ElmDescr.getValue()->Vec;
432 ConstantInt *IdxC =
433 ConstantInt::get(Type::getInt32Ty(Ctx), ElmDescr.getExtractIdx());
434 ValueToInsert = ExtractElementInst::create(
435 VecOp, IdxC, WhereIt, VecOp->getContext(), "VExt");
436 } else {
437 ValueToInsert = ElmDescr.getScalar();
438 }
439 auto NumLanesToInsert = VecUtils::getNumLanes(ValueToInsert);
440 if (NumLanesToInsert == 1) {
441 // If we are inserting a scalar element then we need a single insert.
442 // %VIns = insert %DstVec, %SrcScalar, Lane
443 ConstantInt *LaneC = ConstantInt::get(Type::getInt32Ty(Ctx), Lane);
444 LastV = InsertElementInst::create(LastV, ValueToInsert, LaneC,
445 WhereIt, Ctx, "VIns");
446 } else {
447 // If we are inserting a vector element then we need to extract and
448 // insert each vector element one by one with a chain of extracts and
449 // inserts, for example:
450 // %VExt0 = extract %SrcVec, 0
451 // %VIns0 = insert %DstVec, %Vect0, Lane + 0
452 // %VExt1 = extract %SrcVec, 1
453 // %VIns1 = insert %VIns0, %Vect0, Lane + 1
454 for (unsigned LnCnt = 0; LnCnt != NumLanesToInsert; ++LnCnt) {
455 auto *ExtrIdxC = ConstantInt::get(Type::getInt32Ty(Ctx), LnCnt);
456 auto *ExtrI = ExtractElementInst::create(ValueToInsert, ExtrIdxC,
457 WhereIt, Ctx, "VExt");
458 unsigned InsLane = Lane + LnCnt;
459 auto *InsLaneC = ConstantInt::get(Type::getInt32Ty(Ctx), InsLane);
460 LastV = InsertElementInst::create(LastV, ExtrI, InsLaneC, WhereIt,
461 Ctx, "VIns");
462 }
463 }
464 Lane += NumLanesToInsert;
465 }
466 NewVec = LastV;
467 break;
468 }
470 // If we can't vectorize the seeds then just return.
471 if (Depth == 0)
472 return nullptr;
473 NewVec = createPack(Bndl, UserBB);
474 break;
475 }
476 }
477 if (NewVec != nullptr) {
478 Change = true;
479 ActionPtr->Vec = NewVec;
480 }
481#ifndef NDEBUG
482 if (Opts->sbvec_always_verify) {
483 // This helps find broken IR by constantly verifying the function. Note
484 // that this is very expensive and should only be used for debugging.
485 Instruction *I0 = isa<Instruction>(Bndl[0])
486 ? cast<Instruction>(Bndl[0])
487 : cast<Instruction>(UserBndl[0]);
488 assert(!Utils::verifyFunction(I0->getParent()->getParent(), dbgs()) &&
489 "Broken function!");
490 }
491#endif // NDEBUG
492 }
493 return NewVec;
494}
495
496bool BundleVec::tryVectorize(BndlRef<Value *> Bndl,
497 LegalityAnalysis &Legality) {
498 Change = false;
499 if (LLVM_UNLIKELY(InvocationCnt++ >= Opts->sbvec_stop_at &&
500 Opts->sbvec_stop_at != SBVecStopDisabled))
501 return false;
502 Legality.clear();
503 Actions.clear();
504 DebugBndlCnt = 0;
505 vectorizeRec(Bndl, {}, /*Depth=*/0, Legality);
507 << "Vec: Vectorization Actions:\n";
508 Actions.dump());
509 emitVectors();
510 DeadInstrMorgue.tryEraseDeadInstrs();
511 return Change;
512}
513
515 Opts = &getVectorizeOptions(Rgn.getContext());
516 const auto &SeedSlice = Rgn.getAux();
517 if (SeedSlice.size() < 2)
518 return false;
519 Function &F = *SeedSlice[0]->getParent()->getParent();
520 IMaps = std::make_unique<InstrMaps>();
521 LegalityAnalysis Legality(A.getAA(), A.getScalarEvolution(),
522 F.getParent()->getDataLayout(), A.getTTI(),
523 F.getContext(), *IMaps, Dir);
524
525 // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
526 SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
527 // Try to vectorize starting from the seed slice. The returned value
528 // is true if we found vectorizable code and generated some vector
529 // code for it. It does not mean that the code is profitable.
530 return tryVectorize(SeedSliceVals, Legality);
531}
532
533} // namespace sandboxir
534} // namespace llvm
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define LLVM_UNLIKELY(EXPR)
Definition Compiler.h:348
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
SI Fold Operands
This file defines the SmallVector class.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
#define DEBUG_PREFIX
Definition Debug.h:19
Value * RHS
Value * LHS
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
LLVM Value Representation.
Definition Value.h:75
static LLVM_ABI Value * createWithCopiedFlags(Instruction::Opcode Op, Value *LHS, Value *RHS, Value *CopyFrom, InsertPosition Pos, Context &Ctx, const Twine &Name="")
An ArrayRef of Values or Instructions that we can print/dump for debugging.
Definition VecUtils.h:39
bool runOnRegion(Region &Rgn, const Analyses &A) final
\Returns true if it modifies R.
static LLVM_ABI Value * create(Type *DestTy, Opcode Op, Value *Operand, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Value * create(Predicate Pred, Value *S1, Value *S2, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition Constant.cpp:48
static LLVM_ABI ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition Constant.cpp:56
static LLVM_ABI Value * create(Value *Vec, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Value * create(Value *Vec, Value *NewElt, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
LLVM_ABI BBIterator getIterator() const
\Returns a BasicBlock::iterator for this Instruction.
Performs the legality analysis and returns a LegalityResult object.
Definition Legality.h:322
static LLVM_ABI LoadInst * create(Type *Ty, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx, const Twine &Name="")
virtual void print(raw_ostream &OS) const
Definition Pass.h:68
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constant.cpp:263
Context & getContext() const
Definition Region.h:169
const SmallVector< Instruction * > & getAux() const
\Returns the auxiliary vector.
Definition Region.h:177
static LLVM_ABI Value * create(Value *Cond, Value *True, Value *False, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI Value * create(Value *V1, Value *V2, Value *Mask, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI StoreInst * create(Value *V, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx)
static LLVM_ABI IntegerType * getInt32Ty(Context &Ctx)
Definition Type.cpp:21
static LLVM_ABI Value * createWithCopiedFlags(Instruction::Opcode Op, Value *OpV, Value *CopyFrom, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static Type * getExpectedType(const Value *V)
\Returns the expected type of Value V.
Definition Utils.h:34
static bool verifyFunction(const Function *F, raw_ostream &OS)
Equivalent to llvm::verifyFunction().
Definition Utils.h:133
A SandboxIR Value has users. This is the base class.
Definition Value.h:72
static Type * getCommonScalarType(ArrayRef< Value * > Bndl)
Similar to tryGetCommonScalarType() but will assert that there is a common type.
Definition VecUtils.h:331
static Instruction * getLastPHIOrSelf(Instruction *I)
If I is not a PHI it returns it.
Definition VecUtils.h:290
static unsigned getNumLanes(Type *Ty)
\Returns the number of vector lanes of Ty or 1 if not a vector.
Definition VecUtils.h:134
static Value * unpack(Value *FromVec, Type *ExtrTy, unsigned Lane, BasicBlock::iterator WhereIt)
Emits the necessary instruction sequence to extract element of type ExtrTy at Lane from FromVec.
Definition VecUtils.h:418
static LLVM_DUMP_METHOD void dump(ArrayRef< Value * > Bndl)
Helper dump function for debugging.
Definition VecUtils.cpp:166
static Type * getWideType(Type *ElemTy, unsigned NumElts)
\Returns <NumElts x ElemTy>.
Definition VecUtils.h:157
static auto enumerateLanes(const ValueContainerT &Range)
Helper for creating LaneValueEnumerator ranges.
Definition VecUtils.h:536
static Type * getElementType(Type *Ty)
Returns Ty if scalar or its element type if vector.
Definition VecUtils.h:94
static Align getVecAlignment(AlignRangeT AlignRange, TypeRangeT TypeRange)
Given the alignments of the instructions before vectorization, this returns the alignment of the vect...
Definition VecUtils.h:547
static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef< Value * > Vals, BasicBlock *BB)
\Returns the BB iterator after the lowest instruction in Vals (skipping instructions not in BB),...
Definition VecUtils.h:302
static LLVM_ABI SmallVector< BundleTy > getNextUserBundles(ArrayRef< Value * > Bndl, const InstrMaps &IMaps, SmallPtrSet< Instruction *, 4 > &Claimed)
For each user of lane 0 in Bndl, try to form a bundle of matching users for all lanes.
Definition VecUtils.cpp:64
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
BndlRef(const T &OneElt) -> BndlRef< T >
LLVM_ABI Function * getParent() const
StringLiteral schedDirectionToStr(SchedDirection Dir)
BasicBlock(llvm::BasicBlock *BB, Context &SBCtx)
Definition BasicBlock.h:75
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
static BundleTy getOperand(BndlRef< Value * > Bndl, unsigned OpIdx)
Definition BundleVec.cpp:25
SmallVector< Value *, 4 > BundleTy
Definition VecUtils.h:82
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2575
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
constexpr unsigned long SBVecStopDisabled
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
const VectorizeOptions & getVectorizeOptions(const LLVMContext &C)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341