26#define DEBUG_PREFIX_LOCAL DEBUG_PREFIX "LoadStoreVec: "
39 const auto &LegalityRes = Legality->canVectorizeLSV(Bndl);
47void LoadStoreVec::saveIR(Region &R) {
50 CostBefore = SB.getAfterCost() - SB.getBeforeCost();
54bool LoadStoreVec::acceptOrRevert() {
59 <<
" (After=" << CostAfter <<
" Before=" << CostBefore
62 if (CostGain > Opts.sbvec_cost_threshold) {
89 Context &Ctx = Ty->getContext();
99SameSizeReinterpret::getCastChain(Type *SrcTy, Type *DestTy,
100 const DataLayout &
DL) {
102 "Cannot reinterpret a non-integral pointer!");
104 "Expected types of the same size!");
109 if (Ty->getScalarType()->isPointerTy()) {
110 Ty = getIntTypeFor(Ty,
DL);
111 Chain.emplace_back(Instruction::Opcode::PtrToInt, Ty);
113 bool DestIsPtr = DestTy->getScalarType()->isPointerTy();
114 Type *BitCastTy = DestIsPtr ? getIntTypeFor(DestTy,
DL) : DestTy;
116 Chain.emplace_back(Instruction::Opcode::BitCast, BitCastTy);
118 Chain.emplace_back(Instruction::Opcode::IntToPtr, DestTy);
121 assert(Chain.size() <= 3 &&
"Unexpected reinterpret cast chain!");
130 for (
auto [
Opc, Ty] : getCastChain(
C->getType(), DestTy,
DL)) {
132 case Instruction::Opcode::PtrToInt:
135 case Instruction::Opcode::BitCast:
138 case Instruction::Opcode::IntToPtr:
151 auto *LaneTy = VecTy->getScalarType();
157 "Chain contains a constant which cannot be converted or split");
167 for (
Value *Elm : AggrCOp->operands())
171 ConstantElements.push_back(SeqCOp->getElementAsConstant(ElmIdx));
173 auto *ZeroElm =
Zero->getSequentialElement();
174 for ([[maybe_unused]]
auto Cnt :
176 ConstantElements.push_back(ZeroElm);
179 for ([[maybe_unused]]
auto Cnt :
183 ConstantElements.push_back(Elm);
186 for ([[maybe_unused]]
auto Cnt :
190 ConstantElements.push_back(Elm);
194 for ([[maybe_unused]]
auto Cnt :
198 ConstantElements.push_back(Elm);
203 ConstantElements.push_back(COp);
214 LaneTyConstants.
reserve(ConstantElements.size());
215 for (Constant *
C : ConstantElements) {
217 if (Bits == LaneBits) {
221 LaneTyConstants.push_back(Lane);
222 }
else if (Bits > LaneBits) {
231 Val = CI->getValue();
233 Val = CFP->getValue().bitcastToAPInt();
236 unsigned NumSlices =
Bits / LaneBits;
238 unsigned EndianAwareSlice =
239 DL->isLittleEndian() ? SliceIdx : NumSlices - 1 - SliceIdx;
243 *Ctx, Val.extractBits(LaneBits, EndianAwareSlice * LaneBits));
246 if (SliceC ==
nullptr)
248 LaneTyConstants.push_back(SliceC);
258 if (!canVectorize(Stores))
262 for (
auto *
I : Stores) {
275 if (LI->getParent() != BB)
277 if (LI->hasNUsesOrMore(2))
283 if (!AllLoads && !AllConstants)
289 Value *VecOp =
nullptr;
290 SmallVector<Instruction *, 8> Loads;
295 if (!canVectorize(Loads)) {
299 VecOp = createVectorLoad(Loads);
300 assert(VecOp !=
nullptr &&
"Expected vector value!");
301 }
else if (AllConstants) {
306 VecOp = createConstantVector(Constants);
307 if (VecOp ==
nullptr) {
320 DeadInstrMorgue.collectPotentiallyDeadInstrs(Stores);
322 DeadInstrMorgue.collectPotentiallyDeadInstrs<
Instruction>(Loads);
323 DeadInstrMorgue.tryEraseDeadInstrs();
325 return acceptOrRevert();
330 auto VecTy = canVectorize(Loads);
343 auto *
VecLoad = createVectorLoad(Loads);
344 assert(VecLoad !=
nullptr &&
"Expected vector load");
349 if (OrigLoad->hasNUses(0))
353 OrigLoad->replaceAllUsesWith(Unpacked);
356 DeadInstrMorgue.collectPotentiallyDeadInstrs(Loads);
357 DeadInstrMorgue.tryEraseDeadInstrs();
359 if (!acceptOrRevert())
368 Function &
F = *Bndl[0]->getParent()->getParent();
369 DL = &
F.getParent()->getDataLayout();
370 Ctx = &
F.getContext();
373 auto Opc = Bndl[0]->getOpcode();
376 "Expected a homogeneous seed slice!");
380 case Instruction::Opcode::Load:
383 case Instruction::Opcode::Store:
390 Legality = std::make_unique<LegalityAnalysis>(
391 A->getAA(), A->getScalarEvolution(),
F.getParent()->getDataLayout(),
392 A->getTTI(),
F.getContext(), IMaps, Dir);
396 case Instruction::Opcode::Load:
397 Changed = vectorizeLoads(Bndl, Rgn) !=
nullptr;
399 case Instruction::Opcode::Store:
400 Changed = vectorizeStores(Bndl, Rgn);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static constexpr Value * getValue(Ty &ValueOrUse)
#define DEBUG_PREFIX_LOCAL
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
InstListType::iterator iterator
Instruction iterators...
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
void reserve(size_type N)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
LLVM Value Representation.
An ArrayRef of Values or Instructions that we can print/dump for debugging.
static LLVM_ABI Constant * getBitCast(Constant *C, Type *Ty)
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty)
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty)
static LLVM_ABI Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
static LLVM_ABI Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
void save()
Convenience function for getTracker().save()
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Maps the original instructions to the vectorized instrs and the reverse.
A sandboxir::User with operands, opcode and linked with previous/next instructions in an instruction ...
static LLVM_ABI IntegerType * get(Context &C, unsigned NumBits)
static LLVM_ABI LoadInst * create(Type *Ty, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx, const Twine &Name="")
bool runOnRegion(Region &Rgn, const Analyses &A) final
\Returns true if it modifies R.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Context & getContext() const
static LLVM_ABI Constant * reinterpretConstant(Constant *C, Type *DestTy, const DataLayout &DL)
Reinterprets constant C as DestTy, e.g., float 1.0 to i32 0x3f800000, or a null pointer to i64 0.
static LLVM_ABI bool canReinterpret(Type *SrcTy, Type *DestTy, const DataLayout &DL)
static LLVM_ABI StoreInst * create(Value *V, Value *Ptr, MaybeAlign Align, InsertPosition Pos, bool IsVolatile, Context &Ctx)
Just like llvm::Type these are immutable, unique, never get freed and can only be created via static ...
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
LLVM_ABI Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isPointerTy() const
True if this is an instance of PointerType.
static void propagateMetadata(Instruction *I, const ValueRangeT &VL)
static unsigned getNumBits(Type *Ty, const DataLayout &DL)
\Returns the number of bits of Ty.
A SandboxIR Value has users. This is the base class.
static Instruction * getLowest(ArrayRef< Instruction * > Instrs)
\Returns the instruction in Instrs that is lowest in the BB.
static Value * unpack(Value *FromVec, Type *ExtrTy, unsigned Lane, BasicBlock::iterator WhereIt)
Emits the necessary instruction sequence to extract element of type ExtrTy at Lane from FromVec.
static LLVM_DUMP_METHOD void dump(ArrayRef< Value * > Bndl)
Helper dump function for debugging.
static auto enumerateLanes(const ValueContainerT &Range)
Helper for creating LaneValueEnumerator ranges.
static Type * getElementType(Type *Ty)
Returns Ty if scalar or its element type if vector.
static Align getVecAlignment(AlignRangeT AlignRange, TypeRangeT TypeRange)
Given the alignments of the instructions before vectorization, this returns the alignment of the vect...
static Type * getCombinedVectorTypeFor(BndlRef< T * > Bndl, const DataLayout &DL)
\Returns the combined vector type for Bndl, even when the element types differ.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
BndlRef(const T &OneElt) -> BndlRef< T >
BasicBlock(llvm::BasicBlock *BB, Context &SBCtx)
friend class Instruction
Iterator for Instructions in a `BasicBlock.
static bool isNonIntegralPtr(Type *Ty, const DataLayout &DL)
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
DWARFExpression::Operation Op
const VectorizeOptions & getVectorizeOptions(const LLVMContext &C)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.