35#include "llvm/IR/IntrinsicsARM.h"
47#define DEBUG_TYPE "arm-mve-gather-scatter-lowering"
51 cl::desc(
"Enable the generation of masked gathers and scatters"));
66 return "MVE gather/scatter lowering";
81 bool isLegalTypeAndAlignment(
unsigned NumElements,
unsigned ElemSize,
84 void lookThroughBitcast(
Value *&
Ptr);
97 int computeScale(
unsigned GEPElemSize,
unsigned MemoryElemSize);
100 std::optional<int64_t> getIfConst(
const Value *V);
104 std::pair<Value *, int64_t> getVarAndConst(
Value *Inst,
int TypeScale);
114 int64_t Increment = 0);
118 int64_t Increment = 0);
127 int64_t Increment = 0);
131 int64_t Increment = 0);
152 void pushOutAdd(
PHINode *&Phi,
Value *OffsSecondOperand,
unsigned StartIndex);
154 void pushOutMulShl(
unsigned Opc,
PHINode *&Phi,
Value *IncrementPerRound,
155 Value *OffsSecondOperand,
unsigned LoopIncrement,
161char MVEGatherScatterLowering::ID = 0;
164 "MVE gather/scattering lowering pass",
false,
false)
167 return new MVEGatherScatterLowering();
170bool MVEGatherScatterLowering::isLegalTypeAndAlignment(
unsigned NumElements,
173 if (((NumElements == 4 &&
174 (ElemSize == 32 || ElemSize == 16 || ElemSize == 8)) ||
175 (NumElements == 8 && (ElemSize == 16 || ElemSize == 8)) ||
176 (NumElements == 16 && ElemSize == 8)) &&
177 Alignment >= ElemSize / 8)
179 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: instruction does not have "
180 <<
"valid alignment or vector type \n");
195 unsigned TargetElemSize = 128 / TargetElemCount;
196 unsigned OffsetElemSize = cast<FixedVectorType>(Offsets->getType())
198 ->getScalarSizeInBits();
199 if (OffsetElemSize != TargetElemSize || OffsetElemSize != 32) {
200 Constant *ConstOff = dyn_cast<Constant>(Offsets);
203 int64_t TargetElemMaxSize = (1ULL << TargetElemSize);
204 auto CheckValueSize = [TargetElemMaxSize](
Value *OffsetElem) {
205 ConstantInt *OConst = dyn_cast<ConstantInt>(OffsetElem);
209 if (SExtValue >= TargetElemMaxSize || SExtValue < 0)
213 if (isa<FixedVectorType>(ConstOff->
getType())) {
214 for (
unsigned i = 0; i < TargetElemCount; i++) {
219 if (!CheckValueSize(ConstOff))
230 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
Ptr)) {
233 computeScale(
GEP->getSourceElementType()->getPrimitiveSizeInBits(),
235 return Scale == -1 ? nullptr :
V;
253Value *MVEGatherScatterLowering::decomposeGEP(
Value *&Offsets,
258 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: no getelementpointer "
262 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: getelementpointer found."
263 <<
" Looking at intrinsic for base + vector of offsets\n");
264 Value *GEPPtr =
GEP->getPointerOperand();
267 !isa<FixedVectorType>(
Offsets->getType()))
270 if (
GEP->getNumOperands() != 2) {
271 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: getelementptr with too many"
272 <<
" operands. Expanding.\n");
276 unsigned OffsetsElemCount =
277 cast<FixedVectorType>(
Offsets->getType())->getNumElements();
281 ZExtInst *ZextOffs = dyn_cast<ZExtInst>(Offsets);
288 if (!ZextOffs || cast<FixedVectorType>(ZextOffs->
getDestTy())
290 ->getScalarSizeInBits() != 32)
296 if (Ty !=
Offsets->getType()) {
305 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: found correct offsets\n");
309void MVEGatherScatterLowering::lookThroughBitcast(
Value *&
Ptr) {
311 if (
auto *BitCast = dyn_cast<BitCastInst>(
Ptr)) {
312 auto *BCTy = cast<FixedVectorType>(BitCast->getType());
313 auto *BCSrcTy = cast<FixedVectorType>(BitCast->getOperand(0)->getType());
314 if (BCTy->getNumElements() == BCSrcTy->getNumElements()) {
317 Ptr = BitCast->getOperand(0);
322int MVEGatherScatterLowering::computeScale(
unsigned GEPElemSize,
323 unsigned MemoryElemSize) {
326 if (GEPElemSize == 32 && MemoryElemSize == 32)
328 else if (GEPElemSize == 16 && MemoryElemSize == 16)
330 else if (GEPElemSize == 8)
332 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: incorrect scale. Can't "
333 <<
"create intrinsic\n");
337std::optional<int64_t> MVEGatherScatterLowering::getIfConst(
const Value *V) {
338 const Constant *
C = dyn_cast<Constant>(V);
339 if (
C &&
C->getSplatValue())
340 return std::optional<int64_t>{
C->getUniqueInteger().getSExtValue()};
341 if (!isa<Instruction>(V))
342 return std::optional<int64_t>{};
345 if (
I->getOpcode() == Instruction::Add ||
I->getOpcode() == Instruction::Or ||
346 I->getOpcode() == Instruction::Mul ||
347 I->getOpcode() == Instruction::Shl) {
348 std::optional<int64_t> Op0 = getIfConst(
I->getOperand(0));
349 std::optional<int64_t> Op1 = getIfConst(
I->getOperand(1));
351 return std::optional<int64_t>{};
352 if (
I->getOpcode() == Instruction::Add)
353 return std::optional<int64_t>{*Op0 + *Op1};
354 if (
I->getOpcode() == Instruction::Mul)
355 return std::optional<int64_t>{*Op0 * *Op1};
356 if (
I->getOpcode() == Instruction::Shl)
357 return std::optional<int64_t>{*Op0 << *Op1};
358 if (
I->getOpcode() == Instruction::Or)
359 return std::optional<int64_t>{*Op0 | *Op1};
361 return std::optional<int64_t>{};
367 return I->getOpcode() == Instruction::Or &&
371std::pair<Value *, int64_t>
372MVEGatherScatterLowering::getVarAndConst(
Value *Inst,
int TypeScale) {
373 std::pair<Value *, int64_t> ReturnFalse =
374 std::pair<Value *, int64_t>(
nullptr, 0);
378 if (
Add ==
nullptr ||
383 std::optional<int64_t>
Const;
385 if ((Const = getIfConst(
Add->getOperand(0))))
386 Summand =
Add->getOperand(1);
387 else if ((Const = getIfConst(
Add->getOperand(1))))
388 Summand =
Add->getOperand(0);
393 int64_t Immediate = *
Const << TypeScale;
394 if (Immediate > 512 || Immediate < -512 || Immediate % 4 != 0)
397 return std::pair<Value *, int64_t>(Summand, Immediate);
401 using namespace PatternMatch;
402 LLVM_DEBUG(
dbgs() <<
"masked gathers: checking transform preconditions\n"
408 auto *Ty = cast<FixedVectorType>(
I->getType());
410 Align Alignment = cast<ConstantInt>(
I->getArgOperand(1))->getAlignValue();
412 Value *PassThru =
I->getArgOperand(3);
417 lookThroughBitcast(
Ptr);
418 assert(
Ptr->getType()->isVectorTy() &&
"Unexpected pointer type");
428 Load = tryCreateMaskedGatherOffset(
I,
Ptr, Root, Builder);
430 Load = tryCreateMaskedGatherBase(
I,
Ptr, Builder);
434 if (!isa<UndefValue>(PassThru) && !
match(PassThru,
m_Zero())) {
435 LLVM_DEBUG(
dbgs() <<
"masked gathers: found non-trivial passthru - "
436 <<
"creating select\n");
446 I->eraseFromParent();
448 LLVM_DEBUG(
dbgs() <<
"masked gathers: successfully built masked gather\n"
453Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
455 using namespace PatternMatch;
456 auto *Ty = cast<FixedVectorType>(
I->getType());
457 LLVM_DEBUG(
dbgs() <<
"masked gathers: loading from vector of pointers\n");
464 {Ty,
Ptr->getType()},
468 Intrinsic::arm_mve_vldr_gather_base_predicated,
469 {Ty,
Ptr->getType(),
Mask->getType()},
473Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
475 using namespace PatternMatch;
476 auto *Ty = cast<FixedVectorType>(
I->getType());
477 LLVM_DEBUG(
dbgs() <<
"masked gathers: loading from vector of pointers with "
485 {Ty,
Ptr->getType()},
489 Intrinsic::arm_mve_vldr_gather_base_wb_predicated,
490 {Ty,
Ptr->getType(),
Mask->getType()},
494Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
496 using namespace PatternMatch;
498 Type *MemoryTy =
I->getType();
499 Type *ResultTy = MemoryTy;
505 bool TruncResult =
false;
507 if (
I->hasOneUse()) {
512 if (isa<SExtInst>(
User) &&
519 }
else if (isa<ZExtInst>(
User) &&
522 << *ResultTy <<
"\n");
535 LLVM_DEBUG(
dbgs() <<
"masked gathers: Small input type, truncing to: "
536 << *ResultTy <<
"\n");
541 LLVM_DEBUG(
dbgs() <<
"masked gathers: Extend needed but not provided "
542 "from the correct type. Expanding\n");
550 Ptr, Offsets, Scale, cast<FixedVectorType>(ResultTy), MemoryTy, Builder);
559 Intrinsic::arm_mve_vldr_gather_offset_predicated,
565 Intrinsic::arm_mve_vldr_gather_offset,
571 Load = TruncInst::Create(Instruction::Trunc, Load, MemoryTy);
578 using namespace PatternMatch;
579 LLVM_DEBUG(
dbgs() <<
"masked scatters: checking transform preconditions\n"
585 Value *Input =
I->getArgOperand(0);
587 Align Alignment = cast<ConstantInt>(
I->getArgOperand(2))->getAlignValue();
588 auto *Ty = cast<FixedVectorType>(Input->
getType());
594 lookThroughBitcast(
Ptr);
595 assert(
Ptr->getType()->isVectorTy() &&
"Unexpected pointer type");
603 Store = tryCreateMaskedScatterOffset(
I,
Ptr, Builder);
605 Store = tryCreateMaskedScatterBase(
I,
Ptr, Builder);
609 LLVM_DEBUG(
dbgs() <<
"masked scatters: successfully built masked scatter\n"
611 I->eraseFromParent();
615Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
617 using namespace PatternMatch;
618 Value *Input =
I->getArgOperand(0);
619 auto *Ty = cast<FixedVectorType>(Input->
getType());
627 LLVM_DEBUG(
dbgs() <<
"masked scatters: storing to a vector of pointers\n");
634 Intrinsic::arm_mve_vstr_scatter_base_predicated,
639Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
641 using namespace PatternMatch;
642 Value *Input =
I->getArgOperand(0);
643 auto *Ty = cast<FixedVectorType>(Input->
getType());
644 LLVM_DEBUG(
dbgs() <<
"masked scatters: storing to a vector of pointers "
645 <<
"with writeback\n");
651 return Builder.
CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base_wb,
656 Intrinsic::arm_mve_vstr_scatter_base_wb_predicated,
661Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
663 using namespace PatternMatch;
664 Value *Input =
I->getArgOperand(0);
667 Type *MemoryTy = InputTy;
669 LLVM_DEBUG(
dbgs() <<
"masked scatters: getelementpointer found. Storing"
670 <<
" to base + vector of offsets\n");
673 if (
TruncInst *Trunc = dyn_cast<TruncInst>(Input)) {
674 Value *PreTrunc = Trunc->getOperand(0);
678 InputTy = PreTruncTy;
681 bool ExtendInput =
false;
691 LLVM_DEBUG(
dbgs() <<
"masked scatters: Small input type, will extend:\n"
695 LLVM_DEBUG(
dbgs() <<
"masked scatters: cannot create scatters for "
696 "non-standard input types. Expanding.\n");
703 Ptr, Offsets, Scale, cast<FixedVectorType>(InputTy), MemoryTy, Builder);
711 Intrinsic::arm_mve_vstr_scatter_offset_predicated,
719 Intrinsic::arm_mve_vstr_scatter_offset,
726Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
729 if (
I->getIntrinsicID() == Intrinsic::masked_gather)
730 Ty = cast<FixedVectorType>(
I->getType());
732 Ty = cast<FixedVectorType>(
I->getArgOperand(0)->getType());
738 Loop *
L = LI->getLoopFor(
I->getParent());
749 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: trying to build incrementing "
750 "wb gather/scatter\n");
755 computeScale(
DL->getTypeSizeInBits(
GEP->getOperand(0)->getType()),
756 DL->getTypeSizeInBits(
GEP->getType()) /
757 cast<FixedVectorType>(
GEP->getType())->getNumElements());
761 if (
GEP->hasOneUse()) {
765 if (
auto *Load = tryCreateIncrementingWBGatScat(
I, BasePtr, Offsets,
770 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: trying to build incrementing "
771 "non-wb gather/scatter\n");
773 std::pair<Value *, int64_t>
Add = getVarAndConst(Offsets, TypeScale);
774 if (
Add.first ==
nullptr)
777 int64_t Immediate =
Add.second;
781 Instruction::Shl, OffsetsIncoming,
784 "ScaledIndex",
I->getIterator());
787 Instruction::Add, ScaledOffsets,
792 cast<VectorType>(ScaledOffsets->
getType())->getElementType())),
793 "StartIndex",
I->getIterator());
795 if (
I->getIntrinsicID() == Intrinsic::masked_gather)
796 return tryCreateMaskedGatherBase(
I, OffsetsIncoming, Builder, Immediate);
798 return tryCreateMaskedScatterBase(
I, OffsetsIncoming, Builder, Immediate);
801Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
806 Loop *
L = LI->getLoopFor(
I->getParent());
810 if (Phi ==
nullptr ||
Phi->getNumIncomingValues() != 2 ||
811 Phi->getParent() !=
L->getHeader() ||
Phi->getNumUses() != 2)
818 unsigned IncrementIndex =
819 Phi->getIncomingBlock(0) ==
L->getLoopLatch() ? 0 : 1;
821 Offsets =
Phi->getIncomingValue(IncrementIndex);
823 std::pair<Value *, int64_t>
Add = getVarAndConst(Offsets, TypeScale);
824 if (
Add.first ==
nullptr)
827 int64_t Immediate =
Add.second;
828 if (OffsetsIncoming != Phi)
835 cast<FixedVectorType>(OffsetsIncoming->
getType())->getNumElements();
839 Instruction::Shl,
Phi->getIncomingValue(1 - IncrementIndex),
842 Phi->getIncomingBlock(1 - IncrementIndex)->back().getIterator());
845 Instruction::Add, ScaledOffsets,
850 cast<VectorType>(ScaledOffsets->
getType())->getElementType())),
852 Phi->getIncomingBlock(1 - IncrementIndex)->back().getIterator());
855 Instruction::Sub, OffsetsIncoming,
857 "PreIncrementStartIndex",
858 Phi->getIncomingBlock(1 - IncrementIndex)->back().getIterator());
859 Phi->setIncomingValue(1 - IncrementIndex, OffsetsIncoming);
865 if (
I->getIntrinsicID() == Intrinsic::masked_gather) {
867 Value *
Load = tryCreateMaskedGatherBaseWB(
I, Phi, Builder, Immediate);
872 Builder.
Insert(EndResult);
873 Builder.
Insert(NewInduction);
876 EndResult = NewInduction =
877 tryCreateMaskedScatterBaseWB(
I, Phi, Builder, Immediate);
882 Phi->setIncomingValue(IncrementIndex, NewInduction);
887void MVEGatherScatterLowering::pushOutAdd(
PHINode *&Phi,
888 Value *OffsSecondOperand,
889 unsigned StartIndex) {
890 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: optimising add instruction\n");
892 Phi->getIncomingBlock(StartIndex)->back().getIterator();
895 Instruction::Add,
Phi->getIncomingValue(StartIndex), OffsSecondOperand,
897 unsigned IncrementIndex = StartIndex == 0 ? 1 : 0;
900 Phi->addIncoming(NewIndex,
Phi->getIncomingBlock(StartIndex));
901 Phi->addIncoming(
Phi->getIncomingValue(IncrementIndex),
902 Phi->getIncomingBlock(IncrementIndex));
903 Phi->removeIncomingValue(1);
904 Phi->removeIncomingValue((
unsigned)0);
907void MVEGatherScatterLowering::pushOutMulShl(
unsigned Opcode,
PHINode *&Phi,
908 Value *IncrementPerRound,
909 Value *OffsSecondOperand,
910 unsigned LoopIncrement,
912 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: optimising mul instruction\n");
917 Phi->getIncomingBlock(LoopIncrement == 1 ? 0 : 1)->back().getIterator();
922 Phi->getIncomingValue(LoopIncrement == 1 ? 0 : 1),
930 Phi->getIncomingBlock(LoopIncrement)->back().getIterator();
931 NewIncrInsertPt = std::prev(NewIncrInsertPt);
935 Instruction::Add, Phi, Product,
"IncrementPushedOutMul", NewIncrInsertPt);
937 Phi->addIncoming(StartIndex,
938 Phi->getIncomingBlock(LoopIncrement == 1 ? 0 : 1));
939 Phi->addIncoming(NewIncrement,
Phi->getIncomingBlock(LoopIncrement));
940 Phi->removeIncomingValue((
unsigned)0);
941 Phi->removeIncomingValue((
unsigned)0);
947 if (
I->hasNUses(0)) {
951 for (
User *U :
I->users()) {
952 if (!isa<Instruction>(U))
954 if (isa<GetElementPtrInst>(U) ||
958 unsigned OpCode = cast<Instruction>(U)->getOpcode();
959 if ((OpCode == Instruction::Add || OpCode == Instruction::Mul ||
960 OpCode == Instruction::Shl ||
971bool MVEGatherScatterLowering::optimiseOffsets(
Value *Offsets,
BasicBlock *BB,
973 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: trying to optimize: "
974 << *Offsets <<
"\n");
977 if (!isa<Instruction>(Offsets))
999 }
else if (isa<PHINode>(Offs->
getOperand(1))) {
1003 bool Changed =
false;
1005 L->contains(cast<Instruction>(Offs->
getOperand(0))))
1006 Changed |= optimiseOffsets(Offs->
getOperand(0), BB, LI);
1008 L->contains(cast<Instruction>(Offs->
getOperand(1))))
1009 Changed |= optimiseOffsets(Offs->
getOperand(1), BB, LI);
1015 }
else if (isa<PHINode>(Offs->
getOperand(1))) {
1024 if (
Phi->getParent() !=
L->getHeader())
1029 Value *Start, *IncrementPerRound;
1031 IncInstruction->
getOpcode() != Instruction::Add)
1034 int IncrementingBlock =
Phi->getIncomingValue(0) == IncInstruction ? 0 : 1;
1039 if (IncrementPerRound->
getType() != OffsSecondOperand->
getType() ||
1040 !
L->isLoopInvariant(OffsSecondOperand))
1046 if (!isa<Constant>(IncrementPerRound) &&
1047 !(isa<Instruction>(IncrementPerRound) &&
1048 !
L->contains(cast<Instruction>(IncrementPerRound))))
1054 if (
Phi->getNumUses() == 2) {
1062 IncrementPerRound,
"LoopIncrement", IncInstruction->
getIterator());
1063 Phi->setIncomingValue(IncrementingBlock, IncInstruction);
1070 NewPhi->
addIncoming(
Phi->getIncomingValue(IncrementingBlock == 1 ? 0 : 1),
1071 Phi->getIncomingBlock(IncrementingBlock == 1 ? 0 : 1));
1074 IncrementPerRound,
"LoopIncrement", IncInstruction->
getIterator());
1076 Phi->getIncomingBlock(IncrementingBlock));
1077 IncrementingBlock = 1;
1085 case Instruction::Add:
1086 case Instruction::Or:
1087 pushOutAdd(NewPhi, OffsSecondOperand, IncrementingBlock == 1 ? 0 : 1);
1089 case Instruction::Mul:
1090 case Instruction::Shl:
1091 pushOutMulShl(Offs->
getOpcode(), NewPhi, IncrementPerRound,
1092 OffsSecondOperand, IncrementingBlock, Builder);
1097 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: simplified loop variable "
1119 if ((Const = dyn_cast<ConstantInt>(NonVectorVal)) &&
1123 if (
N < (
unsigned)(1 << (TargetElemSize - 1))) {
1137 if (XElType && !YElType) {
1138 FixSummands(XElType,
Y);
1139 YElType = cast<FixedVectorType>(
Y->getType());
1140 }
else if (YElType && !XElType) {
1141 FixSummands(YElType,
X);
1142 XElType = cast<FixedVectorType>(
X->getType());
1144 assert(XElType && YElType &&
"Unknown vector types");
1146 if (XElType != YElType) {
1147 LLVM_DEBUG(
dbgs() <<
"masked gathers/scatters: incompatible gep offsets\n");
1154 Constant *ConstX = dyn_cast<Constant>(
X);
1155 Constant *ConstY = dyn_cast<Constant>(
Y);
1156 if (!ConstX || !ConstY)
1164 if (!ConstXEl || !ConstYEl ||
1167 (
unsigned)(1 << (TargetElemSize - 1)))
1188 Value *&Offsets,
unsigned &Scale,
1190 Value *GEPPtr =
GEP->getPointerOperand();
1192 Scale =
DL->getTypeAllocSize(
GEP->getSourceElementType());
1195 if (
GEP->getNumIndices() != 1 || !isa<Constant>(Offsets))
1199 Value *BaseBasePtr = foldGEP(BaseGEP, Offsets, Scale, Builder);
1203 Offsets, Scale,
GEP->getOperand(1),
1204 DL->getTypeAllocSize(
GEP->getSourceElementType()), Builder);
1205 if (Offsets ==
nullptr)
1213bool MVEGatherScatterLowering::optimiseAddress(
Value *Address,
BasicBlock *BB,
1218 bool Changed =
false;
1219 if (
GEP->hasOneUse() && isa<GetElementPtrInst>(
GEP->getPointerOperand())) {
1231 assert(Scale == 1 &&
"Expected to fold GEP to a scale of 1");
1233 if (
auto *VecTy = dyn_cast<FixedVectorType>(
Base->getType()))
1237 "gep.merged",
GEP->getIterator());
1239 <<
"\n new : " << *NewAddress <<
"\n");
1240 GEP->replaceAllUsesWith(
1246 Changed |= optimiseOffsets(
GEP->getOperand(1),
GEP->getParent(), LI);
1250bool MVEGatherScatterLowering::runOnFunction(
Function &
F) {
1253 auto &TPC = getAnalysis<TargetPassConfig>();
1256 if (!
ST->hasMVEIntegerOps())
1258 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1259 DL = &
F.getDataLayout();
1263 bool Changed =
false;
1270 if (
II &&
II->getIntrinsicID() == Intrinsic::masked_gather &&
1271 isa<FixedVectorType>(
II->getType())) {
1273 Changed |= optimiseAddress(
II->getArgOperand(0),
II->getParent(), LI);
1274 }
else if (
II &&
II->getIntrinsicID() == Intrinsic::masked_scatter &&
1275 isa<FixedVectorType>(
II->getArgOperand(0)->getType())) {
1277 Changed |= optimiseAddress(
II->getArgOperand(1),
II->getParent(), LI);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static Decomposition decomposeGEP(GEPOperator &GEP, SmallVectorImpl< ConditionTy > &Preconditions, bool IsSigned, const DataLayout &DL)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isAddLikeOr(Instruction *I, const DataLayout &DL)
static bool hasAllGatScatUsers(Instruction *I, const DataLayout &DL)
static bool checkOffsetSize(Value *Offsets, unsigned TargetElemCount)
static Value * CheckAndCreateOffsetAdd(Value *X, unsigned ScaleX, Value *Y, unsigned ScaleY, IRBuilder<> &Builder)
cl::opt< bool > EnableMaskedGatherScatters("enable-arm-maskedgatscat", cl::Hidden, cl::init(true), cl::desc("Enable the generation of masked gathers and scatters"))
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static unsigned getNumElements(Type *Ty)
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
LLVM Basic Block Representation.
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
BinaryOps getOpcode() const
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
Type * getDestTy() const
Return the destination type, as a convenience.
This is the shared class of boolean and integer constants.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
A parsed version of the target data layout string in and methods for querying it.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionPass class - This class is used to implement most global optimizations.
virtual bool runOnFunction(Function &F)=0
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
A wrapper class for inspecting calls to intrinsic functions.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
The legacy pass manager's analysis pass to compute loop information.
Represents a single loop in the control flow graph.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, Instruction *MDFrom=nullptr)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Primary interface to the complete machine description for the target machine.
Target-Independent Code Generator Pass Configuration Options.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
unsigned getNumUses() const
This method computes the number of uses of this Value.
Type * getElementType() const
This class represents zero extension of integer types.
const ParentTy * getParent() const
self_iterator getIterator()
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
initializer< Ty > init(const Ty &Val)
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
bool haveNoCommonBitsSet(const WithCache< const Value * > &LHSCache, const WithCache< const Value * > &RHSCache, const SimplifyQuery &SQ)
Return true if LHS and RHS have no common bits set.
Pass * createMVEGatherScatterLoweringPass()
bool SimplifyInstructionsInBlock(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr)
Scan the specified basic block and try to simplify any instructions in it and recursively delete dead...
bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
bool isGatherScatter(IntrinsicInst *IntInst)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void initializeMVEGatherScatterLoweringPass(PassRegistry &)
This struct is a compact representation of a valid (non-zero power of two) alignment.