LLVM 24.0.0git
CodeGenPrepare.cpp
Go to the documentation of this file.
1//===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass munges the code in the input function to better prepare it for
10// SelectionDAG-based code generation. This works around limitations in it's
11// basic-block-at-a-time approach. It should eventually be removed.
12//
13//===----------------------------------------------------------------------===//
14
16#include "CodeGenOptions.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/Statistic.h"
47#include "llvm/Config/llvm-config.h"
48#include "llvm/IR/Argument.h"
49#include "llvm/IR/Attributes.h"
50#include "llvm/IR/BasicBlock.h"
51#include "llvm/IR/CFG.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/CycleInfo.h"
55#include "llvm/IR/DataLayout.h"
56#include "llvm/IR/DebugInfo.h"
58#include "llvm/IR/Dominators.h"
59#include "llvm/IR/Function.h"
61#include "llvm/IR/GlobalValue.h"
63#include "llvm/IR/IRBuilder.h"
64#include "llvm/IR/InlineAsm.h"
65#include "llvm/IR/InstrTypes.h"
66#include "llvm/IR/Instruction.h"
69#include "llvm/IR/Intrinsics.h"
70#include "llvm/IR/IntrinsicsAArch64.h"
71#include "llvm/IR/LLVMContext.h"
72#include "llvm/IR/MDBuilder.h"
73#include "llvm/IR/Module.h"
74#include "llvm/IR/Operator.h"
77#include "llvm/IR/Statepoint.h"
78#include "llvm/IR/Type.h"
79#include "llvm/IR/Use.h"
80#include "llvm/IR/User.h"
81#include "llvm/IR/Value.h"
82#include "llvm/IR/ValueHandle.h"
83#include "llvm/IR/ValueMap.h"
85#include "llvm/Pass.h"
91#include "llvm/Support/Debug.h"
101#include <algorithm>
102#include <cassert>
103#include <cstdint>
104#include <iterator>
105#include <limits>
106#include <memory>
107#include <optional>
108#include <utility>
109#include <vector>
110
111using namespace llvm;
112using namespace llvm::PatternMatch;
113
114#define DEBUG_TYPE "codegenprepare"
115
116STATISTIC(NumBlocksElim, "Number of blocks eliminated");
117STATISTIC(NumPHIsElim, "Number of trivial PHIs eliminated");
118STATISTIC(NumGEPsElim, "Number of GEPs converted to casts");
119STATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of "
120 "sunken Cmps");
121STATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses "
122 "of sunken Casts");
123STATISTIC(NumMemoryInsts, "Number of memory instructions whose address "
124 "computations were sunk");
125STATISTIC(NumMemoryInstsPhiCreated,
126 "Number of phis created when address "
127 "computations were sunk to memory instructions");
128STATISTIC(NumMemoryInstsSelectCreated,
129 "Number of select created when address "
130 "computations were sunk to memory instructions");
131STATISTIC(NumExtsMoved, "Number of [s|z]ext instructions combined with loads");
132STATISTIC(NumExtUses, "Number of uses of [s|z]ext instructions optimized");
133STATISTIC(NumAndsAdded,
134 "Number of and mask instructions added to form ext loads");
135STATISTIC(NumAndUses, "Number of uses of and mask instructions optimized");
136STATISTIC(NumRetsDup, "Number of return instructions duplicated");
137STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");
138STATISTIC(NumSelectsExpanded, "Number of selects turned into branches");
139STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed");
140
141namespace {
142
143enum ExtType {
144 ZeroExtension, // Zero extension has been seen.
145 SignExtension, // Sign extension has been seen.
146 BothExtension // This extension type is used if we saw sext after
147 // ZeroExtension had been set, or if we saw zext after
148 // SignExtension had been set. It makes the type
149 // information of a promoted instruction invalid.
150};
151
152enum ModifyDT {
153 NotModifyDT, // Not Modify any DT.
154 ModifyBBDT, // Modify the Basic Block Dominator Tree.
155 ModifyInstDT // Modify the Instruction Dominator in a Basic Block,
156 // This usually means we move/delete/insert instruction
157 // in a Basic Block. So we should re-iterate instructions
158 // in such Basic Block.
159};
160
161using SetOfInstrs = SmallPtrSet<Instruction *, 16>;
162using TypeIsSExt = PointerIntPair<Type *, 2, ExtType>;
163using InstrToOrigTy = DenseMap<Instruction *, TypeIsSExt>;
165using ValueToSExts = MapVector<Value *, SExts>;
166
167class TypePromotionTransaction;
168
169class CodeGenPrepare {
170 friend class CodeGenPrepareLegacyPass;
171 const CodeGenOptions &Opts;
172 const TargetMachine *TM = nullptr;
173 const TargetSubtargetInfo *SubtargetInfo = nullptr;
174 const TargetLowering *TLI = nullptr;
175 const TargetRegisterInfo *TRI = nullptr;
176 const TargetTransformInfo *TTI = nullptr;
177 const BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr;
178 const TargetLibraryInfo *TLInfo = nullptr;
179 DomTreeUpdater *DTU = nullptr;
180 LoopInfo *LI = nullptr;
181 BlockFrequencyInfo *BFI;
182 BranchProbabilityInfo *BPI;
183 ProfileSummaryInfo *PSI = nullptr;
184
185 /// As we scan instructions optimizing them, this is the next instruction
186 /// to optimize. Transforms that can invalidate this should update it.
187 BasicBlock::iterator CurInstIterator;
188
189 /// Keeps track of non-local addresses that have been sunk into a block.
190 /// This allows us to avoid inserting duplicate code for blocks with
191 /// multiple load/stores of the same address. The usage of WeakTrackingVH
192 /// enables SunkAddrs to be treated as a cache whose entries can be
193 /// invalidated if a sunken address computation has been erased.
194 ValueMap<Value *, WeakTrackingVH> SunkAddrs;
195
196 /// Keeps track of all instructions inserted for the current function.
197 SetOfInstrs InsertedInsts;
198
199 /// Keeps track of the type of the related instruction before their
200 /// promotion for the current function.
201 InstrToOrigTy PromotedInsts;
202
203 /// Keep track of instructions removed during promotion.
204 SetOfInstrs RemovedInsts;
205
206 /// Keep track of sext chains based on their initial value.
207 DenseMap<Value *, Instruction *> SeenChainsForSExt;
208
209 /// Keep track of GEPs accessing the same data structures such as structs or
210 /// arrays that are candidates to be split later because of their large
211 /// size.
212 MapVector<AssertingVH<Value>,
214 LargeOffsetGEPMap;
215
216 /// Keep track of new GEP base after splitting the GEPs having large offset.
217 SmallSet<AssertingVH<Value>, 2> NewGEPBases;
218
219 /// Map serial numbers to Large offset GEPs.
220 DenseMap<AssertingVH<GetElementPtrInst>, int> LargeOffsetGEPID;
221
222 /// Keep track of SExt promoted.
223 ValueToSExts ValToSExtendedUses;
224
225 /// True if the function has the OptSize attribute.
226 bool OptSize;
227
228 /// DataLayout for the Function being processed.
229 const DataLayout *DL = nullptr;
230
231public:
232 CodeGenPrepare(const TargetMachine *TM)
233 : Opts(cl::LibraryOptionsContext::get<CodeGenOptions>(
234 TM->Options.MCOptions.LibraryOptions)),
235 TM(TM) {}
236 /// If encounter huge function, we need to limit the build time.
237 bool IsHugeFunc = false;
238
239 /// FreshBBs is like worklist, it collected the updated BBs which need
240 /// to be optimized again.
241 /// Note: Consider building time in this pass, when a BB updated, we need
242 /// to insert such BB into FreshBBs for huge function.
243 SmallPtrSet<BasicBlock *, 32> FreshBBs;
244
245 void releaseMemory() {
246 // Clear per function information.
247 InsertedInsts.clear();
248 PromotedInsts.clear();
249 FreshBBs.clear();
250 }
251
253
254private:
255 template <typename F>
256 void resetIteratorIfInvalidatedWhileCalling(BasicBlock *BB, F f) {
257 // Substituting can cause recursive simplifications, which can invalidate
258 // our iterator. Use a WeakTrackingVH to hold onto it in case this
259 // happens.
260 Value *CurValue = &*CurInstIterator;
261 WeakTrackingVH IterHandle(CurValue);
262
263 f();
264
265 // If the iterator instruction was recursively deleted, start over at the
266 // start of the block.
267 if (IterHandle != CurValue) {
268 CurInstIterator = BB->begin();
269 SunkAddrs.clear();
270 }
271 }
272
273 // Get the DominatorTree, updating it if necessary.
274 DominatorTree &getDT() { return DTU->getDomTree(); }
275
276 void removeAllAssertingVHReferences(Value *V);
277 bool eliminateAssumptions(Function &F);
278 bool eliminateFallThrough(Function &F);
279 bool eliminateMostlyEmptyBlocks(Function &F, bool &ResetLI);
280 BasicBlock *findDestBlockOfMergeableEmptyBlock(BasicBlock *BB);
281 bool canMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const;
282 bool eliminateMostlyEmptyBlock(BasicBlock *BB);
283 bool isMergingEmptyBlockProfitable(BasicBlock *BB, BasicBlock *DestBB,
284 bool isPreheader);
285 bool makeBitReverse(Instruction &I);
286 bool optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT);
287 bool optimizeInst(Instruction *I, ModifyDT &ModifiedDT);
288 bool optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, Type *AccessTy,
289 unsigned AddrSpace);
290 bool optimizeGatherScatterInst(Instruction *MemoryInst, Value *Ptr);
291 bool optimizeMulWithOverflow(Instruction *I, bool IsSigned,
292 ModifyDT &ModifiedDT);
293 bool optimizeInlineAsmInst(CallInst *CS);
294 bool optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT);
295 bool optimizeExt(Instruction *&I);
296 bool optimizeExtUses(Instruction *I);
297 bool optimizeLoadExt(LoadInst *Load);
298 bool optimizeShiftInst(BinaryOperator *BO);
299 bool optimizeFunnelShift(IntrinsicInst *Fsh);
300 bool optimizeSelectInst(SelectInst *SI);
301 bool optimizeShuffleVectorInst(ShuffleVectorInst *SVI);
302 bool optimizeSwitchType(SwitchInst *SI);
303 bool optimizeSwitchPhiConstants(SwitchInst *SI);
304 bool optimizeSwitchInst(SwitchInst *SI);
305 bool optimizeExtractElementInst(Instruction *Inst);
306 bool dupRetToEnableTailCallOpts(BasicBlock *BB, ModifyDT &ModifiedDT);
307 bool fixupDbgVariableRecord(DbgVariableRecord &I);
308 bool fixupDbgVariableRecordsOnInst(Instruction &I);
309 bool placeDbgValues(Function &F);
310 bool placePseudoProbes(Function &F);
311 bool canFormExtLd(const SmallVectorImpl<Instruction *> &MovedExts,
312 LoadInst *&LI, Instruction *&Inst, bool HasPromoted);
313 bool tryToPromoteExts(TypePromotionTransaction &TPT,
314 const SmallVectorImpl<Instruction *> &Exts,
315 SmallVectorImpl<Instruction *> &ProfitablyMovedExts,
316 unsigned CreatedInstsCost = 0);
317 bool mergeSExts(Function &F);
318 bool splitLargeGEPOffsets();
319 bool optimizePhiType(PHINode *Inst, SmallPtrSetImpl<PHINode *> &Visited,
320 SmallPtrSetImpl<Instruction *> &DeletedInstrs);
321 bool optimizePhiTypes(Function &F);
322 bool performAddressTypePromotion(
323 Instruction *&Inst, bool AllowPromotionWithoutCommonHeader,
324 bool HasPromoted, TypePromotionTransaction &TPT,
325 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts);
326 bool splitBranchCondition(Function &F);
327 bool simplifyOffsetableRelocate(GCStatepointInst &I);
328
329 bool tryToSinkFreeOperands(Instruction *I);
330 bool replaceMathCmpWithIntrinsic(BinaryOperator *BO, Value *Arg0, Value *Arg1,
331 CmpInst *Cmp, Intrinsic::ID IID);
332 bool optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT);
333 bool optimizeURem(Instruction *Rem);
334 bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
335 bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
336 bool unfoldPowerOf2Test(CmpInst *Cmp);
337 void verifyBFIUpdates(Function &F);
338 bool _run(Function &F);
339};
340
341class CodeGenPrepareLegacyPass : public FunctionPass {
342public:
343 static char ID; // Pass identification, replacement for typeid
344
345 CodeGenPrepareLegacyPass() : FunctionPass(ID) {}
346
347 bool runOnFunction(Function &F) override;
348
349 StringRef getPassName() const override { return "CodeGen Prepare"; }
350
351 void getAnalysisUsage(AnalysisUsage &AU) const override {
352 // FIXME: When we can selectively preserve passes, preserve the domtree.
353 AU.addRequired<ProfileSummaryInfoWrapperPass>();
354 AU.addRequired<TargetLibraryInfoWrapperPass>();
355 AU.addRequired<TargetPassConfig>();
356 AU.addRequired<TargetTransformInfoWrapperPass>();
357 AU.addRequired<DominatorTreeWrapperPass>();
358 AU.addRequired<LoopInfoWrapperPass>();
359 AU.addRequired<BranchProbabilityInfoWrapperPass>();
360 AU.addRequired<BlockFrequencyInfoWrapperPass>();
361 AU.addUsedIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();
362 }
363};
364
365} // end anonymous namespace
366
367char CodeGenPrepareLegacyPass::ID = 0;
368
369bool CodeGenPrepareLegacyPass::runOnFunction(Function &F) {
370 if (skipFunction(F))
371 return false;
372 auto TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
373 CodeGenPrepare CGP(TM);
374 CGP.DL = &F.getDataLayout();
375 CGP.SubtargetInfo = TM->getSubtargetImpl(F);
376 CGP.TLI = CGP.SubtargetInfo->getTargetLowering();
377 CGP.TRI = CGP.SubtargetInfo->getRegisterInfo();
378 CGP.TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
379 CGP.TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
380 CGP.LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
381 CGP.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
382 CGP.BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
383 CGP.PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
384 auto BBSPRWP =
385 getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();
386 CGP.BBSectionsProfileReader = BBSPRWP ? &BBSPRWP->getBBSPR() : nullptr;
387 DomTreeUpdater DTUpdater(
388 &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
389 DomTreeUpdater::UpdateStrategy::Lazy);
390 CGP.DTU = &DTUpdater;
391
392 return CGP._run(F);
393}
394
395INITIALIZE_PASS_BEGIN(CodeGenPrepareLegacyPass, DEBUG_TYPE,
396 "Optimize for code generation", false, false)
404INITIALIZE_PASS_END(CodeGenPrepareLegacyPass, DEBUG_TYPE,
405 "Optimize for code generation", false, false)
406
408 return new CodeGenPrepareLegacyPass();
409}
410
413 CodeGenPrepare CGP(TM);
414
415 bool Changed = CGP.run(F, AM);
416 if (!Changed)
417 return PreservedAnalyses::all();
418
422 return PA;
423}
424
425bool CodeGenPrepare::run(Function &F, FunctionAnalysisManager &AM) {
426 DL = &F.getDataLayout();
427 SubtargetInfo = TM->getSubtargetImpl(F);
428 TLI = SubtargetInfo->getTargetLowering();
429 TRI = SubtargetInfo->getRegisterInfo();
430 TLInfo = &AM.getResult<TargetLibraryAnalysis>(F);
432 LI = &AM.getResult<LoopAnalysis>(F);
435 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
436 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
437 if (!PSI)
438 reportFatalUsageError("this pass requires the profile-summary module "
439 "analysis to be available");
440 BBSectionsProfileReader =
443 DomTreeUpdater::UpdateStrategy::Lazy);
444 DTU = &DTUpdater;
445 return _run(F);
446}
447
448bool CodeGenPrepare::_run(Function &F) {
449 bool EverMadeChange = false;
450
451 OptSize = F.hasOptSize();
452 // Use the basic-block-sections profile to promote hot functions to .text.hot
453 // if requested.
454 if (Opts.bbsections_guided_section_prefix && BBSectionsProfileReader &&
455 BBSectionsProfileReader->isFunctionHot(F.getName())) {
456 (void)F.setSectionPrefix("hot");
457 } else if (Opts.profile_guided_section_prefix) {
458 // The hot attribute overwrites profile count based hotness while profile
459 // counts based hotness overwrite the cold attribute.
460 // This is a conservative behabvior.
461 if (F.hasFnAttribute(Attribute::Hot) ||
462 PSI->isFunctionHotInCallGraph(&F, *BFI))
463 (void)F.setSectionPrefix("hot");
464 // If PSI shows this function is not hot, we will placed the function
465 // into unlikely section if (1) PSI shows this is a cold function, or
466 // (2) the function has a attribute of cold.
467 else if (PSI->isFunctionColdInCallGraph(&F, *BFI) ||
468 F.hasFnAttribute(Attribute::Cold))
469 (void)F.setSectionPrefix("unlikely");
470 else if (Opts.profile_unknown_in_special_section &&
471 PSI->hasPartialSampleProfile() && PSI->isFunctionHotnessUnknown(F))
472 (void)F.setSectionPrefix("unknown");
473 }
474
475 /// This optimization identifies DIV instructions that can be
476 /// profitably bypassed and carried out with a shorter, faster divide.
477 if (!OptSize && !PSI->hasHugeWorkingSetSize() && TLI->isSlowDivBypassed()) {
478 const DenseMap<unsigned int, unsigned int> &BypassWidths =
480 BasicBlock *BB = &*F.begin();
481 while (BB != nullptr) {
482 // bypassSlowDivision may create new BBs, but we don't want to reapply the
483 // optimization to those blocks.
484 BasicBlock *Next = BB->getNextNode();
485 if (!llvm::shouldOptimizeForSize(BB, PSI, BFI))
486 EverMadeChange |= bypassSlowDivision(BB, BypassWidths, DTU, LI, BPI);
487 BB = Next;
488 }
489 }
490
491 // Get rid of @llvm.assume builtins before attempting to eliminate empty
492 // blocks, since there might be blocks that only contain @llvm.assume calls
493 // (plus arguments that we can get rid of).
494 EverMadeChange |= eliminateAssumptions(F);
495
496 auto resetLoopInfo = [this]() {
497 LI->releaseMemory();
498 LI->analyze(DTU->getDomTree());
499 };
500
501 // Eliminate blocks that contain only PHI nodes and an
502 // unconditional branch.
503 bool ResetLI = false;
504 EverMadeChange |= eliminateMostlyEmptyBlocks(F, ResetLI);
505 if (ResetLI)
506 resetLoopInfo();
507
508 if (Opts.cgp_branch_opts)
509 EverMadeChange |= splitBranchCondition(F);
510
511 // Split some critical edges where one of the sources is an indirect branch,
512 // to help generate sane code for PHIs involving such edges.
513 bool Split = SplitIndirectBrCriticalEdges(F, /*IgnoreBlocksWithoutPHI=*/true,
514 BPI, BFI, DTU);
515 EverMadeChange |= Split;
516 if (Split)
517 resetLoopInfo();
518
519#ifndef NDEBUG
520 if (VerifyDomInfo)
521 assert(getDT().verify(DominatorTree::VerificationLevel::Fast) &&
522 "Incorrect DominatorTree updates in CGP");
523
524 if (VerifyLoopInfo)
525 LI->verify();
526#endif
527
528 // If we are optimzing huge function, we need to consider the build time.
529 // Because the basic algorithm's complex is near O(N!).
530 IsHugeFunc = F.size() > Opts.cgp_huge_func;
531
532 bool MadeChange = true;
533 bool FuncIterated = false;
534 while (MadeChange) {
535 MadeChange = false;
536
537 // This is required because optimizeBlock() calls getDT() inside the loop
538 // below, which flushes pending updates and may delete dead blocks, leading
539 // to iterator invalidation.
540 DTU->flush();
541
542 for (BasicBlock &BB : llvm::make_early_inc_range(F)) {
543 if (FuncIterated && !FreshBBs.contains(&BB))
544 continue;
545
546 ModifyDT ModifiedDTOnIteration = ModifyDT::NotModifyDT;
547 bool Changed = optimizeBlock(BB, ModifiedDTOnIteration);
548
549 MadeChange |= Changed;
550 if (IsHugeFunc) {
551 // If the BB is updated, it may still has chance to be optimized.
552 // This usually happen at sink optimization.
553 // For example:
554 //
555 // bb0:
556 // %and = and i32 %a, 4
557 // %cmp = icmp eq i32 %and, 0
558 //
559 // If the %cmp sink to other BB, the %and will has chance to sink.
560 if (Changed)
561 FreshBBs.insert(&BB);
562 else if (FuncIterated)
563 FreshBBs.erase(&BB);
564 } else {
565 // For small/normal functions, we restart BB iteration if the dominator
566 // tree of the Function was changed.
567 if (ModifiedDTOnIteration != ModifyDT::NotModifyDT)
568 break;
569 }
570 }
571 // We have iterated all the BB in the (only work for huge) function.
572 FuncIterated = IsHugeFunc;
573
574 if (Opts.cgp_type_promotion_merge && !ValToSExtendedUses.empty())
575 MadeChange |= mergeSExts(F);
576 if (!LargeOffsetGEPMap.empty())
577 MadeChange |= splitLargeGEPOffsets();
578 MadeChange |= optimizePhiTypes(F);
579
580 if (MadeChange)
581 eliminateFallThrough(F);
582
583#ifndef NDEBUG
584 if (VerifyDomInfo)
585 assert(getDT().verify(DominatorTree::VerificationLevel::Fast) &&
586 "Incorrect DominatorTree updates in CGP");
587
588 if (VerifyLoopInfo)
589 LI->verify();
590#endif
591
592 // Really free removed instructions during promotion.
593 for (Instruction *I : RemovedInsts)
594 I->deleteValue();
595
596 EverMadeChange |= MadeChange;
597 SeenChainsForSExt.clear();
598 ValToSExtendedUses.clear();
599 RemovedInsts.clear();
600 LargeOffsetGEPMap.clear();
601 LargeOffsetGEPID.clear();
602 }
603
604 NewGEPBases.clear();
605 SunkAddrs.clear();
606
607 // LoopInfo is not needed anymore and ConstantFoldTerminator can break it.
608 LI = nullptr;
609
610 if (Opts.cgp_branch_opts) {
611 MadeChange = false;
612 // Use a set vector to get deterministic iteration order. The order the
613 // blocks are removed may affect whether or not PHI nodes in successors
614 // are removed.
615 SmallSetVector<BasicBlock *, 8> WorkList;
616 for (BasicBlock &BB : F) {
618 MadeChange |= ConstantFoldTerminator(&BB, true, nullptr, DTU);
619 if (!MadeChange)
620 continue;
621
622 for (BasicBlock *Succ : Successors)
623 if (pred_empty(Succ))
624 WorkList.insert(Succ);
625 }
626
627 // Delete the dead blocks and any of their dead successors.
628 MadeChange |= !WorkList.empty();
629 while (!WorkList.empty()) {
630 BasicBlock *BB = WorkList.pop_back_val();
632
633 DeleteDeadBlock(BB, DTU);
634
635 for (BasicBlock *Succ : Successors)
636 if (pred_empty(Succ))
637 WorkList.insert(Succ);
638 }
639
640 // Flush pending DT updates in order to finalise deletion of dead blocks.
641 DTU->flush();
642
643 // Merge pairs of basic blocks with unconditional branches, connected by
644 // a single edge.
645 if (EverMadeChange || MadeChange)
646 MadeChange |= eliminateFallThrough(F);
647
648 EverMadeChange |= MadeChange;
649 }
650
651 if (Opts.cgp_gc_opts) {
653 for (BasicBlock &BB : F)
654 for (Instruction &I : BB)
655 if (auto *SP = dyn_cast<GCStatepointInst>(&I))
656 Statepoints.push_back(SP);
657 for (auto &I : Statepoints)
658 EverMadeChange |= simplifyOffsetableRelocate(*I);
659 }
660
661 // Do this last to clean up use-before-def scenarios introduced by other
662 // preparatory transforms.
663 EverMadeChange |= placeDbgValues(F);
664 EverMadeChange |= placePseudoProbes(F);
665
666#ifndef NDEBUG
667 if (Opts.cgp_verify_bfi_updates)
668 verifyBFIUpdates(F);
669#endif
670
671 return EverMadeChange;
672}
673
674bool CodeGenPrepare::eliminateAssumptions(Function &F) {
675 bool MadeChange = false;
676 for (BasicBlock &BB : F) {
677 CurInstIterator = BB.begin();
678 while (CurInstIterator != BB.end()) {
679 Instruction *I = &*(CurInstIterator++);
680 if (auto *Assume = dyn_cast<AssumeInst>(I)) {
681 MadeChange = true;
682 Value *Operand = Assume->getOperand(0);
683 Assume->eraseFromParent();
684
685 resetIteratorIfInvalidatedWhileCalling(&BB, [&]() {
686 RecursivelyDeleteTriviallyDeadInstructions(Operand, TLInfo, nullptr);
687 });
688 }
689 }
690 }
691 return MadeChange;
692}
693
694/// An instruction is about to be deleted, so remove all references to it in our
695/// GEP-tracking data strcutures.
696void CodeGenPrepare::removeAllAssertingVHReferences(Value *V) {
697 LargeOffsetGEPMap.erase(V);
698 NewGEPBases.erase(V);
699
701 if (!GEP)
702 return;
703
704 LargeOffsetGEPID.erase(GEP);
705
706 auto VecI = LargeOffsetGEPMap.find(GEP->getPointerOperand());
707 if (VecI == LargeOffsetGEPMap.end())
708 return;
709
710 auto &GEPVector = VecI->second;
711 llvm::erase_if(GEPVector, [=](auto &Elt) { return Elt.first == GEP; });
712
713 if (GEPVector.empty())
714 LargeOffsetGEPMap.erase(VecI);
715}
716
717// Verify BFI has been updated correctly by recomputing BFI and comparing them.
718[[maybe_unused]] void CodeGenPrepare::verifyBFIUpdates(Function &F) {
719 CycleInfo NewCI;
720 NewCI.compute(F);
721 BranchProbabilityInfo NewBPI(F, NewCI, TLInfo);
722 BlockFrequencyInfo NewBFI(F, NewBPI, NewCI);
723 NewBFI.verifyMatch(*BFI);
724}
725
726/// Merge basic blocks which are connected by a single edge, where one of the
727/// basic blocks has a single successor pointing to the other basic block,
728/// which has a single predecessor.
729bool CodeGenPrepare::eliminateFallThrough(Function &F) {
730 bool Changed = false;
731 SmallPtrSet<BasicBlock *, 8> Preds;
732 // Scan all of the blocks in the function, except for the entry block.
733 for (auto &Block : llvm::drop_begin(F)) {
734 auto *BB = &Block;
735 if (DTU->isBBPendingDeletion(BB))
736 continue;
737 // If the destination block has a single pred, then this is a trivial
738 // edge, just collapse it.
739 BasicBlock *SinglePred = BB->getSinglePredecessor();
740
741 // Don't merge if BB's address is taken.
742 if (!SinglePred || SinglePred == BB || BB->hasAddressTaken())
743 continue;
744
745 if (isa<UncondBrInst>(SinglePred->getTerminator())) {
746 Changed = true;
747 LLVM_DEBUG(dbgs() << "To merge:\n" << *BB << "\n\n\n");
748
749 // Merge BB into SinglePred and delete it.
750 MergeBlockIntoPredecessor(BB, DTU, LI);
751 Preds.insert(SinglePred);
752
753 if (IsHugeFunc) {
754 // Update FreshBBs to optimize the merged BB.
755 FreshBBs.insert(SinglePred);
756 FreshBBs.erase(BB);
757 }
758 }
759 }
760
761 // (Repeatedly) merging blocks into their predecessors can create redundant
762 // debug intrinsics.
763 for (auto *Pred : Preds)
764 if (!DTU->isBBPendingDeletion(Pred))
766
767 return Changed;
768}
769
770/// Find a destination block from BB if BB is mergeable empty block.
771BasicBlock *CodeGenPrepare::findDestBlockOfMergeableEmptyBlock(BasicBlock *BB) {
772 // If this block doesn't end with an uncond branch, ignore it.
773 UncondBrInst *BI = dyn_cast<UncondBrInst>(BB->getTerminator());
774 if (!BI)
775 return nullptr;
776
777 // If the instruction before the branch (skipping debug info) isn't a phi
778 // node, then other stuff is happening here.
780 if (BBI != BB->begin()) {
781 --BBI;
782 if (!isa<PHINode>(BBI))
783 return nullptr;
784 }
785
786 // Do not break infinite loops.
787 BasicBlock *DestBB = BI->getSuccessor();
788 if (DestBB == BB)
789 return nullptr;
790
791 if (!canMergeBlocks(BB, DestBB))
792 DestBB = nullptr;
793
794 return DestBB;
795}
796
797/// Eliminate blocks that contain only PHI nodes, debug info directives, and an
798/// unconditional branch. Passes before isel (e.g. LSR/loopsimplify) often split
799/// edges in ways that are non-optimal for isel. Start by eliminating these
800/// blocks so we can split them the way we want them.
801bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F, bool &ResetLI) {
802 SmallPtrSet<BasicBlock *, 16> Preheaders;
803 SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end());
804 while (!LoopList.empty()) {
805 Loop *L = LoopList.pop_back_val();
806 llvm::append_range(LoopList, *L);
807 if (BasicBlock *Preheader = L->getLoopPreheader())
808 Preheaders.insert(Preheader);
809 }
810
811 ResetLI = false;
812 bool MadeChange = false;
813 SmallPtrSet<PHINode *, 32> KnownNonDeadPHIs;
814 // Note that this intentionally skips the entry block.
815 for (auto &Block : llvm::drop_begin(F)) {
816 // Delete phi nodes that could block deleting other empty blocks.
817 if (Opts.cgp_delete_phis)
818 MadeChange |= DeleteDeadPHIs(&Block, TLInfo, nullptr, &KnownNonDeadPHIs);
819 }
820
821 for (auto &Block : llvm::drop_begin(F)) {
822 auto *BB = &Block;
823 if (DTU->isBBPendingDeletion(BB))
824 continue;
825 BasicBlock *DestBB = findDestBlockOfMergeableEmptyBlock(BB);
826 if (!DestBB ||
827 !isMergingEmptyBlockProfitable(BB, DestBB, Preheaders.count(BB)))
828 continue;
829
830 ResetLI |= eliminateMostlyEmptyBlock(BB);
831 MadeChange = true;
832 }
833 return MadeChange;
834}
835
836bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB,
837 BasicBlock *DestBB,
838 bool isPreheader) {
839 // Do not delete loop preheaders if doing so would create a critical edge.
840 // Loop preheaders can be good locations to spill registers. If the
841 // preheader is deleted and we create a critical edge, registers may be
842 // spilled in the loop body instead.
843 if (Opts.cgp_preheader_prot && isPreheader &&
844 !(BB->getSinglePredecessor() &&
846 return false;
847
848 // Skip merging if the block's successor is also a successor to any callbr
849 // that leads to this block.
850 // FIXME: Is this really needed? Is this a correctness issue?
851 for (BasicBlock *Pred : predecessors(BB)) {
852 if (isa<CallBrInst>(Pred->getTerminator()) &&
854 return false;
855 }
856
857 // Try to skip merging if the unique predecessor of BB is terminated by a
858 // switch or indirect branch instruction, and BB is used as an incoming block
859 // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to
860 // add COPY instructions in the predecessor of BB instead of BB (if it is not
861 // merged). Note that the critical edge created by merging such blocks wont be
862 // split in MachineSink because the jump table is not analyzable. By keeping
863 // such empty block (BB), ISel will place COPY instructions in BB, not in the
864 // predecessor of BB.
866 if (!Pred || !(isa<SwitchInst>(Pred->getTerminator()) ||
867 isa<IndirectBrInst>(Pred->getTerminator())))
868 return true;
869
870 if (BB->getTerminator() != &*BB->getFirstNonPHIOrPseudoProbe())
871 return true;
872
873 // We use a simple cost heuristic which determine skipping merging is
874 // profitable if the cost of skipping merging is less than the cost of
875 // merging : Cost(skipping merging) < Cost(merging BB), where the
876 // Cost(skipping merging) is Freq(BB) * (Cost(Copy) + Cost(Branch)), and
877 // the Cost(merging BB) is Freq(Pred) * Cost(Copy).
878 // Assuming Cost(Copy) == Cost(Branch), we could simplify it to :
879 // Freq(Pred) / Freq(BB) > 2.
880 // Note that if there are multiple empty blocks sharing the same incoming
881 // value for the PHIs in the DestBB, we consider them together. In such
882 // case, Cost(merging BB) will be the sum of their frequencies.
883
884 if (!isa<PHINode>(DestBB->begin()))
885 return true;
886
887 SmallPtrSet<BasicBlock *, 16> SameIncomingValueBBs;
888
889 // Find all other incoming blocks from which incoming values of all PHIs in
890 // DestBB are the same as the ones from BB.
891 for (BasicBlock *DestBBPred : predecessors(DestBB)) {
892 if (DestBBPred == BB)
893 continue;
894
895 if (llvm::all_of(DestBB->phis(), [&](const PHINode &DestPN) {
896 return DestPN.getIncomingValueForBlock(BB) ==
897 DestPN.getIncomingValueForBlock(DestBBPred);
898 }))
899 SameIncomingValueBBs.insert(DestBBPred);
900 }
901
902 // See if all BB's incoming values are same as the value from Pred. In this
903 // case, no reason to skip merging because COPYs are expected to be place in
904 // Pred already.
905 if (SameIncomingValueBBs.count(Pred))
906 return true;
907
908 BlockFrequency PredFreq = BFI->getBlockFreq(Pred);
909 BlockFrequency BBFreq = BFI->getBlockFreq(BB);
910
911 for (auto *SameValueBB : SameIncomingValueBBs)
912 if (SameValueBB->getUniquePredecessor() == Pred &&
913 DestBB == findDestBlockOfMergeableEmptyBlock(SameValueBB))
914 BBFreq += BFI->getBlockFreq(SameValueBB);
915
916 std::optional<BlockFrequency> Limit =
917 BBFreq.mul(Opts.cgp_freq_ratio_to_skip_merge);
918 return !Limit || PredFreq <= *Limit;
919}
920
921/// Return true if we can merge BB into DestBB if there is a single
922/// unconditional branch between them, and BB contains no other non-phi
923/// instructions.
924bool CodeGenPrepare::canMergeBlocks(const BasicBlock *BB,
925 const BasicBlock *DestBB) const {
926 // We only want to eliminate blocks whose phi nodes are used by phi nodes in
927 // the successor. If there are more complex condition (e.g. preheaders),
928 // don't mess around with them.
929 for (const PHINode &PN : BB->phis()) {
930 for (const User *U : PN.users()) {
931 const Instruction *UI = cast<Instruction>(U);
932 if (UI->getParent() != DestBB || !isa<PHINode>(UI))
933 return false;
934 // If User is inside DestBB block and it is a PHINode then check
935 // incoming value. If incoming value is not from BB then this is
936 // a complex condition (e.g. preheaders) we want to avoid here.
937 if (UI->getParent() == DestBB) {
938 if (const PHINode *UPN = dyn_cast<PHINode>(UI))
939 for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) {
940 Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I));
941 if (Insn && Insn->getParent() == BB &&
942 Insn->getParent() != UPN->getIncomingBlock(I))
943 return false;
944 }
945 }
946 }
947 }
948
949 // If BB and DestBB contain any common predecessors, then the phi nodes in BB
950 // and DestBB may have conflicting incoming values for the block. If so, we
951 // can't merge the block.
952 const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin());
953 if (!DestBBPN)
954 return true; // no conflict.
955
956 // Collect the preds of BB.
957 SmallPtrSet<const BasicBlock *, 16> BBPreds;
958 if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
959 // It is faster to get preds from a PHI than with pred_iterator.
960 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
961 BBPreds.insert(BBPN->getIncomingBlock(i));
962 } else {
963 BBPreds.insert_range(predecessors(BB));
964 }
965
966 // Walk the preds of DestBB.
967 for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) {
968 BasicBlock *Pred = DestBBPN->getIncomingBlock(i);
969 if (BBPreds.count(Pred)) { // Common predecessor?
970 for (const PHINode &PN : DestBB->phis()) {
971 const Value *V1 = PN.getIncomingValueForBlock(Pred);
972 const Value *V2 = PN.getIncomingValueForBlock(BB);
973
974 // If V2 is a phi node in BB, look up what the mapped value will be.
975 if (const PHINode *V2PN = dyn_cast<PHINode>(V2))
976 if (V2PN->getParent() == BB)
977 V2 = V2PN->getIncomingValueForBlock(Pred);
978
979 // If there is a conflict, bail out.
980 if (V1 != V2)
981 return false;
982 }
983 }
984 }
985
986 return true;
987}
988
989/// Replace all old uses with new ones, and push the updated BBs into FreshBBs.
990static void replaceAllUsesWith(Value *Old, Value *New,
992 bool IsHuge) {
993 auto *OldI = dyn_cast<Instruction>(Old);
994 if (OldI) {
995 for (Instruction::user_iterator UI = OldI->user_begin(),
996 E = OldI->user_end();
997 UI != E; ++UI) {
999 if (IsHuge)
1000 FreshBBs.insert(User->getParent());
1001 }
1002 }
1003 Old->replaceAllUsesWith(New);
1004}
1005
1006/// Eliminate a basic block that has only phi's and an unconditional branch in
1007/// it.
1008/// Indicate that the LoopInfo was modified only if it wasn't updated.
1009bool CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
1010 UncondBrInst *BI = cast<UncondBrInst>(BB->getTerminator());
1011 BasicBlock *DestBB = BI->getSuccessor();
1012
1013 LLVM_DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n"
1014 << *BB << *DestBB);
1015
1016 // If the destination block has a single pred, then this is a trivial edge,
1017 // just collapse it.
1018 if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) {
1019 if (SinglePred != DestBB) {
1020 assert(SinglePred == BB &&
1021 "Single predecessor not the same as predecessor");
1022 // Merge DestBB into SinglePred/BB and delete it.
1023 MergeBlockIntoPredecessor(DestBB, DTU, LI);
1024 // Note: BB(=SinglePred) will not be deleted on this path.
1025 // DestBB(=its single successor) is the one that was deleted.
1026 LLVM_DEBUG(dbgs() << "AFTER:\n" << *SinglePred << "\n\n\n");
1027
1028 if (IsHugeFunc) {
1029 // Update FreshBBs to optimize the merged BB.
1030 FreshBBs.insert(SinglePred);
1031 FreshBBs.erase(DestBB);
1032 }
1033 return false;
1034 }
1035 }
1036
1037 // Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB
1038 // to handle the new incoming edges it is about to have.
1039 for (PHINode &PN : DestBB->phis()) {
1040 // Remove the incoming value for BB, and remember it.
1041 Value *InVal = PN.removeIncomingValue(BB, false);
1042
1043 // Two options: either the InVal is a phi node defined in BB or it is some
1044 // value that dominates BB.
1045 PHINode *InValPhi = dyn_cast<PHINode>(InVal);
1046 if (InValPhi && InValPhi->getParent() == BB) {
1047 // Add all of the input values of the input PHI as inputs of this phi.
1048 for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i)
1049 PN.addIncoming(InValPhi->getIncomingValue(i),
1050 InValPhi->getIncomingBlock(i));
1051 } else {
1052 // Otherwise, add one instance of the dominating value for each edge that
1053 // we will be adding.
1054 if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
1055 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
1056 PN.addIncoming(InVal, BBPN->getIncomingBlock(i));
1057 } else {
1058 for (BasicBlock *Pred : predecessors(BB))
1059 PN.addIncoming(InVal, Pred);
1060 }
1061 }
1062 }
1063
1064 // Preserve loop Metadata.
1065 if (BI->hasMetadata(LLVMContext::MD_loop)) {
1066 for (auto *Pred : predecessors(BB))
1067 Pred->getTerminator()->copyMetadata(*BI, LLVMContext::MD_loop);
1068 }
1069
1070 // The PHIs are now updated, change everything that refers to BB to use
1071 // DestBB and remove BB.
1073 SmallPtrSet<BasicBlock *, 8> SeenPreds;
1074 SmallPtrSet<BasicBlock *, 8> PredOfDestBB(llvm::from_range,
1075 predecessors(DestBB));
1076 for (auto *Pred : predecessors(BB)) {
1077 if (!PredOfDestBB.contains(Pred)) {
1078 if (SeenPreds.insert(Pred).second)
1079 DTUpdates.push_back({DominatorTree::Insert, Pred, DestBB});
1080 }
1081 }
1082 SeenPreds.clear();
1083 for (auto *Pred : predecessors(BB)) {
1084 if (SeenPreds.insert(Pred).second)
1085 DTUpdates.push_back({DominatorTree::Delete, Pred, BB});
1086 }
1087 DTUpdates.push_back({DominatorTree::Delete, BB, DestBB});
1088 BB->replaceAllUsesWith(DestBB);
1089 DTU->applyUpdates(DTUpdates);
1090 DTU->deleteBB(BB);
1091 ++NumBlocksElim;
1092
1093 LLVM_DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");
1094 return true;
1095}
1096
1097// Computes a map of base pointer relocation instructions to corresponding
1098// derived pointer relocation instructions given a vector of all relocate calls
1100 const SmallVectorImpl<GCRelocateInst *> &AllRelocateCalls,
1102 &RelocateInstMap) {
1103 // Collect information in two maps: one primarily for locating the base object
1104 // while filling the second map; the second map is the final structure holding
1105 // a mapping between Base and corresponding Derived relocate calls
1107 for (auto *ThisRelocate : AllRelocateCalls) {
1108 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),
1109 ThisRelocate->getDerivedPtrIndex());
1110 RelocateIdxMap.insert(std::make_pair(K, ThisRelocate));
1111 }
1112 for (auto &Item : RelocateIdxMap) {
1113 std::pair<unsigned, unsigned> Key = Item.first;
1114 if (Key.first == Key.second)
1115 // Base relocation: nothing to insert
1116 continue;
1117
1118 GCRelocateInst *I = Item.second;
1119 auto BaseKey = std::make_pair(Key.first, Key.first);
1120
1121 // We're iterating over RelocateIdxMap so we cannot modify it.
1122 auto MaybeBase = RelocateIdxMap.find(BaseKey);
1123 if (MaybeBase == RelocateIdxMap.end())
1124 // TODO: We might want to insert a new base object relocate and gep off
1125 // that, if there are enough derived object relocates.
1126 continue;
1127
1128 RelocateInstMap[MaybeBase->second].push_back(I);
1129 }
1130}
1131
1132// Accepts a GEP and extracts the operands into a vector provided they're all
1133// small integer constants
1135 SmallVectorImpl<Value *> &OffsetV) {
1136 for (unsigned i = 1; i < GEP->getNumOperands(); i++) {
1137 // Only accept small constant integer operands
1138 auto *Op = dyn_cast<ConstantInt>(GEP->getOperand(i));
1139 if (!Op || Op->getZExtValue() > 20)
1140 return false;
1141 }
1142
1143 for (unsigned i = 1; i < GEP->getNumOperands(); i++)
1144 OffsetV.push_back(GEP->getOperand(i));
1145 return true;
1146}
1147
1148// Takes a RelocatedBase (base pointer relocation instruction) and Targets to
1149// replace, computes a replacement, and affects it.
1150static bool
1152 const SmallVectorImpl<GCRelocateInst *> &Targets) {
1153 bool MadeChange = false;
1154 // We must ensure the relocation of derived pointer is defined after
1155 // relocation of base pointer. If we find a relocation corresponding to base
1156 // defined earlier than relocation of base then we move relocation of base
1157 // right before found relocation. We consider only relocation in the same
1158 // basic block as relocation of base. Relocations from other basic block will
1159 // be skipped by optimization and we do not care about them.
1160 for (auto R = RelocatedBase->getParent()->getFirstInsertionPt();
1161 &*R != RelocatedBase; ++R)
1162 if (auto *RI = dyn_cast<GCRelocateInst>(R))
1163 if (RI->getStatepoint() == RelocatedBase->getStatepoint())
1164 if (RI->getBasePtrIndex() == RelocatedBase->getBasePtrIndex()) {
1165 RelocatedBase->moveBefore(RI->getIterator());
1166 MadeChange = true;
1167 break;
1168 }
1169
1170 for (GCRelocateInst *ToReplace : Targets) {
1171 assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() &&
1172 "Not relocating a derived object of the original base object");
1173 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {
1174 // A duplicate relocate call. TODO: coalesce duplicates.
1175 continue;
1176 }
1177
1178 if (RelocatedBase->getParent() != ToReplace->getParent()) {
1179 // Base and derived relocates are in different basic blocks.
1180 // In this case transform is only valid when base dominates derived
1181 // relocate. However it would be too expensive to check dominance
1182 // for each such relocate, so we skip the whole transformation.
1183 continue;
1184 }
1185
1186 Value *Base = ToReplace->getBasePtr();
1187 auto *Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr());
1188 if (!Derived || Derived->getPointerOperand() != Base)
1189 continue;
1190
1192 if (!getGEPSmallConstantIntOffsetV(Derived, OffsetV))
1193 continue;
1194
1195 // Create a Builder and replace the target callsite with a gep
1196 assert(RelocatedBase->getNextNode() &&
1197 "Should always have one since it's not a terminator");
1198
1199 // Insert after RelocatedBase
1200 IRBuilder<> Builder(RelocatedBase->getNextNode());
1201 Builder.disableGEPCanonicalization();
1202 Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
1203
1204 Value *Replacement = Builder.CreateGEP(Derived->getSourceElementType(),
1205 RelocatedBase, ArrayRef(OffsetV));
1206 Replacement->takeName(ToReplace);
1207 ToReplace->replaceAllUsesWith(Replacement);
1208 ToReplace->eraseFromParent();
1209
1210 MadeChange = true;
1211 }
1212 return MadeChange;
1213}
1214
1215// Turns this:
1216//
1217// %base = ...
1218// %ptr = gep %base + 15
1219// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
1220// %base' = relocate(%tok, i32 4, i32 4)
1221// %ptr' = relocate(%tok, i32 4, i32 5)
1222// %val = load %ptr'
1223//
1224// into this:
1225//
1226// %base = ...
1227// %ptr = gep %base + 15
1228// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
1229// %base' = gc.relocate(%tok, i32 4, i32 4)
1230// %ptr' = gep %base' + 15
1231// %val = load %ptr'
1232bool CodeGenPrepare::simplifyOffsetableRelocate(GCStatepointInst &I) {
1233 bool MadeChange = false;
1234 SmallVector<GCRelocateInst *, 2> AllRelocateCalls;
1235 for (auto *U : I.users())
1236 if (GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U))
1237 // Collect all the relocate calls associated with a statepoint
1238 AllRelocateCalls.push_back(Relocate);
1239
1240 // We need at least one base pointer relocation + one derived pointer
1241 // relocation to mangle
1242 if (AllRelocateCalls.size() < 2)
1243 return false;
1244
1245 // RelocateInstMap is a mapping from the base relocate instruction to the
1246 // corresponding derived relocate instructions
1247 MapVector<GCRelocateInst *, SmallVector<GCRelocateInst *, 0>> RelocateInstMap;
1248 computeBaseDerivedRelocateMap(AllRelocateCalls, RelocateInstMap);
1249 if (RelocateInstMap.empty())
1250 return false;
1251
1252 for (auto &Item : RelocateInstMap)
1253 // Item.first is the RelocatedBase to offset against
1254 // Item.second is the vector of Targets to replace
1255 MadeChange = simplifyRelocatesOffABase(Item.first, Item.second);
1256 return MadeChange;
1257}
1258
1259/// Sink the specified cast instruction into its user blocks.
1260static bool SinkCast(CastInst *CI) {
1261 BasicBlock *DefBB = CI->getParent();
1262
1263 /// InsertedCasts - Only insert a cast in each block once.
1265
1266 bool MadeChange = false;
1267 for (Instruction::user_iterator UI = CI->user_begin(), E = CI->user_end();
1268 UI != E;) {
1269 Use &TheUse = UI.getUse();
1271
1272 // Figure out which BB this cast is used in. For PHI's this is the
1273 // appropriate predecessor block.
1274 BasicBlock *UserBB = User->getParent();
1275 if (PHINode *PN = dyn_cast<PHINode>(User)) {
1276 UserBB = PN->getIncomingBlock(TheUse);
1277 }
1278
1279 // Preincrement use iterator so we don't invalidate it.
1280 ++UI;
1281
1282 // The first insertion point of a block containing an EH pad is after the
1283 // pad. If the pad is the user, we cannot sink the cast past the pad.
1284 if (User->isEHPad())
1285 continue;
1286
1287 // If the block selected to receive the cast is an EH pad that does not
1288 // allow non-PHI instructions before the terminator, we can't sink the
1289 // cast.
1290 if (UserBB->getTerminator()->isEHPad())
1291 continue;
1292
1293 // If this user is in the same block as the cast, don't change the cast.
1294 if (UserBB == DefBB)
1295 continue;
1296
1297 // If we have already inserted a cast into this block, use it.
1298 CastInst *&InsertedCast = InsertedCasts[UserBB];
1299
1300 if (!InsertedCast) {
1301 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
1302 assert(InsertPt != UserBB->end());
1303 InsertedCast = cast<CastInst>(CI->clone());
1304 InsertedCast->insertBefore(*UserBB, InsertPt);
1305 }
1306
1307 // Replace a use of the cast with a use of the new cast.
1308 TheUse = InsertedCast;
1309 MadeChange = true;
1310 ++NumCastUses;
1311 }
1312
1313 // If we removed all uses, nuke the cast.
1314 if (CI->use_empty()) {
1315 salvageDebugInfo(*CI);
1316 CI->eraseFromParent();
1317 MadeChange = true;
1318 }
1319
1320 return MadeChange;
1321}
1322
1323/// Hoists bitcasts to the source block to reduce register pressure
1325 const DataLayout &DL) {
1326 auto *SrcInst = dyn_cast<Instruction>(BCI->getOperand(0));
1327 if (!SrcInst || SrcInst->getParent() == BCI->getParent() ||
1328 SrcInst->isTerminator())
1329 return false;
1330
1331 Type *DestTy = BCI->getType();
1332 Type *SrcTy = SrcInst->getType();
1333 EVT SrcVT = TLI.getValueType(DL, SrcTy);
1334 EVT DestVT = TLI.getValueType(DL, DestTy);
1335
1336 // Bail out on scalable vectors and illegal destination types
1337 if (SrcVT.isScalableVector() || DestVT.isScalableVector())
1338 return false;
1339
1340 // Only hoist if it reduces physical register count
1341 if (TLI.getNumRegisters(BCI->getContext(), SrcVT) <=
1342 TLI.getNumRegisters(BCI->getContext(), DestVT))
1343 return false;
1344
1345 // Block large or cross-domain scalars to prevent spills and broken atomics.
1346 bool IsCrossDomain = DestTy->isFPOrFPVectorTy() != SrcTy->isFPOrFPVectorTy();
1347
1348 // A scalar is large if it requires more than one native register.
1349 unsigned NativeWidth = DL.getPointerSizeInBits();
1350 bool IsLargeScalar =
1351 !DestTy->isVectorTy() &&
1352 DL.getTypeSizeInBits(DestTy).getFixedValue() > NativeWidth;
1353
1354 if (IsCrossDomain || IsLargeScalar)
1355 return false;
1356
1357 // Hoist the bitcast
1358 BasicBlock *SrcBB = SrcInst->getParent();
1359 auto InsertPt = isa<PHINode>(SrcInst) ? SrcBB->getFirstInsertionPt()
1360 : std::next(SrcInst->getIterator());
1361 BCI->moveBefore(*SrcBB, InsertPt);
1362
1363 return true;
1364}
1365
1366/// If the specified cast instruction is a noop copy (e.g. it's casting from
1367/// one pointer type to another, i32->i8 on PPC), sink it into user blocks to
1368/// reduce the number of virtual registers that must be created and coalesced.
1369///
1370/// Return true if any changes are made.
1372 const DataLayout &DL) {
1373 // Sink only "cheap" (or nop) address-space casts. This is a weaker condition
1374 // than sinking only nop casts, but is helpful on some platforms.
1375 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(CI)) {
1376 if (!TLI.isFreeAddrSpaceCast(DL, ASC->getSrcAddressSpace(),
1377 ASC->getDestAddressSpace()))
1378 return false;
1379 }
1380
1381 // If this is a noop copy,
1382 EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType());
1383 EVT DstVT = TLI.getValueType(DL, CI->getType());
1384
1385 // This is an fp<->int conversion?
1386 if (SrcVT.isInteger() != DstVT.isInteger())
1387 return false;
1388
1389 // If this is an extension, it will be a zero or sign extension, which
1390 // isn't a noop.
1391 if (SrcVT.bitsLT(DstVT))
1392 return false;
1393
1394 // If these values will be promoted, find out what they will be promoted
1395 // to. This helps us consider truncates on PPC as noop copies when they
1396 // are.
1397 if (TLI.getTypeAction(CI->getContext(), SrcVT) ==
1399 SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT);
1400 if (TLI.getTypeAction(CI->getContext(), DstVT) ==
1402 DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT);
1403
1404 // If, after promotion, these are the same types, this is a noop copy.
1405 if (SrcVT != DstVT)
1406 return false;
1407
1408 return SinkCast(CI);
1409}
1410
1411// Match a simple increment by constant operation. Note that if a sub is
1412// matched, the step is negated (as if the step had been canonicalized to
1413// an add, even though we leave the instruction alone.)
1414static bool matchIncrement(const Instruction *IVInc, Instruction *&LHS,
1415 Constant *&Step) {
1416 if (match(IVInc, m_Add(m_Instruction(LHS), m_Constant(Step))) ||
1418 m_Instruction(LHS), m_Constant(Step)))))
1419 return true;
1420 if (match(IVInc, m_Sub(m_Instruction(LHS), m_Constant(Step))) ||
1422 m_Instruction(LHS), m_Constant(Step))))) {
1423 Step = ConstantExpr::getNeg(Step);
1424 return true;
1425 }
1426 return false;
1427}
1428
1429/// If given \p PN is an inductive variable with value IVInc coming from the
1430/// backedge, and on each iteration it gets increased by Step, return pair
1431/// <IVInc, Step>. Otherwise, return std::nullopt.
1432static std::optional<std::pair<Instruction *, Constant *>>
1433getIVIncrement(const PHINode *PN, const LoopInfo *LI) {
1434 const Loop *L = LI->getLoopFor(PN->getParent());
1435 if (!L || L->getHeader() != PN->getParent() || !L->getLoopLatch())
1436 return std::nullopt;
1437 auto *IVInc =
1438 dyn_cast<Instruction>(PN->getIncomingValueForBlock(L->getLoopLatch()));
1439 if (!IVInc || LI->getLoopFor(IVInc->getParent()) != L)
1440 return std::nullopt;
1441 Instruction *LHS = nullptr;
1442 Constant *Step = nullptr;
1443 if (matchIncrement(IVInc, LHS, Step) && LHS == PN)
1444 return std::make_pair(IVInc, Step);
1445 return std::nullopt;
1446}
1447
1448static bool isIVIncrement(const Value *V, const LoopInfo *LI) {
1449 auto *I = dyn_cast<Instruction>(V);
1450 if (!I)
1451 return false;
1452 Instruction *LHS = nullptr;
1453 Constant *Step = nullptr;
1454 if (!matchIncrement(I, LHS, Step))
1455 return false;
1456 if (auto *PN = dyn_cast<PHINode>(LHS))
1457 if (auto IVInc = getIVIncrement(PN, LI))
1458 return IVInc->first == I;
1459 return false;
1460}
1461
1462bool CodeGenPrepare::replaceMathCmpWithIntrinsic(BinaryOperator *BO,
1463 Value *Arg0, Value *Arg1,
1464 CmpInst *Cmp,
1465 Intrinsic::ID IID) {
1466 auto IsReplacableIVIncrement = [this, &Cmp](BinaryOperator *BO) {
1467 if (!isIVIncrement(BO, LI))
1468 return false;
1469 const Loop *L = LI->getLoopFor(BO->getParent());
1470 assert(L && "L should not be null after isIVIncrement()");
1471 // Do not risk on moving increment into a child loop.
1472 if (LI->getLoopFor(Cmp->getParent()) != L)
1473 return false;
1474
1475 // Finally, we need to ensure that the insert point will dominate all
1476 // existing uses of the increment.
1477
1478 auto &DT = getDT();
1479 if (DT.dominates(Cmp->getParent(), BO->getParent()))
1480 // If we're moving up the dom tree, all uses are trivially dominated.
1481 // (This is the common case for code produced by LSR.)
1482 return true;
1483
1484 // Otherwise, special case the single use in the phi recurrence.
1485 return BO->hasOneUse() && DT.dominates(Cmp->getParent(), L->getLoopLatch());
1486 };
1487 if (BO->getParent() != Cmp->getParent() && !IsReplacableIVIncrement(BO)) {
1488 // We used to use a dominator tree here to allow multi-block optimization.
1489 // But that was problematic because:
1490 // 1. It could cause a perf regression by hoisting the math op into the
1491 // critical path.
1492 // 2. It could cause a perf regression by creating a value that was live
1493 // across multiple blocks and increasing register pressure.
1494 // 3. Use of a dominator tree could cause large compile-time regression.
1495 // This is because we recompute the DT on every change in the main CGP
1496 // run-loop. The recomputing is probably unnecessary in many cases, so if
1497 // that was fixed, using a DT here would be ok.
1498 //
1499 // There is one important particular case we still want to handle: if BO is
1500 // the IV increment. Important properties that make it profitable:
1501 // - We can speculate IV increment anywhere in the loop (as long as the
1502 // indvar Phi is its only user);
1503 // - Upon computing Cmp, we effectively compute something equivalent to the
1504 // IV increment (despite it loops differently in the IR). So moving it up
1505 // to the cmp point does not really increase register pressure.
1506 return false;
1507 }
1508
1509 // We allow matching the canonical IR (add X, C) back to (usubo X, -C).
1510 if (BO->getOpcode() == Instruction::Add &&
1511 IID == Intrinsic::usub_with_overflow) {
1512 assert(isa<Constant>(Arg1) && "Unexpected input for usubo");
1514 }
1515
1516 // Insert at the first instruction of the pair.
1517 Instruction *InsertPt = nullptr;
1518 for (Instruction &Iter : *Cmp->getParent()) {
1519 // If BO is an XOR, it is not guaranteed that it comes after both inputs to
1520 // the overflow intrinsic are defined.
1521 if ((BO->getOpcode() != Instruction::Xor && &Iter == BO) || &Iter == Cmp) {
1522 InsertPt = &Iter;
1523 break;
1524 }
1525 }
1526 assert(InsertPt != nullptr && "Parent block did not contain cmp or binop");
1527
1528 IRBuilder<> Builder(InsertPt);
1529 Value *MathOV = Builder.CreateBinaryIntrinsic(IID, Arg0, Arg1);
1530 if (BO->getOpcode() != Instruction::Xor) {
1531 Value *Math = Builder.CreateExtractValue(MathOV, 0, "math");
1532 replaceAllUsesWith(BO, Math, FreshBBs, IsHugeFunc);
1533 } else
1534 assert(BO->hasOneUse() &&
1535 "Patterns with XOr should use the BO only in the compare");
1536 Value *OV = Builder.CreateExtractValue(MathOV, 1, "ov");
1537 replaceAllUsesWith(Cmp, OV, FreshBBs, IsHugeFunc);
1538 Cmp->eraseFromParent();
1539 BO->eraseFromParent();
1540 return true;
1541}
1542
1543/// Match special-case patterns that check for unsigned add overflow.
1545 BinaryOperator *&Add) {
1546 // Add = add A, 1; Cmp = icmp eq A,-1 (overflow if A is max val)
1547 // Add = add A,-1; Cmp = icmp ne A, 0 (overflow if A is non-zero)
1548 Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1);
1549
1550 // We are not expecting non-canonical/degenerate code. Just bail out.
1551 if (isa<Constant>(A))
1552 return false;
1553
1554 ICmpInst::Predicate Pred = Cmp->getPredicate();
1555 if (Pred == ICmpInst::ICMP_EQ && match(B, m_AllOnes()))
1556 B = ConstantInt::get(B->getType(), 1);
1557 else if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt()))
1558 B = Constant::getAllOnesValue(B->getType());
1559 else
1560 return false;
1561
1562 // Check the users of the variable operand of the compare looking for an add
1563 // with the adjusted constant.
1564 for (User *U : A->users()) {
1565 if (match(U, m_Add(m_Specific(A), m_Specific(B)))) {
1567 return true;
1568 }
1569 }
1570 return false;
1571}
1572
1573/// Try to combine the compare into a call to the llvm.uadd.with.overflow
1574/// intrinsic. Return true if any changes were made.
1575bool CodeGenPrepare::combineToUAddWithOverflow(CmpInst *Cmp,
1576 ModifyDT &ModifiedDT) {
1577 bool EdgeCase = false;
1578 Value *A, *B;
1579 BinaryOperator *Add;
1580 if (!match(Cmp, m_UAddWithOverflow(m_Value(A), m_Value(B), m_BinOp(Add)))) {
1582 return false;
1583 // Set A and B in case we match matchUAddWithOverflowConstantEdgeCases.
1584 A = Add->getOperand(0);
1585 B = Add->getOperand(1);
1586 EdgeCase = true;
1587 }
1588
1590 TLI->getValueType(*DL, Add->getType()),
1591 Add->hasNUsesOrMore(EdgeCase ? 1 : 2)))
1592 return false;
1593
1594 // We don't want to move around uses of condition values this late, so we
1595 // check if it is legal to create the call to the intrinsic in the basic
1596 // block containing the icmp.
1597 if (Add->getParent() != Cmp->getParent() && !Add->hasOneUse())
1598 return false;
1599
1600 if (!replaceMathCmpWithIntrinsic(Add, A, B, Cmp,
1601 Intrinsic::uadd_with_overflow))
1602 return false;
1603
1604 // Reset callers - do not crash by iterating over a dead instruction.
1605 ModifiedDT = ModifyDT::ModifyInstDT;
1606 return true;
1607}
1608
1609bool CodeGenPrepare::combineToUSubWithOverflow(CmpInst *Cmp,
1610 ModifyDT &ModifiedDT) {
1611 // We are not expecting non-canonical/degenerate code. Just bail out.
1612 Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1);
1613 if (isa<Constant>(A) && isa<Constant>(B))
1614 return false;
1615
1616 // Convert (A u> B) to (A u< B) to simplify pattern matching.
1617 ICmpInst::Predicate Pred = Cmp->getPredicate();
1618 if (Pred == ICmpInst::ICMP_UGT) {
1619 std::swap(A, B);
1620 Pred = ICmpInst::ICMP_ULT;
1621 }
1622 // Convert special-case: (A == 0) is the same as (A u< 1).
1623 if (Pred == ICmpInst::ICMP_EQ && match(B, m_ZeroInt())) {
1624 B = ConstantInt::get(B->getType(), 1);
1625 Pred = ICmpInst::ICMP_ULT;
1626 }
1627 // Convert special-case: (A != 0) is the same as (0 u< A).
1628 if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt())) {
1629 std::swap(A, B);
1630 Pred = ICmpInst::ICMP_ULT;
1631 }
1632 if (Pred != ICmpInst::ICMP_ULT)
1633 return false;
1634
1635 // Walk the users of a variable operand of a compare looking for a subtract or
1636 // add with that same operand. Also match the 2nd operand of the compare to
1637 // the add/sub, but that may be a negated constant operand of an add.
1638 Value *CmpVariableOperand = isa<Constant>(A) ? B : A;
1639 BinaryOperator *Sub = nullptr;
1640 for (User *U : CmpVariableOperand->users()) {
1641 // A - B, A u< B --> usubo(A, B)
1642 if (match(U, m_Sub(m_Specific(A), m_Specific(B)))) {
1644 break;
1645 }
1646
1647 // A + (-C), A u< C (canonicalized form of (sub A, C))
1648 const APInt *CmpC, *AddC;
1649 if (match(U, m_Add(m_Specific(A), m_APInt(AddC))) &&
1650 match(B, m_APInt(CmpC)) && *AddC == -(*CmpC)) {
1652 break;
1653 }
1654 }
1655 if (!Sub)
1656 return false;
1657
1659 TLI->getValueType(*DL, Sub->getType()),
1660 Sub->hasNUsesOrMore(1)))
1661 return false;
1662
1663 // We don't want to move around uses of condition values this late, so we
1664 // check if it is legal to create the call to the intrinsic in the basic
1665 // block containing the icmp.
1666 if (Sub->getParent() != Cmp->getParent() && !Sub->hasOneUse())
1667 return false;
1668
1669 if (!replaceMathCmpWithIntrinsic(Sub, Sub->getOperand(0), Sub->getOperand(1),
1670 Cmp, Intrinsic::usub_with_overflow))
1671 return false;
1672
1673 // Reset callers - do not crash by iterating over a dead instruction.
1674 ModifiedDT = ModifyDT::ModifyInstDT;
1675 return true;
1676}
1677
1678// Decanonicalizes icmp+ctpop power-of-two test if ctpop is slow.
1679// The same transformation exists in DAG combiner, but we repeat it here because
1680// DAG builder can break the pattern by moving icmp into a successor block.
1681bool CodeGenPrepare::unfoldPowerOf2Test(CmpInst *Cmp) {
1682 CmpPredicate Pred;
1683 Value *X;
1684 const APInt *C;
1685
1686 // (icmp (ctpop x), c)
1688 return false;
1689
1690 // We're only interested in "is power of 2 [or zero]" patterns.
1691 bool IsStrictlyPowerOf2Test = ICmpInst::isEquality(Pred) && *C == 1;
1692 bool IsPowerOf2OrZeroTest = (Pred == CmpInst::ICMP_ULT && *C == 2) ||
1693 (Pred == CmpInst::ICMP_UGT && *C == 1);
1694 if (!IsStrictlyPowerOf2Test && !IsPowerOf2OrZeroTest)
1695 return false;
1696
1697 // Some targets have better codegen for `ctpop(x) u</u>= 2/1`than for
1698 // `ctpop(x) ==/!= 1`. If ctpop is fast, only try changing the comparison,
1699 // and otherwise expand ctpop into a few simple instructions.
1700 Type *OpTy = X->getType();
1701 if (TLI->isCtpopFast(TLI->getValueType(*DL, OpTy))) {
1702 // Look for `ctpop(x) ==/!= 1`, where `ctpop(x)` is known to be non-zero.
1703 if (!IsStrictlyPowerOf2Test || !isKnownNonZero(Cmp->getOperand(0), *DL))
1704 return false;
1705
1706 // ctpop(x) == 1 -> ctpop(x) u< 2
1707 // ctpop(x) != 1 -> ctpop(x) u> 1
1708 if (Pred == ICmpInst::ICMP_EQ) {
1709 Cmp->setOperand(1, ConstantInt::get(OpTy, 2));
1710 Cmp->setPredicate(ICmpInst::ICMP_ULT);
1711 } else {
1712 Cmp->setPredicate(ICmpInst::ICMP_UGT);
1713 }
1714 return true;
1715 }
1716
1717 Value *NewCmp;
1718 if (IsPowerOf2OrZeroTest ||
1719 (IsStrictlyPowerOf2Test && isKnownNonZero(Cmp->getOperand(0), *DL))) {
1720 // ctpop(x) u< 2 -> (x & (x - 1)) == 0
1721 // ctpop(x) u> 1 -> (x & (x - 1)) != 0
1722 IRBuilder<> Builder(Cmp);
1723 Value *Sub = Builder.CreateAdd(X, Constant::getAllOnesValue(OpTy));
1724 Value *And = Builder.CreateAnd(X, Sub);
1725 CmpInst::Predicate NewPred =
1728 : CmpInst::ICMP_NE;
1729 NewCmp = Builder.CreateICmp(NewPred, And, ConstantInt::getNullValue(OpTy));
1730 } else {
1731 // ctpop(x) == 1 -> (x ^ (x - 1)) u> (x - 1)
1732 // ctpop(x) != 1 -> (x ^ (x - 1)) u<= (x - 1)
1733 IRBuilder<> Builder(Cmp);
1734 Value *Sub = Builder.CreateAdd(X, Constant::getAllOnesValue(OpTy));
1735 Value *Xor = Builder.CreateXor(X, Sub);
1736 CmpInst::Predicate NewPred =
1738 NewCmp = Builder.CreateICmp(NewPred, Xor, Sub);
1739 }
1740
1741 Cmp->replaceAllUsesWith(NewCmp);
1743 return true;
1744}
1745
1746/// Sink the given CmpInst into user blocks to reduce the number of virtual
1747/// registers that must be created and coalesced. This is a clear win except on
1748/// targets with multiple condition code registers (PowerPC), where it might
1749/// lose; some adjustment may be wanted there.
1750///
1751/// Return true if any changes are made.
1753 const DataLayout &DL) {
1754 if (TLI.hasMultipleConditionRegisters(EVT::getEVT(Cmp->getType())))
1755 return false;
1756
1757 // Avoid sinking soft-FP comparisons, since this can move them into a loop.
1758 if (TLI.useSoftFloat() && isa<FCmpInst>(Cmp))
1759 return false;
1760
1761 bool UsedInPhiOrCurrentBlock = any_of(Cmp->users(), [Cmp](User *U) {
1762 return isa<PHINode>(U) ||
1763 cast<Instruction>(U)->getParent() == Cmp->getParent();
1764 });
1765
1766 // Avoid sinking larger than legal integer comparisons unless its ONLY used in
1767 // another BB.
1768 if (UsedInPhiOrCurrentBlock && Cmp->getOperand(0)->getType()->isIntegerTy() &&
1769 Cmp->getOperand(0)->getType()->getScalarSizeInBits() >
1770 DL.getLargestLegalIntTypeSizeInBits())
1771 return false;
1772
1773 // Only insert a cmp in each block once.
1775
1776 bool MadeChange = false;
1777 for (Instruction::user_iterator UI = Cmp->user_begin(), E = Cmp->user_end();
1778 UI != E;) {
1779 Use &TheUse = UI.getUse();
1781
1782 // Preincrement use iterator so we don't invalidate it.
1783 ++UI;
1784
1785 // Don't bother for PHI nodes.
1786 if (isa<PHINode>(User))
1787 continue;
1788
1789 // Figure out which BB this cmp is used in.
1790 BasicBlock *UserBB = User->getParent();
1791 BasicBlock *DefBB = Cmp->getParent();
1792
1793 // If this user is in the same block as the cmp, don't change the cmp.
1794 if (UserBB == DefBB)
1795 continue;
1796
1797 // If we have already inserted a cmp into this block, use it.
1798 CmpInst *&InsertedCmp = InsertedCmps[UserBB];
1799
1800 if (!InsertedCmp) {
1801 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
1802 assert(InsertPt != UserBB->end());
1803 InsertedCmp = CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(),
1804 Cmp->getOperand(0), Cmp->getOperand(1), "");
1805 InsertedCmp->insertBefore(*UserBB, InsertPt);
1806 // Propagate the debug info.
1807 InsertedCmp->setDebugLoc(Cmp->getDebugLoc());
1808 }
1809
1810 // Replace a use of the cmp with a use of the new cmp.
1811 TheUse = InsertedCmp;
1812 MadeChange = true;
1813 ++NumCmpUses;
1814 }
1815
1816 // If we removed all uses, nuke the cmp.
1817 if (Cmp->use_empty()) {
1818 Cmp->eraseFromParent();
1819 MadeChange = true;
1820 }
1821
1822 return MadeChange;
1823}
1824
1825/// For pattern like:
1826///
1827/// DomCond = icmp sgt/slt CmpOp0, CmpOp1 (might not be in DomBB)
1828/// ...
1829/// DomBB:
1830/// ...
1831/// br DomCond, TrueBB, CmpBB
1832/// CmpBB: (with DomBB being the single predecessor)
1833/// ...
1834/// Cmp = icmp eq CmpOp0, CmpOp1
1835/// ...
1836///
1837/// It would use two comparison on targets that lowering of icmp sgt/slt is
1838/// different from lowering of icmp eq (PowerPC). This function try to convert
1839/// 'Cmp = icmp eq CmpOp0, CmpOp1' to ' Cmp = icmp slt/sgt CmpOp0, CmpOp1'.
1840/// After that, DomCond and Cmp can use the same comparison so reduce one
1841/// comparison.
1842///
1843/// Return true if any changes are made.
1845 bool EnableICmpEqToICmpSt) {
1846 if (!EnableICmpEqToICmpSt && TLI.isEqualityCmpFoldedWithSignedCmp())
1847 return false;
1848
1849 ICmpInst::Predicate Pred = Cmp->getPredicate();
1850 if (Pred != ICmpInst::ICMP_EQ)
1851 return false;
1852
1853 // If icmp eq has users other than CondBrInst and SelectInst, converting it to
1854 // icmp slt/sgt would introduce more redundant LLVM IR.
1855 for (User *U : Cmp->users()) {
1856 if (isa<CondBrInst>(U))
1857 continue;
1858 if (isa<SelectInst>(U) && cast<SelectInst>(U)->getCondition() == Cmp)
1859 continue;
1860 return false;
1861 }
1862
1863 // This is a cheap/incomplete check for dominance - just match a single
1864 // predecessor with a conditional branch.
1865 BasicBlock *CmpBB = Cmp->getParent();
1866 BasicBlock *DomBB = CmpBB->getSinglePredecessor();
1867 if (!DomBB)
1868 return false;
1869
1870 // We want to ensure that the only way control gets to the comparison of
1871 // interest is that a less/greater than comparison on the same operands is
1872 // false.
1873 Value *DomCond;
1874 BasicBlock *TrueBB, *FalseBB;
1875 if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB)))
1876 return false;
1877 if (CmpBB != FalseBB)
1878 return false;
1879
1880 Value *CmpOp0 = Cmp->getOperand(0), *CmpOp1 = Cmp->getOperand(1);
1881 CmpPredicate DomPred;
1882 if (!match(DomCond, m_ICmp(DomPred, m_Specific(CmpOp0), m_Specific(CmpOp1))))
1883 return false;
1884 if (DomPred != ICmpInst::ICMP_SGT && DomPred != ICmpInst::ICMP_SLT)
1885 return false;
1886
1887 // Convert the equality comparison to the opposite of the dominating
1888 // comparison and swap the direction for all branch/select users.
1889 // We have conceptually converted:
1890 // Res = (a < b) ? <LT_RES> : (a == b) ? <EQ_RES> : <GT_RES>;
1891 // to
1892 // Res = (a < b) ? <LT_RES> : (a > b) ? <GT_RES> : <EQ_RES>;
1893 // And similarly for branches.
1894 for (User *U : Cmp->users()) {
1895 if (auto *BI = dyn_cast<CondBrInst>(U)) {
1896 BI->swapSuccessors();
1897 continue;
1898 }
1899 if (auto *SI = dyn_cast<SelectInst>(U)) {
1900 // Swap operands
1901 SI->swapValues();
1902 SI->swapProfMetadata();
1903 continue;
1904 }
1905 llvm_unreachable("Must be a branch or a select");
1906 }
1907 Cmp->setPredicate(CmpInst::getSwappedPredicate(DomPred));
1908 return true;
1909}
1910
1911/// Many architectures use the same instruction for both subtract and cmp. Try
1912/// to swap cmp operands to match subtract operations to allow for CSE.
1914 Value *Op0 = Cmp->getOperand(0);
1915 Value *Op1 = Cmp->getOperand(1);
1916 if (!Op0->getType()->isIntegerTy() || isa<Constant>(Op0) ||
1917 isa<Constant>(Op1) || Op0 == Op1)
1918 return false;
1919
1920 // If a subtract already has the same operands as a compare, swapping would be
1921 // bad. If a subtract has the same operands as a compare but in reverse order,
1922 // then swapping is good.
1923 int GoodToSwap = 0;
1924 unsigned NumInspected = 0;
1925 for (const User *U : Op0->users()) {
1926 // Avoid walking many users.
1927 if (++NumInspected > 128)
1928 return false;
1929 if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0))))
1930 GoodToSwap++;
1931 else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1))))
1932 GoodToSwap--;
1933 }
1934
1935 if (GoodToSwap > 0) {
1936 Cmp->swapOperands();
1937 return true;
1938 }
1939 return false;
1940}
1941
1943 const DataLayout &DL) {
1944 FCmpInst *FCmp = dyn_cast<FCmpInst>(Cmp);
1945 if (!FCmp)
1946 return false;
1947
1948 // Don't fold if the target offers free fabs and the predicate is legal.
1949 EVT VT = TLI.getValueType(DL, Cmp->getOperand(0)->getType());
1950 if (TLI.isFAbsFree(VT) &&
1951 TLI.isCondCodeLegal(getFCmpCondCode(FCmp->getPredicate()),
1952 VT.getSimpleVT()))
1953 return false;
1954
1955 // Reverse the canonicalization if it is a FP class test
1956 auto ShouldReverseTransform = [](FPClassTest ClassTest) {
1957 return ClassTest == fcInf || ClassTest == (fcInf | fcNan);
1958 };
1959 auto [ClassVal, ClassTest] =
1960 fcmpToClassTest(FCmp->getPredicate(), *FCmp->getParent()->getParent(),
1961 FCmp->getOperand(0), FCmp->getOperand(1));
1962 if (!ClassVal)
1963 return false;
1964
1965 if (!ShouldReverseTransform(ClassTest) && !ShouldReverseTransform(~ClassTest))
1966 return false;
1967
1968 IRBuilder<> Builder(Cmp);
1969 Value *IsFPClass = Builder.createIsFPClass(ClassVal, ClassTest);
1970 Cmp->replaceAllUsesWith(IsFPClass);
1972 return true;
1973}
1974
1976 Instruction *Rem, const LoopInfo *LI, Value *&RemAmtOut, Value *&AddInstOut,
1977 Value *&AddOffsetOut, PHINode *&LoopIncrPNOut, Constant *&StepOut) {
1978 Value *Incr, *RemAmt;
1979 // NB: If RemAmt is a power of 2 it *should* have been transformed by now.
1980 if (!match(Rem, m_URem(m_Value(Incr), m_Value(RemAmt))))
1981 return false;
1982
1983 Value *AddInst, *AddOffset;
1984 // Find out loop increment PHI.
1985 PHINode *PN = dyn_cast<PHINode>(Incr);
1986 if (PN != nullptr) {
1987 AddInst = nullptr;
1988 AddOffset = nullptr;
1989 } else {
1990 // Search through a NUW add (or disjoint or) on top of the loop increment.
1991 if (!match(Incr, m_c_NUWAddLike(m_Phi(PN), m_Value(AddOffset))))
1992 return false;
1993 AddInst = Incr;
1994 }
1995
1996 if (!PN)
1997 return false;
1998
1999 // This isn't strictly necessary, what we really need is one increment and any
2000 // amount of initial values all being the same.
2001 if (PN->getNumIncomingValues() != 2)
2002 return false;
2003
2004 // Only trivially analyzable loops.
2005 Loop *L = LI->getLoopFor(PN->getParent());
2006 if (!L || !L->getLoopPreheader() || !L->getLoopLatch())
2007 return false;
2008
2009 // Req that the remainder is in the loop
2010 if (!L->contains(Rem))
2011 return false;
2012
2013 // Only works if the remainder amount is a loop invaraint
2014 if (!L->isLoopInvariant(RemAmt))
2015 return false;
2016
2017 // Only works if the AddOffset is a loop invaraint
2018 if (AddOffset && !L->isLoopInvariant(AddOffset))
2019 return false;
2020
2021 // Is the PHI a loop increment?
2022 auto LoopIncrInfo = getIVIncrement(PN, LI);
2023 if (!LoopIncrInfo)
2024 return false;
2025
2026 // The remainder must wrap at most once per iteration. That needs a known
2027 // remainder amount unless the step is one.
2028 Constant *Step = LoopIncrInfo->second;
2029 if (!match(Step, m_One())) {
2030 auto *StepC = dyn_cast<ConstantInt>(Step);
2031 auto *RemC = dyn_cast<ConstantInt>(RemAmt);
2032 if (!StepC || !RemC || StepC->isZero() ||
2033 StepC->getValue().uge(RemC->getValue()))
2034 return false;
2035 // `rem + Step` must not overflow.
2036 bool Overflow;
2037 (void)(RemC->getValue() - 1).uadd_ov(StepC->getValue(), Overflow);
2038 if (Overflow)
2039 return false;
2040 }
2041
2042 // Need the increment to not overflow.
2043 if (!match(LoopIncrInfo->first, m_c_NUWAdd(m_Specific(PN), m_Value())))
2044 return false;
2045
2046 // Set output variables.
2047 RemAmtOut = RemAmt;
2048 LoopIncrPNOut = PN;
2049 AddInstOut = AddInst;
2050 AddOffsetOut = AddOffset;
2051 StepOut = Step;
2052
2053 return true;
2054}
2055
2056// Try to transform:
2057//
2058// for(i = Start; i < End; i += Step)
2059// Rem = (i nuw+ IncrLoopInvariant) u% RemAmtLoopInvariant;
2060//
2061// ->
2062//
2063// Rem = (Start nuw+ IncrLoopInvariant) % RemAmtLoopInvariant;
2064// for(i = Start; i < End; i += Step, rem += Step)
2065// Rem = rem u>= RemAmtLoopInvariant ? rem - RemAmtLoopInvariant : rem;
2066//
2067// Step must be 1 unless RemAmtLoopInvariant is a constant greater than Step.
2069 const LoopInfo *LI, DominatorTree &DT,
2071 bool IsHuge) {
2072 Value *AddOffset, *RemAmt, *AddInst;
2073 PHINode *LoopIncrPN;
2074 Constant *Step;
2075 if (!isRemOfLoopIncrementWithLoopInvariant(Rem, LI, RemAmt, AddInst,
2076 AddOffset, LoopIncrPN, Step))
2077 return false;
2078
2079 Loop *L = LI->getLoopFor(LoopIncrPN->getParent());
2080 Type *Ty = Rem->getType();
2081
2082 // `urem` of a constant ends up as `mul` + `shift` + `sub`, so the extra IV
2083 // is only worth it if the `urem` runs every iteration.
2084 //
2085 // Potential TODO: Should we have a check for how "nested" a non-constant
2086 // remainder operation is? The new code runs every iteration so if the
2087 // remainder is guarded behind unlikely conditions this might not be worth
2088 // it.
2089 bool IsConstRem = isa<ConstantInt>(RemAmt);
2090 if (match(RemAmt, m_ImmConstant())) {
2091 // A power-of-2 `urem` is a single mask.
2092 if (!IsConstRem || cast<ConstantInt>(RemAmt)->getValue().isPowerOf2() ||
2093 !DT.dominates(Rem->getParent(), L->getLoopLatch()))
2094 return false;
2095 }
2096
2097 Value *Start = LoopIncrPN->getIncomingValueForBlock(L->getLoopPreheader());
2098 // If we have add create initial value for remainder.
2099 // The logic here is:
2100 // (urem (add nuw Start, IncrLoopInvariant), RemAmtLoopInvariant
2101 //
2102 // For a non-constant remainder amount, only proceed if the expression
2103 // simplifies (otherwise we can't fully optimize out the urem). A constant
2104 // one is cheap enough to compute once in the preheader.
2105 IRBuilder<> PHBuilder(L->getLoopPreheader()->getTerminator());
2106 if (AddInst) {
2107 assert(AddOffset && "We found an add but missing values");
2108 // Without dom-condition/assumption cache we aren't likely to get much out
2109 // of a context instruction.
2110 Value *NewStart = simplifyAddInst(
2111 Start, AddOffset, match(AddInst, m_NSWAdd(m_Value(), m_Value())),
2112 /*IsNUW=*/true, *DL);
2113 if (!NewStart && !IsConstRem)
2114 return false;
2115 Start = NewStart ? NewStart : PHBuilder.CreateNUWAdd(Start, AddOffset);
2116 }
2117
2118 Value *StartRem = simplifyURemInst(Start, RemAmt, *DL);
2119 if (!StartRem) {
2120 // If we can't fully optimize out the `rem`, skip this transform.
2121 if (!IsConstRem)
2122 return false;
2123 StartRem = PHBuilder.CreateURem(Start, RemAmt);
2124 }
2125
2126 // Create new remainder with induction variable.
2127 IRBuilder<> Builder(LoopIncrPN);
2128 PHINode *NewRem = Builder.CreatePHI(Ty, 2);
2129
2130 Builder.SetInsertPoint(cast<Instruction>(
2131 LoopIncrPN->getIncomingValueForBlock(L->getLoopLatch())));
2132 // `(add (urem x, y), Step)` is nuw, see
2133 // isRemOfLoopIncrementWithLoopInvariant.
2134 Value *RemAdd = Builder.CreateNUWAdd(NewRem, Step);
2135 Value *RemSel;
2136 if (match(Step, m_One())) {
2137 Value *RemCmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, RemAdd, RemAmt);
2138 RemSel = Builder.CreateSelect(RemCmp, Constant::getNullValue(Ty), RemAdd);
2139 } else {
2140 Value *RemCmp = Builder.CreateICmp(ICmpInst::ICMP_UGE, RemAdd, RemAmt);
2141 Value *RemSub = Builder.CreateNUWSub(RemAdd, RemAmt);
2142 RemSel = Builder.CreateSelect(RemCmp, RemSub, RemAdd);
2143 }
2144 if (auto *SI = dyn_cast<SelectInst>(RemSel))
2146
2147 NewRem->addIncoming(StartRem, L->getLoopPreheader());
2148 NewRem->addIncoming(RemSel, L->getLoopLatch());
2149
2150 // Insert all touched BBs.
2151 FreshBBs.insert(L->getLoopPreheader());
2152 FreshBBs.insert(LoopIncrPN->getParent());
2153 FreshBBs.insert(L->getLoopLatch());
2154 FreshBBs.insert(Rem->getParent());
2155 if (AddInst)
2156 FreshBBs.insert(cast<Instruction>(AddInst)->getParent());
2157 replaceAllUsesWith(Rem, NewRem, FreshBBs, IsHuge);
2158 Rem->eraseFromParent();
2159 if (AddInst && AddInst->use_empty())
2160 cast<Instruction>(AddInst)->eraseFromParent();
2161 return true;
2162}
2163
2164bool CodeGenPrepare::optimizeURem(Instruction *Rem) {
2165 if (foldURemOfLoopIncrement(Rem, DL, LI, getDT(), FreshBBs, IsHugeFunc))
2166 return true;
2167 return false;
2168}
2169
2170bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {
2171 if (sinkCmpExpression(Cmp, *TLI, *DL))
2172 return true;
2173
2174 if (combineToUAddWithOverflow(Cmp, ModifiedDT))
2175 return true;
2176
2177 if (combineToUSubWithOverflow(Cmp, ModifiedDT))
2178 return true;
2179
2180 if (unfoldPowerOf2Test(Cmp))
2181 return true;
2182
2183 if (foldICmpWithDominatingICmp(Cmp, *TLI, Opts.cgp_icmp_eq2icmp_st))
2184 return true;
2185
2187 return true;
2188
2189 if (foldFCmpToFPClassTest(Cmp, *TLI, *DL))
2190 return true;
2191
2192 return false;
2193}
2194
2195/// Duplicate and sink the given 'and' instruction into user blocks where it is
2196/// used in a compare to allow isel to generate better code for targets where
2197/// this operation can be combined.
2198///
2199/// Return true if any changes are made.
2201 SetOfInstrs &InsertedInsts) {
2202 // Double-check that we're not trying to optimize an instruction that was
2203 // already optimized by some other part of this pass.
2204 assert(!InsertedInsts.count(AndI) &&
2205 "Attempting to optimize already optimized and instruction");
2206 (void)InsertedInsts;
2207
2208 // Nothing to do for single use in same basic block.
2209 if (AndI->hasOneUse() &&
2210 AndI->getParent() == cast<Instruction>(*AndI->user_begin())->getParent())
2211 return false;
2212
2213 // Try to avoid cases where sinking/duplicating is likely to increase register
2214 // pressure.
2215 if (!isa<ConstantInt>(AndI->getOperand(0)) &&
2216 !isa<ConstantInt>(AndI->getOperand(1)) &&
2217 AndI->getOperand(0)->hasOneUse() && AndI->getOperand(1)->hasOneUse())
2218 return false;
2219
2220 for (auto *U : AndI->users()) {
2222
2223 // Only sink 'and' feeding icmp with 0.
2224 if (!isa<ICmpInst>(User))
2225 return false;
2226
2227 auto *CmpC = dyn_cast<ConstantInt>(User->getOperand(1));
2228 if (!CmpC || !CmpC->isZero())
2229 return false;
2230 }
2231
2232 if (!TLI.isMaskAndCmp0FoldingBeneficial(*AndI))
2233 return false;
2234
2235 LLVM_DEBUG(dbgs() << "found 'and' feeding only icmp 0;\n");
2236 LLVM_DEBUG(AndI->getParent()->dump());
2237
2238 // Push the 'and' into the same block as the icmp 0. There should only be
2239 // one (icmp (and, 0)) in each block, since CSE/GVN should have removed any
2240 // others, so we don't need to keep track of which BBs we insert into.
2241 for (Instruction::user_iterator UI = AndI->user_begin(), E = AndI->user_end();
2242 UI != E;) {
2243 Use &TheUse = UI.getUse();
2245
2246 // Preincrement use iterator so we don't invalidate it.
2247 ++UI;
2248
2249 LLVM_DEBUG(dbgs() << "sinking 'and' use: " << *User << "\n");
2250
2251 // Keep the 'and' in the same place if the use is already in the same block.
2252 Instruction *InsertPt =
2253 User->getParent() == AndI->getParent() ? AndI : User;
2254 Instruction *InsertedAnd = BinaryOperator::Create(
2255 Instruction::And, AndI->getOperand(0), AndI->getOperand(1), "",
2256 InsertPt->getIterator());
2257 // Propagate the debug info.
2258 InsertedAnd->setDebugLoc(AndI->getDebugLoc());
2259
2260 // Replace a use of the 'and' with a use of the new 'and'.
2261 TheUse = InsertedAnd;
2262 ++NumAndUses;
2263 LLVM_DEBUG(User->getParent()->dump());
2264 }
2265
2266 // We removed all uses, nuke the and.
2267 AndI->eraseFromParent();
2268 return true;
2269}
2270
2271/// Check if the candidates could be combined with a shift instruction, which
2272/// includes:
2273/// 1. Truncate instruction
2274/// 2. And instruction and the imm is a mask of the low bits:
2275/// imm & (imm+1) == 0
2277 if (!isa<TruncInst>(User)) {
2278 if (User->getOpcode() != Instruction::And ||
2280 return false;
2281
2282 const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue();
2283
2284 if ((Cimm & (Cimm + 1)).getBoolValue())
2285 return false;
2286 }
2287 return true;
2288}
2289
2290/// Sink both shift and truncate instruction to the use of truncate's BB.
2291static bool
2294 const TargetLowering &TLI, const DataLayout &DL) {
2295 BasicBlock *UserBB = User->getParent();
2297 auto *TruncI = cast<TruncInst>(User);
2298 bool MadeChange = false;
2299
2300 for (Instruction::user_iterator TruncUI = TruncI->user_begin(),
2301 TruncE = TruncI->user_end();
2302 TruncUI != TruncE;) {
2303
2304 Use &TruncTheUse = TruncUI.getUse();
2305 Instruction *TruncUser = cast<Instruction>(*TruncUI);
2306 // Preincrement use iterator so we don't invalidate it.
2307
2308 ++TruncUI;
2309
2310 int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode());
2311 if (!ISDOpcode)
2312 continue;
2313
2314 // If the use is actually a legal node, there will not be an
2315 // implicit truncate.
2316 // FIXME: always querying the result type is just an
2317 // approximation; some nodes' legality is determined by the
2318 // operand or other means. There's no good way to find out though.
2319 if (TLI.isOperationLegalOrCustom(
2320 ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true)))
2321 continue;
2322
2323 // Don't bother for PHI nodes.
2324 if (isa<PHINode>(TruncUser))
2325 continue;
2326
2327 BasicBlock *TruncUserBB = TruncUser->getParent();
2328
2329 if (UserBB == TruncUserBB)
2330 continue;
2331
2332 BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB];
2333 CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];
2334
2335 if (!InsertedShift && !InsertedTrunc) {
2336 BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt();
2337 assert(InsertPt != TruncUserBB->end());
2338 // Sink the shift
2339 if (ShiftI->getOpcode() == Instruction::AShr)
2340 InsertedShift =
2341 BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "");
2342 else
2343 InsertedShift =
2344 BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "");
2345 InsertedShift->setDebugLoc(ShiftI->getDebugLoc());
2346 InsertedShift->insertBefore(*TruncUserBB, InsertPt);
2347
2348 // Sink the trunc
2349 BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt();
2350 TruncInsertPt++;
2351 // It will go ahead of any debug-info.
2352 TruncInsertPt.setHeadBit(true);
2353 assert(TruncInsertPt != TruncUserBB->end());
2354
2355 InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift,
2356 TruncI->getType(), "");
2357 InsertedTrunc->insertBefore(*TruncUserBB, TruncInsertPt);
2358 InsertedTrunc->setDebugLoc(TruncI->getDebugLoc());
2359
2360 MadeChange = true;
2361
2362 TruncTheUse = InsertedTrunc;
2363 }
2364 }
2365 return MadeChange;
2366}
2367
2368/// Sink the shift *right* instruction into user blocks if the uses could
2369/// potentially be combined with this shift instruction and generate BitExtract
2370/// instruction. It will only be applied if the architecture supports BitExtract
2371/// instruction. Here is an example:
2372/// BB1:
2373/// %x.extract.shift = lshr i64 %arg1, 32
2374/// BB2:
2375/// %x.extract.trunc = trunc i64 %x.extract.shift to i16
2376/// ==>
2377///
2378/// BB2:
2379/// %x.extract.shift.1 = lshr i64 %arg1, 32
2380/// %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16
2381///
2382/// CodeGen will recognize the pattern in BB2 and generate BitExtract
2383/// instruction.
2384/// Return true if any changes are made.
2386 const TargetLowering &TLI,
2387 const DataLayout &DL) {
2388 BasicBlock *DefBB = ShiftI->getParent();
2389
2390 /// Only insert instructions in each block once.
2392
2393 bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType()));
2394
2395 bool MadeChange = false;
2396 for (Instruction::user_iterator UI = ShiftI->user_begin(),
2397 E = ShiftI->user_end();
2398 UI != E;) {
2399 Use &TheUse = UI.getUse();
2401 // Preincrement use iterator so we don't invalidate it.
2402 ++UI;
2403
2404 // Don't bother for PHI nodes.
2405 if (isa<PHINode>(User))
2406 continue;
2407
2409 continue;
2410
2411 BasicBlock *UserBB = User->getParent();
2412
2413 if (UserBB == DefBB) {
2414 // If the shift and truncate instruction are in the same BB. The use of
2415 // the truncate(TruncUse) may still introduce another truncate if not
2416 // legal. In this case, we would like to sink both shift and truncate
2417 // instruction to the BB of TruncUse.
2418 // for example:
2419 // BB1:
2420 // i64 shift.result = lshr i64 opnd, imm
2421 // trunc.result = trunc shift.result to i16
2422 //
2423 // BB2:
2424 // ----> We will have an implicit truncate here if the architecture does
2425 // not have i16 compare.
2426 // cmp i16 trunc.result, opnd2
2427 //
2428 if (isa<TruncInst>(User) &&
2429 shiftIsLegal
2430 // If the type of the truncate is legal, no truncate will be
2431 // introduced in other basic blocks.
2432 && (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType()))))
2433 MadeChange =
2434 SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI, DL);
2435
2436 continue;
2437 }
2438 // If we have already inserted a shift into this block, use it.
2439 BinaryOperator *&InsertedShift = InsertedShifts[UserBB];
2440
2441 if (!InsertedShift) {
2442 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
2443 assert(InsertPt != UserBB->end());
2444
2445 if (ShiftI->getOpcode() == Instruction::AShr)
2446 InsertedShift =
2447 BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "");
2448 else
2449 InsertedShift =
2450 BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "");
2451 InsertedShift->insertBefore(*UserBB, InsertPt);
2452 InsertedShift->setDebugLoc(ShiftI->getDebugLoc());
2453
2454 MadeChange = true;
2455 }
2456
2457 // Replace a use of the shift with a use of the new shift.
2458 TheUse = InsertedShift;
2459 }
2460
2461 // If we removed all uses, or there are none, nuke the shift.
2462 if (ShiftI->use_empty()) {
2463 salvageDebugInfo(*ShiftI);
2464 ShiftI->eraseFromParent();
2465 MadeChange = true;
2466 }
2467
2468 return MadeChange;
2469}
2470
2471/// If counting leading or trailing zeros is an expensive operation and a zero
2472/// input is defined, add a check for zero to avoid calling the intrinsic.
2473///
2474/// We want to transform:
2475/// %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)
2476///
2477/// into:
2478/// entry:
2479/// %cmpz = icmp eq i64 %A, 0
2480/// br i1 %cmpz, label %cond.end, label %cond.false
2481/// cond.false:
2482/// %z = call i64 @llvm.cttz.i64(i64 %A, i1 true)
2483/// br label %cond.end
2484/// cond.end:
2485/// %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
2486///
2487/// If the transform is performed, return true and set ModifiedDT to true.
2488static bool despeculateCountZeros(IntrinsicInst *CountZeros,
2489 DomTreeUpdater *DTU, LoopInfo *LI,
2490 const TargetLowering *TLI,
2491 const DataLayout *DL, ModifyDT &ModifiedDT,
2493 bool IsHugeFunc) {
2494 // If a zero input is undefined, it doesn't make sense to despeculate that.
2495 if (match(CountZeros->getOperand(1), m_One()))
2496 return false;
2497
2498 // If it's cheap to speculate, there's nothing to do.
2499 Type *Ty = CountZeros->getType();
2500 auto IntrinsicID = CountZeros->getIntrinsicID();
2501 if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz(Ty)) ||
2502 (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz(Ty)))
2503 return false;
2504
2505 // Only handle scalar cases. Anything else requires too much work.
2506 unsigned SizeInBits = Ty->getScalarSizeInBits();
2507 if (Ty->isVectorTy())
2508 return false;
2509
2510 // Bail if the value is never zero.
2511 Use &Op = CountZeros->getOperandUse(0);
2512 if (isKnownNonZero(Op, *DL))
2513 return false;
2514
2515 // The intrinsic will be sunk behind a compare against zero and branch.
2516 BasicBlock *StartBlock = CountZeros->getParent();
2517 BasicBlock *CallBlock = SplitBlock(StartBlock, CountZeros, DTU, LI,
2518 /* MSSAU */ nullptr, "cond.false");
2519 if (IsHugeFunc)
2520 FreshBBs.insert(CallBlock);
2521
2522 // Create another block after the count zero intrinsic. A PHI will be added
2523 // in this block to select the result of the intrinsic or the bit-width
2524 // constant if the input to the intrinsic is zero.
2525 BasicBlock::iterator SplitPt = std::next(BasicBlock::iterator(CountZeros));
2526 // Any debug-info after CountZeros should not be included.
2527 SplitPt.setHeadBit(true);
2528 BasicBlock *EndBlock = SplitBlock(CallBlock, &*SplitPt, DTU, LI,
2529 /* MSSAU */ nullptr, "cond.end");
2530 if (IsHugeFunc)
2531 FreshBBs.insert(EndBlock);
2532
2533 // Set up a builder to create a compare, conditional branch, and PHI.
2534 IRBuilder<> Builder(StartBlock->getTerminator());
2535 Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc());
2536
2537 // Replace the unconditional branch that was created by the first split with
2538 // a compare against zero and a conditional branch.
2540 // Avoid introducing branch on poison. This also replaces the ctz operand.
2542 Op = Builder.CreateFreeze(Op, Op->getName() + ".fr");
2543 Value *Cmp = Builder.CreateICmpEQ(Op, Zero, "cmpz");
2544 Builder.CreateCondBr(Cmp, EndBlock, CallBlock);
2545 StartBlock->getTerminator()->eraseFromParent();
2546 DTU->applyUpdates({{DominatorTree::Insert, StartBlock, EndBlock}});
2547
2548 // Create a PHI in the end block to select either the output of the intrinsic
2549 // or the bit width of the operand.
2550 Builder.SetInsertPoint(EndBlock->begin());
2551 PHINode *PN = Builder.CreatePHI(Ty, 2, "ctz");
2552 replaceAllUsesWith(CountZeros, PN, FreshBBs, IsHugeFunc);
2553 Value *BitWidth = Builder.getInt(APInt(SizeInBits, SizeInBits));
2554 PN->addIncoming(BitWidth, StartBlock);
2555 PN->addIncoming(CountZeros, CallBlock);
2556
2557 // We are explicitly handling the zero case, so we can set the intrinsic's
2558 // undefined zero argument to 'true'. This will also prevent reprocessing the
2559 // intrinsic; we only despeculate when a zero input is defined.
2560 CountZeros->setArgOperand(1, Builder.getTrue());
2561 ModifiedDT = ModifyDT::ModifyBBDT;
2562 return true;
2563}
2564
2565bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
2566 BasicBlock *BB = CI->getParent();
2567
2568 // Sink address computing for memory operands into the block.
2569 if (CI->isInlineAsm() && optimizeInlineAsmInst(CI))
2570 return true;
2571
2572 // Align the pointer arguments to this call if the target thinks it's a good
2573 // idea
2574 unsigned MinSize;
2575 Align PrefAlign;
2576 if (TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {
2577 for (auto &Arg : CI->args()) {
2578 // We want to align both objects whose address is used directly and
2579 // objects whose address is used in casts and GEPs, though it only makes
2580 // sense for GEPs if the offset is a multiple of the desired alignment and
2581 // if size - offset meets the size threshold.
2582 if (!Arg->getType()->isPointerTy())
2583 continue;
2584 APInt Offset(DL->getIndexSizeInBits(
2585 cast<PointerType>(Arg->getType())->getAddressSpace()),
2586 0);
2587 Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset);
2588 uint64_t Offset2 = Offset.getLimitedValue();
2589 if (!isAligned(PrefAlign, Offset2))
2590 continue;
2591 AllocaInst *AI;
2592 if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlign() < PrefAlign) {
2593 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(*DL);
2594 if (AllocaSize && AllocaSize->getKnownMinValue() >= MinSize + Offset2)
2595 AI->setAlignment(PrefAlign);
2596 }
2597 // Global variables can only be aligned if they are defined in this
2598 // object (i.e. they are uniquely initialized in this object), and
2599 // over-aligning global variables that have an explicit section is
2600 // forbidden.
2601 GlobalVariable *GV;
2602 if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() &&
2603 GV->getPointerAlignment(*DL) < PrefAlign &&
2604 GV->getGlobalSize(*DL) >= MinSize + Offset2)
2605 GV->setAlignment(PrefAlign);
2606 }
2607 }
2608 // If this is a memcpy (or similar) then we may be able to improve the
2609 // alignment.
2610 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) {
2611 Align DestAlign = getKnownAlignment(MI->getDest(), *DL);
2612 MaybeAlign MIDestAlign = MI->getDestAlign();
2613 if (!MIDestAlign || DestAlign > *MIDestAlign)
2614 MI->setDestAlignment(DestAlign);
2615 if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) {
2616 MaybeAlign MTISrcAlign = MTI->getSourceAlign();
2617 Align SrcAlign = getKnownAlignment(MTI->getSource(), *DL);
2618 if (!MTISrcAlign || SrcAlign > *MTISrcAlign)
2619 MTI->setSourceAlignment(SrcAlign);
2620 }
2621 }
2622
2623 // If we have a cold call site, try to sink addressing computation into the
2624 // cold block. This interacts with our handling for loads and stores to
2625 // ensure that we can fold all uses of a potential addressing computation
2626 // into their uses. TODO: generalize this to work over profiling data
2627 if (CI->hasFnAttr(Attribute::Cold) &&
2628 !llvm::shouldOptimizeForSize(BB, PSI, BFI))
2629 for (auto &Arg : CI->args()) {
2630 if (!Arg->getType()->isPointerTy())
2631 continue;
2632 unsigned AS = Arg->getType()->getPointerAddressSpace();
2633 if (optimizeMemoryInst(CI, Arg, Arg->getType(), AS))
2634 return true;
2635 }
2636
2637 IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
2638 if (II) {
2639 switch (II->getIntrinsicID()) {
2640 default:
2641 break;
2642 case Intrinsic::assume:
2643 llvm_unreachable("llvm.assume should have been removed already");
2644 case Intrinsic::allow_runtime_check:
2645 case Intrinsic::allow_ubsan_check:
2646 case Intrinsic::experimental_widenable_condition: {
2647 // Give up on future widening opportunities so that we can fold away dead
2648 // paths and merge blocks before going into block-local instruction
2649 // selection.
2650 if (II->use_empty()) {
2651 II->eraseFromParent();
2652 return true;
2653 }
2654 Constant *RetVal = ConstantInt::getTrue(II->getContext());
2655 resetIteratorIfInvalidatedWhileCalling(BB, [&]() {
2656 replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr);
2657 });
2658 return true;
2659 }
2660 case Intrinsic::objectsize:
2661 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2662 case Intrinsic::is_constant:
2663 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2664 case Intrinsic::aarch64_stlxr:
2665 case Intrinsic::aarch64_stxr: {
2666 ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0));
2667 if (!ExtVal || !ExtVal->hasOneUse() ||
2668 ExtVal->getParent() == CI->getParent())
2669 return false;
2670 // Sink a zext feeding stlxr/stxr before it, so it can be folded into it.
2671 ExtVal->moveBefore(CI->getIterator());
2672 // Mark this instruction as "inserted by CGP", so that other
2673 // optimizations don't touch it.
2674 InsertedInsts.insert(ExtVal);
2675 return true;
2676 }
2677
2678 case Intrinsic::launder_invariant_group: {
2679 Value *ArgVal = II->getArgOperand(0);
2680 auto it = LargeOffsetGEPMap.find(II);
2681 if (it != LargeOffsetGEPMap.end()) {
2682 // Merge entries in LargeOffsetGEPMap to reflect the RAUW.
2683 // Make sure not to have to deal with iterator invalidation
2684 // after possibly adding ArgVal to LargeOffsetGEPMap.
2685 auto GEPs = std::move(it->second);
2686 LargeOffsetGEPMap[ArgVal].append(GEPs.begin(), GEPs.end());
2687 LargeOffsetGEPMap.erase(II);
2688 }
2689
2690 replaceAllUsesWith(II, ArgVal, FreshBBs, IsHugeFunc);
2691 II->eraseFromParent();
2692 return true;
2693 }
2694 case Intrinsic::cttz:
2695 case Intrinsic::ctlz:
2696 // If counting zeros is expensive, try to avoid it.
2697 return despeculateCountZeros(II, DTU, LI, TLI, DL, ModifiedDT, FreshBBs,
2698 IsHugeFunc);
2699 case Intrinsic::fshl:
2700 case Intrinsic::fshr:
2701 return optimizeFunnelShift(II);
2702 case Intrinsic::masked_gather:
2703 return optimizeGatherScatterInst(II, II->getArgOperand(0));
2704 case Intrinsic::masked_scatter:
2705 return optimizeGatherScatterInst(II, II->getArgOperand(1));
2706 case Intrinsic::masked_load:
2707 // Treat v1X masked load as load X type.
2708 if (auto *VT = dyn_cast<FixedVectorType>(II->getType())) {
2709 if (VT->getNumElements() == 1) {
2710 Value *PtrVal = II->getArgOperand(0);
2711 unsigned AS = PtrVal->getType()->getPointerAddressSpace();
2712 if (optimizeMemoryInst(II, PtrVal, VT->getElementType(), AS))
2713 return true;
2714 }
2715 }
2716 return false;
2717 case Intrinsic::masked_store:
2718 // Treat v1X masked store as store X type.
2719 if (auto *VT =
2720 dyn_cast<FixedVectorType>(II->getArgOperand(0)->getType())) {
2721 if (VT->getNumElements() == 1) {
2722 Value *PtrVal = II->getArgOperand(1);
2723 unsigned AS = PtrVal->getType()->getPointerAddressSpace();
2724 if (optimizeMemoryInst(II, PtrVal, VT->getElementType(), AS))
2725 return true;
2726 }
2727 }
2728 return false;
2729 case Intrinsic::umul_with_overflow:
2730 return optimizeMulWithOverflow(II, /*IsSigned=*/false, ModifiedDT);
2731 case Intrinsic::smul_with_overflow:
2732 return optimizeMulWithOverflow(II, /*IsSigned=*/true, ModifiedDT);
2733 }
2734
2735 SmallVector<Value *, 2> PtrOps;
2736 Type *AccessTy;
2737 if (TLI->getAddrModeArguments(II, PtrOps, AccessTy))
2738 while (!PtrOps.empty()) {
2739 Value *PtrVal = PtrOps.pop_back_val();
2740 unsigned AS = PtrVal->getType()->getPointerAddressSpace();
2741 if (optimizeMemoryInst(II, PtrVal, AccessTy, AS))
2742 return true;
2743 }
2744 }
2745
2746 // From here on out we're working with named functions.
2747 auto *Callee = CI->getCalledFunction();
2748 if (!Callee)
2749 return false;
2750
2751 // Lower all default uses of _chk calls. This is very similar
2752 // to what InstCombineCalls does, but here we are only lowering calls
2753 // to fortified library functions (e.g. __memcpy_chk) that have the default
2754 // "don't know" as the objectsize. Anything else should be left alone.
2755 FortifiedLibCallSimplifier Simplifier(TLInfo, true);
2756 IRBuilder<> Builder(CI);
2757 if (Value *V = Simplifier.optimizeCall(CI, Builder)) {
2758 replaceAllUsesWith(CI, V, FreshBBs, IsHugeFunc);
2759 CI->eraseFromParent();
2760 return true;
2761 }
2762
2763 // SCCP may have propagated, among other things, C++ static variables across
2764 // calls. If this happens to be the case, we may want to undo it in order to
2765 // avoid redundant pointer computation of the constant, as the function method
2766 // returning the constant needs to be executed anyways.
2767 auto GetUniformReturnValue = [](const Function *F) -> GlobalVariable * {
2768 if (!F->getReturnType()->isPointerTy())
2769 return nullptr;
2770
2771 GlobalVariable *UniformValue = nullptr;
2772 for (auto &BB : *F) {
2773 if (auto *RI = dyn_cast<ReturnInst>(BB.getTerminator())) {
2774 if (auto *V = dyn_cast<GlobalVariable>(RI->getReturnValue())) {
2775 if (!UniformValue)
2776 UniformValue = V;
2777 else if (V != UniformValue)
2778 return nullptr;
2779 } else {
2780 return nullptr;
2781 }
2782 }
2783 }
2784
2785 return UniformValue;
2786 };
2787
2788 if (Callee->hasExactDefinition()) {
2789 if (GlobalVariable *RV = GetUniformReturnValue(Callee)) {
2790 bool MadeChange = false;
2791 for (Use &U : make_early_inc_range(RV->uses())) {
2792 auto *I = dyn_cast<Instruction>(U.getUser());
2793 if (!I || I->getParent() != CI->getParent()) {
2794 // Limit to the same basic block to avoid extending the call-site live
2795 // range, which otherwise could increase register pressure.
2796 continue;
2797 }
2798 if (CI->comesBefore(I)) {
2799 U.set(CI);
2800 MadeChange = true;
2801 }
2802 }
2803
2804 return MadeChange;
2805 }
2806 }
2807
2808 return false;
2809}
2810
2812 const CallInst *CI) {
2813 assert(CI && CI->use_empty());
2814
2815 if (const auto *II = dyn_cast<IntrinsicInst>(CI))
2816 switch (II->getIntrinsicID()) {
2817 case Intrinsic::memset:
2818 case Intrinsic::memcpy:
2819 case Intrinsic::memmove:
2820 return true;
2821 default:
2822 return false;
2823 }
2824
2825 Function *Callee = CI->getCalledFunction();
2826 if (Callee && TLInfo)
2827 switch (TLInfo->getLibFunc(*Callee)) {
2828 case LibFunc_strcpy:
2829 case LibFunc_strncpy:
2830 case LibFunc_strcat:
2831 case LibFunc_strncat:
2832 return true;
2833 default:
2834 return false;
2835 }
2836
2837 return false;
2838}
2839
2840/// Look for opportunities to duplicate return instructions to the predecessor
2841/// to enable tail call optimizations. The case it is currently looking for is
2842/// the following one. Known intrinsics or library function that may be tail
2843/// called are taken into account as well.
2844/// @code
2845/// bb0:
2846/// %tmp0 = tail call i32 @f0()
2847/// br label %return
2848/// bb1:
2849/// %tmp1 = tail call i32 @f1()
2850/// br label %return
2851/// bb2:
2852/// %tmp2 = tail call i32 @f2()
2853/// br label %return
2854/// return:
2855/// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
2856/// ret i32 %retval
2857/// @endcode
2858///
2859/// =>
2860///
2861/// @code
2862/// bb0:
2863/// %tmp0 = tail call i32 @f0()
2864/// ret i32 %tmp0
2865/// bb1:
2866/// %tmp1 = tail call i32 @f1()
2867/// ret i32 %tmp1
2868/// bb2:
2869/// %tmp2 = tail call i32 @f2()
2870/// ret i32 %tmp2
2871/// @endcode
2872bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB,
2873 ModifyDT &ModifiedDT) {
2874 ReturnInst *RetI = dyn_cast<ReturnInst>(BB->getTerminator());
2875 if (!RetI)
2876 return false;
2877
2878 assert(LI->getLoopFor(BB) == nullptr && "A return block cannot be in a loop");
2879
2880 PHINode *PN = nullptr;
2881 ExtractValueInst *EVI = nullptr;
2882 BitCastInst *BCI = nullptr;
2883 Value *V = RetI->getReturnValue();
2884 if (V) {
2885 BCI = dyn_cast<BitCastInst>(V);
2886 if (BCI)
2887 V = BCI->getOperand(0);
2888
2890 if (EVI) {
2891 V = EVI->getOperand(0);
2892 if (!llvm::all_of(EVI->indices(), equal_to(0)))
2893 return false;
2894 }
2895
2896 PN = dyn_cast<PHINode>(V);
2897 }
2898
2899 if (PN && PN->getParent() != BB)
2900 return false;
2901
2903
2904 auto isFakeUse = [&FakeUses](const Instruction *Inst) {
2905 if (auto *II = dyn_cast<IntrinsicInst>(Inst);
2906 II && II->getIntrinsicID() == Intrinsic::fake_use) {
2907 // Record the instruction so it can be preserved when the exit block is
2908 // removed. Do not preserve the fake use that uses the result of the
2909 // PHI instruction.
2910 // Do not copy fake uses that use the result of a PHI node.
2911 // FIXME: If we do want to copy the fake use into the return blocks, we
2912 // have to figure out which of the PHI node operands to use for each
2913 // copy.
2914 if (!isa<PHINode>(II->getOperand(0))) {
2915 FakeUses.push_back(II);
2916 }
2917 return true;
2918 }
2919
2920 return false;
2921 };
2922
2923 // Make sure there are no instructions between the first instruction
2924 // and return.
2926 // Skip over pseudo-probes and the bitcast.
2927 while (&*BI == BCI || &*BI == EVI || isa<PseudoProbeInst>(BI) ||
2928 match(&*BI, m_Intrinsic<Intrinsic::lifetime_end>()) || isFakeUse(&*BI))
2929 BI = std::next(BI);
2930 if (&*BI != RetI)
2931 return false;
2932
2933 // Only dup the ReturnInst if the CallInst is likely to be emitted as a tail
2934 // call.
2935 auto MayBePermittedAsTailCall = [&](const auto *CI) {
2936 return TLI->mayBeEmittedAsTailCall(CI) &&
2937 attributesPermitTailCall(BB->getParent(), CI, RetI, *TLI);
2938 };
2939
2940 SmallVector<BasicBlock *, 4> TailCallBBs;
2941 // Record the call instructions so we can insert any fake uses
2942 // that need to be preserved before them.
2944 if (PN) {
2945 for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) {
2946 // Look through bitcasts.
2947 Value *IncomingVal = PN->getIncomingValue(I)->stripPointerCasts();
2948 CallInst *CI = dyn_cast<CallInst>(IncomingVal);
2949 BasicBlock *PredBB = PN->getIncomingBlock(I);
2950 // Make sure the phi value is indeed produced by the tail call.
2951 if (CI && CI->hasOneUse() && CI->getParent() == PredBB &&
2952 MayBePermittedAsTailCall(CI)) {
2953 TailCallBBs.push_back(PredBB);
2954 CallInsts.push_back(CI);
2955 } else {
2956 // Consider the cases in which the phi value is indirectly produced by
2957 // the tail call, for example when encountering memset(), memmove(),
2958 // strcpy(), whose return value may have been optimized out. In such
2959 // cases, the value needs to be the first function argument.
2960 //
2961 // bb0:
2962 // tail call void @llvm.memset.p0.i64(ptr %0, i8 0, i64 %1)
2963 // br label %return
2964 // return:
2965 // %phi = phi ptr [ %0, %bb0 ], [ %2, %entry ]
2966 if (PredBB && PredBB->getSingleSuccessor() == BB)
2968 PredBB->getTerminator()->getPrevNode());
2969
2970 if (CI && CI->use_empty() &&
2971 isIntrinsicOrLFToBeTailCalled(TLInfo, CI) &&
2972 IncomingVal == CI->getArgOperand(0) &&
2973 MayBePermittedAsTailCall(CI)) {
2974 TailCallBBs.push_back(PredBB);
2975 CallInsts.push_back(CI);
2976 }
2977 }
2978 }
2979 } else {
2980 SmallPtrSet<BasicBlock *, 4> VisitedBBs;
2981 for (BasicBlock *Pred : predecessors(BB)) {
2982 if (!VisitedBBs.insert(Pred).second)
2983 continue;
2984 if (Instruction *I = Pred->rbegin()->getPrevNode()) {
2985 CallInst *CI = dyn_cast<CallInst>(I);
2986 if (CI && CI->use_empty() && MayBePermittedAsTailCall(CI)) {
2987 // Either we return void or the return value must be the first
2988 // argument of a known intrinsic or library function.
2989 if (!V || isa<UndefValue>(V) ||
2990 (isIntrinsicOrLFToBeTailCalled(TLInfo, CI) &&
2991 V == CI->getArgOperand(0))) {
2992 TailCallBBs.push_back(Pred);
2993 CallInsts.push_back(CI);
2994 }
2995 }
2996 }
2997 }
2998 }
2999
3000 bool Changed = false;
3001 for (auto const &TailCallBB : TailCallBBs) {
3002 // Make sure the call instruction is followed by an unconditional branch to
3003 // the return block.
3004 UncondBrInst *BI = dyn_cast<UncondBrInst>(TailCallBB->getTerminator());
3005 if (!BI || BI->getSuccessor() != BB)
3006 continue;
3007
3008 // Duplicate the return into TailCallBB.
3009 (void)FoldReturnIntoUncondBranch(RetI, BB, TailCallBB, DTU);
3010 assert(!Opts.cgp_verify_bfi_updates ||
3011 BFI->getBlockFreq(BB) >= BFI->getBlockFreq(TailCallBB));
3012 BFI->setBlockFreq(BB,
3013 (BFI->getBlockFreq(BB) - BFI->getBlockFreq(TailCallBB)));
3014 ModifiedDT = ModifyDT::ModifyBBDT;
3015 Changed = true;
3016 ++NumRetsDup;
3017 }
3018
3019 // If we eliminated all predecessors of the block, delete the block now.
3020 if (Changed && !BB->hasAddressTaken() && pred_empty(BB)) {
3021 // Copy the fake uses found in the original return block to all blocks
3022 // that contain tail calls.
3023 for (auto *CI : CallInsts) {
3024 for (auto const *FakeUse : FakeUses) {
3025 auto *ClonedInst = FakeUse->clone();
3026 ClonedInst->insertBefore(CI->getIterator());
3027 }
3028 }
3029 DTU->deleteBB(BB);
3030 }
3031
3032 return Changed;
3033}
3034
3035//===----------------------------------------------------------------------===//
3036// Memory Optimization
3037//===----------------------------------------------------------------------===//
3038
3039namespace {
3040
3041/// This is an extended version of TargetLowering::AddrMode
3042/// which holds actual Value*'s for register values.
3043struct ExtAddrMode : public TargetLowering::AddrMode {
3044 Value *BaseReg = nullptr;
3045 Value *ScaledReg = nullptr;
3046 Value *OriginalValue = nullptr;
3047 bool InBounds = true;
3048
3049 enum FieldName {
3050 NoField = 0x00,
3051 BaseRegField = 0x01,
3052 BaseGVField = 0x02,
3053 BaseOffsField = 0x04,
3054 ScaledRegField = 0x08,
3055 ScaleField = 0x10,
3056 MultipleFields = 0xff
3057 };
3058
3059 ExtAddrMode() = default;
3060
3061 void print(raw_ostream &OS) const;
3062 void dump() const;
3063
3064 // Replace From in ExtAddrMode with To.
3065 // E.g., SExt insts may be promoted and deleted. We should replace them with
3066 // the promoted values.
3067 void replaceWith(Value *From, Value *To) {
3068 if (ScaledReg == From)
3069 ScaledReg = To;
3070 }
3071
3072 FieldName compare(const ExtAddrMode &other) {
3073 // First check that the types are the same on each field, as differing types
3074 // is something we can't cope with later on.
3075 if (BaseReg && other.BaseReg &&
3076 BaseReg->getType() != other.BaseReg->getType())
3077 return MultipleFields;
3078 if (BaseGV && other.BaseGV && BaseGV->getType() != other.BaseGV->getType())
3079 return MultipleFields;
3080 if (ScaledReg && other.ScaledReg &&
3081 ScaledReg->getType() != other.ScaledReg->getType())
3082 return MultipleFields;
3083
3084 // Conservatively reject 'inbounds' mismatches.
3085 if (InBounds != other.InBounds)
3086 return MultipleFields;
3087
3088 // Check each field to see if it differs.
3089 unsigned Result = NoField;
3090 if (BaseReg != other.BaseReg)
3091 Result |= BaseRegField;
3092 if (BaseGV != other.BaseGV)
3093 Result |= BaseGVField;
3094 if (BaseOffs != other.BaseOffs)
3095 Result |= BaseOffsField;
3096 if (ScaledReg != other.ScaledReg)
3097 Result |= ScaledRegField;
3098 // Don't count 0 as being a different scale, because that actually means
3099 // unscaled (which will already be counted by having no ScaledReg).
3100 if (Scale && other.Scale && Scale != other.Scale)
3101 Result |= ScaleField;
3102
3103 if (llvm::popcount(Result) > 1)
3104 return MultipleFields;
3105 else
3106 return static_cast<FieldName>(Result);
3107 }
3108
3109 // An AddrMode is trivial if it involves no calculation i.e. it is just a base
3110 // with no offset.
3111 bool isTrivial() {
3112 // An AddrMode is (BaseGV + BaseReg + BaseOffs + ScaleReg * Scale) so it is
3113 // trivial if at most one of these terms is nonzero, except that BaseGV and
3114 // BaseReg both being zero actually means a null pointer value, which we
3115 // consider to be 'non-zero' here.
3116 return !BaseOffs && !Scale && !(BaseGV && BaseReg);
3117 }
3118
3119 Value *GetFieldAsValue(FieldName Field, Type *IntPtrTy) {
3120 switch (Field) {
3121 default:
3122 return nullptr;
3123 case BaseRegField:
3124 return BaseReg;
3125 case BaseGVField:
3126 return BaseGV;
3127 case ScaledRegField:
3128 return ScaledReg;
3129 case BaseOffsField:
3130 return ConstantInt::getSigned(IntPtrTy, BaseOffs);
3131 }
3132 }
3133
3134 void SetCombinedField(FieldName Field, Value *V,
3135 const SmallVectorImpl<ExtAddrMode> &AddrModes) {
3136 switch (Field) {
3137 default:
3138 llvm_unreachable("Unhandled fields are expected to be rejected earlier");
3139 break;
3140 case ExtAddrMode::BaseRegField:
3141 BaseReg = V;
3142 break;
3143 case ExtAddrMode::BaseGVField:
3144 // A combined BaseGV is an Instruction, not a GlobalValue, so it goes
3145 // in the BaseReg field.
3146 assert(BaseReg == nullptr);
3147 BaseReg = V;
3148 BaseGV = nullptr;
3149 break;
3150 case ExtAddrMode::ScaledRegField:
3151 ScaledReg = V;
3152 // If we have a mix of scaled and unscaled addrmodes then we want scale
3153 // to be the scale and not zero.
3154 if (!Scale)
3155 for (const ExtAddrMode &AM : AddrModes)
3156 if (AM.Scale) {
3157 Scale = AM.Scale;
3158 break;
3159 }
3160 break;
3161 case ExtAddrMode::BaseOffsField:
3162 // The offset is no longer a constant, so it goes in ScaledReg with a
3163 // scale of 1.
3164 assert(ScaledReg == nullptr);
3165 ScaledReg = V;
3166 Scale = 1;
3167 BaseOffs = 0;
3168 break;
3169 }
3170 }
3171};
3172
3173#ifndef NDEBUG
3174static inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) {
3175 AM.print(OS);
3176 return OS;
3177}
3178#endif
3179
3180#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3181void ExtAddrMode::print(raw_ostream &OS) const {
3182 bool NeedPlus = false;
3183 OS << "[";
3184 if (InBounds)
3185 OS << "inbounds ";
3186 if (BaseGV) {
3187 OS << "GV:";
3188 BaseGV->printAsOperand(OS, /*PrintType=*/false);
3189 NeedPlus = true;
3190 }
3191
3192 if (BaseOffs) {
3193 OS << (NeedPlus ? " + " : "") << BaseOffs;
3194 NeedPlus = true;
3195 }
3196
3197 if (BaseReg) {
3198 OS << (NeedPlus ? " + " : "") << "Base:";
3199 BaseReg->printAsOperand(OS, /*PrintType=*/false);
3200 NeedPlus = true;
3201 }
3202 if (Scale) {
3203 OS << (NeedPlus ? " + " : "") << Scale << "*";
3204 ScaledReg->printAsOperand(OS, /*PrintType=*/false);
3205 }
3206
3207 OS << ']';
3208}
3209
3210LLVM_DUMP_METHOD void ExtAddrMode::dump() const {
3211 print(dbgs());
3212 dbgs() << '\n';
3213}
3214#endif
3215
3216} // end anonymous namespace
3217
3218namespace {
3219
3220/// This class provides transaction based operation on the IR.
3221/// Every change made through this class is recorded in the internal state and
3222/// can be undone (rollback) until commit is called.
3223/// CGP does not check if instructions could be speculatively executed when
3224/// moved. Preserving the original location would pessimize the debugging
3225/// experience, as well as negatively impact the quality of sample PGO.
3226class TypePromotionTransaction {
3227 /// This represents the common interface of the individual transaction.
3228 /// Each class implements the logic for doing one specific modification on
3229 /// the IR via the TypePromotionTransaction.
3230 class TypePromotionAction {
3231 protected:
3232 /// The Instruction modified.
3233 Instruction *Inst;
3234
3235 public:
3236 /// Constructor of the action.
3237 /// The constructor performs the related action on the IR.
3238 TypePromotionAction(Instruction *Inst) : Inst(Inst) {}
3239
3240 virtual ~TypePromotionAction() = default;
3241
3242 /// Undo the modification done by this action.
3243 /// When this method is called, the IR must be in the same state as it was
3244 /// before this action was applied.
3245 /// \pre Undoing the action works if and only if the IR is in the exact same
3246 /// state as it was directly after this action was applied.
3247 virtual void undo() = 0;
3248
3249 /// Advocate every change made by this action.
3250 /// When the results on the IR of the action are to be kept, it is important
3251 /// to call this function, otherwise hidden information may be kept forever.
3252 virtual void commit() {
3253 // Nothing to be done, this action is not doing anything.
3254 }
3255 };
3256
3257 /// Utility to remember the position of an instruction.
3258 class InsertionHandler {
3259 /// Position of an instruction.
3260 /// Either an instruction:
3261 /// - Is the first in a basic block: BB is used.
3262 /// - Has a previous instruction: PrevInst is used.
3263 struct {
3264 BasicBlock::iterator PrevInst;
3265 BasicBlock *BB;
3266 } Point;
3267 std::optional<DbgRecord::self_iterator> BeforeDbgRecord = std::nullopt;
3268
3269 /// Remember whether or not the instruction had a previous instruction.
3270 bool HasPrevInstruction;
3271
3272 public:
3273 /// Record the position of \p Inst.
3274 InsertionHandler(Instruction *Inst) {
3275 HasPrevInstruction = (Inst != &*(Inst->getParent()->begin()));
3276 BasicBlock *BB = Inst->getParent();
3277
3278 // Record where we would have to re-insert the instruction in the sequence
3279 // of DbgRecords, if we ended up reinserting.
3280 BeforeDbgRecord = Inst->getDbgReinsertionPosition();
3281
3282 if (HasPrevInstruction) {
3283 Point.PrevInst = std::prev(Inst->getIterator());
3284 } else {
3285 Point.BB = BB;
3286 }
3287 }
3288
3289 /// Insert \p Inst at the recorded position.
3290 void insert(Instruction *Inst) {
3291 if (HasPrevInstruction) {
3292 if (Inst->getParent())
3293 Inst->removeFromParent();
3294 Inst->insertAfter(Point.PrevInst);
3295 } else {
3296 BasicBlock::iterator Position = Point.BB->getFirstInsertionPt();
3297 if (Inst->getParent())
3298 Inst->moveBefore(*Point.BB, Position);
3299 else
3300 Inst->insertBefore(*Point.BB, Position);
3301 }
3302
3303 Inst->getParent()->reinsertInstInDbgRecords(Inst, BeforeDbgRecord);
3304 }
3305 };
3306
3307 /// Set the operand of an instruction with a new value.
3308 class OperandSetter : public TypePromotionAction {
3309 /// Original operand of the instruction.
3310 Value *Origin;
3311
3312 /// Index of the modified instruction.
3313 unsigned Idx;
3314
3315 public:
3316 /// Set \p Idx operand of \p Inst with \p NewVal.
3317 OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal)
3318 : TypePromotionAction(Inst), Idx(Idx) {
3319 LLVM_DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n"
3320 << "for:" << *Inst << "\n"
3321 << "with:" << *NewVal << "\n");
3322 Origin = Inst->getOperand(Idx);
3323 Inst->setOperand(Idx, NewVal);
3324 }
3325
3326 /// Restore the original value of the instruction.
3327 void undo() override {
3328 LLVM_DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n"
3329 << "for: " << *Inst << "\n"
3330 << "with: " << *Origin << "\n");
3331 Inst->setOperand(Idx, Origin);
3332 }
3333 };
3334
3335 /// Hide the operands of an instruction.
3336 /// Do as if this instruction was not using any of its operands.
3337 class OperandsHider : public TypePromotionAction {
3338 /// The list of original operands.
3339 SmallVector<Value *, 4> OriginalValues;
3340
3341 public:
3342 /// Remove \p Inst from the uses of the operands of \p Inst.
3343 OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) {
3344 LLVM_DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n");
3345 unsigned NumOpnds = Inst->getNumOperands();
3346 OriginalValues.reserve(NumOpnds);
3347 for (unsigned It = 0; It < NumOpnds; ++It) {
3348 // Save the current operand.
3349 Value *Val = Inst->getOperand(It);
3350 OriginalValues.push_back(Val);
3351 // Set a dummy one.
3352 // We could use OperandSetter here, but that would imply an overhead
3353 // that we are not willing to pay.
3354 Inst->setOperand(It, PoisonValue::get(Val->getType()));
3355 }
3356 }
3357
3358 /// Restore the original list of uses.
3359 void undo() override {
3360 LLVM_DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n");
3361 for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It)
3362 Inst->setOperand(It, OriginalValues[It]);
3363 }
3364 };
3365
3366 /// Build a truncate instruction.
3367 class TruncBuilder : public TypePromotionAction {
3368 Value *Val;
3369
3370 public:
3371 /// Build a truncate instruction of \p Opnd producing a \p Ty
3372 /// result.
3373 /// trunc Opnd to Ty.
3374 TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) {
3375 IRBuilder<> Builder(Opnd);
3376 Builder.SetCurrentDebugLocation(DebugLoc());
3377 Val = Builder.CreateTrunc(Opnd, Ty, "promoted");
3378 LLVM_DEBUG(dbgs() << "Do: TruncBuilder: " << *Val << "\n");
3379 }
3380
3381 /// Get the built value.
3382 Value *getBuiltValue() { return Val; }
3383
3384 /// Remove the built instruction.
3385 void undo() override {
3386 LLVM_DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val << "\n");
3387 if (Instruction *IVal = dyn_cast<Instruction>(Val))
3388 IVal->eraseFromParent();
3389 }
3390 };
3391
3392 /// Build a sign extension instruction.
3393 class SExtBuilder : public TypePromotionAction {
3394 Value *Val;
3395
3396 public:
3397 /// Build a sign extension instruction of \p Opnd producing a \p Ty
3398 /// result.
3399 /// sext Opnd to Ty.
3400 SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
3401 : TypePromotionAction(InsertPt) {
3402 IRBuilder<> Builder(InsertPt);
3403 Val = Builder.CreateSExt(Opnd, Ty, "promoted");
3404 LLVM_DEBUG(dbgs() << "Do: SExtBuilder: " << *Val << "\n");
3405 }
3406
3407 /// Get the built value.
3408 Value *getBuiltValue() { return Val; }
3409
3410 /// Remove the built instruction.
3411 void undo() override {
3412 LLVM_DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val << "\n");
3413 if (Instruction *IVal = dyn_cast<Instruction>(Val))
3414 IVal->eraseFromParent();
3415 }
3416 };
3417
3418 /// Build a zero extension instruction.
3419 class ZExtBuilder : public TypePromotionAction {
3420 Value *Val;
3421
3422 public:
3423 /// Build a zero extension instruction of \p Opnd producing a \p Ty
3424 /// result.
3425 /// zext Opnd to Ty.
3426 ZExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
3427 : TypePromotionAction(InsertPt) {
3428 IRBuilder<> Builder(InsertPt);
3429 Builder.SetCurrentDebugLocation(DebugLoc());
3430 Val = Builder.CreateZExt(Opnd, Ty, "promoted");
3431 LLVM_DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val << "\n");
3432 }
3433
3434 /// Get the built value.
3435 Value *getBuiltValue() { return Val; }
3436
3437 /// Remove the built instruction.
3438 void undo() override {
3439 LLVM_DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val << "\n");
3440 if (Instruction *IVal = dyn_cast<Instruction>(Val))
3441 IVal->eraseFromParent();
3442 }
3443 };
3444
3445 /// Mutate an instruction to another type.
3446 class TypeMutator : public TypePromotionAction {
3447 /// Record the original type.
3448 Type *OrigTy;
3449
3450 public:
3451 /// Mutate the type of \p Inst into \p NewTy.
3452 TypeMutator(Instruction *Inst, Type *NewTy)
3453 : TypePromotionAction(Inst), OrigTy(Inst->getType()) {
3454 LLVM_DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy
3455 << "\n");
3456 Inst->mutateType(NewTy);
3457 }
3458
3459 /// Mutate the instruction back to its original type.
3460 void undo() override {
3461 LLVM_DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy
3462 << "\n");
3463 Inst->mutateType(OrigTy);
3464 }
3465 };
3466
3467 /// Replace the uses of an instruction by another instruction.
3468 class UsesReplacer : public TypePromotionAction {
3469 /// Helper structure to keep track of the replaced uses.
3470 struct InstructionAndIdx {
3471 /// The instruction using the instruction.
3472 Instruction *Inst;
3473
3474 /// The index where this instruction is used for Inst.
3475 unsigned Idx;
3476
3477 InstructionAndIdx(Instruction *Inst, unsigned Idx)
3478 : Inst(Inst), Idx(Idx) {}
3479 };
3480
3481 /// Keep track of the original uses (pair Instruction, Index).
3483 /// Keep track of the debug users.
3484 SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
3485
3486 /// Keep track of the new value so that we can undo it by replacing
3487 /// instances of the new value with the original value.
3488 Value *New;
3489
3490 public:
3491 /// Replace all the use of \p Inst by \p New.
3492 UsesReplacer(Instruction *Inst, Value *New)
3493 : TypePromotionAction(Inst), New(New) {
3494 LLVM_DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New
3495 << "\n");
3496 // Record the original uses.
3497 for (Use &U : Inst->uses()) {
3498 Instruction *UserI = cast<Instruction>(U.getUser());
3499 OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo()));
3500 }
3501 // Record the debug uses separately. They are not in the instruction's
3502 // use list, but they are replaced by RAUW.
3503 findDbgValues(Inst, DbgVariableRecords);
3504
3505 // Now, we can replace the uses.
3506 Inst->replaceAllUsesWith(New);
3507 }
3508
3509 /// Reassign the original uses of Inst to Inst.
3510 void undo() override {
3511 LLVM_DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n");
3512 for (InstructionAndIdx &Use : OriginalUses)
3513 Use.Inst->setOperand(Use.Idx, Inst);
3514 // RAUW has replaced all original uses with references to the new value,
3515 // including the debug uses. Since we are undoing the replacements,
3516 // the original debug uses must also be reinstated to maintain the
3517 // correctness and utility of debug value records.
3518 for (DbgVariableRecord *DVR : DbgVariableRecords)
3519 DVR->replaceVariableLocationOp(New, Inst);
3520 }
3521 };
3522
3523 /// Remove an instruction from the IR.
3524 class InstructionRemover : public TypePromotionAction {
3525 /// Original position of the instruction.
3526 InsertionHandler Inserter;
3527
3528 /// Helper structure to hide all the link to the instruction. In other
3529 /// words, this helps to do as if the instruction was removed.
3530 OperandsHider Hider;
3531
3532 /// Keep track of the uses replaced, if any.
3533 UsesReplacer *Replacer = nullptr;
3534
3535 /// Keep track of instructions removed.
3536 SetOfInstrs &RemovedInsts;
3537
3538 public:
3539 /// Remove all reference of \p Inst and optionally replace all its
3540 /// uses with New.
3541 /// \p RemovedInsts Keep track of the instructions removed by this Action.
3542 /// \pre If !Inst->use_empty(), then New != nullptr
3543 InstructionRemover(Instruction *Inst, SetOfInstrs &RemovedInsts,
3544 Value *New = nullptr)
3545 : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst),
3546 RemovedInsts(RemovedInsts) {
3547 if (New)
3548 Replacer = new UsesReplacer(Inst, New);
3549 LLVM_DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n");
3550 RemovedInsts.insert(Inst);
3551 /// The instructions removed here will be freed after completing
3552 /// optimizeBlock() for all blocks as we need to keep track of the
3553 /// removed instructions during promotion.
3554 Inst->removeFromParent();
3555 }
3556
3557 ~InstructionRemover() override { delete Replacer; }
3558
3559 InstructionRemover &operator=(const InstructionRemover &other) = delete;
3560 InstructionRemover(const InstructionRemover &other) = delete;
3561
3562 /// Resurrect the instruction and reassign it to the proper uses if
3563 /// new value was provided when build this action.
3564 void undo() override {
3565 LLVM_DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n");
3566 Inserter.insert(Inst);
3567 if (Replacer)
3568 Replacer->undo();
3569 Hider.undo();
3570 RemovedInsts.erase(Inst);
3571 }
3572 };
3573
3574public:
3575 /// Restoration point.
3576 /// The restoration point is a pointer to an action instead of an iterator
3577 /// because the iterator may be invalidated but not the pointer.
3578 using ConstRestorationPt = const TypePromotionAction *;
3579
3580 TypePromotionTransaction(SetOfInstrs &RemovedInsts)
3581 : RemovedInsts(RemovedInsts) {}
3582
3583 /// Advocate every changes made in that transaction. Return true if any change
3584 /// happen.
3585 bool commit();
3586
3587 /// Undo all the changes made after the given point.
3588 void rollback(ConstRestorationPt Point);
3589
3590 /// Get the current restoration point.
3591 ConstRestorationPt getRestorationPoint() const;
3592
3593 /// \name API for IR modification with state keeping to support rollback.
3594 /// @{
3595 /// Same as Instruction::setOperand.
3596 void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal);
3597
3598 /// Same as Instruction::eraseFromParent.
3599 void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr);
3600
3601 /// Same as Value::replaceAllUsesWith.
3602 void replaceAllUsesWith(Instruction *Inst, Value *New);
3603
3604 /// Same as Value::mutateType.
3605 void mutateType(Instruction *Inst, Type *NewTy);
3606
3607 /// Same as IRBuilder::createTrunc.
3608 Value *createTrunc(Instruction *Opnd, Type *Ty);
3609
3610 /// Same as IRBuilder::createSExt.
3611 Value *createSExt(Instruction *Inst, Value *Opnd, Type *Ty);
3612
3613 /// Same as IRBuilder::createZExt.
3614 Value *createZExt(Instruction *Inst, Value *Opnd, Type *Ty);
3615
3616private:
3617 /// The ordered list of actions made so far.
3619
3620 SetOfInstrs &RemovedInsts;
3621};
3622
3623} // end anonymous namespace
3624
3625void TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx,
3626 Value *NewVal) {
3627 Actions.push_back(std::make_unique<TypePromotionTransaction::OperandSetter>(
3628 Inst, Idx, NewVal));
3629}
3630
3631void TypePromotionTransaction::eraseInstruction(Instruction *Inst,
3632 Value *NewVal) {
3633 Actions.push_back(
3634 std::make_unique<TypePromotionTransaction::InstructionRemover>(
3635 Inst, RemovedInsts, NewVal));
3636}
3637
3638void TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst,
3639 Value *New) {
3640 Actions.push_back(
3641 std::make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));
3642}
3643
3644void TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) {
3645 Actions.push_back(
3646 std::make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));
3647}
3648
3649Value *TypePromotionTransaction::createTrunc(Instruction *Opnd, Type *Ty) {
3650 std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty));
3651 Value *Val = Ptr->getBuiltValue();
3652 Actions.push_back(std::move(Ptr));
3653 return Val;
3654}
3655
3656Value *TypePromotionTransaction::createSExt(Instruction *Inst, Value *Opnd,
3657 Type *Ty) {
3658 std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty));
3659 Value *Val = Ptr->getBuiltValue();
3660 Actions.push_back(std::move(Ptr));
3661 return Val;
3662}
3663
3664Value *TypePromotionTransaction::createZExt(Instruction *Inst, Value *Opnd,
3665 Type *Ty) {
3666 std::unique_ptr<ZExtBuilder> Ptr(new ZExtBuilder(Inst, Opnd, Ty));
3667 Value *Val = Ptr->getBuiltValue();
3668 Actions.push_back(std::move(Ptr));
3669 return Val;
3670}
3671
3672TypePromotionTransaction::ConstRestorationPt
3673TypePromotionTransaction::getRestorationPoint() const {
3674 return !Actions.empty() ? Actions.back().get() : nullptr;
3675}
3676
3677bool TypePromotionTransaction::commit() {
3678 for (std::unique_ptr<TypePromotionAction> &Action : Actions)
3679 Action->commit();
3680 bool Modified = !Actions.empty();
3681 Actions.clear();
3682 return Modified;
3683}
3684
3685void TypePromotionTransaction::rollback(
3686 TypePromotionTransaction::ConstRestorationPt Point) {
3687 while (!Actions.empty() && Point != Actions.back().get()) {
3688 std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();
3689 Curr->undo();
3690 }
3691}
3692
3693namespace {
3694
3695/// A helper class for matching addressing modes.
3696///
3697/// This encapsulates the logic for matching the target-legal addressing modes.
3698class AddressingModeMatcher {
3699 SmallVectorImpl<Instruction *> &AddrModeInsts;
3700 const TargetLowering &TLI;
3701 const TargetRegisterInfo &TRI;
3702 const DataLayout &DL;
3703 const LoopInfo &LI;
3704 const std::function<const DominatorTree &()> getDTFn;
3705
3706 /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
3707 /// the memory instruction that we're computing this address for.
3708 Type *AccessTy;
3709 unsigned AddrSpace;
3710 Instruction *MemoryInst;
3711
3712 /// This is the addressing mode that we're building up. This is
3713 /// part of the return value of this addressing mode matching stuff.
3714 ExtAddrMode &AddrMode;
3715
3716 /// The instructions inserted by other CodeGenPrepare optimizations.
3717 const SetOfInstrs &InsertedInsts;
3718
3719 /// A map from the instructions to their type before promotion.
3720 InstrToOrigTy &PromotedInsts;
3721
3722 /// The ongoing transaction where every action should be registered.
3723 TypePromotionTransaction &TPT;
3724
3725 // A GEP which has too large offset to be folded into the addressing mode.
3726 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP;
3727
3728 /// This is set to true when we should not do profitability checks.
3729 /// When true, IsProfitableToFoldIntoAddressingMode always returns true.
3730 bool IgnoreProfitability;
3731
3732 /// True if we are optimizing for size.
3733 bool OptSize = false;
3734
3735 ProfileSummaryInfo *PSI;
3736 BlockFrequencyInfo *BFI;
3737 const CodeGenOptions &Opts;
3738
3739 AddressingModeMatcher(
3740 SmallVectorImpl<Instruction *> &AMI, const TargetLowering &TLI,
3741 const TargetRegisterInfo &TRI, const LoopInfo &LI,
3742 const std::function<const DominatorTree &()> getDTFn, Type *AT,
3743 unsigned AS, Instruction *MI, ExtAddrMode &AM,
3744 const SetOfInstrs &InsertedInsts, InstrToOrigTy &PromotedInsts,
3745 TypePromotionTransaction &TPT,
3746 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP,
3747 bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI,
3748 const CodeGenOptions &Opts)
3749 : AddrModeInsts(AMI), TLI(TLI), TRI(TRI), DL(MI->getDataLayout()), LI(LI),
3750 getDTFn(getDTFn), AccessTy(AT), AddrSpace(AS), MemoryInst(MI),
3751 AddrMode(AM), InsertedInsts(InsertedInsts),
3752 PromotedInsts(PromotedInsts), TPT(TPT), LargeOffsetGEP(LargeOffsetGEP),
3753 OptSize(OptSize), PSI(PSI), BFI(BFI), Opts(Opts) {
3754 IgnoreProfitability = false;
3755 }
3756
3757public:
3758 /// Find the maximal addressing mode that a load/store of V can fold,
3759 /// give an access type of AccessTy. This returns a list of involved
3760 /// instructions in AddrModeInsts.
3761 /// \p InsertedInsts The instructions inserted by other CodeGenPrepare
3762 /// optimizations.
3763 /// \p PromotedInsts maps the instructions to their type before promotion.
3764 /// \p The ongoing transaction where every action should be registered.
3765 static ExtAddrMode
3766 Match(Value *V, Type *AccessTy, unsigned AS, Instruction *MemoryInst,
3767 SmallVectorImpl<Instruction *> &AddrModeInsts,
3768 const TargetLowering &TLI, const LoopInfo &LI,
3769 const std::function<const DominatorTree &()> getDTFn,
3770 const TargetRegisterInfo &TRI, const SetOfInstrs &InsertedInsts,
3771 InstrToOrigTy &PromotedInsts, TypePromotionTransaction &TPT,
3772 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP,
3773 bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI,
3774 const CodeGenOptions &Opts) {
3775 ExtAddrMode Result;
3776
3777 bool Success = AddressingModeMatcher(
3778 AddrModeInsts, TLI, TRI, LI, getDTFn, AccessTy, AS,
3779 MemoryInst, Result, InsertedInsts, PromotedInsts, TPT,
3780 LargeOffsetGEP, OptSize, PSI, BFI, Opts)
3781 .matchAddr(V, 0);
3782 (void)Success;
3783 assert(Success && "Couldn't select *anything*?");
3784 return Result;
3785 }
3786
3787private:
3788 bool matchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth);
3789 bool matchAddr(Value *Addr, unsigned Depth);
3790 bool matchOperationAddr(User *AddrInst, unsigned Opcode, unsigned Depth,
3791 bool *MovedAway = nullptr);
3792 bool isProfitableToFoldIntoAddressingMode(Instruction *I,
3793 ExtAddrMode &AMBefore,
3794 ExtAddrMode &AMAfter);
3795 bool valueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2);
3796 bool isPromotionProfitable(unsigned NewCost, unsigned OldCost,
3797 Value *PromotedOperand) const;
3798};
3799
3800class PhiNodeSet;
3801
3802/// An iterator for PhiNodeSet.
3803class PhiNodeSetIterator {
3804 PhiNodeSet *const Set;
3805 size_t CurrentIndex = 0;
3806
3807public:
3808 /// The constructor. Start should point to either a valid element, or be equal
3809 /// to the size of the underlying SmallVector of the PhiNodeSet.
3810 PhiNodeSetIterator(PhiNodeSet *const Set, size_t Start);
3811 PHINode *operator*() const;
3812 PhiNodeSetIterator &operator++();
3813 bool operator==(const PhiNodeSetIterator &RHS) const;
3814 bool operator!=(const PhiNodeSetIterator &RHS) const;
3815};
3816
3817/// Keeps a set of PHINodes.
3818///
3819/// This is a minimal set implementation for a specific use case:
3820/// It is very fast when there are very few elements, but also provides good
3821/// performance when there are many. It is similar to SmallPtrSet, but also
3822/// provides iteration by insertion order, which is deterministic and stable
3823/// across runs. It is also similar to SmallSetVector, but provides removing
3824/// elements in O(1) time. This is achieved by not actually removing the element
3825/// from the underlying vector, so comes at the cost of using more memory, but
3826/// that is fine, since PhiNodeSets are used as short lived objects.
3827class PhiNodeSet {
3828 friend class PhiNodeSetIterator;
3829
3830 using MapType = SmallDenseMap<PHINode *, size_t, 32>;
3831 using iterator = PhiNodeSetIterator;
3832
3833 /// Keeps the elements in the order of their insertion in the underlying
3834 /// vector. To achieve constant time removal, it never deletes any element.
3836
3837 /// Keeps the elements in the underlying set implementation. This (and not the
3838 /// NodeList defined above) is the source of truth on whether an element
3839 /// is actually in the collection.
3840 MapType NodeMap;
3841
3842 /// Points to the first valid (not deleted) element when the set is not empty
3843 /// and the value is not zero. Equals to the size of the underlying vector
3844 /// when the set is empty. When the value is 0, as in the beginning, the
3845 /// first element may or may not be valid.
3846 size_t FirstValidElement = 0;
3847
3848public:
3849 /// Inserts a new element to the collection.
3850 /// \returns true if the element is actually added, i.e. was not in the
3851 /// collection before the operation.
3852 bool insert(PHINode *Ptr) {
3853 if (NodeMap.insert(std::make_pair(Ptr, NodeList.size())).second) {
3854 NodeList.push_back(Ptr);
3855 return true;
3856 }
3857 return false;
3858 }
3859
3860 /// Removes the element from the collection.
3861 /// \returns whether the element is actually removed, i.e. was in the
3862 /// collection before the operation.
3863 bool erase(PHINode *Ptr) {
3864 if (NodeMap.erase(Ptr)) {
3865 SkipRemovedElements(FirstValidElement);
3866 return true;
3867 }
3868 return false;
3869 }
3870
3871 /// Removes all elements and clears the collection.
3872 void clear() {
3873 NodeMap.clear();
3874 NodeList.clear();
3875 FirstValidElement = 0;
3876 }
3877
3878 /// \returns an iterator that will iterate the elements in the order of
3879 /// insertion.
3880 iterator begin() {
3881 if (FirstValidElement == 0)
3882 SkipRemovedElements(FirstValidElement);
3883 return PhiNodeSetIterator(this, FirstValidElement);
3884 }
3885
3886 /// \returns an iterator that points to the end of the collection.
3887 iterator end() { return PhiNodeSetIterator(this, NodeList.size()); }
3888
3889 /// Returns the number of elements in the collection.
3890 size_t size() const { return NodeMap.size(); }
3891
3892 /// \returns 1 if the given element is in the collection, and 0 if otherwise.
3893 size_t count(PHINode *Ptr) const { return NodeMap.count(Ptr); }
3894
3895private:
3896 /// Updates the CurrentIndex so that it will point to a valid element.
3897 ///
3898 /// If the element of NodeList at CurrentIndex is valid, it does not
3899 /// change it. If there are no more valid elements, it updates CurrentIndex
3900 /// to point to the end of the NodeList.
3901 void SkipRemovedElements(size_t &CurrentIndex) {
3902 while (CurrentIndex < NodeList.size()) {
3903 auto it = NodeMap.find(NodeList[CurrentIndex]);
3904 // If the element has been deleted and added again later, NodeMap will
3905 // point to a different index, so CurrentIndex will still be invalid.
3906 if (it != NodeMap.end() && it->second == CurrentIndex)
3907 break;
3908 ++CurrentIndex;
3909 }
3910 }
3911};
3912
3913PhiNodeSetIterator::PhiNodeSetIterator(PhiNodeSet *const Set, size_t Start)
3914 : Set(Set), CurrentIndex(Start) {}
3915
3916PHINode *PhiNodeSetIterator::operator*() const {
3917 assert(CurrentIndex < Set->NodeList.size() &&
3918 "PhiNodeSet access out of range");
3919 return Set->NodeList[CurrentIndex];
3920}
3921
3922PhiNodeSetIterator &PhiNodeSetIterator::operator++() {
3923 assert(CurrentIndex < Set->NodeList.size() &&
3924 "PhiNodeSet access out of range");
3925 ++CurrentIndex;
3926 Set->SkipRemovedElements(CurrentIndex);
3927 return *this;
3928}
3929
3930bool PhiNodeSetIterator::operator==(const PhiNodeSetIterator &RHS) const {
3931 return CurrentIndex == RHS.CurrentIndex;
3932}
3933
3934bool PhiNodeSetIterator::operator!=(const PhiNodeSetIterator &RHS) const {
3935 return !((*this) == RHS);
3936}
3937
3938/// Keep track of simplification of Phi nodes.
3939/// Accept the set of all phi nodes and erase phi node from this set
3940/// if it is simplified.
3941class SimplificationTracker {
3942 DenseMap<Value *, Value *> Storage;
3943 // Tracks newly created Phi nodes. The elements are iterated by insertion
3944 // order.
3945 PhiNodeSet AllPhiNodes;
3946 // Tracks newly created Select nodes.
3947 SmallPtrSet<SelectInst *, 32> AllSelectNodes;
3948
3949public:
3950 Value *Get(Value *V) {
3951 do {
3952 auto SV = Storage.find(V);
3953 if (SV == Storage.end())
3954 return V;
3955 V = SV->second;
3956 } while (true);
3957 }
3958
3959 void Put(Value *From, Value *To) { Storage.insert({From, To}); }
3960
3961 void ReplacePhi(PHINode *From, PHINode *To) {
3962 Value *OldReplacement = Get(From);
3963 while (OldReplacement != From) {
3964 From = To;
3965 To = dyn_cast<PHINode>(OldReplacement);
3966 OldReplacement = Get(From);
3967 }
3968 assert(To && Get(To) == To && "Replacement PHI node is already replaced.");
3969 Put(From, To);
3970 From->replaceAllUsesWith(To);
3971 AllPhiNodes.erase(From);
3972 From->eraseFromParent();
3973 }
3974
3975 PhiNodeSet &newPhiNodes() { return AllPhiNodes; }
3976
3977 void insertNewPhi(PHINode *PN) { AllPhiNodes.insert(PN); }
3978
3979 void insertNewSelect(SelectInst *SI) { AllSelectNodes.insert(SI); }
3980
3981 unsigned countNewPhiNodes() const { return AllPhiNodes.size(); }
3982
3983 unsigned countNewSelectNodes() const { return AllSelectNodes.size(); }
3984
3985 void destroyNewNodes(Type *CommonType) {
3986 // For safe erasing, replace the uses with dummy value first.
3987 auto *Dummy = PoisonValue::get(CommonType);
3988 for (auto *I : AllPhiNodes) {
3989 I->replaceAllUsesWith(Dummy);
3990 I->eraseFromParent();
3991 }
3992 AllPhiNodes.clear();
3993 for (auto *I : AllSelectNodes) {
3994 I->replaceAllUsesWith(Dummy);
3995 I->eraseFromParent();
3996 }
3997 AllSelectNodes.clear();
3998 }
3999};
4000
4001/// A helper class for combining addressing modes.
4002class AddressingModeCombiner {
4003 typedef DenseMap<Value *, Value *> FoldAddrToValueMapping;
4004 typedef std::pair<PHINode *, PHINode *> PHIPair;
4005
4006private:
4007 /// The addressing modes we've collected.
4009
4010 /// The field in which the AddrModes differ, when we have more than one.
4011 ExtAddrMode::FieldName DifferentField = ExtAddrMode::NoField;
4012
4013 /// Are the AddrModes that we have all just equal to their original values?
4014 bool AllAddrModesTrivial = true;
4015
4016 /// Common Type for all different fields in addressing modes.
4017 Type *CommonType = nullptr;
4018
4019 const DataLayout &DL;
4020 const CodeGenOptions &Opts;
4021
4022 /// Original Address.
4023 Value *Original;
4024
4025 /// Common value among addresses
4026 Value *CommonValue = nullptr;
4027
4028public:
4029 AddressingModeCombiner(const DataLayout &DL, const CodeGenOptions &Opts,
4030 Value *OriginalValue)
4031 : DL(DL), Opts(Opts), Original(OriginalValue) {}
4032
4033 ~AddressingModeCombiner() { eraseCommonValueIfDead(); }
4034
4035 /// Get the combined AddrMode
4036 const ExtAddrMode &getAddrMode() const { return AddrModes[0]; }
4037
4038 /// Add a new AddrMode if it's compatible with the AddrModes we already
4039 /// have.
4040 /// \return True iff we succeeded in doing so.
4041 bool addNewAddrMode(ExtAddrMode &NewAddrMode) {
4042 // Take note of if we have any non-trivial AddrModes, as we need to detect
4043 // when all AddrModes are trivial as then we would introduce a phi or select
4044 // which just duplicates what's already there.
4045 AllAddrModesTrivial = AllAddrModesTrivial && NewAddrMode.isTrivial();
4046
4047 // If this is the first addrmode then everything is fine.
4048 if (AddrModes.empty()) {
4049 AddrModes.emplace_back(NewAddrMode);
4050 return true;
4051 }
4052
4053 // Figure out how different this is from the other address modes, which we
4054 // can do just by comparing against the first one given that we only care
4055 // about the cumulative difference.
4056 ExtAddrMode::FieldName ThisDifferentField =
4057 AddrModes[0].compare(NewAddrMode);
4058 if (DifferentField == ExtAddrMode::NoField)
4059 DifferentField = ThisDifferentField;
4060 else if (DifferentField != ThisDifferentField)
4061 DifferentField = ExtAddrMode::MultipleFields;
4062
4063 // If NewAddrMode differs in more than one dimension we cannot handle it.
4064 bool CanHandle = DifferentField != ExtAddrMode::MultipleFields;
4065
4066 // If Scale Field is different then we reject.
4067 CanHandle = CanHandle && DifferentField != ExtAddrMode::ScaleField;
4068
4069 // We also must reject the case when base offset is different and
4070 // scale reg is not null, we cannot handle this case due to merge of
4071 // different offsets will be used as ScaleReg.
4072 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseOffsField ||
4073 !NewAddrMode.ScaledReg);
4074
4075 // We also must reject the case when GV is different and BaseReg installed
4076 // due to we want to use base reg as a merge of GV values.
4077 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseGVField ||
4078 !NewAddrMode.HasBaseReg);
4079
4080 // Even if NewAddMode is the same we still need to collect it due to
4081 // original value is different. And later we will need all original values
4082 // as anchors during finding the common Phi node.
4083 if (CanHandle)
4084 AddrModes.emplace_back(NewAddrMode);
4085 else
4086 AddrModes.clear();
4087
4088 return CanHandle;
4089 }
4090
4091 /// Combine the addressing modes we've collected into a single
4092 /// addressing mode.
4093 /// \return True iff we successfully combined them or we only had one so
4094 /// didn't need to combine them anyway.
4095 bool combineAddrModes() {
4096 // If we have no AddrModes then they can't be combined.
4097 if (AddrModes.size() == 0)
4098 return false;
4099
4100 // A single AddrMode can trivially be combined.
4101 if (AddrModes.size() == 1 || DifferentField == ExtAddrMode::NoField)
4102 return true;
4103
4104 // If the AddrModes we collected are all just equal to the value they are
4105 // derived from then combining them wouldn't do anything useful.
4106 if (AllAddrModesTrivial)
4107 return false;
4108
4109 if (!addrModeCombiningAllowed())
4110 return false;
4111
4112 // Build a map between <original value, basic block where we saw it> to
4113 // value of base register.
4114 // Bail out if there is no common type.
4115 FoldAddrToValueMapping Map;
4116 if (!initializeMap(Map))
4117 return false;
4118
4119 CommonValue = findCommon(Map);
4120 if (CommonValue)
4121 AddrModes[0].SetCombinedField(DifferentField, CommonValue, AddrModes);
4122 return CommonValue != nullptr;
4123 }
4124
4125private:
4126 /// `CommonValue` may be a placeholder inserted by us.
4127 /// If the placeholder is not used, we should remove this dead instruction.
4128 void eraseCommonValueIfDead() {
4129 if (CommonValue && CommonValue->use_empty())
4130 if (Instruction *CommonInst = dyn_cast<Instruction>(CommonValue))
4131 CommonInst->eraseFromParent();
4132 }
4133
4134 /// Initialize Map with anchor values. For address seen
4135 /// we set the value of different field saw in this address.
4136 /// At the same time we find a common type for different field we will
4137 /// use to create new Phi/Select nodes. Keep it in CommonType field.
4138 /// Return false if there is no common type found.
4139 bool initializeMap(FoldAddrToValueMapping &Map) {
4140 // Keep track of keys where the value is null. We will need to replace it
4141 // with constant null when we know the common type.
4142 SmallVector<Value *, 2> NullValue;
4143 Type *IntPtrTy = DL.getIntPtrType(AddrModes[0].OriginalValue->getType());
4144 for (auto &AM : AddrModes) {
4145 Value *DV = AM.GetFieldAsValue(DifferentField, IntPtrTy);
4146 if (DV) {
4147 auto *Type = DV->getType();
4148 if (CommonType && CommonType != Type)
4149 return false;
4150 CommonType = Type;
4151 Map[AM.OriginalValue] = DV;
4152 } else {
4153 NullValue.push_back(AM.OriginalValue);
4154 }
4155 }
4156 assert(CommonType && "At least one non-null value must be!");
4157 for (auto *V : NullValue)
4158 Map[V] = Constant::getNullValue(CommonType);
4159 return true;
4160 }
4161
4162 /// We have mapping between value A and other value B where B was a field in
4163 /// addressing mode represented by A. Also we have an original value C
4164 /// representing an address we start with. Traversing from C through phi and
4165 /// selects we ended up with A's in a map. This utility function tries to find
4166 /// a value V which is a field in addressing mode C and traversing through phi
4167 /// nodes and selects we will end up in corresponded values B in a map.
4168 /// The utility will create a new Phi/Selects if needed.
4169 // The simple example looks as follows:
4170 // BB1:
4171 // p1 = b1 + 40
4172 // br cond BB2, BB3
4173 // BB2:
4174 // p2 = b2 + 40
4175 // br BB3
4176 // BB3:
4177 // p = phi [p1, BB1], [p2, BB2]
4178 // v = load p
4179 // Map is
4180 // p1 -> b1
4181 // p2 -> b2
4182 // Request is
4183 // p -> ?
4184 // The function tries to find or build phi [b1, BB1], [b2, BB2] in BB3.
4185 Value *findCommon(FoldAddrToValueMapping &Map) {
4186 // Tracks the simplification of newly created phi nodes. The reason we use
4187 // this mapping is because we will add new created Phi nodes in AddrToBase.
4188 // Simplification of Phi nodes is recursive, so some Phi node may
4189 // be simplified after we added it to AddrToBase. In reality this
4190 // simplification is possible only if original phi/selects were not
4191 // simplified yet.
4192 // Using this mapping we can find the current value in AddrToBase.
4193 SimplificationTracker ST;
4194
4195 // First step, DFS to create PHI nodes for all intermediate blocks.
4196 // Also fill traverse order for the second step.
4197 SmallVector<Value *, 32> TraverseOrder;
4198 InsertPlaceholders(Map, TraverseOrder, ST);
4199
4200 // Second Step, fill new nodes by merged values and simplify if possible.
4201 FillPlaceholders(Map, TraverseOrder, ST);
4202
4203 if (!Opts.cgp_addr_sink_new_select && ST.countNewSelectNodes() > 0) {
4204 ST.destroyNewNodes(CommonType);
4205 return nullptr;
4206 }
4207
4208 // Now we'd like to match New Phi nodes to existed ones.
4209 unsigned PhiNotMatchedCount = 0;
4210 if (!MatchPhiSet(ST, Opts.cgp_addr_sink_new_phis, PhiNotMatchedCount)) {
4211 ST.destroyNewNodes(CommonType);
4212 return nullptr;
4213 }
4214
4215 auto *Result = ST.Get(Map.find(Original)->second);
4216 if (Result) {
4217 NumMemoryInstsPhiCreated += ST.countNewPhiNodes() + PhiNotMatchedCount;
4218 NumMemoryInstsSelectCreated += ST.countNewSelectNodes();
4219 }
4220 return Result;
4221 }
4222
4223 /// Try to match PHI node to Candidate.
4224 /// Matcher tracks the matched Phi nodes.
4225 bool MatchPhiNode(PHINode *PHI, PHINode *Candidate,
4226 SmallSetVector<PHIPair, 8> &Matcher,
4227 PhiNodeSet &PhiNodesToMatch) {
4228 SmallVector<PHIPair, 8> WorkList;
4229 Matcher.insert({PHI, Candidate});
4230 SmallPtrSet<PHINode *, 8> MatchedPHIs;
4231 MatchedPHIs.insert(PHI);
4232 WorkList.push_back({PHI, Candidate});
4233 SmallSet<PHIPair, 8> Visited;
4234 while (!WorkList.empty()) {
4235 auto Item = WorkList.pop_back_val();
4236 if (!Visited.insert(Item).second)
4237 continue;
4238 // We iterate over all incoming values to Phi to compare them.
4239 // If values are different and both of them Phi and the first one is a
4240 // Phi we added (subject to match) and both of them is in the same basic
4241 // block then we can match our pair if values match. So we state that
4242 // these values match and add it to work list to verify that.
4243 for (auto *B : Item.first->blocks()) {
4244 Value *FirstValue = Item.first->getIncomingValueForBlock(B);
4245 Value *SecondValue = Item.second->getIncomingValueForBlock(B);
4246 if (FirstValue == SecondValue)
4247 continue;
4248
4249 PHINode *FirstPhi = dyn_cast<PHINode>(FirstValue);
4250 PHINode *SecondPhi = dyn_cast<PHINode>(SecondValue);
4251
4252 // One of them is not Phi or
4253 // The first one is not Phi node from the set we'd like to match or
4254 // Phi nodes from different basic blocks then
4255 // we will not be able to match.
4256 if (!FirstPhi || !SecondPhi || !PhiNodesToMatch.count(FirstPhi) ||
4257 FirstPhi->getParent() != SecondPhi->getParent())
4258 return false;
4259
4260 // If we already matched them then continue.
4261 if (Matcher.count({FirstPhi, SecondPhi}))
4262 continue;
4263 // So the values are different and does not match. So we need them to
4264 // match. (But we register no more than one match per PHI node, so that
4265 // we won't later try to replace them twice.)
4266 if (MatchedPHIs.insert(FirstPhi).second)
4267 Matcher.insert({FirstPhi, SecondPhi});
4268 // But me must check it.
4269 WorkList.push_back({FirstPhi, SecondPhi});
4270 }
4271 }
4272 return true;
4273 }
4274
4275 /// For the given set of PHI nodes (in the SimplificationTracker) try
4276 /// to find their equivalents.
4277 /// Returns false if this matching fails and creation of new Phi is disabled.
4278 bool MatchPhiSet(SimplificationTracker &ST, bool AllowNewPhiNodes,
4279 unsigned &PhiNotMatchedCount) {
4280 // Matched and PhiNodesToMatch iterate their elements in a deterministic
4281 // order, so the replacements (ReplacePhi) are also done in a deterministic
4282 // order.
4283 SmallSetVector<PHIPair, 8> Matched;
4284 SmallPtrSet<PHINode *, 8> WillNotMatch;
4285 PhiNodeSet &PhiNodesToMatch = ST.newPhiNodes();
4286 while (PhiNodesToMatch.size()) {
4287 PHINode *PHI = *PhiNodesToMatch.begin();
4288
4289 // Add us, if no Phi nodes in the basic block we do not match.
4290 WillNotMatch.clear();
4291 WillNotMatch.insert(PHI);
4292
4293 // Traverse all Phis until we found equivalent or fail to do that.
4294 bool IsMatched = false;
4295 for (auto &P : PHI->getParent()->phis()) {
4296 // Skip new Phi nodes.
4297 if (PhiNodesToMatch.count(&P))
4298 continue;
4299 if ((IsMatched = MatchPhiNode(PHI, &P, Matched, PhiNodesToMatch)))
4300 break;
4301 // If it does not match, collect all Phi nodes from matcher.
4302 // if we end up with no match, them all these Phi nodes will not match
4303 // later.
4304 WillNotMatch.insert_range(llvm::make_first_range(Matched));
4305 Matched.clear();
4306 }
4307 if (IsMatched) {
4308 // Replace all matched values and erase them.
4309 for (auto MV : Matched)
4310 ST.ReplacePhi(MV.first, MV.second);
4311 Matched.clear();
4312 continue;
4313 }
4314 // If we are not allowed to create new nodes then bail out.
4315 if (!AllowNewPhiNodes)
4316 return false;
4317 // Just remove all seen values in matcher. They will not match anything.
4318 PhiNotMatchedCount += WillNotMatch.size();
4319 for (auto *P : WillNotMatch)
4320 PhiNodesToMatch.erase(P);
4321 }
4322 return true;
4323 }
4324 /// Fill the placeholders with values from predecessors and simplify them.
4325 void FillPlaceholders(FoldAddrToValueMapping &Map,
4326 SmallVectorImpl<Value *> &TraverseOrder,
4327 SimplificationTracker &ST) {
4328 while (!TraverseOrder.empty()) {
4329 Value *Current = TraverseOrder.pop_back_val();
4330 assert(Map.contains(Current) && "No node to fill!!!");
4331 Value *V = Map[Current];
4332
4333 if (SelectInst *Select = dyn_cast<SelectInst>(V)) {
4334 // CurrentValue also must be Select.
4335 auto *CurrentSelect = cast<SelectInst>(Current);
4336 auto *TrueValue = CurrentSelect->getTrueValue();
4337 assert(Map.contains(TrueValue) && "No True Value!");
4338 Select->setTrueValue(ST.Get(Map[TrueValue]));
4339 auto *FalseValue = CurrentSelect->getFalseValue();
4340 assert(Map.contains(FalseValue) && "No False Value!");
4341 Select->setFalseValue(ST.Get(Map[FalseValue]));
4342 } else {
4343 // Must be a Phi node then.
4344 auto *PHI = cast<PHINode>(V);
4345 // Fill the Phi node with values from predecessors.
4346 for (auto *B : predecessors(PHI->getParent())) {
4347 Value *PV = cast<PHINode>(Current)->getIncomingValueForBlock(B);
4348 assert(Map.contains(PV) && "No predecessor Value!");
4349 PHI->addIncoming(ST.Get(Map[PV]), B);
4350 }
4351 }
4352 }
4353 }
4354
4355 /// Starting from original value recursively iterates over def-use chain up to
4356 /// known ending values represented in a map. For each traversed phi/select
4357 /// inserts a placeholder Phi or Select.
4358 /// Reports all new created Phi/Select nodes by adding them to set.
4359 /// Also reports and order in what values have been traversed.
4360 void InsertPlaceholders(FoldAddrToValueMapping &Map,
4361 SmallVectorImpl<Value *> &TraverseOrder,
4362 SimplificationTracker &ST) {
4363 SmallVector<Value *, 32> Worklist;
4364 assert((isa<PHINode>(Original) || isa<SelectInst>(Original)) &&
4365 "Address must be a Phi or Select node");
4366 auto *Dummy = PoisonValue::get(CommonType);
4367 Worklist.push_back(Original);
4368 while (!Worklist.empty()) {
4369 Value *Current = Worklist.pop_back_val();
4370 // if it is already visited or it is an ending value then skip it.
4371 if (Map.contains(Current))
4372 continue;
4373 TraverseOrder.push_back(Current);
4374
4375 // CurrentValue must be a Phi node or select. All others must be covered
4376 // by anchors.
4377 if (SelectInst *CurrentSelect = dyn_cast<SelectInst>(Current)) {
4378 // Is it OK to get metadata from OrigSelect?!
4379 // Create a Select placeholder with dummy value.
4380 SelectInst *Select =
4381 SelectInst::Create(CurrentSelect->getCondition(), Dummy, Dummy,
4382 CurrentSelect->getName(),
4383 CurrentSelect->getIterator(), CurrentSelect);
4384 Map[Current] = Select;
4385 ST.insertNewSelect(Select);
4386 // We are interested in True and False values.
4387 Worklist.push_back(CurrentSelect->getTrueValue());
4388 Worklist.push_back(CurrentSelect->getFalseValue());
4389 } else {
4390 // It must be a Phi node then.
4391 PHINode *CurrentPhi = cast<PHINode>(Current);
4392 unsigned PredCount = CurrentPhi->getNumIncomingValues();
4393 PHINode *PHI =
4394 PHINode::Create(CommonType, PredCount, "sunk_phi", CurrentPhi->getIterator());
4395 Map[Current] = PHI;
4396 ST.insertNewPhi(PHI);
4397 append_range(Worklist, CurrentPhi->incoming_values());
4398 }
4399 }
4400 }
4401
4402 bool addrModeCombiningAllowed() {
4403 if (!Opts.cgp_complex_addr_modes)
4404 return false;
4405 switch (DifferentField) {
4406 default:
4407 return false;
4408 case ExtAddrMode::BaseRegField:
4409 return Opts.cgp_addr_sink_combine_base_reg;
4410 case ExtAddrMode::BaseGVField:
4411 return Opts.cgp_addr_sink_combine_base_gv;
4412 case ExtAddrMode::BaseOffsField:
4413 return Opts.cgp_addr_sink_combine_base_offs;
4414 case ExtAddrMode::ScaledRegField:
4415 return Opts.cgp_addr_sink_combine_scaled_reg;
4416 }
4417 }
4418};
4419} // end anonymous namespace
4420
4421/// Try adding ScaleReg*Scale to the current addressing mode.
4422/// Return true and update AddrMode if this addr mode is legal for the target,
4423/// false if not.
4424bool AddressingModeMatcher::matchScaledValue(Value *ScaleReg, int64_t Scale,
4425 unsigned Depth) {
4426 // If Scale is 1, then this is the same as adding ScaleReg to the addressing
4427 // mode. Just process that directly.
4428 if (Scale == 1)
4429 return matchAddr(ScaleReg, Depth);
4430
4431 // If the scale is 0, it takes nothing to add this.
4432 if (Scale == 0)
4433 return true;
4434
4435 // If we already have a scale of this value, we can add to it, otherwise, we
4436 // need an available scale field.
4437 if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg)
4438 return false;
4439
4440 ExtAddrMode TestAddrMode = AddrMode;
4441
4442 // Add scale to turn X*4+X*3 -> X*7. This could also do things like
4443 // [A+B + A*7] -> [B+A*8].
4444 TestAddrMode.Scale += Scale;
4445 TestAddrMode.ScaledReg = ScaleReg;
4446
4447 // If the new address isn't legal, bail out.
4448 if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace))
4449 return false;
4450
4451 // It was legal, so commit it.
4452 AddrMode = TestAddrMode;
4453
4454 // Okay, we decided that we can add ScaleReg+Scale to AddrMode. Check now
4455 // to see if ScaleReg is actually X+C. If so, we can turn this into adding
4456 // X*Scale + C*Scale to addr mode. If we found available IV increment, do not
4457 // go any further: we can reuse it and cannot eliminate it.
4458 ConstantInt *CI = nullptr;
4459 Value *AddLHS = nullptr;
4460 if (isa<Instruction>(ScaleReg) && // not a constant expr.
4461 match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI))) &&
4462 !isIVIncrement(ScaleReg, &LI) && CI->getValue().isSignedIntN(64)) {
4463 TestAddrMode.InBounds = false;
4464 TestAddrMode.ScaledReg = AddLHS;
4465 TestAddrMode.BaseOffs += CI->getSExtValue() * TestAddrMode.Scale;
4466
4467 // If this addressing mode is legal, commit it and remember that we folded
4468 // this instruction.
4469 if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) {
4470 AddrModeInsts.push_back(cast<Instruction>(ScaleReg));
4471 AddrMode = TestAddrMode;
4472 return true;
4473 }
4474 // Restore status quo.
4475 TestAddrMode = AddrMode;
4476 }
4477
4478 // If this is an add recurrence with a constant step, return the increment
4479 // instruction and the canonicalized step.
4480 auto GetConstantStep =
4481 [this](const Value *V) -> std::optional<std::pair<Instruction *, APInt>> {
4482 auto *PN = dyn_cast<PHINode>(V);
4483 if (!PN)
4484 return std::nullopt;
4485 auto IVInc = getIVIncrement(PN, &LI);
4486 if (!IVInc)
4487 return std::nullopt;
4488 // TODO: The result of the intrinsics above is two-complement. However when
4489 // IV inc is expressed as add or sub, iv.next is potentially a poison value.
4490 // If it has nuw or nsw flags, we need to make sure that these flags are
4491 // inferrable at the point of memory instruction. Otherwise we are replacing
4492 // well-defined two-complement computation with poison. Currently, to avoid
4493 // potentially complex analysis needed to prove this, we reject such cases.
4494 if (auto *OIVInc = dyn_cast<OverflowingBinaryOperator>(IVInc->first))
4495 if (OIVInc->hasNoSignedWrap() || OIVInc->hasNoUnsignedWrap())
4496 return std::nullopt;
4497 if (auto *ConstantStep = dyn_cast<ConstantInt>(IVInc->second))
4498 return std::make_pair(IVInc->first, ConstantStep->getValue());
4499 return std::nullopt;
4500 };
4501
4502 // Try to account for the following special case:
4503 // 1. ScaleReg is an inductive variable;
4504 // 2. We use it with non-zero offset;
4505 // 3. IV's increment is available at the point of memory instruction.
4506 //
4507 // In this case, we may reuse the IV increment instead of the IV Phi to
4508 // achieve the following advantages:
4509 // 1. If IV step matches the offset, we will have no need in the offset;
4510 // 2. Even if they don't match, we will reduce the overlap of living IV
4511 // and IV increment, that will potentially lead to better register
4512 // assignment.
4513 if (AddrMode.BaseOffs) {
4514 if (auto IVStep = GetConstantStep(ScaleReg)) {
4515 Instruction *IVInc = IVStep->first;
4516 // The following assert is important to ensure a lack of infinite loops.
4517 // This transforms is (intentionally) the inverse of the one just above.
4518 // If they don't agree on the definition of an increment, we'd alternate
4519 // back and forth indefinitely.
4520 assert(isIVIncrement(IVInc, &LI) && "implied by GetConstantStep");
4521 APInt Step = IVStep->second;
4522 APInt Offset = Step * AddrMode.Scale;
4523 if (Offset.isSignedIntN(64)) {
4524 TestAddrMode.InBounds = false;
4525 TestAddrMode.ScaledReg = IVInc;
4526 TestAddrMode.BaseOffs -= Offset.getLimitedValue();
4527 // If this addressing mode is legal, commit it..
4528 // (Note that we defer the (expensive) domtree base legality check
4529 // to the very last possible point.)
4530 if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace) &&
4531 getDTFn().dominates(IVInc, MemoryInst)) {
4532 AddrModeInsts.push_back(cast<Instruction>(IVInc));
4533 AddrMode = TestAddrMode;
4534 return true;
4535 }
4536 // Restore status quo.
4537 TestAddrMode = AddrMode;
4538 }
4539 }
4540 }
4541
4542 // Otherwise, just return what we have.
4543 return true;
4544}
4545
4546/// This is a little filter, which returns true if an addressing computation
4547/// involving I might be folded into a load/store accessing it.
4548/// This doesn't need to be perfect, but needs to accept at least
4549/// the set of instructions that MatchOperationAddr can.
4551 switch (I->getOpcode()) {
4552 case Instruction::BitCast:
4553 case Instruction::AddrSpaceCast:
4554 // Don't touch identity bitcasts.
4555 if (I->getType() == I->getOperand(0)->getType())
4556 return false;
4557 return I->getType()->isIntOrPtrTy();
4558 case Instruction::PtrToInt:
4559 // PtrToInt is always a noop, as we know that the int type is pointer sized.
4560 return true;
4561 case Instruction::IntToPtr:
4562 // We know the input is intptr_t, so this is foldable.
4563 return true;
4564 case Instruction::Add:
4565 return true;
4566 case Instruction::Or:
4567 return cast<PossiblyDisjointInst>(I)->isDisjoint();
4568 case Instruction::Mul:
4569 case Instruction::Shl:
4570 // Can only handle X*C and X << C.
4571 return isa<ConstantInt>(I->getOperand(1));
4572 case Instruction::GetElementPtr:
4573 return true;
4574 default:
4575 return false;
4576 }
4577}
4578
4579/// Check whether or not \p Val is a legal instruction for \p TLI.
4580/// \note \p Val is assumed to be the product of some type promotion.
4581/// Therefore if \p Val has an undefined state in \p TLI, this is assumed
4582/// to be legal, as the non-promoted value would have had the same state.
4584 const DataLayout &DL, Value *Val) {
4585 Instruction *PromotedInst = dyn_cast<Instruction>(Val);
4586 if (!PromotedInst)
4587 return false;
4588 int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode());
4589 // If the ISDOpcode is undefined, it was undefined before the promotion.
4590 if (!ISDOpcode)
4591 return true;
4592 // Otherwise, check if the promoted instruction is legal or not.
4593 return TLI.isOperationLegalOrCustom(
4594 ISDOpcode, TLI.getValueType(DL, PromotedInst->getType()));
4595}
4596
4597namespace {
4598
4599/// Hepler class to perform type promotion.
4600class TypePromotionHelper {
4601 /// Utility function to add a promoted instruction \p ExtOpnd to
4602 /// \p PromotedInsts and record the type of extension we have seen.
4603 static void addPromotedInst(InstrToOrigTy &PromotedInsts,
4604 Instruction *ExtOpnd, bool IsSExt) {
4605 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;
4606 auto [It, Inserted] = PromotedInsts.try_emplace(ExtOpnd);
4607 if (!Inserted) {
4608 // If the new extension is same as original, the information in
4609 // PromotedInsts[ExtOpnd] is still correct.
4610 if (It->second.getInt() == ExtTy)
4611 return;
4612
4613 // Now the new extension is different from old extension, we make
4614 // the type information invalid by setting extension type to
4615 // BothExtension.
4616 ExtTy = BothExtension;
4617 }
4618 It->second = TypeIsSExt(ExtOpnd->getType(), ExtTy);
4619 }
4620
4621 /// Utility function to query the original type of instruction \p Opnd
4622 /// with a matched extension type. If the extension doesn't match, we
4623 /// cannot use the information we had on the original type.
4624 /// BothExtension doesn't match any extension type.
4625 static const Type *getOrigType(const InstrToOrigTy &PromotedInsts,
4626 Instruction *Opnd, bool IsSExt) {
4627 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;
4628 InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
4629 if (It != PromotedInsts.end() && It->second.getInt() == ExtTy)
4630 return It->second.getPointer();
4631 return nullptr;
4632 }
4633
4634 /// Utility function to check whether or not a sign or zero extension
4635 /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by
4636 /// either using the operands of \p Inst or promoting \p Inst.
4637 /// The type of the extension is defined by \p IsSExt.
4638 /// In other words, check if:
4639 /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType.
4640 /// #1 Promotion applies:
4641 /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...).
4642 /// #2 Operand reuses:
4643 /// ext opnd1 to ConsideredExtType.
4644 /// \p PromotedInsts maps the instructions to their type before promotion.
4645 static bool canGetThrough(const Instruction *Inst, Type *ConsideredExtType,
4646 const InstrToOrigTy &PromotedInsts, bool IsSExt);
4647
4648 /// Utility function to determine if \p OpIdx should be promoted when
4649 /// promoting \p Inst.
4650 static bool shouldExtOperand(const Instruction *Inst, int OpIdx) {
4651 return !(isa<SelectInst>(Inst) && OpIdx == 0);
4652 }
4653
4654 /// Utility function to promote the operand of \p Ext when this
4655 /// operand is a promotable trunc or sext or zext.
4656 /// \p PromotedInsts maps the instructions to their type before promotion.
4657 /// \p CreatedInstsCost[out] contains the cost of all instructions
4658 /// created to promote the operand of Ext.
4659 /// Newly added extensions are inserted in \p Exts.
4660 /// Newly added truncates are inserted in \p Truncs.
4661 /// Should never be called directly.
4662 /// \return The promoted value which is used instead of Ext.
4663 static Value *promoteOperandForTruncAndAnyExt(
4664 Instruction *Ext, TypePromotionTransaction &TPT,
4665 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
4666 SmallVectorImpl<Instruction *> *Exts,
4667 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI);
4668
4669 /// Utility function to promote the operand of \p Ext when this
4670 /// operand is promotable and is not a supported trunc or sext.
4671 /// \p PromotedInsts maps the instructions to their type before promotion.
4672 /// \p CreatedInstsCost[out] contains the cost of all the instructions
4673 /// created to promote the operand of Ext.
4674 /// Newly added extensions are inserted in \p Exts.
4675 /// Newly added truncates are inserted in \p Truncs.
4676 /// Should never be called directly.
4677 /// \return The promoted value which is used instead of Ext.
4678 static Value *promoteOperandForOther(Instruction *Ext,
4679 TypePromotionTransaction &TPT,
4680 InstrToOrigTy &PromotedInsts,
4681 unsigned &CreatedInstsCost,
4682 SmallVectorImpl<Instruction *> *Exts,
4683 SmallVectorImpl<Instruction *> *Truncs,
4684 const TargetLowering &TLI, bool IsSExt);
4685
4686 /// \see promoteOperandForOther.
4687 static Value *signExtendOperandForOther(
4688 Instruction *Ext, TypePromotionTransaction &TPT,
4689 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
4690 SmallVectorImpl<Instruction *> *Exts,
4691 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
4692 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
4693 Exts, Truncs, TLI, true);
4694 }
4695
4696 /// \see promoteOperandForOther.
4697 static Value *zeroExtendOperandForOther(
4698 Instruction *Ext, TypePromotionTransaction &TPT,
4699 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
4700 SmallVectorImpl<Instruction *> *Exts,
4701 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
4702 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
4703 Exts, Truncs, TLI, false);
4704 }
4705
4706public:
4707 /// Type for the utility function that promotes the operand of Ext.
4708 using Action = Value *(*)(Instruction *Ext, TypePromotionTransaction &TPT,
4709 InstrToOrigTy &PromotedInsts,
4710 unsigned &CreatedInstsCost,
4711 SmallVectorImpl<Instruction *> *Exts,
4712 SmallVectorImpl<Instruction *> *Truncs,
4713 const TargetLowering &TLI);
4714
4715 /// Given a sign/zero extend instruction \p Ext, return the appropriate
4716 /// action to promote the operand of \p Ext instead of using Ext.
4717 /// \return NULL if no promotable action is possible with the current
4718 /// sign extension.
4719 /// \p InsertedInsts keeps track of all the instructions inserted by the
4720 /// other CodeGenPrepare optimizations. This information is important
4721 /// because we do not want to promote these instructions as CodeGenPrepare
4722 /// will reinsert them later. Thus creating an infinite loop: create/remove.
4723 /// \p PromotedInsts maps the instructions to their type before promotion.
4724 static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedInsts,
4725 const TargetLowering &TLI,
4726 const InstrToOrigTy &PromotedInsts);
4727};
4728
4729} // end anonymous namespace
4730
4731bool TypePromotionHelper::canGetThrough(const Instruction *Inst,
4732 Type *ConsideredExtType,
4733 const InstrToOrigTy &PromotedInsts,
4734 bool IsSExt) {
4735 // The promotion helper does not know how to deal with vector types yet.
4736 // To be able to fix that, we would need to fix the places where we
4737 // statically extend, e.g., constants and such.
4738 if (Inst->getType()->isVectorTy())
4739 return false;
4740
4741 // We can always get through zext.
4742 if (isa<ZExtInst>(Inst))
4743 return true;
4744
4745 // sext(sext) is ok too.
4746 if (IsSExt && isa<SExtInst>(Inst))
4747 return true;
4748
4749 // We can get through binary operator, if it is legal. In other words, the
4750 // binary operator must have a nuw or nsw flag.
4751 if (const auto *BinOp = dyn_cast<BinaryOperator>(Inst))
4752 if (isa<OverflowingBinaryOperator>(BinOp) &&
4753 ((!IsSExt && BinOp->hasNoUnsignedWrap()) ||
4754 (IsSExt && BinOp->hasNoSignedWrap())))
4755 return true;
4756
4757 // ext(and(opnd, cst)) --> and(ext(opnd), ext(cst))
4758 if ((Inst->getOpcode() == Instruction::And ||
4759 Inst->getOpcode() == Instruction::Or))
4760 return true;
4761
4762 // ext(xor(opnd, cst)) --> xor(ext(opnd), ext(cst))
4763 if (Inst->getOpcode() == Instruction::Xor) {
4764 // Make sure it is not a NOT.
4765 if (const auto *Cst = dyn_cast<ConstantInt>(Inst->getOperand(1)))
4766 if (!Cst->getValue().isAllOnes())
4767 return true;
4768 }
4769
4770 // zext(shrl(opnd, cst)) --> shrl(zext(opnd), zext(cst))
4771 // It may change a poisoned value into a regular value, like
4772 // zext i32 (shrl i8 %val, 12) --> shrl i32 (zext i8 %val), 12
4773 // poisoned value regular value
4774 // It should be OK since undef covers valid value.
4775 if (Inst->getOpcode() == Instruction::LShr && !IsSExt)
4776 return true;
4777
4778 // and(ext(shl(opnd, cst)), cst) --> and(shl(ext(opnd), ext(cst)), cst)
4779 // It may change a poisoned value into a regular value, like
4780 // zext i32 (shl i8 %val, 12) --> shl i32 (zext i8 %val), 12
4781 // poisoned value regular value
4782 // It should be OK since undef covers valid value.
4783 if (Inst->getOpcode() == Instruction::Shl && Inst->hasOneUse()) {
4784 const auto *ExtInst = cast<const Instruction>(*Inst->user_begin());
4785 if (ExtInst->hasOneUse()) {
4786 const auto *AndInst = dyn_cast<const Instruction>(*ExtInst->user_begin());
4787 if (AndInst && AndInst->getOpcode() == Instruction::And) {
4788 const auto *Cst = dyn_cast<ConstantInt>(AndInst->getOperand(1));
4789 if (Cst &&
4790 Cst->getValue().isIntN(Inst->getType()->getIntegerBitWidth()))
4791 return true;
4792 }
4793 }
4794 }
4795
4796 // Check if we can do the following simplification.
4797 // ext(trunc(opnd)) --> ext(opnd)
4798 if (!isa<TruncInst>(Inst))
4799 return false;
4800
4801 Value *OpndVal = Inst->getOperand(0);
4802 // Check if we can use this operand in the extension.
4803 // If the type is larger than the result type of the extension, we cannot.
4804 if (!OpndVal->getType()->isIntegerTy() ||
4805 OpndVal->getType()->getIntegerBitWidth() >
4806 ConsideredExtType->getIntegerBitWidth())
4807 return false;
4808
4809 // If the operand of the truncate is not an instruction, we will not have
4810 // any information on the dropped bits.
4811 // (Actually we could for constant but it is not worth the extra logic).
4812 Instruction *Opnd = dyn_cast<Instruction>(OpndVal);
4813 if (!Opnd)
4814 return false;
4815
4816 // Check if the source of the type is narrow enough.
4817 // I.e., check that trunc just drops extended bits of the same kind of
4818 // the extension.
4819 // #1 get the type of the operand and check the kind of the extended bits.
4820 const Type *OpndType = getOrigType(PromotedInsts, Opnd, IsSExt);
4821 if (OpndType)
4822 ;
4823 else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))
4824 OpndType = Opnd->getOperand(0)->getType();
4825 else
4826 return false;
4827
4828 // #2 check that the truncate just drops extended bits.
4829 return Inst->getType()->getIntegerBitWidth() >=
4830 OpndType->getIntegerBitWidth();
4831}
4832
4833TypePromotionHelper::Action TypePromotionHelper::getAction(
4834 Instruction *Ext, const SetOfInstrs &InsertedInsts,
4835 const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) {
4836 assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
4837 "Unexpected instruction type");
4838 Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0));
4839 Type *ExtTy = Ext->getType();
4840 bool IsSExt = isa<SExtInst>(Ext);
4841 // If the operand of the extension is not an instruction, we cannot
4842 // get through.
4843 // If it, check we can get through.
4844 if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))
4845 return nullptr;
4846
4847 // Do not promote if the operand has been added by codegenprepare.
4848 // Otherwise, it means we are undoing an optimization that is likely to be
4849 // redone, thus causing potential infinite loop.
4850 if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))
4851 return nullptr;
4852
4853 // SExt or Trunc instructions.
4854 // Return the related handler.
4855 if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) ||
4856 isa<ZExtInst>(ExtOpnd))
4857 return promoteOperandForTruncAndAnyExt;
4858
4859 // Regular instruction.
4860 // Abort early if we will have to insert non-free instructions.
4861 if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType()))
4862 return nullptr;
4863 return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;
4864}
4865
4866Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(
4867 Instruction *SExt, TypePromotionTransaction &TPT,
4868 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
4869 SmallVectorImpl<Instruction *> *Exts,
4870 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
4871 // By construction, the operand of SExt is an instruction. Otherwise we cannot
4872 // get through it and this method should not be called.
4873 Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0));
4874 Value *ExtVal = SExt;
4875 bool HasMergedNonFreeExt = false;
4876 if (isa<ZExtInst>(SExtOpnd)) {
4877 // Replace s|zext(zext(opnd))
4878 // => zext(opnd).
4879 HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd);
4880 Value *ZExt =
4881 TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType());
4882 TPT.replaceAllUsesWith(SExt, ZExt);
4883 TPT.eraseInstruction(SExt);
4884 ExtVal = ZExt;
4885 } else {
4886 // Replace z|sext(trunc(opnd)) or sext(sext(opnd))
4887 // => z|sext(opnd).
4888 TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0));
4889 }
4890 CreatedInstsCost = 0;
4891
4892 // Remove dead code.
4893 if (SExtOpnd->use_empty())
4894 TPT.eraseInstruction(SExtOpnd);
4895
4896 // Check if the extension is still needed.
4897 Instruction *ExtInst = dyn_cast<Instruction>(ExtVal);
4898 if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) {
4899 if (ExtInst) {
4900 if (Exts)
4901 Exts->push_back(ExtInst);
4902 CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt;
4903 }
4904 return ExtVal;
4905 }
4906
4907 // At this point we have: ext ty opnd to ty.
4908 // Reassign the uses of ExtInst to the opnd and remove ExtInst.
4909 Value *NextVal = ExtInst->getOperand(0);
4910 TPT.eraseInstruction(ExtInst, NextVal);
4911 return NextVal;
4912}
4913
4914Value *TypePromotionHelper::promoteOperandForOther(
4915 Instruction *Ext, TypePromotionTransaction &TPT,
4916 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
4917 SmallVectorImpl<Instruction *> *Exts,
4918 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI,
4919 bool IsSExt) {
4920 // By construction, the operand of Ext is an instruction. Otherwise we cannot
4921 // get through it and this method should not be called.
4922 Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0));
4923 CreatedInstsCost = 0;
4924 if (!ExtOpnd->hasOneUse()) {
4925 // ExtOpnd will be promoted.
4926 // All its uses, but Ext, will need to use a truncated value of the
4927 // promoted version.
4928 // Create the truncate now.
4929 Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType());
4930 if (Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) {
4931 // Insert it just after the definition.
4932 ITrunc->moveAfter(ExtOpnd);
4933 if (Truncs)
4934 Truncs->push_back(ITrunc);
4935 }
4936
4937 TPT.replaceAllUsesWith(ExtOpnd, Trunc);
4938 // Restore the operand of Ext (which has been replaced by the previous call
4939 // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext.
4940 TPT.setOperand(Ext, 0, ExtOpnd);
4941 }
4942
4943 // Get through the Instruction:
4944 // 1. Update its type.
4945 // 2. Replace the uses of Ext by Inst.
4946 // 3. Extend each operand that needs to be extended.
4947
4948 // Remember the original type of the instruction before promotion.
4949 // This is useful to know that the high bits are sign extended bits.
4950 addPromotedInst(PromotedInsts, ExtOpnd, IsSExt);
4951 // Step #1.
4952 TPT.mutateType(ExtOpnd, Ext->getType());
4953 // Step #2.
4954 TPT.replaceAllUsesWith(Ext, ExtOpnd);
4955 // Step #3.
4956 LLVM_DEBUG(dbgs() << "Propagate Ext to operands\n");
4957 for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx;
4958 ++OpIdx) {
4959 LLVM_DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n');
4960 if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() ||
4961 !shouldExtOperand(ExtOpnd, OpIdx)) {
4962 LLVM_DEBUG(dbgs() << "No need to propagate\n");
4963 continue;
4964 }
4965 // Check if we can statically extend the operand.
4966 Value *Opnd = ExtOpnd->getOperand(OpIdx);
4967 if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) {
4968 LLVM_DEBUG(dbgs() << "Statically extend\n");
4969 unsigned BitWidth = Ext->getType()->getIntegerBitWidth();
4970 APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth)
4971 : Cst->getValue().zext(BitWidth);
4972 TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal));
4973 continue;
4974 }
4975 // UndefValue are typed, so we have to statically sign extend them.
4976 if (isa<UndefValue>(Opnd)) {
4977 LLVM_DEBUG(dbgs() << "Statically extend\n");
4978 TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType()));
4979 continue;
4980 }
4981
4982 // Otherwise we have to explicitly sign extend the operand.
4983 Value *ValForExtOpnd = IsSExt
4984 ? TPT.createSExt(ExtOpnd, Opnd, Ext->getType())
4985 : TPT.createZExt(ExtOpnd, Opnd, Ext->getType());
4986 TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);
4987 Instruction *InstForExtOpnd = dyn_cast<Instruction>(ValForExtOpnd);
4988 if (!InstForExtOpnd)
4989 continue;
4990
4991 if (Exts)
4992 Exts->push_back(InstForExtOpnd);
4993
4994 CreatedInstsCost += !TLI.isExtFree(InstForExtOpnd);
4995 }
4996 LLVM_DEBUG(dbgs() << "Extension is useless now\n");
4997 TPT.eraseInstruction(Ext);
4998 return ExtOpnd;
4999}
5000
5001/// Check whether or not promoting an instruction to a wider type is profitable.
5002/// \p NewCost gives the cost of extension instructions created by the
5003/// promotion.
5004/// \p OldCost gives the cost of extension instructions before the promotion
5005/// plus the number of instructions that have been
5006/// matched in the addressing mode the promotion.
5007/// \p PromotedOperand is the value that has been promoted.
5008/// \return True if the promotion is profitable, false otherwise.
5009bool AddressingModeMatcher::isPromotionProfitable(
5010 unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const {
5011 LLVM_DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost
5012 << '\n');
5013 // The cost of the new extensions is greater than the cost of the
5014 // old extension plus what we folded.
5015 // This is not profitable.
5016 if (NewCost > OldCost)
5017 return false;
5018 if (NewCost < OldCost)
5019 return true;
5020 // The promotion is neutral but it may help folding the sign extension in
5021 // loads for instance.
5022 // Check that we did not create an illegal instruction.
5023 return isPromotedInstructionLegal(TLI, DL, PromotedOperand);
5024}
5025
5026/// Given an instruction or constant expr, see if we can fold the operation
5027/// into the addressing mode. If so, update the addressing mode and return
5028/// true, otherwise return false without modifying AddrMode.
5029/// If \p MovedAway is not NULL, it contains the information of whether or
5030/// not AddrInst has to be folded into the addressing mode on success.
5031/// If \p MovedAway == true, \p AddrInst will not be part of the addressing
5032/// because it has been moved away.
5033/// Thus AddrInst must not be added in the matched instructions.
5034/// This state can happen when AddrInst is a sext, since it may be moved away.
5035/// Therefore, AddrInst may not be valid when MovedAway is true and it must
5036/// not be referenced anymore.
5037bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,
5038 unsigned Depth,
5039 bool *MovedAway) {
5040 // Avoid exponential behavior on extremely deep expression trees.
5041 if (Depth >= 5)
5042 return false;
5043
5044 // By default, all matched instructions stay in place.
5045 if (MovedAway)
5046 *MovedAway = false;
5047
5048 switch (Opcode) {
5049 case Instruction::PtrToInt:
5050 // PtrToInt is always a noop, as we know that the int type is pointer sized.
5051 return matchAddr(AddrInst->getOperand(0), Depth);
5052 case Instruction::IntToPtr: {
5053 auto AS = AddrInst->getType()->getPointerAddressSpace();
5054 auto PtrTy = MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
5055 // This inttoptr is a no-op if the integer type is pointer sized.
5056 if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy)
5057 return matchAddr(AddrInst->getOperand(0), Depth);
5058 return false;
5059 }
5060 case Instruction::BitCast:
5061 // BitCast is always a noop, and we can handle it as long as it is
5062 // int->int or pointer->pointer (we don't want int<->fp or something).
5063 if (AddrInst->getOperand(0)->getType()->isIntOrPtrTy() &&
5064 // Don't touch identity bitcasts. These were probably put here by LSR,
5065 // and we don't want to mess around with them. Assume it knows what it
5066 // is doing.
5067 AddrInst->getOperand(0)->getType() != AddrInst->getType())
5068 return matchAddr(AddrInst->getOperand(0), Depth);
5069 return false;
5070 case Instruction::AddrSpaceCast: {
5071 unsigned SrcAS =
5072 AddrInst->getOperand(0)->getType()->getPointerAddressSpace();
5073 unsigned DestAS = AddrInst->getType()->getPointerAddressSpace();
5074 if (TLI.getTargetMachine().isNoopAddrSpaceCast(DL, SrcAS, DestAS))
5075 return matchAddr(AddrInst->getOperand(0), Depth);
5076 return false;
5077 }
5078 case Instruction::Or:
5079 if (!cast<PossiblyDisjointInst>(AddrInst)->isDisjoint())
5080 return false;
5081 [[fallthrough]];
5082 case Instruction::Add: {
5083 // Check to see if we can merge in one operand, then the other. If so, we
5084 // win.
5085 ExtAddrMode BackupAddrMode = AddrMode;
5086 unsigned OldSize = AddrModeInsts.size();
5087 // Start a transaction at this point.
5088 // The LHS may match but not the RHS.
5089 // Therefore, we need a higher level restoration point to undo partially
5090 // matched operation.
5091 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5092 TPT.getRestorationPoint();
5093
5094 // Try to match an integer constant second to increase its chance of ending
5095 // up in `BaseOffs`, resp. decrease its chance of ending up in `BaseReg`.
5096 int First = 0, Second = 1;
5097 if (isa<ConstantInt>(AddrInst->getOperand(First))
5098 && !isa<ConstantInt>(AddrInst->getOperand(Second)))
5099 std::swap(First, Second);
5100 AddrMode.InBounds = false;
5101 if (matchAddr(AddrInst->getOperand(First), Depth + 1) &&
5102 matchAddr(AddrInst->getOperand(Second), Depth + 1))
5103 return true;
5104
5105 // Restore the old addr mode info.
5106 AddrMode = BackupAddrMode;
5107 AddrModeInsts.resize(OldSize);
5108 TPT.rollback(LastKnownGood);
5109
5110 // Otherwise this was over-aggressive. Try merging operands in the opposite
5111 // order.
5112 if (matchAddr(AddrInst->getOperand(Second), Depth + 1) &&
5113 matchAddr(AddrInst->getOperand(First), Depth + 1))
5114 return true;
5115
5116 // Otherwise we definitely can't merge the ADD in.
5117 AddrMode = BackupAddrMode;
5118 AddrModeInsts.resize(OldSize);
5119 TPT.rollback(LastKnownGood);
5120 break;
5121 }
5122 case Instruction::Mul:
5123 case Instruction::Shl: {
5124 // Can only handle X*C and X << C.
5125 AddrMode.InBounds = false;
5126 ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1));
5127 if (!RHS || RHS->getBitWidth() > 64)
5128 return false;
5129 int64_t Scale = Opcode == Instruction::Shl
5130 ? 1LL << RHS->getLimitedValue(RHS->getBitWidth() - 1)
5131 : RHS->getSExtValue();
5132
5133 return matchScaledValue(AddrInst->getOperand(0), Scale, Depth);
5134 }
5135 case Instruction::GetElementPtr: {
5136 // Scan the GEP. We check it if it contains constant offsets and at most
5137 // one variable offset.
5138 int VariableOperand = -1;
5139 unsigned VariableScale = 0;
5140
5141 int64_t ConstantOffset = 0;
5142 gep_type_iterator GTI = gep_type_begin(AddrInst);
5143 for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {
5144 if (StructType *STy = GTI.getStructTypeOrNull()) {
5145 const StructLayout *SL = DL.getStructLayout(STy);
5146 unsigned Idx =
5147 cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();
5148 ConstantOffset += SL->getElementOffset(Idx);
5149 } else {
5150 TypeSize TS = GTI.getSequentialElementStride(DL);
5151 if (TS.isNonZero()) {
5152 // The optimisations below currently only work for fixed offsets.
5153 if (TS.isScalable())
5154 return false;
5155 int64_t TypeSize = TS.getFixedValue();
5156 if (ConstantInt *CI =
5157 dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {
5158 const APInt &CVal = CI->getValue();
5159 if (CVal.getSignificantBits() <= 64) {
5160 ConstantOffset += CVal.getSExtValue() * TypeSize;
5161 continue;
5162 }
5163 }
5164 // We only allow one variable index at the moment.
5165 if (VariableOperand != -1)
5166 return false;
5167
5168 // Remember the variable index.
5169 VariableOperand = i;
5170 VariableScale = TypeSize;
5171 }
5172 }
5173 }
5174
5175 // A common case is for the GEP to only do a constant offset. In this case,
5176 // just add it to the disp field and check validity.
5177 if (VariableOperand == -1) {
5178 AddrMode.BaseOffs += ConstantOffset;
5179 if (matchAddr(AddrInst->getOperand(0), Depth + 1)) {
5180 if (!cast<GEPOperator>(AddrInst)->isInBounds())
5181 AddrMode.InBounds = false;
5182 return true;
5183 }
5184 AddrMode.BaseOffs -= ConstantOffset;
5185
5186 if (Opts.cgp_split_large_offset_gep && isa<GetElementPtrInst>(AddrInst) &&
5188 ConstantOffset != 0) {
5189 // Record GEPs with non-zero offsets as candidates for splitting in
5190 // the event that the offset cannot fit into the r+i addressing mode.
5191 // Simple and common case that only one GEP is used in calculating the
5192 // address for the memory access.
5193 Value *Base = AddrInst->getOperand(0);
5194 auto *BaseI = dyn_cast<Instruction>(Base);
5195 auto *GEP = cast<GetElementPtrInst>(AddrInst);
5197 (BaseI && !isa<CastInst>(BaseI) &&
5198 !isa<GetElementPtrInst>(BaseI))) {
5199 // Make sure the parent block allows inserting non-PHI instructions
5200 // before the terminator.
5201 BasicBlock *Parent =
5202 BaseI ? BaseI->getParent() : &GEP->getFunction()->getEntryBlock();
5203 if (!Parent->getTerminator()->isEHPad())
5204 LargeOffsetGEP = std::make_pair(GEP, ConstantOffset);
5205 }
5206 }
5207
5208 return false;
5209 }
5210
5211 // Save the valid addressing mode in case we can't match.
5212 ExtAddrMode BackupAddrMode = AddrMode;
5213 unsigned OldSize = AddrModeInsts.size();
5214
5215 // See if the scale and offset amount is valid for this target.
5216 AddrMode.BaseOffs += ConstantOffset;
5217 if (!cast<GEPOperator>(AddrInst)->isInBounds())
5218 AddrMode.InBounds = false;
5219
5220 // Match the base operand of the GEP.
5221 if (!matchAddr(AddrInst->getOperand(0), Depth + 1)) {
5222 // If it couldn't be matched, just stuff the value in a register.
5223 if (AddrMode.HasBaseReg) {
5224 AddrMode = BackupAddrMode;
5225 AddrModeInsts.resize(OldSize);
5226 return false;
5227 }
5228 AddrMode.HasBaseReg = true;
5229 AddrMode.BaseReg = AddrInst->getOperand(0);
5230 }
5231
5232 // Match the remaining variable portion of the GEP.
5233 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale,
5234 Depth)) {
5235 // If it couldn't be matched, try stuffing the base into a register
5236 // instead of matching it, and retrying the match of the scale.
5237 AddrMode = BackupAddrMode;
5238 AddrModeInsts.resize(OldSize);
5239 if (AddrMode.HasBaseReg)
5240 return false;
5241 AddrMode.HasBaseReg = true;
5242 AddrMode.BaseReg = AddrInst->getOperand(0);
5243 AddrMode.BaseOffs += ConstantOffset;
5244 if (!matchScaledValue(AddrInst->getOperand(VariableOperand),
5245 VariableScale, Depth)) {
5246 // If even that didn't work, bail.
5247 AddrMode = BackupAddrMode;
5248 AddrModeInsts.resize(OldSize);
5249 return false;
5250 }
5251 }
5252
5253 return true;
5254 }
5255 case Instruction::SExt:
5256 case Instruction::ZExt: {
5257 Instruction *Ext = dyn_cast<Instruction>(AddrInst);
5258 if (!Ext)
5259 return false;
5260
5261 // Try to move this ext out of the way of the addressing mode.
5262 // Ask for a method for doing so.
5263 TypePromotionHelper::Action TPH =
5264 TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts);
5265 if (!TPH)
5266 return false;
5267
5268 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5269 TPT.getRestorationPoint();
5270 unsigned CreatedInstsCost = 0;
5271 unsigned ExtCost = !TLI.isExtFree(Ext);
5272 Value *PromotedOperand =
5273 TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI);
5274 // SExt has been moved away.
5275 // Thus either it will be rematched later in the recursive calls or it is
5276 // gone. Anyway, we must not fold it into the addressing mode at this point.
5277 // E.g.,
5278 // op = add opnd, 1
5279 // idx = ext op
5280 // addr = gep base, idx
5281 // is now:
5282 // promotedOpnd = ext opnd <- no match here
5283 // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls)
5284 // addr = gep base, op <- match
5285 if (MovedAway)
5286 *MovedAway = true;
5287
5288 assert(PromotedOperand &&
5289 "TypePromotionHelper should have filtered out those cases");
5290
5291 ExtAddrMode BackupAddrMode = AddrMode;
5292 unsigned OldSize = AddrModeInsts.size();
5293
5294 if (!matchAddr(PromotedOperand, Depth) ||
5295 // The total of the new cost is equal to the cost of the created
5296 // instructions.
5297 // The total of the old cost is equal to the cost of the extension plus
5298 // what we have saved in the addressing mode.
5299 !isPromotionProfitable(CreatedInstsCost,
5300 ExtCost + (AddrModeInsts.size() - OldSize),
5301 PromotedOperand)) {
5302 AddrMode = BackupAddrMode;
5303 AddrModeInsts.resize(OldSize);
5304 LLVM_DEBUG(dbgs() << "Sign extension does not pay off: rollback\n");
5305 TPT.rollback(LastKnownGood);
5306 return false;
5307 }
5308
5309 // SExt has been deleted. Make sure it is not referenced by the AddrMode.
5310 AddrMode.replaceWith(Ext, PromotedOperand);
5311 return true;
5312 }
5313 case Instruction::Call:
5314 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(AddrInst)) {
5315 if (II->getIntrinsicID() == Intrinsic::threadlocal_address) {
5316 GlobalValue &GV = cast<GlobalValue>(*II->getArgOperand(0));
5318 return matchAddr(AddrInst->getOperand(0), Depth);
5319 }
5320 }
5321 break;
5322 }
5323 return false;
5324}
5325
5326/// If we can, try to add the value of 'Addr' into the current addressing mode.
5327/// If Addr can't be added to AddrMode this returns false and leaves AddrMode
5328/// unmodified. This assumes that Addr is either a pointer type or intptr_t
5329/// for the target.
5330///
5331bool AddressingModeMatcher::matchAddr(Value *Addr, unsigned Depth) {
5332 // Start a transaction at this point that we will rollback if the matching
5333 // fails.
5334 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5335 TPT.getRestorationPoint();
5336 if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {
5337 if (CI->getValue().isSignedIntN(64)) {
5338 // Check if the addition would result in a signed overflow.
5339 int64_t Result;
5340 bool Overflow =
5341 AddOverflow(AddrMode.BaseOffs, CI->getSExtValue(), Result);
5342 if (!Overflow) {
5343 // Fold in immediates if legal for the target.
5344 AddrMode.BaseOffs = Result;
5345 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
5346 return true;
5347 AddrMode.BaseOffs -= CI->getSExtValue();
5348 }
5349 }
5350 } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {
5351 // If this is a global variable, try to fold it into the addressing mode.
5352 if (!AddrMode.BaseGV) {
5353 AddrMode.BaseGV = GV;
5354 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
5355 return true;
5356 AddrMode.BaseGV = nullptr;
5357 }
5358 } else if (Instruction *I = dyn_cast<Instruction>(Addr)) {
5359 ExtAddrMode BackupAddrMode = AddrMode;
5360 unsigned OldSize = AddrModeInsts.size();
5361
5362 // Check to see if it is possible to fold this operation.
5363 bool MovedAway = false;
5364 if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) {
5365 // This instruction may have been moved away. If so, there is nothing
5366 // to check here.
5367 if (MovedAway)
5368 return true;
5369 // Okay, it's possible to fold this. Check to see if it is actually
5370 // *profitable* to do so. We use a simple cost model to avoid increasing
5371 // register pressure too much.
5372 if (I->hasOneUse() ||
5373 isProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) {
5374 AddrModeInsts.push_back(I);
5375 return true;
5376 }
5377
5378 // It isn't profitable to do this, roll back.
5379 AddrMode = BackupAddrMode;
5380 AddrModeInsts.resize(OldSize);
5381 TPT.rollback(LastKnownGood);
5382 }
5383 } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) {
5384 if (matchOperationAddr(CE, CE->getOpcode(), Depth))
5385 return true;
5386 TPT.rollback(LastKnownGood);
5387 } else if (isa<ConstantPointerNull>(Addr)) {
5388 // Null pointer gets folded without affecting the addressing mode.
5389 return true;
5390 }
5391
5392 // Worse case, the target should support [reg] addressing modes. :)
5393 if (!AddrMode.HasBaseReg) {
5394 AddrMode.HasBaseReg = true;
5395 AddrMode.BaseReg = Addr;
5396 // Still check for legality in case the target supports [imm] but not [i+r].
5397 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
5398 return true;
5399 AddrMode.HasBaseReg = false;
5400 AddrMode.BaseReg = nullptr;
5401 }
5402
5403 // If the base register is already taken, see if we can do [r+r].
5404 if (AddrMode.Scale == 0) {
5405 AddrMode.Scale = 1;
5406 AddrMode.ScaledReg = Addr;
5407 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
5408 return true;
5409 AddrMode.Scale = 0;
5410 AddrMode.ScaledReg = nullptr;
5411 }
5412 // Couldn't match.
5413 TPT.rollback(LastKnownGood);
5414 return false;
5415}
5416
5417/// Check to see if all uses of OpVal by the specified inline asm call are due
5418/// to memory operands. If so, return true, otherwise return false.
5420 const TargetLowering &TLI,
5421 const TargetRegisterInfo &TRI) {
5422 const Function *F = CI->getFunction();
5423 TargetLowering::AsmOperandInfoVector TargetConstraints =
5424 TLI.ParseConstraints(F->getDataLayout(), &TRI, *CI);
5425
5426 for (TargetLowering::AsmOperandInfo &OpInfo : TargetConstraints) {
5427 // Compute the constraint code and ConstraintType to use.
5428 TLI.ComputeConstraintToUse(OpInfo, SDValue());
5429
5430 // If this asm operand is our Value*, and if it isn't an indirect memory
5431 // operand, we can't fold it! TODO: Also handle C_Address?
5432 if (OpInfo.CallOperandVal == OpVal &&
5433 (OpInfo.ConstraintType != TargetLowering::C_Memory ||
5434 !OpInfo.isIndirect))
5435 return false;
5436 }
5437
5438 return true;
5439}
5440
5441/// Recursively walk all the uses of I until we find a memory use.
5442/// If we find an obviously non-foldable instruction, return true.
5443/// Add accessed addresses and types to MemoryUses.
5445 Instruction *I, SmallVectorImpl<std::pair<Use *, Type *>> &MemoryUses,
5446 SmallPtrSetImpl<Instruction *> &ConsideredInsts, const TargetLowering &TLI,
5447 const TargetRegisterInfo &TRI, bool OptSize, ProfileSummaryInfo *PSI,
5448 BlockFrequencyInfo *BFI, unsigned &SeenInsts, unsigned MaxUsersToScan) {
5449 // If we already considered this instruction, we're done.
5450 if (!ConsideredInsts.insert(I).second)
5451 return false;
5452
5453 // If this is an obviously unfoldable instruction, bail out.
5454 if (!MightBeFoldableInst(I))
5455 return true;
5456
5457 // Loop over all the uses, recursively processing them.
5458 for (Use &U : I->uses()) {
5459 // Conservatively return true if we're seeing a large number or a deep chain
5460 // of users. This avoids excessive compilation times in pathological cases.
5461 if (SeenInsts++ >= MaxUsersToScan)
5462 return true;
5463
5464 Instruction *UserI = cast<Instruction>(U.getUser());
5465 if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) {
5466 MemoryUses.push_back({&U, LI->getType()});
5467 continue;
5468 }
5469
5470 if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) {
5471 if (U.getOperandNo() != StoreInst::getPointerOperandIndex())
5472 return true; // Storing addr, not into addr.
5473 MemoryUses.push_back({&U, SI->getValueOperand()->getType()});
5474 continue;
5475 }
5476
5477 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(UserI)) {
5478 if (U.getOperandNo() != AtomicRMWInst::getPointerOperandIndex())
5479 return true; // Storing addr, not into addr.
5480 MemoryUses.push_back({&U, RMW->getValOperand()->getType()});
5481 continue;
5482 }
5483
5485 if (U.getOperandNo() != AtomicCmpXchgInst::getPointerOperandIndex())
5486 return true; // Storing addr, not into addr.
5487 MemoryUses.push_back({&U, CmpX->getCompareOperand()->getType()});
5488 continue;
5489 }
5490
5493 Type *AccessTy;
5494 if (!TLI.getAddrModeArguments(II, PtrOps, AccessTy))
5495 return true;
5496
5497 if (!find(PtrOps, U.get()))
5498 return true;
5499
5500 MemoryUses.push_back({&U, AccessTy});
5501 continue;
5502 }
5503
5504 if (CallInst *CI = dyn_cast<CallInst>(UserI)) {
5505 if (CI->hasFnAttr(Attribute::Cold)) {
5506 // If this is a cold call, we can sink the addressing calculation into
5507 // the cold path. See optimizeCallInst
5508 if (!llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI))
5509 continue;
5510 }
5511
5512 InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledOperand());
5513 if (!IA)
5514 return true;
5515
5516 // If this is a memory operand, we're cool, otherwise bail out.
5517 if (!IsOperandAMemoryOperand(CI, IA, I, TLI, TRI))
5518 return true;
5519 continue;
5520 }
5521
5522 if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TLI, TRI, OptSize,
5523 PSI, BFI, SeenInsts, MaxUsersToScan))
5524 return true;
5525 }
5526
5527 return false;
5528}
5529
5531 Instruction *I, SmallVectorImpl<std::pair<Use *, Type *>> &MemoryUses,
5532 const TargetLowering &TLI, const TargetRegisterInfo &TRI, bool OptSize,
5533 ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, unsigned MaxUsersToScan) {
5534 unsigned SeenInsts = 0;
5535 SmallPtrSet<Instruction *, 16> ConsideredInsts;
5536 return FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TLI, TRI, OptSize,
5537 PSI, BFI, SeenInsts, MaxUsersToScan);
5538}
5539
5540/// Return true if Val is already known to be live at the use site that we're
5541/// folding it into. If so, there is no cost to include it in the addressing
5542/// mode. KnownLive1 and KnownLive2 are two values that we know are live at the
5543/// instruction already.
5544bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value *Val,
5545 Value *KnownLive1,
5546 Value *KnownLive2) {
5547 // If Val is either of the known-live values, we know it is live!
5548 if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2)
5549 return true;
5550
5551 // All values other than instructions and arguments (e.g. constants) are live.
5552 if (!isa<Instruction>(Val) && !isa<Argument>(Val))
5553 return true;
5554
5555 // If Val is a constant sized alloca in the entry block, it is live, this is
5556 // true because it is just a reference to the stack/frame pointer, which is
5557 // live for the whole function.
5558 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val))
5559 if (AI->isStaticAlloca())
5560 return true;
5561
5562 // Check to see if this value is already used in the memory instruction's
5563 // block. If so, it's already live into the block at the very least, so we
5564 // can reasonably fold it.
5565 return Val->isUsedInBasicBlock(MemoryInst->getParent());
5566}
5567
5568/// It is possible for the addressing mode of the machine to fold the specified
5569/// instruction into a load or store that ultimately uses it.
5570/// However, the specified instruction has multiple uses.
5571/// Given this, it may actually increase register pressure to fold it
5572/// into the load. For example, consider this code:
5573///
5574/// X = ...
5575/// Y = X+1
5576/// use(Y) -> nonload/store
5577/// Z = Y+1
5578/// load Z
5579///
5580/// In this case, Y has multiple uses, and can be folded into the load of Z
5581/// (yielding load [X+2]). However, doing this will cause both "X" and "X+1" to
5582/// be live at the use(Y) line. If we don't fold Y into load Z, we use one
5583/// fewer register. Since Y can't be folded into "use(Y)" we don't increase the
5584/// number of computations either.
5585///
5586/// Note that this (like most of CodeGenPrepare) is just a rough heuristic. If
5587/// X was live across 'load Z' for other reasons, we actually *would* want to
5588/// fold the addressing mode in the Z case. This would make Y die earlier.
5589bool AddressingModeMatcher::isProfitableToFoldIntoAddressingMode(
5590 Instruction *I, ExtAddrMode &AMBefore, ExtAddrMode &AMAfter) {
5591 if (IgnoreProfitability)
5592 return true;
5593
5594 // AMBefore is the addressing mode before this instruction was folded into it,
5595 // and AMAfter is the addressing mode after the instruction was folded. Get
5596 // the set of registers referenced by AMAfter and subtract out those
5597 // referenced by AMBefore: this is the set of values which folding in this
5598 // address extends the lifetime of.
5599 //
5600 // Note that there are only two potential values being referenced here,
5601 // BaseReg and ScaleReg (global addresses are always available, as are any
5602 // folded immediates).
5603 Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg;
5604
5605 // If the BaseReg or ScaledReg was referenced by the previous addrmode, their
5606 // lifetime wasn't extended by adding this instruction.
5607 if (valueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg))
5608 BaseReg = nullptr;
5609 if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg))
5610 ScaledReg = nullptr;
5611
5612 // If folding this instruction (and it's subexprs) didn't extend any live
5613 // ranges, we're ok with it.
5614 if (!BaseReg && !ScaledReg)
5615 return true;
5616
5617 // If all uses of this instruction can have the address mode sunk into them,
5618 // we can remove the addressing mode and effectively trade one live register
5619 // for another (at worst.) In this context, folding an addressing mode into
5620 // the use is just a particularly nice way of sinking it.
5622 if (FindAllMemoryUses(I, MemoryUses, TLI, TRI, OptSize, PSI, BFI,
5623 Opts.cgp_max_address_users_to_scan))
5624 return false; // Has a non-memory, non-foldable use!
5625
5626 // Now that we know that all uses of this instruction are part of a chain of
5627 // computation involving only operations that could theoretically be folded
5628 // into a memory use, loop over each of these memory operation uses and see
5629 // if they could *actually* fold the instruction. The assumption is that
5630 // addressing modes are cheap and that duplicating the computation involved
5631 // many times is worthwhile, even on a fastpath. For sinking candidates
5632 // (i.e. cold call sites), this serves as a way to prevent excessive code
5633 // growth since most architectures have some reasonable small and fast way to
5634 // compute an effective address. (i.e LEA on x86)
5635 SmallVector<Instruction *, 32> MatchedAddrModeInsts;
5636 for (const std::pair<Use *, Type *> &Pair : MemoryUses) {
5637 Value *Address = Pair.first->get();
5638 Instruction *UserI = cast<Instruction>(Pair.first->getUser());
5639 Type *AddressAccessTy = Pair.second;
5640 unsigned AS = Address->getType()->getPointerAddressSpace();
5641
5642 // Do a match against the root of this address, ignoring profitability. This
5643 // will tell us if the addressing mode for the memory operation will
5644 // *actually* cover the shared instruction.
5645 ExtAddrMode Result;
5646 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(nullptr,
5647 0);
5648 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5649 TPT.getRestorationPoint();
5650 AddressingModeMatcher Matcher(MatchedAddrModeInsts, TLI, TRI, LI, getDTFn,
5651 AddressAccessTy, AS, UserI, Result,
5652 InsertedInsts, PromotedInsts, TPT,
5653 LargeOffsetGEP, OptSize, PSI, BFI, Opts);
5654 Matcher.IgnoreProfitability = true;
5655 bool Success = Matcher.matchAddr(Address, 0);
5656 (void)Success;
5657 assert(Success && "Couldn't select *anything*?");
5658
5659 // The match was to check the profitability, the changes made are not
5660 // part of the original matcher. Therefore, they should be dropped
5661 // otherwise the original matcher will not present the right state.
5662 TPT.rollback(LastKnownGood);
5663
5664 // If the match didn't cover I, then it won't be shared by it.
5665 if (!is_contained(MatchedAddrModeInsts, I))
5666 return false;
5667
5668 MatchedAddrModeInsts.clear();
5669 }
5670
5671 return true;
5672}
5673
5674/// Return true if the specified values are defined in a
5675/// different basic block than BB.
5676static bool IsNonLocalValue(Value *V, BasicBlock *BB) {
5678 return I->getParent() != BB;
5679 return false;
5680}
5681
5682// Find an insert position of Addr for MemoryInst. We can't guarantee MemoryInst
5683// is the first instruction that will use Addr. So we need to find the first
5684// user of Addr in current BB.
5686 Value *SunkAddr) {
5687 if (Addr->hasOneUse())
5688 return MemoryInst->getIterator();
5689
5690 // We already have a SunkAddr in current BB, but we may need to insert cast
5691 // instruction after it.
5692 if (SunkAddr) {
5693 if (Instruction *AddrInst = dyn_cast<Instruction>(SunkAddr))
5694 return std::next(AddrInst->getIterator());
5695 }
5696
5697 // Find the first user of Addr in current BB.
5698 Instruction *Earliest = MemoryInst;
5699 for (User *U : Addr->users()) {
5700 Instruction *UserInst = dyn_cast<Instruction>(U);
5701 if (UserInst && UserInst->getParent() == MemoryInst->getParent()) {
5702 if (isa<PHINode>(UserInst))
5703 continue;
5704 if (UserInst->comesBefore(Earliest))
5705 Earliest = UserInst;
5706 }
5707 }
5708 return Earliest->getIterator();
5709}
5710
5711/// Sink addressing mode computation immediate before MemoryInst if doing so
5712/// can be done without increasing register pressure. The need for the
5713/// register pressure constraint means this can end up being an all or nothing
5714/// decision for all uses of the same addressing computation.
5715///
5716/// Load and Store Instructions often have addressing modes that can do
5717/// significant amounts of computation. As such, instruction selection will try
5718/// to get the load or store to do as much computation as possible for the
5719/// program. The problem is that isel can only see within a single block. As
5720/// such, we sink as much legal addressing mode work into the block as possible.
5721///
5722/// This method is used to optimize both load/store and inline asms with memory
5723/// operands. It's also used to sink addressing computations feeding into cold
5724/// call sites into their (cold) basic block.
5725///
5726/// The motivation for handling sinking into cold blocks is that doing so can
5727/// both enable other address mode sinking (by satisfying the register pressure
5728/// constraint above), and reduce register pressure globally (by removing the
5729/// addressing mode computation from the fast path entirely.).
5730bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
5731 Type *AccessTy, unsigned AddrSpace) {
5732 Value *Repl = Addr;
5733
5734 // Try to collapse single-value PHI nodes. This is necessary to undo
5735 // unprofitable PRE transformations.
5736 SmallVector<Value *, 8> worklist;
5737 SmallPtrSet<Value *, 16> Visited;
5738 worklist.push_back(Addr);
5739
5740 // Use a worklist to iteratively look through PHI and select nodes, and
5741 // ensure that the addressing mode obtained from the non-PHI/select roots of
5742 // the graph are compatible.
5743 bool PhiOrSelectSeen = false;
5744 SmallVector<Instruction *, 16> AddrModeInsts;
5745 AddressingModeCombiner AddrModes(*DL, Opts, Addr);
5746 TypePromotionTransaction TPT(RemovedInsts);
5747 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5748 TPT.getRestorationPoint();
5749 while (!worklist.empty()) {
5750 Value *V = worklist.pop_back_val();
5751
5752 // We allow traversing cyclic Phi nodes.
5753 // In case of success after this loop we ensure that traversing through
5754 // Phi nodes ends up with all cases to compute address of the form
5755 // BaseGV + Base + Scale * Index + Offset
5756 // where Scale and Offset are constans and BaseGV, Base and Index
5757 // are exactly the same Values in all cases.
5758 // It means that BaseGV, Scale and Offset dominate our memory instruction
5759 // and have the same value as they had in address computation represented
5760 // as Phi. So we can safely sink address computation to memory instruction.
5761 if (!Visited.insert(V).second)
5762 continue;
5763
5764 // For a PHI node, push all of its incoming values.
5765 if (PHINode *P = dyn_cast<PHINode>(V)) {
5766 append_range(worklist, P->incoming_values());
5767 PhiOrSelectSeen = true;
5768 continue;
5769 }
5770 // Similar for select.
5771 if (SelectInst *SI = dyn_cast<SelectInst>(V)) {
5772 worklist.push_back(SI->getFalseValue());
5773 worklist.push_back(SI->getTrueValue());
5774 PhiOrSelectSeen = true;
5775 continue;
5776 }
5777
5778 // For non-PHIs, determine the addressing mode being computed. Note that
5779 // the result may differ depending on what other uses our candidate
5780 // addressing instructions might have.
5781 AddrModeInsts.clear();
5782 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(nullptr,
5783 0);
5784 // Defer the query (and possible computation of) the dom tree to point of
5785 // actual use. It's expected that most address matches don't actually need
5786 // the domtree.
5787 auto getDTFn = [this]() -> const DominatorTree & { return getDT(); };
5788 ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
5789 V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *TLI, *LI, getDTFn,
5790 *TRI, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, OptSize, PSI,
5791 BFI, Opts);
5792
5793 GetElementPtrInst *GEP = LargeOffsetGEP.first;
5794 if (GEP && !NewGEPBases.count(GEP)) {
5795 // If splitting the underlying data structure can reduce the offset of a
5796 // GEP, collect the GEP. Skip the GEPs that are the new bases of
5797 // previously split data structures.
5798 LargeOffsetGEPMap[GEP->getPointerOperand()].push_back(LargeOffsetGEP);
5799 LargeOffsetGEPID.insert(std::make_pair(GEP, LargeOffsetGEPID.size()));
5800 }
5801
5802 NewAddrMode.OriginalValue = V;
5803 if (!AddrModes.addNewAddrMode(NewAddrMode))
5804 break;
5805 }
5806
5807 // Try to combine the AddrModes we've collected. If we couldn't collect any,
5808 // or we have multiple but either couldn't combine them or combining them
5809 // wouldn't do anything useful, bail out now.
5810 if (!AddrModes.combineAddrModes()) {
5811 TPT.rollback(LastKnownGood);
5812 return false;
5813 }
5814 bool Modified = TPT.commit();
5815
5816 // Get the combined AddrMode (or the only AddrMode, if we only had one).
5817 ExtAddrMode AddrMode = AddrModes.getAddrMode();
5818
5819 // If all the instructions matched are already in this BB, don't do anything.
5820 // If we saw a Phi node then it is not local definitely, and if we saw a
5821 // select then we want to push the address calculation past it even if it's
5822 // already in this BB.
5823 if (!PhiOrSelectSeen && none_of(AddrModeInsts, [&](Value *V) {
5824 return IsNonLocalValue(V, MemoryInst->getParent());
5825 })) {
5826 LLVM_DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode
5827 << "\n");
5828 return Modified;
5829 }
5830
5831 // Now that we determined the addressing expression we want to use and know
5832 // that we have to sink it into this block. Check to see if we have already
5833 // done this for some other load/store instr in this block. If so, reuse
5834 // the computation. Before attempting reuse, check if the address is valid
5835 // as it may have been erased.
5836
5837 WeakTrackingVH SunkAddrVH = SunkAddrs[Addr];
5838
5839 Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;
5840 Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
5841 Type *IdxTy = DL->getIndexType(Addr->getType());
5842
5843 // The current BB may be optimized multiple times, we can't guarantee the
5844 // reuse of Addr happens later, call findInsertPos to find an appropriate
5845 // insert position.
5846 auto InsertPos = findInsertPos(Addr, MemoryInst, SunkAddr);
5847
5848 // TODO: Adjust insert point considering (Base|Scaled)Reg if possible.
5849 if (!SunkAddr) {
5850 auto &DT = getDT();
5851 if ((AddrMode.BaseReg && !DT.dominates(AddrMode.BaseReg, &*InsertPos)) ||
5852 (AddrMode.ScaledReg && !DT.dominates(AddrMode.ScaledReg, &*InsertPos)))
5853 return Modified;
5854 }
5855
5856 IRBuilder<> Builder(InsertPos);
5857
5858 if (SunkAddr) {
5859 LLVM_DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode
5860 << " for " << *MemoryInst << "\n");
5861 if (SunkAddr->getType() != Addr->getType()) {
5862 if (SunkAddr->getType()->getPointerAddressSpace() !=
5863 Addr->getType()->getPointerAddressSpace() &&
5864 !DL->isNonIntegralPointerType(Addr->getType())) {
5865 // There are two reasons the address spaces might not match: a no-op
5866 // addrspacecast, or a ptrtoint/inttoptr pair. Either way, we emit a
5867 // ptrtoint/inttoptr pair to ensure we match the original semantics.
5868 // TODO: allow bitcast between different address space pointers with the
5869 // same size.
5870 SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr");
5871 SunkAddr =
5872 Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr");
5873 } else
5874 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType());
5875 }
5876 } else {
5877 LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode
5878 << " for " << *MemoryInst << "\n");
5879 Value *ResultPtr = nullptr, *ResultIndex = nullptr;
5880
5881 // First, find the pointer.
5882 if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) {
5883 ResultPtr = AddrMode.BaseReg;
5884 AddrMode.BaseReg = nullptr;
5885 }
5886
5887 if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) {
5888 // We can't add more than one pointer together, nor can we scale a
5889 // pointer (both of which seem meaningless).
5890 if (ResultPtr || AddrMode.Scale != 1)
5891 return Modified;
5892
5893 ResultPtr = AddrMode.ScaledReg;
5894 AddrMode.Scale = 0;
5895 }
5896
5897 // It is only safe to sign extend the BaseReg if we know that the math
5898 // required to create it did not overflow before we extend it. Since
5899 // the original IR value was tossed in favor of a constant back when
5900 // the AddrMode was created we need to bail out gracefully if widths
5901 // do not match instead of extending it.
5902 //
5903 // (See below for code to add the scale.)
5904 if (AddrMode.Scale) {
5905 Type *ScaledRegTy = AddrMode.ScaledReg->getType();
5906 if (cast<IntegerType>(IdxTy)->getBitWidth() >
5907 cast<IntegerType>(ScaledRegTy)->getBitWidth())
5908 return Modified;
5909 }
5910
5911 GlobalValue *BaseGV = AddrMode.BaseGV;
5912 if (BaseGV != nullptr) {
5913 if (ResultPtr)
5914 return Modified;
5915
5916 if (BaseGV->isThreadLocal()) {
5917 ResultPtr = Builder.CreateThreadLocalAddress(BaseGV);
5918 } else {
5919 ResultPtr = BaseGV;
5920 }
5921 }
5922
5923 // If the real base value actually came from an inttoptr, then the matcher
5924 // will look through it and provide only the integer value. In that case,
5925 // use it here.
5926 if (!DL->isNonIntegralPointerType(Addr->getType())) {
5927 if (!ResultPtr && AddrMode.BaseReg) {
5928 ResultPtr = Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(),
5929 "sunkaddr");
5930 AddrMode.BaseReg = nullptr;
5931 } else if (!ResultPtr && AddrMode.Scale == 1) {
5932 ResultPtr = Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(),
5933 "sunkaddr");
5934 AddrMode.Scale = 0;
5935 }
5936 }
5937
5938 if (!ResultPtr && !AddrMode.BaseReg && !AddrMode.Scale &&
5939 !AddrMode.BaseOffs) {
5940 SunkAddr = Constant::getNullValue(Addr->getType());
5941 } else if (!ResultPtr) {
5942 return Modified;
5943 } else {
5944 Type *I8PtrTy =
5945 Builder.getPtrTy(Addr->getType()->getPointerAddressSpace());
5946
5947 // Start with the base register. Do this first so that subsequent address
5948 // matching finds it last, which will prevent it from trying to match it
5949 // as the scaled value in case it happens to be a mul. That would be
5950 // problematic if we've sunk a different mul for the scale, because then
5951 // we'd end up sinking both muls.
5952 if (AddrMode.BaseReg) {
5953 Value *V = AddrMode.BaseReg;
5954 if (V->getType() != IdxTy)
5955 V = Builder.CreateIntCast(V, IdxTy, /*isSigned=*/true, "sunkaddr");
5956
5957 ResultIndex = V;
5958 }
5959
5960 // Add the scale value.
5961 if (AddrMode.Scale) {
5962 Value *V = AddrMode.ScaledReg;
5963 if (V->getType() == IdxTy) {
5964 // done.
5965 } else {
5967 cast<IntegerType>(V->getType())->getBitWidth() &&
5968 "We can't transform if ScaledReg is too narrow");
5969 V = Builder.CreateTrunc(V, IdxTy, "sunkaddr");
5970 }
5971
5972 if (AddrMode.Scale != 1)
5973 V = Builder.CreateMul(
5974 V, ConstantInt::getSigned(IdxTy, AddrMode.Scale), "sunkaddr");
5975 if (ResultIndex)
5976 ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr");
5977 else
5978 ResultIndex = V;
5979 }
5980
5981 // Add in the Base Offset if present.
5982 if (AddrMode.BaseOffs) {
5983 Value *V = ConstantInt::getSigned(IdxTy, AddrMode.BaseOffs);
5984 if (ResultIndex) {
5985 // We need to add this separately from the scale above to help with
5986 // SDAG consecutive load/store merging.
5987 if (ResultPtr->getType() != I8PtrTy)
5988 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);
5989 ResultPtr = Builder.CreatePtrAdd(ResultPtr, ResultIndex, "sunkaddr",
5990 AddrMode.InBounds);
5991 }
5992
5993 ResultIndex = V;
5994 }
5995
5996 if (!ResultIndex) {
5997 auto PtrInst = dyn_cast<Instruction>(ResultPtr);
5998 // We know that we have a pointer without any offsets. If this pointer
5999 // originates from a different basic block than the current one, we
6000 // must be able to recreate it in the current basic block.
6001 // We do not support the recreation of any instructions yet.
6002 if (PtrInst && PtrInst->getParent() != MemoryInst->getParent())
6003 return Modified;
6004 SunkAddr = ResultPtr;
6005 } else {
6006 if (ResultPtr->getType() != I8PtrTy)
6007 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);
6008 SunkAddr = Builder.CreatePtrAdd(ResultPtr, ResultIndex, "sunkaddr",
6009 AddrMode.InBounds);
6010 }
6011
6012 if (SunkAddr->getType() != Addr->getType()) {
6013 if (SunkAddr->getType()->getPointerAddressSpace() !=
6014 Addr->getType()->getPointerAddressSpace() &&
6015 !DL->isNonIntegralPointerType(Addr->getType())) {
6016 // There are two reasons the address spaces might not match: a no-op
6017 // addrspacecast, or a ptrtoint/inttoptr pair. Either way, we emit a
6018 // ptrtoint/inttoptr pair to ensure we match the original semantics.
6019 // TODO: allow bitcast between different address space pointers with
6020 // the same size.
6021 SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr");
6022 SunkAddr =
6023 Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr");
6024 } else
6025 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType());
6026 }
6027 }
6028 }
6029
6030 MemoryInst->replaceUsesOfWith(Repl, SunkAddr);
6031 // Store the newly computed address into the cache. In the case we reused a
6032 // value, this should be idempotent.
6033 SunkAddrs[Addr] = WeakTrackingVH(SunkAddr);
6034
6035 // If we have no uses, recursively delete the value and all dead instructions
6036 // using it.
6037 if (Repl->use_empty()) {
6038 resetIteratorIfInvalidatedWhileCalling(CurInstIterator->getParent(), [&]() {
6039 RecursivelyDeleteTriviallyDeadInstructions(
6040 Repl, TLInfo, nullptr,
6041 [&](Value *V) { removeAllAssertingVHReferences(V); });
6042 });
6043 }
6044 ++NumMemoryInsts;
6045 return true;
6046}
6047
6048/// Rewrite GEP input to gather/scatter to enable SelectionDAGBuilder to find
6049/// a uniform base to use for ISD::MGATHER/MSCATTER. SelectionDAGBuilder can
6050/// only handle a 2 operand GEP in the same basic block or a splat constant
6051/// vector. The 2 operands to the GEP must have a scalar pointer and a vector
6052/// index.
6053///
6054/// If the existing GEP has a vector base pointer that is splat, we can look
6055/// through the splat to find the scalar pointer. If we can't find a scalar
6056/// pointer there's nothing we can do.
6057///
6058/// If we have a GEP with more than 2 indices where the middle indices are all
6059/// zeroes, we can replace it with 2 GEPs where the second has 2 operands.
6060///
6061/// If the final index isn't a vector or is a splat, we can emit a scalar GEP
6062/// followed by a GEP with an all zeroes vector index. This will enable
6063/// SelectionDAGBuilder to use the scalar GEP as the uniform base and have a
6064/// zero index.
6065bool CodeGenPrepare::optimizeGatherScatterInst(Instruction *MemoryInst,
6066 Value *Ptr) {
6067 Value *NewAddr;
6068
6069 if (const auto *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {
6070 // Don't optimize GEPs that don't have indices.
6071 if (!GEP->hasIndices())
6072 return false;
6073
6074 // If the GEP and the gather/scatter aren't in the same BB, don't optimize.
6075 // FIXME: We should support this by sinking the GEP.
6076 if (MemoryInst->getParent() != GEP->getParent())
6077 return false;
6078
6079 SmallVector<Value *, 2> Ops(GEP->operands());
6080
6081 bool RewriteGEP = false;
6082
6083 if (Ops[0]->getType()->isVectorTy()) {
6084 Ops[0] = getSplatValue(Ops[0]);
6085 if (!Ops[0])
6086 return false;
6087 RewriteGEP = true;
6088 }
6089
6090 unsigned FinalIndex = Ops.size() - 1;
6091
6092 // Ensure all but the last index is 0.
6093 // FIXME: This isn't strictly required. All that's required is that they are
6094 // all scalars or splats.
6095 for (unsigned i = 1; i < FinalIndex; ++i) {
6096 auto *C = dyn_cast<Constant>(Ops[i]);
6097 if (!C)
6098 return false;
6099 if (isa<VectorType>(C->getType()))
6100 C = C->getSplatValue();
6101 auto *CI = dyn_cast_or_null<ConstantInt>(C);
6102 if (!CI || !CI->isZero())
6103 return false;
6104 // Scalarize the index if needed.
6105 Ops[i] = CI;
6106 }
6107
6108 // Try to scalarize the final index.
6109 if (Ops[FinalIndex]->getType()->isVectorTy()) {
6110 if (Value *V = getSplatValue(Ops[FinalIndex])) {
6111 auto *C = dyn_cast<ConstantInt>(V);
6112 // Don't scalarize all zeros vector.
6113 if (!C || !C->isZero()) {
6114 Ops[FinalIndex] = V;
6115 RewriteGEP = true;
6116 }
6117 }
6118 }
6119
6120 // If we made any changes or the we have extra operands, we need to generate
6121 // new instructions.
6122 if (!RewriteGEP && Ops.size() == 2)
6123 return false;
6124
6125 auto NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
6126
6127 IRBuilder<> Builder(MemoryInst);
6128 Builder.disableGEPCanonicalization();
6129
6130 Type *SourceTy = GEP->getSourceElementType();
6131 Type *ScalarIndexTy = DL->getIndexType(Ops[0]->getType()->getScalarType());
6132
6133 // If the final index isn't a vector, emit a scalar GEP containing all ops
6134 // and a vector GEP with all zeroes final index.
6135 if (!Ops[FinalIndex]->getType()->isVectorTy()) {
6136 NewAddr = Builder.CreateGEP(SourceTy, Ops[0], ArrayRef(Ops).drop_front());
6137 auto *IndexTy = VectorType::get(ScalarIndexTy, NumElts);
6138 auto *SecondTy = GetElementPtrInst::getIndexedType(
6139 SourceTy, ArrayRef(Ops).drop_front());
6140 NewAddr =
6141 Builder.CreateGEP(SecondTy, NewAddr, Constant::getNullValue(IndexTy));
6142 } else {
6143 Value *Base = Ops[0];
6144 Value *Index = Ops[FinalIndex];
6145
6146 // Create a scalar GEP if there are more than 2 operands.
6147 if (Ops.size() != 2) {
6148 // Replace the last index with 0.
6149 Ops[FinalIndex] =
6150 Constant::getNullValue(Ops[FinalIndex]->getType()->getScalarType());
6151 Base = Builder.CreateGEP(SourceTy, Base, ArrayRef(Ops).drop_front());
6153 SourceTy, ArrayRef(Ops).drop_front());
6154 }
6155
6156 // Now create the GEP with scalar pointer and vector index.
6157 NewAddr = Builder.CreateGEP(SourceTy, Base, Index);
6158 }
6159 } else if (!isa<Constant>(Ptr)) {
6160 // Not a GEP, maybe its a splat and we can create a GEP to enable
6161 // SelectionDAGBuilder to use it as a uniform base.
6162 Value *V = getSplatValue(Ptr);
6163 if (!V)
6164 return false;
6165
6166 auto NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
6167
6168 IRBuilder<> Builder(MemoryInst);
6169 Builder.disableGEPCanonicalization();
6170
6171 // Emit a vector GEP with a scalar pointer and all 0s vector index.
6172 Type *ScalarIndexTy = DL->getIndexType(V->getType()->getScalarType());
6173 auto *IndexTy = VectorType::get(ScalarIndexTy, NumElts);
6174 Type *ScalarTy;
6175 if (cast<IntrinsicInst>(MemoryInst)->getIntrinsicID() ==
6176 Intrinsic::masked_gather) {
6177 ScalarTy = MemoryInst->getType()->getScalarType();
6178 } else {
6179 assert(cast<IntrinsicInst>(MemoryInst)->getIntrinsicID() ==
6180 Intrinsic::masked_scatter);
6181 ScalarTy = MemoryInst->getOperand(0)->getType()->getScalarType();
6182 }
6183 NewAddr = Builder.CreateGEP(ScalarTy, V, Constant::getNullValue(IndexTy));
6184 } else {
6185 // Constant, SelectionDAGBuilder knows to check if its a splat.
6186 return false;
6187 }
6188
6189 MemoryInst->replaceUsesOfWith(Ptr, NewAddr);
6190
6191 // If we have no uses, recursively delete the value and all dead instructions
6192 // using it.
6193 if (Ptr->use_empty())
6195 Ptr, TLInfo, nullptr,
6196 [&](Value *V) { removeAllAssertingVHReferences(V); });
6197
6198 return true;
6199}
6200
6201// This is a helper for CodeGenPrepare::optimizeMulWithOverflow.
6202// Check the pattern we are interested in where there are maximum 2 uses
6203// of the intrinsic which are the extract instructions.
6205 ExtractValueInst *&OverflowExtract) {
6206 // Bail out if it's more than 2 users:
6207 if (I->hasNUsesOrMore(3))
6208 return false;
6209
6210 for (User *U : I->users()) {
6211 auto *Extract = dyn_cast<ExtractValueInst>(U);
6212 if (!Extract || Extract->getNumIndices() != 1)
6213 return false;
6214
6215 unsigned Index = Extract->getIndices()[0];
6216 if (Index == 0)
6217 MulExtract = Extract;
6218 else if (Index == 1)
6219 OverflowExtract = Extract;
6220 else
6221 return false;
6222 }
6223 return true;
6224}
6225
6226// Rewrite the mul_with_overflow intrinsic by checking if both of the
6227// operands' value ranges are within the legal type. If so, we can optimize the
6228// multiplication algorithm. This code is supposed to be written during the step
6229// of type legalization, but given that we need to reconstruct the IR which is
6230// not doable there, we do it here.
6231// The IR after the optimization will look like:
6232// entry:
6233// if signed:
6234// ( (lhs_lo>>BW-1) ^ lhs_hi) || ( (rhs_lo>>BW-1) ^ rhs_hi) ? overflow,
6235// overflow_no
6236// else:
6237// (lhs_hi != 0) || (rhs_hi != 0) ? overflow, overflow_no
6238// overflow_no:
6239// overflow:
6240// overflow.res:
6241// \returns true if optimization was applied
6242// TODO: This optimization can be further improved to optimize branching on
6243// overflow where the 'overflow_no' BB can branch directly to the false
6244// successor of overflow, but that would add additional complexity so we leave
6245// it for future work.
6246bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,
6247 ModifyDT &ModifiedDT) {
6248 // Check if target supports this optimization.
6250 I->getContext(),
6251 TLI->getValueType(*DL, I->getType()->getContainedType(0))))
6252 return false;
6253
6254 ExtractValueInst *MulExtract = nullptr, *OverflowExtract = nullptr;
6255 if (!matchOverflowPattern(I, MulExtract, OverflowExtract))
6256 return false;
6257
6258 // Keep track of the instruction to stop reoptimizing it again.
6259 InsertedInsts.insert(I);
6260
6261 Value *LHS = I->getOperand(0);
6262 Value *RHS = I->getOperand(1);
6263 Type *Ty = LHS->getType();
6264 unsigned VTHalfBitWidth = Ty->getScalarSizeInBits() / 2;
6265 Type *LegalTy = Ty->getWithNewBitWidth(VTHalfBitWidth);
6266
6267 // New BBs:
6268 BasicBlock *OverflowEntryBB =
6269 splitBlockBefore(I->getParent(), I, DTU, LI, nullptr, "");
6270 OverflowEntryBB->takeName(I->getParent());
6271 // Keep the 'br' instruction that is generated as a result of the split to be
6272 // erased/replaced later.
6273 Instruction *OldTerminator = OverflowEntryBB->getTerminator();
6274 BasicBlock *NoOverflowBB =
6275 BasicBlock::Create(I->getContext(), "overflow.no", I->getFunction());
6276 NoOverflowBB->moveAfter(OverflowEntryBB);
6277 BasicBlock *OverflowBB =
6278 BasicBlock::Create(I->getContext(), "overflow", I->getFunction());
6279 OverflowBB->moveAfter(NoOverflowBB);
6280
6281 // BB overflow.entry:
6282 IRBuilder<> Builder(OverflowEntryBB);
6283 // Extract low and high halves of LHS:
6284 Value *LoLHS = Builder.CreateTrunc(LHS, LegalTy, "lo.lhs");
6285 Value *HiLHS = Builder.CreateLShr(LHS, VTHalfBitWidth, "lhs.lsr");
6286 HiLHS = Builder.CreateTrunc(HiLHS, LegalTy, "hi.lhs");
6287
6288 // Extract low and high halves of RHS:
6289 Value *LoRHS = Builder.CreateTrunc(RHS, LegalTy, "lo.rhs");
6290 Value *HiRHS = Builder.CreateLShr(RHS, VTHalfBitWidth, "rhs.lsr");
6291 HiRHS = Builder.CreateTrunc(HiRHS, LegalTy, "hi.rhs");
6292
6293 Value *IsAnyBitTrue;
6294 if (IsSigned) {
6295 Value *SignLoLHS =
6296 Builder.CreateAShr(LoLHS, VTHalfBitWidth - 1, "sign.lo.lhs");
6297 Value *SignLoRHS =
6298 Builder.CreateAShr(LoRHS, VTHalfBitWidth - 1, "sign.lo.rhs");
6299 Value *XorLHS = Builder.CreateXor(HiLHS, SignLoLHS);
6300 Value *XorRHS = Builder.CreateXor(HiRHS, SignLoRHS);
6301 Value *Or = Builder.CreateOr(XorLHS, XorRHS, "or.lhs.rhs");
6302 IsAnyBitTrue = Builder.CreateCmp(ICmpInst::ICMP_NE, Or,
6303 ConstantInt::getNullValue(Or->getType()));
6304 } else {
6305 Value *CmpLHS = Builder.CreateCmp(ICmpInst::ICMP_NE, HiLHS,
6306 ConstantInt::getNullValue(LegalTy));
6307 Value *CmpRHS = Builder.CreateCmp(ICmpInst::ICMP_NE, HiRHS,
6308 ConstantInt::getNullValue(LegalTy));
6309 IsAnyBitTrue = Builder.CreateOr(CmpLHS, CmpRHS, "or.lhs.rhs");
6310 }
6311 Builder.CreateCondBr(IsAnyBitTrue, OverflowBB, NoOverflowBB);
6312
6313 // BB overflow.no:
6314 Builder.SetInsertPoint(NoOverflowBB);
6315 Value *ExtLoLHS, *ExtLoRHS;
6316 if (IsSigned) {
6317 ExtLoLHS = Builder.CreateSExt(LoLHS, Ty, "lo.lhs.ext");
6318 ExtLoRHS = Builder.CreateSExt(LoRHS, Ty, "lo.rhs.ext");
6319 } else {
6320 ExtLoLHS = Builder.CreateZExt(LoLHS, Ty, "lo.lhs.ext");
6321 ExtLoRHS = Builder.CreateZExt(LoRHS, Ty, "lo.rhs.ext");
6322 }
6323
6324 Value *Mul = Builder.CreateMul(ExtLoLHS, ExtLoRHS, "mul.overflow.no");
6325
6326 // Create the 'overflow.res' BB to merge the results of
6327 // the two paths:
6328 BasicBlock *OverflowResBB = I->getParent();
6329 OverflowResBB->setName("overflow.res");
6330
6331 // BB overflow.no: jump to overflow.res BB
6332 Builder.CreateBr(OverflowResBB);
6333 // No we don't need the old terminator in overflow.entry BB, erase it:
6334 OldTerminator->eraseFromParent();
6335
6336 // BB overflow.res:
6337 Builder.SetInsertPoint(OverflowResBB->getFirstInsertionPt());
6338 // Create PHI nodes to merge results from no.overflow BB and overflow BB to
6339 // replace the extract instructions.
6340 PHINode *OverflowResPHI = Builder.CreatePHI(Ty, 2),
6341 *OverflowFlagPHI =
6342 Builder.CreatePHI(IntegerType::getInt1Ty(I->getContext()), 2);
6343
6344 // Add the incoming values from no.overflow BB and later from overflow BB.
6345 OverflowResPHI->addIncoming(Mul, NoOverflowBB);
6346 OverflowFlagPHI->addIncoming(ConstantInt::getFalse(I->getContext()),
6347 NoOverflowBB);
6348
6349 // Replace all users of MulExtract and OverflowExtract to use the PHI nodes.
6350 if (MulExtract) {
6351 MulExtract->replaceAllUsesWith(OverflowResPHI);
6352 MulExtract->eraseFromParent();
6353 }
6354 if (OverflowExtract) {
6355 OverflowExtract->replaceAllUsesWith(OverflowFlagPHI);
6356 OverflowExtract->eraseFromParent();
6357 }
6358
6359 // Remove the intrinsic from parent (overflow.res BB) as it will be part of
6360 // overflow BB
6361 I->removeFromParent();
6362 // BB overflow:
6363 I->insertInto(OverflowBB, OverflowBB->end());
6364 Builder.SetInsertPoint(OverflowBB->end());
6365 Value *MulOverflow = Builder.CreateExtractValue(I, {0}, "mul.overflow");
6366 Value *OverflowFlag = Builder.CreateExtractValue(I, {1}, "overflow.flag");
6367 Builder.CreateBr(OverflowResBB);
6368
6369 // Add The Extracted values to the PHINodes in the overflow.res BB.
6370 OverflowResPHI->addIncoming(MulOverflow, OverflowBB);
6371 OverflowFlagPHI->addIncoming(OverflowFlag, OverflowBB);
6372
6373 DTU->applyUpdates({{DominatorTree::Insert, OverflowEntryBB, OverflowBB},
6374 {DominatorTree::Insert, OverflowEntryBB, NoOverflowBB},
6375 {DominatorTree::Insert, NoOverflowBB, OverflowResBB},
6376 {DominatorTree::Delete, OverflowEntryBB, OverflowResBB},
6377 {DominatorTree::Insert, OverflowBB, OverflowResBB}});
6378
6379 ModifiedDT = ModifyDT::ModifyBBDT;
6380 return true;
6381}
6382
6383/// If there are any memory operands, use OptimizeMemoryInst to sink their
6384/// address computing into the block when possible / profitable.
6385bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
6386 bool MadeChange = false;
6387
6388 const TargetRegisterInfo *TRI =
6390 TargetLowering::AsmOperandInfoVector TargetConstraints =
6391 TLI->ParseConstraints(*DL, TRI, *CS);
6392 unsigned ArgNo = 0;
6393 for (TargetLowering::AsmOperandInfo &OpInfo : TargetConstraints) {
6394 // Compute the constraint code and ConstraintType to use.
6396
6397 // TODO: Also handle C_Address?
6398 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6399 OpInfo.isIndirect) {
6400 Value *OpVal = CS->getArgOperand(ArgNo++);
6401 MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u);
6402 } else if (OpInfo.Type == InlineAsm::isInput)
6403 ArgNo++;
6404 }
6405
6406 return MadeChange;
6407}
6408
6409/// Check if all the uses of \p Val are equivalent (or free) zero or
6410/// sign extensions.
6411static bool hasSameExtUse(Value *Val, const TargetLowering &TLI) {
6412 assert(!Val->use_empty() && "Input must have at least one use");
6413 const Instruction *FirstUser = cast<Instruction>(*Val->user_begin());
6414 bool IsSExt = isa<SExtInst>(FirstUser);
6415 Type *ExtTy = FirstUser->getType();
6416 for (const User *U : Val->users()) {
6417 const Instruction *UI = cast<Instruction>(U);
6418 if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI)))
6419 return false;
6420 Type *CurTy = UI->getType();
6421 // Same input and output types: Same instruction after CSE.
6422 if (CurTy == ExtTy)
6423 continue;
6424
6425 // If IsSExt is true, we are in this situation:
6426 // a = Val
6427 // b = sext ty1 a to ty2
6428 // c = sext ty1 a to ty3
6429 // Assuming ty2 is shorter than ty3, this could be turned into:
6430 // a = Val
6431 // b = sext ty1 a to ty2
6432 // c = sext ty2 b to ty3
6433 // However, the last sext is not free.
6434 if (IsSExt)
6435 return false;
6436
6437 // This is a ZExt, maybe this is free to extend from one type to another.
6438 // In that case, we would not account for a different use.
6439 Type *NarrowTy;
6440 Type *LargeTy;
6441 if (ExtTy->getScalarType()->getIntegerBitWidth() >
6442 CurTy->getScalarType()->getIntegerBitWidth()) {
6443 NarrowTy = CurTy;
6444 LargeTy = ExtTy;
6445 } else {
6446 NarrowTy = ExtTy;
6447 LargeTy = CurTy;
6448 }
6449
6450 if (!TLI.isZExtFree(NarrowTy, LargeTy))
6451 return false;
6452 }
6453 // All uses are the same or can be derived from one another for free.
6454 return true;
6455}
6456
6457/// Try to speculatively promote extensions in \p Exts and continue
6458/// promoting through newly promoted operands recursively as far as doing so is
6459/// profitable. Save extensions profitably moved up, in \p ProfitablyMovedExts.
6460/// When some promotion happened, \p TPT contains the proper state to revert
6461/// them.
6462///
6463/// \return true if some promotion happened, false otherwise.
6464bool CodeGenPrepare::tryToPromoteExts(
6465 TypePromotionTransaction &TPT, const SmallVectorImpl<Instruction *> &Exts,
6466 SmallVectorImpl<Instruction *> &ProfitablyMovedExts,
6467 unsigned CreatedInstsCost) {
6468 bool Promoted = false;
6469
6470 // Iterate over all the extensions to try to promote them.
6471 for (auto *I : Exts) {
6472 // Early check if we directly have ext(load).
6473 if (isa<LoadInst>(I->getOperand(0))) {
6474 ProfitablyMovedExts.push_back(I);
6475 continue;
6476 }
6477
6478 // Check whether or not we want to do any promotion. The reason we have
6479 // this check inside the for loop is to catch the case where an extension
6480 // is directly fed by a load because in such case the extension can be moved
6481 // up without any promotion on its operands.
6482 if (!TLI->enableExtLdPromotion() || !Opts.cgp_ext_ld_promotion)
6483 return false;
6484
6485 // Get the action to perform the promotion.
6486 TypePromotionHelper::Action TPH =
6487 TypePromotionHelper::getAction(I, InsertedInsts, *TLI, PromotedInsts);
6488 // Check if we can promote.
6489 if (!TPH) {
6490 // Save the current extension as we cannot move up through its operand.
6491 ProfitablyMovedExts.push_back(I);
6492 continue;
6493 }
6494
6495 // Save the current state.
6496 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
6497 TPT.getRestorationPoint();
6498 SmallVector<Instruction *, 4> NewExts;
6499 unsigned NewCreatedInstsCost = 0;
6500 unsigned ExtCost = !TLI->isExtFree(I);
6501 // Promote.
6502 Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost,
6503 &NewExts, nullptr, *TLI);
6504 assert(PromotedVal &&
6505 "TypePromotionHelper should have filtered out those cases");
6506
6507 // We would be able to merge only one extension in a load.
6508 // Therefore, if we have more than 1 new extension we heuristically
6509 // cut this search path, because it means we degrade the code quality.
6510 // With exactly 2, the transformation is neutral, because we will merge
6511 // one extension but leave one. However, we optimistically keep going,
6512 // because the new extension may be removed too. Also avoid replacing a
6513 // single free extension with multiple extensions, as this increases the
6514 // number of IR instructions while not providing any savings.
6515 long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost;
6516 // FIXME: It would be possible to propagate a negative value instead of
6517 // conservatively ceiling it to 0.
6518 TotalCreatedInstsCost =
6519 std::max((long long)0, (TotalCreatedInstsCost - ExtCost));
6520 if (!Opts.cgp_stress_ext_ld_promotion &&
6521 (TotalCreatedInstsCost > 1 ||
6522 !isPromotedInstructionLegal(*TLI, *DL, PromotedVal) ||
6523 (ExtCost == 0 && NewExts.size() > 1))) {
6524 // This promotion is not profitable, rollback to the previous state, and
6525 // save the current extension in ProfitablyMovedExts as the latest
6526 // speculative promotion turned out to be unprofitable.
6527 TPT.rollback(LastKnownGood);
6528 ProfitablyMovedExts.push_back(I);
6529 continue;
6530 }
6531 // Continue promoting NewExts as far as doing so is profitable.
6532 SmallVector<Instruction *, 2> NewlyMovedExts;
6533 (void)tryToPromoteExts(TPT, NewExts, NewlyMovedExts, TotalCreatedInstsCost);
6534 bool NewPromoted = false;
6535 for (auto *ExtInst : NewlyMovedExts) {
6536 Instruction *MovedExt = cast<Instruction>(ExtInst);
6537 Value *ExtOperand = MovedExt->getOperand(0);
6538 // If we have reached to a load, we need this extra profitability check
6539 // as it could potentially be merged into an ext(load).
6540 if (isa<LoadInst>(ExtOperand) &&
6541 !(Opts.cgp_stress_ext_ld_promotion ||
6542 NewCreatedInstsCost <= ExtCost ||
6543 (ExtOperand->hasOneUse() || hasSameExtUse(ExtOperand, *TLI))))
6544 continue;
6545
6546 ProfitablyMovedExts.push_back(MovedExt);
6547 NewPromoted = true;
6548 }
6549
6550 // If none of speculative promotions for NewExts is profitable, rollback
6551 // and save the current extension (I) as the last profitable extension.
6552 if (!NewPromoted) {
6553 TPT.rollback(LastKnownGood);
6554 ProfitablyMovedExts.push_back(I);
6555 continue;
6556 }
6557 // The promotion is profitable.
6558 Promoted = true;
6559 }
6560 return Promoted;
6561}
6562
6563/// Merging redundant sexts when one is dominating the other.
6564bool CodeGenPrepare::mergeSExts(Function &F) {
6565 bool Changed = false;
6566 for (auto &Entry : ValToSExtendedUses) {
6567 SExts &Insts = Entry.second;
6568 SExts CurPts;
6569 for (Instruction *Inst : Insts) {
6570 if (RemovedInsts.count(Inst) || !isa<SExtInst>(Inst) ||
6571 Inst->getOperand(0) != Entry.first)
6572 continue;
6573 bool inserted = false;
6574 for (auto &Pt : CurPts) {
6575 if (getDT().dominates(Inst, Pt)) {
6576 replaceAllUsesWith(Pt, Inst, FreshBBs, IsHugeFunc);
6577 RemovedInsts.insert(Pt);
6578 Pt->removeFromParent();
6579 Pt = Inst;
6580 inserted = true;
6581 Changed = true;
6582 break;
6583 }
6584 if (!getDT().dominates(Pt, Inst))
6585 // Give up if we need to merge in a common dominator as the
6586 // experiments show it is not profitable.
6587 continue;
6588 replaceAllUsesWith(Inst, Pt, FreshBBs, IsHugeFunc);
6589 RemovedInsts.insert(Inst);
6590 Inst->removeFromParent();
6591 inserted = true;
6592 Changed = true;
6593 break;
6594 }
6595 if (!inserted)
6596 CurPts.push_back(Inst);
6597 }
6598 }
6599 return Changed;
6600}
6601
6602// Splitting large data structures so that the GEPs accessing them can have
6603// smaller offsets so that they can be sunk to the same blocks as their users.
6604// For example, a large struct starting from %base is split into two parts
6605// where the second part starts from %new_base.
6606//
6607// Before:
6608// BB0:
6609// %base =
6610//
6611// BB1:
6612// %gep0 = gep %base, off0
6613// %gep1 = gep %base, off1
6614// %gep2 = gep %base, off2
6615//
6616// BB2:
6617// %load1 = load %gep0
6618// %load2 = load %gep1
6619// %load3 = load %gep2
6620//
6621// After:
6622// BB0:
6623// %base =
6624// %new_base = gep %base, off0
6625//
6626// BB1:
6627// %new_gep0 = %new_base
6628// %new_gep1 = gep %new_base, off1 - off0
6629// %new_gep2 = gep %new_base, off2 - off0
6630//
6631// BB2:
6632// %load1 = load i32, i32* %new_gep0
6633// %load2 = load i32, i32* %new_gep1
6634// %load3 = load i32, i32* %new_gep2
6635//
6636// %new_gep1 and %new_gep2 can be sunk to BB2 now after the splitting because
6637// their offsets are smaller enough to fit into the addressing mode.
6638bool CodeGenPrepare::splitLargeGEPOffsets() {
6639 bool Changed = false;
6640 for (auto &Entry : LargeOffsetGEPMap) {
6641 Value *OldBase = Entry.first;
6642 SmallVectorImpl<std::pair<AssertingVH<GetElementPtrInst>, int64_t>>
6643 &LargeOffsetGEPs = Entry.second;
6644 auto compareGEPOffset =
6645 [&](const std::pair<GetElementPtrInst *, int64_t> &LHS,
6646 const std::pair<GetElementPtrInst *, int64_t> &RHS) {
6647 if (LHS.first == RHS.first)
6648 return false;
6649 if (LHS.second != RHS.second)
6650 return LHS.second < RHS.second;
6651 return LargeOffsetGEPID[LHS.first] < LargeOffsetGEPID[RHS.first];
6652 };
6653 // Sorting all the GEPs of the same data structures based on the offsets.
6654 llvm::sort(LargeOffsetGEPs, compareGEPOffset);
6655 LargeOffsetGEPs.erase(llvm::unique(LargeOffsetGEPs), LargeOffsetGEPs.end());
6656 // Skip if all the GEPs have the same offsets.
6657 if (LargeOffsetGEPs.front().second == LargeOffsetGEPs.back().second)
6658 continue;
6659 GetElementPtrInst *BaseGEP = LargeOffsetGEPs.begin()->first;
6660 int64_t BaseOffset = LargeOffsetGEPs.begin()->second;
6661 Value *NewBaseGEP = nullptr;
6662
6663 auto createNewBase = [&](int64_t BaseOffset, Value *OldBase,
6664 GetElementPtrInst *GEP) {
6665 LLVMContext &Ctx = GEP->getContext();
6666 Type *PtrIdxTy = DL->getIndexType(GEP->getType());
6667 Type *I8PtrTy =
6668 PointerType::get(Ctx, GEP->getType()->getPointerAddressSpace());
6669
6670 BasicBlock::iterator NewBaseInsertPt;
6671 BasicBlock *NewBaseInsertBB;
6672 if (auto *BaseI = dyn_cast<Instruction>(OldBase)) {
6673 // If the base of the struct is an instruction, the new base will be
6674 // inserted close to it.
6675 NewBaseInsertBB = BaseI->getParent();
6676 if (isa<PHINode>(BaseI))
6677 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt();
6678 else if (InvokeInst *Invoke = dyn_cast<InvokeInst>(BaseI)) {
6679 NewBaseInsertBB =
6680 SplitEdge(NewBaseInsertBB, Invoke->getNormalDest(), &getDT(), LI);
6681 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt();
6682 } else
6683 NewBaseInsertPt = std::next(BaseI->getIterator());
6684 } else {
6685 // If the current base is an argument or global value, the new base
6686 // will be inserted to the entry block.
6687 NewBaseInsertBB = &BaseGEP->getFunction()->getEntryBlock();
6688 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt();
6689 }
6690 IRBuilder<> NewBaseBuilder(NewBaseInsertPt);
6691 // Create a new base.
6692 // TODO: Avoid implicit trunc?
6693 // See https://github.com/llvm/llvm-project/issues/112510.
6694 Value *BaseIndex =
6695 ConstantInt::getSigned(PtrIdxTy, BaseOffset, /*ImplicitTrunc=*/true);
6696 NewBaseGEP = OldBase;
6697 if (NewBaseGEP->getType() != I8PtrTy)
6698 NewBaseGEP = NewBaseBuilder.CreatePointerCast(NewBaseGEP, I8PtrTy);
6699 NewBaseGEP =
6700 NewBaseBuilder.CreatePtrAdd(NewBaseGEP, BaseIndex, "splitgep");
6701 NewGEPBases.insert(NewBaseGEP);
6702 return;
6703 };
6704
6705 // Check whether all the offsets can be encoded with prefered common base.
6706 if (int64_t PreferBase = TLI->getPreferredLargeGEPBaseOffset(
6707 LargeOffsetGEPs.front().second, LargeOffsetGEPs.back().second)) {
6708 BaseOffset = PreferBase;
6709 // Create a new base if the offset of the BaseGEP can be decoded with one
6710 // instruction.
6711 createNewBase(BaseOffset, OldBase, BaseGEP);
6712 }
6713
6714 auto *LargeOffsetGEP = LargeOffsetGEPs.begin();
6715 while (LargeOffsetGEP != LargeOffsetGEPs.end()) {
6716 GetElementPtrInst *GEP = LargeOffsetGEP->first;
6717 int64_t Offset = LargeOffsetGEP->second;
6718 if (Offset != BaseOffset) {
6719 TargetLowering::AddrMode AddrMode;
6720 AddrMode.HasBaseReg = true;
6721 AddrMode.BaseOffs = Offset - BaseOffset;
6722 // The result type of the GEP might not be the type of the memory
6723 // access.
6725 GEP->getResultElementType(),
6726 GEP->getAddressSpace())) {
6727 // We need to create a new base if the offset to the current base is
6728 // too large to fit into the addressing mode. So, a very large struct
6729 // may be split into several parts.
6730 BaseGEP = GEP;
6731 BaseOffset = Offset;
6732 NewBaseGEP = nullptr;
6733 }
6734 }
6735
6736 // Generate a new GEP to replace the current one.
6737 Type *PtrIdxTy = DL->getIndexType(GEP->getType());
6738
6739 if (!NewBaseGEP) {
6740 // Create a new base if we don't have one yet. Find the insertion
6741 // pointer for the new base first.
6742 createNewBase(BaseOffset, OldBase, GEP);
6743 }
6744
6745 IRBuilder<> Builder(GEP);
6746 Value *NewGEP = NewBaseGEP;
6747 if (Offset != BaseOffset) {
6748 // Calculate the new offset for the new GEP.
6749 Value *Index = ConstantInt::get(PtrIdxTy, Offset - BaseOffset);
6750 NewGEP = Builder.CreatePtrAdd(NewBaseGEP, Index);
6751 }
6752 replaceAllUsesWith(GEP, NewGEP, FreshBBs, IsHugeFunc);
6753 LargeOffsetGEPID.erase(GEP);
6754 LargeOffsetGEP = LargeOffsetGEPs.erase(LargeOffsetGEP);
6755 GEP->eraseFromParent();
6756 Changed = true;
6757 }
6758 }
6759 return Changed;
6760}
6761
6762bool CodeGenPrepare::optimizePhiType(
6763 PHINode *I, SmallPtrSetImpl<PHINode *> &Visited,
6764 SmallPtrSetImpl<Instruction *> &DeletedInstrs) {
6765 // We are looking for a collection on interconnected phi nodes that together
6766 // only use loads/bitcasts and are used by stores/bitcasts, and the bitcasts
6767 // are of the same type. Convert the whole set of nodes to the type of the
6768 // bitcast.
6769 Type *PhiTy = I->getType();
6770 Type *ConvertTy = nullptr;
6771 if (Visited.count(I) ||
6772 (!I->getType()->isIntegerTy() && !I->getType()->isFloatingPointTy()))
6773 return false;
6774
6775 SmallVector<Instruction *, 4> Worklist;
6776 Worklist.push_back(cast<Instruction>(I));
6777 SmallPtrSet<PHINode *, 4> PhiNodes;
6778 SmallPtrSet<ConstantData *, 4> Constants;
6779 PhiNodes.insert(I);
6780 Visited.insert(I);
6781 SmallPtrSet<Instruction *, 4> Defs;
6782 SmallPtrSet<Instruction *, 4> Uses;
6783 // This works by adding extra bitcasts between load/stores and removing
6784 // existing bitcasts. If we have a phi(bitcast(load)) or a store(bitcast(phi))
6785 // we can get in the situation where we remove a bitcast in one iteration
6786 // just to add it again in the next. We need to ensure that at least one
6787 // bitcast we remove are anchored to something that will not change back.
6788 bool AnyAnchored = false;
6789
6790 while (!Worklist.empty()) {
6791 Instruction *II = Worklist.pop_back_val();
6792
6793 if (auto *Phi = dyn_cast<PHINode>(II)) {
6794 // Handle Defs, which might also be PHI's
6795 for (Value *V : Phi->incoming_values()) {
6796 if (auto *OpPhi = dyn_cast<PHINode>(V)) {
6797 if (!PhiNodes.count(OpPhi)) {
6798 if (!Visited.insert(OpPhi).second)
6799 return false;
6800 PhiNodes.insert(OpPhi);
6801 Worklist.push_back(OpPhi);
6802 }
6803 } else if (auto *OpLoad = dyn_cast<LoadInst>(V)) {
6804 if (!OpLoad->isSimple())
6805 return false;
6806 if (Defs.insert(OpLoad).second)
6807 Worklist.push_back(OpLoad);
6808 } else if (auto *OpEx = dyn_cast<ExtractElementInst>(V)) {
6809 if (Defs.insert(OpEx).second)
6810 Worklist.push_back(OpEx);
6811 } else if (auto *OpBC = dyn_cast<BitCastInst>(V)) {
6812 if (!ConvertTy)
6813 ConvertTy = OpBC->getOperand(0)->getType();
6814 if (OpBC->getOperand(0)->getType() != ConvertTy)
6815 return false;
6816 if (Defs.insert(OpBC).second) {
6817 Worklist.push_back(OpBC);
6818 AnyAnchored |= !isa<LoadInst>(OpBC->getOperand(0)) &&
6819 !isa<ExtractElementInst>(OpBC->getOperand(0));
6820 }
6821 } else if (auto *OpC = dyn_cast<ConstantData>(V))
6822 Constants.insert(OpC);
6823 else
6824 return false;
6825 }
6826 }
6827
6828 // Handle uses which might also be phi's
6829 for (User *V : II->users()) {
6830 if (auto *OpPhi = dyn_cast<PHINode>(V)) {
6831 if (!PhiNodes.count(OpPhi)) {
6832 if (Visited.count(OpPhi))
6833 return false;
6834 PhiNodes.insert(OpPhi);
6835 Visited.insert(OpPhi);
6836 Worklist.push_back(OpPhi);
6837 }
6838 } else if (auto *OpStore = dyn_cast<StoreInst>(V)) {
6839 if (!OpStore->isSimple() || OpStore->getOperand(0) != II)
6840 return false;
6841 Uses.insert(OpStore);
6842 } else if (auto *OpBC = dyn_cast<BitCastInst>(V)) {
6843 if (!ConvertTy)
6844 ConvertTy = OpBC->getType();
6845 if (OpBC->getType() != ConvertTy)
6846 return false;
6847 Uses.insert(OpBC);
6848 AnyAnchored |=
6849 any_of(OpBC->users(), [](User *U) { return !isa<StoreInst>(U); });
6850 } else {
6851 return false;
6852 }
6853 }
6854 }
6855
6856 if (!ConvertTy || !AnyAnchored || PhiTy == ConvertTy ||
6857 !TLI->shouldConvertPhiType(PhiTy, ConvertTy))
6858 return false;
6859
6860 LLVM_DEBUG(dbgs() << "Converting " << *I << "\n and connected nodes to "
6861 << *ConvertTy << "\n");
6862
6863 // Create all the new phi nodes of the new type, and bitcast any loads to the
6864 // correct type.
6865 ValueToValueMap ValMap;
6866 for (ConstantData *C : Constants)
6867 ValMap[C] = ConstantExpr::getBitCast(C, ConvertTy);
6868 for (Instruction *D : Defs) {
6869 if (isa<BitCastInst>(D)) {
6870 ValMap[D] = D->getOperand(0);
6871 DeletedInstrs.insert(D);
6872 } else {
6873 BasicBlock::iterator insertPt = std::next(D->getIterator());
6874 ValMap[D] = new BitCastInst(D, ConvertTy, D->getName() + ".bc", insertPt);
6875 }
6876 }
6877 for (PHINode *Phi : PhiNodes)
6878 ValMap[Phi] = PHINode::Create(ConvertTy, Phi->getNumIncomingValues(),
6879 Phi->getName() + ".tc", Phi->getIterator());
6880 // Pipe together all the PhiNodes.
6881 for (PHINode *Phi : PhiNodes) {
6882 PHINode *NewPhi = cast<PHINode>(ValMap[Phi]);
6883 for (int i = 0, e = Phi->getNumIncomingValues(); i < e; i++)
6884 NewPhi->addIncoming(ValMap[Phi->getIncomingValue(i)],
6885 Phi->getIncomingBlock(i));
6886 Visited.insert(NewPhi);
6887 }
6888 // And finally pipe up the stores and bitcasts
6889 for (Instruction *U : Uses) {
6890 if (isa<BitCastInst>(U)) {
6891 DeletedInstrs.insert(U);
6892 replaceAllUsesWith(U, ValMap[U->getOperand(0)], FreshBBs, IsHugeFunc);
6893 } else {
6894 U->setOperand(0, new BitCastInst(ValMap[U->getOperand(0)], PhiTy, "bc",
6895 U->getIterator()));
6896 }
6897 }
6898
6899 // Save the removed phis to be deleted later.
6900 DeletedInstrs.insert_range(PhiNodes);
6901 return true;
6902}
6903
6904bool CodeGenPrepare::optimizePhiTypes(Function &F) {
6905 if (!Opts.cgp_optimize_phi_types)
6906 return false;
6907
6908 bool Changed = false;
6909 SmallPtrSet<PHINode *, 4> Visited;
6910 SmallPtrSet<Instruction *, 4> DeletedInstrs;
6911
6912 // Attempt to optimize all the phis in the functions to the correct type.
6913 for (auto &BB : F)
6914 for (auto &Phi : BB.phis())
6915 Changed |= optimizePhiType(&Phi, Visited, DeletedInstrs);
6916
6917 // Remove any old phi's that have been converted.
6918 for (auto *I : DeletedInstrs) {
6919 replaceAllUsesWith(I, PoisonValue::get(I->getType()), FreshBBs, IsHugeFunc);
6920 I->eraseFromParent();
6921 }
6922
6923 return Changed;
6924}
6925
6926/// Return true, if an ext(load) can be formed from an extension in
6927/// \p MovedExts.
6928bool CodeGenPrepare::canFormExtLd(
6929 const SmallVectorImpl<Instruction *> &MovedExts, LoadInst *&LI,
6930 Instruction *&Inst, bool HasPromoted) {
6931 for (auto *MovedExtInst : MovedExts) {
6932 if (isa<LoadInst>(MovedExtInst->getOperand(0))) {
6933 LI = cast<LoadInst>(MovedExtInst->getOperand(0));
6934 Inst = MovedExtInst;
6935 break;
6936 }
6937 }
6938 if (!LI)
6939 return false;
6940
6941 // If they're already in the same block, there's nothing to do.
6942 // Make the cheap checks first if we did not promote.
6943 // If we promoted, we need to check if it is indeed profitable.
6944 if (!HasPromoted && LI->getParent() == Inst->getParent())
6945 return false;
6946
6947 return TLI->isExtLoad(LI, Inst, *DL);
6948}
6949
6950/// Move a zext or sext fed by a load into the same basic block as the load,
6951/// unless conditions are unfavorable. This allows SelectionDAG to fold the
6952/// extend into the load.
6953///
6954/// E.g.,
6955/// \code
6956/// %ld = load i32* %addr
6957/// %add = add nuw i32 %ld, 4
6958/// %zext = zext i32 %add to i64
6959// \endcode
6960/// =>
6961/// \code
6962/// %ld = load i32* %addr
6963/// %zext = zext i32 %ld to i64
6964/// %add = add nuw i64 %zext, 4
6965/// \encode
6966/// Note that the promotion in %add to i64 is done in tryToPromoteExts(), which
6967/// allow us to match zext(load i32*) to i64.
6968///
6969/// Also, try to promote the computations used to obtain a sign extended
6970/// value used into memory accesses.
6971/// E.g.,
6972/// \code
6973/// a = add nsw i32 b, 3
6974/// d = sext i32 a to i64
6975/// e = getelementptr ..., i64 d
6976/// \endcode
6977/// =>
6978/// \code
6979/// f = sext i32 b to i64
6980/// a = add nsw i64 f, 3
6981/// e = getelementptr ..., i64 a
6982/// \endcode
6983///
6984/// \p Inst[in/out] the extension may be modified during the process if some
6985/// promotions apply.
6986bool CodeGenPrepare::optimizeExt(Instruction *&Inst) {
6987 bool AllowPromotionWithoutCommonHeader = false;
6988 /// See if it is an interesting sext operations for the address type
6989 /// promotion before trying to promote it, e.g., the ones with the right
6990 /// type and used in memory accesses.
6991 bool ATPConsiderable = TTI->shouldConsiderAddressTypePromotion(
6992 *Inst, AllowPromotionWithoutCommonHeader);
6993 TypePromotionTransaction TPT(RemovedInsts);
6994 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
6995 TPT.getRestorationPoint();
6997 SmallVector<Instruction *, 2> SpeculativelyMovedExts;
6998 Exts.push_back(Inst);
6999
7000 bool HasPromoted = tryToPromoteExts(TPT, Exts, SpeculativelyMovedExts);
7001
7002 // Look for a load being extended.
7003 LoadInst *LI = nullptr;
7004 Instruction *ExtFedByLoad;
7005
7006 // Try to promote a chain of computation if it allows to form an extended
7007 // load.
7008 if (canFormExtLd(SpeculativelyMovedExts, LI, ExtFedByLoad, HasPromoted)) {
7009 assert(LI && ExtFedByLoad && "Expect a valid load and extension");
7010 TPT.commit();
7011 // Move the extend into the same block as the load.
7012 ExtFedByLoad->moveAfter(LI);
7013 ++NumExtsMoved;
7014 Inst = ExtFedByLoad;
7015 return true;
7016 }
7017
7018 // Continue promoting SExts if known as considerable depending on targets.
7019 if (ATPConsiderable &&
7020 performAddressTypePromotion(Inst, AllowPromotionWithoutCommonHeader,
7021 HasPromoted, TPT, SpeculativelyMovedExts))
7022 return true;
7023
7024 TPT.rollback(LastKnownGood);
7025 return false;
7026}
7027
7028// Perform address type promotion if doing so is profitable.
7029// If AllowPromotionWithoutCommonHeader == false, we should find other sext
7030// instructions that sign extended the same initial value. However, if
7031// AllowPromotionWithoutCommonHeader == true, we expect promoting the
7032// extension is just profitable.
7033bool CodeGenPrepare::performAddressTypePromotion(
7034 Instruction *&Inst, bool AllowPromotionWithoutCommonHeader,
7035 bool HasPromoted, TypePromotionTransaction &TPT,
7036 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts) {
7037 bool Promoted = false;
7038 SmallPtrSet<Instruction *, 1> UnhandledExts;
7039 bool AllSeenFirst = true;
7040 for (auto *I : SpeculativelyMovedExts) {
7041 Value *HeadOfChain = I->getOperand(0);
7042 auto AlreadySeen = SeenChainsForSExt.find(HeadOfChain);
7043 // If there is an unhandled SExt which has the same header, try to promote
7044 // it as well.
7045 if (AlreadySeen != SeenChainsForSExt.end()) {
7046 if (AlreadySeen->second != nullptr)
7047 UnhandledExts.insert(AlreadySeen->second);
7048 AllSeenFirst = false;
7049 }
7050 }
7051
7052 if (!AllSeenFirst || (AllowPromotionWithoutCommonHeader &&
7053 SpeculativelyMovedExts.size() == 1)) {
7054 TPT.commit();
7055 if (HasPromoted)
7056 Promoted = true;
7057 for (auto *I : SpeculativelyMovedExts) {
7058 Value *HeadOfChain = I->getOperand(0);
7059 SeenChainsForSExt[HeadOfChain] = nullptr;
7060 ValToSExtendedUses[HeadOfChain].push_back(I);
7061 }
7062 // Update Inst as promotion happen.
7063 Inst = SpeculativelyMovedExts.pop_back_val();
7064 } else {
7065 // This is the first chain visited from the header, keep the current chain
7066 // as unhandled. Defer to promote this until we encounter another SExt
7067 // chain derived from the same header.
7068 for (auto *I : SpeculativelyMovedExts) {
7069 Value *HeadOfChain = I->getOperand(0);
7070 SeenChainsForSExt[HeadOfChain] = Inst;
7071 }
7072 return false;
7073 }
7074
7075 if (!AllSeenFirst && !UnhandledExts.empty())
7076 for (auto *VisitedSExt : UnhandledExts) {
7077 if (RemovedInsts.count(VisitedSExt))
7078 continue;
7079 TypePromotionTransaction TPT(RemovedInsts);
7081 SmallVector<Instruction *, 2> Chains;
7082 Exts.push_back(VisitedSExt);
7083 bool HasPromoted = tryToPromoteExts(TPT, Exts, Chains);
7084 TPT.commit();
7085 if (HasPromoted)
7086 Promoted = true;
7087 for (auto *I : Chains) {
7088 Value *HeadOfChain = I->getOperand(0);
7089 // Mark this as handled.
7090 SeenChainsForSExt[HeadOfChain] = nullptr;
7091 ValToSExtendedUses[HeadOfChain].push_back(I);
7092 }
7093 }
7094 return Promoted;
7095}
7096
7097bool CodeGenPrepare::optimizeExtUses(Instruction *I) {
7098 BasicBlock *DefBB = I->getParent();
7099
7100 // If the result of a {s|z}ext and its source are both live out, rewrite all
7101 // other uses of the source with result of extension.
7102 Value *Src = I->getOperand(0);
7103 if (Src->hasOneUse())
7104 return false;
7105
7106 // Only do this xform if truncating is free.
7107 if (!TLI->isTruncateFree(I->getType(), Src->getType()))
7108 return false;
7109
7110 // Only safe to perform the optimization if the source is also defined in
7111 // this block.
7112 if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent())
7113 return false;
7114
7115 bool DefIsLiveOut = false;
7116 for (User *U : I->users()) {
7118
7119 // Figure out which BB this ext is used in.
7120 BasicBlock *UserBB = UI->getParent();
7121 if (UserBB == DefBB)
7122 continue;
7123 DefIsLiveOut = true;
7124 break;
7125 }
7126 if (!DefIsLiveOut)
7127 return false;
7128
7129 // Make sure none of the uses are PHI nodes.
7130 for (User *U : Src->users()) {
7132 BasicBlock *UserBB = UI->getParent();
7133 if (UserBB == DefBB)
7134 continue;
7135 // Be conservative. We don't want this xform to end up introducing
7136 // reloads just before load / store instructions.
7137 if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI))
7138 return false;
7139 }
7140
7141 // InsertedTruncs - Only insert one trunc in each block once.
7142 DenseMap<BasicBlock *, Instruction *> InsertedTruncs;
7143
7144 bool MadeChange = false;
7145 for (Use &U : make_early_inc_range(Src->uses())) {
7146 Instruction *User = cast<Instruction>(U.getUser());
7147
7148 // Figure out which BB this ext is used in.
7149 BasicBlock *UserBB = User->getParent();
7150 if (UserBB == DefBB)
7151 continue;
7152
7153 // Both src and def are live in this block. Rewrite the use.
7154 Instruction *&InsertedTrunc = InsertedTruncs[UserBB];
7155
7156 if (!InsertedTrunc) {
7157 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
7158 assert(InsertPt != UserBB->end());
7159 InsertedTrunc = new TruncInst(I, Src->getType(), "");
7160 InsertedTrunc->insertBefore(*UserBB, InsertPt);
7161 InsertedInsts.insert(InsertedTrunc);
7162 }
7163
7164 // Replace a use of the {s|z}ext source with a use of the result.
7165 U = InsertedTrunc;
7166 ++NumExtUses;
7167 MadeChange = true;
7168 }
7169
7170 // The uses rewritten above read trunc(I), which equals Src only where I isn't
7171 // poison. A poison-generating flag such as zext nneg can only hold under a
7172 // guard, such as a sign check of Src, that itself is one of those uses.
7173 if (MadeChange)
7174 I->dropPoisonGeneratingFlags();
7175
7176 return MadeChange;
7177}
7178
7179// Find loads whose uses only use some of the loaded value's bits. Add an "and"
7180// just after the load if the target can fold this into one extload instruction,
7181// with the hope of eliminating some of the other later "and" instructions using
7182// the loaded value. "and"s that are made trivially redundant by the insertion
7183// of the new "and" are removed by this function, while others (e.g. those whose
7184// path from the load goes through a phi) are left for isel to potentially
7185// remove.
7186//
7187// For example:
7188//
7189// b0:
7190// x = load i32
7191// ...
7192// b1:
7193// y = and x, 0xff
7194// z = use y
7195//
7196// becomes:
7197//
7198// b0:
7199// x = load i32
7200// x' = and x, 0xff
7201// ...
7202// b1:
7203// z = use x'
7204//
7205// whereas:
7206//
7207// b0:
7208// x1 = load i32
7209// ...
7210// b1:
7211// x2 = load i32
7212// ...
7213// b2:
7214// x = phi x1, x2
7215// y = and x, 0xff
7216//
7217// becomes (after a call to optimizeLoadExt for each load):
7218//
7219// b0:
7220// x1 = load i32
7221// x1' = and x1, 0xff
7222// ...
7223// b1:
7224// x2 = load i32
7225// x2' = and x2, 0xff
7226// ...
7227// b2:
7228// x = phi x1', x2'
7229// y = and x, 0xff
7230bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
7231 if (!Load->isSimple() || !Load->getType()->isIntOrPtrTy())
7232 return false;
7233
7234 // Skip loads we've already transformed.
7235 if (Load->hasOneUse() &&
7236 InsertedInsts.count(cast<Instruction>(*Load->user_begin())))
7237 return false;
7238
7239 // Look at all uses of Load, looking through phis, to determine how many bits
7240 // of the loaded value are needed.
7241 SmallVector<Instruction *, 8> WorkList;
7242 SmallPtrSet<Instruction *, 16> Visited;
7243 SmallVector<Instruction *, 8> AndsToMaybeRemove;
7244 SmallVector<Instruction *, 8> DropFlags;
7245 for (auto *U : Load->users())
7246 WorkList.push_back(cast<Instruction>(U));
7247
7248 EVT LoadResultVT = TLI->getValueType(*DL, Load->getType());
7249 unsigned BitWidth = LoadResultVT.getSizeInBits();
7250 // If the BitWidth is 0, do not try to optimize the type
7251 if (BitWidth == 0)
7252 return false;
7253
7254 APInt DemandBits(BitWidth, 0);
7255 APInt WidestAndBits(BitWidth, 0);
7256
7257 while (!WorkList.empty()) {
7258 Instruction *I = WorkList.pop_back_val();
7259
7260 // Break use-def graph loops.
7261 if (!Visited.insert(I).second)
7262 continue;
7263
7264 // For a PHI node, push all of its users.
7265 if (auto *Phi = dyn_cast<PHINode>(I)) {
7266 for (auto *U : Phi->users())
7267 WorkList.push_back(cast<Instruction>(U));
7268 continue;
7269 }
7270
7271 switch (I->getOpcode()) {
7272 case Instruction::And: {
7273 auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1));
7274 if (!AndC)
7275 return false;
7276 APInt AndBits = AndC->getValue();
7277 DemandBits |= AndBits;
7278 // Keep track of the widest and mask we see.
7279 if (AndBits.ugt(WidestAndBits))
7280 WidestAndBits = AndBits;
7281 if (AndBits == WidestAndBits && I->getOperand(0) == Load)
7282 AndsToMaybeRemove.push_back(I);
7283 break;
7284 }
7285
7286 case Instruction::Shl: {
7287 auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1));
7288 if (!ShlC)
7289 return false;
7290 uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1);
7291 DemandBits.setLowBits(BitWidth - ShiftAmt);
7292 DropFlags.push_back(I);
7293 break;
7294 }
7295
7296 case Instruction::Trunc: {
7297 EVT TruncVT = TLI->getValueType(*DL, I->getType());
7298 unsigned TruncBitWidth = TruncVT.getSizeInBits();
7299 DemandBits.setLowBits(TruncBitWidth);
7300 DropFlags.push_back(I);
7301 break;
7302 }
7303
7304 default:
7305 return false;
7306 }
7307 }
7308
7309 uint32_t ActiveBits = DemandBits.getActiveBits();
7310 // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the
7311 // target even if isLoadLegal says an i1 EXTLOAD is valid. For example,
7312 // for the AArch64 target isLoadLegal(i32, i1, ..., ZEXTLOAD, false) returns
7313 // true, but (and (load x) 1) is not matched as a single instruction, rather
7314 // as a LDR followed by an AND.
7315 // TODO: Look into removing this restriction by fixing backends to either
7316 // return false for isLoadLegal for i1 or have them select this pattern to
7317 // a single instruction.
7318 //
7319 // Also avoid hoisting if we didn't see any ands with the exact DemandBits
7320 // mask, since these are the only ands that will be removed by isel.
7321 if (ActiveBits <= 1 || !DemandBits.isMask(ActiveBits) ||
7322 WidestAndBits != DemandBits)
7323 return false;
7324
7325 LLVMContext &Ctx = Load->getType()->getContext();
7326 Type *TruncTy = Type::getIntNTy(Ctx, ActiveBits);
7327 EVT TruncVT = TLI->getValueType(*DL, TruncTy);
7328
7329 // Reject cases that won't be matched as extloads.
7330 if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() ||
7331 !TLI->isLoadLegal(LoadResultVT, TruncVT, Load->getAlign(),
7332 Load->getPointerAddressSpace(), ISD::ZEXTLOAD, false))
7333 return false;
7334
7335 IRBuilder<> Builder(Load->getNextNode());
7336 auto *NewAnd = cast<Instruction>(
7337 Builder.CreateAnd(Load, ConstantInt::get(Ctx, DemandBits)));
7338 // Mark this instruction as "inserted by CGP", so that other
7339 // optimizations don't touch it.
7340 InsertedInsts.insert(NewAnd);
7341
7342 // Replace all uses of load with new and (except for the use of load in the
7343 // new and itself).
7344 replaceAllUsesWith(Load, NewAnd, FreshBBs, IsHugeFunc);
7345 NewAnd->setOperand(0, Load);
7346
7347 // Remove any and instructions that are now redundant.
7348 for (auto *And : AndsToMaybeRemove)
7349 // Check that the and mask is the same as the one we decided to put on the
7350 // new and.
7351 if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) {
7352 replaceAllUsesWith(And, NewAnd, FreshBBs, IsHugeFunc);
7353 if (&*CurInstIterator == And)
7354 CurInstIterator = std::next(And->getIterator());
7355 And->eraseFromParent();
7356 ++NumAndUses;
7357 }
7358
7359 // NSW flags may not longer hold.
7360 for (auto *Inst : DropFlags)
7361 Inst->setHasNoSignedWrap(false);
7362
7363 ++NumAndsAdded;
7364 return true;
7365}
7366
7367/// Check if V (an operand of a select instruction) is an expensive instruction
7368/// that is only used once.
7370 auto *I = dyn_cast<Instruction>(V);
7371 // If it's safe to speculatively execute, then it should not have side
7372 // effects; therefore, it's safe to sink and possibly *not* execute.
7373 return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) &&
7374 TTI->isExpensiveToSpeculativelyExecute(I);
7375}
7376
7377/// Returns true if a SelectInst should be turned into an explicit branch.
7379 const TargetLowering *TLI,
7380 SelectInst *SI) {
7381 // If even a predictable select is cheap, then a branch can't be cheaper.
7382 if (!TLI->isPredictableSelectExpensive())
7383 return false;
7384
7385 // FIXME: This should use the same heuristics as IfConversion to determine
7386 // whether a select is better represented as a branch.
7387
7388 // If metadata tells us that the select condition is obviously predictable,
7389 // then we want to replace the select with a branch.
7390 uint64_t TrueWeight, FalseWeight;
7391 if (extractBranchWeights(*SI, TrueWeight, FalseWeight)) {
7392 uint64_t Max = std::max(TrueWeight, FalseWeight);
7393 uint64_t Sum = TrueWeight + FalseWeight;
7394 if (Sum != 0) {
7395 auto Probability = BranchProbability::getBranchProbability(Max, Sum);
7396 if (Probability > TTI->getPredictableBranchThreshold())
7397 return true;
7398 }
7399 }
7400
7401 CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition());
7402
7403 // If a branch is predictable, an out-of-order CPU can avoid blocking on its
7404 // comparison condition. If the compare has more than one use, there's
7405 // probably another cmov or setcc around, so it's not worth emitting a branch.
7406 if (!Cmp || !Cmp->hasOneUse())
7407 return false;
7408
7409 // If either operand of the select is expensive and only needed on one side
7410 // of the select, we should form a branch.
7411 if (sinkSelectOperand(TTI, SI->getTrueValue()) ||
7412 sinkSelectOperand(TTI, SI->getFalseValue()))
7413 return true;
7414
7415 return false;
7416}
7417
7418/// If \p isTrue is true, return the true value of \p SI, otherwise return
7419/// false value of \p SI. If the true/false value of \p SI is defined by any
7420/// select instructions in \p Selects, look through the defining select
7421/// instruction until the true/false value is not defined in \p Selects.
7422static Value *
7424 const SmallPtrSet<const Instruction *, 2> &Selects) {
7425 Value *V = nullptr;
7426
7427 for (SelectInst *DefSI = SI; DefSI != nullptr && Selects.count(DefSI);
7428 DefSI = dyn_cast<SelectInst>(V)) {
7429 assert(DefSI->getCondition() == SI->getCondition() &&
7430 "The condition of DefSI does not match with SI");
7431 V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue());
7432 }
7433
7434 assert(V && "Failed to get select true/false value");
7435 return V;
7436}
7437
7438bool CodeGenPrepare::optimizeShiftInst(BinaryOperator *Shift) {
7439 assert(Shift->isShift() && "Expected a shift");
7440
7441 // If this is (1) a vector shift, (2) shifts by scalars are cheaper than
7442 // general vector shifts, and (3) the shift amount is a select-of-splatted
7443 // values, hoist the shifts before the select:
7444 // shift Op0, (select Cond, TVal, FVal) -->
7445 // select Cond, (shift Op0, TVal), (shift Op0, FVal)
7446 //
7447 // This is inverting a generic IR transform when we know that the cost of a
7448 // general vector shift is more than the cost of 2 shift-by-scalars.
7449 // We can't do this effectively in SDAG because we may not be able to
7450 // determine if the select operands are splats from within a basic block.
7451 Type *Ty = Shift->getType();
7452 if (!Ty->isVectorTy() || !TTI->isVectorShiftByScalarCheap(Ty))
7453 return false;
7454 Value *Cond, *TVal, *FVal;
7455 if (!match(Shift->getOperand(1),
7456 m_OneUse(m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))))
7457 return false;
7458 if (!isSplatValue(TVal) || !isSplatValue(FVal))
7459 return false;
7460
7461 IRBuilder<> Builder(Shift);
7462 BinaryOperator::BinaryOps Opcode = Shift->getOpcode();
7463 Value *NewTVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), TVal);
7464 Value *NewFVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), FVal);
7465 Value *NewSel = Builder.CreateSelect(Cond, NewTVal, NewFVal);
7466 replaceAllUsesWith(Shift, NewSel, FreshBBs, IsHugeFunc);
7467 Shift->eraseFromParent();
7468 return true;
7469}
7470
7471bool CodeGenPrepare::optimizeFunnelShift(IntrinsicInst *Fsh) {
7472 Intrinsic::ID Opcode = Fsh->getIntrinsicID();
7473 assert((Opcode == Intrinsic::fshl || Opcode == Intrinsic::fshr) &&
7474 "Expected a funnel shift");
7475
7476 // If this is (1) a vector funnel shift, (2) shifts by scalars are cheaper
7477 // than general vector shifts, and (3) the shift amount is select-of-splatted
7478 // values, hoist the funnel shifts before the select:
7479 // fsh Op0, Op1, (select Cond, TVal, FVal) -->
7480 // select Cond, (fsh Op0, Op1, TVal), (fsh Op0, Op1, FVal)
7481 //
7482 // This is inverting a generic IR transform when we know that the cost of a
7483 // general vector shift is more than the cost of 2 shift-by-scalars.
7484 // We can't do this effectively in SDAG because we may not be able to
7485 // determine if the select operands are splats from within a basic block.
7486 Type *Ty = Fsh->getType();
7487 if (!Ty->isVectorTy() || !TTI->isVectorShiftByScalarCheap(Ty))
7488 return false;
7489 Value *Cond, *TVal, *FVal;
7490 if (!match(Fsh->getOperand(2),
7491 m_OneUse(m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))))
7492 return false;
7493 if (!isSplatValue(TVal) || !isSplatValue(FVal))
7494 return false;
7495
7496 IRBuilder<> Builder(Fsh);
7497 Value *X = Fsh->getOperand(0), *Y = Fsh->getOperand(1);
7498 Value *NewTVal = Builder.CreateIntrinsic(Opcode, Ty, {X, Y, TVal});
7499 Value *NewFVal = Builder.CreateIntrinsic(Opcode, Ty, {X, Y, FVal});
7500 Value *NewSel = Builder.CreateSelect(Cond, NewTVal, NewFVal);
7501 replaceAllUsesWith(Fsh, NewSel, FreshBBs, IsHugeFunc);
7502 Fsh->eraseFromParent();
7503 return true;
7504}
7505
7506/// If we have a SelectInst that will likely profit from branch prediction,
7507/// turn it into a branch.
7508bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
7509 if (!Opts.cgp_select2branch)
7510 return false;
7511
7512 // If the SelectOptimize pass is enabled, selects have already been optimized.
7513 if (!Opts.disable_select_optimize)
7514 return false;
7515
7516 // Find all consecutive select instructions that share the same condition.
7518 ASI.push_back(SI);
7520 It != SI->getParent()->end(); ++It) {
7521 SelectInst *I = dyn_cast<SelectInst>(&*It);
7522 if (I && SI->getCondition() == I->getCondition()) {
7523 ASI.push_back(I);
7524 } else {
7525 break;
7526 }
7527 }
7528
7529 SelectInst *LastSI = ASI.back();
7530 // Increment the current iterator to skip all the rest of select instructions
7531 // because they will be either "not lowered" or "all lowered" to branch.
7532 CurInstIterator = std::next(LastSI->getIterator());
7533 // Examine debug-info attached to the consecutive select instructions. They
7534 // won't be individually optimised by optimizeInst, so we need to perform
7535 // DbgVariableRecord maintenence here instead.
7536 for (SelectInst *SI : ArrayRef(ASI).drop_front())
7537 fixupDbgVariableRecordsOnInst(*SI);
7538
7539 bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1);
7540
7541 // Can we convert the 'select' to CF ?
7542 if (VectorCond || SI->getMetadata(LLVMContext::MD_unpredictable))
7543 return false;
7544
7545 TargetLowering::SelectSupportKind SelectKind;
7546 if (SI->getType()->isVectorTy())
7547 SelectKind = TargetLowering::ScalarCondVectorVal;
7548 else
7549 SelectKind = TargetLowering::ScalarValSelect;
7550
7551 if (TLI->isSelectSupported(SelectKind) &&
7553 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI)))
7554 return false;
7555
7556 // Transform a sequence like this:
7557 // start:
7558 // %cmp = cmp uge i32 %a, %b
7559 // %sel = select i1 %cmp, i32 %c, i32 %d
7560 //
7561 // Into:
7562 // start:
7563 // %cmp = cmp uge i32 %a, %b
7564 // %cmp.frozen = freeze %cmp
7565 // br i1 %cmp.frozen, label %select.true, label %select.false
7566 // select.true:
7567 // br label %select.end
7568 // select.false:
7569 // br label %select.end
7570 // select.end:
7571 // %sel = phi i32 [ %c, %select.true ], [ %d, %select.false ]
7572 //
7573 // %cmp should be frozen, otherwise it may introduce undefined behavior.
7574 // In addition, we may sink instructions that produce %c or %d from
7575 // the entry block into the destination(s) of the new branch.
7576 // If the true or false blocks do not contain a sunken instruction, that
7577 // block and its branch may be optimized away. In that case, one side of the
7578 // first branch will point directly to select.end, and the corresponding PHI
7579 // predecessor block will be the start block.
7580 // The CFG is altered here and we update the DominatorTree and the LoopInfo,
7581 // but we don't set a ModifiedDT flag to avoid restarting the function walk in
7582 // runOnFunction for each select optimized.
7583
7584 // Collect values that go on the true side and the values that go on the false
7585 // side.
7586 SmallVector<Instruction *> TrueInstrs, FalseInstrs;
7587 for (SelectInst *SI : ASI) {
7588 if (Value *V = SI->getTrueValue(); sinkSelectOperand(TTI, V))
7589 TrueInstrs.push_back(cast<Instruction>(V));
7590 if (Value *V = SI->getFalseValue(); sinkSelectOperand(TTI, V))
7591 FalseInstrs.push_back(cast<Instruction>(V));
7592 }
7593
7594 // Split the select block, according to how many (if any) values go on each
7595 // side.
7596 BasicBlock *StartBlock = SI->getParent();
7597 BasicBlock::iterator SplitPt = std::next(BasicBlock::iterator(LastSI));
7598 // We should split before any debug-info.
7599 SplitPt.setHeadBit(true);
7600
7601 IRBuilder<> IB(SI);
7602 auto *CondFr = IB.CreateFreeze(SI->getCondition(), SI->getName() + ".frozen");
7603
7604 BasicBlock *TrueBlock = nullptr;
7605 BasicBlock *FalseBlock = nullptr;
7606 BasicBlock *EndBlock = nullptr;
7607 UncondBrInst *TrueBranch = nullptr;
7608 UncondBrInst *FalseBranch = nullptr;
7609 if (TrueInstrs.size() == 0) {
7610 FalseBranch = cast<UncondBrInst>(
7611 SplitBlockAndInsertIfElse(CondFr, SplitPt, false, nullptr, DTU, LI));
7612 FalseBlock = FalseBranch->getParent();
7613 EndBlock = cast<BasicBlock>(FalseBranch->getOperand(0));
7614 } else if (FalseInstrs.size() == 0) {
7615 TrueBranch = cast<UncondBrInst>(
7616 SplitBlockAndInsertIfThen(CondFr, SplitPt, false, nullptr, DTU, LI));
7617 TrueBlock = TrueBranch->getParent();
7618 EndBlock = TrueBranch->getSuccessor();
7619 } else {
7620 Instruction *ThenTerm = nullptr;
7621 Instruction *ElseTerm = nullptr;
7622 SplitBlockAndInsertIfThenElse(CondFr, SplitPt, &ThenTerm, &ElseTerm,
7623 nullptr, DTU, LI);
7624 TrueBranch = cast<UncondBrInst>(ThenTerm);
7625 FalseBranch = cast<UncondBrInst>(ElseTerm);
7626 TrueBlock = TrueBranch->getParent();
7627 FalseBlock = FalseBranch->getParent();
7628 EndBlock = TrueBranch->getSuccessor();
7629 }
7630
7631 EndBlock->setName("select.end");
7632 if (TrueBlock)
7633 TrueBlock->setName("select.true.sink");
7634 if (FalseBlock)
7635 FalseBlock->setName(FalseInstrs.size() == 0 ? "select.false"
7636 : "select.false.sink");
7637
7638 if (IsHugeFunc) {
7639 if (TrueBlock)
7640 FreshBBs.insert(TrueBlock);
7641 if (FalseBlock)
7642 FreshBBs.insert(FalseBlock);
7643 FreshBBs.insert(EndBlock);
7644 }
7645
7646 BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock));
7647
7648 static const unsigned MD[] = {
7649 LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
7650 LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
7651 StartBlock->getTerminator()->copyMetadata(*SI, MD);
7652
7653 // Sink expensive instructions into the conditional blocks to avoid executing
7654 // them speculatively.
7655 for (Instruction *I : TrueInstrs)
7656 I->moveBefore(TrueBranch->getIterator());
7657 for (Instruction *I : FalseInstrs)
7658 I->moveBefore(FalseBranch->getIterator());
7659
7660 // If we did not create a new block for one of the 'true' or 'false' paths
7661 // of the condition, it means that side of the branch goes to the end block
7662 // directly and the path originates from the start block from the point of
7663 // view of the new PHI.
7664 if (TrueBlock == nullptr)
7665 TrueBlock = StartBlock;
7666 else if (FalseBlock == nullptr)
7667 FalseBlock = StartBlock;
7668
7669 SmallPtrSet<const Instruction *, 2> INS(llvm::from_range, ASI);
7670 // Use reverse iterator because later select may use the value of the
7671 // earlier select, and we need to propagate value through earlier select
7672 // to get the PHI operand.
7673 for (SelectInst *SI : llvm::reverse(ASI)) {
7674 // The select itself is replaced with a PHI Node.
7675 PHINode *PN = PHINode::Create(SI->getType(), 2, "");
7676 PN->insertBefore(EndBlock->begin());
7677 PN->takeName(SI);
7678 PN->addIncoming(getTrueOrFalseValue(SI, true, INS), TrueBlock);
7679 PN->addIncoming(getTrueOrFalseValue(SI, false, INS), FalseBlock);
7680 PN->setDebugLoc(SI->getDebugLoc());
7681
7682 replaceAllUsesWith(SI, PN, FreshBBs, IsHugeFunc);
7683 SI->eraseFromParent();
7684 INS.erase(SI);
7685 ++NumSelectsExpanded;
7686 }
7687
7688 // Instruct OptimizeBlock to skip to the next block.
7689 CurInstIterator = StartBlock->end();
7690 return true;
7691}
7692
7693/// Some targets only accept certain types for splat inputs. For example a VDUP
7694/// in MVE takes a GPR (integer) register, and the instruction that incorporate
7695/// a VDUP (such as a VADD qd, qm, rm) also require a gpr register.
7696bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
7697 // Accept shuf(insertelem(undef/poison, val, 0), undef/poison, <0,0,..>) only
7699 m_Undef(), m_ZeroMask())))
7700 return false;
7701 Type *NewType = TLI->shouldConvertSplatType(SVI);
7702 if (!NewType)
7703 return false;
7704
7705 auto *SVIVecType = cast<FixedVectorType>(SVI->getType());
7706 assert(!NewType->isVectorTy() && "Expected a scalar type!");
7707 assert(NewType->getScalarSizeInBits() == SVIVecType->getScalarSizeInBits() &&
7708 "Expected a type of the same size!");
7709 auto *NewVecType =
7710 FixedVectorType::get(NewType, SVIVecType->getNumElements());
7711
7712 // Create a bitcast (shuffle (insert (bitcast(..))))
7713 IRBuilder<> Builder(SVI);
7714 Value *BC1 = Builder.CreateBitCast(
7715 cast<Instruction>(SVI->getOperand(0))->getOperand(1), NewType);
7716 Value *Shuffle = Builder.CreateVectorSplat(NewVecType->getNumElements(), BC1);
7717 Value *BC2 = Builder.CreateBitCast(Shuffle, SVIVecType);
7718
7719 replaceAllUsesWith(SVI, BC2, FreshBBs, IsHugeFunc);
7721 SVI, TLInfo, nullptr,
7722 [&](Value *V) { removeAllAssertingVHReferences(V); });
7723
7724 // Also hoist the bitcast up to its operand if it they are not in the same
7725 // block.
7726 if (auto *BCI = dyn_cast<Instruction>(BC1))
7727 if (auto *Op = dyn_cast<Instruction>(BCI->getOperand(0)))
7728 if (BCI->getParent() != Op->getParent() && !isa<PHINode>(Op) &&
7729 !Op->isTerminator() && !Op->isEHPad())
7730 BCI->moveAfter(Op);
7731
7732 return true;
7733}
7734
7735bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {
7736 // If the operands of I can be folded into a target instruction together with
7737 // I, duplicate and sink them.
7738 SmallVector<Use *, 4> OpsToSink;
7739 if (!TTI->isProfitableToSinkOperands(I, OpsToSink, LI))
7740 return false;
7741
7742 // OpsToSink can contain multiple uses in a use chain (e.g.
7743 // (%u1 with %u1 = shufflevector), (%u2 with %u2 = zext %u1)). The dominating
7744 // uses must come first, so we process the ops in reverse order so as to not
7745 // create invalid IR.
7746 BasicBlock *TargetBB = I->getParent();
7747 bool Changed = false;
7748 SmallVector<Use *, 4> ToReplace;
7749 Instruction *InsertPoint = I;
7750 for (Use *U : reverse(OpsToSink)) {
7751 auto *UI = cast<Instruction>(U->get());
7752 if (isa<PHINode>(UI) || UI->mayHaveSideEffects() || UI->mayReadFromMemory())
7753 continue;
7754 if (UI->getParent() == TargetBB) {
7755 if (UI->comesBefore(InsertPoint))
7756 InsertPoint = UI;
7757 continue;
7758 }
7759 ToReplace.push_back(U);
7760 }
7761
7762 SetVector<Instruction *> MaybeDead;
7763 DenseMap<Instruction *, Instruction *> NewInstructions;
7764 for (Use *U : ToReplace) {
7765 auto *UI = cast<Instruction>(U->get());
7766 Instruction *NI = UI->clone();
7767
7768 if (IsHugeFunc) {
7769 // Now we clone an instruction, its operands' defs may sink to this BB
7770 // now. So we put the operands defs' BBs into FreshBBs to do optimization.
7771 for (Value *Op : NI->operands())
7772 if (auto *OpDef = dyn_cast<Instruction>(Op))
7773 FreshBBs.insert(OpDef->getParent());
7774 }
7775
7776 NewInstructions[UI] = NI;
7777 MaybeDead.insert(UI);
7778 LLVM_DEBUG(dbgs() << "Sinking " << *UI << " to user " << *I << "\n");
7779 NI->insertBefore(InsertPoint->getIterator());
7780 InsertPoint = NI;
7781 InsertedInsts.insert(NI);
7782
7783 // Update the use for the new instruction, making sure that we update the
7784 // sunk instruction uses, if it is part of a chain that has already been
7785 // sunk.
7786 Instruction *OldI = cast<Instruction>(U->getUser());
7787 if (auto It = NewInstructions.find(OldI); It != NewInstructions.end())
7788 It->second->setOperand(U->getOperandNo(), NI);
7789 else
7790 U->set(NI);
7791 Changed = true;
7792 }
7793
7794 // Remove instructions that are dead after sinking.
7795 for (auto *I : MaybeDead) {
7796 if (!I->hasNUsesOrMore(1)) {
7797 LLVM_DEBUG(dbgs() << "Removing dead instruction: " << *I << "\n");
7798 I->eraseFromParent();
7799 }
7800 }
7801
7802 return Changed;
7803}
7804
7805bool CodeGenPrepare::optimizeSwitchType(SwitchInst *SI) {
7806 Value *Cond = SI->getCondition();
7807 Type *OldType = Cond->getType();
7808 LLVMContext &Context = Cond->getContext();
7809 EVT OldVT = TLI->getValueType(*DL, OldType);
7811 unsigned RegWidth = RegType.getSizeInBits();
7812
7813 if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth())
7814 return false;
7815
7816 // If the register width is greater than the type width, expand the condition
7817 // of the switch instruction and each case constant to the width of the
7818 // register. By widening the type of the switch condition, subsequent
7819 // comparisons (for case comparisons) will not need to be extended to the
7820 // preferred register width, so we will potentially eliminate N-1 extends,
7821 // where N is the number of cases in the switch.
7822 auto *NewType = Type::getIntNTy(Context, RegWidth);
7823
7824 // Extend the switch condition and case constants using the target preferred
7825 // extend unless the switch condition is a function argument with an extend
7826 // attribute. In that case, we can avoid an unnecessary mask/extension by
7827 // matching the argument extension instead.
7828 Instruction::CastOps ExtType = Instruction::ZExt;
7829 // Some targets prefer SExt over ZExt.
7830 if (TLI->isSExtCheaperThanZExt(OldVT, RegType))
7831 ExtType = Instruction::SExt;
7832
7833 if (auto *Arg = dyn_cast<Argument>(Cond)) {
7834 if (Arg->hasSExtAttr())
7835 ExtType = Instruction::SExt;
7836 if (Arg->hasZExtAttr())
7837 ExtType = Instruction::ZExt;
7838 }
7839
7840 auto *ExtInst = CastInst::Create(ExtType, Cond, NewType);
7841 ExtInst->insertBefore(SI->getIterator());
7842 ExtInst->setDebugLoc(SI->getDebugLoc());
7843 SI->setCondition(ExtInst);
7844 for (auto Case : SI->cases()) {
7845 const APInt &NarrowConst = Case.getCaseValue()->getValue();
7846 APInt WideConst = (ExtType == Instruction::ZExt)
7847 ? NarrowConst.zext(RegWidth)
7848 : NarrowConst.sext(RegWidth);
7849 Case.setValue(ConstantInt::get(Context, WideConst));
7850 }
7851
7852 return true;
7853}
7854
7855bool CodeGenPrepare::optimizeSwitchPhiConstants(SwitchInst *SI) {
7856 // The SCCP optimization tends to produce code like this:
7857 // switch(x) { case 42: phi(42, ...) }
7858 // Materializing the constant for the phi-argument needs instructions; So we
7859 // change the code to:
7860 // switch(x) { case 42: phi(x, ...) }
7861
7862 Value *Condition = SI->getCondition();
7863 // Avoid endless loop in degenerate case.
7864 if (isa<ConstantInt>(*Condition))
7865 return false;
7866
7867 bool Changed = false;
7868 BasicBlock *SwitchBB = SI->getParent();
7869 Type *ConditionType = Condition->getType();
7870
7871 for (const SwitchInst::CaseHandle &Case : SI->cases()) {
7872 ConstantInt *CaseValue = Case.getCaseValue();
7873 BasicBlock *CaseBB = Case.getCaseSuccessor();
7874 // Set to true if we previously checked that `CaseBB` is only reached by
7875 // a single case from this switch.
7876 bool CheckedForSinglePred = false;
7877 for (PHINode &PHI : CaseBB->phis()) {
7878 Type *PHIType = PHI.getType();
7879 // If ZExt is free then we can also catch patterns like this:
7880 // switch((i32)x) { case 42: phi((i64)42, ...); }
7881 // and replace `(i64)42` with `zext i32 %x to i64`.
7882 bool TryZExt =
7883 PHIType->isIntegerTy() &&
7884 PHIType->getIntegerBitWidth() > ConditionType->getIntegerBitWidth() &&
7885 TLI->isZExtFree(ConditionType, PHIType);
7886 if (PHIType == ConditionType || TryZExt) {
7887 // Set to true to skip this case because of multiple preds.
7888 bool SkipCase = false;
7889 Value *Replacement = nullptr;
7890 for (unsigned I = 0, E = PHI.getNumIncomingValues(); I != E; I++) {
7891 Value *PHIValue = PHI.getIncomingValue(I);
7892 if (PHIValue != CaseValue) {
7893 if (!TryZExt)
7894 continue;
7895 ConstantInt *PHIValueInt = dyn_cast<ConstantInt>(PHIValue);
7896 if (!PHIValueInt ||
7897 PHIValueInt->getValue() !=
7898 CaseValue->getValue().zext(PHIType->getIntegerBitWidth()))
7899 continue;
7900 }
7901 if (PHI.getIncomingBlock(I) != SwitchBB)
7902 continue;
7903 // We cannot optimize if there are multiple case labels jumping to
7904 // this block. This check may get expensive when there are many
7905 // case labels so we test for it last.
7906 if (!CheckedForSinglePred) {
7907 CheckedForSinglePred = true;
7908 if (SI->findCaseDest(CaseBB) == nullptr) {
7909 SkipCase = true;
7910 break;
7911 }
7912 }
7913
7914 if (Replacement == nullptr) {
7915 if (PHIValue == CaseValue) {
7916 Replacement = Condition;
7917 } else {
7918 IRBuilder<> Builder(SI);
7919 Replacement = Builder.CreateZExt(Condition, PHIType);
7920 }
7921 }
7922 PHI.setIncomingValue(I, Replacement);
7923 Changed = true;
7924 }
7925 if (SkipCase)
7926 break;
7927 }
7928 }
7929 }
7930 return Changed;
7931}
7932
7933bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) {
7934 bool Changed = optimizeSwitchType(SI);
7935 Changed |= optimizeSwitchPhiConstants(SI);
7936 return Changed;
7937}
7938
7939namespace {
7940
7941/// Helper class to promote a scalar operation to a vector one.
7942/// This class is used to move downward extractelement transition.
7943/// E.g.,
7944/// a = vector_op <2 x i32>
7945/// b = extractelement <2 x i32> a, i32 0
7946/// c = scalar_op b
7947/// store c
7948///
7949/// =>
7950/// a = vector_op <2 x i32>
7951/// c = vector_op a (equivalent to scalar_op on the related lane)
7952/// * d = extractelement <2 x i32> c, i32 0
7953/// * store d
7954/// Assuming both extractelement and store can be combine, we get rid of the
7955/// transition.
7956class VectorPromoteHelper {
7957 /// DataLayout associated with the current module.
7958 const DataLayout &DL;
7959
7960 /// Used to perform some checks on the legality of vector operations.
7961 const TargetLowering &TLI;
7962
7963 /// Used to estimated the cost of the promoted chain.
7964 const TargetTransformInfo &TTI;
7965
7966 /// The transition being moved downwards.
7967 Instruction *Transition;
7968
7969 /// The sequence of instructions to be promoted.
7970 SmallVector<Instruction *, 4> InstsToBePromoted;
7971
7972 /// Cost of combining a store and an extract.
7973 unsigned StoreExtractCombineCost;
7974
7975 bool StressStoreExtract;
7976
7977 /// Instruction that will be combined with the transition.
7978 Instruction *CombineInst = nullptr;
7979
7980 /// The instruction that represents the current end of the transition.
7981 /// Since we are faking the promotion until we reach the end of the chain
7982 /// of computation, we need a way to get the current end of the transition.
7983 Instruction *getEndOfTransition() const {
7984 if (InstsToBePromoted.empty())
7985 return Transition;
7986 return InstsToBePromoted.back();
7987 }
7988
7989 /// Return the index of the original value in the transition.
7990 /// E.g., for "extractelement <2 x i32> c, i32 1" the original value,
7991 /// c, is at index 0.
7992 unsigned getTransitionOriginalValueIdx() const {
7993 assert(isa<ExtractElementInst>(Transition) &&
7994 "Other kind of transitions are not supported yet");
7995 return 0;
7996 }
7997
7998 /// Return the index of the index in the transition.
7999 /// E.g., for "extractelement <2 x i32> c, i32 0" the index
8000 /// is at index 1.
8001 unsigned getTransitionIdx() const {
8002 assert(isa<ExtractElementInst>(Transition) &&
8003 "Other kind of transitions are not supported yet");
8004 return 1;
8005 }
8006
8007 /// Get the type of the transition.
8008 /// This is the type of the original value.
8009 /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the
8010 /// transition is <2 x i32>.
8011 Type *getTransitionType() const {
8012 return Transition->getOperand(getTransitionOriginalValueIdx())->getType();
8013 }
8014
8015 /// Promote \p ToBePromoted by moving \p Def downward through.
8016 /// I.e., we have the following sequence:
8017 /// Def = Transition <ty1> a to <ty2>
8018 /// b = ToBePromoted <ty2> Def, ...
8019 /// =>
8020 /// b = ToBePromoted <ty1> a, ...
8021 /// Def = Transition <ty1> ToBePromoted to <ty2>
8022 void promoteImpl(Instruction *ToBePromoted);
8023
8024 /// Check whether or not it is profitable to promote all the
8025 /// instructions enqueued to be promoted.
8026 bool isProfitableToPromote() {
8027 Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx());
8028 unsigned Index = isa<ConstantInt>(ValIdx)
8029 ? cast<ConstantInt>(ValIdx)->getZExtValue()
8030 : -1;
8031 Type *PromotedType = getTransitionType();
8032
8033 StoreInst *ST = cast<StoreInst>(CombineInst);
8034 unsigned AS = ST->getPointerAddressSpace();
8035 // Check if this store is supported.
8037 TLI.getValueType(DL, ST->getValueOperand()->getType()), AS,
8038 ST->getAlign())) {
8039 // If this is not supported, there is no way we can combine
8040 // the extract with the store.
8041 return false;
8042 }
8043
8044 // The scalar chain of computation has to pay for the transition
8045 // scalar to vector.
8046 // The vector chain has to account for the combining cost.
8049 InstructionCost ScalarCost =
8050 TTI.getVectorInstrCost(*Transition, PromotedType, CostKind, Index);
8051 InstructionCost VectorCost = StoreExtractCombineCost;
8052 for (const auto &Inst : InstsToBePromoted) {
8053 // Compute the cost.
8054 // By construction, all instructions being promoted are arithmetic ones.
8055 // Moreover, one argument is a constant that can be viewed as a splat
8056 // constant.
8057 Value *Arg0 = Inst->getOperand(0);
8058 bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) ||
8059 isa<ConstantFP>(Arg0);
8060 TargetTransformInfo::OperandValueInfo Arg0Info, Arg1Info;
8061 if (IsArg0Constant)
8063 else
8065
8066 ScalarCost += TTI.getArithmeticInstrCost(
8067 Inst->getOpcode(), Inst->getType(), CostKind, Arg0Info, Arg1Info);
8068 VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType,
8069 CostKind, Arg0Info, Arg1Info);
8070 }
8071 LLVM_DEBUG(
8072 dbgs() << "Estimated cost of computation to be promoted:\nScalar: "
8073 << ScalarCost << "\nVector: " << VectorCost << '\n');
8074 return ScalarCost > VectorCost;
8075 }
8076
8077 /// Generate a constant vector with \p Val with the same
8078 /// number of elements as the transition.
8079 /// \p UseSplat defines whether or not \p Val should be replicated
8080 /// across the whole vector.
8081 /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>,
8082 /// otherwise we generate a vector with as many poison as possible:
8083 /// <poison, ..., poison, Val, poison, ..., poison> where \p Val is only
8084 /// used at the index of the extract.
8085 Value *getConstantVector(Constant *Val, bool UseSplat) const {
8086 unsigned ExtractIdx = std::numeric_limits<unsigned>::max();
8087 if (!UseSplat) {
8088 // If we cannot determine where the constant must be, we have to
8089 // use a splat constant.
8090 Value *ValExtractIdx = Transition->getOperand(getTransitionIdx());
8091 if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx))
8092 ExtractIdx = CstVal->getSExtValue();
8093 else
8094 UseSplat = true;
8095 }
8096
8097 ElementCount EC = cast<VectorType>(getTransitionType())->getElementCount();
8098 if (UseSplat)
8099 return ConstantVector::getSplat(EC, Val);
8100
8101 if (!EC.isScalable()) {
8102 SmallVector<Constant *, 4> ConstVec;
8103 PoisonValue *PoisonVal = PoisonValue::get(Val->getType());
8104 for (unsigned Idx = 0; Idx != EC.getKnownMinValue(); ++Idx) {
8105 if (Idx == ExtractIdx)
8106 ConstVec.push_back(Val);
8107 else
8108 ConstVec.push_back(PoisonVal);
8109 }
8110 return ConstantVector::get(ConstVec);
8111 } else
8113 "Generate scalable vector for non-splat is unimplemented");
8114 }
8115
8116 /// Check if promoting to a vector type an operand at \p OperandIdx
8117 /// in \p Use can trigger undefined behavior.
8118 static bool canCauseUndefinedBehavior(const Instruction *Use,
8119 unsigned OperandIdx) {
8120 // This is not safe to introduce undef when the operand is on
8121 // the right hand side of a division-like instruction.
8122 if (OperandIdx != 1)
8123 return false;
8124 switch (Use->getOpcode()) {
8125 default:
8126 return false;
8127 case Instruction::SDiv:
8128 case Instruction::UDiv:
8129 case Instruction::SRem:
8130 case Instruction::URem:
8131 return true;
8132 case Instruction::FDiv:
8133 case Instruction::FRem:
8134 return !Use->hasNoNaNs();
8135 }
8136 llvm_unreachable(nullptr);
8137 }
8138
8139public:
8140 VectorPromoteHelper(const DataLayout &DL, const TargetLowering &TLI,
8141 const TargetTransformInfo &TTI, Instruction *Transition,
8142 unsigned CombineCost, bool StressStoreExtract)
8143 : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition),
8144 StoreExtractCombineCost(CombineCost),
8145 StressStoreExtract(StressStoreExtract) {
8146 assert(Transition && "Do not know how to promote null");
8147 }
8148
8149 /// Check if we can promote \p ToBePromoted to \p Type.
8150 bool canPromote(const Instruction *ToBePromoted) const {
8151 // We could support CastInst too.
8152 return isa<BinaryOperator>(ToBePromoted);
8153 }
8154
8155 /// Check if it is profitable to promote \p ToBePromoted
8156 /// by moving downward the transition through.
8157 bool shouldPromote(const Instruction *ToBePromoted) const {
8159 return false;
8160 // Promote only if all the operands can be statically expanded.
8161 // Indeed, we do not want to introduce any new kind of transitions.
8162 for (const Use &U : ToBePromoted->operands()) {
8163 const Value *Val = U.get();
8164 if (Val == getEndOfTransition()) {
8165 continue;
8166 }
8167 if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) &&
8168 !isa<ConstantFP>(Val))
8169 return false;
8170 }
8171 // Check that the resulting operation is legal.
8172 int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode());
8173 if (!ISDOpcode)
8174 return false;
8175 return StressStoreExtract ||
8177 ISDOpcode, TLI.getValueType(DL, getTransitionType(), true));
8178 }
8179
8180 /// Check whether or not \p Use can be combined
8181 /// with the transition.
8182 /// I.e., is it possible to do Use(Transition) => AnotherUse?
8183 bool canCombine(const Instruction *Use) { return isa<StoreInst>(Use); }
8184
8185 /// Record \p ToBePromoted as part of the chain to be promoted.
8186 void enqueueForPromotion(Instruction *ToBePromoted) {
8187 InstsToBePromoted.push_back(ToBePromoted);
8188 }
8189
8190 /// Set the instruction that will be combined with the transition.
8191 void recordCombineInstruction(Instruction *ToBeCombined) {
8192 assert(canCombine(ToBeCombined) && "Unsupported instruction to combine");
8193 CombineInst = ToBeCombined;
8194 }
8195
8196 /// Promote all the instructions enqueued for promotion if it is
8197 /// is profitable.
8198 /// \return True if the promotion happened, false otherwise.
8199 bool promote() {
8200 // Check if there is something to promote.
8201 // Right now, if we do not have anything to combine with,
8202 // we assume the promotion is not profitable.
8203 if (InstsToBePromoted.empty() || !CombineInst)
8204 return false;
8205
8206 // Check cost.
8207 if (!StressStoreExtract && !isProfitableToPromote())
8208 return false;
8209
8210 // Promote.
8211 for (auto &ToBePromoted : InstsToBePromoted)
8212 promoteImpl(ToBePromoted);
8213 InstsToBePromoted.clear();
8214 return true;
8215 }
8216};
8217
8218} // end anonymous namespace
8219
8220void VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) {
8221 // At this point, we know that all the operands of ToBePromoted but Def
8222 // can be statically promoted.
8223 // For Def, we need to use its parameter in ToBePromoted:
8224 // b = ToBePromoted ty1 a
8225 // Def = Transition ty1 b to ty2
8226 // Move the transition down.
8227 // 1. Replace all uses of the promoted operation by the transition.
8228 // = ... b => = ... Def.
8229 assert(ToBePromoted->getType() == Transition->getType() &&
8230 "The type of the result of the transition does not match "
8231 "the final type");
8232 ToBePromoted->replaceAllUsesWith(Transition);
8233 // 2. Update the type of the uses.
8234 // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def.
8235 Type *TransitionTy = getTransitionType();
8236 ToBePromoted->mutateType(TransitionTy);
8237 // 3. Update all the operands of the promoted operation with promoted
8238 // operands.
8239 // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a.
8240 for (Use &U : ToBePromoted->operands()) {
8241 Value *Val = U.get();
8242 Value *NewVal = nullptr;
8243 if (Val == Transition)
8244 NewVal = Transition->getOperand(getTransitionOriginalValueIdx());
8245 else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) ||
8246 isa<ConstantFP>(Val)) {
8247 // Use a splat constant if it is not safe to use undef.
8248 NewVal = getConstantVector(
8249 cast<Constant>(Val),
8250 isa<UndefValue>(Val) ||
8251 canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()));
8252 } else
8253 llvm_unreachable("Did you modified shouldPromote and forgot to update "
8254 "this?");
8255 ToBePromoted->setOperand(U.getOperandNo(), NewVal);
8256 }
8257 Transition->moveAfter(ToBePromoted);
8258 Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted);
8259}
8260
8261/// Some targets can do store(extractelement) with one instruction.
8262/// Try to push the extractelement towards the stores when the target
8263/// has this feature and this is profitable.
8264bool CodeGenPrepare::optimizeExtractElementInst(Instruction *Inst) {
8265 unsigned CombineCost = std::numeric_limits<unsigned>::max();
8266 if (!Opts.cgp_store_extract ||
8267 (!Opts.cgp_stress_store_extract &&
8269 Inst->getOperand(1), CombineCost)))
8270 return false;
8271
8272 // At this point we know that Inst is a vector to scalar transition.
8273 // Try to move it down the def-use chain, until:
8274 // - We can combine the transition with its single use
8275 // => we got rid of the transition.
8276 // - We escape the current basic block
8277 // => we would need to check that we are moving it at a cheaper place and
8278 // we do not do that for now.
8279 BasicBlock *Parent = Inst->getParent();
8280 LLVM_DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n');
8281 VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost,
8282 Opts.cgp_stress_store_extract);
8283 // If the transition has more than one use, assume this is not going to be
8284 // beneficial.
8285 while (Inst->hasOneUse()) {
8286 Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin());
8287 LLVM_DEBUG(dbgs() << "Use: " << *ToBePromoted << '\n');
8288
8289 if (ToBePromoted->getParent() != Parent) {
8290 LLVM_DEBUG(dbgs() << "Instruction to promote is in a different block ("
8291 << ToBePromoted->getParent()->getName()
8292 << ") than the transition (" << Parent->getName()
8293 << ").\n");
8294 return false;
8295 }
8296
8297 if (VPH.canCombine(ToBePromoted)) {
8298 LLVM_DEBUG(dbgs() << "Assume " << *Inst << '\n'
8299 << "will be combined with: " << *ToBePromoted << '\n');
8300 VPH.recordCombineInstruction(ToBePromoted);
8301 bool Changed = VPH.promote();
8302 NumStoreExtractExposed += Changed;
8303 return Changed;
8304 }
8305
8306 LLVM_DEBUG(dbgs() << "Try promoting.\n");
8307 if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted))
8308 return false;
8309
8310 LLVM_DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n");
8311
8312 VPH.enqueueForPromotion(ToBePromoted);
8313 Inst = ToBePromoted;
8314 }
8315 return false;
8316}
8317
8318/// For the instruction sequence of store below, F and I values
8319/// are bundled together as an i64 value before being stored into memory.
8320/// Sometimes it is more efficient to generate separate stores for F and I,
8321/// which can remove the bitwise instructions or sink them to colder places.
8322///
8323/// (store (or (zext (bitcast F to i32) to i64),
8324/// (shl (zext I to i64), 32)), addr) -->
8325/// (store F, addr) and (store I, addr+4)
8326///
8327/// Similarly, splitting for other merged store can also be beneficial, like:
8328/// For pair of {i32, i32}, i64 store --> two i32 stores.
8329/// For pair of {i32, i16}, i64 store --> two i32 stores.
8330/// For pair of {i16, i16}, i32 store --> two i16 stores.
8331/// For pair of {i16, i8}, i32 store --> two i16 stores.
8332/// For pair of {i8, i8}, i16 store --> two i8 stores.
8333///
8334/// We allow each target to determine specifically which kind of splitting is
8335/// supported.
8336///
8337/// The store patterns are commonly seen from the simple code snippet below
8338/// if only std::make_pair(...) is sroa transformed before inlined into hoo.
8339/// void goo(const std::pair<int, float> &);
8340/// hoo() {
8341/// ...
8342/// goo(std::make_pair(tmp, ftmp));
8343/// ...
8344/// }
8345///
8346/// Although we already have similar splitting in DAG Combine, we duplicate
8347/// it in CodeGenPrepare to catch the case in which pattern is across
8348/// multiple BBs. The logic in DAG Combine is kept to catch case generated
8349/// during code expansion.
8351 const TargetLowering &TLI,
8352 bool ForceSplitStore) {
8353 // Handle simple but common cases only.
8354 Type *StoreType = SI.getValueOperand()->getType();
8355
8356 // The code below assumes shifting a value by <number of bits>,
8357 // whereas scalable vectors would have to be shifted by
8358 // <2log(vscale) + number of bits> in order to store the
8359 // low/high parts. Bailing out for now.
8360 if (StoreType->isScalableTy())
8361 return false;
8362
8363 if (!DL.typeSizeEqualsStoreSize(StoreType) ||
8364 DL.getTypeSizeInBits(StoreType) == 0)
8365 return false;
8366
8367 unsigned HalfValBitSize = DL.getTypeSizeInBits(StoreType) / 2;
8368 Type *SplitStoreType = Type::getIntNTy(SI.getContext(), HalfValBitSize);
8369 if (!DL.typeSizeEqualsStoreSize(SplitStoreType))
8370 return false;
8371
8372 // Don't split the store if it is volatile or atomic.
8373 if (!SI.isSimple())
8374 return false;
8375
8376 // Match the following patterns:
8377 // (store (or (zext LValue to i64),
8378 // (shl (zext HValue to i64), 32)), HalfValBitSize)
8379 // or
8380 // (store (or (shl (zext HValue to i64), 32)), HalfValBitSize)
8381 // (zext LValue to i64),
8382 // Expect both operands of OR and the first operand of SHL have only
8383 // one use.
8384 Value *LValue, *HValue;
8385 if (!match(SI.getValueOperand(),
8388 m_SpecificInt(HalfValBitSize))))))
8389 return false;
8390
8391 // Check LValue and HValue are int with size less or equal than 32.
8392 if (!LValue->getType()->isIntegerTy() ||
8393 DL.getTypeSizeInBits(LValue->getType()) > HalfValBitSize ||
8394 !HValue->getType()->isIntegerTy() ||
8395 DL.getTypeSizeInBits(HValue->getType()) > HalfValBitSize)
8396 return false;
8397
8398 // If LValue/HValue is a bitcast instruction, use the EVT before bitcast
8399 // as the input of target query.
8400 auto *LBC = dyn_cast<BitCastInst>(LValue);
8401 auto *HBC = dyn_cast<BitCastInst>(HValue);
8402 EVT LowTy = LBC ? EVT::getEVT(LBC->getOperand(0)->getType())
8403 : EVT::getEVT(LValue->getType());
8404 EVT HighTy = HBC ? EVT::getEVT(HBC->getOperand(0)->getType())
8405 : EVT::getEVT(HValue->getType());
8406 if (!ForceSplitStore && !TLI.isMultiStoresCheaperThanBitsMerge(LowTy, HighTy))
8407 return false;
8408
8409 // Start to split store.
8410 IRBuilder<> Builder(&SI);
8411 Builder.disableGEPCanonicalization();
8412
8413 // If LValue/HValue is a bitcast in another BB, create a new one in current
8414 // BB so it may be merged with the splitted stores by dag combiner.
8415 if (LBC && LBC->getParent() != SI.getParent())
8416 LValue = Builder.CreateBitCast(LBC->getOperand(0), LBC->getType());
8417 if (HBC && HBC->getParent() != SI.getParent())
8418 HValue = Builder.CreateBitCast(HBC->getOperand(0), HBC->getType());
8419
8420 bool IsLE = SI.getDataLayout().isLittleEndian();
8421 auto CreateSplitStore = [&](Value *V, bool Upper) {
8422 V = Builder.CreateZExtOrBitCast(V, SplitStoreType);
8423 Value *Addr = SI.getPointerOperand();
8424 Align Alignment = SI.getAlign();
8425 const bool IsOffsetStore = (IsLE && Upper) || (!IsLE && !Upper);
8426 if (IsOffsetStore) {
8427 Addr = Builder.CreateGEP(
8428 SplitStoreType, Addr,
8429 ConstantInt::get(Type::getInt32Ty(SI.getContext()), 1));
8430
8431 // When splitting the store in half, naturally one half will retain the
8432 // alignment of the original wider store, regardless of whether it was
8433 // over-aligned or not, while the other will require adjustment.
8434 Alignment = commonAlignment(Alignment, HalfValBitSize / 8);
8435 }
8436 Builder.CreateAlignedStore(V, Addr, Alignment);
8437 };
8438
8439 CreateSplitStore(LValue, false);
8440 CreateSplitStore(HValue, true);
8441
8442 // Delete the old store.
8443 SI.eraseFromParent();
8444 return true;
8445}
8446
8447// Try unmerging GEPs to reduce liveness interference (register pressure) across
8448// IndirectBr edges. Since IndirectBr edges tend to touch on many blocks,
8449// reducing liveness interference across those edges benefits global register
8450// allocation. Currently handles only certain cases.
8451//
8452// For example, unmerge %GEPI and %UGEPI as below.
8453//
8454// ---------- BEFORE ----------
8455// SrcBlock:
8456// ...
8457// %GEPIOp = ...
8458// ...
8459// %GEPI = gep %GEPIOp, Idx
8460// ...
8461// indirectbr ... [ label %DstB0, label %DstB1, ... label %DstBi ... ]
8462// (* %GEPI is alive on the indirectbr edges due to other uses ahead)
8463// (* %GEPIOp is alive on the indirectbr edges only because of it's used by
8464// %UGEPI)
8465//
8466// DstB0: ... (there may be a gep similar to %UGEPI to be unmerged)
8467// DstB1: ... (there may be a gep similar to %UGEPI to be unmerged)
8468// ...
8469//
8470// DstBi:
8471// ...
8472// %UGEPI = gep %GEPIOp, UIdx
8473// ...
8474// ---------------------------
8475//
8476// ---------- AFTER ----------
8477// SrcBlock:
8478// ... (same as above)
8479// (* %GEPI is still alive on the indirectbr edges)
8480// (* %GEPIOp is no longer alive on the indirectbr edges as a result of the
8481// unmerging)
8482// ...
8483//
8484// DstBi:
8485// ...
8486// %UGEPI = gep %GEPI, (UIdx-Idx)
8487// ...
8488// ---------------------------
8489//
8490// The register pressure on the IndirectBr edges is reduced because %GEPIOp is
8491// no longer alive on them.
8492//
8493// We try to unmerge GEPs here in CodGenPrepare, as opposed to limiting merging
8494// of GEPs in the first place in InstCombiner::visitGetElementPtrInst() so as
8495// not to disable further simplications and optimizations as a result of GEP
8496// merging.
8497//
8498// Note this unmerging may increase the length of the data flow critical path
8499// (the path from %GEPIOp to %UGEPI would go through %GEPI), which is a tradeoff
8500// between the register pressure and the length of data-flow critical
8501// path. Restricting this to the uncommon IndirectBr case would minimize the
8502// impact of potentially longer critical path, if any, and the impact on compile
8503// time.
8505 const TargetTransformInfo *TTI) {
8506 const DataLayout &DL = GEPI->getDataLayout();
8507 BasicBlock *SrcBlock = GEPI->getParent();
8508 // Check that SrcBlock ends with an IndirectBr. If not, give up. The common
8509 // (non-IndirectBr) cases exit early here.
8510 if (!isa<IndirectBrInst>(SrcBlock->getTerminator()))
8511 return false;
8512 // Check that GEPI is a constant ptradd with canonical index type.
8513 Instruction *GEPIOp;
8514 ConstantInt *GEPIIdx;
8515 if (!match(GEPI, m_PtrAdd(DL, m_Instruction(GEPIOp), m_ConstantInt(GEPIIdx))))
8516 return false;
8517 // Check that GEPI is a cheap one.
8518 if (TTI->getIntImmCost(GEPIIdx->getValue(), GEPIIdx->getType(),
8521 return false;
8522 // Check that GEPIOp is also defined in SrcBlock.
8523 if (GEPIOp->getParent() != SrcBlock)
8524 return false;
8525 // Check that GEP is used outside the block, meaning it's alive on the
8526 // IndirectBr edge(s).
8527 if (llvm::none_of(GEPI->users(), [&](User *Usr) {
8528 if (auto *I = dyn_cast<Instruction>(Usr)) {
8529 if (I->getParent() != SrcBlock) {
8530 return true;
8531 }
8532 }
8533 return false;
8534 }))
8535 return false;
8536 // The second elements of the GEP chains to be unmerged.
8537 std::vector<GetElementPtrInst *> UGEPIs;
8538 // Check each user of GEPIOp to check if unmerging would make GEPIOp not alive
8539 // on IndirectBr edges.
8540 for (User *Usr : GEPIOp->users()) {
8541 if (Usr == GEPI)
8542 continue;
8543 // Check if Usr is an Instruction. If not, give up.
8544 if (!isa<Instruction>(Usr))
8545 return false;
8546 auto *UI = cast<Instruction>(Usr);
8547 // Check if Usr in the same block as GEPIOp, which is fine, skip.
8548 if (UI->getParent() == SrcBlock)
8549 continue;
8550 // Check if Usr is also a constant ptradd with GEPIOp as pointer operand.
8551 // If so, record it in the vector. If not, give up.
8552 ConstantInt *UGEPIIdx;
8553 if (!match(Usr, m_PtrAdd(DL, m_Specific(GEPIOp), m_ConstantInt(UGEPIIdx))))
8554 return false;
8555 if (GEPIIdx->getType() != UGEPIIdx->getType())
8556 return false;
8557 if (TTI->getIntImmCost(UGEPIIdx->getValue(), UGEPIIdx->getType(),
8560 return false;
8561 UGEPIs.push_back(cast<GetElementPtrInst>(Usr));
8562 }
8563 if (UGEPIs.size() == 0)
8564 return false;
8565 // Check the materializing cost of (Uidx-Idx).
8566 for (GetElementPtrInst *UGEPI : UGEPIs) {
8567 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));
8568 APInt NewIdx = UGEPIIdx->getValue() - GEPIIdx->getValue();
8570 NewIdx, GEPIIdx->getType(), TargetTransformInfo::TCK_SizeAndLatency);
8571 if (ImmCost > TargetTransformInfo::TCC_Basic)
8572 return false;
8573 }
8574 // Now unmerge between GEPI and UGEPIs.
8575 for (GetElementPtrInst *UGEPI : UGEPIs) {
8576 UGEPI->setOperand(0, GEPI);
8577 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));
8578 auto NewIdx = UGEPIIdx->getValue() - GEPIIdx->getValue();
8579 Constant *NewUGEPIIdx = ConstantInt::get(GEPIIdx->getType(), NewIdx);
8580 UGEPI->setOperand(1, NewUGEPIIdx);
8581
8582 auto SourceFlags = GEPI->getNoWrapFlags();
8583 // Intersect flags to avoid UB in updated GEP.
8584 auto TargetFlags =
8585 UGEPI->getNoWrapFlags().intersectForOffsetAdd(SourceFlags);
8586 // If UGEPI now has a negative index, drop the nuw flag.
8587 if (NewIdx.isNegative() && TargetFlags.hasNoUnsignedWrap())
8588 TargetFlags = TargetFlags.withoutNoUnsignedWrap();
8589 UGEPI->setNoWrapFlags(TargetFlags);
8590 }
8591 // After unmerging, verify that GEPIOp is actually only used in SrcBlock (not
8592 // alive on IndirectBr edges).
8593 assert(llvm::none_of(GEPIOp->users(),
8594 [&](User *Usr) {
8595 return cast<Instruction>(Usr)->getParent() != SrcBlock;
8596 }) &&
8597 "GEPIOp is used outside SrcBlock");
8598 return true;
8599}
8600
8601static bool optimizeBranch(CondBrInst *Branch, const TargetLowering &TLI,
8603 bool IsHugeFunc) {
8604 // Try and convert
8605 // %c = icmp ult %x, 8
8606 // br %c, bla, blb
8607 // %tc = lshr %x, 3
8608 // to
8609 // %tc = lshr %x, 3
8610 // %c = icmp eq %tc, 0
8611 // br %c, bla, blb
8612 // Creating the cmp to zero can be better for the backend, especially if the
8613 // lshr produces flags that can be used automatically.
8614 if (!TLI.preferZeroCompareBranch())
8615 return false;
8616
8617 ICmpInst *Cmp = dyn_cast<ICmpInst>(Branch->getCondition());
8618 if (!Cmp || !isa<ConstantInt>(Cmp->getOperand(1)) || !Cmp->hasOneUse())
8619 return false;
8620
8621 Value *X = Cmp->getOperand(0);
8622 if (!X->hasUseList())
8623 return false;
8624
8625 APInt CmpC = cast<ConstantInt>(Cmp->getOperand(1))->getValue();
8626
8627 for (auto *U : X->users()) {
8629 // A quick dominance check
8630 if (!UI ||
8631 (UI->getParent() != Branch->getParent() &&
8632 UI->getParent() != Branch->getSuccessor(0) &&
8633 UI->getParent() != Branch->getSuccessor(1)) ||
8634 (UI->getParent() != Branch->getParent() &&
8635 !UI->getParent()->getSinglePredecessor()))
8636 continue;
8637
8638 if (CmpC.isPowerOf2() && Cmp->getPredicate() == ICmpInst::ICMP_ULT &&
8639 match(UI, m_Shr(m_Specific(X), m_SpecificInt(CmpC.logBase2())))) {
8640 IRBuilder<> Builder(Branch);
8641 if (UI->getParent() != Branch->getParent())
8642 UI->moveBefore(Branch->getIterator());
8644 Value *NewCmp = Builder.CreateCmp(ICmpInst::ICMP_EQ, UI,
8645 ConstantInt::get(UI->getType(), 0));
8646 LLVM_DEBUG(dbgs() << "Converting " << *Cmp << "\n");
8647 LLVM_DEBUG(dbgs() << " to compare on zero: " << *NewCmp << "\n");
8648 replaceAllUsesWith(Cmp, NewCmp, FreshBBs, IsHugeFunc);
8649 return true;
8650 }
8651 if (Cmp->isEquality() &&
8652 (match(UI, m_Add(m_Specific(X), m_SpecificInt(-CmpC))) ||
8653 match(UI, m_Sub(m_Specific(X), m_SpecificInt(CmpC))) ||
8654 match(UI, m_Xor(m_Specific(X), m_SpecificInt(CmpC))))) {
8655 IRBuilder<> Builder(Branch);
8656 if (UI->getParent() != Branch->getParent())
8657 UI->moveBefore(Branch->getIterator());
8659 Value *NewCmp = Builder.CreateCmp(Cmp->getPredicate(), UI,
8660 ConstantInt::get(UI->getType(), 0));
8661 LLVM_DEBUG(dbgs() << "Converting " << *Cmp << "\n");
8662 LLVM_DEBUG(dbgs() << " to compare on zero: " << *NewCmp << "\n");
8663 replaceAllUsesWith(Cmp, NewCmp, FreshBBs, IsHugeFunc);
8664 return true;
8665 }
8666 }
8667 return false;
8668}
8669
8670// Performs very basic loop strength reduction to vector values with a known
8671// evolution from one iteration to the next that are used as the value operand
8672// for a store.
8673// A simple example to illustrate:
8674//
8675// %vec.ind = phi <vscale x 2 x i64> [ %start, %vector.ph ],
8676// [ %vec.ind.next, %vector.body ]
8677// %data.gep = getelementptr inbounds nuw [72 x i8], ptr %datap,
8678// <vscale x 2 x i64> %vec.ind
8679// %addr.gep = getelementptr inbounds nuw [8 x i8], ptr %addrp, i64 %index
8680// store <vscale x 2 x ptr> %data.gep, ptr %addr.gep, align 8
8681// %vec.ind.next = add nuw nsw <vscale x 2 x i64> %vec.ind, %elt.cnt.splat
8682//
8683// We want to change this to:
8684// vector.ph:
8685// %base = gep i8, %datap, (<stepvector> * 72)
8686// %stride = splat(elementcount) * 72
8687//
8688// vector.body:
8689// %vec.lsr.phi = phi <vscale x 2 x ptr> [ %base, %vector.ph ],
8690// [ %vec.lsr.next, %vector.body ]
8691// %addr.gep = gep inbounds nuw [8 x i8], ptr %addrp, i64 %index
8692// store <vscale x 2 x ptr> %vec.lsr.phi, ptr %addr.gep, align 8
8693// %vec.lsr.next = gep i8, %vec.lsr.phi, %stride
8694//
8695// Doing so will remove a multiply from the loop, and leave the update as just
8696// an add.
8697//
8699 // We're only interested in header phis in innermost loops.
8700 // We want a loop with an identifiable preheader and single latch.
8701 Loop *L = LI->getLoopFor(Phi->getParent());
8702 if (!L || !L->isInnermost())
8703 return false;
8704
8705 BasicBlock *Header = L->getHeader();
8706 BasicBlock *PreHeader = L->getLoopPreheader();
8707 BasicBlock *Latch = L->getLoopLatch();
8708 if (!PreHeader || !Latch || Phi->getParent() != Header ||
8709 Phi->getNumIncomingValues() != 2)
8710 return false;
8711
8712 // Check the progression of the phi.
8713 Value *Start = Phi->getIncomingValueForBlock(PreHeader);
8714 Value *Step = Phi->getIncomingValueForBlock(Latch);
8715
8716 // Check that the elements are integers equal in size to pointers.
8717 DataLayout DL = Phi->getFunction()->getDataLayout();
8718 VectorType *VTy = dyn_cast<VectorType>(Start->getType());
8719 if (!VTy)
8720 return false;
8721 Type *ITy = VTy->getElementType();
8722 if (DL.getTypeStoreSize(ITy) != DL.getPointerSize())
8723 return false;
8724
8725 // TODO: Handle offsets from stepvector.
8727 return false;
8728
8729 Value *LoopStride = nullptr;
8730 if (!match(Step, m_OneUse(m_c_Add(m_Specific(Phi), m_Value(LoopStride)))))
8731 return false;
8732
8733 // Make sure the loop stride matches the pointer stride.
8734 auto EltCnt = cast<VectorType>(Phi->getType())->getElementCount();
8735 const APInt *ShiftAmt = nullptr;
8736 Value *PtrStride = nullptr;
8737 if (!match(
8738 LoopStride,
8739 m_Splat(m_Value(PtrStride, m_Shl(m_VScale(), m_APInt(ShiftAmt))))) ||
8740 (1ULL << ShiftAmt->getZExtValue()) != EltCnt.getKnownMinValue())
8741 return false;
8742
8743 // Make sure the stride isn't defined inside the loop, so we don't have to
8744 // worry about dominance.
8745 if (L->contains(cast<Instruction>(PtrStride)))
8746 return false;
8747
8748 // For the simplest case, we're only interested if the phi has two users;
8749 // The Step forming the incoming value for the backedge, and a GEP.
8750 GetElementPtrInst *GEP = nullptr;
8751 Value *Base = nullptr;
8752 for (User *U : Phi->users()) {
8753 // Skip over the step, handled above.
8754 if (Step == U)
8755 continue;
8756
8757 // If we've already found a GEP, bail out.
8758 // TODO: Support multiple GEPs.
8759 if (GEP)
8760 return false;
8761
8762 // We're only interested in single index GEPs
8764 if (!GEP || GEP->getNumOperands() != 2)
8765 return false;
8766
8767 // Reject if the base isn't loop invariant.
8768 Base = GEP->getOperand(0);
8769 if (!L->isLoopInvariant(Base))
8770 return false;
8771 }
8772
8773 // Make sure we found a GEP.
8774 if (!GEP)
8775 return false;
8776
8777 // Multiply the start by the size of the struct, and add the base pointer.
8778 unsigned Size = DL.getTypeAllocSize(GEP->getSourceElementType());
8779 IRBuilder<> Builder(PreHeader->getTerminator());
8780 Value *StructSize = ConstantInt::get(ITy, Size);
8781 Value *SizeSplat =
8782 Builder.CreateVectorSplat(VTy->getElementCount(), StructSize);
8783 Value *NewStart = Builder.CreateMul(Start, SizeSplat);
8784 NewStart = Builder.CreatePtrAdd(Base, NewStart);
8785
8786 // Create a new step based on the total size of all struct addresses per
8787 // iteration.
8788 Value *StructStride = Builder.CreateMul(PtrStride, StructSize);
8789 StructStride =
8790 Builder.CreateVectorSplat(VTy->getElementCount(), StructStride);
8791
8792 Builder.SetInsertPoint(Phi);
8793 PHINode *NewPhi = Builder.CreatePHI(GEP->getType(), 2, "vec.lsr.phi");
8794 Builder.SetInsertPoint(cast<Instruction>(Step));
8795 StructStride = Builder.CreatePtrAdd(NewPhi, StructStride);
8796
8797 // Set up the new PHI.
8798 NewPhi->addIncoming(NewStart, PreHeader);
8799 NewPhi->addIncoming(StructStride, Latch);
8800
8801 // Replace the GEP with the new phi.
8802 Builder.SetInsertPoint(GEP);
8803 GEP->replaceAllUsesWith(NewPhi);
8804
8805 // Remove old phi, gep, and step.
8806 Phi->replaceAllUsesWith(llvm::PoisonValue::get(Phi->getType()));
8808
8809 return true;
8810}
8811
8812bool CodeGenPrepare::optimizeInst(Instruction *I, ModifyDT &ModifiedDT) {
8813 bool AnyChange = false;
8814 AnyChange = fixupDbgVariableRecordsOnInst(*I);
8815
8816 // Bail out if we inserted the instruction to prevent optimizations from
8817 // stepping on each other's toes.
8818 if (InsertedInsts.count(I))
8819 return AnyChange;
8820
8821 // TODO: Move into the switch on opcode below here.
8822 if (PHINode *P = dyn_cast<PHINode>(I)) {
8823 // It is possible for very late stage optimizations (such as SimplifyCFG)
8824 // to introduce PHI nodes too late to be cleaned up. If we detect such a
8825 // trivial PHI, go ahead and zap it here.
8826 if (Value *V = simplifyInstruction(P, {*DL, TLInfo})) {
8827 LargeOffsetGEPMap.erase(P);
8828 replaceAllUsesWith(P, V, FreshBBs, IsHugeFunc);
8829 P->eraseFromParent();
8830 ++NumPHIsElim;
8831 return true;
8832 }
8833
8834 // Look for simple GEPs on scalable vectors used as data which may
8835 // introduce unnecessary multiplies in the loop.
8836 // TODO: Support fixed vectors too.
8837 if (P->getType()->isScalableTy())
8838 AnyChange |= strengthReduceVectorPhiUsers(P, LI);
8839 return AnyChange;
8840 }
8841
8842 if (CastInst *CI = dyn_cast<CastInst>(I)) {
8843 // If the source of the cast is a constant, then this should have
8844 // already been constant folded. The only reason NOT to constant fold
8845 // it is if something (e.g. LSR) was careful to place the constant
8846 // evaluation in a block other than then one that uses it (e.g. to hoist
8847 // the address of globals out of a loop). If this is the case, we don't
8848 // want to forward-subst the cast.
8849 if (auto *BCI = dyn_cast<BitCastInst>(CI)) {
8850 // Hoist bitcasts of illegal types to reduce cross-block register pressure
8851 // and prevent register splitting.
8852 if (optimizeBitCast(BCI, *TLI, *DL)) {
8853 return true;
8854 }
8855 }
8856
8857 if (isa<Constant>(CI->getOperand(0)))
8858 return AnyChange;
8859
8860 if (OptimizeNoopCopyExpression(CI, *TLI, *DL))
8861 return true;
8862
8864 isa<TruncInst>(I)) &&
8866 I, LI->getLoopFor(I->getParent()), *TTI))
8867 return true;
8868
8869 if (isa<ZExtInst>(I) || isa<SExtInst>(I)) {
8870 /// Sink a zext or sext into its user blocks if the target type doesn't
8871 /// fit in one register
8872 if (TLI->getTypeAction(CI->getContext(),
8873 TLI->getValueType(*DL, CI->getType())) ==
8874 TargetLowering::TypeExpandInteger) {
8875 return SinkCast(CI);
8876 } else {
8878 I, LI->getLoopFor(I->getParent()), *TTI))
8879 return true;
8880
8881 bool MadeChange = optimizeExt(I);
8882 return MadeChange | optimizeExtUses(I);
8883 }
8884 }
8885 return AnyChange;
8886 }
8887
8888 if (auto *Cmp = dyn_cast<CmpInst>(I))
8889 if (optimizeCmp(Cmp, ModifiedDT))
8890 return true;
8891
8892 if (match(I, m_URem(m_Value(), m_Value())))
8893 if (optimizeURem(I))
8894 return true;
8895
8896 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
8897 LI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
8898 bool Modified = optimizeLoadExt(LI);
8899 unsigned AS = LI->getPointerAddressSpace();
8900 Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS);
8901 return Modified;
8902 }
8903
8904 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
8905 if (splitMergedValStore(*SI, *DL, *TLI, Opts.cgp_force_split_store))
8906 return true;
8907 bool Modified = tryToSinkFreeOperands(I);
8908 SI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
8909 unsigned AS = SI->getPointerAddressSpace();
8910 Modified |= optimizeMemoryInst(I, SI->getOperand(1),
8911 SI->getOperand(0)->getType(), AS);
8912 return Modified;
8913 }
8914
8915 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
8916 unsigned AS = RMW->getPointerAddressSpace();
8917 return optimizeMemoryInst(I, RMW->getPointerOperand(), RMW->getType(), AS);
8918 }
8919
8920 if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(I)) {
8921 unsigned AS = CmpX->getPointerAddressSpace();
8922 return optimizeMemoryInst(I, CmpX->getPointerOperand(),
8923 CmpX->getCompareOperand()->getType(), AS);
8924 }
8925
8926 BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I);
8927
8928 if (BinOp && BinOp->getOpcode() == Instruction::And &&
8929 Opts.cgp_andcmp_sinking &&
8930 sinkAndCmp0Expression(BinOp, *TLI, InsertedInsts))
8931 return true;
8932
8933 // TODO: Move this into the switch on opcode - it handles shifts already.
8934 if (BinOp && (BinOp->getOpcode() == Instruction::AShr ||
8935 BinOp->getOpcode() == Instruction::LShr)) {
8936 ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1));
8937 if (CI && TLI->hasExtractBitsInsn())
8938 if (OptimizeExtractBits(BinOp, CI, *TLI, *DL))
8939 return true;
8940 }
8941
8942 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
8943 if (GEPI->hasAllZeroIndices()) {
8944 /// The GEP operand must be a pointer, so must its result -> BitCast
8945 Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(),
8946 GEPI->getName(), GEPI->getIterator());
8947 NC->setDebugLoc(GEPI->getDebugLoc());
8948 replaceAllUsesWith(GEPI, NC, FreshBBs, IsHugeFunc);
8950 GEPI, TLInfo, nullptr,
8951 [&](Value *V) { removeAllAssertingVHReferences(V); });
8952 ++NumGEPsElim;
8953 optimizeInst(NC, ModifiedDT);
8954 return true;
8955 }
8957 return true;
8958 }
8959 }
8960
8961 if (FreezeInst *FI = dyn_cast<FreezeInst>(I)) {
8962 // freeze(icmp a, const)) -> icmp (freeze a), const
8963 // This helps generate efficient conditional jumps.
8964 CmpInst *CmpI = dyn_cast<CmpInst>(FI->getOperand(0));
8965 if (CmpI && CmpI->hasOneUse()) {
8966 auto Op0 = CmpI->getOperand(0), Op1 = CmpI->getOperand(1);
8967 bool Const0 = isa<ConstantInt>(Op0) || isa<ConstantFP>(Op0) ||
8969 bool Const1 = isa<ConstantInt>(Op1) || isa<ConstantFP>(Op1) ||
8971 if (Const0 || Const1) {
8972 if (!Const0 || !Const1) {
8973 auto *F = new FreezeInst(Const0 ? Op1 : Op0, "", CmpI->getIterator());
8974 F->takeName(FI);
8975 CmpI->setOperand(Const0 ? 1 : 0, F);
8976 }
8978 replaceAllUsesWith(FI, CmpI, FreshBBs, IsHugeFunc);
8979 FI->eraseFromParent();
8980 return true;
8981 }
8982 }
8983 return AnyChange;
8984 }
8985
8986 // Try to fold some cross-lane vector operations if in a loop.
8987 if (I->getType()->isVectorTy() &&
8988 TLI->optimizeVectorCrossLaneOperation(I, LI->getLoopFor(I->getParent())))
8989 return true;
8990
8991 if (tryToSinkFreeOperands(I))
8992 return true;
8993
8994 switch (I->getOpcode()) {
8995 case Instruction::Shl:
8996 case Instruction::LShr:
8997 case Instruction::AShr:
8998 return optimizeShiftInst(cast<BinaryOperator>(I));
8999 case Instruction::Call:
9000 return optimizeCallInst(cast<CallInst>(I), ModifiedDT);
9001 case Instruction::Select:
9002 return optimizeSelectInst(cast<SelectInst>(I));
9003 case Instruction::ShuffleVector:
9004 return optimizeShuffleVectorInst(cast<ShuffleVectorInst>(I));
9005 case Instruction::Switch:
9006 return optimizeSwitchInst(cast<SwitchInst>(I));
9007 case Instruction::ExtractElement:
9008 return optimizeExtractElementInst(cast<ExtractElementInst>(I));
9009 case Instruction::CondBr:
9010 return optimizeBranch(cast<CondBrInst>(I), *TLI, FreshBBs, IsHugeFunc);
9011 }
9012
9013 return AnyChange;
9014}
9015
9016/// Given an OR instruction, check to see if this is a bitreverse
9017/// idiom. If so, insert the new intrinsic and return true.
9018bool CodeGenPrepare::makeBitReverse(Instruction &I) {
9019 if (!I.getType()->isIntegerTy() ||
9021 TLI->getValueType(*DL, I.getType(), true)))
9022 return false;
9023
9024 SmallVector<Instruction *, 4> Insts;
9025 if (!recognizeBSwapOrBitReverseIdiom(&I, false, true, Insts))
9026 return false;
9027 Instruction *LastInst = Insts.back();
9028 replaceAllUsesWith(&I, LastInst, FreshBBs, IsHugeFunc);
9030 &I, TLInfo, nullptr,
9031 [&](Value *V) { removeAllAssertingVHReferences(V); });
9032 return true;
9033}
9034
9035// In this pass we look for GEP and cast instructions that are used
9036// across basic blocks and rewrite them to improve basic-block-at-a-time
9037// selection.
9038bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT) {
9039 SunkAddrs.clear();
9040 bool MadeChange = false;
9041
9042 do {
9043 CurInstIterator = BB.begin();
9044 ModifiedDT = ModifyDT::NotModifyDT;
9045 while (CurInstIterator != BB.end()) {
9046 MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT);
9047 if (ModifiedDT != ModifyDT::NotModifyDT) {
9048 // For huge function we tend to quickly go though the inner optmization
9049 // opportunities in the BB. So we go back to the BB head to re-optimize
9050 // each instruction instead of go back to the function head.
9051 if (IsHugeFunc)
9052 break;
9053 return true;
9054 }
9055 }
9056 } while (ModifiedDT == ModifyDT::ModifyInstDT);
9057
9058 bool MadeBitReverse = true;
9059 while (MadeBitReverse) {
9060 MadeBitReverse = false;
9061 for (auto &I : reverse(BB)) {
9062 if (makeBitReverse(I)) {
9063 MadeBitReverse = MadeChange = true;
9064 break;
9065 }
9066 }
9067 }
9068 MadeChange |= dupRetToEnableTailCallOpts(&BB, ModifiedDT);
9069
9070 return MadeChange;
9071}
9072
9073bool CodeGenPrepare::fixupDbgVariableRecordsOnInst(Instruction &I) {
9074 bool AnyChange = false;
9075 for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
9076 AnyChange |= fixupDbgVariableRecord(DVR);
9077 return AnyChange;
9078}
9079
9080// FIXME: should updating debug-info really cause the "changed" flag to fire,
9081// which can cause a function to be reprocessed?
9082bool CodeGenPrepare::fixupDbgVariableRecord(DbgVariableRecord &DVR) {
9083 if (DVR.Type != DbgVariableRecord::LocationType::Value &&
9084 DVR.Type != DbgVariableRecord::LocationType::Assign)
9085 return false;
9086
9087 // Does this DbgVariableRecord refer to a sunk address calculation?
9088 bool AnyChange = false;
9089 SmallDenseSet<Value *> LocationOps(DVR.location_ops().begin(),
9090 DVR.location_ops().end());
9091 for (Value *Location : LocationOps) {
9092 WeakTrackingVH SunkAddrVH = SunkAddrs[Location];
9093 Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;
9094 if (SunkAddr) {
9095 // Point dbg.value at locally computed address, which should give the best
9096 // opportunity to be accurately lowered. This update may change the type
9097 // of pointer being referred to; however this makes no difference to
9098 // debugging information, and we can't generate bitcasts that may affect
9099 // codegen.
9100 DVR.replaceVariableLocationOp(Location, SunkAddr);
9101 AnyChange = true;
9102 }
9103 }
9104 return AnyChange;
9105}
9106
9108 DVR->removeFromParent();
9109 BasicBlock *VIBB = VI->getParent();
9110 if (isa<PHINode>(VI))
9111 VIBB->insertDbgRecordBefore(DVR, VIBB->getFirstInsertionPt());
9112 else
9113 VIBB->insertDbgRecordAfter(DVR, &*VI);
9114}
9115
9116// A llvm.dbg.value may be using a value before its definition, due to
9117// optimizations in this pass and others. Scan for such dbg.values, and rescue
9118// them by moving the dbg.value to immediately after the value definition.
9119// FIXME: Ideally this should never be necessary, and this has the potential
9120// to re-order dbg.value intrinsics.
9121bool CodeGenPrepare::placeDbgValues(Function &F) {
9122 bool MadeChange = false;
9123 DominatorTree &DT = getDT();
9124
9125 auto DbgProcessor = [&](auto *DbgItem, Instruction *Position) {
9126 SmallVector<Instruction *, 4> VIs;
9127 for (Value *V : DbgItem->location_ops())
9128 if (Instruction *VI = dyn_cast_or_null<Instruction>(V))
9129 VIs.push_back(VI);
9130
9131 // This item may depend on multiple instructions, complicating any
9132 // potential sink. This block takes the defensive approach, opting to
9133 // "undef" the item if it has more than one instruction and any of them do
9134 // not dominate iem.
9135 for (Instruction *VI : VIs) {
9136 if (VI->isTerminator())
9137 continue;
9138
9139 // If VI is a phi in a block with an EHPad terminator, we can't insert
9140 // after it.
9141 if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad())
9142 continue;
9143
9144 // If the defining instruction dominates the dbg.value, we do not need
9145 // to move the dbg.value.
9146 if (DT.dominates(VI, Position))
9147 continue;
9148
9149 // If we depend on multiple instructions and any of them doesn't
9150 // dominate this DVI, we probably can't salvage it: moving it to
9151 // after any of the instructions could cause us to lose the others.
9152 if (VIs.size() > 1) {
9153 LLVM_DEBUG(
9154 dbgs()
9155 << "Unable to find valid location for Debug Value, undefing:\n"
9156 << *DbgItem);
9157 DbgItem->setKillLocation();
9158 break;
9159 }
9160
9161 LLVM_DEBUG(dbgs() << "Moving Debug Value before :\n"
9162 << *DbgItem << ' ' << *VI);
9163 DbgInserterHelper(DbgItem, VI->getIterator());
9164 MadeChange = true;
9165 ++NumDbgValueMoved;
9166 }
9167 };
9168
9169 for (BasicBlock &BB : F) {
9170 for (Instruction &Insn : llvm::make_early_inc_range(BB)) {
9171 // Process any DbgVariableRecord records attached to this
9172 // instruction.
9173 for (DbgVariableRecord &DVR : llvm::make_early_inc_range(
9174 filterDbgVars(Insn.getDbgRecordRange()))) {
9175 if (DVR.Type != DbgVariableRecord::LocationType::Value)
9176 continue;
9177 DbgProcessor(&DVR, &Insn);
9178 }
9179 }
9180 }
9181
9182 return MadeChange;
9183}
9184
9185// Group scattered pseudo probes in a block to favor SelectionDAG. Scattered
9186// probes can be chained dependencies of other regular DAG nodes and block DAG
9187// combine optimizations.
9188bool CodeGenPrepare::placePseudoProbes(Function &F) {
9189 bool MadeChange = false;
9190 for (auto &Block : F) {
9191 // Move the rest probes to the beginning of the block.
9192 auto FirstInst = Block.getFirstInsertionPt();
9193 while (FirstInst != Block.end() && isa<PseudoProbeInst>(FirstInst))
9194 ++FirstInst;
9195 BasicBlock::iterator I(FirstInst);
9196 I++;
9197 while (I != Block.end()) {
9198 if (auto *II = dyn_cast<PseudoProbeInst>(I++)) {
9199 II->moveBefore(FirstInst);
9200 MadeChange = true;
9201 }
9202 }
9203 }
9204 return MadeChange;
9205}
9206
9207/// Some targets prefer to split a conditional branch like:
9208/// \code
9209/// %0 = icmp ne i32 %a, 0
9210/// %1 = icmp ne i32 %b, 0
9211/// %or.cond = or i1 %0, %1
9212/// br i1 %or.cond, label %TrueBB, label %FalseBB
9213/// \endcode
9214/// into multiple branch instructions like:
9215/// \code
9216/// bb1:
9217/// %0 = icmp ne i32 %a, 0
9218/// br i1 %0, label %TrueBB, label %bb2
9219/// bb2:
9220/// %1 = icmp ne i32 %b, 0
9221/// br i1 %1, label %TrueBB, label %FalseBB
9222/// \endcode
9223/// This usually allows instruction selection to do even further optimizations
9224/// and combine the compare with the branch instruction. Currently this is
9225/// applied for targets which have "cheap" jump instructions.
9226///
9227/// FIXME: Remove the (equivalent?) implementation in SelectionDAG.
9228///
9229bool CodeGenPrepare::splitBranchCondition(Function &F) {
9231 return false;
9232
9233 bool MadeChange = false;
9234 for (auto &BB : F) {
9235 // Does this BB end with the following?
9236 // %cond1 = icmp|fcmp|binary instruction ...
9237 // %cond2 = icmp|fcmp|binary instruction ...
9238 // %cond.or = or|and i1 %cond1, cond2
9239 // br i1 %cond.or label %dest1, label %dest2"
9240 Instruction *LogicOp;
9241 BasicBlock *TBB, *FBB;
9242 if (!match(BB.getTerminator(),
9243 m_Br(m_OneUse(m_Instruction(LogicOp)), TBB, FBB)))
9244 continue;
9245
9246 auto *Br1 = cast<CondBrInst>(BB.getTerminator());
9247 if (Br1->getMetadata(LLVMContext::MD_unpredictable))
9248 continue;
9249
9250 // The merging of mostly empty BB can cause a degenerate branch.
9251 if (TBB == FBB)
9252 continue;
9253
9254 unsigned Opc;
9255 Value *Cond1, *Cond2;
9256 if (match(LogicOp,
9257 m_LogicalAnd(m_OneUse(m_Value(Cond1)), m_OneUse(m_Value(Cond2)))))
9258 Opc = Instruction::And;
9259 else if (match(LogicOp, m_LogicalOr(m_OneUse(m_Value(Cond1)),
9260 m_OneUse(m_Value(Cond2)))))
9261 Opc = Instruction::Or;
9262 else
9263 continue;
9264
9265 auto IsGoodCond = [](Value *Cond) {
9266 return match(
9267 Cond,
9269 m_LogicalOr(m_Value(), m_Value()))));
9270 };
9271 if (!IsGoodCond(Cond1) || !IsGoodCond(Cond2))
9272 continue;
9273
9274 LLVM_DEBUG(dbgs() << "Before branch condition splitting\n"; BB.dump());
9275
9276 // Create a new BB.
9277 auto *TmpBB =
9278 BasicBlock::Create(BB.getContext(), BB.getName() + ".cond.split",
9279 BB.getParent(), BB.getNextNode());
9280 if (IsHugeFunc)
9281 FreshBBs.insert(TmpBB);
9282
9283 // Update original basic block by using the first condition directly by the
9284 // branch instruction and removing the no longer needed and/or instruction.
9285 Br1->setCondition(Cond1);
9286 LogicOp->eraseFromParent();
9287
9288 // Depending on the condition we have to either replace the true or the
9289 // false successor of the original branch instruction.
9290 if (Opc == Instruction::And)
9291 Br1->setSuccessor(0, TmpBB);
9292 else
9293 Br1->setSuccessor(1, TmpBB);
9294
9295 // Fill in the new basic block.
9296 auto *Br2 = IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB);
9297 if (auto *I = dyn_cast<Instruction>(Cond2)) {
9298 I->removeFromParent();
9299 I->insertBefore(Br2->getIterator());
9300 }
9301
9302 // Update PHI nodes in both successors. The original BB needs to be
9303 // replaced in one successor's PHI nodes, because the branch comes now from
9304 // the newly generated BB (NewBB). In the other successor we need to add one
9305 // incoming edge to the PHI nodes, because both branch instructions target
9306 // now the same successor. Depending on the original branch condition
9307 // (and/or) we have to swap the successors (TrueDest, FalseDest), so that
9308 // we perform the correct update for the PHI nodes.
9309 // This doesn't change the successor order of the just created branch
9310 // instruction (or any other instruction).
9311 if (Opc == Instruction::Or)
9312 std::swap(TBB, FBB);
9313
9314 // Replace the old BB with the new BB.
9315 TBB->replacePhiUsesWith(&BB, TmpBB);
9316
9317 // Add another incoming edge from the new BB.
9318 for (PHINode &PN : FBB->phis()) {
9319 auto *Val = PN.getIncomingValueForBlock(&BB);
9320 PN.addIncoming(Val, TmpBB);
9321 }
9322
9323 if (Loop *L = LI->getLoopFor(&BB))
9324 L->addBasicBlockToLoop(TmpBB, *LI);
9325
9326 // The edge we need to delete starts at BB and ends at whatever TBB ends
9327 // up pointing to.
9328 DTU->applyUpdates({{DominatorTree::Insert, &BB, TmpBB},
9329 {DominatorTree::Insert, TmpBB, TBB},
9330 {DominatorTree::Insert, TmpBB, FBB},
9331 {DominatorTree::Delete, &BB, TBB}});
9332
9333 // Update the branch weights (from SelectionDAGBuilder::
9334 // FindMergedConditions).
9335 if (Opc == Instruction::Or) {
9336 // Codegen X | Y as:
9337 // BB1:
9338 // jmp_if_X TBB
9339 // jmp TmpBB
9340 // TmpBB:
9341 // jmp_if_Y TBB
9342 // jmp FBB
9343 //
9344
9345 // We have flexibility in setting Prob for BB1 and Prob for NewBB.
9346 // The requirement is that
9347 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
9348 // = TrueProb for original BB.
9349 // Assuming the original weights are A and B, one choice is to set BB1's
9350 // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
9351 // assumes that
9352 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
9353 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
9354 // TmpBB, but the math is more complicated.
9355 uint64_t TrueWeight, FalseWeight;
9356 if (extractBranchWeights(*Br1, TrueWeight, FalseWeight)) {
9357 uint64_t NewTrueWeight = TrueWeight;
9358 uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight;
9359 setFittedBranchWeights(*Br1, {NewTrueWeight, NewFalseWeight},
9360 hasBranchWeightOrigin(*Br1));
9361
9362 NewTrueWeight = TrueWeight;
9363 NewFalseWeight = 2 * FalseWeight;
9364 setFittedBranchWeights(*Br2, {NewTrueWeight, NewFalseWeight},
9365 /*IsExpected=*/false);
9366 }
9367 } else {
9368 // Codegen X & Y as:
9369 // BB1:
9370 // jmp_if_X TmpBB
9371 // jmp FBB
9372 // TmpBB:
9373 // jmp_if_Y TBB
9374 // jmp FBB
9375 //
9376 // This requires creation of TmpBB after CurBB.
9377
9378 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
9379 // The requirement is that
9380 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
9381 // = FalseProb for original BB.
9382 // Assuming the original weights are A and B, one choice is to set BB1's
9383 // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
9384 // assumes that
9385 // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
9386 uint64_t TrueWeight, FalseWeight;
9387 if (extractBranchWeights(*Br1, TrueWeight, FalseWeight)) {
9388 uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight;
9389 uint64_t NewFalseWeight = FalseWeight;
9390 setFittedBranchWeights(*Br1, {NewTrueWeight, NewFalseWeight},
9391 /*IsExpected=*/false);
9392
9393 NewTrueWeight = 2 * TrueWeight;
9394 NewFalseWeight = FalseWeight;
9395 setFittedBranchWeights(*Br2, {NewTrueWeight, NewFalseWeight},
9396 /*IsExpected=*/false);
9397 }
9398 }
9399
9400 MadeChange = true;
9401
9402 LLVM_DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump();
9403 TmpBB->dump());
9404 }
9405 return MadeChange;
9406}
#define Success
return SDValue()
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool sinkAndCmp0Expression(Instruction *AndI, const TargetLowering &TLI, SetOfInstrs &InsertedInsts)
Duplicate and sink the given 'and' instruction into user blocks where it is used in a compare to allo...
static bool SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI, DenseMap< BasicBlock *, BinaryOperator * > &InsertedShifts, const TargetLowering &TLI, const DataLayout &DL)
Sink both shift and truncate instruction to the use of truncate's BB.
static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP, SmallVectorImpl< Value * > &OffsetV)
static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V)
Check if V (an operand of a select instruction) is an expensive instruction that is only used once.
static bool isExtractBitsCandidateUse(Instruction *User)
Check if the candidates could be combined with a shift instruction, which includes:
static bool optimizeBitCast(BitCastInst *BCI, const TargetLowering &TLI, const DataLayout &DL)
Hoists bitcasts to the source block to reduce register pressure.
static bool foldFCmpToFPClassTest(CmpInst *Cmp, const TargetLowering &TLI, const DataLayout &DL)
static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI, const TargetLowering &TLI, const DataLayout &DL)
Sink the shift right instruction into user blocks if the uses could potentially be combined with this...
static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI, const DataLayout &DL)
If the specified cast instruction is a noop copy (e.g.
static bool SinkCast(CastInst *CI)
Sink the specified cast instruction into its user blocks.
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI, bool ForceSplitStore)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
static bool swapICmpOperandsToExposeCSEOpportunities(CmpInst *Cmp)
Many architectures use the same instruction for both subtract and cmp.
static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI, const TargetLowering *TLI, SelectInst *SI)
Returns true if a SelectInst should be turned into an explicit branch.
static bool FindAllMemoryUses(Instruction *I, SmallVectorImpl< std::pair< Use *, Type * > > &MemoryUses, SmallPtrSetImpl< Instruction * > &ConsideredInsts, const TargetLowering &TLI, const TargetRegisterInfo &TRI, bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, unsigned &SeenInsts, unsigned MaxUsersToScan)
Recursively walk all the uses of I until we find a memory use.
static std::optional< std::pair< Instruction *, Constant * > > getIVIncrement(const PHINode *PN, const LoopInfo *LI)
If given PN is an inductive variable with value IVInc coming from the backedge, and on each iteration...
static Value * getTrueOrFalseValue(SelectInst *SI, bool isTrue, const SmallPtrSet< const Instruction *, 2 > &Selects)
If isTrue is true, return the true value of SI, otherwise return false value of SI.
static bool foldICmpWithDominatingICmp(CmpInst *Cmp, const TargetLowering &TLI, bool EnableICmpEqToICmpSt)
For pattern like:
static bool strengthReduceVectorPhiUsers(PHINode *Phi, LoopInfo *LI)
static bool optimizeBranch(CondBrInst *Branch, const TargetLowering &TLI, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHugeFunc)
static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI, const TargetTransformInfo *TTI)
static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal, const TargetLowering &TLI, const TargetRegisterInfo &TRI)
Check to see if all uses of OpVal by the specified inline asm call are due to memory operands.
static bool isIntrinsicOrLFToBeTailCalled(const TargetLibraryInfo *TLInfo, const CallInst *CI)
static void replaceAllUsesWith(Value *Old, Value *New, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHuge)
Replace all old uses with new ones, and push the updated BBs into FreshBBs.
static bool matchOverflowPattern(Instruction *&I, ExtractValueInst *&MulExtract, ExtractValueInst *&OverflowExtract)
static void computeBaseDerivedRelocateMap(const SmallVectorImpl< GCRelocateInst * > &AllRelocateCalls, MapVector< GCRelocateInst *, SmallVector< GCRelocateInst *, 0 > > &RelocateInstMap)
static bool simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase, const SmallVectorImpl< GCRelocateInst * > &Targets)
static bool MightBeFoldableInst(Instruction *I)
This is a little filter, which returns true if an addressing computation involving I might be folded ...
static bool matchIncrement(const Instruction *IVInc, Instruction *&LHS, Constant *&Step)
static bool isIVIncrement(const Value *V, const LoopInfo *LI)
static void DbgInserterHelper(DbgVariableRecord *DVR, BasicBlock::iterator VI)
static bool isPromotedInstructionLegal(const TargetLowering &TLI, const DataLayout &DL, Value *Val)
Check whether or not Val is a legal instruction for TLI.
static BasicBlock::iterator findInsertPos(Value *Addr, Instruction *MemoryInst, Value *SunkAddr)
static bool isRemOfLoopIncrementWithLoopInvariant(Instruction *Rem, const LoopInfo *LI, Value *&RemAmtOut, Value *&AddInstOut, Value *&AddOffsetOut, PHINode *&LoopIncrPNOut, Constant *&StepOut)
static bool IsNonLocalValue(Value *V, BasicBlock *BB)
Return true if the specified values are defined in a different basic block than BB.
static bool despeculateCountZeros(IntrinsicInst *CountZeros, DomTreeUpdater *DTU, LoopInfo *LI, const TargetLowering *TLI, const DataLayout *DL, ModifyDT &ModifiedDT, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHugeFunc)
If counting leading or trailing zeros is an expensive operation and a zero input is defined,...
static bool sinkCmpExpression(CmpInst *Cmp, const TargetLowering &TLI, const DataLayout &DL)
Sink the given CmpInst into user blocks to reduce the number of virtual registers that must be create...
static bool hasSameExtUse(Value *Val, const TargetLowering &TLI)
Check if all the uses of Val are equivalent (or free) zero or sign extensions.
static bool foldURemOfLoopIncrement(Instruction *Rem, const DataLayout *DL, const LoopInfo *LI, DominatorTree &DT, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHuge)
static bool matchUAddWithOverflowConstantEdgeCases(CmpInst *Cmp, BinaryOperator *&Add)
Match special-case patterns that check for unsigned add overflow.
Defines an IR pass for CodeGen Prepare.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:702
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file declares the LLVM IR specialization of the GenericCycle templates.
This file defines the DenseMap class.
static bool runOnFunction(Function &F, bool PostInlining)
#define DEBUG_TYPE
static Value * getCondition(Instruction *I)
Hexagon Common GEP
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
iv users
Definition IVUsers.cpp:48
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static void eraseInstruction(Instruction &I, const LICMDependencies &Deps)
Definition LICM.cpp:1188
static LVOptions Options
Definition LVOptions.cpp:25
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
#define P(N)
ppc ctr loops verify
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file defines the PointerIntPair class.
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
static DominatorTree getDomTree(Function &F)
static bool dominates(InstrPosIndexes &PosIndexes, const MachineInstr &A, const MachineInstr &B)
Remove Loads Into Fake Uses
This file contains some templates that are useful if you are working with the STL at all.
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc=0)
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
This pass exposes codegen information to IR-level passes.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
static Constant * getConstantVector(MVT VT, ArrayRef< APInt > Bits, const APInt &Undefs, LLVMContext &C)
Value * RHS
Value * LHS
BinaryOperator * Mul
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1186
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:325
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
Definition APInt.h:431
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition APInt.h:1551
unsigned logBase2() const
Definition APInt.h:1781
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:1030
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
LLVM_ABI bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
void setAlignment(Align Align)
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
An instruction that atomically checks whether a specified value is in a memory location,...
static unsigned getPointerOperandIndex()
an instruction that atomically reads a memory location, combines it with another value,...
static unsigned getPointerOperandIndex()
Analysis pass providing the BasicBlockSectionsProfileReader.
LLVM_ABI bool isFunctionHot(StringRef FuncName) const
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator end()
Definition BasicBlock.h:453
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:509
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition BasicBlock.h:666
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI void insertDbgRecordBefore(DbgRecord *DR, InstListType::iterator Here)
Insert a DbgRecord into a block at the position given by Here.
InstListType::const_iterator const_iterator
Definition BasicBlock.h:171
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition BasicBlock.h:206
LLVM_ABI void moveAfter(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it right after MovePos in the function M...
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI InstListType::const_iterator getFirstNonPHIOrPseudoProbe() const
Returns an iterator to the first instruction in this block that is not a PHINode or a pseudo operatio...
LLVM_ABI void insertDbgRecordAfter(DbgRecord *DR, Instruction *I)
Insert a DbgRecord into a block at the position given by I.
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
BinaryOps getOpcode() const
Definition InstrTypes.h:409
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
This class represents a no-op cast from one type to another.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
LLVM_ABI void setBlockFreq(const BasicBlock *BB, BlockFrequency Freq)
LLVM_ABI BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
Analysis pass which computes BranchProbabilityInfo.
static LLVM_ABI BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Definition InstrTypes.h:512
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
static LLVM_ABI CmpInst * Create(OtherOps Op, Predicate Pred, Value *S1, Value *S2, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
Predicate getPredicate() const
Return the predicate for this instruction.
Definition InstrTypes.h:828
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Conditional Branch instruction.
static LLVM_ABI Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:174
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
LLVM_ABI void removeFromParent()
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LocationType Type
Classification of the debug-info record that this DbgVariableRecord represents.
LLVM_ABI void replaceVariableLocationOp(Value *OldValue, Value *NewValue, bool AllowEmpty=false)
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
unsigned size() const
Definition DenseMap.h:717
bool erase(const KeyT &Val)
Definition DenseMap.h:930
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
LLVM_ABI void deleteBB(BasicBlock *DelBB)
Delete DelBB.
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:277
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
This instruction extracts a struct member or array element value from an aggregate value.
iterator_range< idx_iterator > indices() const
This instruction compares its operands according to the predicate given to the constructor.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
const BasicBlock & getEntryBlock() const
Definition Function.h:803
LLVM_ABI const Value * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Represents calls to the gc.relocate intrinsic.
unsigned getBasePtrIndex() const
The index into the associate statepoint's argument list which contains the base pointer of the pointe...
void compute(FunctionT &F)
Compute the cycle info for a function.
DomTreeT & getDomTree()
Flush DomTree updates and return DomTree.
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
void flush()
Apply all pending updates to available trees and flush all BasicBlocks awaiting deletion.
bool isBBPendingDeletion(BasicBlockT *DelBB) const
Returns true if DelBB is awaiting deletion.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static LLVM_ABI Type * getIndexedType(Type *Ty, ArrayRef< Value * > IdxList)
Returns the result type of a getelementptr with the given source element type and indexes.
LLVM_ABI bool canIncreaseAlignment() const
Returns true if the alignment of the value can be unilaterally increased.
Definition Globals.cpp:422
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
LLVM_ABI uint64_t getGlobalSize(const DataLayout &DL) const
Get the size of this global variable in bytes.
Definition Globals.cpp:640
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalVariable.
This instruction compares its operands according to the predicate given to the constructor.
bool isEquality() const
Return true if this predicate is either EQ or NE.
Value * CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:1437
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:1501
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
LLVM_ABI void setHasNoSignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
user_iterator_impl< Instruction > user_iterator
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI void moveAfter(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI void insertBefore(InstListType::iterator InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified position.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
LLVM_ABI bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
LLVM_ABI bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
user_iterator user_begin()
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
user_iterator user_end()
bool isShift() const
LLVM_ABI void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
LLVM_ABI std::optional< simple_ilist< DbgRecord >::iterator > getDbgReinsertionPosition()
Return an iterator to the position of the "Next" DbgRecord after this instruction,...
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:598
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
The legacy pass manager's analysis pass to compute loop information.
Definition LoopInfo.h:623
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
static MVT getIntegerVT(unsigned BitWidth)
This class implements a map that also provides access to all stored values in a deterministic order.
Definition MapVector.h:39
iterator find(const KeyT &Key)
Definition MapVector.h:157
iterator end()
Definition MapVector.h:70
bool empty() const
Definition MapVector.h:80
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition MapVector.h:127
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
Definition MapVector.h:211
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
PointerIntPair - This class implements a pair of a pointer and small integer.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:133
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
Definition SetVector.h:268
void clear()
Completely clear the SetVector.
Definition SetVector.h:273
bool empty() const
Determine if the SetVector is empty or not.
Definition SetVector.h:100
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
value_type pop_back_val()
Definition SetVector.h:285
VectorType * getType() const
Overload to return most specific vector type.
size_type size() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void insert_range(Range &&R)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition SmallSet.h:176
bool erase(const T &V)
Definition SmallSet.h:200
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void resize(size_type N)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
static unsigned getPointerOperandIndex()
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:777
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LibFunc getLibFunc(StringRef funcName) const
Searches for a particular function name.
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool isSelectSupported(SelectSupportKind) const
virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const
Try to convert math with an overflow comparison into the corresponding DAG node operation.
virtual bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context, EVT VT) const
bool isExtLoad(const LoadInst *Load, const Instruction *Ext, const DataLayout &DL) const
Return true if Load and Ext can form an ExtLoad.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
const TargetMachine & getTargetMachine() const
virtual bool isCtpopFast(EVT VT) const
Return true if ctpop instruction is fast.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool enableExtLdPromotion() const
Return true if the target wants to use the optimization that turns ext(promotableInst1(....
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
bool hasExtractBitsInsn() const
Return true if the target has BitExtract instructions.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isSlowDivBypassed() const
Returns true if target has indicated at least one type should be bypassed.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const
Returns preferred type for switch condition.
virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const
Return true if the target can combine store(extractelement VectorTy,Idx).
virtual bool shouldConsiderGEPOffsetSplit() const
bool isExtFree(const Instruction *I) const
Return true if the extension represented by I is free.
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool getAddrModeArguments(const IntrinsicInst *, SmallVectorImpl< Value * > &, Type *&) const
CodeGenPrepare sinks address calculations into the same BB as Load/Store instructions reading the add...
const DenseMap< unsigned int, unsigned int > & getBypassSlowDivWidths() const
Returns map of slow types for division or remainder with corresponding fast types.
virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) const
Return the prefered common base offset.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool optimizeVectorCrossLaneOperation(Instruction *I, Loop *L) const
virtual bool shouldAlignPointerArgs(CallInst *, unsigned &, Align &) const
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
virtual Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal on this target.
virtual bool addressingModeSupportsTLS(const GlobalValue &) const
Returns true if the targets addressing mode can target thread local storage (TLS).
virtual bool shouldConvertPhiType(Type *From, Type *To) const
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool optimizeExtendOrTruncateConversion(Instruction *I, Loop *L, const TargetTransformInfo &TTI) const
Try to optimize extending or truncating conversion instructions (like zext, trunc,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::vector< AsmOperandInfo > AsmOperandInfoVector
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual bool mayBeEmittedAsTailCall(const CallInst *) const
Return true if the target may be able emit the call instruction as a tail call.
virtual bool isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
TargetOptions Options
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
Target-Independent Code Generator Pass Configuration Options.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
virtual const TargetLowering * getTargetLowering() const
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
LLVM_ABI InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index=-1, const Value *Op0=nullptr, const Value *Op1=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
LLVM_ABI InstructionCost getIntImmCost(const APInt &Imm, Type *Ty, TargetCostKind CostKind) const
Return the expected cost of materializing for the given integer immediate of the specified type.
LLVM_ABI bool shouldConsiderAddressTypePromotion(const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const
LLVM_ABI InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, const TargetLibraryInfo *TLibInfo=nullptr) const
This is an approximation of reciprocal throughput of a math/logic op.
@ TCC_Basic
The cost of a typical 'add' instruction.
LLVM_ABI bool isVectorShiftByScalarCheap(Type *Ty) const
Return true if it's significantly cheaper to shift a vector by a uniform scalar than by an amount whi...
LLVM_ABI bool isProfitableToSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops, const LoopInfo *LI=nullptr) const
Return true if sinking I's operands to the same basic block as I is profitable, e....
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVM_ABI bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
Definition Type.cpp:61
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
Definition Type.h:265
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
BasicBlock * getSuccessor(unsigned i=0) const
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:267
const Use & getOperandUse(unsigned i) const
Definition User.h:220
void setOperand(unsigned i, Value *Val)
Definition User.h:212
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:25
Value * getOperand(unsigned i) const
Definition User.h:207
unsigned getNumOperands() const
Definition User.h:229
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
user_iterator user_begin()
Definition Value.h:404
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:389
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition Value.cpp:985
LLVM_ABI bool isUsedInBasicBlock(const BasicBlock *BB) const
Check if this value is used in the specified basic block.
Definition Value.cpp:234
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
bool use_empty() const
Definition Value.h:348
iterator_range< use_iterator > uses()
Definition Value.h:382
void mutateType(Type *Ty)
Mutate the type of this Value to be of the specified type.
Definition Value.h:809
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
LLVM_ABI void dump() const
Support for debugging, callable in GDB: V->dump()
bool pointsToAliveValue() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isNonZero() const
Definition TypeSize.h:156
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
TypeSize getSequentialElementStride(const DataLayout &DL) const
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ Entry
Definition COFF.h:862
unsigned getAddrMode(MCInstrInfo const &MCII, MCInst const &MCI)
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
AllOnesConstantMatch m_AllOnes()
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
match_bind< PHINode > m_Phi(PHINode *&PN)
Match a PHI node, capturing it if we match.
auto m_Cmp()
Matches any compare instruction and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap, true > m_c_NUWAdd(const LHS &L, const RHS &R)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_Ctpop(const Opnd0 &Op0)
auto m_Constant()
Match an arbitrary Constant and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
auto m_VScale()
Matches a call to llvm.vscale().
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
Splat_match< T > m_Splat(const T &SubPattern)
Match a vector splat.
PtrAdd_match< PointerOpTy, OffsetOpTy > m_PtrAdd(const DataLayout &DL, const PointerOpTy &PointerOp, const OffsetOpTy &OffsetOp)
Matches GEP with i8 source element type and canonical index type.
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap, true >, DisjointOr_match< LHS, RHS, true > > m_c_NUWAddLike(const LHS &L, const RHS &R)
Match either "add nuw" or "or disjoint", with LHS and RHS commutable.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap > m_NSWAdd(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
UAddWithOverflow_match< LHS_t, RHS_t, Sum_t > m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S)
Match an icmp instruction checking for unsigned overflow on addition.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
brc_match< Cond_t, match_bind< BasicBlock >, match_bind< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
auto m_Undef()
Match an arbitrary undef constant.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
int compare(DigitsT LDigits, int16_t LScale, DigitsT RDigits, int16_t RScale)
Compare two scaled numbers.
@ CE
Windows NT (Windows on ARM)
Definition MCAsmInfo.h:51
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
@ User
could "use" a pointer
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
SmallVector< Node, 4 > NodeList
Definition RDFGraph.h:550
iterator end() const
Definition BasicBlock.h:89
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
LLVM_ABI iterator begin() const
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition DWP.cpp:577
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1781
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
Definition Local.cpp:522
LLVM_ABI bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr, DomTreeUpdater *DTU=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
Definition Local.cpp:133
LLVM_ABI bool bypassSlowDivision(BasicBlock *BB, const DenseMap< unsigned int, unsigned int > &BypassWidth, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BranchProbabilityInfo *BPI=nullptr)
This optimization identifies DIV instructions in a BB that can be profitably bypassed and carried out...
LLVM_ABI void findDbgValues(Value *V, SmallVectorImpl< DbgVariableRecord * > &DbgVariableRecords)
Finds the dbg.values describing a value.
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
APInt operator*(APInt a, uint64_t RHS)
Definition APInt.h:2261
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
Definition Alignment.h:139
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition Utils.cpp:1837
auto successors(const MachineBasicBlock *BB)
@ Load
The value being inserted comes from a load (InsertElement only).
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
LLVM_ABI ReturnInst * FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, BasicBlock *Pred, DomTreeUpdater *DTU=nullptr)
This method duplicates the specified return instruction into a predecessor which ends in an unconditi...
bool operator!=(uint64_t V1, const APInt &V2)
Definition APInt.h:2139
constexpr from_range_t from_range
LLVM_ABI BasicBlock * splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt, DomTreeUpdater *DTU, LoopInfo *LI, MemorySSAUpdater *MSSAU, const Twine &BBName="")
Split the specified block at the specified instruction SplitPt.
LLVM_ABI Instruction * SplitBlockAndInsertIfElse(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ElseBlock=nullptr)
Similar to SplitBlockAndInsertIfThen, but the inserted block is on the false path of the branch.
LLVM_ABI bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr, DomTreeUpdater *DTU=nullptr)
LLVM_ABI bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, SmallPtrSetImpl< PHINode * > *KnownNonDeadPHIs=nullptr)
Examine each PHI in the given block and delete it if it is dead.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
constexpr std::enable_if_t< std::is_signed_v< T >, std::pair< T, bool > > AddOverflow(T X, T Y)
Add two signed integers, computing the two's complement truncated result, returning a pair {result,...
Definition MathExtras.h:703
LLVM_ABI void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified block, which must have no predecessors.
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
auto unique(Range &&R, Predicate P)
Definition STLExtras.h:2150
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I)
Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
Definition STLExtras.h:2189
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
LLVM_ABI Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition STLExtras.h:2216
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_ABI bool replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr, SmallSetVector< Instruction *, 8 > *UnsimplifiedUsers=nullptr)
Replace all uses of 'I' with 'SimpleV' and simplify the uses recursively.
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI bool recognizeBSwapOrBitReverseIdiom(Instruction *I, bool MatchBSwaps, bool MatchBitReversals, SmallVectorImpl< Instruction * > &InsertedInsts)
Try to match a bswap or bitreverse idiom.
Definition Local.cpp:3860
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
auto make_first_range(ContainerTy &&c)
Given a container of pairs, return a range over the first elements.
Definition STLExtras.h:1415
generic_gep_type_iterator<> gep_type_iterator
LLVM_ABI FunctionPass * createCodeGenPrepareLegacyPass()
createCodeGenPrepareLegacyPass - Transform the code to expose more pattern matching during instructio...
LLVM_ABI ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition Analysis.cpp:214
LLVM_ABI bool VerifyLoopInfo
Enable verification of loop info.
Definition LoopInfo.cpp:54
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
LLVM_ABI bool attributesPermitTailCall(const Function *F, const Instruction *I, const ReturnInst *Ret, const TargetLoweringBase &TLI, bool *AllowDifferingSizes=nullptr)
Test if given that the input instruction is in the tail call position, if there is an attribute misma...
Definition Analysis.cpp:757
TargetTransformInfo TTI
LLVM_ABI bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false, DominatorTree *DT=nullptr)
Attempts to merge a block into its predecessor, if possible.
@ Or
Bitwise or logical OR of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition STLExtras.h:2028
IntPtrTy
Definition InstrProf.h:82
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I, bool IgnoreUBImplyingAttrs=true)
Don't use information from its non-constant operands.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI bool VerifyDomInfo
Enables verification of dominator trees.
constexpr unsigned BitWidth
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition Local.h:240
constexpr std::enable_if_t< std::is_signed_v< T >, std::pair< T, bool > > MulOverflow(T X, T Y)
Multiply two signed integers, computing the two's complement truncated result, returning a pair {resu...
Definition MathExtras.h:777
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
bool pred_empty(const BasicBlock *BB)
Definition CFG.h:107
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)
Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.
std::pair< Value *, FPClassTest > fcmpToClassTest(FCmpInst::Predicate Pred, const Function &F, Value *LHS, Value *RHS, bool LookThroughSrc=true)
Returns a pair of values, which if passed to llvm.is.fpclass, returns the same result as an fcmp with...
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI Value * simplifyURemInst(Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a URem, fold the result or return null.
DenseMap< const Value *, Value * > ValueToValueMap
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define NC
Definition regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool isRound() const
Return true if the size is a power-of-two number of bytes.
Definition ValueTypes.h:286
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
This contains information for each constraint that we are lowering.