LLVM 20.0.0git
SIFixSGPRCopies.cpp
Go to the documentation of this file.
1//===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Copies from VGPR to SGPR registers are illegal and the register coalescer
11/// will sometimes generate these illegal copies in situations like this:
12///
13/// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14///
15/// BB0:
16/// %0 <sgpr> = SCALAR_INST
17/// %1 <vsrc> = COPY %0 <sgpr>
18/// ...
19/// BRANCH %cond BB1, BB2
20/// BB1:
21/// %2 <vgpr> = VECTOR_INST
22/// %3 <vsrc> = COPY %2 <vgpr>
23/// BB2:
24/// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25/// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26///
27///
28/// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29/// code will look like this:
30///
31/// BB0:
32/// %0 <sgpr> = SCALAR_INST
33/// ...
34/// BRANCH %cond BB1, BB2
35/// BB1:
36/// %2 <vgpr> = VECTOR_INST
37/// %3 <vsrc> = COPY %2 <vgpr>
38/// BB2:
39/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41///
42/// Now that the result of the PHI instruction is an SGPR, the register
43/// allocator is now forced to constrain the register class of %3 to
44/// <sgpr> so we end up with final code like this:
45///
46/// BB0:
47/// %0 <sgpr> = SCALAR_INST
48/// ...
49/// BRANCH %cond BB1, BB2
50/// BB1:
51/// %2 <vgpr> = VECTOR_INST
52/// %3 <sgpr> = COPY %2 <vgpr>
53/// BB2:
54/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56///
57/// Now this code contains an illegal copy from a VGPR to an SGPR.
58///
59/// In order to avoid this problem, this pass searches for PHI instructions
60/// which define a <vsrc> register and constrains its definition class to
61/// <vgpr> if the user of the PHI's definition register is a vector instruction.
62/// If the PHI's definition class is constrained to <vgpr> then the coalescer
63/// will be unable to perform the COPY removal from the above example which
64/// ultimately led to the creation of an illegal COPY.
65//===----------------------------------------------------------------------===//
66
67#include "SIFixSGPRCopies.h"
68#include "AMDGPU.h"
69#include "GCNSubtarget.h"
75
76using namespace llvm;
77
78#define DEBUG_TYPE "si-fix-sgpr-copies"
79
81 "amdgpu-enable-merge-m0",
82 cl::desc("Merge and hoist M0 initializations"),
83 cl::init(true));
84
85namespace {
86
87class V2SCopyInfo {
88public:
89 // VGPR to SGPR copy being processed
90 MachineInstr *Copy;
91 // All SALU instructions reachable from this copy in SSA graph
93 // Number of SGPR to VGPR copies that are used to put the SALU computation
94 // results back to VALU.
95 unsigned NumSVCopies;
96
97 unsigned Score;
98 // Actual count of v_readfirstlane_b32
99 // which need to be inserted to keep SChain SALU
100 unsigned NumReadfirstlanes;
101 // Current score state. To speedup selection V2SCopyInfos for processing
102 bool NeedToBeConvertedToVALU = false;
103 // Unique ID. Used as a key for mapping to keep permanent order.
104 unsigned ID;
105
106 // Count of another VGPR to SGPR copies that contribute to the
107 // current copy SChain
108 unsigned SiblingPenalty = 0;
109 SetVector<unsigned> Siblings;
110 V2SCopyInfo() : Copy(nullptr), ID(0){};
111 V2SCopyInfo(unsigned Id, MachineInstr *C, unsigned Width)
112 : Copy(C), NumSVCopies(0), NumReadfirstlanes(Width / 32), ID(Id){};
113#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
114 void dump() {
115 dbgs() << ID << " : " << *Copy << "\n\tS:" << SChain.size()
116 << "\n\tSV:" << NumSVCopies << "\n\tSP: " << SiblingPenalty
117 << "\nScore: " << Score << "\n";
118 }
119#endif
120};
121
122class SIFixSGPRCopies {
128 unsigned NextVGPRToSGPRCopyID = 0;
131
132public:
134 const SIRegisterInfo *TRI;
135 const SIInstrInfo *TII;
136
137 SIFixSGPRCopies(MachineDominatorTree *MDT) : MDT(MDT) {}
138
139 bool run(MachineFunction &MF);
140 void fixSCCCopies(MachineFunction &MF);
141 void prepareRegSequenceAndPHIs(MachineFunction &MF);
142 unsigned getNextVGPRToSGPRCopyId() { return ++NextVGPRToSGPRCopyID; }
143 bool needToBeConvertedToVALU(V2SCopyInfo *I);
144 void analyzeVGPRToSGPRCopy(MachineInstr *MI);
145 void lowerVGPR2SGPRCopies(MachineFunction &MF);
146 // Handles copies which source register is:
147 // 1. Physical register
148 // 2. AGPR
149 // 3. Defined by the instruction the merely moves the immediate
150 bool lowerSpecialCase(MachineInstr &MI, MachineBasicBlock::iterator &I);
151
152 void processPHINode(MachineInstr &MI);
153
154 // Check if MO is an immediate materialized into a VGPR, and if so replace it
155 // with an SGPR immediate. The VGPR immediate is also deleted if it does not
156 // have any other uses.
157 bool tryMoveVGPRConstToSGPR(MachineOperand &MO, Register NewDst,
158 MachineBasicBlock *BlockToInsertTo,
159 MachineBasicBlock::iterator PointToInsertTo);
160};
161
162class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
163public:
164 static char ID;
165
166 SIFixSGPRCopiesLegacy() : MachineFunctionPass(ID) {}
167
168 bool runOnMachineFunction(MachineFunction &MF) override {
170 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
171 SIFixSGPRCopies Impl(MDT);
172 return Impl.run(MF);
173 }
174
175 StringRef getPassName() const override { return "SI Fix SGPR copies"; }
176
177 void getAnalysisUsage(AnalysisUsage &AU) const override {
180 AU.setPreservesCFG();
182 }
183};
184
185} // end anonymous namespace
186
187INITIALIZE_PASS_BEGIN(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
188 false, false)
190INITIALIZE_PASS_END(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
192
193char SIFixSGPRCopiesLegacy::ID = 0;
194
195char &llvm::SIFixSGPRCopiesLegacyID = SIFixSGPRCopiesLegacy::ID;
196
198 return new SIFixSGPRCopiesLegacy();
199}
200
201static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
203 const SIRegisterInfo &TRI,
204 const MachineRegisterInfo &MRI) {
205 Register DstReg = Copy.getOperand(0).getReg();
206 Register SrcReg = Copy.getOperand(1).getReg();
207
208 const TargetRegisterClass *SrcRC = SrcReg.isVirtual()
209 ? MRI.getRegClass(SrcReg)
210 : TRI.getPhysRegBaseClass(SrcReg);
211
212 // We don't really care about the subregister here.
213 // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
214
215 const TargetRegisterClass *DstRC = DstReg.isVirtual()
216 ? MRI.getRegClass(DstReg)
217 : TRI.getPhysRegBaseClass(DstReg);
218
219 return std::pair(SrcRC, DstRC);
220}
221
222static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
223 const TargetRegisterClass *DstRC,
224 const SIRegisterInfo &TRI) {
225 return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(DstRC) &&
226 TRI.hasVectorRegisters(SrcRC);
227}
228
229static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
230 const TargetRegisterClass *DstRC,
231 const SIRegisterInfo &TRI) {
232 return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(SrcRC) &&
233 TRI.hasVectorRegisters(DstRC);
234}
235
237 const SIRegisterInfo *TRI,
238 const SIInstrInfo *TII) {
239 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
240 auto &Src = MI.getOperand(1);
241 Register DstReg = MI.getOperand(0).getReg();
242 Register SrcReg = Src.getReg();
243 if (!SrcReg.isVirtual() || !DstReg.isVirtual())
244 return false;
245
246 for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
247 const auto *UseMI = MO.getParent();
248 if (UseMI == &MI)
249 continue;
250 if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
251 UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
252 return false;
253
254 unsigned OpIdx = MO.getOperandNo();
255 if (OpIdx >= UseMI->getDesc().getNumOperands() ||
256 !TII->isOperandLegal(*UseMI, OpIdx, &Src))
257 return false;
258 }
259 // Change VGPR to SGPR destination.
260 MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
261 return true;
262}
263
264// Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
265//
266// SGPRx = ...
267// SGPRy = REG_SEQUENCE SGPRx, sub0 ...
268// VGPRz = COPY SGPRy
269//
270// ==>
271//
272// VGPRx = COPY SGPRx
273// VGPRz = REG_SEQUENCE VGPRx, sub0
274//
275// This exposes immediate folding opportunities when materializing 64-bit
276// immediates.
278 const SIRegisterInfo *TRI,
279 const SIInstrInfo *TII,
281 assert(MI.isRegSequence());
282
283 Register DstReg = MI.getOperand(0).getReg();
284 if (!TRI->isSGPRClass(MRI.getRegClass(DstReg)))
285 return false;
286
287 if (!MRI.hasOneUse(DstReg))
288 return false;
289
290 MachineInstr &CopyUse = *MRI.use_instr_begin(DstReg);
291 if (!CopyUse.isCopy())
292 return false;
293
294 // It is illegal to have vreg inputs to a physreg defining reg_sequence.
295 if (CopyUse.getOperand(0).getReg().isPhysical())
296 return false;
297
298 const TargetRegisterClass *SrcRC, *DstRC;
299 std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
300
301 if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
302 return false;
303
304 if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
305 return true;
306
307 // TODO: Could have multiple extracts?
308 unsigned SubReg = CopyUse.getOperand(1).getSubReg();
309 if (SubReg != AMDGPU::NoSubRegister)
310 return false;
311
312 MRI.setRegClass(DstReg, DstRC);
313
314 // SGPRx = ...
315 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
316 // VGPRz = COPY SGPRy
317
318 // =>
319 // VGPRx = COPY SGPRx
320 // VGPRz = REG_SEQUENCE VGPRx, sub0
321
322 MI.getOperand(0).setReg(CopyUse.getOperand(0).getReg());
323 bool IsAGPR = TRI->isAGPRClass(DstRC);
324
325 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
326 const TargetRegisterClass *SrcRC =
327 TRI->getRegClassForOperandReg(MRI, MI.getOperand(I));
328 assert(TRI->isSGPRClass(SrcRC) &&
329 "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
330 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SrcRC);
331
332 Register TmpReg = MRI.createVirtualRegister(NewSrcRC);
333
334 BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
335 TmpReg)
336 .add(MI.getOperand(I));
337
338 if (IsAGPR) {
339 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SrcRC);
340 Register TmpAReg = MRI.createVirtualRegister(NewSrcRC);
341 unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
342 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::COPY;
343 BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(Opc),
344 TmpAReg)
345 .addReg(TmpReg, RegState::Kill);
346 TmpReg = TmpAReg;
347 }
348
349 MI.getOperand(I).setReg(TmpReg);
350 }
351
352 CopyUse.eraseFromParent();
353 return true;
354}
355
356static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
357 const MachineInstr *MoveImm,
358 const SIInstrInfo *TII,
359 unsigned &SMovOp,
360 int64_t &Imm) {
361 if (Copy->getOpcode() != AMDGPU::COPY)
362 return false;
363
364 if (!MoveImm->isMoveImmediate())
365 return false;
366
367 const MachineOperand *ImmOp =
368 TII->getNamedOperand(*MoveImm, AMDGPU::OpName::src0);
369 if (!ImmOp->isImm())
370 return false;
371
372 // FIXME: Handle copies with sub-regs.
373 if (Copy->getOperand(1).getSubReg())
374 return false;
375
376 switch (MoveImm->getOpcode()) {
377 default:
378 return false;
379 case AMDGPU::V_MOV_B32_e32:
380 SMovOp = AMDGPU::S_MOV_B32;
381 break;
382 case AMDGPU::V_MOV_B64_PSEUDO:
383 SMovOp = AMDGPU::S_MOV_B64_IMM_PSEUDO;
384 break;
385 }
386 Imm = ImmOp->getImm();
387 return true;
388}
389
390template <class UnaryPredicate>
392 const MachineBasicBlock *CutOff,
393 UnaryPredicate Predicate) {
394 if (MBB == CutOff)
395 return false;
396
399
400 while (!Worklist.empty()) {
401 MachineBasicBlock *MBB = Worklist.pop_back_val();
402
403 if (!Visited.insert(MBB).second)
404 continue;
405 if (MBB == CutOff)
406 continue;
407 if (Predicate(MBB))
408 return true;
409
410 Worklist.append(MBB->pred_begin(), MBB->pred_end());
411 }
412
413 return false;
414}
415
416// Checks if there is potential path From instruction To instruction.
417// If CutOff is specified and it sits in between of that path we ignore
418// a higher portion of the path and report it is not reachable.
419static bool isReachable(const MachineInstr *From,
420 const MachineInstr *To,
421 const MachineBasicBlock *CutOff,
423 if (MDT.dominates(From, To))
424 return true;
425
426 const MachineBasicBlock *MBBFrom = From->getParent();
427 const MachineBasicBlock *MBBTo = To->getParent();
428
429 // Do predecessor search.
430 // We should almost never get here since we do not usually produce M0 stores
431 // other than -1.
432 return searchPredecessors(MBBTo, CutOff, [MBBFrom]
433 (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
434}
435
436// Return the first non-prologue instruction in the block.
440 while (I != MBB->end() && TII->isBasicBlockPrologue(*I))
441 ++I;
442
443 return I;
444}
445
446// Hoist and merge identical SGPR initializations into a common predecessor.
447// This is intended to combine M0 initializations, but can work with any
448// SGPR. A VGPR cannot be processed since we cannot guarantee vector
449// executioon.
450static bool hoistAndMergeSGPRInits(unsigned Reg,
452 const TargetRegisterInfo *TRI,
454 const TargetInstrInfo *TII) {
455 // List of inits by immediate value.
456 using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
457 InitListMap Inits;
458 // List of clobbering instructions.
460 // List of instructions marked for deletion.
461 SmallSet<MachineInstr*, 8> MergedInstrs;
462
463 bool Changed = false;
464
465 for (auto &MI : MRI.def_instructions(Reg)) {
466 MachineOperand *Imm = nullptr;
467 for (auto &MO : MI.operands()) {
468 if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
469 (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
470 Imm = nullptr;
471 break;
472 }
473 if (MO.isImm())
474 Imm = &MO;
475 }
476 if (Imm)
477 Inits[Imm->getImm()].push_front(&MI);
478 else
479 Clobbers.push_back(&MI);
480 }
481
482 for (auto &Init : Inits) {
483 auto &Defs = Init.second;
484
485 for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
486 MachineInstr *MI1 = *I1;
487
488 for (auto I2 = std::next(I1); I2 != E; ) {
489 MachineInstr *MI2 = *I2;
490
491 // Check any possible interference
492 auto interferes = [&](MachineBasicBlock::iterator From,
493 MachineBasicBlock::iterator To) -> bool {
494
495 assert(MDT.dominates(&*To, &*From));
496
497 auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
498 const MachineBasicBlock *MBBFrom = From->getParent();
499 const MachineBasicBlock *MBBTo = To->getParent();
500 bool MayClobberFrom = isReachable(Clobber, &*From, MBBTo, MDT);
501 bool MayClobberTo = isReachable(Clobber, &*To, MBBTo, MDT);
502 if (!MayClobberFrom && !MayClobberTo)
503 return false;
504 if ((MayClobberFrom && !MayClobberTo) ||
505 (!MayClobberFrom && MayClobberTo))
506 return true;
507 // Both can clobber, this is not an interference only if both are
508 // dominated by Clobber and belong to the same block or if Clobber
509 // properly dominates To, given that To >> From, so it dominates
510 // both and located in a common dominator.
511 return !((MBBFrom == MBBTo &&
512 MDT.dominates(Clobber, &*From) &&
513 MDT.dominates(Clobber, &*To)) ||
514 MDT.properlyDominates(Clobber->getParent(), MBBTo));
515 };
516
517 return (llvm::any_of(Clobbers, interferes)) ||
518 (llvm::any_of(Inits, [&](InitListMap::value_type &C) {
519 return C.first != Init.first &&
520 llvm::any_of(C.second, interferes);
521 }));
522 };
523
524 if (MDT.dominates(MI1, MI2)) {
525 if (!interferes(MI2, MI1)) {
527 << "Erasing from "
528 << printMBBReference(*MI2->getParent()) << " " << *MI2);
529 MergedInstrs.insert(MI2);
530 Changed = true;
531 ++I2;
532 continue;
533 }
534 } else if (MDT.dominates(MI2, MI1)) {
535 if (!interferes(MI1, MI2)) {
537 << "Erasing from "
538 << printMBBReference(*MI1->getParent()) << " " << *MI1);
539 MergedInstrs.insert(MI1);
540 Changed = true;
541 ++I1;
542 break;
543 }
544 } else {
545 auto *MBB = MDT.findNearestCommonDominator(MI1->getParent(),
546 MI2->getParent());
547 if (!MBB) {
548 ++I2;
549 continue;
550 }
551
553 if (!interferes(MI1, I) && !interferes(MI2, I)) {
555 << "Erasing from "
556 << printMBBReference(*MI1->getParent()) << " " << *MI1
557 << "and moving from "
558 << printMBBReference(*MI2->getParent()) << " to "
559 << printMBBReference(*I->getParent()) << " " << *MI2);
560 I->getParent()->splice(I, MI2->getParent(), MI2);
561 MergedInstrs.insert(MI1);
562 Changed = true;
563 ++I1;
564 break;
565 }
566 }
567 ++I2;
568 }
569 ++I1;
570 }
571 }
572
573 // Remove initializations that were merged into another.
574 for (auto &Init : Inits) {
575 auto &Defs = Init.second;
576 auto I = Defs.begin();
577 while (I != Defs.end()) {
578 if (MergedInstrs.count(*I)) {
579 (*I)->eraseFromParent();
580 I = Defs.erase(I);
581 } else
582 ++I;
583 }
584 }
585
586 // Try to schedule SGPR initializations as early as possible in the MBB.
587 for (auto &Init : Inits) {
588 auto &Defs = Init.second;
589 for (auto *MI : Defs) {
590 auto MBB = MI->getParent();
591 MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
593 // Check if B should actually be a boundary. If not set the previous
594 // instruction as the boundary instead.
595 if (!TII->isBasicBlockPrologue(*B))
596 B++;
597
598 auto R = std::next(MI->getReverseIterator());
599 const unsigned Threshold = 50;
600 // Search until B or Threshold for a place to insert the initialization.
601 for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
602 if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) ||
604 break;
605
606 // Move to directly after R.
607 if (&*--R != MI)
608 MBB->splice(*R, MBB, MI);
609 }
610 }
611
612 if (Changed)
613 MRI.clearKillFlags(Reg);
614
615 return Changed;
616}
617
618bool SIFixSGPRCopies::run(MachineFunction &MF) {
619 // Only need to run this in SelectionDAG path.
620 if (MF.getProperties().hasProperty(
621 MachineFunctionProperties::Property::Selected))
622 return false;
623
625 MRI = &MF.getRegInfo();
626 TRI = ST.getRegisterInfo();
627 TII = ST.getInstrInfo();
628
629 for (MachineBasicBlock &MBB : MF) {
630 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
631 ++I) {
632 MachineInstr &MI = *I;
633
634 switch (MI.getOpcode()) {
635 default:
636 continue;
637 case AMDGPU::COPY:
638 case AMDGPU::WQM:
639 case AMDGPU::STRICT_WQM:
640 case AMDGPU::SOFT_WQM:
641 case AMDGPU::STRICT_WWM: {
642 const TargetRegisterClass *SrcRC, *DstRC;
643 std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, *MRI);
644
645 if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
646 // Since VGPR to SGPR copies affect VGPR to SGPR copy
647 // score and, hence the lowering decision, let's try to get rid of
648 // them as early as possible
650 continue;
651
652 // Collect those not changed to try them after VGPR to SGPR copies
653 // lowering as there will be more opportunities.
654 S2VCopies.push_back(&MI);
655 }
656 if (!isVGPRToSGPRCopy(SrcRC, DstRC, *TRI))
657 continue;
658 if (lowerSpecialCase(MI, I))
659 continue;
660
661 analyzeVGPRToSGPRCopy(&MI);
662
663 break;
664 }
665 case AMDGPU::INSERT_SUBREG:
666 case AMDGPU::PHI:
667 case AMDGPU::REG_SEQUENCE: {
668 if (TRI->isSGPRClass(TII->getOpRegClass(MI, 0))) {
669 for (MachineOperand &MO : MI.operands()) {
670 if (!MO.isReg() || !MO.getReg().isVirtual())
671 continue;
672 const TargetRegisterClass *SrcRC = MRI->getRegClass(MO.getReg());
673 if (TRI->hasVectorRegisters(SrcRC)) {
674 const TargetRegisterClass *DestRC =
675 TRI->getEquivalentSGPRClass(SrcRC);
676 Register NewDst = MRI->createVirtualRegister(DestRC);
677 MachineBasicBlock *BlockToInsertCopy =
678 MI.isPHI() ? MI.getOperand(MO.getOperandNo() + 1).getMBB()
679 : &MBB;
680 MachineBasicBlock::iterator PointToInsertCopy =
681 MI.isPHI() ? BlockToInsertCopy->getFirstInstrTerminator() : I;
682
683 if (!tryMoveVGPRConstToSGPR(MO, NewDst, BlockToInsertCopy,
684 PointToInsertCopy)) {
685 MachineInstr *NewCopy =
686 BuildMI(*BlockToInsertCopy, PointToInsertCopy,
687 PointToInsertCopy->getDebugLoc(),
688 TII->get(AMDGPU::COPY), NewDst)
689 .addReg(MO.getReg());
690 MO.setReg(NewDst);
691 analyzeVGPRToSGPRCopy(NewCopy);
692 }
693 }
694 }
695 }
696
697 if (MI.isPHI())
698 PHINodes.push_back(&MI);
699 else if (MI.isRegSequence())
700 RegSequences.push_back(&MI);
701
702 break;
703 }
704 case AMDGPU::V_WRITELANE_B32: {
705 // Some architectures allow more than one constant bus access without
706 // SGPR restriction
707 if (ST.getConstantBusLimit(MI.getOpcode()) != 1)
708 break;
709
710 // Writelane is special in that it can use SGPR and M0 (which would
711 // normally count as using the constant bus twice - but in this case it
712 // is allowed since the lane selector doesn't count as a use of the
713 // constant bus). However, it is still required to abide by the 1 SGPR
714 // rule. Apply a fix here as we might have multiple SGPRs after
715 // legalizing VGPRs to SGPRs
716 int Src0Idx =
717 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
718 int Src1Idx =
719 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
720 MachineOperand &Src0 = MI.getOperand(Src0Idx);
721 MachineOperand &Src1 = MI.getOperand(Src1Idx);
722
723 // Check to see if the instruction violates the 1 SGPR rule
724 if ((Src0.isReg() && TRI->isSGPRReg(*MRI, Src0.getReg()) &&
725 Src0.getReg() != AMDGPU::M0) &&
726 (Src1.isReg() && TRI->isSGPRReg(*MRI, Src1.getReg()) &&
727 Src1.getReg() != AMDGPU::M0)) {
728
729 // Check for trivially easy constant prop into one of the operands
730 // If this is the case then perform the operation now to resolve SGPR
731 // issue. If we don't do that here we will always insert a mov to m0
732 // that can't be resolved in later operand folding pass
733 bool Resolved = false;
734 for (MachineOperand *MO : {&Src0, &Src1}) {
735 if (MO->getReg().isVirtual()) {
736 MachineInstr *DefMI = MRI->getVRegDef(MO->getReg());
737 if (DefMI && TII->isFoldableCopy(*DefMI)) {
738 const MachineOperand &Def = DefMI->getOperand(0);
739 if (Def.isReg() &&
740 MO->getReg() == Def.getReg() &&
741 MO->getSubReg() == Def.getSubReg()) {
742 const MachineOperand &Copied = DefMI->getOperand(1);
743 if (Copied.isImm() &&
744 TII->isInlineConstant(APInt(64, Copied.getImm(), true))) {
745 MO->ChangeToImmediate(Copied.getImm());
746 Resolved = true;
747 break;
748 }
749 }
750 }
751 }
752 }
753
754 if (!Resolved) {
755 // Haven't managed to resolve by replacing an SGPR with an immediate
756 // Move src1 to be in M0
757 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
758 TII->get(AMDGPU::COPY), AMDGPU::M0)
759 .add(Src1);
760 Src1.ChangeToRegister(AMDGPU::M0, false);
761 }
762 }
763 break;
764 }
765 }
766 }
767 }
768
769 lowerVGPR2SGPRCopies(MF);
770 // Postprocessing
771 fixSCCCopies(MF);
772 for (auto MI : S2VCopies) {
773 // Check if it is still valid
774 if (MI->isCopy()) {
775 const TargetRegisterClass *SrcRC, *DstRC;
776 std::tie(SrcRC, DstRC) = getCopyRegClasses(*MI, *TRI, *MRI);
777 if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
779 }
780 }
781 for (auto MI : RegSequences) {
782 // Check if it is still valid
783 if (MI->isRegSequence())
785 }
786 for (auto MI : PHINodes) {
787 processPHINode(*MI);
788 }
789 if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
790 hoistAndMergeSGPRInits(AMDGPU::M0, *MRI, TRI, *MDT, TII);
791
792 SiblingPenalty.clear();
793 V2SCopies.clear();
794 SCCCopies.clear();
795 RegSequences.clear();
796 PHINodes.clear();
797 S2VCopies.clear();
798
799 return true;
800}
801
802void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
803 bool AllAGPRUses = true;
806 SetVector<MachineInstr *> PHIOperands;
807 worklist.insert(&MI);
808 Visited.insert(&MI);
809 // HACK to make MIR tests with no uses happy
810 bool HasUses = false;
811 while (!worklist.empty()) {
812 const MachineInstr *Instr = worklist.pop_back_val();
813 Register Reg = Instr->getOperand(0).getReg();
814 for (const auto &Use : MRI->use_operands(Reg)) {
815 HasUses = true;
816 const MachineInstr *UseMI = Use.getParent();
817 AllAGPRUses &= (UseMI->isCopy() &&
818 TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg())) ||
819 TRI->isAGPR(*MRI, Use.getReg());
820 if (UseMI->isCopy() || UseMI->isRegSequence()) {
821 if (Visited.insert(UseMI).second)
822 worklist.insert(UseMI);
823
824 continue;
825 }
826 }
827 }
828
829 Register PHIRes = MI.getOperand(0).getReg();
830 const TargetRegisterClass *RC0 = MRI->getRegClass(PHIRes);
831 if (HasUses && AllAGPRUses && !TRI->isAGPRClass(RC0)) {
832 LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
833 MRI->setRegClass(PHIRes, TRI->getEquivalentAGPRClass(RC0));
834 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
835 MachineInstr *DefMI = MRI->getVRegDef(MI.getOperand(I).getReg());
836 if (DefMI && DefMI->isPHI())
837 PHIOperands.insert(DefMI);
838 }
839 }
840
841 if (TRI->isVectorRegister(*MRI, PHIRes) ||
842 RC0 == &AMDGPU::VReg_1RegClass) {
843 LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
844 TII->legalizeOperands(MI, MDT);
845 }
846
847 // Propagate register class back to PHI operands which are PHI themselves.
848 while (!PHIOperands.empty()) {
849 processPHINode(*PHIOperands.pop_back_val());
850 }
851}
852
853bool SIFixSGPRCopies::tryMoveVGPRConstToSGPR(
854 MachineOperand &MaybeVGPRConstMO, Register DstReg,
855 MachineBasicBlock *BlockToInsertTo,
856 MachineBasicBlock::iterator PointToInsertTo) {
857
858 MachineInstr *DefMI = MRI->getVRegDef(MaybeVGPRConstMO.getReg());
859 if (!DefMI || !DefMI->isMoveImmediate())
860 return false;
861
862 MachineOperand *SrcConst = TII->getNamedOperand(*DefMI, AMDGPU::OpName::src0);
863 if (SrcConst->isReg())
864 return false;
865
866 const TargetRegisterClass *SrcRC =
867 MRI->getRegClass(MaybeVGPRConstMO.getReg());
868 unsigned MoveSize = TRI->getRegSizeInBits(*SrcRC);
869 unsigned MoveOp = MoveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
870 BuildMI(*BlockToInsertTo, PointToInsertTo, PointToInsertTo->getDebugLoc(),
871 TII->get(MoveOp), DstReg)
872 .add(*SrcConst);
873 if (MRI->hasOneUse(MaybeVGPRConstMO.getReg()))
875 MaybeVGPRConstMO.setReg(DstReg);
876 return true;
877}
878
879bool SIFixSGPRCopies::lowerSpecialCase(MachineInstr &MI,
881 Register DstReg = MI.getOperand(0).getReg();
882 Register SrcReg = MI.getOperand(1).getReg();
883 if (!DstReg.isVirtual()) {
884 // If the destination register is a physical register there isn't
885 // really much we can do to fix this.
886 // Some special instructions use M0 as an input. Some even only use
887 // the first lane. Insert a readfirstlane and hope for the best.
888 if (DstReg == AMDGPU::M0 &&
889 TRI->hasVectorRegisters(MRI->getRegClass(SrcReg))) {
890 Register TmpReg =
891 MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
892 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
893 TII->get(AMDGPU::V_READFIRSTLANE_B32), TmpReg)
894 .add(MI.getOperand(1));
895 MI.getOperand(1).setReg(TmpReg);
896 } else if (tryMoveVGPRConstToSGPR(MI.getOperand(1), DstReg, MI.getParent(),
897 MI)) {
898 I = std::next(I);
899 MI.eraseFromParent();
900 }
901 return true;
902 }
903 if (!SrcReg.isVirtual() || TRI->isAGPR(*MRI, SrcReg)) {
904 SIInstrWorklist worklist;
905 worklist.insert(&MI);
906 TII->moveToVALU(worklist, MDT);
907 return true;
908 }
909
910 unsigned SMovOp;
911 int64_t Imm;
912 // If we are just copying an immediate, we can replace the copy with
913 // s_mov_b32.
914 if (isSafeToFoldImmIntoCopy(&MI, MRI->getVRegDef(SrcReg), TII, SMovOp, Imm)) {
915 MI.getOperand(1).ChangeToImmediate(Imm);
916 MI.addImplicitDefUseOperands(*MI.getParent()->getParent());
917 MI.setDesc(TII->get(SMovOp));
918 return true;
919 }
920 return false;
921}
922
923void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
924 Register DstReg = MI->getOperand(0).getReg();
925 const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg);
926
927 V2SCopyInfo Info(getNextVGPRToSGPRCopyId(), MI,
928 TRI->getRegSizeInBits(*DstRC));
929 SmallVector<MachineInstr *, 8> AnalysisWorklist;
930 // Needed because the SSA is not a tree but a graph and may have
931 // forks and joins. We should not then go same way twice.
933 AnalysisWorklist.push_back(Info.Copy);
934 while (!AnalysisWorklist.empty()) {
935
936 MachineInstr *Inst = AnalysisWorklist.pop_back_val();
937
938 if (!Visited.insert(Inst).second)
939 continue;
940
941 // Copies and REG_SEQUENCE do not contribute to the final assembly
942 // So, skip them but take care of the SGPR to VGPR copies bookkeeping.
943 if (Inst->isCopy() || Inst->isRegSequence()) {
944 if (TRI->isVGPR(*MRI, Inst->getOperand(0).getReg())) {
945 if (!Inst->isCopy() ||
947 Info.NumSVCopies++;
948 continue;
949 }
950 }
951 }
952
953 SiblingPenalty[Inst].insert(Info.ID);
954
956 if ((TII->isSALU(*Inst) && Inst->isCompare()) ||
957 (Inst->isCopy() && Inst->getOperand(0).getReg() == AMDGPU::SCC)) {
958 auto I = Inst->getIterator();
959 auto E = Inst->getParent()->end();
960 while (++I != E &&
961 !I->findRegisterDefOperand(AMDGPU::SCC, /*TRI=*/nullptr)) {
962 if (I->readsRegister(AMDGPU::SCC, /*TRI=*/nullptr))
963 Users.push_back(&*I);
964 }
965 } else if (Inst->getNumExplicitDefs() != 0) {
966 Register Reg = Inst->getOperand(0).getReg();
967 if (TRI->isSGPRReg(*MRI, Reg) && !TII->isVALU(*Inst))
968 for (auto &U : MRI->use_instructions(Reg))
969 Users.push_back(&U);
970 }
971 for (auto U : Users) {
972 if (TII->isSALU(*U))
973 Info.SChain.insert(U);
974 AnalysisWorklist.push_back(U);
975 }
976 }
977 V2SCopies[Info.ID] = Info;
978}
979
980// The main function that computes the VGPR to SGPR copy score
981// and determines copy further lowering way: v_readfirstlane_b32 or moveToVALU
982bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
983 if (Info->SChain.empty()) {
984 Info->Score = 0;
985 return true;
986 }
987 Info->Siblings = SiblingPenalty[*llvm::max_element(
988 Info->SChain, [&](MachineInstr *A, MachineInstr *B) -> bool {
989 return SiblingPenalty[A].size() < SiblingPenalty[B].size();
990 })];
991 Info->Siblings.remove_if([&](unsigned ID) { return ID == Info->ID; });
992 // The loop below computes the number of another VGPR to SGPR V2SCopies
993 // which contribute to the current copy SALU chain. We assume that all the
994 // V2SCopies with the same source virtual register will be squashed to one
995 // by regalloc. Also we take care of the V2SCopies of the differnt subregs
996 // of the same register.
998 for (auto J : Info->Siblings) {
999 auto InfoIt = V2SCopies.find(J);
1000 if (InfoIt != V2SCopies.end()) {
1001 MachineInstr *SiblingCopy = InfoIt->second.Copy;
1002 if (SiblingCopy->isImplicitDef())
1003 // the COPY has already been MoveToVALUed
1004 continue;
1005
1006 SrcRegs.insert(std::pair(SiblingCopy->getOperand(1).getReg(),
1007 SiblingCopy->getOperand(1).getSubReg()));
1008 }
1009 }
1010 Info->SiblingPenalty = SrcRegs.size();
1011
1012 unsigned Penalty =
1013 Info->NumSVCopies + Info->SiblingPenalty + Info->NumReadfirstlanes;
1014 unsigned Profit = Info->SChain.size();
1015 Info->Score = Penalty > Profit ? 0 : Profit - Penalty;
1016 Info->NeedToBeConvertedToVALU = Info->Score < 3;
1017 return Info->NeedToBeConvertedToVALU;
1018}
1019
1020void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
1021
1022 SmallVector<unsigned, 8> LoweringWorklist;
1023 for (auto &C : V2SCopies) {
1024 if (needToBeConvertedToVALU(&C.second))
1025 LoweringWorklist.push_back(C.second.ID);
1026 }
1027
1028 // Store all the V2S copy instructions that need to be moved to VALU
1029 // in the Copies worklist.
1031
1032 while (!LoweringWorklist.empty()) {
1033 unsigned CurID = LoweringWorklist.pop_back_val();
1034 auto CurInfoIt = V2SCopies.find(CurID);
1035 if (CurInfoIt != V2SCopies.end()) {
1036 V2SCopyInfo C = CurInfoIt->second;
1037 LLVM_DEBUG(dbgs() << "Processing ...\n"; C.dump());
1038 for (auto S : C.Siblings) {
1039 auto SibInfoIt = V2SCopies.find(S);
1040 if (SibInfoIt != V2SCopies.end()) {
1041 V2SCopyInfo &SI = SibInfoIt->second;
1042 LLVM_DEBUG(dbgs() << "Sibling:\n"; SI.dump());
1043 if (!SI.NeedToBeConvertedToVALU) {
1044 SI.SChain.set_subtract(C.SChain);
1045 if (needToBeConvertedToVALU(&SI))
1046 LoweringWorklist.push_back(SI.ID);
1047 }
1048 SI.Siblings.remove_if([&](unsigned ID) { return ID == C.ID; });
1049 }
1050 }
1051 LLVM_DEBUG(dbgs() << "V2S copy " << *C.Copy
1052 << " is being turned to VALU\n");
1053 // TODO: MapVector::erase is inefficient. Do bulk removal with remove_if
1054 // instead.
1055 V2SCopies.erase(C.ID);
1056 Copies.insert(C.Copy);
1057 }
1058 }
1059
1060 TII->moveToVALU(Copies, MDT);
1061 Copies.clear();
1062
1063 // Now do actual lowering
1064 for (auto C : V2SCopies) {
1065 MachineInstr *MI = C.second.Copy;
1066 MachineBasicBlock *MBB = MI->getParent();
1067 // We decide to turn V2S copy to v_readfirstlane_b32
1068 // remove it from the V2SCopies and remove it from all its siblings
1069 LLVM_DEBUG(dbgs() << "V2S copy " << *MI
1070 << " is being turned to v_readfirstlane_b32"
1071 << " Score: " << C.second.Score << "\n");
1072 Register DstReg = MI->getOperand(0).getReg();
1073 Register SrcReg = MI->getOperand(1).getReg();
1074 unsigned SubReg = MI->getOperand(1).getSubReg();
1075 const TargetRegisterClass *SrcRC =
1076 TRI->getRegClassForOperandReg(*MRI, MI->getOperand(1));
1077 size_t SrcSize = TRI->getRegSizeInBits(*SrcRC);
1078 if (SrcSize == 16) {
1079 // HACK to handle possible 16bit VGPR source
1080 auto MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
1081 TII->get(AMDGPU::V_READFIRSTLANE_B32), DstReg);
1082 MIB.addReg(SrcReg, 0, AMDGPU::NoSubRegister);
1083 } else if (SrcSize == 32) {
1084 auto MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
1085 TII->get(AMDGPU::V_READFIRSTLANE_B32), DstReg);
1086 MIB.addReg(SrcReg, 0, SubReg);
1087 } else {
1088 auto Result = BuildMI(*MBB, MI, MI->getDebugLoc(),
1089 TII->get(AMDGPU::REG_SEQUENCE), DstReg);
1090 int N = TRI->getRegSizeInBits(*SrcRC) / 32;
1091 for (int i = 0; i < N; i++) {
1092 Register PartialSrc = TII->buildExtractSubReg(
1093 Result, *MRI, MI->getOperand(1), SrcRC,
1094 TRI->getSubRegFromChannel(i), &AMDGPU::VGPR_32RegClass);
1095 Register PartialDst =
1096 MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1097 BuildMI(*MBB, *Result, Result->getDebugLoc(),
1098 TII->get(AMDGPU::V_READFIRSTLANE_B32), PartialDst)
1099 .addReg(PartialSrc);
1100 Result.addReg(PartialDst).addImm(TRI->getSubRegFromChannel(i));
1101 }
1102 }
1103 MI->eraseFromParent();
1104 }
1105}
1106
1107void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
1108 bool IsWave32 = MF.getSubtarget<GCNSubtarget>().isWave32();
1109 for (MachineBasicBlock &MBB : MF) {
1110 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
1111 ++I) {
1112 MachineInstr &MI = *I;
1113 // May already have been lowered.
1114 if (!MI.isCopy())
1115 continue;
1116 Register SrcReg = MI.getOperand(1).getReg();
1117 Register DstReg = MI.getOperand(0).getReg();
1118 if (SrcReg == AMDGPU::SCC) {
1119 Register SCCCopy = MRI->createVirtualRegister(
1120 TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID));
1121 I = BuildMI(*MI.getParent(), std::next(MachineBasicBlock::iterator(MI)),
1122 MI.getDebugLoc(),
1123 TII->get(IsWave32 ? AMDGPU::S_CSELECT_B32
1124 : AMDGPU::S_CSELECT_B64),
1125 SCCCopy)
1126 .addImm(-1)
1127 .addImm(0);
1128 I = BuildMI(*MI.getParent(), std::next(I), I->getDebugLoc(),
1129 TII->get(AMDGPU::COPY), DstReg)
1130 .addReg(SCCCopy);
1131 MI.eraseFromParent();
1132 continue;
1133 }
1134 if (DstReg == AMDGPU::SCC) {
1135 unsigned Opcode = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
1136 Register Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1137 Register Tmp = MRI->createVirtualRegister(TRI->getBoolRC());
1138 I = BuildMI(*MI.getParent(), std::next(MachineBasicBlock::iterator(MI)),
1139 MI.getDebugLoc(), TII->get(Opcode))
1140 .addReg(Tmp, getDefRegState(true))
1141 .addReg(SrcReg)
1142 .addReg(Exec);
1143 MI.eraseFromParent();
1144 }
1145 }
1146 }
1147}
1148
1153 SIFixSGPRCopies Impl(&MDT);
1154 bool Changed = Impl.run(MF);
1155 if (!Changed)
1156 return PreservedAnalyses::all();
1157
1158 // TODO: We could detect CFG changed.
1160 return PA;
1161}
unsigned SubReg
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
Falkor HW Prefetch Fix
Provides AMDGPU specific target descriptions.
MachineBasicBlock & MBB
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
#define LLVM_DEBUG(X)
Definition: Debug.h:101
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
iv Induction Variable Users
Definition: IVUsers.cpp:48
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:57
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
static std::pair< const TargetRegisterClass *, const TargetRegisterClass * > getCopyRegClasses(const MachineInstr &Copy, const SIRegisterInfo &TRI, const MachineRegisterInfo &MRI)
SI Fix SGPR copies
static cl::opt< bool > EnableM0Merge("amdgpu-enable-merge-m0", cl::desc("Merge and hoist M0 initializations"), cl::init(true))
static bool hoistAndMergeSGPRInits(unsigned Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo *TRI, MachineDominatorTree &MDT, const TargetInstrInfo *TII)
static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII, MachineRegisterInfo &MRI)
bool searchPredecessors(const MachineBasicBlock *MBB, const MachineBasicBlock *CutOff, UnaryPredicate Predicate)
static bool isReachable(const MachineInstr *From, const MachineInstr *To, const MachineBasicBlock *CutOff, MachineDominatorTree &MDT)
static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII)
static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy, const MachineInstr *MoveImm, const SIInstrInfo *TII, unsigned &SMovOp, int64_t &Imm)
#define DEBUG_TYPE
static MachineBasicBlock::iterator getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI Lower i1 Copies
This file defines generic set operations that may be used on set's of different types,...
Class for arbitrary precision integers.
Definition: APInt.h:78
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:256
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
Test if the given instruction should be considered a scheduling boundary.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:237
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Analysis pass which computes a MachineDominatorTree.
Analysis pass which computes a MachineDominatorTree.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B.
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
bool properlyDominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
bool hasProperty(Property P) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineFunctionProperties & getProperties() const
Get the function properties.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:569
bool isImplicitDef() const
bool isCopy() const
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:346
bool isCompare(QueryType Type=IgnoreBundle) const
Return true if this instruction is a comparison.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:566
bool isRegSequence() const
unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
bool isMoveImmediate(QueryType Type=IgnoreBundle) const
Return true if this instruction is a move immediate (including conditional moves) instruction.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
bool isPHI() const
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:579
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:93
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
value_type pop_back_val()
Definition: SetVector.h:285
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_type size() const
Definition: SmallSet.h:161
bool empty() const
Definition: SmallVector.h:95
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:697
void push_back(const T &Elt)
Definition: SmallVector.h:427
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
self_iterator getIterator()
Definition: ilist_node.h:132
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
@ Resolved
Queried, materialization begun.
NodeAddr< InstrNode * > Instr
Definition: RDFGraph.h:389
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
unsigned getDefRegState(bool B)
auto max_element(R &&Range)
Definition: STLExtras.h:1986
char & SIFixSGPRCopiesLegacyID
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
FunctionPass * createSIFixSGPRCopiesLegacyPass()
#define N
Utility to store machine instructions worklist.
Definition: SIInstrInfo.h:49
void insert(MachineInstr *MI)