LLVM 17.0.0git
PPCInstrInfo.cpp
Go to the documentation of this file.
1//===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the PowerPC implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCInstrInfo.h"
15#include "PPC.h"
17#include "PPCInstrBuilder.h"
19#include "PPCTargetMachine.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/Statistic.h"
38#include "llvm/MC/MCAsmInfo.h"
39#include "llvm/MC/MCInst.h"
42#include "llvm/Support/Debug.h"
45
46using namespace llvm;
47
48#define DEBUG_TYPE "ppc-instr-info"
49
50#define GET_INSTRMAP_INFO
51#define GET_INSTRINFO_CTOR_DTOR
52#include "PPCGenInstrInfo.inc"
53
54STATISTIC(NumStoreSPILLVSRRCAsVec,
55 "Number of spillvsrrc spilled to stack as vec");
56STATISTIC(NumStoreSPILLVSRRCAsGpr,
57 "Number of spillvsrrc spilled to stack as gpr");
58STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
59STATISTIC(CmpIselsConverted,
60 "Number of ISELs that depend on comparison of constants converted");
61STATISTIC(MissedConvertibleImmediateInstrs,
62 "Number of compare-immediate instructions fed by constants");
63STATISTIC(NumRcRotatesConvertedToRcAnd,
64 "Number of record-form rotates converted to record-form andi");
65
66static cl::
67opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
68 cl::desc("Disable analysis for CTR loops"));
69
70static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
71cl::desc("Disable compare instruction optimization"), cl::Hidden);
72
73static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
74cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
76
77static cl::opt<bool>
78UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
79 cl::desc("Use the old (incorrect) instruction latency calculation"));
80
81static cl::opt<float>
82 FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
83 cl::desc("register pressure factor for the transformations."));
84
86 "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
87 cl::desc("enable register pressure reduce in machine combiner pass."));
88
89// Pin the vtable to this file.
90void PPCInstrInfo::anchor() {}
91
93 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
94 /* CatchRetOpcode */ -1,
95 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
96 Subtarget(STI), RI(STI.getTargetMachine()) {}
97
98/// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
99/// this target when scheduling the DAG.
102 const ScheduleDAG *DAG) const {
103 unsigned Directive =
104 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
107 const InstrItineraryData *II =
108 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
109 return new ScoreboardHazardRecognizer(II, DAG);
110 }
111
113}
114
115/// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
116/// to use for this target when scheduling the DAG.
119 const ScheduleDAG *DAG) const {
120 unsigned Directive =
121 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
122
123 // FIXME: Leaving this as-is until we have POWER9 scheduling info
125 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
126
127 // Most subtargets use a PPC970 recognizer.
130 assert(DAG->TII && "No InstrInfo?");
131
132 return new PPCHazardRecognizer970(*DAG);
133 }
134
135 return new ScoreboardHazardRecognizer(II, DAG);
136}
137
139 const MachineInstr &MI,
140 unsigned *PredCost) const {
141 if (!ItinData || UseOldLatencyCalc)
142 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
143
144 // The default implementation of getInstrLatency calls getStageLatency, but
145 // getStageLatency does not do the right thing for us. While we have
146 // itinerary, most cores are fully pipelined, and so the itineraries only
147 // express the first part of the pipeline, not every stage. Instead, we need
148 // to use the listed output operand cycle number (using operand 0 here, which
149 // is an output).
150
151 unsigned Latency = 1;
152 unsigned DefClass = MI.getDesc().getSchedClass();
153 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
154 const MachineOperand &MO = MI.getOperand(i);
155 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
156 continue;
157
158 int Cycle = ItinData->getOperandCycle(DefClass, i);
159 if (Cycle < 0)
160 continue;
161
162 Latency = std::max(Latency, (unsigned) Cycle);
163 }
164
165 return Latency;
166}
167
169 const MachineInstr &DefMI, unsigned DefIdx,
170 const MachineInstr &UseMI,
171 unsigned UseIdx) const {
172 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
173 UseMI, UseIdx);
174
175 if (!DefMI.getParent())
176 return Latency;
177
178 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
179 Register Reg = DefMO.getReg();
180
181 bool IsRegCR;
182 if (Reg.isVirtual()) {
183 const MachineRegisterInfo *MRI =
184 &DefMI.getParent()->getParent()->getRegInfo();
185 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
186 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
187 } else {
188 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
189 PPC::CRBITRCRegClass.contains(Reg);
190 }
191
192 if (UseMI.isBranch() && IsRegCR) {
193 if (Latency < 0)
194 Latency = getInstrLatency(ItinData, DefMI);
195
196 // On some cores, there is an additional delay between writing to a condition
197 // register, and using it from a branch.
198 unsigned Directive = Subtarget.getCPUDirective();
199 switch (Directive) {
200 default: break;
201 case PPC::DIR_7400:
202 case PPC::DIR_750:
203 case PPC::DIR_970:
204 case PPC::DIR_E5500:
205 case PPC::DIR_PWR4:
206 case PPC::DIR_PWR5:
207 case PPC::DIR_PWR5X:
208 case PPC::DIR_PWR6:
209 case PPC::DIR_PWR6X:
210 case PPC::DIR_PWR7:
211 case PPC::DIR_PWR8:
212 // FIXME: Is this needed for POWER9?
213 Latency += 2;
214 break;
215 }
216 }
217
218 return Latency;
219}
220
221/// This is an architecture-specific helper function of reassociateOps.
222/// Set special operand attributes for new instructions after reassociation.
224 MachineInstr &OldMI2,
225 MachineInstr &NewMI1,
226 MachineInstr &NewMI2) const {
227 // Propagate FP flags from the original instructions.
228 // But clear poison-generating flags because those may not be valid now.
229 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
230 NewMI1.setFlags(IntersectedFlags);
234
235 NewMI2.setFlags(IntersectedFlags);
239}
240
242 uint16_t Flags) const {
243 MI.setFlags(Flags);
247}
248
249// This function does not list all associative and commutative operations, but
250// only those worth feeding through the machine combiner in an attempt to
251// reduce the critical path. Mostly, this means floating-point operations,
252// because they have high latencies(>=5) (compared to other operations, such as
253// and/or, which are also associative and commutative, but have low latencies).
255 bool Invert) const {
256 if (Invert)
257 return false;
258 switch (Inst.getOpcode()) {
259 // Floating point:
260 // FP Add:
261 case PPC::FADD:
262 case PPC::FADDS:
263 // FP Multiply:
264 case PPC::FMUL:
265 case PPC::FMULS:
266 // Altivec Add:
267 case PPC::VADDFP:
268 // VSX Add:
269 case PPC::XSADDDP:
270 case PPC::XVADDDP:
271 case PPC::XVADDSP:
272 case PPC::XSADDSP:
273 // VSX Multiply:
274 case PPC::XSMULDP:
275 case PPC::XVMULDP:
276 case PPC::XVMULSP:
277 case PPC::XSMULSP:
280 // Fixed point:
281 // Multiply:
282 case PPC::MULHD:
283 case PPC::MULLD:
284 case PPC::MULHW:
285 case PPC::MULLW:
286 return true;
287 default:
288 return false;
289 }
290}
291
292#define InfoArrayIdxFMAInst 0
293#define InfoArrayIdxFAddInst 1
294#define InfoArrayIdxFMULInst 2
295#define InfoArrayIdxAddOpIdx 3
296#define InfoArrayIdxMULOpIdx 4
297#define InfoArrayIdxFSubInst 5
298// Array keeps info for FMA instructions:
299// Index 0(InfoArrayIdxFMAInst): FMA instruction;
300// Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
301// Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
302// Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
303// Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
304// second MUL operand index is plus 1;
305// Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
306static const uint16_t FMAOpIdxInfo[][6] = {
307 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
308 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
309 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
310 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
311 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
312 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
313 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
314
315// Check if an opcode is a FMA instruction. If it is, return the index in array
316// FMAOpIdxInfo. Otherwise, return -1.
317int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
318 for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
319 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
320 return I;
321 return -1;
322}
323
324// On PowerPC target, we have two kinds of patterns related to FMA:
325// 1: Improve ILP.
326// Try to reassociate FMA chains like below:
327//
328// Pattern 1:
329// A = FADD X, Y (Leaf)
330// B = FMA A, M21, M22 (Prev)
331// C = FMA B, M31, M32 (Root)
332// -->
333// A = FMA X, M21, M22
334// B = FMA Y, M31, M32
335// C = FADD A, B
336//
337// Pattern 2:
338// A = FMA X, M11, M12 (Leaf)
339// B = FMA A, M21, M22 (Prev)
340// C = FMA B, M31, M32 (Root)
341// -->
342// A = FMUL M11, M12
343// B = FMA X, M21, M22
344// D = FMA A, M31, M32
345// C = FADD B, D
346//
347// breaking the dependency between A and B, allowing FMA to be executed in
348// parallel (or back-to-back in a pipeline) instead of depending on each other.
349//
350// 2: Reduce register pressure.
351// Try to reassociate FMA with FSUB and a constant like below:
352// C is a floating point const.
353//
354// Pattern 1:
355// A = FSUB X, Y (Leaf)
356// D = FMA B, C, A (Root)
357// -->
358// A = FMA B, Y, -C
359// D = FMA A, X, C
360//
361// Pattern 2:
362// A = FSUB X, Y (Leaf)
363// D = FMA B, A, C (Root)
364// -->
365// A = FMA B, Y, -C
366// D = FMA A, X, C
367//
368// Before the transformation, A must be assigned with different hardware
369// register with D. After the transformation, A and D must be assigned with
370// same hardware register due to TIE attribute of FMA instructions.
371//
374 bool DoRegPressureReduce) const {
378
379 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
380 for (const auto &MO : Instr.explicit_operands())
381 if (!(MO.isReg() && MO.getReg().isVirtual()))
382 return false;
383 return true;
384 };
385
386 auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
387 unsigned OpType) {
388 if (Instr.getOpcode() !=
389 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
390 return false;
391
392 // Instruction can be reassociated.
393 // fast math flags may prohibit reassociation.
396 return false;
397
398 // Instruction operands are virtual registers for reassociation.
399 if (!IsAllOpsVirtualReg(Instr))
400 return false;
401
402 // For register pressure reassociation, the FSub must have only one use as
403 // we want to delete the sub to save its def.
404 if (OpType == InfoArrayIdxFSubInst &&
405 !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
406 return false;
407
408 return true;
409 };
410
411 auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
412 int16_t &MulOpIdx, bool IsLeaf) {
413 int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
414 if (Idx < 0)
415 return false;
416
417 // Instruction can be reassociated.
418 // fast math flags may prohibit reassociation.
421 return false;
422
423 // Instruction operands are virtual registers for reassociation.
424 if (!IsAllOpsVirtualReg(Instr))
425 return false;
426
428 if (IsLeaf)
429 return true;
430
432
433 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
434 MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
435 // If 'add' operand's def is not in current block, don't do ILP related opt.
436 if (!MIAdd || MIAdd->getParent() != MBB)
437 return false;
438
439 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
440 // as this fma will be changed later.
441 return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
442 };
443
444 int16_t AddOpIdx = -1;
445 int16_t MulOpIdx = -1;
446
447 bool IsUsedOnceL = false;
448 bool IsUsedOnceR = false;
449 MachineInstr *MULInstrL = nullptr;
450 MachineInstr *MULInstrR = nullptr;
451
452 auto IsRPReductionCandidate = [&]() {
453 // Currently, we only support float and double.
454 // FIXME: add support for other types.
455 unsigned Opcode = Root.getOpcode();
456 if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
457 return false;
458
459 // Root must be a valid FMA like instruction.
460 // Treat it as leaf as we don't care its add operand.
461 if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
462 assert((MulOpIdx >= 0) && "mul operand index not right!");
463 Register MULRegL = TRI->lookThruSingleUseCopyChain(
464 Root.getOperand(MulOpIdx).getReg(), MRI);
465 Register MULRegR = TRI->lookThruSingleUseCopyChain(
466 Root.getOperand(MulOpIdx + 1).getReg(), MRI);
467 if (!MULRegL && !MULRegR)
468 return false;
469
470 if (MULRegL && !MULRegR) {
471 MULRegR =
472 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
473 IsUsedOnceL = true;
474 } else if (!MULRegL && MULRegR) {
475 MULRegL =
476 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
477 IsUsedOnceR = true;
478 } else {
479 IsUsedOnceL = true;
480 IsUsedOnceR = true;
481 }
482
483 if (!MULRegL.isVirtual() || !MULRegR.isVirtual())
484 return false;
485
486 MULInstrL = MRI->getVRegDef(MULRegL);
487 MULInstrR = MRI->getVRegDef(MULRegR);
488 return true;
489 }
490 return false;
491 };
492
493 // Register pressure fma reassociation patterns.
494 if (DoRegPressureReduce && IsRPReductionCandidate()) {
495 assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
496 // Register pressure pattern 1
497 if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
498 IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
499 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
501 return true;
502 }
503
504 // Register pressure pattern 2
505 if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
506 IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
507 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
509 return true;
510 }
511 }
512
513 // ILP fma reassociation patterns.
514 // Root must be a valid FMA like instruction.
515 AddOpIdx = -1;
516 if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
517 return false;
518
519 assert((AddOpIdx >= 0) && "add operand index not right!");
520
521 Register RegB = Root.getOperand(AddOpIdx).getReg();
522 MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
523
524 // Prev must be a valid FMA like instruction.
525 AddOpIdx = -1;
526 if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
527 return false;
528
529 assert((AddOpIdx >= 0) && "add operand index not right!");
530
531 Register RegA = Prev->getOperand(AddOpIdx).getReg();
532 MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
533 AddOpIdx = -1;
534 if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
536 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
537 return true;
538 }
539 if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
541 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
542 return true;
543 }
544 return false;
545}
546
549 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
550 assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
551
552 MachineFunction *MF = Root.getMF();
556
557 int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
558 if (Idx < 0)
559 return;
560
562
563 // For now we only need to fix up placeholder for register pressure reduce
564 // patterns.
565 Register ConstReg = 0;
566 switch (P) {
568 ConstReg =
569 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
570 break;
572 ConstReg =
573 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
574 break;
575 default:
576 // Not register pressure reduce patterns.
577 return;
578 }
579
580 MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
581 // Get const value from const pool.
582 const Constant *C = getConstantFromConstantPool(ConstDefInstr);
583 assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
584
585 // Get negative fp const.
586 APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
587 F1.changeSign();
588 Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
589 Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
590
591 // Put negative fp const into constant pool.
592 unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
593
594 MachineOperand *Placeholder = nullptr;
595 // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
596 for (auto *Inst : InsInstrs) {
597 for (MachineOperand &Operand : Inst->explicit_operands()) {
598 assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
599 if (Operand.getReg() == PPC::ZERO8) {
600 Placeholder = &Operand;
601 break;
602 }
603 }
604 }
605
606 assert(Placeholder && "Placeholder does not exist!");
607
608 // Generate instructions to load the const fp from constant pool.
609 // We only support PPC64 and medium code model.
610 Register LoadNewConst =
611 generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
612
613 // Fill the placeholder with the new load from constant pool.
614 Placeholder->setReg(LoadNewConst);
615}
616
618 const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
619
621 return false;
622
623 // Currently, we only enable register pressure reducing in machine combiner
624 // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
625 // support.
626 //
627 // So we need following instructions to access a TOC entry:
628 //
629 // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
630 // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
631 // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
632 //
633 // FIXME: add more supported targets, like Small and Large code model, PPC32,
634 // AIX.
635 if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
637 return false;
638
640 const MachineFunction *MF = MBB->getParent();
641 const MachineRegisterInfo *MRI = &MF->getRegInfo();
642
643 auto GetMBBPressure =
644 [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
645 RegionPressure Pressure;
646 RegPressureTracker RPTracker(Pressure);
647
648 // Initialize the register pressure tracker.
649 RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
650 /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
651
652 for (const auto &MI : reverse(*MBB)) {
653 if (MI.isDebugValue() || MI.isDebugLabel())
654 continue;
655 RegisterOperands RegOpers;
656 RegOpers.collect(MI, *TRI, *MRI, false, false);
657 RPTracker.recedeSkipDebugValues();
658 assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
659 RPTracker.recede(RegOpers);
660 }
661
662 // Close the RPTracker to finalize live ins.
663 RPTracker.closeRegion();
664
665 return RPTracker.getPressure().MaxSetPressure;
666 };
667
668 // For now we only care about float and double type fma.
669 unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
670 *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
671
672 // Only reduce register pressure when pressure is high.
673 return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
674 (float)VSSRCLimit * FMARPFactor;
675}
676
677bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
678 // I has only one memory operand which is load from constant pool.
679 if (!I->hasOneMemOperand())
680 return false;
681
682 MachineMemOperand *Op = I->memoperands()[0];
683 return Op->isLoad() && Op->getPseudoValue() &&
684 Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
685}
686
687Register PPCInstrInfo::generateLoadForNewConst(
688 unsigned Idx, MachineInstr *MI, Type *Ty,
689 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
690 // Now we only support PPC64, Medium code model and P9 with vector.
691 // We have immutable pattern to access const pool. See function
692 // shouldReduceRegisterPressure.
693 assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
695 "Target not supported!\n");
696
697 MachineFunction *MF = MI->getMF();
699
700 // Generate ADDIStocHA8
701 Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
702 MachineInstrBuilder TOCOffset =
703 BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
704 .addReg(PPC::X2)
706
707 assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
708 "Only float and double are supported!");
709
710 unsigned LoadOpcode;
711 // Should be float type or double type.
712 if (Ty->isFloatTy())
713 LoadOpcode = PPC::DFLOADf32;
714 else
715 LoadOpcode = PPC::DFLOADf64;
716
717 const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
718 Register VReg2 = MRI->createVirtualRegister(RC);
722
723 // Generate Load from constant pool.
725 BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
727 .addReg(VReg1, getKillRegState(true))
728 .addMemOperand(MMO);
729
730 Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
731
732 // Insert the toc load instructions into InsInstrs.
733 InsInstrs.insert(InsInstrs.begin(), Load);
734 InsInstrs.insert(InsInstrs.begin(), TOCOffset);
735 return VReg2;
736}
737
738// This function returns the const value in constant pool if the \p I is a load
739// from constant pool.
740const Constant *
741PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
742 MachineFunction *MF = I->getMF();
745 assert(I->mayLoad() && "Should be a load instruction.\n");
746 for (auto MO : I->uses()) {
747 if (!MO.isReg())
748 continue;
749 Register Reg = MO.getReg();
750 if (Reg == 0 || !Reg.isVirtual())
751 continue;
752 // Find the toc address.
753 MachineInstr *DefMI = MRI->getVRegDef(Reg);
754 for (auto MO2 : DefMI->uses())
755 if (MO2.isCPI())
756 return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
757 }
758 return nullptr;
759}
760
763 bool DoRegPressureReduce) const {
764 // Using the machine combiner in this way is potentially expensive, so
765 // restrict to when aggressive optimizations are desired.
767 return false;
768
769 if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
770 return true;
771
773 DoRegPressureReduce);
774}
775
780 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
781 switch (Pattern) {
786 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
787 break;
788 default:
789 // Reassociate default patterns.
791 DelInstrs, InstrIdxForVirtReg);
792 break;
793 }
794}
795
796void PPCInstrInfo::reassociateFMA(
800 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
801 MachineFunction *MF = Root.getMF();
804 MachineOperand &OpC = Root.getOperand(0);
805 Register RegC = OpC.getReg();
806 const TargetRegisterClass *RC = MRI.getRegClass(RegC);
807 MRI.constrainRegClass(RegC, RC);
808
809 unsigned FmaOp = Root.getOpcode();
810 int16_t Idx = getFMAOpIdxInfo(FmaOp);
811 assert(Idx >= 0 && "Root must be a FMA instruction");
812
813 bool IsILPReassociate =
816
819
820 MachineInstr *Prev = nullptr;
821 MachineInstr *Leaf = nullptr;
822 switch (Pattern) {
823 default:
824 llvm_unreachable("not recognized pattern!");
827 Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
828 Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
829 break;
831 Register MULReg =
832 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
833 Leaf = MRI.getVRegDef(MULReg);
834 break;
835 }
837 Register MULReg = TRI->lookThruCopyLike(
838 Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
839 Leaf = MRI.getVRegDef(MULReg);
840 break;
841 }
842 }
843
844 uint16_t IntersectedFlags = 0;
845 if (IsILPReassociate)
846 IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
847 else
848 IntersectedFlags = Root.getFlags() & Leaf->getFlags();
849
850 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
851 bool &KillFlag) {
852 Reg = Operand.getReg();
853 MRI.constrainRegClass(Reg, RC);
854 KillFlag = Operand.isKill();
855 };
856
857 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
858 Register &MulOp2, Register &AddOp,
859 bool &MulOp1KillFlag, bool &MulOp2KillFlag,
860 bool &AddOpKillFlag) {
861 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
862 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
863 GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
864 };
865
866 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
867 RegA21, RegB;
868 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
869 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
870 KillA11 = false, KillA21 = false, KillB = false;
871
872 GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
873
874 if (IsILPReassociate)
875 GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
876
878 GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
879 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
881 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
882 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
883 } else {
884 // Get FSUB instruction info.
885 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
886 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
887 }
888
889 // Create new virtual registers for the new results instead of
890 // recycling legacy ones because the MachineCombiner's computation of the
891 // critical path requires a new register definition rather than an existing
892 // one.
893 // For register pressure reassociation, we only need create one virtual
894 // register for the new fma.
895 Register NewVRA = MRI.createVirtualRegister(RC);
896 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
897
898 Register NewVRB = 0;
899 if (IsILPReassociate) {
900 NewVRB = MRI.createVirtualRegister(RC);
901 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
902 }
903
904 Register NewVRD = 0;
906 NewVRD = MRI.createVirtualRegister(RC);
907 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
908 }
909
910 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
911 Register RegMul1, bool KillRegMul1,
912 Register RegMul2, bool KillRegMul2) {
913 MI->getOperand(AddOpIdx).setReg(RegAdd);
914 MI->getOperand(AddOpIdx).setIsKill(KillAdd);
915 MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
916 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
917 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
918 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
919 };
920
921 MachineInstrBuilder NewARegPressure, NewCRegPressure;
922 switch (Pattern) {
923 default:
924 llvm_unreachable("not recognized pattern!");
926 // Create new instructions for insertion.
927 MachineInstrBuilder MINewB =
928 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
929 .addReg(RegX, getKillRegState(KillX))
930 .addReg(RegM21, getKillRegState(KillM21))
931 .addReg(RegM22, getKillRegState(KillM22));
932 MachineInstrBuilder MINewA =
933 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
934 .addReg(RegY, getKillRegState(KillY))
935 .addReg(RegM31, getKillRegState(KillM31))
936 .addReg(RegM32, getKillRegState(KillM32));
937 // If AddOpIdx is not 1, adjust the order.
938 if (AddOpIdx != 1) {
939 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
940 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
941 }
942
943 MachineInstrBuilder MINewC =
944 BuildMI(*MF, Root.getDebugLoc(),
946 .addReg(NewVRB, getKillRegState(true))
947 .addReg(NewVRA, getKillRegState(true));
948
949 // Update flags for newly created instructions.
950 setSpecialOperandAttr(*MINewA, IntersectedFlags);
951 setSpecialOperandAttr(*MINewB, IntersectedFlags);
952 setSpecialOperandAttr(*MINewC, IntersectedFlags);
953
954 // Record new instructions for insertion.
955 InsInstrs.push_back(MINewA);
956 InsInstrs.push_back(MINewB);
957 InsInstrs.push_back(MINewC);
958 break;
959 }
961 assert(NewVRD && "new FMA register not created!");
962 // Create new instructions for insertion.
963 MachineInstrBuilder MINewA =
964 BuildMI(*MF, Leaf->getDebugLoc(),
966 .addReg(RegM11, getKillRegState(KillM11))
967 .addReg(RegM12, getKillRegState(KillM12));
968 MachineInstrBuilder MINewB =
969 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
970 .addReg(RegX, getKillRegState(KillX))
971 .addReg(RegM21, getKillRegState(KillM21))
972 .addReg(RegM22, getKillRegState(KillM22));
973 MachineInstrBuilder MINewD =
974 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
975 .addReg(NewVRA, getKillRegState(true))
976 .addReg(RegM31, getKillRegState(KillM31))
977 .addReg(RegM32, getKillRegState(KillM32));
978 // If AddOpIdx is not 1, adjust the order.
979 if (AddOpIdx != 1) {
980 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
981 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
982 KillM32);
983 }
984
985 MachineInstrBuilder MINewC =
986 BuildMI(*MF, Root.getDebugLoc(),
988 .addReg(NewVRB, getKillRegState(true))
989 .addReg(NewVRD, getKillRegState(true));
990
991 // Update flags for newly created instructions.
992 setSpecialOperandAttr(*MINewA, IntersectedFlags);
993 setSpecialOperandAttr(*MINewB, IntersectedFlags);
994 setSpecialOperandAttr(*MINewD, IntersectedFlags);
995 setSpecialOperandAttr(*MINewC, IntersectedFlags);
996
997 // Record new instructions for insertion.
998 InsInstrs.push_back(MINewA);
999 InsInstrs.push_back(MINewB);
1000 InsInstrs.push_back(MINewD);
1001 InsInstrs.push_back(MINewC);
1002 break;
1003 }
1006 Register VarReg;
1007 bool KillVarReg = false;
1009 VarReg = RegM31;
1010 KillVarReg = KillM31;
1011 } else {
1012 VarReg = RegM32;
1013 KillVarReg = KillM32;
1014 }
1015 // We don't want to get negative const from memory pool too early, as the
1016 // created entry will not be deleted even if it has no users. Since all
1017 // operand of Leaf and Root are virtual register, we use zero register
1018 // here as a placeholder. When the InsInstrs is selected in
1019 // MachineCombiner, we call finalizeInsInstrs to replace the zero register
1020 // with a virtual register which is a load from constant pool.
1021 NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
1022 .addReg(RegB, getKillRegState(RegB))
1023 .addReg(RegY, getKillRegState(KillY))
1024 .addReg(PPC::ZERO8);
1025 NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
1026 .addReg(NewVRA, getKillRegState(true))
1027 .addReg(RegX, getKillRegState(KillX))
1028 .addReg(VarReg, getKillRegState(KillVarReg));
1029 // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1030 // both at index 1, no need to adjust.
1031 // FIXME: when add more fma instructions support, like fma/fmas, adjust
1032 // the operand index here.
1033 break;
1034 }
1035 }
1036
1037 if (!IsILPReassociate) {
1038 setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
1039 setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
1040
1041 InsInstrs.push_back(NewARegPressure);
1042 InsInstrs.push_back(NewCRegPressure);
1043 }
1044
1045 assert(!InsInstrs.empty() &&
1046 "Insertion instructions set should not be empty!");
1047
1048 // Record old instructions for deletion.
1049 DelInstrs.push_back(Leaf);
1050 if (IsILPReassociate)
1051 DelInstrs.push_back(Prev);
1052 DelInstrs.push_back(&Root);
1053}
1054
1055// Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
1057 Register &SrcReg, Register &DstReg,
1058 unsigned &SubIdx) const {
1059 switch (MI.getOpcode()) {
1060 default: return false;
1061 case PPC::EXTSW:
1062 case PPC::EXTSW_32:
1063 case PPC::EXTSW_32_64:
1064 SrcReg = MI.getOperand(1).getReg();
1065 DstReg = MI.getOperand(0).getReg();
1066 SubIdx = PPC::sub_32;
1067 return true;
1068 }
1069}
1070
1072 int &FrameIndex) const {
1073 if (llvm::is_contained(getLoadOpcodesForSpillArray(), MI.getOpcode())) {
1074 // Check for the operands added by addFrameReference (the immediate is the
1075 // offset which defaults to 0).
1076 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1077 MI.getOperand(2).isFI()) {
1078 FrameIndex = MI.getOperand(2).getIndex();
1079 return MI.getOperand(0).getReg();
1080 }
1081 }
1082 return 0;
1083}
1084
1085// For opcodes with the ReMaterializable flag set, this function is called to
1086// verify the instruction is really rematable.
1088 const MachineInstr &MI) const {
1089 switch (MI.getOpcode()) {
1090 default:
1091 // This function should only be called for opcodes with the ReMaterializable
1092 // flag set.
1093 llvm_unreachable("Unknown rematerializable operation!");
1094 break;
1095 case PPC::LI:
1096 case PPC::LI8:
1097 case PPC::PLI:
1098 case PPC::PLI8:
1099 case PPC::LIS:
1100 case PPC::LIS8:
1101 case PPC::ADDIStocHA:
1102 case PPC::ADDIStocHA8:
1103 case PPC::ADDItocL:
1104 case PPC::LOAD_STACK_GUARD:
1105 case PPC::XXLXORz:
1106 case PPC::XXLXORspz:
1107 case PPC::XXLXORdpz:
1108 case PPC::XXLEQVOnes:
1109 case PPC::XXSPLTI32DX:
1110 case PPC::XXSPLTIW:
1111 case PPC::XXSPLTIDP:
1112 case PPC::V_SET0B:
1113 case PPC::V_SET0H:
1114 case PPC::V_SET0:
1115 case PPC::V_SETALLONESB:
1116 case PPC::V_SETALLONESH:
1117 case PPC::V_SETALLONES:
1118 case PPC::CRSET:
1119 case PPC::CRUNSET:
1120 case PPC::XXSETACCZ:
1121 case PPC::XXSETACCZW:
1122 return true;
1123 }
1124 return false;
1125}
1126
1128 int &FrameIndex) const {
1129 if (llvm::is_contained(getStoreOpcodesForSpillArray(), MI.getOpcode())) {
1130 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1131 MI.getOperand(2).isFI()) {
1132 FrameIndex = MI.getOperand(2).getIndex();
1133 return MI.getOperand(0).getReg();
1134 }
1135 }
1136 return 0;
1137}
1138
1140 unsigned OpIdx1,
1141 unsigned OpIdx2) const {
1142 MachineFunction &MF = *MI.getParent()->getParent();
1143
1144 // Normal instructions can be commuted the obvious way.
1145 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1146 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1147 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1148 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1149 // changing the relative order of the mask operands might change what happens
1150 // to the high-bits of the mask (and, thus, the result).
1151
1152 // Cannot commute if it has a non-zero rotate count.
1153 if (MI.getOperand(3).getImm() != 0)
1154 return nullptr;
1155
1156 // If we have a zero rotate count, we have:
1157 // M = mask(MB,ME)
1158 // Op0 = (Op1 & ~M) | (Op2 & M)
1159 // Change this to:
1160 // M = mask((ME+1)&31, (MB-1)&31)
1161 // Op0 = (Op2 & ~M) | (Op1 & M)
1162
1163 // Swap op1/op2
1164 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1165 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1166 Register Reg0 = MI.getOperand(0).getReg();
1167 Register Reg1 = MI.getOperand(1).getReg();
1168 Register Reg2 = MI.getOperand(2).getReg();
1169 unsigned SubReg1 = MI.getOperand(1).getSubReg();
1170 unsigned SubReg2 = MI.getOperand(2).getSubReg();
1171 bool Reg1IsKill = MI.getOperand(1).isKill();
1172 bool Reg2IsKill = MI.getOperand(2).isKill();
1173 bool ChangeReg0 = false;
1174 // If machine instrs are no longer in two-address forms, update
1175 // destination register as well.
1176 if (Reg0 == Reg1) {
1177 // Must be two address instruction!
1178 assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
1179 "Expecting a two-address instruction!");
1180 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1181 Reg2IsKill = false;
1182 ChangeReg0 = true;
1183 }
1184
1185 // Masks.
1186 unsigned MB = MI.getOperand(4).getImm();
1187 unsigned ME = MI.getOperand(5).getImm();
1188
1189 // We can't commute a trivial mask (there is no way to represent an all-zero
1190 // mask).
1191 if (MB == 0 && ME == 31)
1192 return nullptr;
1193
1194 if (NewMI) {
1195 // Create a new instruction.
1196 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
1197 bool Reg0IsDead = MI.getOperand(0).isDead();
1198 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
1199 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
1200 .addReg(Reg2, getKillRegState(Reg2IsKill))
1201 .addReg(Reg1, getKillRegState(Reg1IsKill))
1202 .addImm((ME + 1) & 31)
1203 .addImm((MB - 1) & 31);
1204 }
1205
1206 if (ChangeReg0) {
1207 MI.getOperand(0).setReg(Reg2);
1208 MI.getOperand(0).setSubReg(SubReg2);
1209 }
1210 MI.getOperand(2).setReg(Reg1);
1211 MI.getOperand(1).setReg(Reg2);
1212 MI.getOperand(2).setSubReg(SubReg1);
1213 MI.getOperand(1).setSubReg(SubReg2);
1214 MI.getOperand(2).setIsKill(Reg1IsKill);
1215 MI.getOperand(1).setIsKill(Reg2IsKill);
1216
1217 // Swap the mask around.
1218 MI.getOperand(4).setImm((ME + 1) & 31);
1219 MI.getOperand(5).setImm((MB - 1) & 31);
1220 return &MI;
1221}
1222
1224 unsigned &SrcOpIdx1,
1225 unsigned &SrcOpIdx2) const {
1226 // For VSX A-Type FMA instructions, it is the first two operands that can be
1227 // commuted, however, because the non-encoded tied input operand is listed
1228 // first, the operands to swap are actually the second and third.
1229
1230 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
1231 if (AltOpc == -1)
1232 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1233
1234 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1235 // and SrcOpIdx2.
1236 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
1237}
1238
1241 // This function is used for scheduling, and the nop wanted here is the type
1242 // that terminates dispatch groups on the POWER cores.
1243 unsigned Directive = Subtarget.getCPUDirective();
1244 unsigned Opcode;
1245 switch (Directive) {
1246 default: Opcode = PPC::NOP; break;
1247 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1248 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1249 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1250 // FIXME: Update when POWER9 scheduling model is ready.
1251 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1252 }
1253
1254 DebugLoc DL;
1255 BuildMI(MBB, MI, DL, get(Opcode));
1256}
1257
1258/// Return the noop instruction to use for a noop.
1260 MCInst Nop;
1261 Nop.setOpcode(PPC::NOP);
1262 return Nop;
1263}
1264
1265// Branch analysis.
1266// Note: If the condition register is set to CTR or CTR8 then this is a
1267// BDNZ (imm == 1) or BDZ (imm == 0) branch.
1270 MachineBasicBlock *&FBB,
1272 bool AllowModify) const {
1273 bool isPPC64 = Subtarget.isPPC64();
1274
1275 // If the block has no terminators, it just falls into the block after it.
1277 if (I == MBB.end())
1278 return false;
1279
1280 if (!isUnpredicatedTerminator(*I))
1281 return false;
1282
1283 if (AllowModify) {
1284 // If the BB ends with an unconditional branch to the fallthrough BB,
1285 // we eliminate the branch instruction.
1286 if (I->getOpcode() == PPC::B &&
1287 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
1288 I->eraseFromParent();
1289
1290 // We update iterator after deleting the last branch.
1292 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1293 return false;
1294 }
1295 }
1296
1297 // Get the last instruction in the block.
1298 MachineInstr &LastInst = *I;
1299
1300 // If there is only one terminator instruction, process it.
1301 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
1302 if (LastInst.getOpcode() == PPC::B) {
1303 if (!LastInst.getOperand(0).isMBB())
1304 return true;
1305 TBB = LastInst.getOperand(0).getMBB();
1306 return false;
1307 } else if (LastInst.getOpcode() == PPC::BCC) {
1308 if (!LastInst.getOperand(2).isMBB())
1309 return true;
1310 // Block ends with fall-through condbranch.
1311 TBB = LastInst.getOperand(2).getMBB();
1312 Cond.push_back(LastInst.getOperand(0));
1313 Cond.push_back(LastInst.getOperand(1));
1314 return false;
1315 } else if (LastInst.getOpcode() == PPC::BC) {
1316 if (!LastInst.getOperand(1).isMBB())
1317 return true;
1318 // Block ends with fall-through condbranch.
1319 TBB = LastInst.getOperand(1).getMBB();
1321 Cond.push_back(LastInst.getOperand(0));
1322 return false;
1323 } else if (LastInst.getOpcode() == PPC::BCn) {
1324 if (!LastInst.getOperand(1).isMBB())
1325 return true;
1326 // Block ends with fall-through condbranch.
1327 TBB = LastInst.getOperand(1).getMBB();
1329 Cond.push_back(LastInst.getOperand(0));
1330 return false;
1331 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1332 LastInst.getOpcode() == PPC::BDNZ) {
1333 if (!LastInst.getOperand(0).isMBB())
1334 return true;
1336 return true;
1337 TBB = LastInst.getOperand(0).getMBB();
1338 Cond.push_back(MachineOperand::CreateImm(1));
1339 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1340 true));
1341 return false;
1342 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1343 LastInst.getOpcode() == PPC::BDZ) {
1344 if (!LastInst.getOperand(0).isMBB())
1345 return true;
1347 return true;
1348 TBB = LastInst.getOperand(0).getMBB();
1349 Cond.push_back(MachineOperand::CreateImm(0));
1350 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1351 true));
1352 return false;
1353 }
1354
1355 // Otherwise, don't know what this is.
1356 return true;
1357 }
1358
1359 // Get the instruction before it if it's a terminator.
1360 MachineInstr &SecondLastInst = *I;
1361
1362 // If there are three terminators, we don't know what sort of block this is.
1363 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
1364 return true;
1365
1366 // If the block ends with PPC::B and PPC:BCC, handle it.
1367 if (SecondLastInst.getOpcode() == PPC::BCC &&
1368 LastInst.getOpcode() == PPC::B) {
1369 if (!SecondLastInst.getOperand(2).isMBB() ||
1370 !LastInst.getOperand(0).isMBB())
1371 return true;
1372 TBB = SecondLastInst.getOperand(2).getMBB();
1373 Cond.push_back(SecondLastInst.getOperand(0));
1374 Cond.push_back(SecondLastInst.getOperand(1));
1375 FBB = LastInst.getOperand(0).getMBB();
1376 return false;
1377 } else if (SecondLastInst.getOpcode() == PPC::BC &&
1378 LastInst.getOpcode() == PPC::B) {
1379 if (!SecondLastInst.getOperand(1).isMBB() ||
1380 !LastInst.getOperand(0).isMBB())
1381 return true;
1382 TBB = SecondLastInst.getOperand(1).getMBB();
1384 Cond.push_back(SecondLastInst.getOperand(0));
1385 FBB = LastInst.getOperand(0).getMBB();
1386 return false;
1387 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1388 LastInst.getOpcode() == PPC::B) {
1389 if (!SecondLastInst.getOperand(1).isMBB() ||
1390 !LastInst.getOperand(0).isMBB())
1391 return true;
1392 TBB = SecondLastInst.getOperand(1).getMBB();
1394 Cond.push_back(SecondLastInst.getOperand(0));
1395 FBB = LastInst.getOperand(0).getMBB();
1396 return false;
1397 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1398 SecondLastInst.getOpcode() == PPC::BDNZ) &&
1399 LastInst.getOpcode() == PPC::B) {
1400 if (!SecondLastInst.getOperand(0).isMBB() ||
1401 !LastInst.getOperand(0).isMBB())
1402 return true;
1404 return true;
1405 TBB = SecondLastInst.getOperand(0).getMBB();
1406 Cond.push_back(MachineOperand::CreateImm(1));
1407 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1408 true));
1409 FBB = LastInst.getOperand(0).getMBB();
1410 return false;
1411 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1412 SecondLastInst.getOpcode() == PPC::BDZ) &&
1413 LastInst.getOpcode() == PPC::B) {
1414 if (!SecondLastInst.getOperand(0).isMBB() ||
1415 !LastInst.getOperand(0).isMBB())
1416 return true;
1418 return true;
1419 TBB = SecondLastInst.getOperand(0).getMBB();
1420 Cond.push_back(MachineOperand::CreateImm(0));
1421 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1422 true));
1423 FBB = LastInst.getOperand(0).getMBB();
1424 return false;
1425 }
1426
1427 // If the block ends with two PPC:Bs, handle it. The second one is not
1428 // executed, so remove it.
1429 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1430 if (!SecondLastInst.getOperand(0).isMBB())
1431 return true;
1432 TBB = SecondLastInst.getOperand(0).getMBB();
1433 I = LastInst;
1434 if (AllowModify)
1435 I->eraseFromParent();
1436 return false;
1437 }
1438
1439 // Otherwise, can't handle this.
1440 return true;
1441}
1442
1444 int *BytesRemoved) const {
1445 assert(!BytesRemoved && "code size not handled");
1446
1448 if (I == MBB.end())
1449 return 0;
1450
1451 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1452 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1453 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1454 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1455 return 0;
1456
1457 // Remove the branch.
1458 I->eraseFromParent();
1459
1460 I = MBB.end();
1461
1462 if (I == MBB.begin()) return 1;
1463 --I;
1464 if (I->getOpcode() != PPC::BCC &&
1465 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1466 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1467 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1468 return 1;
1469
1470 // Remove the branch.
1471 I->eraseFromParent();
1472 return 2;
1473}
1474
1477 MachineBasicBlock *FBB,
1479 const DebugLoc &DL,
1480 int *BytesAdded) const {
1481 // Shouldn't be a fall through.
1482 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1483 assert((Cond.size() == 2 || Cond.size() == 0) &&
1484 "PPC branch conditions have two components!");
1485 assert(!BytesAdded && "code size not handled");
1486
1487 bool isPPC64 = Subtarget.isPPC64();
1488
1489 // One-way branch.
1490 if (!FBB) {
1491 if (Cond.empty()) // Unconditional branch
1492 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1493 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1494 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1495 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1496 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1497 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1498 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1499 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1500 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1501 else // Conditional branch
1502 BuildMI(&MBB, DL, get(PPC::BCC))
1503 .addImm(Cond[0].getImm())
1504 .add(Cond[1])
1505 .addMBB(TBB);
1506 return 1;
1507 }
1508
1509 // Two-way Conditional Branch.
1510 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1511 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1512 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1513 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1514 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1515 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1516 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1517 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1518 else
1519 BuildMI(&MBB, DL, get(PPC::BCC))
1520 .addImm(Cond[0].getImm())
1521 .add(Cond[1])
1522 .addMBB(TBB);
1523 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1524 return 2;
1525}
1526
1527// Select analysis.
1530 Register DstReg, Register TrueReg,
1531 Register FalseReg, int &CondCycles,
1532 int &TrueCycles, int &FalseCycles) const {
1533 if (Cond.size() != 2)
1534 return false;
1535
1536 // If this is really a bdnz-like condition, then it cannot be turned into a
1537 // select.
1538 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1539 return false;
1540
1541 // If the conditional branch uses a physical register, then it cannot be
1542 // turned into a select.
1543 if (Cond[1].getReg().isPhysical())
1544 return false;
1545
1546 // Check register classes.
1548 const TargetRegisterClass *RC =
1549 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1550 if (!RC)
1551 return false;
1552
1553 // isel is for regular integer GPRs only.
1554 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1555 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1556 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1557 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1558 return false;
1559
1560 // FIXME: These numbers are for the A2, how well they work for other cores is
1561 // an open question. On the A2, the isel instruction has a 2-cycle latency
1562 // but single-cycle throughput. These numbers are used in combination with
1563 // the MispredictPenalty setting from the active SchedMachineModel.
1564 CondCycles = 1;
1565 TrueCycles = 1;
1566 FalseCycles = 1;
1567
1568 return true;
1569}
1570
1573 const DebugLoc &dl, Register DestReg,
1575 Register FalseReg) const {
1576 assert(Cond.size() == 2 &&
1577 "PPC branch conditions have two components!");
1578
1579 // Get the register classes.
1581 const TargetRegisterClass *RC =
1582 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1583 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1584
1585 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1586 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1587 assert((Is64Bit ||
1588 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1589 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1590 "isel is for regular integer GPRs only");
1591
1592 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1593 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1594
1595 unsigned SubIdx = 0;
1596 bool SwapOps = false;
1597 switch (SelectPred) {
1598 case PPC::PRED_EQ:
1599 case PPC::PRED_EQ_MINUS:
1600 case PPC::PRED_EQ_PLUS:
1601 SubIdx = PPC::sub_eq; SwapOps = false; break;
1602 case PPC::PRED_NE:
1603 case PPC::PRED_NE_MINUS:
1604 case PPC::PRED_NE_PLUS:
1605 SubIdx = PPC::sub_eq; SwapOps = true; break;
1606 case PPC::PRED_LT:
1607 case PPC::PRED_LT_MINUS:
1608 case PPC::PRED_LT_PLUS:
1609 SubIdx = PPC::sub_lt; SwapOps = false; break;
1610 case PPC::PRED_GE:
1611 case PPC::PRED_GE_MINUS:
1612 case PPC::PRED_GE_PLUS:
1613 SubIdx = PPC::sub_lt; SwapOps = true; break;
1614 case PPC::PRED_GT:
1615 case PPC::PRED_GT_MINUS:
1616 case PPC::PRED_GT_PLUS:
1617 SubIdx = PPC::sub_gt; SwapOps = false; break;
1618 case PPC::PRED_LE:
1619 case PPC::PRED_LE_MINUS:
1620 case PPC::PRED_LE_PLUS:
1621 SubIdx = PPC::sub_gt; SwapOps = true; break;
1622 case PPC::PRED_UN:
1623 case PPC::PRED_UN_MINUS:
1624 case PPC::PRED_UN_PLUS:
1625 SubIdx = PPC::sub_un; SwapOps = false; break;
1626 case PPC::PRED_NU:
1627 case PPC::PRED_NU_MINUS:
1628 case PPC::PRED_NU_PLUS:
1629 SubIdx = PPC::sub_un; SwapOps = true; break;
1630 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1631 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1632 }
1633
1634 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1635 SecondReg = SwapOps ? TrueReg : FalseReg;
1636
1637 // The first input register of isel cannot be r0. If it is a member
1638 // of a register class that can be r0, then copy it first (the
1639 // register allocator should eliminate the copy).
1640 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1641 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1642 const TargetRegisterClass *FirstRC =
1643 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1644 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1645 Register OldFirstReg = FirstReg;
1646 FirstReg = MRI.createVirtualRegister(FirstRC);
1647 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1648 .addReg(OldFirstReg);
1649 }
1650
1651 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1652 .addReg(FirstReg).addReg(SecondReg)
1653 .addReg(Cond[1].getReg(), 0, SubIdx);
1654}
1655
1656static unsigned getCRBitValue(unsigned CRBit) {
1657 unsigned Ret = 4;
1658 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1659 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1660 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1661 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1662 Ret = 3;
1663 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1664 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1665 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1666 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1667 Ret = 2;
1668 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1669 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1670 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1671 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1672 Ret = 1;
1673 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1674 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1675 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1676 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1677 Ret = 0;
1678
1679 assert(Ret != 4 && "Invalid CR bit register");
1680 return Ret;
1681}
1682
1685 const DebugLoc &DL, MCRegister DestReg,
1686 MCRegister SrcReg, bool KillSrc) const {
1687 // We can end up with self copies and similar things as a result of VSX copy
1688 // legalization. Promote them here.
1690 if (PPC::F8RCRegClass.contains(DestReg) &&
1691 PPC::VSRCRegClass.contains(SrcReg)) {
1692 MCRegister SuperReg =
1693 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1694
1695 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1696 llvm_unreachable("nop VSX copy");
1697
1698 DestReg = SuperReg;
1699 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1700 PPC::VSRCRegClass.contains(DestReg)) {
1701 MCRegister SuperReg =
1702 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1703
1704 if (VSXSelfCopyCrash && DestReg == SuperReg)
1705 llvm_unreachable("nop VSX copy");
1706
1707 SrcReg = SuperReg;
1708 }
1709
1710 // Different class register copy
1711 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1712 PPC::GPRCRegClass.contains(DestReg)) {
1713 MCRegister CRReg = getCRFromCRBit(SrcReg);
1714 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1715 getKillRegState(KillSrc);
1716 // Rotate the CR bit in the CR fields to be the least significant bit and
1717 // then mask with 0x1 (MB = ME = 31).
1718 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1719 .addReg(DestReg, RegState::Kill)
1720 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1721 .addImm(31)
1722 .addImm(31);
1723 return;
1724 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1725 (PPC::G8RCRegClass.contains(DestReg) ||
1726 PPC::GPRCRegClass.contains(DestReg))) {
1727 bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
1728 unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1729 unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1730 unsigned CRNum = TRI->getEncodingValue(SrcReg);
1731 BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
1732 getKillRegState(KillSrc);
1733 if (CRNum == 7)
1734 return;
1735 // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1736 BuildMI(MBB, I, DL, get(ShCode), DestReg)
1737 .addReg(DestReg, RegState::Kill)
1738 .addImm(CRNum * 4 + 4)
1739 .addImm(28)
1740 .addImm(31);
1741 return;
1742 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1743 PPC::VSFRCRegClass.contains(DestReg)) {
1744 assert(Subtarget.hasDirectMove() &&
1745 "Subtarget doesn't support directmove, don't know how to copy.");
1746 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1747 NumGPRtoVSRSpill++;
1748 getKillRegState(KillSrc);
1749 return;
1750 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1751 PPC::G8RCRegClass.contains(DestReg)) {
1752 assert(Subtarget.hasDirectMove() &&
1753 "Subtarget doesn't support directmove, don't know how to copy.");
1754 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1755 getKillRegState(KillSrc);
1756 return;
1757 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1758 PPC::GPRCRegClass.contains(DestReg)) {
1759 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1760 getKillRegState(KillSrc);
1761 return;
1762 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1763 PPC::SPERCRegClass.contains(DestReg)) {
1764 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1765 getKillRegState(KillSrc);
1766 return;
1767 }
1768
1769 unsigned Opc;
1770 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1771 Opc = PPC::OR;
1772 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1773 Opc = PPC::OR8;
1774 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1775 Opc = PPC::FMR;
1776 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1777 Opc = PPC::MCRF;
1778 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1779 Opc = PPC::VOR;
1780 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1781 // There are two different ways this can be done:
1782 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1783 // issue in VSU pipeline 0.
1784 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1785 // can go to either pipeline.
1786 // We'll always use xxlor here, because in practically all cases where
1787 // copies are generated, they are close enough to some use that the
1788 // lower-latency form is preferable.
1789 Opc = PPC::XXLOR;
1790 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1791 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1792 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1793 else if (Subtarget.pairedVectorMemops() &&
1794 PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
1795 if (SrcReg > PPC::VSRp15)
1796 SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1797 else
1798 SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1799 if (DestReg > PPC::VSRp15)
1800 DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1801 else
1802 DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1803 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
1804 addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1805 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
1806 addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
1807 return;
1808 }
1809 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1810 Opc = PPC::CROR;
1811 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1812 Opc = PPC::EVOR;
1813 else if ((PPC::ACCRCRegClass.contains(DestReg) ||
1814 PPC::UACCRCRegClass.contains(DestReg)) &&
1815 (PPC::ACCRCRegClass.contains(SrcReg) ||
1816 PPC::UACCRCRegClass.contains(SrcReg))) {
1817 // If primed, de-prime the source register, copy the individual registers
1818 // and prime the destination if needed. The vector subregisters are
1819 // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1820 // source is primed, we need to re-prime it after the copy as well.
1821 PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1822 bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
1823 bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
1824 MCRegister VSLSrcReg =
1825 PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1826 MCRegister VSLDestReg =
1827 PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1828 if (SrcPrimed)
1829 BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
1830 for (unsigned Idx = 0; Idx < 4; Idx++)
1831 BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
1832 .addReg(VSLSrcReg + Idx)
1833 .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
1834 if (DestPrimed)
1835 BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
1836 if (SrcPrimed && !KillSrc)
1837 BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
1838 return;
1839 } else if (PPC::G8pRCRegClass.contains(DestReg) &&
1840 PPC::G8pRCRegClass.contains(SrcReg)) {
1841 // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1842 unsigned DestRegIdx = DestReg - PPC::G8p0;
1843 MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1844 MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1845 unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1846 MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1847 MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1848 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
1849 .addReg(SrcRegSub0)
1850 .addReg(SrcRegSub0, getKillRegState(KillSrc));
1851 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
1852 .addReg(SrcRegSub1)
1853 .addReg(SrcRegSub1, getKillRegState(KillSrc));
1854 return;
1855 } else
1856 llvm_unreachable("Impossible reg-to-reg copy");
1857
1858 const MCInstrDesc &MCID = get(Opc);
1859 if (MCID.getNumOperands() == 3)
1860 BuildMI(MBB, I, DL, MCID, DestReg)
1861 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1862 else
1863 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1864}
1865
1866unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1867 int OpcodeIndex = 0;
1868
1869 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1870 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1872 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1873 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1875 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1877 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1879 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1881 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1883 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1885 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1887 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1889 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1891 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1893 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1895 } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1896 assert(Subtarget.pairedVectorMemops() &&
1897 "Register unexpected when paired memops are disabled.");
1899 } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1900 assert(Subtarget.pairedVectorMemops() &&
1901 "Register unexpected when paired memops are disabled.");
1903 } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
1904 assert(Subtarget.pairedVectorMemops() &&
1905 "Register unexpected when paired memops are disabled.");
1907 } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1908 assert(Subtarget.pairedVectorMemops() &&
1909 "Register unexpected when paired memops are disabled.");
1911 } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1913 } else {
1914 llvm_unreachable("Unknown regclass!");
1915 }
1916 return OpcodeIndex;
1917}
1918
1919unsigned
1921 ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
1922 return OpcodesForSpill[getSpillIndex(RC)];
1923}
1924
1925unsigned
1927 ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
1928 return OpcodesForSpill[getSpillIndex(RC)];
1929}
1930
1931void PPCInstrInfo::StoreRegToStackSlot(
1932 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1933 const TargetRegisterClass *RC,
1934 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1935 unsigned Opcode = getStoreOpcodeForSpill(RC);
1936 DebugLoc DL;
1937
1938 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1939 FuncInfo->setHasSpills();
1940
1942 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1943 FrameIdx));
1944
1945 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1946 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1947 FuncInfo->setSpillsCR();
1948
1949 if (isXFormMemOp(Opcode))
1950 FuncInfo->setHasNonRISpills();
1951}
1952
1955 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1956 const TargetRegisterInfo *TRI) const {
1957 MachineFunction &MF = *MBB.getParent();
1959
1960 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1961
1962 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1963 MBB.insert(MI, NewMIs[i]);
1964
1965 const MachineFrameInfo &MFI = MF.getFrameInfo();
1969 MFI.getObjectAlign(FrameIdx));
1970 NewMIs.back()->addMemOperand(MF, MMO);
1971}
1972
1975 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1976 const TargetRegisterInfo *TRI, Register VReg) const {
1977 // We need to avoid a situation in which the value from a VRRC register is
1978 // spilled using an Altivec instruction and reloaded into a VSRC register
1979 // using a VSX instruction. The issue with this is that the VSX
1980 // load/store instructions swap the doublewords in the vector and the Altivec
1981 // ones don't. The register classes on the spill/reload may be different if
1982 // the register is defined using an Altivec instruction and is then used by a
1983 // VSX instruction.
1984 RC = updatedRC(RC);
1985 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1986}
1987
1988void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1989 unsigned DestReg, int FrameIdx,
1990 const TargetRegisterClass *RC,
1992 const {
1993 unsigned Opcode = getLoadOpcodeForSpill(RC);
1994 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1995 FrameIdx));
1996 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1997
1998 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1999 PPC::CRBITRCRegClass.hasSubClassEq(RC))
2000 FuncInfo->setSpillsCR();
2001
2002 if (isXFormMemOp(Opcode))
2003 FuncInfo->setHasNonRISpills();
2004}
2005
2008 int FrameIdx, const TargetRegisterClass *RC,
2009 const TargetRegisterInfo *TRI) const {
2010 MachineFunction &MF = *MBB.getParent();
2012 DebugLoc DL;
2013 if (MI != MBB.end()) DL = MI->getDebugLoc();
2014
2015 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2016 FuncInfo->setHasSpills();
2017
2018 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
2019
2020 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
2021 MBB.insert(MI, NewMIs[i]);
2022
2023 const MachineFrameInfo &MFI = MF.getFrameInfo();
2027 MFI.getObjectAlign(FrameIdx));
2028 NewMIs.back()->addMemOperand(MF, MMO);
2029}
2030
2033 Register DestReg, int FrameIdx,
2034 const TargetRegisterClass *RC,
2035 const TargetRegisterInfo *TRI,
2036 Register VReg) const {
2037 // We need to avoid a situation in which the value from a VRRC register is
2038 // spilled using an Altivec instruction and reloaded into a VSRC register
2039 // using a VSX instruction. The issue with this is that the VSX
2040 // load/store instructions swap the doublewords in the vector and the Altivec
2041 // ones don't. The register classes on the spill/reload may be different if
2042 // the register is defined using an Altivec instruction and is then used by a
2043 // VSX instruction.
2044 RC = updatedRC(RC);
2045
2046 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
2047}
2048
2051 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2052 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2053 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2054 else
2055 // Leave the CR# the same, but invert the condition.
2056 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
2057 return false;
2058}
2059
2060// For some instructions, it is legal to fold ZERO into the RA register field.
2061// This function performs that fold by replacing the operand with PPC::ZERO,
2062// it does not consider whether the load immediate zero is no longer in use.
2064 Register Reg) const {
2065 // A zero immediate should always be loaded with a single li.
2066 unsigned DefOpc = DefMI.getOpcode();
2067 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2068 return false;
2069 if (!DefMI.getOperand(1).isImm())
2070 return false;
2071 if (DefMI.getOperand(1).getImm() != 0)
2072 return false;
2073
2074 // Note that we cannot here invert the arguments of an isel in order to fold
2075 // a ZERO into what is presented as the second argument. All we have here
2076 // is the condition bit, and that might come from a CR-logical bit operation.
2077
2078 const MCInstrDesc &UseMCID = UseMI.getDesc();
2079
2080 // Only fold into real machine instructions.
2081 if (UseMCID.isPseudo())
2082 return false;
2083
2084 // We need to find which of the User's operands is to be folded, that will be
2085 // the operand that matches the given register ID.
2086 unsigned UseIdx;
2087 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2088 if (UseMI.getOperand(UseIdx).isReg() &&
2089 UseMI.getOperand(UseIdx).getReg() == Reg)
2090 break;
2091
2092 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2093 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2094
2095 const MCOperandInfo *UseInfo = &UseMCID.operands()[UseIdx];
2096
2097 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2098 // register (which might also be specified as a pointer class kind).
2099 if (UseInfo->isLookupPtrRegClass()) {
2100 if (UseInfo->RegClass /* Kind */ != 1)
2101 return false;
2102 } else {
2103 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
2104 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
2105 return false;
2106 }
2107
2108 // Make sure this is not tied to an output register (or otherwise
2109 // constrained). This is true for ST?UX registers, for example, which
2110 // are tied to their output registers.
2111 if (UseInfo->Constraints != 0)
2112 return false;
2113
2114 MCRegister ZeroReg;
2115 if (UseInfo->isLookupPtrRegClass()) {
2116 bool isPPC64 = Subtarget.isPPC64();
2117 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
2118 } else {
2119 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
2120 PPC::ZERO8 : PPC::ZERO;
2121 }
2122
2123 LLVM_DEBUG(dbgs() << "Folded immediate zero for: ");
2124 LLVM_DEBUG(UseMI.dump());
2125 UseMI.getOperand(UseIdx).setReg(ZeroReg);
2126 LLVM_DEBUG(dbgs() << "Into: ");
2127 LLVM_DEBUG(UseMI.dump());
2128 return true;
2129}
2130
2131// Folds zero into instructions which have a load immediate zero as an operand
2132// but also recognize zero as immediate zero. If the definition of the load
2133// has no more users it is deleted.
2135 Register Reg, MachineRegisterInfo *MRI) const {
2136 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2137 if (MRI->use_nodbg_empty(Reg))
2138 DefMI.eraseFromParent();
2139 return Changed;
2140}
2141
2143 for (MachineInstr &MI : MBB)
2144 if (MI.definesRegister(PPC::CTR) || MI.definesRegister(PPC::CTR8))
2145 return true;
2146 return false;
2147}
2148
2149// We should make sure that, if we're going to predicate both sides of a
2150// condition (a diamond), that both sides don't define the counter register. We
2151// can predicate counter-decrement-based branches, but while that predicates
2152// the branching, it does not predicate the counter decrement. If we tried to
2153// merge the triangle into one predicated block, we'd decrement the counter
2154// twice.
2156 unsigned NumT, unsigned ExtraT,
2157 MachineBasicBlock &FMBB,
2158 unsigned NumF, unsigned ExtraF,
2159 BranchProbability Probability) const {
2160 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
2161}
2162
2163
2165 // The predicated branches are identified by their type, not really by the
2166 // explicit presence of a predicate. Furthermore, some of them can be
2167 // predicated more than once. Because if conversion won't try to predicate
2168 // any instruction which already claims to be predicated (by returning true
2169 // here), always return false. In doing so, we let isPredicable() be the
2170 // final word on whether not the instruction can be (further) predicated.
2171
2172 return false;
2173}
2174
2176 const MachineBasicBlock *MBB,
2177 const MachineFunction &MF) const {
2178 // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2179 // across them, since some FP operations may change content of FPSCR.
2180 // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2181 if (MI.getOpcode() == PPC::MFFS || MI.getOpcode() == PPC::MTFSF)
2182 return true;
2184}
2185
2187 ArrayRef<MachineOperand> Pred) const {
2188 unsigned OpC = MI.getOpcode();
2189 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2190 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2191 bool isPPC64 = Subtarget.isPPC64();
2192 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2193 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2194 // Need add Def and Use for CTR implicit operand.
2195 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2196 .addReg(Pred[1].getReg(), RegState::Implicit)
2198 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2199 MI.setDesc(get(PPC::BCLR));
2200 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2201 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2202 MI.setDesc(get(PPC::BCLRn));
2203 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2204 } else {
2205 MI.setDesc(get(PPC::BCCLR));
2206 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2207 .addImm(Pred[0].getImm())
2208 .add(Pred[1]);
2209 }
2210
2211 return true;
2212 } else if (OpC == PPC::B) {
2213 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2214 bool isPPC64 = Subtarget.isPPC64();
2215 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2216 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2217 // Need add Def and Use for CTR implicit operand.
2218 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2219 .addReg(Pred[1].getReg(), RegState::Implicit)
2221 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2222 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2223 MI.removeOperand(0);
2224
2225 MI.setDesc(get(PPC::BC));
2226 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2227 .add(Pred[1])
2228 .addMBB(MBB);
2229 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2230 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2231 MI.removeOperand(0);
2232
2233 MI.setDesc(get(PPC::BCn));
2234 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2235 .add(Pred[1])
2236 .addMBB(MBB);
2237 } else {
2238 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2239 MI.removeOperand(0);
2240
2241 MI.setDesc(get(PPC::BCC));
2242 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2243 .addImm(Pred[0].getImm())
2244 .add(Pred[1])
2245 .addMBB(MBB);
2246 }
2247
2248 return true;
2249 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2250 OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2251 OpC == PPC::BCTRL8_RM) {
2252 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2253 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2254
2255 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2256 OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2257 bool isPPC64 = Subtarget.isPPC64();
2258
2259 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2260 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2261 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2262 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2263 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2264 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2265 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2266 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2267 } else {
2268 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2269 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2270 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2271 .addImm(Pred[0].getImm())
2272 .add(Pred[1]);
2273 }
2274
2275 // Need add Def and Use for LR implicit operand.
2276 if (setLR)
2277 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2278 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
2279 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
2280 if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2281 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2283
2284 return true;
2285 }
2286
2287 return false;
2288}
2289
2291 ArrayRef<MachineOperand> Pred2) const {
2292 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2293 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2294
2295 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2296 return false;
2297 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2298 return false;
2299
2300 // P1 can only subsume P2 if they test the same condition register.
2301 if (Pred1[1].getReg() != Pred2[1].getReg())
2302 return false;
2303
2304 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2305 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2306
2307 if (P1 == P2)
2308 return true;
2309
2310 // Does P1 subsume P2, e.g. GE subsumes GT.
2311 if (P1 == PPC::PRED_LE &&
2312 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2313 return true;
2314 if (P1 == PPC::PRED_GE &&
2315 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2316 return true;
2317
2318 return false;
2319}
2320
2322 std::vector<MachineOperand> &Pred,
2323 bool SkipDead) const {
2324 // Note: At the present time, the contents of Pred from this function is
2325 // unused by IfConversion. This implementation follows ARM by pushing the
2326 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2327 // predicate, instructions defining CTR or CTR8 are also included as
2328 // predicate-defining instructions.
2329
2330 const TargetRegisterClass *RCs[] =
2331 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2332 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2333
2334 bool Found = false;
2335 for (const MachineOperand &MO : MI.operands()) {
2336 for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
2337 const TargetRegisterClass *RC = RCs[c];
2338 if (MO.isReg()) {
2339 if (MO.isDef() && RC->contains(MO.getReg())) {
2340 Pred.push_back(MO);
2341 Found = true;
2342 }
2343 } else if (MO.isRegMask()) {
2344 for (MCPhysReg R : *RC)
2345 if (MO.clobbersPhysReg(R)) {
2346 Pred.push_back(MO);
2347 Found = true;
2348 }
2349 }
2350 }
2351 }
2352
2353 return Found;
2354}
2355
2357 Register &SrcReg2, int64_t &Mask,
2358 int64_t &Value) const {
2359 unsigned Opc = MI.getOpcode();
2360
2361 switch (Opc) {
2362 default: return false;
2363 case PPC::CMPWI:
2364 case PPC::CMPLWI:
2365 case PPC::CMPDI:
2366 case PPC::CMPLDI:
2367 SrcReg = MI.getOperand(1).getReg();
2368 SrcReg2 = 0;
2369 Value = MI.getOperand(2).getImm();
2370 Mask = 0xFFFF;
2371 return true;
2372 case PPC::CMPW:
2373 case PPC::CMPLW:
2374 case PPC::CMPD:
2375 case PPC::CMPLD:
2376 case PPC::FCMPUS:
2377 case PPC::FCMPUD:
2378 SrcReg = MI.getOperand(1).getReg();
2379 SrcReg2 = MI.getOperand(2).getReg();
2380 Value = 0;
2381 Mask = 0;
2382 return true;
2383 }
2384}
2385
2387 Register SrcReg2, int64_t Mask,
2388 int64_t Value,
2389 const MachineRegisterInfo *MRI) const {
2390 if (DisableCmpOpt)
2391 return false;
2392
2393 int OpC = CmpInstr.getOpcode();
2394 Register CRReg = CmpInstr.getOperand(0).getReg();
2395
2396 // FP record forms set CR1 based on the exception status bits, not a
2397 // comparison with zero.
2398 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2399 return false;
2400
2402 // The record forms set the condition register based on a signed comparison
2403 // with zero (so says the ISA manual). This is not as straightforward as it
2404 // seems, however, because this is always a 64-bit comparison on PPC64, even
2405 // for instructions that are 32-bit in nature (like slw for example).
2406 // So, on PPC32, for unsigned comparisons, we can use the record forms only
2407 // for equality checks (as those don't depend on the sign). On PPC64,
2408 // we are restricted to equality for unsigned 64-bit comparisons and for
2409 // signed 32-bit comparisons the applicability is more restricted.
2410 bool isPPC64 = Subtarget.isPPC64();
2411 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2412 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2413 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2414
2415 // Look through copies unless that gets us to a physical register.
2416 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
2417 if (ActualSrc.isVirtual())
2418 SrcReg = ActualSrc;
2419
2420 // Get the unique definition of SrcReg.
2421 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2422 if (!MI) return false;
2423
2424 bool equalityOnly = false;
2425 bool noSub = false;
2426 if (isPPC64) {
2427 if (is32BitSignedCompare) {
2428 // We can perform this optimization only if SrcReg is sign-extending.
2429 if (isSignExtended(SrcReg, MRI))
2430 noSub = true;
2431 else
2432 return false;
2433 } else if (is32BitUnsignedCompare) {
2434 // We can perform this optimization, equality only, if SrcReg is
2435 // zero-extending.
2436 if (isZeroExtended(SrcReg, MRI)) {
2437 noSub = true;
2438 equalityOnly = true;
2439 } else
2440 return false;
2441 } else
2442 equalityOnly = is64BitUnsignedCompare;
2443 } else
2444 equalityOnly = is32BitUnsignedCompare;
2445
2446 if (equalityOnly) {
2447 // We need to check the uses of the condition register in order to reject
2448 // non-equality comparisons.
2450 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2451 I != IE; ++I) {
2452 MachineInstr *UseMI = &*I;
2453 if (UseMI->getOpcode() == PPC::BCC) {
2455 unsigned PredCond = PPC::getPredicateCondition(Pred);
2456 // We ignore hint bits when checking for non-equality comparisons.
2457 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2458 return false;
2459 } else if (UseMI->getOpcode() == PPC::ISEL ||
2460 UseMI->getOpcode() == PPC::ISEL8) {
2461 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
2462 if (SubIdx != PPC::sub_eq)
2463 return false;
2464 } else
2465 return false;
2466 }
2467 }
2468
2469 MachineBasicBlock::iterator I = CmpInstr;
2470
2471 // Scan forward to find the first use of the compare.
2472 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2473 ++I) {
2474 bool FoundUse = false;
2476 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
2477 J != JE; ++J)
2478 if (&*J == &*I) {
2479 FoundUse = true;
2480 break;
2481 }
2482
2483 if (FoundUse)
2484 break;
2485 }
2486
2489
2490 // There are two possible candidates which can be changed to set CR[01].
2491 // One is MI, the other is a SUB instruction.
2492 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2493 MachineInstr *Sub = nullptr;
2494 if (SrcReg2 != 0)
2495 // MI is not a candidate for CMPrr.
2496 MI = nullptr;
2497 // FIXME: Conservatively refuse to convert an instruction which isn't in the
2498 // same BB as the comparison. This is to allow the check below to avoid calls
2499 // (and other explicit clobbers); instead we should really check for these
2500 // more explicitly (in at least a few predecessors).
2501 else if (MI->getParent() != CmpInstr.getParent())
2502 return false;
2503 else if (Value != 0) {
2504 // The record-form instructions set CR bit based on signed comparison
2505 // against 0. We try to convert a compare against 1 or -1 into a compare
2506 // against 0 to exploit record-form instructions. For example, we change
2507 // the condition "greater than -1" into "greater than or equal to 0"
2508 // and "less than 1" into "less than or equal to 0".
2509
2510 // Since we optimize comparison based on a specific branch condition,
2511 // we don't optimize if condition code is used by more than once.
2512 if (equalityOnly || !MRI->hasOneUse(CRReg))
2513 return false;
2514
2515 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
2516 if (UseMI->getOpcode() != PPC::BCC)
2517 return false;
2518
2520 unsigned PredCond = PPC::getPredicateCondition(Pred);
2521 unsigned PredHint = PPC::getPredicateHint(Pred);
2522 int16_t Immed = (int16_t)Value;
2523
2524 // When modifying the condition in the predicate, we propagate hint bits
2525 // from the original predicate to the new one.
2526 if (Immed == -1 && PredCond == PPC::PRED_GT)
2527 // We convert "greater than -1" into "greater than or equal to 0",
2528 // since we are assuming signed comparison by !equalityOnly
2529 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
2530 else if (Immed == -1 && PredCond == PPC::PRED_LE)
2531 // We convert "less than or equal to -1" into "less than 0".
2532 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
2533 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2534 // We convert "less than 1" into "less than or equal to 0".
2535 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2536 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2537 // We convert "greater than or equal to 1" into "greater than 0".
2538 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2539 else
2540 return false;
2541
2542 // Convert the comparison and its user to a compare against zero with the
2543 // appropriate predicate on the branch. Zero comparison might provide
2544 // optimization opportunities post-RA (see optimization in
2545 // PPCPreEmitPeephole.cpp).
2546 UseMI->getOperand(0).setImm(Pred);
2547 CmpInstr.getOperand(2).setImm(0);
2548 }
2549
2550 // Search for Sub.
2551 --I;
2552
2553 // Get ready to iterate backward from CmpInstr.
2554 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2555
2556 for (; I != E && !noSub; --I) {
2557 const MachineInstr &Instr = *I;
2558 unsigned IOpC = Instr.getOpcode();
2559
2560 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2561 Instr.readsRegister(PPC::CR0, TRI)))
2562 // This instruction modifies or uses the record condition register after
2563 // the one we want to change. While we could do this transformation, it
2564 // would likely not be profitable. This transformation removes one
2565 // instruction, and so even forcing RA to generate one move probably
2566 // makes it unprofitable.
2567 return false;
2568
2569 // Check whether CmpInstr can be made redundant by the current instruction.
2570 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2571 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2572 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2573 ((Instr.getOperand(1).getReg() == SrcReg &&
2574 Instr.getOperand(2).getReg() == SrcReg2) ||
2575 (Instr.getOperand(1).getReg() == SrcReg2 &&
2576 Instr.getOperand(2).getReg() == SrcReg))) {
2577 Sub = &*I;
2578 break;
2579 }
2580
2581 if (I == B)
2582 // The 'and' is below the comparison instruction.
2583 return false;
2584 }
2585
2586 // Return false if no candidates exist.
2587 if (!MI && !Sub)
2588 return false;
2589
2590 // The single candidate is called MI.
2591 if (!MI) MI = Sub;
2592
2593 int NewOpC = -1;
2594 int MIOpC = MI->getOpcode();
2595 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2596 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2597 NewOpC = MIOpC;
2598 else {
2599 NewOpC = PPC::getRecordFormOpcode(MIOpC);
2600 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2601 NewOpC = MIOpC;
2602 }
2603
2604 // FIXME: On the non-embedded POWER architectures, only some of the record
2605 // forms are fast, and we should use only the fast ones.
2606
2607 // The defining instruction has a record form (or is already a record
2608 // form). It is possible, however, that we'll need to reverse the condition
2609 // code of the users.
2610 if (NewOpC == -1)
2611 return false;
2612
2613 // This transformation should not be performed if `nsw` is missing and is not
2614 // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2615 // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2616 // CRReg can reflect if compared values are equal, this optz is still valid.
2617 if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2618 Sub && !Sub->getFlag(MachineInstr::NoSWrap))
2619 return false;
2620
2621 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2622 // needs to be updated to be based on SUB. Push the condition code
2623 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2624 // condition code of these operands will be modified.
2625 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2626 // comparison against 0, which may modify predicate.
2627 bool ShouldSwap = false;
2628 if (Sub && Value == 0) {
2629 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2630 Sub->getOperand(2).getReg() == SrcReg;
2631
2632 // The operands to subf are the opposite of sub, so only in the fixed-point
2633 // case, invert the order.
2634 ShouldSwap = !ShouldSwap;
2635 }
2636
2637 if (ShouldSwap)
2639 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2640 I != IE; ++I) {
2641 MachineInstr *UseMI = &*I;
2642 if (UseMI->getOpcode() == PPC::BCC) {
2644 unsigned PredCond = PPC::getPredicateCondition(Pred);
2645 assert((!equalityOnly ||
2646 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2647 "Invalid predicate for equality-only optimization");
2648 (void)PredCond; // To suppress warning in release build.
2649 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2651 } else if (UseMI->getOpcode() == PPC::ISEL ||
2652 UseMI->getOpcode() == PPC::ISEL8) {
2653 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2654 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2655 "Invalid CR bit for equality-only optimization");
2656
2657 if (NewSubReg == PPC::sub_lt)
2658 NewSubReg = PPC::sub_gt;
2659 else if (NewSubReg == PPC::sub_gt)
2660 NewSubReg = PPC::sub_lt;
2661
2662 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2663 NewSubReg));
2664 } else // We need to abort on a user we don't understand.
2665 return false;
2666 }
2667 assert(!(Value != 0 && ShouldSwap) &&
2668 "Non-zero immediate support and ShouldSwap"
2669 "may conflict in updating predicate");
2670
2671 // Create a new virtual register to hold the value of the CR set by the
2672 // record-form instruction. If the instruction was not previously in
2673 // record form, then set the kill flag on the CR.
2674 CmpInstr.eraseFromParent();
2675
2677 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2678 get(TargetOpcode::COPY), CRReg)
2679 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2680
2681 // Even if CR0 register were dead before, it is alive now since the
2682 // instruction we just built uses it.
2683 MI->clearRegisterDeads(PPC::CR0);
2684
2685 if (MIOpC != NewOpC) {
2686 // We need to be careful here: we're replacing one instruction with
2687 // another, and we need to make sure that we get all of the right
2688 // implicit uses and defs. On the other hand, the caller may be holding
2689 // an iterator to this instruction, and so we can't delete it (this is
2690 // specifically the case if this is the instruction directly after the
2691 // compare).
2692
2693 // Rotates are expensive instructions. If we're emitting a record-form
2694 // rotate that can just be an andi/andis, we should just emit that.
2695 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2696 Register GPRRes = MI->getOperand(0).getReg();
2697 int64_t SH = MI->getOperand(2).getImm();
2698 int64_t MB = MI->getOperand(3).getImm();
2699 int64_t ME = MI->getOperand(4).getImm();
2700 // We can only do this if both the start and end of the mask are in the
2701 // same halfword.
2702 bool MBInLoHWord = MB >= 16;
2703 bool MEInLoHWord = ME >= 16;
2704 uint64_t Mask = ~0LLU;
2705
2706 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2707 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2708 // The mask value needs to shift right 16 if we're emitting andis.
2709 Mask >>= MBInLoHWord ? 0 : 16;
2710 NewOpC = MIOpC == PPC::RLWINM
2711 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2712 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2713 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2714 (ME - MB + 1 == SH) && (MB >= 16)) {
2715 // If we are rotating by the exact number of bits as are in the mask
2716 // and the mask is in the least significant bits of the register,
2717 // that's just an andis. (as long as the GPR result has no uses).
2718 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2719 Mask >>= 16;
2720 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2721 }
2722 // If we've set the mask, we can transform.
2723 if (Mask != ~0LLU) {
2724 MI->removeOperand(4);
2725 MI->removeOperand(3);
2726 MI->getOperand(2).setImm(Mask);
2727 NumRcRotatesConvertedToRcAnd++;
2728 }
2729 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2730 int64_t MB = MI->getOperand(3).getImm();
2731 if (MB >= 48) {
2732 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2733 NewOpC = PPC::ANDI8_rec;
2734 MI->removeOperand(3);
2735 MI->getOperand(2).setImm(Mask);
2736 NumRcRotatesConvertedToRcAnd++;
2737 }
2738 }
2739
2740 const MCInstrDesc &NewDesc = get(NewOpC);
2741 MI->setDesc(NewDesc);
2742
2743 for (MCPhysReg ImpDef : NewDesc.implicit_defs()) {
2744 if (!MI->definesRegister(ImpDef)) {
2745 MI->addOperand(*MI->getParent()->getParent(),
2746 MachineOperand::CreateReg(ImpDef, true, true));
2747 }
2748 }
2749 for (MCPhysReg ImpUse : NewDesc.implicit_uses()) {
2750 if (!MI->readsRegister(ImpUse)) {
2751 MI->addOperand(*MI->getParent()->getParent(),
2752 MachineOperand::CreateReg(ImpUse, false, true));
2753 }
2754 }
2755 }
2756 assert(MI->definesRegister(PPC::CR0) &&
2757 "Record-form instruction does not define cr0?");
2758
2759 // Modify the condition code of operands in OperandsToUpdate.
2760 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2761 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2762 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2763 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2764
2765 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2766 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2767
2768 return true;
2769}
2770
2773 if (MRI->isSSA())
2774 return false;
2775
2776 Register SrcReg, SrcReg2;
2777 int64_t CmpMask, CmpValue;
2778 if (!analyzeCompare(CmpMI, SrcReg, SrcReg2, CmpMask, CmpValue))
2779 return false;
2780
2781 // Try to optimize the comparison against 0.
2782 if (CmpValue || !CmpMask || SrcReg2)
2783 return false;
2784
2785 // The record forms set the condition register based on a signed comparison
2786 // with zero (see comments in optimizeCompareInstr). Since we can't do the
2787 // equality checks in post-RA, we are more restricted on a unsigned
2788 // comparison.
2789 unsigned Opc = CmpMI.getOpcode();
2790 if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
2791 return false;
2792
2793 // The record forms are always based on a 64-bit comparison on PPC64
2794 // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
2795 // comparison. Since we can't do the equality checks in post-RA, we bail out
2796 // the case.
2797 if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
2798 return false;
2799
2800 // CmpMI can't be deleted if it has implicit def.
2801 if (CmpMI.hasImplicitDef())
2802 return false;
2803
2804 bool SrcRegHasOtherUse = false;
2805 MachineInstr *SrcMI = getDefMIPostRA(SrcReg, CmpMI, SrcRegHasOtherUse);
2806 if (!SrcMI || !SrcMI->definesRegister(SrcReg))
2807 return false;
2808
2809 MachineOperand RegMO = CmpMI.getOperand(0);
2810 Register CRReg = RegMO.getReg();
2811 if (CRReg != PPC::CR0)
2812 return false;
2813
2814 // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
2815 bool SeenUseOfCRReg = false;
2816 bool IsCRRegKilled = false;
2817 if (!isRegElgibleForForwarding(RegMO, *SrcMI, CmpMI, false, IsCRRegKilled,
2818 SeenUseOfCRReg) ||
2819 SrcMI->definesRegister(CRReg) || SeenUseOfCRReg)
2820 return false;
2821
2822 int SrcMIOpc = SrcMI->getOpcode();
2823 int NewOpC = PPC::getRecordFormOpcode(SrcMIOpc);
2824 if (NewOpC == -1)
2825 return false;
2826
2827 LLVM_DEBUG(dbgs() << "Replace Instr: ");
2828 LLVM_DEBUG(SrcMI->dump());
2829
2830 const MCInstrDesc &NewDesc = get(NewOpC);
2831 SrcMI->setDesc(NewDesc);
2832 MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
2834 SrcMI->clearRegisterDeads(CRReg);
2835
2836 // Fix up killed/dead flag for SrcReg after transformation.
2837 if (SrcRegHasOtherUse || CmpMI.getOperand(1).isKill())
2838 fixupIsDeadOrKill(SrcMI, &CmpMI, SrcReg);
2839
2840 assert(SrcMI->definesRegister(PPC::CR0) &&
2841 "Record-form instruction does not define cr0?");
2842
2843 LLVM_DEBUG(dbgs() << "with: ");
2844 LLVM_DEBUG(SrcMI->dump());
2845 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
2846 LLVM_DEBUG(CmpMI.dump());
2847 return true;
2848}
2849
2852 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2853 const TargetRegisterInfo *TRI) const {
2854 const MachineOperand *BaseOp;
2855 OffsetIsScalable = false;
2856 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2857 return false;
2858 BaseOps.push_back(BaseOp);
2859 return true;
2860}
2861
2862static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2863 const TargetRegisterInfo *TRI) {
2864 // If this is a volatile load/store, don't mess with it.
2865 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2866 return false;
2867
2868 if (LdSt.getOperand(2).isFI())
2869 return true;
2870
2871 assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2872 // Can't cluster if the instruction modifies the base register
2873 // or it is update form. e.g. ld r2,3(r2)
2874 if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
2875 return false;
2876
2877 return true;
2878}
2879
2880// Only cluster instruction pair that have the same opcode, and they are
2881// clusterable according to PowerPC specification.
2882static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2883 const PPCSubtarget &Subtarget) {
2884 switch (FirstOpc) {
2885 default:
2886 return false;
2887 case PPC::STD:
2888 case PPC::STFD:
2889 case PPC::STXSD:
2890 case PPC::DFSTOREf64:
2891 return FirstOpc == SecondOpc;
2892 // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2893 // 32bit and 64bit instruction selection. They are clusterable pair though
2894 // they are different opcode.
2895 case PPC::STW:
2896 case PPC::STW8:
2897 return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2898 }
2899}
2900
2903 ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2904 unsigned NumBytes) const {
2905
2906 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2907 const MachineOperand &BaseOp1 = *BaseOps1.front();
2908 const MachineOperand &BaseOp2 = *BaseOps2.front();
2909 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2910 "Only base registers and frame indices are supported.");
2911
2912 // The NumLoads means the number of loads that has been clustered.
2913 // Don't cluster memory op if there are already two ops clustered at least.
2914 if (NumLoads > 2)
2915 return false;
2916
2917 // Cluster the load/store only when they have the same base
2918 // register or FI.
2919 if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2920 (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2921 (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2922 return false;
2923
2924 // Check if the load/store are clusterable according to the PowerPC
2925 // specification.
2926 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2927 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2928 unsigned FirstOpc = FirstLdSt.getOpcode();
2929 unsigned SecondOpc = SecondLdSt.getOpcode();
2931 // Cluster the load/store only when they have the same opcode, and they are
2932 // clusterable opcode according to PowerPC specification.
2933 if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2934 return false;
2935
2936 // Can't cluster load/store that have ordered or volatile memory reference.
2937 if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
2938 !isLdStSafeToCluster(SecondLdSt, TRI))
2939 return false;
2940
2941 int64_t Offset1 = 0, Offset2 = 0;
2942 unsigned Width1 = 0, Width2 = 0;
2943 const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2944 if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
2945 !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
2946 Width1 != Width2)
2947 return false;
2948
2949 assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2950 "getMemOperandWithOffsetWidth return incorrect base op");
2951 // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2952 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2953 return Offset1 + Width1 == Offset2;
2954}
2955
2956/// GetInstSize - Return the number of bytes of code the specified
2957/// instruction may be. This returns the maximum number of bytes.
2958///
2960 unsigned Opcode = MI.getOpcode();
2961
2962 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2963 const MachineFunction *MF = MI.getParent()->getParent();
2964 const char *AsmStr = MI.getOperand(0).getSymbolName();
2965 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2966 } else if (Opcode == TargetOpcode::STACKMAP) {
2967 StackMapOpers Opers(&MI);
2968 return Opers.getNumPatchBytes();
2969 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2970 PatchPointOpers Opers(&MI);
2971 return Opers.getNumPatchBytes();
2972 } else {
2973 return get(Opcode).getSize();
2974 }
2975}
2976
2977std::pair<unsigned, unsigned>
2979 const unsigned Mask = PPCII::MO_ACCESS_MASK;
2980 return std::make_pair(TF & Mask, TF & ~Mask);
2981}
2982
2985 using namespace PPCII;
2986 static const std::pair<unsigned, const char *> TargetFlags[] = {
2987 {MO_LO, "ppc-lo"},
2988 {MO_HA, "ppc-ha"},
2989 {MO_TPREL_LO, "ppc-tprel-lo"},
2990 {MO_TPREL_HA, "ppc-tprel-ha"},
2991 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2992 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2993 {MO_TOC_LO, "ppc-toc-lo"},
2994 {MO_TLS, "ppc-tls"}};
2995 return ArrayRef(TargetFlags);
2996}
2997
3000 using namespace PPCII;
3001 static const std::pair<unsigned, const char *> TargetFlags[] = {
3002 {MO_PLT, "ppc-plt"},
3003 {MO_PIC_FLAG, "ppc-pic"},
3004 {MO_PCREL_FLAG, "ppc-pcrel"},
3005 {MO_GOT_FLAG, "ppc-got"},
3006 {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
3007 {MO_TLSGD_FLAG, "ppc-tlsgd"},
3008 {MO_TLSLD_FLAG, "ppc-tlsld"},
3009 {MO_TPREL_FLAG, "ppc-tprel"},
3010 {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
3011 {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
3012 {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
3013 {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}};
3014 return ArrayRef(TargetFlags);
3015}
3016
3017// Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
3018// The VSX versions have the advantage of a full 64-register target whereas
3019// the FP ones have the advantage of lower latency and higher throughput. So
3020// what we are after is using the faster instructions in low register pressure
3021// situations and using the larger register file in high register pressure
3022// situations.
3024 unsigned UpperOpcode, LowerOpcode;
3025 switch (MI.getOpcode()) {
3026 case PPC::DFLOADf32:
3027 UpperOpcode = PPC::LXSSP;
3028 LowerOpcode = PPC::LFS;
3029 break;
3030 case PPC::DFLOADf64:
3031 UpperOpcode = PPC::LXSD;
3032 LowerOpcode = PPC::LFD;
3033 break;
3034 case PPC::DFSTOREf32:
3035 UpperOpcode = PPC::STXSSP;
3036 LowerOpcode = PPC::STFS;
3037 break;
3038 case PPC::DFSTOREf64:
3039 UpperOpcode = PPC::STXSD;
3040 LowerOpcode = PPC::STFD;
3041 break;
3042 case PPC::XFLOADf32:
3043 UpperOpcode = PPC::LXSSPX;
3044 LowerOpcode = PPC::LFSX;
3045 break;
3046 case PPC::XFLOADf64:
3047 UpperOpcode = PPC::LXSDX;
3048 LowerOpcode = PPC::LFDX;
3049 break;
3050 case PPC::XFSTOREf32:
3051 UpperOpcode = PPC::STXSSPX;
3052 LowerOpcode = PPC::STFSX;
3053 break;
3054 case PPC::XFSTOREf64:
3055 UpperOpcode = PPC::STXSDX;
3056 LowerOpcode = PPC::STFDX;
3057 break;
3058 case PPC::LIWAX:
3059 UpperOpcode = PPC::LXSIWAX;
3060 LowerOpcode = PPC::LFIWAX;
3061 break;
3062 case PPC::LIWZX:
3063 UpperOpcode = PPC::LXSIWZX;
3064 LowerOpcode = PPC::LFIWZX;
3065 break;
3066 case PPC::STIWX:
3067 UpperOpcode = PPC::STXSIWX;
3068 LowerOpcode = PPC::STFIWX;
3069 break;
3070 default:
3071 llvm_unreachable("Unknown Operation!");
3072 }
3073
3074 Register TargetReg = MI.getOperand(0).getReg();
3075 unsigned Opcode;
3076 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
3077 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
3078 Opcode = LowerOpcode;
3079 else
3080 Opcode = UpperOpcode;
3081 MI.setDesc(get(Opcode));
3082 return true;
3083}
3084
3085static bool isAnImmediateOperand(const MachineOperand &MO) {
3086 return MO.isCPI() || MO.isGlobal() || MO.isImm();
3087}
3088
3090 auto &MBB = *MI.getParent();
3091 auto DL = MI.getDebugLoc();
3092
3093 switch (MI.getOpcode()) {
3094 case PPC::BUILD_UACC: {
3095 MCRegister ACC = MI.getOperand(0).getReg();
3096 MCRegister UACC = MI.getOperand(1).getReg();
3097 if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3098 MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3099 MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3100 // FIXME: This can easily be improved to look up to the top of the MBB
3101 // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3102 // we can just re-target any such XXLOR's to DstVSR + offset.
3103 for (int VecNo = 0; VecNo < 4; VecNo++)
3104 BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
3105 .addReg(SrcVSR + VecNo)
3106 .addReg(SrcVSR + VecNo);
3107 }
3108 // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3109 // So after building the 4 copies, we can replace the BUILD_UACC instruction
3110 // with a NOP.
3111 [[fallthrough]];
3112 }
3113 case PPC::KILL_PAIR: {
3114 MI.setDesc(get(PPC::UNENCODED_NOP));
3115 MI.removeOperand(1);
3116 MI.removeOperand(0);
3117 return true;
3118 }
3119 case TargetOpcode::LOAD_STACK_GUARD: {
3120 assert(Subtarget.isTargetLinux() &&
3121 "Only Linux target is expected to contain LOAD_STACK_GUARD");
3122 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3123 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3124 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3125 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3126 .addImm(Offset)
3127 .addReg(Reg);
3128 return true;
3129 }
3130 case PPC::DFLOADf32:
3131 case PPC::DFLOADf64:
3132 case PPC::DFSTOREf32:
3133 case PPC::DFSTOREf64: {
3134 assert(Subtarget.hasP9Vector() &&
3135 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3136 assert(MI.getOperand(2).isReg() &&
3137 isAnImmediateOperand(MI.getOperand(1)) &&
3138 "D-form op must have register and immediate operands");
3139 return expandVSXMemPseudo(MI);
3140 }
3141 case PPC::XFLOADf32:
3142 case PPC::XFSTOREf32:
3143 case PPC::LIWAX:
3144 case PPC::LIWZX:
3145 case PPC::STIWX: {
3146 assert(Subtarget.hasP8Vector() &&
3147 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3148 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3149 "X-form op must have register and register operands");
3150 return expandVSXMemPseudo(MI);
3151 }
3152 case PPC::XFLOADf64:
3153 case PPC::XFSTOREf64: {
3154 assert(Subtarget.hasVSX() &&
3155 "Invalid X-Form Pseudo-ops on target that has no VSX.");
3156 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3157 "X-form op must have register and register operands");
3158 return expandVSXMemPseudo(MI);
3159 }
3160 case PPC::SPILLTOVSR_LD: {
3161 Register TargetReg = MI.getOperand(0).getReg();
3162 if (PPC::VSFRCRegClass.contains(TargetReg)) {
3163 MI.setDesc(get(PPC::DFLOADf64));
3164 return expandPostRAPseudo(MI);
3165 }
3166 else
3167 MI.setDesc(get(PPC::LD));
3168 return true;
3169 }
3170 case PPC::SPILLTOVSR_ST: {
3171 Register SrcReg = MI.getOperand(0).getReg();
3172 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3173 NumStoreSPILLVSRRCAsVec++;
3174 MI.setDesc(get(PPC::DFSTOREf64));
3175 return expandPostRAPseudo(MI);
3176 } else {
3177 NumStoreSPILLVSRRCAsGpr++;
3178 MI.setDesc(get(PPC::STD));
3179 }
3180 return true;
3181 }
3182 case PPC::SPILLTOVSR_LDX: {
3183 Register TargetReg = MI.getOperand(0).getReg();
3184 if (PPC::VSFRCRegClass.contains(TargetReg))
3185 MI.setDesc(get(PPC::LXSDX));
3186 else
3187 MI.setDesc(get(PPC::LDX));
3188 return true;
3189 }
3190 case PPC::SPILLTOVSR_STX: {
3191 Register SrcReg = MI.getOperand(0).getReg();
3192 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3193 NumStoreSPILLVSRRCAsVec++;
3194 MI.setDesc(get(PPC::STXSDX));
3195 } else {
3196 NumStoreSPILLVSRRCAsGpr++;
3197 MI.setDesc(get(PPC::STDX));
3198 }
3199 return true;
3200 }
3201
3202 // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3203 case PPC::CFENCE8: {
3204 auto Val = MI.getOperand(0).getReg();
3205 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
3206 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
3208 .addReg(PPC::CR7)
3209 .addImm(1);
3210 MI.setDesc(get(PPC::ISYNC));
3211 MI.removeOperand(0);
3212 return true;
3213 }
3214 }
3215 return false;
3216}
3217
3218// Essentially a compile-time implementation of a compare->isel sequence.
3219// It takes two constants to compare, along with the true/false registers
3220// and the comparison type (as a subreg to a CR field) and returns one
3221// of the true/false registers, depending on the comparison results.
3222static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3223 unsigned TrueReg, unsigned FalseReg,
3224 unsigned CRSubReg) {
3225 // Signed comparisons. The immediates are assumed to be sign-extended.
3226 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3227 switch (CRSubReg) {
3228 default: llvm_unreachable("Unknown integer comparison type.");
3229 case PPC::sub_lt:
3230 return Imm1 < Imm2 ? TrueReg : FalseReg;
3231 case PPC::sub_gt:
3232 return Imm1 > Imm2 ? TrueReg : FalseReg;
3233 case PPC::sub_eq:
3234 return Imm1 == Imm2 ? TrueReg : FalseReg;
3235 }
3236 }
3237 // Unsigned comparisons.
3238 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3239 switch (CRSubReg) {
3240 default: llvm_unreachable("Unknown integer comparison type.");
3241 case PPC::sub_lt:
3242 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3243 case PPC::sub_gt:
3244 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3245 case PPC::sub_eq:
3246 return Imm1 == Imm2 ? TrueReg : FalseReg;
3247 }
3248 }
3249 return PPC::NoRegister;
3250}
3251
3253 unsigned OpNo,
3254 int64_t Imm) const {
3255 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3256 // Replace the REG with the Immediate.
3257 Register InUseReg = MI.getOperand(OpNo).getReg();
3258 MI.getOperand(OpNo).ChangeToImmediate(Imm);
3259
3260 // We need to make sure that the MI didn't have any implicit use
3261 // of this REG any more. We don't call MI.implicit_operands().empty() to
3262 // return early, since MI's MCID might be changed in calling context, as a
3263 // result its number of explicit operands may be changed, thus the begin of
3264 // implicit operand is changed.
3266 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
3267 if (UseOpIdx >= 0) {
3268 MachineOperand &MO = MI.getOperand(UseOpIdx);
3269 if (MO.isImplicit())
3270 // The operands must always be in the following order:
3271 // - explicit reg defs,
3272 // - other explicit operands (reg uses, immediates, etc.),
3273 // - implicit reg defs
3274 // - implicit reg uses
3275 // Therefore, removing the implicit operand won't change the explicit
3276 // operands layout.
3277 MI.removeOperand(UseOpIdx);
3278 }
3279}
3280
3281// Replace an instruction with one that materializes a constant (and sets
3282// CR0 if the original instruction was a record-form instruction).
3284 const LoadImmediateInfo &LII) const {
3285 // Remove existing operands.
3286 int OperandToKeep = LII.SetCR ? 1 : 0;
3287 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3288 MI.removeOperand(i);
3289
3290 // Replace the instruction.
3291 if (LII.SetCR) {
3292 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3293 // Set the immediate.
3294 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3295 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
3296 return;
3297 }
3298 else
3299 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
3300
3301 // Set the immediate.
3302 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3303 .addImm(LII.Imm);
3304}
3305
3307 bool &SeenIntermediateUse) const {
3308 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3309 "Should be called after register allocation.");
3311 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3312 It++;
3313 SeenIntermediateUse = false;
3314 for (; It != E; ++It) {
3315 if (It->modifiesRegister(Reg, TRI))
3316 return &*It;
3317 if (It->readsRegister(Reg, TRI))
3318 SeenIntermediateUse = true;
3319 }
3320 return nullptr;
3321}
3322
3325 const DebugLoc &DL, Register Reg,
3326 int64_t Imm) const {
3328 "Register should be in non-SSA form after RA");
3329 bool isPPC64 = Subtarget.isPPC64();
3330 // FIXME: Materialization here is not optimal.
3331 // For some special bit patterns we can use less instructions.
3332 // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3333 if (isInt<16>(Imm)) {
3334 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm);
3335 } else if (isInt<32>(Imm)) {
3336 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg)
3337 .addImm(Imm >> 16);
3338 if (Imm & 0xFFFF)
3339 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg)
3340 .addReg(Reg, RegState::Kill)
3341 .addImm(Imm & 0xFFFF);
3342 } else {
3343 assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3344 "only supported in PPC64");
3345 BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48);
3346 if ((Imm >> 32) & 0xFFFF)
3347 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3348 .addReg(Reg, RegState::Kill)
3349 .addImm((Imm >> 32) & 0xFFFF);
3350 BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg)
3351 .addReg(Reg, RegState::Kill)
3352 .addImm(32)
3353 .addImm(31);
3354 BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg)
3355 .addReg(Reg, RegState::Kill)
3356 .addImm((Imm >> 16) & 0xFFFF);
3357 if (Imm & 0xFFFF)
3358 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3359 .addReg(Reg, RegState::Kill)
3360 .addImm(Imm & 0xFFFF);
3361 }
3362}
3363
3364MachineInstr *PPCInstrInfo::getForwardingDefMI(
3366 unsigned &OpNoForForwarding,
3367 bool &SeenIntermediateUse) const {
3368 OpNoForForwarding = ~0U;
3369 MachineInstr *DefMI = nullptr;
3370 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3372 // If we're in SSA, get the defs through the MRI. Otherwise, only look
3373 // within the basic block to see if the register is defined using an
3374 // LI/LI8/ADDI/ADDI8.
3375 if (MRI->isSSA()) {
3376 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3377 if (!MI.getOperand(i).isReg())
3378 continue;
3379 Register Reg = MI.getOperand(i).getReg();
3380 if (!Reg.isVirtual())
3381 continue;
3382 Register TrueReg = TRI->lookThruCopyLike(Reg, MRI);
3383 if (TrueReg.isVirtual()) {
3384 MachineInstr *DefMIForTrueReg = MRI->getVRegDef(TrueReg);
3385 if (DefMIForTrueReg->getOpcode() == PPC::LI ||
3386 DefMIForTrueReg->getOpcode() == PPC::LI8 ||
3387 DefMIForTrueReg->getOpcode() == PPC::ADDI ||
3388 DefMIForTrueReg->getOpcode() == PPC::ADDI8) {
3389 OpNoForForwarding = i;
3390 DefMI = DefMIForTrueReg;
3391 // The ADDI and LI operand maybe exist in one instruction at same
3392 // time. we prefer to fold LI operand as LI only has one Imm operand
3393 // and is more possible to be converted. So if current DefMI is
3394 // ADDI/ADDI8, we continue to find possible LI/LI8.
3395 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3396 break;
3397 }
3398 }
3399 }
3400 } else {
3401 // Looking back through the definition for each operand could be expensive,
3402 // so exit early if this isn't an instruction that either has an immediate
3403 // form or is already an immediate form that we can handle.
3404 ImmInstrInfo III;
3405 unsigned Opc = MI.getOpcode();
3406 bool ConvertibleImmForm =
3407 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3408 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3409 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3410 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3411 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3412 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3413 Opc == PPC::RLWINM8_rec;
3414 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
3415 ? isVFRegister(MI.getOperand(0).getReg())
3416 : false;
3417 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
3418 return nullptr;
3419
3420 // Don't convert or %X, %Y, %Y since that's just a register move.
3421 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3422 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
3423 return nullptr;
3424 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3425 MachineOperand &MO = MI.getOperand(i);
3426 SeenIntermediateUse = false;
3427 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3428 Register Reg = MI.getOperand(i).getReg();
3429 // If we see another use of this reg between the def and the MI,
3430 // we want to flag it so the def isn't deleted.
3431 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3432 if (DefMI) {
3433 // Is this register defined by some form of add-immediate (including
3434 // load-immediate) within this basic block?
3435 switch (DefMI->getOpcode()) {
3436 default:
3437 break;
3438 case PPC::LI:
3439 case PPC::LI8:
3440 case PPC::ADDItocL:
3441 case PPC::ADDI:
3442 case PPC::ADDI8:
3443 OpNoForForwarding = i;
3444 return DefMI;
3445 }
3446 }
3447 }
3448 }
3449 }
3450 return OpNoForForwarding == ~0U ? nullptr : DefMI;
3451}
3452
3453unsigned PPCInstrInfo::getSpillTarget() const {
3454 // With P10, we may need to spill paired vector registers or accumulator
3455 // registers. MMA implies paired vectors, so we can just check that.
3456 bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3457 return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
3458 2 : Subtarget.hasP9Vector() ?
3459 1 : 0;
3460}
3461
3462ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3463 return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3464}
3465
3466ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3467 return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3468}
3469
3471 unsigned RegNo) const {
3472 // Conservatively clear kill flag for the register if the instructions are in
3473 // different basic blocks and in SSA form, because the kill flag may no longer
3474 // be right. There is no need to bother with dead flags since defs with no
3475 // uses will be handled by DCE.
3477 if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) {
3478 MRI.clearKillFlags(RegNo);
3479 return;
3480 }
3481
3482 // Instructions between [StartMI, EndMI] should be in same basic block.
3483 assert((StartMI->getParent() == EndMI->getParent()) &&
3484 "Instructions are not in same basic block");
3485
3486 // If before RA, StartMI may be def through COPY, we need to adjust it to the
3487 // real def. See function getForwardingDefMI.
3488 if (MRI.isSSA()) {
3489 bool Reads, Writes;
3490 std::tie(Reads, Writes) = StartMI->readsWritesVirtualRegister(RegNo);
3491 if (!Reads && !Writes) {
3493 "Must be a virtual register");
3494 // Get real def and ignore copies.
3495 StartMI = MRI.getVRegDef(RegNo);
3496 }
3497 }
3498
3499 bool IsKillSet = false;
3500
3501 auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
3502 MachineOperand &MO = MI.getOperand(Index);
3503 if (MO.isReg() && MO.isUse() && MO.isKill() &&
3504 getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
3505 MO.setIsKill(false);
3506 };
3507
3508 // Set killed flag for EndMI.
3509 // No need to do anything if EndMI defines RegNo.
3510 int UseIndex =
3511 EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
3512 if (UseIndex != -1) {
3513 EndMI->getOperand(UseIndex).setIsKill(true);
3514 IsKillSet = true;
3515 // Clear killed flag for other EndMI operands related to RegNo. In some
3516 // upexpected cases, killed may be set multiple times for same register
3517 // operand in same MI.
3518 for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i)
3519 if (i != UseIndex)
3520 clearOperandKillInfo(*EndMI, i);
3521 }
3522
3523 // Walking the inst in reverse order (EndMI -> StartMI].
3526 // EndMI has been handled above, skip it here.
3527 It++;
3528 MachineOperand *MO = nullptr;
3529 for (; It != E; ++It) {
3530 // Skip insturctions which could not be a def/use of RegNo.
3531 if (It->isDebugInstr() || It->isPosition())
3532 continue;
3533
3534 // Clear killed flag for all It operands related to RegNo. In some
3535 // upexpected cases, killed may be set multiple times for same register
3536 // operand in same MI.
3537 for (int i = 0, e = It->getNumOperands(); i != e; ++i)
3538 clearOperandKillInfo(*It, i);
3539
3540 // If killed is not set, set killed for its last use or set dead for its def
3541 // if no use found.
3542 if (!IsKillSet) {
3543 if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
3544 // Use found, set it killed.
3545 IsKillSet = true;
3546 MO->setIsKill(true);
3547 continue;
3548 } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
3549 &getRegisterInfo()))) {
3550 // No use found, set dead for its def.
3551 assert(&*It == StartMI && "No new def between StartMI and EndMI.");
3552 MO->setIsDead(true);
3553 break;
3554 }
3555 }
3556
3557 if ((&*It) == StartMI)
3558 break;
3559 }
3560 // Ensure RegMo liveness is killed after EndMI.
3561 assert((IsKillSet || (MO && MO->isDead())) &&
3562 "RegNo should be killed or dead");
3563}
3564
3565// This opt tries to convert the following imm form to an index form to save an
3566// add for stack variables.
3567// Return false if no such pattern found.
3568//
3569// ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3570// ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3571// Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3572//
3573// can be converted to:
3574//
3575// new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3576// Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3577//
3578// In order to eliminate ADD instr, make sure that:
3579// 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3580// new ADDI instr and ADDI can only take int16 Imm.
3581// 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3582// between ADDI and ADD instr since its original def in ADDI will be changed
3583// in new ADDI instr. And also there should be no new def for it between
3584// ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3585// 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3586// between ADD and Imm instr since ADD instr will be eliminated.
3587// 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3588// moved to Index instr.
3590 MachineFunction *MF = MI.getParent()->getParent();
3592 bool PostRA = !MRI->isSSA();
3593 // Do this opt after PEI which is after RA. The reason is stack slot expansion
3594 // in PEI may expose such opportunities since in PEI, stack slot offsets to
3595 // frame base(OffsetAddi) are determined.
3596 if (!PostRA)
3597 return false;
3598 unsigned ToBeDeletedReg = 0;
3599 int64_t OffsetImm = 0;
3600 unsigned XFormOpcode = 0;
3601 ImmInstrInfo III;
3602
3603 // Check if Imm instr meets requirement.
3604 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
3605 III))
3606 return false;
3607
3608 bool OtherIntermediateUse = false;
3609 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
3610
3611 // Exit if there is other use between ADD and Imm instr or no def found.
3612 if (OtherIntermediateUse || !ADDMI)
3613 return false;
3614
3615 // Check if ADD instr meets requirement.
3616 if (!isADDInstrEligibleForFolding(*ADDMI))
3617 return false;
3618
3619 unsigned ScaleRegIdx = 0;
3620 int64_t OffsetAddi = 0;
3621 MachineInstr *ADDIMI = nullptr;
3622
3623 // Check if there is a valid ToBeChangedReg in ADDMI.
3624 // 1: It must be killed.
3625 // 2: Its definition must be a valid ADDIMI.
3626 // 3: It must satify int16 offset requirement.
3627 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
3628 ScaleRegIdx = 2;
3629 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
3630 ScaleRegIdx = 1;
3631 else
3632 return false;
3633
3634 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3635 Register ToBeChangedReg = ADDIMI->getOperand(0).getReg();
3636 Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
3637 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3639 for (auto It = ++Start; It != End; It++)
3640 if (It->modifiesRegister(Reg, &getRegisterInfo()))
3641 return true;
3642 return false;
3643 };
3644
3645 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3646 // treated as special zero when ScaleReg is R0/X0 register.
3647 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3648 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3649 return false;
3650
3651 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3652 // and Imm Instr.
3653 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3654 return false;
3655
3656 // Now start to do the transformation.
3657 LLVM_DEBUG(dbgs() << "Replace instruction: "
3658 << "\n");
3659 LLVM_DEBUG(ADDIMI->dump());
3660 LLVM_DEBUG(ADDMI->dump());
3661 LLVM_DEBUG(MI.dump());
3662 LLVM_DEBUG(dbgs() << "with: "
3663 << "\n");
3664
3665 // Update ADDI instr.
3666 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
3667
3668 // Update Imm instr.
3669 MI.setDesc(get(XFormOpcode));
3670 MI.getOperand(III.ImmOpNo)
3671 .ChangeToRegister(ScaleReg, false, false,
3672 ADDMI->getOperand(ScaleRegIdx).isKill());
3673
3674 MI.getOperand(III.OpNoForForwarding)
3675 .ChangeToRegister(ToBeChangedReg, false, false, true);
3676
3677 // Eliminate ADD instr.
3678 ADDMI->eraseFromParent();
3679
3680 LLVM_DEBUG(ADDIMI->dump());
3681 LLVM_DEBUG(MI.dump());
3682
3683 return true;
3684}
3685
3687 int64_t &Imm) const {
3688 unsigned Opc = ADDIMI.getOpcode();
3689
3690 // Exit if the instruction is not ADDI.
3691 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3692 return false;
3693
3694 // The operand may not necessarily be an immediate - it could be a relocation.
3695 if (!ADDIMI.getOperand(2).isImm())
3696 return false;
3697
3698 Imm = ADDIMI.getOperand(2).getImm();
3699
3700 return true;
3701}
3702
3704 unsigned Opc = ADDMI.getOpcode();
3705
3706 // Exit if the instruction is not ADD.
3707 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3708}
3709
3711 unsigned &ToBeDeletedReg,
3712 unsigned &XFormOpcode,
3713 int64_t &OffsetImm,
3714 ImmInstrInfo &III) const {
3715 // Only handle load/store.
3716 if (!MI.mayLoadOrStore())
3717 return false;
3718
3719 unsigned Opc = MI.getOpcode();
3720
3721 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
3722
3723 // Exit if instruction has no index form.
3724 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3725 return false;
3726
3727 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3728 if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
3729 III, true))
3730 return false;
3731
3732 if (!III.IsSummingOperands)
3733 return false;
3734
3735 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
3736 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
3737 // Only support imm operands, not relocation slots or others.
3738 if (!ImmOperand.isImm())
3739 return false;
3740
3741 assert(RegOperand.isReg() && "Instruction format is not right");
3742
3743 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3744 if (!RegOperand.isKill())
3745 return false;
3746
3747 ToBeDeletedReg = RegOperand.getReg();
3748 OffsetImm = ImmOperand.getImm();
3749
3750 return true;
3751}
3752
3754 MachineInstr *&ADDIMI,
3755 int64_t &OffsetAddi,
3756 int64_t OffsetImm) const {
3757 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3758 MachineOperand &MO = ADDMI->getOperand(Index);
3759
3760 if (!MO.isKill())
3761 return false;
3762
3763 bool OtherIntermediateUse = false;
3764
3765 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
3766 // Currently handle only one "add + Imminstr" pair case, exit if other
3767 // intermediate use for ToBeChangedReg found.
3768 // TODO: handle the cases where there are other "add + Imminstr" pairs
3769 // with same offset in Imminstr which is like:
3770 //
3771 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3772 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3773 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3774 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3775 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3776 //
3777 // can be converted to:
3778 //
3779 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3780 // (OffsetAddi + OffsetImm)
3781 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3782 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3783
3784 if (OtherIntermediateUse || !ADDIMI)
3785 return false;
3786 // Check if ADDI instr meets requirement.
3787 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
3788 return false;
3789
3790 if (isInt<16>(OffsetAddi + OffsetImm))
3791 return true;
3792 return false;
3793}
3794
3795// If this instruction has an immediate form and one of its operands is a
3796// result of a load-immediate or an add-immediate, convert it to
3797// the immediate form if the constant is in range.
3799 MachineInstr **KilledDef) const {
3800 MachineFunction *MF = MI.getParent()->getParent();
3802 bool PostRA = !MRI->isSSA();
3803 bool SeenIntermediateUse = true;
3804 unsigned ForwardingOperand = ~0U;
3805 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
3806 SeenIntermediateUse);
3807 if (!DefMI)
3808 return false;
3809 assert(ForwardingOperand < MI.getNumOperands() &&
3810 "The forwarding operand needs to be valid at this point");
3811 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
3812 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3813 if (KilledDef && KillFwdDefMI)
3814 *KilledDef = DefMI;
3815
3816 // If this is a imm instruction and its register operands is produced by ADDI,
3817 // put the imm into imm inst directly.
3818 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3819 PPC::INSTRUCTION_LIST_END &&
3820 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3821 return true;
3822
3823 ImmInstrInfo III;
3824 bool IsVFReg = MI.getOperand(0).isReg()
3825 ? isVFRegister(MI.getOperand(0).getReg())
3826 : false;
3827 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3828 // If this is a reg+reg instruction that has a reg+imm form,
3829 // and one of the operands is produced by an add-immediate,
3830 // try to convert it.
3831 if (HasImmForm &&
3832 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3833 KillFwdDefMI))
3834 return true;
3835
3836 // If this is a reg+reg instruction that has a reg+imm form,
3837 // and one of the operands is produced by LI, convert it now.
3838 if (HasImmForm &&
3839 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3840 return true;
3841
3842 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3843 // can be simpified to LI.
3844 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3845 return true;
3846
3847 return false;
3848}
3849
3851 MachineInstr **ToErase) const {
3852 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3853 Register FoldingReg = MI.getOperand(1).getReg();
3854 if (!FoldingReg.isVirtual())
3855 return false;
3856 MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
3857 if (SrcMI->getOpcode() != PPC::RLWINM &&
3858 SrcMI->getOpcode() != PPC::RLWINM_rec &&
3859 SrcMI->getOpcode() != PPC::RLWINM8 &&
3860 SrcMI->getOpcode() != PPC::RLWINM8_rec)
3861 return false;
3862 assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3863 MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3864 SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3865 "Invalid PPC::RLWINM Instruction!");
3866 uint64_t SHSrc = SrcMI->getOperand(2).getImm();
3867 uint64_t SHMI = MI.getOperand(2).getImm();
3868 uint64_t MBSrc = SrcMI->getOperand(3).getImm();
3869 uint64_t MBMI = MI.getOperand(3).getImm();
3870 uint64_t MESrc = SrcMI->getOperand(4).getImm();
3871 uint64_t MEMI = MI.getOperand(4).getImm();
3872
3873 assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3874 "Invalid PPC::RLWINM Instruction!");
3875 // If MBMI is bigger than MEMI, we always can not get run of ones.
3876 // RotatedSrcMask non-wrap:
3877 // 0........31|32........63
3878 // RotatedSrcMask: B---E B---E
3879 // MaskMI: -----------|--E B------
3880 // Result: ----- --- (Bad candidate)
3881 //
3882 // RotatedSrcMask wrap:
3883 // 0........31|32........63
3884 // RotatedSrcMask: --E B----|--E B----
3885 // MaskMI: -----------|--E B------
3886 // Result: --- -----|--- ----- (Bad candidate)
3887 //
3888 // One special case is RotatedSrcMask is a full set mask.
3889 // RotatedSrcMask full:
3890 // 0........31|32........63
3891 // RotatedSrcMask: ------EB---|-------EB---
3892 // MaskMI: -----------|--E B------
3893 // Result: -----------|--- ------- (Good candidate)
3894
3895 // Mark special case.
3896 bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3897
3898 // For other MBMI > MEMI cases, just return.
3899 if ((MBMI > MEMI) && !SrcMaskFull)
3900 return false;
3901
3902 // Handle MBMI <= MEMI cases.
3903 APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
3904 // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3905 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3906 // while in PowerPC ISA, lowerest bit is at index 63.
3907 APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
3908
3909 APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
3910 APInt FinalMask = RotatedSrcMask & MaskMI;
3911 uint32_t NewMB, NewME;
3912 bool Simplified = false;
3913
3914 // If final mask is 0, MI result should be 0 too.
3915 if (FinalMask.isZero()) {
3916 bool Is64Bit =
3917 (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3918 Simplified = true;
3919 LLVM_DEBUG(dbgs() << "Replace Instr: ");
3920 LLVM_DEBUG(MI.dump());
3921
3922 if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3923 // Replace MI with "LI 0"
3924 MI.removeOperand(4);
3925 MI.removeOperand(3);
3926 MI.removeOperand(2);
3927 MI.getOperand(1).ChangeToImmediate(0);
3928 MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
3929 } else {
3930 // Replace MI with "ANDI_rec reg, 0"
3931 MI.removeOperand(4);
3932 MI.removeOperand(3);
3933 MI.getOperand(2).setImm(0);
3934 MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3935 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3936 if (SrcMI->getOperand(1).isKill()) {
3937 MI.getOperand(1).setIsKill(true);
3938 SrcMI->getOperand(1).setIsKill(false);
3939 } else
3940 // About to replace MI.getOperand(1), clear its kill flag.
3941 MI.getOperand(1).setIsKill(false);
3942 }
3943
3944 LLVM_DEBUG(dbgs() << "With: ");
3945 LLVM_DEBUG(MI.dump());
3946
3947 } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
3948 NewMB <= NewME) ||
3949 SrcMaskFull) {
3950 // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3951 // than NewME. Otherwise we get a 64 bit value after folding, but MI
3952 // return a 32 bit value.
3953 Simplified = true;
3954 LLVM_DEBUG(dbgs() << "Converting Instr: ");
3955 LLVM_DEBUG(MI.dump());
3956
3957 uint16_t NewSH = (SHSrc + SHMI) % 32;
3958 MI.getOperand(2).setImm(NewSH);
3959 // If SrcMI mask is full, no need to update MBMI and MEMI.
3960 if (!SrcMaskFull) {
3961 MI.getOperand(3).setImm(NewMB);
3962 MI.getOperand(4).setImm(NewME);
3963 }
3964 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3965 if (SrcMI->getOperand(1).isKill()) {
3966 MI.getOperand(1).setIsKill(true);
3967 SrcMI->getOperand(1).setIsKill(false);
3968 } else
3969 // About to replace MI.getOperand(1), clear its kill flag.
3970 MI.getOperand(1).setIsKill(false);
3971
3972 LLVM_DEBUG(dbgs() << "To: ");
3973 LLVM_DEBUG(MI.dump());
3974 }
3975 if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
3976 !SrcMI->hasImplicitDef()) {
3977 // If FoldingReg has no non-debug use and it has no implicit def (it
3978 // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
3979 // Otherwise keep it.
3980 *ToErase = SrcMI;
3981 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
3982 LLVM_DEBUG(SrcMI->dump());
3983 }
3984 return Simplified;
3985}
3986
3987bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3988 ImmInstrInfo &III, bool PostRA) const {
3989 // The vast majority of the instructions would need their operand 2 replaced
3990 // with an immediate when switching to the reg+imm form. A marked exception
3991 // are the update form loads/stores for which a constant operand 2 would need
3992 // to turn into a displacement and move operand 1 to the operand 2 position.
3993 III.ImmOpNo = 2;
3994 III.OpNoForForwarding = 2;
3995 III.ImmWidth = 16;
3996 III.ImmMustBeMultipleOf = 1;
3997 III.TruncateImmTo = 0;
3998 III.IsSummingOperands = false;
3999 switch (Opc) {
4000 default: return false;
4001 case PPC::ADD4:
4002 case PPC::ADD8:
4003 III.SignedImm = true;
4004 III.ZeroIsSpecialOrig = 0;
4005 III.ZeroIsSpecialNew = 1;
4006 III.IsCommutative = true;
4007 III.IsSummingOperands = true;
4008 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
4009 break;
4010 case PPC::ADDC:
4011 case PPC::ADDC8:
4012 III.SignedImm = true;
4013 III.ZeroIsSpecialOrig = 0;
4014 III.ZeroIsSpecialNew = 0;
4015 III.IsCommutative = true;
4016 III.IsSummingOperands = true;
4017 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
4018 break;
4019 case PPC::ADDC_rec:
4020 III.SignedImm = true;
4021 III.ZeroIsSpecialOrig = 0;
4022 III.ZeroIsSpecialNew = 0;
4023 III.IsCommutative = true;
4024 III.IsSummingOperands = true;
4025 III.ImmOpcode = PPC::ADDIC_rec;
4026 break;
4027 case PPC::SUBFC:
4028 case PPC::SUBFC8:
4029 III.SignedImm = true;
4030 III.ZeroIsSpecialOrig = 0;
4031 III.ZeroIsSpecialNew = 0;
4032 III.IsCommutative = false;
4033 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
4034 break;
4035 case PPC::CMPW:
4036 case PPC::CMPD:
4037 III.SignedImm = true;
4038 III.ZeroIsSpecialOrig = 0;
4039 III.ZeroIsSpecialNew = 0;
4040 III.IsCommutative = false;
4041 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
4042 break;
4043 case PPC::CMPLW:
4044 case PPC::CMPLD:
4045 III.SignedImm = false;
4046 III.ZeroIsSpecialOrig = 0;
4047 III.ZeroIsSpecialNew = 0;
4048 III.IsCommutative = false;
4049 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
4050 break;
4051 case PPC::AND_rec:
4052 case PPC::AND8_rec:
4053 case PPC::OR:
4054 case PPC::OR8:
4055 case PPC::XOR:
4056 case PPC::XOR8:
4057 III.SignedImm = false;
4058 III.ZeroIsSpecialOrig = 0;
4059 III.ZeroIsSpecialNew = 0;
4060 III.IsCommutative = true;
4061 switch(Opc) {
4062 default: llvm_unreachable("Unknown opcode");
4063 case PPC::AND_rec:
4064 III.ImmOpcode = PPC::ANDI_rec;
4065 break;
4066 case PPC::AND8_rec:
4067 III.ImmOpcode = PPC::ANDI8_rec;
4068 break;
4069 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
4070 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
4071 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
4072 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
4073 }
4074 break;
4075 case PPC::RLWNM:
4076 case PPC::RLWNM8:
4077 case PPC::RLWNM_rec:
4078 case PPC::RLWNM8_rec:
4079 case PPC::SLW:
4080 case PPC::SLW8:
4081 case PPC::SLW_rec:
4082 case PPC::SLW8_rec:
4083 case PPC::SRW:
4084 case PPC::SRW8:
4085 case PPC::SRW_rec:
4086 case PPC::SRW8_rec:
4087 case PPC::SRAW:
4088 case PPC::SRAW_rec:
4089 III.SignedImm = false;
4090 III.ZeroIsSpecialOrig = 0;
4091 III.ZeroIsSpecialNew = 0;
4092 III.IsCommutative = false;
4093 // This isn't actually true, but the instructions ignore any of the
4094 // upper bits, so any immediate loaded with an LI is acceptable.
4095 // This does not apply to shift right algebraic because a value
4096 // out of range will produce a -1/0.
4097 III.ImmWidth = 16;
4098 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
4099 Opc == PPC::RLWNM8_rec)
4100 III.TruncateImmTo = 5;
4101 else
4102 III.TruncateImmTo = 6;
4103 switch(Opc) {
4104 default: llvm_unreachable("Unknown opcode");
4105 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
4106 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
4107 case PPC::RLWNM_rec:
4108 III.ImmOpcode = PPC::RLWINM_rec;
4109 break;
4110 case PPC::RLWNM8_rec:
4111 III.ImmOpcode = PPC::RLWINM8_rec;
4112 break;
4113 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4114 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4115 case PPC::SLW_rec:
4116 III.ImmOpcode = PPC::RLWINM_rec;
4117 break;
4118 case PPC::SLW8_rec:
4119 III.ImmOpcode = PPC::RLWINM8_rec;
4120 break;
4121 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4122 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4123 case PPC::SRW_rec:
4124 III.ImmOpcode = PPC::RLWINM_rec;
4125 break;
4126 case PPC::SRW8_rec:
4127 III.ImmOpcode = PPC::RLWINM8_rec;
4128 break;
4129 case PPC::SRAW:
4130 III.ImmWidth = 5;
4131 III.TruncateImmTo = 0;
4132 III.ImmOpcode = PPC::SRAWI;
4133 break;
4134 case PPC::SRAW_rec:
4135 III.ImmWidth = 5;
4136 III.TruncateImmTo = 0;
4137 III.ImmOpcode = PPC::SRAWI_rec;
4138 break;
4139 }
4140 break;
4141 case PPC::RLDCL:
4142 case PPC::RLDCL_rec:
4143 case PPC::RLDCR:
4144 case PPC::RLDCR_rec:
4145 case PPC::SLD:
4146 case PPC::SLD_rec:
4147 case PPC::SRD:
4148 case PPC::SRD_rec:
4149 case PPC::SRAD:
4150 case PPC::SRAD_rec:
4151 III.SignedImm = false;
4152 III.ZeroIsSpecialOrig = 0;
4153 III.ZeroIsSpecialNew = 0;
4154 III.IsCommutative = false;
4155 // This isn't actually true, but the instructions ignore any of the
4156 // upper bits, so any immediate loaded with an LI is acceptable.
4157 // This does not apply to shift right algebraic because a value
4158 // out of range will produce a -1/0.
4159 III.ImmWidth = 16;
4160 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4161 Opc == PPC::RLDCR_rec)
4162 III.TruncateImmTo = 6;
4163 else
4164 III.TruncateImmTo = 7;
4165 switch(Opc) {
4166 default: llvm_unreachable("Unknown opcode");
4167 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4168 case PPC::RLDCL_rec:
4169 III.ImmOpcode = PPC::RLDICL_rec;
4170 break;
4171 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4172 case PPC::RLDCR_rec:
4173 III.ImmOpcode = PPC::RLDICR_rec;
4174 break;
4175 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4176 case PPC::SLD_rec:
4177 III.ImmOpcode = PPC::RLDICR_rec;
4178 break;
4179 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4180 case PPC::SRD_rec:
4181 III.ImmOpcode = PPC::RLDICL_rec;
4182 break;
4183 case PPC::SRAD:
4184 III.ImmWidth = 6;
4185 III.TruncateImmTo = 0;
4186 III.ImmOpcode = PPC::SRADI;
4187 break;
4188 case PPC::SRAD_rec:
4189 III.ImmWidth = 6;
4190 III.TruncateImmTo = 0;
4191 III.ImmOpcode = PPC::SRADI_rec;
4192 break;
4193 }
4194 break;
4195 // Loads and stores:
4196 case PPC::LBZX:
4197 case PPC::LBZX8:
4198 case PPC::LHZX:
4199 case PPC::LHZX8:
4200 case PPC::LHAX:
4201 case PPC::LHAX8:
4202 case PPC::LWZX:
4203 case PPC::LWZX8:
4204 case PPC::LWAX:
4205 case PPC::LDX:
4206 case PPC::LFSX:
4207 case PPC::LFDX:
4208 case PPC::STBX:
4209 case PPC::STBX8:
4210 case PPC::STHX:
4211 case PPC::STHX8:
4212 case PPC::STWX:
4213 case PPC::STWX8:
4214 case PPC::STDX:
4215 case PPC::STFSX:
4216 case PPC::STFDX:
4217 III.SignedImm = true;
4218 III.ZeroIsSpecialOrig = 1;
4219 III.ZeroIsSpecialNew = 2;
4220 III.IsCommutative = true;
4221 III.IsSummingOperands = true;
4222 III.ImmOpNo = 1;
4223 III.OpNoForForwarding = 2;
4224 switch(Opc) {
4225 default: llvm_unreachable("Unknown opcode");
4226 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4227 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4228 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4229 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4230 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4231 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4232 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4233 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4234 case PPC::LWAX:
4235 III.ImmOpcode = PPC::LWA;
4236 III.ImmMustBeMultipleOf = 4;
4237 break;
4238 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4239 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4240 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4241 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4242 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4243 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4244 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4245 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4246 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4247 case PPC::STDX:
4248 III.ImmOpcode = PPC::STD;
4249 III.ImmMustBeMultipleOf = 4;
4250 break;
4251 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4252 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4253 }
4254 break;
4255 case PPC::LBZUX:
4256 case PPC::LBZUX8:
4257 case PPC::LHZUX:
4258 case PPC::LHZUX8:
4259 case PPC::LHAUX:
4260 case PPC::LHAUX8:
4261 case PPC::LWZUX:
4262 case PPC::LWZUX8:
4263 case PPC::LDUX:
4264 case PPC::LFSUX:
4265 case PPC::LFDUX:
4266 case PPC::STBUX:
4267 case PPC::STBUX8:
4268 case PPC::STHUX:
4269 case PPC::STHUX8:
4270 case PPC::STWUX:
4271 case PPC::STWUX8:
4272 case PPC::STDUX:
4273 case PPC::STFSUX:
4274 case PPC::STFDUX:
4275 III.SignedImm = true;
4276 III.ZeroIsSpecialOrig = 2;
4277 III.ZeroIsSpecialNew = 3;
4278 III.IsCommutative = false;
4279 III.IsSummingOperands = true;
4280 III.ImmOpNo = 2;
4281 III.OpNoForForwarding = 3;
4282 switch(Opc) {
4283 default: llvm_unreachable("Unknown opcode");
4284 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4285 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4286 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4287 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4288 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4289 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4290 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4291 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4292 case PPC::LDUX:
4293 III.ImmOpcode = PPC::LDU;
4294 III.ImmMustBeMultipleOf = 4;
4295 break;
4296 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4297 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4298 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4299 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4300 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4301 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4302 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4303 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4304 case PPC::STDUX:
4305 III.ImmOpcode = PPC::STDU;
4306 III.ImmMustBeMultipleOf = 4;
4307 break;
4308 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4309 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4310 }
4311 break;
4312 // Power9 and up only. For some of these, the X-Form version has access to all
4313 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4314 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4315 // into or stored from is one of the VR registers.
4316 case PPC::LXVX:
4317 case PPC::LXSSPX:
4318 case PPC::LXSDX:
4319 case PPC::STXVX:
4320 case PPC::STXSSPX:
4321 case PPC::STXSDX:
4322 case PPC::XFLOADf32:
4323 case PPC::XFLOADf64:
4324 case PPC::XFSTOREf32:
4325 case PPC::XFSTOREf64:
4326 if (!Subtarget.hasP9Vector())
4327 return false;
4328 III.SignedImm = true;
4329 III.ZeroIsSpecialOrig = 1;
4330 III.ZeroIsSpecialNew = 2;
4331 III.IsCommutative = true;
4332 III.IsSummingOperands = true;
4333 III.ImmOpNo = 1;
4334 III.OpNoForForwarding = 2;
4335 III.ImmMustBeMultipleOf = 4;
4336 switch(Opc) {
4337 default: llvm_unreachable("Unknown opcode");
4338 case PPC::LXVX:
4339 III.ImmOpcode = PPC::LXV;
4340 III.ImmMustBeMultipleOf = 16;
4341 break;
4342 case PPC::LXSSPX:
4343 if (PostRA) {
4344 if (IsVFReg)
4345 III.ImmOpcode = PPC::LXSSP;
4346 else {
4347 III.ImmOpcode = PPC::LFS;
4348 III.ImmMustBeMultipleOf = 1;
4349 }
4350 break;
4351 }
4352 [[fallthrough]];
4353 case PPC::XFLOADf32:
4354 III.ImmOpcode = PPC::DFLOADf32;
4355 break;
4356 case PPC::LXSDX:
4357 if (PostRA) {
4358 if (IsVFReg)
4359 III.ImmOpcode = PPC::LXSD;
4360 else {
4361 III.ImmOpcode = PPC::LFD;
4362 III.ImmMustBeMultipleOf = 1;
4363 }
4364 break;
4365 }
4366 [[fallthrough]];
4367 case PPC::XFLOADf64:
4368 III.ImmOpcode = PPC::DFLOADf64;
4369 break;
4370 case PPC::STXVX:
4371 III.ImmOpcode = PPC::STXV;
4372 III.ImmMustBeMultipleOf = 16;
4373 break;
4374 case PPC::STXSSPX:
4375 if (PostRA) {
4376 if (IsVFReg)
4377 III.ImmOpcode = PPC::STXSSP;
4378 else {
4379 III.ImmOpcode = PPC::STFS;
4380 III.ImmMustBeMultipleOf = 1;
4381 }
4382 break;
4383 }
4384 [[fallthrough]];
4385 case PPC::XFSTOREf32:
4386 III.ImmOpcode = PPC::DFSTOREf32;
4387 break;
4388 case PPC::STXSDX:
4389 if (PostRA) {
4390 if (IsVFReg)
4391 III.ImmOpcode = PPC::STXSD;
4392 else {
4393 III.ImmOpcode = PPC::STFD;
4394 III.ImmMustBeMultipleOf = 1;
4395 }
4396 break;
4397 }
4398 [[fallthrough]];
4399 case PPC::XFSTOREf64:
4400 III.ImmOpcode = PPC::DFSTOREf64;
4401 break;
4402 }
4403 break;
4404 }
4405 return true;
4406}
4407
4408// Utility function for swaping two arbitrary operands of an instruction.
4409static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4410 assert(Op1 != Op2 && "Cannot swap operand with itself.");
4411
4412 unsigned MaxOp = std::max(Op1, Op2);
4413 unsigned MinOp = std::min(Op1, Op2);
4414 MachineOperand MOp1 = MI.getOperand(MinOp);
4415 MachineOperand MOp2 = MI.getOperand(MaxOp);
4416 MI.removeOperand(std::max(Op1, Op2));
4417 MI.removeOperand(std::min(Op1, Op2));
4418
4419 // If the operands we are swapping are the two at the end (the common case)
4420 // we can just remove both and add them in the opposite order.
4421 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4422 MI.addOperand(MOp2);
4423 MI.addOperand(MOp1);
4424 } else {
4425 // Store all operands in a temporary vector, remove them and re-add in the
4426 // right order.
4428 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4429 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4430 MOps.push_back(MI.getOperand(i));
4431 MI.removeOperand(i);
4432 }
4433 // MOp2 needs to be added next.
4434 MI.addOperand(MOp2);
4435 // Now add the rest.
4436 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4437 if (i == MaxOp)
4438 MI.addOperand(MOp1);
4439 else {
4440 MI.addOperand(MOps.back());
4441 MOps.pop_back();
4442 }
4443 }
4444 }
4445}
4446
4447// Check if the 'MI' that has the index OpNoForForwarding
4448// meets the requirement described in the ImmInstrInfo.
4449bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4450 const ImmInstrInfo &III,
4451 unsigned OpNoForForwarding
4452 ) const {
4453 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4454 // would not work pre-RA, we can only do the check post RA.
4455 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4456 if (MRI.isSSA())
4457 return false;
4458
4459 // Cannot do the transform if MI isn't summing the operands.
4460 if (!III.IsSummingOperands)
4461 return false;
4462
4463 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4464 if (!III.ZeroIsSpecialOrig)
4465 return false;
4466
4467 // We cannot do the transform if the operand we are trying to replace
4468 // isn't the same as the operand the instruction allows.
4469 if (OpNoForForwarding != III.OpNoForForwarding)
4470 return false;
4471
4472 // Check if the instruction we are trying to transform really has
4473 // the special zero register as its operand.
4474 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4475 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4476 return false;
4477
4478 // This machine instruction is convertible if it is,
4479 // 1. summing the operands.
4480 // 2. one of the operands is special zero register.
4481 // 3. the operand we are trying to replace is allowed by the MI.
4482 return true;
4483}
4484
4485// Check if the DefMI is the add inst and set the ImmMO and RegMO
4486// accordingly.
4487bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4488 const ImmInstrInfo &III,
4489 MachineOperand *&ImmMO,
4490 MachineOperand *&RegMO) const {
4491 unsigned Opc = DefMI.getOpcode();
4492 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4493 return false;
4494
4495 assert(DefMI.getNumOperands() >= 3 &&
4496 "Add inst must have at least three operands");
4497 RegMO = &DefMI.getOperand(1);
4498 ImmMO = &DefMI.getOperand(2);
4499
4500 // Before RA, ADDI first operand could be a frame index.
4501 if (!RegMO->isReg())
4502 return false;
4503
4504 // This DefMI is elgible for forwarding if it is:
4505 // 1. add inst
4506 // 2. one of the operands is Imm/CPI/Global.
4507 return isAnImmediateOperand(*ImmMO);
4508}
4509
4510bool PPCInstrInfo::isRegElgibleForForwarding(
4511 const MachineOperand &RegMO, const MachineInstr &DefMI,
4512 const MachineInstr &MI, bool KillDefMI,
4513 bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
4514 // x = addi y, imm
4515 // ...
4516 // z = lfdx 0, x -> z = lfd imm(y)
4517 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4518 // of "y" between the DEF of "x" and "z".
4519 // The query is only valid post RA.
4520 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4521 if (MRI.isSSA())
4522 return false;
4523
4524 Register Reg = RegMO.getReg();
4525
4526 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4528 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4529 It++;
4530 for (; It != E; ++It) {
4531 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4532 return false;
4533 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4534 IsFwdFeederRegKilled = true;
4535 if (It->readsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4536 SeenIntermediateUse = true;
4537 // Made it to DefMI without encountering a clobber.
4538 if ((&*It) == &DefMI)
4539 break;
4540 }
4541 assert((&*It) == &DefMI && "DefMI is missing");
4542
4543 // If DefMI also defines the register to be forwarded, we can only forward it
4544 // if DefMI is being erased.
4545 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
4546 return KillDefMI;
4547
4548 return true;
4549}
4550
4551bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4552 const MachineInstr &DefMI,
4553 const ImmInstrInfo &III,
4554 int64_t &Imm,
4555 int64_t BaseImm) const {
4556 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4557 if (DefMI.getOpcode() == PPC::ADDItocL) {
4558 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
4559 // However, we know that, it is 16-bit width, and has the alignment of 4.
4560 // Check if the instruction met the requirement.
4561 if (III.ImmMustBeMultipleOf > 4 ||
4562 III.TruncateImmTo || III.ImmWidth != 16)
4563 return false;
4564
4565 // Going from XForm to DForm loads means that the displacement needs to be
4566 // not just an immediate but also a multiple of 4, or 16 depending on the
4567 // load. A DForm load cannot be represented if it is a multiple of say 2.
4568 // XForm loads do not have this restriction.
4569 if (ImmMO.isGlobal()) {
4570 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
4572 return false;
4573 }
4574
4575 return true;
4576 }
4577
4578 if (ImmMO.isImm()) {
4579 // It is Imm, we need to check if the Imm fit the range.
4580 // Sign-extend to 64-bits.
4581 // DefMI may be folded with another imm form instruction, the result Imm is
4582 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4583 APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4584 if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
4585 return false;
4586 if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
4587 return false;
4588 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
4589
4590 if (Imm % III.ImmMustBeMultipleOf)
4591 return false;
4592 if (III.TruncateImmTo)
4593 Imm &= ((1 << III.TruncateImmTo) - 1);
4594 }
4595 else
4596 return false;
4597
4598 // This ImmMO is forwarded if it meets the requriement describle
4599 // in ImmInstrInfo
4600 return true;
4601}
4602
4603bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4604 unsigned OpNoForForwarding,
4605 MachineInstr **KilledDef) const {
4606 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4607 !DefMI.getOperand(1).isImm())
4608 return false;
4609
4610 MachineFunction *MF = MI.getParent()->getParent();
4612 bool PostRA = !MRI->isSSA();
4613
4614 int64_t Immediate = DefMI.getOperand(1).getImm();
4615 // Sign-extend to 64-bits.
4616 int64_t SExtImm = SignExtend64<16>(Immediate);
4617
4618 bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill();
4619 Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg();
4620
4621 bool ReplaceWithLI = false;
4622 bool Is64BitLI = false;
4623 int64_t NewImm = 0;
4624 bool SetCR = false;
4625 unsigned Opc = MI.getOpcode();
4626 switch (Opc) {
4627 default:
4628 return false;
4629
4630 // FIXME: Any branches conditional on such a comparison can be made
4631 // unconditional. At this time, this happens too infrequently to be worth
4632 // the implementation effort, but if that ever changes, we could convert
4633 // such a pattern here.
4634 case PPC::CMPWI:
4635 case PPC::CMPLWI:
4636 case PPC::CMPDI:
4637 case PPC::CMPLDI: {
4638 // Doing this post-RA would require dataflow analysis to reliably find uses
4639 // of the CR register set by the compare.
4640 // No need to fixup killed/dead flag since this transformation is only valid
4641 // before RA.
4642 if (PostRA)
4643 return false;
4644 // If a compare-immediate is fed by an immediate and is itself an input of
4645 // an ISEL (the most common case) into a COPY of the correct register.
4646 bool Changed = false;
4647 Register DefReg = MI.getOperand(0).getReg();
4648 int64_t Comparand = MI.getOperand(2).getImm();
4649 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4650 ? (Comparand | 0xFFFFFFFFFFFF0000)
4651 : Comparand;
4652
4653 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
4654 unsigned UseOpc = CompareUseMI.getOpcode();
4655 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4656 continue;
4657 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
4658 Register TrueReg = CompareUseMI.getOperand(1).getReg();
4659 Register FalseReg = CompareUseMI.getOperand(2).getReg();
4660 unsigned RegToCopy =
4661 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
4662 if (RegToCopy == PPC::NoRegister)
4663 continue;
4664 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4665 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4666 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4667 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
4668 CompareUseMI.removeOperand(3);
4669 CompareUseMI.removeOperand(2);
4670 continue;
4671 }
4672 LLVM_DEBUG(
4673 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4674 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4675 LLVM_DEBUG(dbgs() << "Is converted to:\n");
4676 // Convert to copy and remove unneeded operands.
4677 CompareUseMI.setDesc(get(PPC::COPY));
4678 CompareUseMI.removeOperand(3);
4679 CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1);
4680 CmpIselsConverted++;
4681 Changed = true;
4682 LLVM_DEBUG(CompareUseMI.dump());
4683 }
4684 if (Changed)
4685 return true;
4686 // This may end up incremented multiple times since this function is called
4687 // during a fixed-point transformation, but it is only meant to indicate the
4688 // presence of this opportunity.
4689 MissedConvertibleImmediateInstrs++;
4690 return false;
4691 }
4692
4693 // Immediate forms - may simply be convertable to an LI.
4694 case PPC::ADDI:
4695 case PPC::ADDI8: {
4696 // Does the sum fit in a 16-bit signed field?
4697 int64_t Addend = MI.getOperand(2).getImm();
4698 if (isInt<16>(Addend + SExtImm)) {
4699 ReplaceWithLI = true;
4700 Is64BitLI = Opc == PPC::ADDI8;
4701 NewImm = Addend + SExtImm;