Line data Source code
1 : //===-- TargetInstrInfo.cpp - Target Instruction Information --------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements the TargetInstrInfo class.
11 : //
12 : //===----------------------------------------------------------------------===//
13 :
14 : #include "llvm/CodeGen/TargetInstrInfo.h"
15 : #include "llvm/CodeGen/MachineFrameInfo.h"
16 : #include "llvm/CodeGen/MachineInstrBuilder.h"
17 : #include "llvm/CodeGen/MachineMemOperand.h"
18 : #include "llvm/CodeGen/MachineRegisterInfo.h"
19 : #include "llvm/CodeGen/PseudoSourceValue.h"
20 : #include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
21 : #include "llvm/CodeGen/StackMaps.h"
22 : #include "llvm/CodeGen/TargetFrameLowering.h"
23 : #include "llvm/CodeGen/TargetLowering.h"
24 : #include "llvm/CodeGen/TargetRegisterInfo.h"
25 : #include "llvm/CodeGen/TargetSchedule.h"
26 : #include "llvm/IR/DataLayout.h"
27 : #include "llvm/MC/MCAsmInfo.h"
28 : #include "llvm/MC/MCInstrItineraries.h"
29 : #include "llvm/Support/CommandLine.h"
30 : #include "llvm/Support/ErrorHandling.h"
31 : #include "llvm/Support/raw_ostream.h"
32 : #include "llvm/Target/TargetMachine.h"
33 : #include <cctype>
34 :
35 : using namespace llvm;
36 :
37 : static cl::opt<bool> DisableHazardRecognizer(
38 : "disable-sched-hazard", cl::Hidden, cl::init(false),
39 : cl::desc("Disable hazard detection during preRA scheduling"));
40 :
41 34780 : TargetInstrInfo::~TargetInstrInfo() {
42 34780 : }
43 0 :
44 0 : const TargetRegisterClass*
45 34780 : TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum,
46 34780 : const TargetRegisterInfo *TRI,
47 : const MachineFunction &MF) const {
48 : if (OpNum >= MCID.getNumOperands())
49 80081788 : return nullptr;
50 :
51 : short RegClass = MCID.OpInfo[OpNum].RegClass;
52 160163576 : if (MCID.OpInfo[OpNum].isLookupPtrRegClass())
53 : return TRI->getPointerRegClass(MF, RegClass);
54 :
55 77851200 : // Instructions like INSERT_SUBREG do not have fixed register classes.
56 77851200 : if (RegClass < 0)
57 8594309 : return nullptr;
58 :
59 : // Otherwise just look it up normally.
60 69256891 : return TRI->getRegClass(RegClass);
61 : }
62 :
63 : /// insertNoop - Insert a noop into the instruction stream at the specified
64 110034386 : /// point.
65 : void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB,
66 : MachineBasicBlock::iterator MI) const {
67 : llvm_unreachable("Target didn't implement insertNoop!");
68 : }
69 0 :
70 : static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) {
71 0 : return strncmp(Str, MAI.getCommentString().data(),
72 : MAI.getCommentString().size()) == 0;
73 : }
74 :
75 116867 : /// Measure the specified inline asm to determine an approximation of its
76 : /// length.
77 : /// Comments (which run till the next SeparatorString or newline) do not
78 : /// count as an instruction.
79 : /// Any other non-whitespace text is considered an instruction, with
80 : /// multiple instructions separated by SeparatorString or newlines.
81 : /// Variable-length instructions are not handled here; this function
82 : /// may be overloaded in the target code to do that.
83 : /// We implement a special case of the .space directive which takes only a
84 : /// single integer argument in base 10 that is the size in bytes. This is a
85 : /// restricted form of the GAS directive in that we only interpret
86 : /// simple--i.e. not a logical or arithmetic expression--size values without
87 : /// the optional fill value. This is primarily used for creating arbitrary
88 : /// sized inline asm blocks for testing purposes.
89 : unsigned TargetInstrInfo::getInlineAsmLength(const char *Str,
90 : const MCAsmInfo &MAI) const {
91 : // Count the number of instructions in the asm.
92 : bool AtInsnStart = true;
93 193522 : unsigned Length = 0;
94 : for (; *Str; ++Str) {
95 : if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(),
96 : strlen(MAI.getSeparatorString())) == 0) {
97 : AtInsnStart = true;
98 311666 : } else if (isAsmComment(Str, MAI)) {
99 118144 : // Stop counting as an instruction after a comment until the next
100 : // separator.
101 : AtInsnStart = false;
102 116867 : }
103 :
104 : if (AtInsnStart && !std::isspace(static_cast<unsigned char>(*Str))) {
105 : unsigned AddLength = MAI.getMaxInstLength();
106 : if (strncmp(Str, ".space", 6) == 0) {
107 : char *EStr;
108 116614 : int SpaceSize;
109 20484 : SpaceSize = strtol(Str + 6, &EStr, 10);
110 20484 : SpaceSize = SpaceSize < 0 ? 0 : SpaceSize;
111 : while (*EStr != '\n' && std::isspace(static_cast<unsigned char>(*EStr)))
112 : ++EStr;
113 464 : if (*EStr == '\0' || *EStr == '\n' ||
114 464 : isAsmComment(EStr, MAI)) // Successfully parsed .space argument
115 464 : AddLength = SpaceSize;
116 0 : }
117 464 : Length += AddLength;
118 : AtInsnStart = false;
119 464 : }
120 : }
121 20484 :
122 : return Length;
123 : }
124 :
125 : /// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
126 193522 : /// after it, replacing it with an unconditional branch to NewDest.
127 : void
128 : TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
129 : MachineBasicBlock *NewDest) const {
130 : MachineBasicBlock *MBB = Tail->getParent();
131 :
132 26347 : // Remove all the old successors of MBB from the CFG.
133 : while (!MBB->succ_empty())
134 26347 : MBB->removeSuccessor(MBB->succ_begin());
135 :
136 : // Save off the debug loc before erasing the instruction.
137 34280 : DebugLoc DL = Tail->getDebugLoc();
138 7933 :
139 : // Remove all the dead instructions from the end of MBB.
140 : MBB->erase(Tail, MBB->end());
141 :
142 : // If MBB isn't immediately before MBB, insert a branch to it.
143 : if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
144 : insertBranch(*MBB, NewDest, nullptr, SmallVector<MachineOperand, 0>(), DL);
145 : MBB->addSuccessor(NewDest);
146 : }
147 26347 :
148 51050 : MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI,
149 26347 : bool NewMI, unsigned Idx1,
150 26347 : unsigned Idx2) const {
151 : const MCInstrDesc &MCID = MI.getDesc();
152 535556 : bool HasDef = MCID.getNumDefs();
153 : if (HasDef && !MI.getOperand(0).isReg())
154 : // No idea how to commute this instruction. Target should implement its own.
155 535556 : return nullptr;
156 535556 :
157 535556 : unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1;
158 : unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2;
159 : assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) &&
160 : CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 &&
161 : "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands.");
162 : assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() &&
163 : "This only knows how to commute register operands so far");
164 :
165 : unsigned Reg0 = HasDef ? MI.getOperand(0).getReg() : 0;
166 : unsigned Reg1 = MI.getOperand(Idx1).getReg();
167 : unsigned Reg2 = MI.getOperand(Idx2).getReg();
168 : unsigned SubReg0 = HasDef ? MI.getOperand(0).getSubReg() : 0;
169 535556 : unsigned SubReg1 = MI.getOperand(Idx1).getSubReg();
170 1071112 : unsigned SubReg2 = MI.getOperand(Idx2).getSubReg();
171 535556 : bool Reg1IsKill = MI.getOperand(Idx1).isKill();
172 535556 : bool Reg2IsKill = MI.getOperand(Idx2).isKill();
173 : bool Reg1IsUndef = MI.getOperand(Idx1).isUndef();
174 : bool Reg2IsUndef = MI.getOperand(Idx2).isUndef();
175 : bool Reg1IsInternal = MI.getOperand(Idx1).isInternalRead();
176 : bool Reg2IsInternal = MI.getOperand(Idx2).isInternalRead();
177 : // Avoid calling isRenamable for virtual registers since we assert that
178 : // renamable property is only queried/set for physical registers.
179 : bool Reg1IsRenamable = TargetRegisterInfo::isPhysicalRegister(Reg1)
180 : ? MI.getOperand(Idx1).isRenamable()
181 : : false;
182 : bool Reg2IsRenamable = TargetRegisterInfo::isPhysicalRegister(Reg2)
183 : ? MI.getOperand(Idx2).isRenamable()
184 535556 : : false;
185 : // If destination is tied to either of the commuted source register, then
186 : // it must be updated.
187 535556 : if (HasDef && Reg0 == Reg1 &&
188 : MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) {
189 : Reg2IsKill = false;
190 : Reg0 = Reg2;
191 535804 : SubReg0 = SubReg2;
192 2417 : } else if (HasDef && Reg0 == Reg2 &&
193 : MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) {
194 : Reg1IsKill = false;
195 : Reg0 = Reg1;
196 535308 : SubReg0 = SubReg1;
197 2470 : }
198 :
199 : MachineInstr *CommutedMI = nullptr;
200 : if (NewMI) {
201 : // Create a new instruction.
202 : MachineFunction &MF = *MI.getMF();
203 : CommutedMI = MF.CloneMachineInstr(&MI);
204 535556 : } else {
205 : CommutedMI = &MI;
206 : }
207 0 :
208 : if (HasDef) {
209 : CommutedMI->getOperand(0).setReg(Reg0);
210 : CommutedMI->getOperand(0).setSubReg(SubReg0);
211 : }
212 535556 : CommutedMI->getOperand(Idx2).setReg(Reg1);
213 482380 : CommutedMI->getOperand(Idx1).setReg(Reg2);
214 482380 : CommutedMI->getOperand(Idx2).setSubReg(SubReg1);
215 : CommutedMI->getOperand(Idx1).setSubReg(SubReg2);
216 1071112 : CommutedMI->getOperand(Idx2).setIsKill(Reg1IsKill);
217 1071112 : CommutedMI->getOperand(Idx1).setIsKill(Reg2IsKill);
218 535556 : CommutedMI->getOperand(Idx2).setIsUndef(Reg1IsUndef);
219 535556 : CommutedMI->getOperand(Idx1).setIsUndef(Reg2IsUndef);
220 535556 : CommutedMI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal);
221 535556 : CommutedMI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal);
222 535556 : // Avoid calling setIsRenamable for virtual registers since we assert that
223 535556 : // renamable property is only queried/set for physical registers.
224 535556 : if (TargetRegisterInfo::isPhysicalRegister(Reg1))
225 535556 : CommutedMI->getOperand(Idx2).setIsRenamable(Reg1IsRenamable);
226 : if (TargetRegisterInfo::isPhysicalRegister(Reg2))
227 : CommutedMI->getOperand(Idx1).setIsRenamable(Reg2IsRenamable);
228 535556 : return CommutedMI;
229 23574 : }
230 535556 :
231 23574 : MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI,
232 : unsigned OpIdx1,
233 : unsigned OpIdx2) const {
234 : // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose
235 632207 : // any commutable operand, which is done in findCommutedOpIndices() method
236 : // called below.
237 : if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) &&
238 : !findCommutedOpIndices(MI, OpIdx1, OpIdx2)) {
239 : assert(MI.isCommutable() &&
240 : "Precondition violation: MI must be commutable.");
241 1193319 : return nullptr;
242 561112 : }
243 : return commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
244 : }
245 :
246 : bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1,
247 630252 : unsigned &ResultIdx2,
248 : unsigned CommutableOpIdx1,
249 : unsigned CommutableOpIdx2) {
250 668077 : if (ResultIdx1 == CommuteAnyOperandIndex &&
251 : ResultIdx2 == CommuteAnyOperandIndex) {
252 : ResultIdx1 = CommutableOpIdx1;
253 : ResultIdx2 = CommutableOpIdx2;
254 668077 : } else if (ResultIdx1 == CommuteAnyOperandIndex) {
255 578165 : if (ResultIdx2 == CommutableOpIdx1)
256 578165 : ResultIdx1 = CommutableOpIdx2;
257 578165 : else if (ResultIdx2 == CommutableOpIdx2)
258 89912 : ResultIdx1 = CommutableOpIdx1;
259 0 : else
260 0 : return false;
261 0 : } else if (ResultIdx2 == CommuteAnyOperandIndex) {
262 0 : if (ResultIdx1 == CommutableOpIdx1)
263 : ResultIdx2 = CommutableOpIdx2;
264 : else if (ResultIdx1 == CommutableOpIdx2)
265 89912 : ResultIdx2 = CommutableOpIdx1;
266 30936 : else
267 28795 : return false;
268 2141 : } else
269 1983 : // Check that the result operand indices match the given commutable
270 : // operand indices.
271 : return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) ||
272 : (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1);
273 :
274 : return true;
275 58976 : }
276 59022 :
277 : bool TargetInstrInfo::findCommutedOpIndices(MachineInstr &MI,
278 : unsigned &SrcOpIdx1,
279 : unsigned &SrcOpIdx2) const {
280 : assert(!MI.isBundle() &&
281 342845 : "TargetInstrInfo::findCommutedOpIndices() can't handle bundles");
282 :
283 : const MCInstrDesc &MCID = MI.getDesc();
284 : if (!MCID.isCommutable())
285 : return false;
286 :
287 342845 : // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this
288 685690 : // is not true, then the target must implement this.
289 : unsigned CommutableOpIdx1 = MCID.getNumDefs();
290 : unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1;
291 : if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
292 : CommutableOpIdx1, CommutableOpIdx2))
293 341992 : return false;
294 341992 :
295 341992 : if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg())
296 : // No idea.
297 : return false;
298 : return true;
299 680436 : }
300 :
301 141 : bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const {
302 : if (!MI.isTerminator()) return false;
303 :
304 : // Conditional branch is a special case.
305 1534457 : if (MI.isBranch() && !MI.isBarrier())
306 1534457 : return true;
307 : if (!MI.isPredicable())
308 : return true;
309 924172 : return !isPredicated(MI);
310 : }
311 569471 :
312 : bool TargetInstrInfo::PredicateInstruction(
313 31030 : MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
314 : bool MadeChange = false;
315 :
316 0 : assert(!MI.isBundle() &&
317 : "TargetInstrInfo::PredicateInstruction() can't handle bundles");
318 :
319 : const MCInstrDesc &MCID = MI.getDesc();
320 : if (!MI.isPredicable())
321 : return false;
322 :
323 0 : for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) {
324 0 : if (MCID.OpInfo[i].isPredicate()) {
325 : MachineOperand &MO = MI.getOperand(i);
326 : if (MO.isReg()) {
327 0 : MO.setReg(Pred[j].getReg());
328 0 : MadeChange = true;
329 0 : } else if (MO.isImm()) {
330 0 : MO.setImm(Pred[j].getImm());
331 0 : MadeChange = true;
332 : } else if (MO.isMBB()) {
333 0 : MO.setMBB(Pred[j].getMBB());
334 0 : MadeChange = true;
335 : }
336 0 : ++j;
337 0 : }
338 : }
339 : return MadeChange;
340 0 : }
341 :
342 : bool TargetInstrInfo::hasLoadFromStackSlot(
343 : const MachineInstr &MI,
344 : SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
345 : size_t StartSize = Accesses.size();
346 2041922 : for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
347 : oe = MI.memoperands_end();
348 : o != oe; ++o) {
349 2041922 : if ((*o)->isLoad() &&
350 569482 : dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue()))
351 : Accesses.push_back(*o);
352 2611404 : }
353 1440804 : return Accesses.size() != StartSize;
354 : }
355 69333 :
356 : bool TargetInstrInfo::hasStoreToStackSlot(
357 2041922 : const MachineInstr &MI,
358 : SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
359 : size_t StartSize = Accesses.size();
360 2012321 : for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
361 : oe = MI.memoperands_end();
362 : o != oe; ++o) {
363 2012321 : if ((*o)->isStore() &&
364 539423 : dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue()))
365 : Accesses.push_back(*o);
366 2551744 : }
367 1335591 : return Accesses.size() != StartSize;
368 : }
369 47319 :
370 : bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
371 2012321 : unsigned SubIdx, unsigned &Size,
372 : unsigned &Offset,
373 : const MachineFunction &MF) const {
374 1723 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
375 : if (!SubIdx) {
376 : Size = TRI->getSpillSize(*RC);
377 : Offset = 0;
378 1723 : return true;
379 1723 : }
380 1685 : unsigned BitSize = TRI->getSubRegIdxSize(SubIdx);
381 1685 : // Convert bit size to byte size.
382 1685 : if (BitSize % 8)
383 : return false;
384 38 :
385 : int BitOffset = TRI->getSubRegIdxOffset(SubIdx);
386 38 : if (BitOffset < 0 || BitOffset % 8)
387 : return false;
388 :
389 38 : Size = BitSize /= 8;
390 38 : Offset = (unsigned)BitOffset / 8;
391 :
392 : assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range");
393 38 :
394 38 : if (!MF.getDataLayout().isLittleEndian()) {
395 : Offset = TRI->getSpillSize(*RC) - (Offset + Size);
396 : }
397 : return true;
398 38 : }
399 3 :
400 : void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB,
401 : MachineBasicBlock::iterator I,
402 : unsigned DestReg, unsigned SubIdx,
403 : const MachineInstr &Orig,
404 8546 : const TargetRegisterInfo &TRI) const {
405 : MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
406 : MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI);
407 : MBB.insert(I, MI);
408 : }
409 8546 :
410 8546 : bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0,
411 : const MachineInstr &MI1,
412 8546 : const MachineRegisterInfo *MRI) const {
413 : return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs);
414 269163 : }
415 :
416 : MachineInstr &TargetInstrInfo::duplicate(MachineBasicBlock &MBB,
417 269163 : MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const {
418 : assert(!Orig.isNotDuplicable() && "Instruction cannot be duplicated");
419 : MachineFunction &MF = *MBB.getParent();
420 9356 : return MF.CloneMachineInstrBundle(MBB, InsertBefore, Orig);
421 : }
422 :
423 9356 : // If the COPY instruction in MI can be folded to a stack operation, return
424 9356 : // the register class to use.
425 : static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI,
426 : unsigned FoldIdx) {
427 : assert(MI.isCopy() && "MI must be a COPY instruction");
428 : if (MI.getNumOperands() != 2)
429 19003 : return nullptr;
430 : assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand");
431 :
432 19003 : const MachineOperand &FoldOp = MI.getOperand(FoldIdx);
433 : const MachineOperand &LiveOp = MI.getOperand(1 - FoldIdx);
434 :
435 : if (FoldOp.getSubReg() || LiveOp.getSubReg())
436 19003 : return nullptr;
437 19003 :
438 : unsigned FoldReg = FoldOp.getReg();
439 19003 : unsigned LiveReg = LiveOp.getReg();
440 :
441 : assert(TargetRegisterInfo::isVirtualRegister(FoldReg) &&
442 18785 : "Cannot fold physregs");
443 18785 :
444 : const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
445 : const TargetRegisterClass *RC = MRI.getRegClass(FoldReg);
446 :
447 : if (TargetRegisterInfo::isPhysicalRegister(LiveOp.getReg()))
448 18785 : return RC->contains(LiveOp.getReg()) ? RC : nullptr;
449 :
450 : if (RC->hasSubClassEq(MRI.getRegClass(LiveReg)))
451 37570 : return RC;
452 6743 :
453 : // FIXME: Allow folding when register classes are memory compatible.
454 24084 : return nullptr;
455 11878 : }
456 :
457 : void TargetInstrInfo::getNoop(MCInst &NopInst) const {
458 : llvm_unreachable("Not implemented");
459 : }
460 :
461 0 : static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
462 0 : ArrayRef<unsigned> Ops, int FrameIndex,
463 : const TargetInstrInfo &TII) {
464 : unsigned StartIdx = 0;
465 256 : switch (MI.getOpcode()) {
466 : case TargetOpcode::STACKMAP: {
467 : // StackMapLiveValues are foldable
468 : StartIdx = StackMapOpers(&MI).getVarIdx();
469 512 : break;
470 38 : }
471 : case TargetOpcode::PATCHPOINT: {
472 38 : // For PatchPoint, the call args are not foldable (even if reported in the
473 38 : // stackmap e.g. via anyregcc).
474 : StartIdx = PatchPointOpers(&MI).getVarIdx();
475 195 : break;
476 : }
477 : case TargetOpcode::STATEPOINT: {
478 195 : // For statepoints, fold deopt and gc arguments, but not call arguments.
479 195 : StartIdx = StatepointOpers(&MI).getVarIdx();
480 : break;
481 : }
482 : default:
483 : llvm_unreachable("unexpected stackmap opcode");
484 23 : }
485 :
486 0 : // Return false if any operands requested for folding are not foldable (not
487 0 : // part of the stackmap's live values).
488 : for (unsigned Op : Ops) {
489 : if (Op < StartIdx)
490 : return nullptr;
491 : }
492 390 :
493 256 : MachineInstr *NewMI =
494 : MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);
495 : MachineInstrBuilder MIB(MF, NewMI);
496 :
497 : // No need to fold return, the meta data, and function arguments
498 402 : for (unsigned i = 0; i < StartIdx; ++i)
499 : MIB.add(MI.getOperand(i));
500 :
501 : for (unsigned i = StartIdx; i < MI.getNumOperands(); ++i) {
502 1068 : MachineOperand &MO = MI.getOperand(i);
503 934 : if (is_contained(Ops, i)) {
504 : unsigned SpillSize;
505 4830 : unsigned SpillOffset;
506 4696 : // Compute the spill slot size and offset.
507 4696 : const TargetRegisterClass *RC =
508 : MF.getRegInfo().getRegClass(MO.getReg());
509 : bool Valid =
510 : TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF);
511 : if (!Valid)
512 134 : report_fatal_error("cannot spill patchpoint subregister operand");
513 : MIB.addImm(StackMaps::IndirectMemRefOp);
514 268 : MIB.addImm(SpillSize);
515 134 : MIB.addFrameIndex(FrameIndex);
516 0 : MIB.addImm(SpillOffset);
517 : }
518 134 : else
519 : MIB.add(MO);
520 134 : }
521 : return NewMI;
522 : }
523 :
524 : MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
525 134 : ArrayRef<unsigned> Ops, int FI,
526 : LiveIntervals *LIS) const {
527 : auto Flags = MachineMemOperand::MONone;
528 38320 : for (unsigned OpIdx : Ops)
529 : Flags |= MI.getOperand(OpIdx).isDef() ? MachineMemOperand::MOStore
530 : : MachineMemOperand::MOLoad;
531 :
532 76832 : MachineBasicBlock *MBB = MI.getParent();
533 38512 : assert(MBB && "foldMemoryOperand needs an inserted instruction");
534 38512 : MachineFunction &MF = *MBB->getParent();
535 :
536 38320 : // If we're not folding a load into a subreg, the size of the load is the
537 : // size of the spill slot. But if we are, we need to figure out what the
538 38320 : // actual load size is.
539 : int64_t MemSize = 0;
540 : const MachineFrameInfo &MFI = MF.getFrameInfo();
541 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
542 :
543 38320 : if (Flags & MachineMemOperand::MOStore) {
544 38320 : MemSize = MFI.getObjectSize(FI);
545 38320 : } else {
546 : for (unsigned OpIdx : Ops) {
547 76640 : int64_t OpSize = MFI.getObjectSize(FI);
548 15577 :
549 : if (auto SubReg = MI.getOperand(OpIdx).getSubReg()) {
550 45668 : unsigned SubRegSize = TRI->getSubRegIdxSize(SubReg);
551 22925 : if (SubRegSize > 0 && !(SubRegSize % 8))
552 : OpSize = SubRegSize / 8;
553 45850 : }
554 440 :
555 440 : MemSize = std::max(MemSize, OpSize);
556 440 : }
557 : }
558 :
559 22925 : assert(MemSize && "Did not expect a zero-sized stack slot");
560 :
561 : MachineInstr *NewMI = nullptr;
562 :
563 : if (MI.getOpcode() == TargetOpcode::STACKMAP ||
564 : MI.getOpcode() == TargetOpcode::PATCHPOINT ||
565 : MI.getOpcode() == TargetOpcode::STATEPOINT) {
566 : // Fold stackmap/patchpoint.
567 38320 : NewMI = foldPatchpoint(MF, MI, Ops, FI, *this);
568 76499 : if (NewMI)
569 : MBB->insert(MI, NewMI);
570 : } else {
571 141 : // Ask the target to do the actual folding.
572 141 : NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS);
573 : }
574 :
575 : if (NewMI) {
576 76358 : NewMI->setMemRefs(MF, MI.memoperands());
577 : // Add a memory operand, foldMemoryOperandImpl doesn't do that.
578 : assert((!(Flags & MachineMemOperand::MOStore) ||
579 38320 : NewMI->mayStore()) &&
580 6913 : "Folded a def to a non-store!");
581 : assert((!(Flags & MachineMemOperand::MOLoad) ||
582 : NewMI->mayLoad()) &&
583 : "Folded a use to a non-load!");
584 : assert(MFI.getObjectOffset(FI) != -1);
585 : MachineMemOperand *MMO = MF.getMachineMemOperand(
586 : MachinePointerInfo::getFixedStack(MF, FI), Flags, MemSize,
587 : MFI.getObjectAlignment(FI));
588 : NewMI->addMemOperand(MF, MMO);
589 6913 :
590 : return NewMI;
591 : }
592 6913 :
593 : // Straight COPY may fold as load/store.
594 6913 : if (!MI.isCopy() || Ops.size() != 1)
595 : return nullptr;
596 :
597 : const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]);
598 31407 : if (!RC)
599 : return nullptr;
600 :
601 19003 : const MachineOperand &MO = MI.getOperand(1 - Ops[0]);
602 19003 : MachineBasicBlock::iterator Pos = MI;
603 :
604 : if (Flags == MachineMemOperand::MOStore)
605 18595 : storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI);
606 : else
607 : loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI);
608 18595 : return &*--Pos;
609 17850 : }
610 :
611 9670 : MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
612 18595 : ArrayRef<unsigned> Ops,
613 : MachineInstr &LoadMI,
614 : LiveIntervals *LIS) const {
615 140674 : assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!");
616 : #ifndef NDEBUG
617 : for (unsigned OpIdx : Ops)
618 : assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!");
619 : #endif
620 :
621 : MachineBasicBlock &MBB = *MI.getParent();
622 : MachineFunction &MF = *MBB.getParent();
623 :
624 : // Ask the target to do the actual folding.
625 140674 : MachineInstr *NewMI = nullptr;
626 140674 : int FrameIndex = 0;
627 :
628 : if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
629 : MI.getOpcode() == TargetOpcode::PATCHPOINT ||
630 140674 : MI.getOpcode() == TargetOpcode::STATEPOINT) &&
631 : isLoadFromStackSlot(LoadMI, FrameIndex)) {
632 140674 : // Fold stackmap/patchpoint.
633 140582 : NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
634 140789 : if (NewMI)
635 115 : NewMI = &*MBB.insert(MI, NewMI);
636 : } else {
637 115 : // Ask the target to do the actual folding.
638 115 : NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, LIS);
639 : }
640 :
641 : if (!NewMI)
642 281118 : return nullptr;
643 :
644 : // Copy the memoperands from the load to the folded instruction.
645 140674 : if (MI.memoperands_empty()) {
646 : NewMI->setMemRefs(MF, LoadMI.memoperands());
647 : } else {
648 : // Handle the rare case of folding multiple loads.
649 6834 : NewMI->setMemRefs(MF, MI.memoperands());
650 6771 : for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(),
651 : E = LoadMI.memoperands_end();
652 : I != E; ++I) {
653 63 : NewMI->addMemOperand(MF, *I);
654 63 : }
655 : }
656 126 : return NewMI;
657 63 : }
658 :
659 : bool TargetInstrInfo::hasReassociableOperands(
660 : const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
661 : const MachineOperand &Op1 = Inst.getOperand(1);
662 : const MachineOperand &Op2 = Inst.getOperand(2);
663 41552 : const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
664 :
665 41552 : // We need virtual register definitions for the operands that we will
666 : // reassociate.
667 41552 : MachineInstr *MI1 = nullptr;
668 : MachineInstr *MI2 = nullptr;
669 : if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg()))
670 : MI1 = MRI.getUniqueVRegDef(Op1.getReg());
671 : if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg()))
672 : MI2 = MRI.getUniqueVRegDef(Op2.getReg());
673 41552 :
674 41552 : // And they need to be in the trace (otherwise, they won't have a depth).
675 41552 : return MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB;
676 41552 : }
677 :
678 : bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
679 41552 : bool &Commuted) const {
680 : const MachineBasicBlock *MBB = Inst.getParent();
681 : const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
682 34812 : MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
683 : MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
684 34812 : unsigned AssocOpcode = Inst.getOpcode();
685 34812 :
686 34812 : // If only one operand has the same opcode and it's the second source operand,
687 34812 : // the operands must be commuted.
688 34812 : Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode;
689 : if (Commuted)
690 : std::swap(MI1, MI2);
691 :
692 69624 : // 1. The previous instruction must be the same type as Inst.
693 34812 : // 2. The previous instruction must have virtual register definitions for its
694 : // operands in the same basic block as Inst.
695 : // 3. The previous instruction's result must only be used by Inst.
696 : return MI1->getOpcode() == AssocOpcode &&
697 : hasReassociableOperands(*MI1, MBB) &&
698 : MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg());
699 : }
700 39734 :
701 39655 : // 1. The operation must be associative and commutative.
702 4843 : // 2. The instruction must have virtual register definitions for its
703 : // operands in the same basic block.
704 : // 3. The instruction must have a reassociable sibling.
705 : bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst,
706 : bool &Commuted) const {
707 : return isAssociativeAndCommutative(Inst) &&
708 : hasReassociableOperands(Inst, Inst.getParent()) &&
709 3484520 : hasReassociableSibling(Inst, Commuted);
710 : }
711 3521626 :
712 3519332 : // The concept of the reassociation pass is that these operations can benefit
713 34812 : // from this kind of transformation:
714 : //
715 : // A = ? op ?
716 : // B = A op X (Prev)
717 : // C = B op Y (Root)
718 : // -->
719 : // A = ? op ?
720 : // B = X op Y
721 : // C = A op B
722 : //
723 : // breaking the dependency between A and B, allowing them to be executed in
724 : // parallel (or back-to-back in a pipeline) instead of depending on each other.
725 :
726 : // FIXME: This has the potential to be expensive (compile time) while not
727 : // improving the code at all. Some ways to limit the overhead:
728 : // 1. Track successful transforms; bail out if hit rate gets too low.
729 : // 2. Only enable at -O3 or some other non-default optimization level.
730 : // 3. Pre-screen pattern candidates here: if an operand of the previous
731 : // instruction is known to not increase the critical path, then don't match
732 : // that pattern.
733 : bool TargetInstrInfo::getMachineCombinerPatterns(
734 : MachineInstr &Root,
735 : SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
736 : bool Commute;
737 3484520 : if (isReassociationCandidate(Root, Commute)) {
738 : // We found a sequence of instructions that may be suitable for a
739 : // reassociation of operands to increase ILP. Specify each commutation
740 : // possibility for the Prev instruction in the sequence and let the
741 3484520 : // machine combiner decide if changing the operands is worthwhile.
742 : if (Commute) {
743 : Patterns.push_back(MachineCombinerPattern::REASSOC_AX_YB);
744 : Patterns.push_back(MachineCombinerPattern::REASSOC_XA_YB);
745 : } else {
746 2671 : Patterns.push_back(MachineCombinerPattern::REASSOC_AX_BY);
747 445 : Patterns.push_back(MachineCombinerPattern::REASSOC_XA_BY);
748 445 : }
749 : return true;
750 2226 : }
751 2226 :
752 : return false;
753 2671 : }
754 :
755 : /// Return true when a code sequence can improve loop throughput.
756 : bool
757 : TargetInstrInfo::isThroughputPattern(MachineCombinerPattern Pattern) const {
758 : return false;
759 : }
760 :
761 15 : /// Attempt the reassociation transformation to reduce critical path length.
762 15 : /// See the above comments before getMachineCombinerPatterns().
763 : void TargetInstrInfo::reassociateOps(
764 : MachineInstr &Root, MachineInstr &Prev,
765 : MachineCombinerPattern Pattern,
766 : SmallVectorImpl<MachineInstr *> &InsInstrs,
767 3366 : SmallVectorImpl<MachineInstr *> &DelInstrs,
768 : DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
769 : MachineFunction *MF = Root.getMF();
770 : MachineRegisterInfo &MRI = MF->getRegInfo();
771 : const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
772 : const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
773 : const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
774 3366 :
775 3366 : // This array encodes the operand index for each parameter because the
776 3366 : // operands may be commuted. Each row corresponds to a pattern value,
777 3366 : // and each column specifies the index of A, B, X, Y.
778 : unsigned OpIdx[4][4] = {
779 : { 1, 1, 2, 2 },
780 : { 1, 2, 2, 1 },
781 : { 2, 1, 1, 2 },
782 3366 : { 2, 2, 1, 1 }
783 : };
784 :
785 : int Row;
786 : switch (Pattern) {
787 : case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break;
788 : case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break;
789 : case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break;
790 : case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break;
791 : default: llvm_unreachable("unexpected MachineCombinerPattern");
792 : }
793 :
794 : MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]);
795 0 : MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]);
796 : MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]);
797 : MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]);
798 3366 : MachineOperand &OpC = Root.getOperand(0);
799 3366 :
800 3366 : unsigned RegA = OpA.getReg();
801 3366 : unsigned RegB = OpB.getReg();
802 : unsigned RegX = OpX.getReg();
803 : unsigned RegY = OpY.getReg();
804 3366 : unsigned RegC = OpC.getReg();
805 3366 :
806 3366 : if (TargetRegisterInfo::isVirtualRegister(RegA))
807 3366 : MRI.constrainRegClass(RegA, RC);
808 3366 : if (TargetRegisterInfo::isVirtualRegister(RegB))
809 : MRI.constrainRegClass(RegB, RC);
810 3366 : if (TargetRegisterInfo::isVirtualRegister(RegX))
811 3366 : MRI.constrainRegClass(RegX, RC);
812 3366 : if (TargetRegisterInfo::isVirtualRegister(RegY))
813 3366 : MRI.constrainRegClass(RegY, RC);
814 3366 : if (TargetRegisterInfo::isVirtualRegister(RegC))
815 3366 : MRI.constrainRegClass(RegC, RC);
816 3366 :
817 3366 : // Create a new virtual register for the result of (X op Y) instead of
818 3366 : // recycling RegB because the MachineCombiner's computation of the critical
819 3366 : // path requires a new register definition rather than an existing one.
820 : unsigned NewVR = MRI.createVirtualRegister(RC);
821 : InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
822 :
823 : unsigned Opcode = Root.getOpcode();
824 3366 : bool KillA = OpA.isKill();
825 3366 : bool KillX = OpX.isKill();
826 : bool KillY = OpY.isKill();
827 3366 :
828 : // Create new instructions for insertion.
829 : MachineInstrBuilder MIB1 =
830 : BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
831 : .addReg(RegX, getKillRegState(KillX))
832 : .addReg(RegY, getKillRegState(KillY));
833 : MachineInstrBuilder MIB2 =
834 6732 : BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
835 3366 : .addReg(RegA, getKillRegState(KillA))
836 3366 : .addReg(NewVR, getKillRegState(true));
837 :
838 6732 : setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2);
839 3366 :
840 3366 : // Record new instructions for insertion and old instructions for deletion.
841 : InsInstrs.push_back(MIB1);
842 3366 : InsInstrs.push_back(MIB2);
843 : DelInstrs.push_back(&Prev);
844 : DelInstrs.push_back(&Root);
845 3366 : }
846 3366 :
847 3366 : void TargetInstrInfo::genAlternativeCodeSequence(
848 3366 : MachineInstr &Root, MachineCombinerPattern Pattern,
849 3366 : SmallVectorImpl<MachineInstr *> &InsInstrs,
850 : SmallVectorImpl<MachineInstr *> &DelInstrs,
851 3366 : DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const {
852 : MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
853 :
854 : // Select the previous instruction in the sequence based on the input pattern.
855 : MachineInstr *Prev = nullptr;
856 3366 : switch (Pattern) {
857 : case MachineCombinerPattern::REASSOC_AX_BY:
858 : case MachineCombinerPattern::REASSOC_XA_BY:
859 : Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
860 3366 : break;
861 2464 : case MachineCombinerPattern::REASSOC_AX_YB:
862 : case MachineCombinerPattern::REASSOC_XA_YB:
863 2464 : Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
864 2464 : break;
865 902 : default:
866 : break;
867 902 : }
868 902 :
869 : assert(Prev && "Unknown pattern for machine combiner");
870 :
871 : reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
872 : }
873 :
874 : bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric(
875 3366 : const MachineInstr &MI, AliasAnalysis *AA) const {
876 3366 : const MachineFunction &MF = *MI.getMF();
877 : const MachineRegisterInfo &MRI = MF.getRegInfo();
878 294556 :
879 : // Remat clients assume operand 0 is the defined register.
880 294556 : if (!MI.getNumOperands() || !MI.getOperand(0).isReg())
881 294556 : return false;
882 : unsigned DefReg = MI.getOperand(0).getReg();
883 :
884 294556 : // A sub-register definition can only be rematerialized if the instruction
885 : // doesn't read the other parts of the register. Otherwise it is really a
886 294556 : // read-modify-write operation on the full virtual register which cannot be
887 : // moved safely.
888 : if (TargetRegisterInfo::isVirtualRegister(DefReg) &&
889 : MI.getOperand(0).getSubReg() && MI.readsVirtualRegister(DefReg))
890 : return false;
891 :
892 294556 : // A load from a fixed stack slot can be rematerialized. This may be
893 307617 : // redundant with subsequent checks, but it's target-independent,
894 : // simple, and a common case.
895 : int FrameIdx = 0;
896 : if (isLoadFromStackSlot(MI, FrameIdx) &&
897 : MF.getFrameInfo().isImmutableObjectIndex(FrameIdx))
898 : return true;
899 288052 :
900 321588 : // Avoid instructions obviously unsafe for remat.
901 33951 : if (MI.isNotDuplicable() || MI.mayStore() || MI.hasUnmodeledSideEffects())
902 : return false;
903 :
904 : // Don't remat inline asm. We have no idea how expensive it is
905 269801 : // even if it's side effect free.
906 22 : if (MI.isInlineAsm())
907 : return false;
908 :
909 : // Avoid instructions which load from potentially varying memory.
910 269779 : if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad(AA))
911 : return false;
912 :
913 : // If any of the registers accessed are non-constant, conservatively assume
914 269779 : // the instruction is not rematerializable.
915 : for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
916 : const MachineOperand &MO = MI.getOperand(i);
917 : if (!MO.isReg()) continue;
918 : unsigned Reg = MO.getReg();
919 842357 : if (Reg == 0)
920 725211 : continue;
921 725211 :
922 503602 : // Check for a well-behaved physical register.
923 503602 : if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
924 : if (MO.isUse()) {
925 : // If the physreg has no defs anywhere, it's just an ambient register
926 : // and we can freely move its uses. Alternatively, if it's allocatable,
927 312140 : // it could get allocated to something with a def during allocation.
928 88628 : if (!MRI.isConstantPhysReg(Reg))
929 : return false;
930 : } else {
931 : // A physreg def. We can't remat it.
932 88628 : return false;
933 : }
934 : continue;
935 : }
936 :
937 : // Only allow one virtual-register def. There may be multiple defs of the
938 : // same virtual register, though.
939 : if (MO.isDef() && Reg != DefReg)
940 : return false;
941 :
942 : // Don't allow any virtual-register uses. Rematting an instruction with
943 223512 : // virtual register uses would length the live ranges of the uses, which
944 : // is not necessarily a good idea, certainly not "trivial".
945 : if (MO.isUse())
946 : return false;
947 : }
948 :
949 223512 : // Everything checked out.
950 : return true;
951 : }
952 :
953 : int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const {
954 : const MachineFunction *MF = MI.getMF();
955 : const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
956 : bool StackGrowsDown =
957 1874 : TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
958 1874 :
959 1874 : unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
960 : unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
961 1874 :
962 : if (!isFrameInstr(MI))
963 1874 : return 0;
964 1874 :
965 : int SPAdj = TFI->alignSPAdjust(getFrameSize(MI));
966 :
967 : if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) ||
968 : (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode))
969 100 : SPAdj = -SPAdj;
970 :
971 100 : return SPAdj;
972 100 : }
973 50 :
974 : /// isSchedulingBoundary - Test if the given instruction should be
975 : /// considered a scheduling boundary. This primarily includes labels
976 : /// and terminators.
977 : bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
978 : const MachineBasicBlock *MBB,
979 : const MachineFunction &MF) const {
980 : // Terminators and labels can't be scheduled around.
981 4785995 : if (MI.isTerminator() || MI.isPosition())
982 : return true;
983 :
984 : // Don't attempt to schedule around any instruction that defines
985 4785995 : // a stack-oriented pointer, as it's unlikely to be profitable. This
986 : // saves compile time, because it doesn't require every single
987 : // stack slot reference to depend on the instruction that does the
988 : // modification.
989 : const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
990 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
991 : return MI.modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI);
992 : }
993 4063643 :
994 4063643 : // Provide a global flag for disabling the PreRA hazard recognizer that targets
995 4063643 : // may choose to honor.
996 : bool TargetInstrInfo::usePreRAHazardRecognizer() const {
997 : return !DisableHazardRecognizer;
998 : }
999 :
1000 11322 : // Default implementation of CreateTargetRAHazardRecognizer.
1001 11322 : ScheduleHazardRecognizer *TargetInstrInfo::
1002 : CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
1003 : const ScheduleDAG *DAG) const {
1004 : // Dummy hazard recognizer allows all instructions to issue.
1005 15899 : return new ScheduleHazardRecognizer();
1006 : }
1007 :
1008 : // Default implementation of CreateTargetMIHazardRecognizer.
1009 15899 : ScheduleHazardRecognizer *TargetInstrInfo::
1010 : CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1011 : const ScheduleDAG *DAG) const {
1012 : return (ScheduleHazardRecognizer *)
1013 340371 : new ScoreboardHazardRecognizer(II, DAG, "misched");
1014 : }
1015 :
1016 : // Default implementation of CreateTargetPostRAHazardRecognizer.
1017 340371 : ScheduleHazardRecognizer *TargetInstrInfo::
1018 : CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1019 : const ScheduleDAG *DAG) const {
1020 : return (ScheduleHazardRecognizer *)
1021 8671 : new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched");
1022 : }
1023 :
1024 : //===----------------------------------------------------------------------===//
1025 8671 : // SelectionDAG latency interface.
1026 : //===----------------------------------------------------------------------===//
1027 :
1028 : int
1029 : TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
1030 : SDNode *DefNode, unsigned DefIdx,
1031 : SDNode *UseNode, unsigned UseIdx) const {
1032 : if (!ItinData || ItinData->isEmpty())
1033 98820 : return -1;
1034 :
1035 : if (!DefNode->isMachineOpcode())
1036 98820 : return -1;
1037 :
1038 : unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass();
1039 0 : if (!UseNode->isMachineOpcode())
1040 : return ItinData->getOperandCycle(DefClass, DefIdx);
1041 : unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass();
1042 0 : return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1043 0 : }
1044 :
1045 0 : int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1046 0 : SDNode *N) const {
1047 : if (!ItinData || ItinData->isEmpty())
1048 : return 1;
1049 0 :
1050 : if (!N->isMachineOpcode())
1051 0 : return 1;
1052 :
1053 : return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass());
1054 0 : }
1055 :
1056 : //===----------------------------------------------------------------------===//
1057 0 : // MachineInstr latency interface.
1058 : //===----------------------------------------------------------------------===//
1059 :
1060 : unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
1061 : const MachineInstr &MI) const {
1062 : if (!ItinData || ItinData->isEmpty())
1063 : return 1;
1064 0 :
1065 : unsigned Class = MI.getDesc().getSchedClass();
1066 0 : int UOps = ItinData->Itineraries[Class].NumMicroOps;
1067 : if (UOps >= 0)
1068 : return UOps;
1069 0 :
1070 0 : // The # of u-ops is dynamically determined. The specific target should
1071 0 : // override this function to return the right number.
1072 0 : return 1;
1073 : }
1074 :
1075 : /// Return the default expected latency for a def based on it's opcode.
1076 : unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel,
1077 : const MachineInstr &DefMI) const {
1078 : if (DefMI.isTransient())
1079 : return 0;
1080 4559496 : if (DefMI.mayLoad())
1081 : return SchedModel.LoadLatency;
1082 : if (isHighLatencyDef(DefMI.getOpcode()))
1083 : return SchedModel.HighLatency;
1084 3617675 : return 1;
1085 1379813 : }
1086 4475724 :
1087 1788 : unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const {
1088 : return 0;
1089 : }
1090 :
1091 33999 : unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1092 33999 : const MachineInstr &MI,
1093 : unsigned *PredCost) const {
1094 : // Default to one cycle for no itinerary. However, an "empty" itinerary may
1095 17305 : // still have a MinLatency property, which getStageLatency checks.
1096 : if (!ItinData)
1097 : return MI.mayLoad() ? 2 : 1;
1098 :
1099 : return ItinData->getStageLatency(MI.getDesc().getSchedClass());
1100 17305 : }
1101 8333 :
1102 : bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
1103 17944 : const MachineInstr &DefMI,
1104 : unsigned DefIdx) const {
1105 : const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
1106 24937 : if (!ItinData || ItinData->isEmpty())
1107 : return false;
1108 :
1109 : unsigned DefClass = DefMI.getDesc().getSchedClass();
1110 163 : int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
1111 : return (DefCycle != -1 && DefCycle <= 1);
1112 : }
1113 163 :
1114 : /// Both DefMI and UseMI must be valid. By default, call directly to the
1115 163 : /// itinerary. This may be overriden by the target.
1116 : int TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
1117 : const MachineInstr &DefMI,
1118 : unsigned DefIdx,
1119 : const MachineInstr &UseMI,
1120 307257 : unsigned UseIdx) const {
1121 : unsigned DefClass = DefMI.getDesc().getSchedClass();
1122 : unsigned UseClass = UseMI.getDesc().getSchedClass();
1123 : return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1124 : }
1125 307257 :
1126 307257 : /// If we can determine the operand latency from the def only, without itinerary
1127 307257 : /// lookup, do so. Otherwise return -1.
1128 : int TargetInstrInfo::computeDefOperandLatency(
1129 : const InstrItineraryData *ItinData, const MachineInstr &DefMI) const {
1130 :
1131 : // Let the target hook getInstrLatency handle missing itineraries.
1132 0 : if (!ItinData)
1133 : return getInstrLatency(ItinData, DefMI);
1134 :
1135 : if(ItinData->isEmpty())
1136 0 : return defaultDefLatency(ItinData->SchedModel, DefMI);
1137 0 :
1138 : // ...operand lookup required
1139 0 : return -1;
1140 0 : }
1141 :
1142 : bool TargetInstrInfo::getRegSequenceInputs(
1143 : const MachineInstr &MI, unsigned DefIdx,
1144 : SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
1145 : assert((MI.isRegSequence() ||
1146 244956 : MI.isRegSequenceLike()) && "Instruction do not have the proper type");
1147 :
1148 : if (!MI.isRegSequence())
1149 : return getRegSequenceLikeInputs(MI, DefIdx, InputRegs);
1150 :
1151 : // We are looking at:
1152 244956 : // Def = REG_SEQUENCE v0, sub0, v1, sub1, ...
1153 1595 : assert(DefIdx == 0 && "REG_SEQUENCE only has one def");
1154 : for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx;
1155 : OpIdx += 2) {
1156 : const MachineOperand &MOReg = MI.getOperand(OpIdx);
1157 : if (MOReg.isUndef())
1158 945267 : continue;
1159 701906 : const MachineOperand &MOSubIdx = MI.getOperand(OpIdx + 1);
1160 701906 : assert(MOSubIdx.isImm() &&
1161 701906 : "One of the subindex of the reg_sequence is not an immediate");
1162 : // Record Reg:SubReg, SubIdx.
1163 701906 : InputRegs.push_back(RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(),
1164 : (unsigned)MOSubIdx.getImm()));
1165 : }
1166 : return true;
1167 1403812 : }
1168 701906 :
1169 : bool TargetInstrInfo::getExtractSubregInputs(
1170 : const MachineInstr &MI, unsigned DefIdx,
1171 : RegSubRegPairAndIdx &InputReg) const {
1172 : assert((MI.isExtractSubreg() ||
1173 3121 : MI.isExtractSubregLike()) && "Instruction do not have the proper type");
1174 :
1175 : if (!MI.isExtractSubreg())
1176 : return getExtractSubregLikeInputs(MI, DefIdx, InputReg);
1177 :
1178 : // We are looking at:
1179 3121 : // Def = EXTRACT_SUBREG v0.sub1, sub0.
1180 3121 : assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def");
1181 : const MachineOperand &MOReg = MI.getOperand(1);
1182 : if (MOReg.isUndef())
1183 : return false;
1184 : const MachineOperand &MOSubIdx = MI.getOperand(2);
1185 0 : assert(MOSubIdx.isImm() &&
1186 0 : "The subindex of the extract_subreg is not an immediate");
1187 :
1188 : InputReg.Reg = MOReg.getReg();
1189 : InputReg.SubReg = MOReg.getSubReg();
1190 : InputReg.SubIdx = (unsigned)MOSubIdx.getImm();
1191 : return true;
1192 0 : }
1193 0 :
1194 0 : bool TargetInstrInfo::getInsertSubregInputs(
1195 0 : const MachineInstr &MI, unsigned DefIdx,
1196 : RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const {
1197 : assert((MI.isInsertSubreg() ||
1198 28764 : MI.isInsertSubregLike()) && "Instruction do not have the proper type");
1199 :
1200 : if (!MI.isInsertSubreg())
1201 : return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg);
1202 :
1203 : // We are looking at:
1204 28764 : // Def = INSERT_SEQUENCE v0, v1, sub0.
1205 193 : assert(DefIdx == 0 && "INSERT_SUBREG only has one def");
1206 : const MachineOperand &MOBaseReg = MI.getOperand(1);
1207 : const MachineOperand &MOInsertedReg = MI.getOperand(2);
1208 : if (MOInsertedReg.isUndef())
1209 : return false;
1210 28571 : const MachineOperand &MOSubIdx = MI.getOperand(3);
1211 : assert(MOSubIdx.isImm() &&
1212 28571 : "One of the subindex of the reg_sequence is not an immediate");
1213 : BaseReg.Reg = MOBaseReg.getReg();
1214 : BaseReg.SubReg = MOBaseReg.getSubReg();
1215 :
1216 : InsertedReg.Reg = MOInsertedReg.getReg();
1217 28571 : InsertedReg.SubReg = MOInsertedReg.getSubReg();
1218 28571 : InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm();
1219 : return true;
1220 28571 : }
|
