LLVM 18.0.0git
GCNDPPCombine.cpp
Go to the documentation of this file.
1//=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
9// operand. If any of the use instruction cannot be combined with the mov the
10// whole sequence is reverted.
11//
12// $old = ...
13// $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
14// dpp_controls..., $row_mask, $bank_mask, $bound_ctrl
15// $res = VALU $dpp_value [, src1]
16//
17// to
18//
19// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]
20// dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
21//
22// Combining rules :
23//
24// if $row_mask and $bank_mask are fully enabled (0xF) and
25// $bound_ctrl==DPP_BOUND_ZERO or $old==0
26// -> $combined_old = undef,
27// $combined_bound_ctrl = DPP_BOUND_ZERO
28//
29// if the VALU op is binary and
30// $bound_ctrl==DPP_BOUND_OFF and
31// $old==identity value (immediate) for the VALU op
32// -> $combined_old = src1,
33// $combined_bound_ctrl = DPP_BOUND_OFF
34//
35// Otherwise cancel.
36//
37// The mov_dpp instruction should reside in the same BB as all its uses
38//===----------------------------------------------------------------------===//
39
40#include "AMDGPU.h"
41#include "GCNSubtarget.h"
43#include "llvm/ADT/Statistic.h"
45
46using namespace llvm;
47
48#define DEBUG_TYPE "gcn-dpp-combine"
49
50STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");
51
52namespace {
53
54class GCNDPPCombine : public MachineFunctionPass {
56 const SIInstrInfo *TII;
57 const GCNSubtarget *ST;
58
60
61 MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
62
63 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
64 RegSubRegPair CombOldVGPR,
65 MachineOperand *OldOpnd, bool CombBCZ,
66 bool IsShrinkable) const;
67
68 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
69 RegSubRegPair CombOldVGPR, bool CombBCZ,
70 bool IsShrinkable) const;
71
72 bool hasNoImmOrEqual(MachineInstr &MI,
73 unsigned OpndName,
74 int64_t Value,
75 int64_t Mask = -1) const;
76
77 bool combineDPPMov(MachineInstr &MI) const;
78
79public:
80 static char ID;
81
82 GCNDPPCombine() : MachineFunctionPass(ID) {
84 }
85
86 bool runOnMachineFunction(MachineFunction &MF) override;
87
88 StringRef getPassName() const override { return "GCN DPP Combine"; }
89
90 void getAnalysisUsage(AnalysisUsage &AU) const override {
91 AU.setPreservesCFG();
93 }
94
97 .set(MachineFunctionProperties::Property::IsSSA);
98 }
99
100private:
101 int getDPPOp(unsigned Op, bool IsShrinkable) const;
102 bool isShrinkable(MachineInstr &MI) const;
103};
104
105} // end anonymous namespace
106
107INITIALIZE_PASS(GCNDPPCombine, DEBUG_TYPE, "GCN DPP Combine", false, false)
108
109char GCNDPPCombine::ID = 0;
110
111char &llvm::GCNDPPCombineID = GCNDPPCombine::ID;
112
114 return new GCNDPPCombine();
115}
116
117bool GCNDPPCombine::isShrinkable(MachineInstr &MI) const {
118 unsigned Op = MI.getOpcode();
119 if (!TII->isVOP3(Op)) {
120 return false;
121 }
122 if (!TII->hasVALU32BitEncoding(Op)) {
123 LLVM_DEBUG(dbgs() << " Inst hasn't e32 equivalent\n");
124 return false;
125 }
126 // Do not shrink True16 instructions pre-RA to avoid the restriction in
127 // register allocation from only being able to use 128 VGPRs
129 return false;
130 if (const auto *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) {
131 // Give up if there are any uses of the sdst in carry-out or VOPC.
132 // The shrunken form of the instruction would write it to vcc instead of to
133 // a virtual register. If we rewrote the uses the shrinking would be
134 // possible.
135 if (!MRI->use_nodbg_empty(SDst->getReg()))
136 return false;
137 }
138 // check if other than abs|neg modifiers are set (opsel for example)
139 const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
140 if (!hasNoImmOrEqual(MI, AMDGPU::OpName::src0_modifiers, 0, Mask) ||
141 !hasNoImmOrEqual(MI, AMDGPU::OpName::src1_modifiers, 0, Mask) ||
142 !hasNoImmOrEqual(MI, AMDGPU::OpName::clamp, 0) ||
143 !hasNoImmOrEqual(MI, AMDGPU::OpName::omod, 0)) {
144 LLVM_DEBUG(dbgs() << " Inst has non-default modifiers\n");
145 return false;
146 }
147 return true;
148}
149
150int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {
151 int DPP32 = AMDGPU::getDPPOp32(Op);
152 if (IsShrinkable) {
153 assert(DPP32 == -1);
154 int E32 = AMDGPU::getVOPe32(Op);
155 DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(E32);
156 }
157 if (DPP32 != -1 && TII->pseudoToMCOpcode(DPP32) != -1)
158 return DPP32;
159 int DPP64 = -1;
160 if (ST->hasVOP3DPP())
161 DPP64 = AMDGPU::getDPPOp64(Op);
162 if (DPP64 != -1 && TII->pseudoToMCOpcode(DPP64) != -1)
163 return DPP64;
164 return -1;
165}
166
167// tracks the register operand definition and returns:
168// 1. immediate operand used to initialize the register if found
169// 2. nullptr if the register operand is undef
170// 3. the operand itself otherwise
171MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
172 auto *Def = getVRegSubRegDef(getRegSubRegPair(OldOpnd), *MRI);
173 if (!Def)
174 return nullptr;
175
176 switch(Def->getOpcode()) {
177 default: break;
178 case AMDGPU::IMPLICIT_DEF:
179 return nullptr;
180 case AMDGPU::COPY:
181 case AMDGPU::V_MOV_B32_e32:
182 case AMDGPU::V_MOV_B64_PSEUDO:
183 case AMDGPU::V_MOV_B64_e32:
184 case AMDGPU::V_MOV_B64_e64: {
185 auto &Op1 = Def->getOperand(1);
186 if (Op1.isImm())
187 return &Op1;
188 break;
189 }
190 }
191 return &OldOpnd;
192}
193
194[[maybe_unused]] static unsigned getOperandSize(MachineInstr &MI, unsigned Idx,
196 int16_t RegClass = MI.getDesc().operands()[Idx].RegClass;
197 if (RegClass == -1)
198 return 0;
199
200 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
201 return TRI->getRegSizeInBits(*TRI->getRegClass(RegClass));
202}
203
204MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
205 MachineInstr &MovMI,
206 RegSubRegPair CombOldVGPR,
207 bool CombBCZ,
208 bool IsShrinkable) const {
209 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
210 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
211 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
212
213 bool HasVOP3DPP = ST->hasVOP3DPP();
214 auto OrigOp = OrigMI.getOpcode();
215 auto DPPOp = getDPPOp(OrigOp, IsShrinkable);
216 if (DPPOp == -1) {
217 LLVM_DEBUG(dbgs() << " failed: no DPP opcode\n");
218 return nullptr;
219 }
220 int OrigOpE32 = AMDGPU::getVOPe32(OrigOp);
221 // Prior checks cover Mask with VOPC condition, but not on purpose
222 auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);
223 assert(RowMaskOpnd && RowMaskOpnd->isImm());
224 auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);
225 assert(BankMaskOpnd && BankMaskOpnd->isImm());
226 const bool MaskAllLanes =
227 RowMaskOpnd->getImm() == 0xF && BankMaskOpnd->getImm() == 0xF;
228 (void)MaskAllLanes;
229 assert((MaskAllLanes ||
230 !(TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&
231 TII->isVOPC(OrigOpE32)))) &&
232 "VOPC cannot form DPP unless mask is full");
233
234 auto DPPInst = BuildMI(*OrigMI.getParent(), OrigMI,
235 OrigMI.getDebugLoc(), TII->get(DPPOp))
236 .setMIFlags(OrigMI.getFlags());
237
238 bool Fail = false;
239 do {
240 int NumOperands = 0;
241 if (auto *Dst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst)) {
242 DPPInst.add(*Dst);
243 ++NumOperands;
244 }
245 if (auto *SDst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::sdst)) {
246 if (TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, SDst)) {
247 DPPInst.add(*SDst);
248 ++NumOperands;
249 }
250 // If we shrunk a 64bit vop3b to 32bits, just ignore the sdst
251 }
252
253 const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old);
254 if (OldIdx != -1) {
255 assert(OldIdx == NumOperands);
257 CombOldVGPR,
258 *MRI->getRegClass(
259 TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()),
260 *MRI));
261 auto *Def = getVRegSubRegDef(CombOldVGPR, *MRI);
262 DPPInst.addReg(CombOldVGPR.Reg, Def ? 0 : RegState::Undef,
263 CombOldVGPR.SubReg);
264 ++NumOperands;
265 } else if (TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&
266 TII->isVOPC(OrigOpE32))) {
267 // VOPC DPP and VOPC promoted to VOP3 DPP do not have an old operand
268 // because they write to SGPRs not VGPRs
269 } else {
270 // TODO: this discards MAC/FMA instructions for now, let's add it later
271 LLVM_DEBUG(dbgs() << " failed: no old operand in DPP instruction,"
272 " TBD\n");
273 Fail = true;
274 break;
275 }
276
277 if (auto *Mod0 = TII->getNamedOperand(OrigMI,
278 AMDGPU::OpName::src0_modifiers)) {
279 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
280 AMDGPU::OpName::src0_modifiers));
281 assert(HasVOP3DPP ||
282 (0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
283 DPPInst.addImm(Mod0->getImm());
284 ++NumOperands;
285 } else if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src0_modifiers)) {
286 DPPInst.addImm(0);
287 ++NumOperands;
288 }
289 auto *Src0 = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);
290 assert(Src0);
291 int Src0Idx = NumOperands;
292 if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src0)) {
293 LLVM_DEBUG(dbgs() << " failed: src0 is illegal\n");
294 Fail = true;
295 break;
296 }
297 DPPInst.add(*Src0);
298 DPPInst->getOperand(NumOperands).setIsKill(false);
299 ++NumOperands;
300
301 if (auto *Mod1 = TII->getNamedOperand(OrigMI,
302 AMDGPU::OpName::src1_modifiers)) {
303 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
304 AMDGPU::OpName::src1_modifiers));
305 assert(HasVOP3DPP ||
306 (0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
307 DPPInst.addImm(Mod1->getImm());
308 ++NumOperands;
309 } else if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src1_modifiers)) {
310 DPPInst.addImm(0);
311 ++NumOperands;
312 }
313 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
314 if (Src1) {
315 int OpNum = NumOperands;
316 // If subtarget does not support SGPRs for src1 operand then the
317 // requirements are the same as for src0. We check src0 instead because
318 // pseudos are shared between subtargets and allow SGPR for src1 on all.
319 if (!ST->hasDPPSrc1SGPR()) {
320 assert(getOperandSize(*DPPInst, Src0Idx, *MRI) ==
321 getOperandSize(*DPPInst, NumOperands, *MRI) &&
322 "Src0 and Src1 operands should have the same size");
323 OpNum = Src0Idx;
324 }
325 if (!TII->isOperandLegal(*DPPInst.getInstr(), OpNum, Src1)) {
326 LLVM_DEBUG(dbgs() << " failed: src1 is illegal\n");
327 Fail = true;
328 break;
329 }
330 DPPInst.add(*Src1);
331 ++NumOperands;
332 }
333 if (auto *Mod2 =
334 TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2_modifiers)) {
335 assert(NumOperands ==
336 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::src2_modifiers));
337 assert(HasVOP3DPP ||
338 (0LL == (Mod2->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
339 DPPInst.addImm(Mod2->getImm());
340 ++NumOperands;
341 }
342 auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2);
343 if (Src2) {
344 if (!TII->getNamedOperand(*DPPInst.getInstr(), AMDGPU::OpName::src2) ||
345 !TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) {
346 LLVM_DEBUG(dbgs() << " failed: src2 is illegal\n");
347 Fail = true;
348 break;
349 }
350 DPPInst.add(*Src2);
351 ++NumOperands;
352 }
353 if (HasVOP3DPP) {
354 auto *ClampOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::clamp);
355 if (ClampOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::clamp)) {
356 DPPInst.addImm(ClampOpr->getImm());
357 }
358 auto *VdstInOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst_in);
359 if (VdstInOpr &&
360 AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::vdst_in)) {
361 DPPInst.add(*VdstInOpr);
362 }
363 auto *OmodOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::omod);
364 if (OmodOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::omod)) {
365 DPPInst.addImm(OmodOpr->getImm());
366 }
367 // Validate OP_SEL has to be set to all 0 and OP_SEL_HI has to be set to
368 // all 1.
369 if (auto *OpSelOpr =
370 TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel)) {
371 auto OpSel = OpSelOpr->getImm();
372 if (OpSel != 0) {
373 LLVM_DEBUG(dbgs() << " failed: op_sel must be zero\n");
374 Fail = true;
375 break;
376 }
377 if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::op_sel))
378 DPPInst.addImm(OpSel);
379 }
380 if (auto *OpSelHiOpr =
381 TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel_hi)) {
382 auto OpSelHi = OpSelHiOpr->getImm();
383 // Only vop3p has op_sel_hi, and all vop3p have 3 operands, so check
384 // the bitmask for 3 op_sel_hi bits set
385 assert(Src2 && "Expected vop3p with 3 operands");
386 if (OpSelHi != 7) {
387 LLVM_DEBUG(dbgs() << " failed: op_sel_hi must be all set to one\n");
388 Fail = true;
389 break;
390 }
391 if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::op_sel_hi))
392 DPPInst.addImm(OpSelHi);
393 }
394 auto *NegOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_lo);
395 if (NegOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::neg_lo)) {
396 DPPInst.addImm(NegOpr->getImm());
397 }
398 auto *NegHiOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_hi);
399 if (NegHiOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::neg_hi)) {
400 DPPInst.addImm(NegHiOpr->getImm());
401 }
402 }
403 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl));
404 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask));
405 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask));
406 DPPInst.addImm(CombBCZ ? 1 : 0);
407 } while (false);
408
409 if (Fail) {
410 DPPInst.getInstr()->eraseFromParent();
411 return nullptr;
412 }
413 LLVM_DEBUG(dbgs() << " combined: " << *DPPInst.getInstr());
414 return DPPInst.getInstr();
415}
416
417static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
418 assert(OldOpnd->isImm());
419 switch (OrigMIOp) {
420 default: break;
421 case AMDGPU::V_ADD_U32_e32:
422 case AMDGPU::V_ADD_U32_e64:
423 case AMDGPU::V_ADD_CO_U32_e32:
424 case AMDGPU::V_ADD_CO_U32_e64:
425 case AMDGPU::V_OR_B32_e32:
426 case AMDGPU::V_OR_B32_e64:
427 case AMDGPU::V_SUBREV_U32_e32:
428 case AMDGPU::V_SUBREV_U32_e64:
429 case AMDGPU::V_SUBREV_CO_U32_e32:
430 case AMDGPU::V_SUBREV_CO_U32_e64:
431 case AMDGPU::V_MAX_U32_e32:
432 case AMDGPU::V_MAX_U32_e64:
433 case AMDGPU::V_XOR_B32_e32:
434 case AMDGPU::V_XOR_B32_e64:
435 if (OldOpnd->getImm() == 0)
436 return true;
437 break;
438 case AMDGPU::V_AND_B32_e32:
439 case AMDGPU::V_AND_B32_e64:
440 case AMDGPU::V_MIN_U32_e32:
441 case AMDGPU::V_MIN_U32_e64:
442 if (static_cast<uint32_t>(OldOpnd->getImm()) ==
443 std::numeric_limits<uint32_t>::max())
444 return true;
445 break;
446 case AMDGPU::V_MIN_I32_e32:
447 case AMDGPU::V_MIN_I32_e64:
448 if (static_cast<int32_t>(OldOpnd->getImm()) ==
449 std::numeric_limits<int32_t>::max())
450 return true;
451 break;
452 case AMDGPU::V_MAX_I32_e32:
453 case AMDGPU::V_MAX_I32_e64:
454 if (static_cast<int32_t>(OldOpnd->getImm()) ==
455 std::numeric_limits<int32_t>::min())
456 return true;
457 break;
458 case AMDGPU::V_MUL_I32_I24_e32:
459 case AMDGPU::V_MUL_I32_I24_e64:
460 case AMDGPU::V_MUL_U32_U24_e32:
461 case AMDGPU::V_MUL_U32_U24_e64:
462 if (OldOpnd->getImm() == 1)
463 return true;
464 break;
465 }
466 return false;
467}
468
469MachineInstr *GCNDPPCombine::createDPPInst(
470 MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,
471 MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {
472 assert(CombOldVGPR.Reg);
473 if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
474 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
475 if (!Src1 || !Src1->isReg()) {
476 LLVM_DEBUG(dbgs() << " failed: no src1 or it isn't a register\n");
477 return nullptr;
478 }
479 if (!isIdentityValue(OrigMI.getOpcode(), OldOpndValue)) {
480 LLVM_DEBUG(dbgs() << " failed: old immediate isn't an identity\n");
481 return nullptr;
482 }
483 CombOldVGPR = getRegSubRegPair(*Src1);
484 auto MovDst = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
485 const TargetRegisterClass *RC = MRI->getRegClass(MovDst->getReg());
486 if (!isOfRegClass(CombOldVGPR, *RC, *MRI)) {
487 LLVM_DEBUG(dbgs() << " failed: src1 has wrong register class\n");
488 return nullptr;
489 }
490 }
491 return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);
492}
493
494// returns true if MI doesn't have OpndName immediate operand or the
495// operand has Value
496bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, unsigned OpndName,
497 int64_t Value, int64_t Mask) const {
498 auto *Imm = TII->getNamedOperand(MI, OpndName);
499 if (!Imm)
500 return true;
501
502 assert(Imm->isImm());
503 return (Imm->getImm() & Mask) == Value;
504}
505
506bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
507 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
508 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
509 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
510 LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
511
512 auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
513 assert(DstOpnd && DstOpnd->isReg());
514 auto DPPMovReg = DstOpnd->getReg();
515 if (DPPMovReg.isPhysical()) {
516 LLVM_DEBUG(dbgs() << " failed: dpp move writes physreg\n");
517 return false;
518 }
519 if (execMayBeModifiedBeforeAnyUse(*MRI, DPPMovReg, MovMI)) {
520 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
521 " for all uses\n");
522 return false;
523 }
524
525 if (MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
526 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
527 auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl);
528 assert(DppCtrl && DppCtrl->isImm());
530 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move uses unsupported"
531 " control value\n");
532 // Let it split, then control may become legal.
533 return false;
534 }
535 }
536
537 auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);
538 assert(RowMaskOpnd && RowMaskOpnd->isImm());
539 auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);
540 assert(BankMaskOpnd && BankMaskOpnd->isImm());
541 const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
542 BankMaskOpnd->getImm() == 0xF;
543
544 auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl);
545 assert(BCZOpnd && BCZOpnd->isImm());
546 bool BoundCtrlZero = BCZOpnd->getImm();
547
548 auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old);
549 auto *SrcOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);
550 assert(OldOpnd && OldOpnd->isReg());
551 assert(SrcOpnd && SrcOpnd->isReg());
552 if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) {
553 LLVM_DEBUG(dbgs() << " failed: dpp move reads physreg\n");
554 return false;
555 }
556
557 auto * const OldOpndValue = getOldOpndValue(*OldOpnd);
558 // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else
559 // We could use: assert(!OldOpndValue || OldOpndValue->isImm())
560 // but the third option is used to distinguish undef from non-immediate
561 // to reuse IMPLICIT_DEF instruction later
562 assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
563
564 bool CombBCZ = false;
565
566 if (MaskAllLanes && BoundCtrlZero) { // [1]
567 CombBCZ = true;
568 } else {
569 if (!OldOpndValue || !OldOpndValue->isImm()) {
570 LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n");
571 return false;
572 }
573
574 if (OldOpndValue->getImm() == 0) {
575 if (MaskAllLanes) {
576 assert(!BoundCtrlZero); // by check [1]
577 CombBCZ = true;
578 }
579 } else if (BoundCtrlZero) {
580 assert(!MaskAllLanes); // by check [1]
581 LLVM_DEBUG(dbgs() <<
582 " failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");
583 return false;
584 }
585 }
586
587 LLVM_DEBUG(dbgs() << " old=";
588 if (!OldOpndValue)
589 dbgs() << "undef";
590 else
591 dbgs() << *OldOpndValue;
592 dbgs() << ", bound_ctrl=" << CombBCZ << '\n');
593
594 SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;
596 auto CombOldVGPR = getRegSubRegPair(*OldOpnd);
597 // try to reuse previous old reg if its undefined (IMPLICIT_DEF)
598 if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef
599 const TargetRegisterClass *RC = MRI->getRegClass(DPPMovReg);
600 CombOldVGPR = RegSubRegPair(
601 MRI->createVirtualRegister(RC));
602 auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(),
603 TII->get(AMDGPU::IMPLICIT_DEF), CombOldVGPR.Reg);
604 DPPMIs.push_back(UndefInst.getInstr());
605 }
606
607 OrigMIs.push_back(&MovMI);
608 bool Rollback = true;
610
611 for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) {
612 Uses.push_back(&Use);
613 }
614
615 while (!Uses.empty()) {
616 MachineOperand *Use = Uses.pop_back_val();
617 Rollback = true;
618
619 auto &OrigMI = *Use->getParent();
620 LLVM_DEBUG(dbgs() << " try: " << OrigMI);
621
622 auto OrigOp = OrigMI.getOpcode();
623 assert((TII->get(OrigOp).getSize() != 4 || !AMDGPU::isTrue16Inst(OrigOp)) &&
624 "There should not be e32 True16 instructions pre-RA");
625 if (OrigOp == AMDGPU::REG_SEQUENCE) {
626 Register FwdReg = OrigMI.getOperand(0).getReg();
627 unsigned FwdSubReg = 0;
628
629 if (execMayBeModifiedBeforeAnyUse(*MRI, FwdReg, OrigMI)) {
630 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
631 " for all uses\n");
632 break;
633 }
634
635 unsigned OpNo, E = OrigMI.getNumOperands();
636 for (OpNo = 1; OpNo < E; OpNo += 2) {
637 if (OrigMI.getOperand(OpNo).getReg() == DPPMovReg) {
638 FwdSubReg = OrigMI.getOperand(OpNo + 1).getImm();
639 break;
640 }
641 }
642
643 if (!FwdSubReg)
644 break;
645
646 for (auto &Op : MRI->use_nodbg_operands(FwdReg)) {
647 if (Op.getSubReg() == FwdSubReg)
648 Uses.push_back(&Op);
649 }
650 RegSeqWithOpNos[&OrigMI].push_back(OpNo);
651 continue;
652 }
653
654 bool IsShrinkable = isShrinkable(OrigMI);
655 if (!(IsShrinkable ||
656 ((TII->isVOP3P(OrigOp) || TII->isVOPC(OrigOp) ||
657 TII->isVOP3(OrigOp)) &&
658 ST->hasVOP3DPP()) ||
659 TII->isVOP1(OrigOp) || TII->isVOP2(OrigOp))) {
660 LLVM_DEBUG(dbgs() << " failed: not VOP1/2/3/3P/C\n");
661 break;
662 }
663 if (OrigMI.modifiesRegister(AMDGPU::EXEC, ST->getRegisterInfo())) {
664 LLVM_DEBUG(dbgs() << " failed: can't combine v_cmpx\n");
665 break;
666 }
667
668 auto *Src0 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0);
669 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
670 if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]
671 LLVM_DEBUG(dbgs() << " failed: no suitable operands\n");
672 break;
673 }
674
675 auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2);
676 assert(Src0 && "Src1 without Src0?");
677 if ((Use == Src0 && ((Src1 && Src1->isIdenticalTo(*Src0)) ||
678 (Src2 && Src2->isIdenticalTo(*Src0)))) ||
679 (Use == Src1 && (Src1->isIdenticalTo(*Src0) ||
680 (Src2 && Src2->isIdenticalTo(*Src1))))) {
682 dbgs()
683 << " " << OrigMI
684 << " failed: DPP register is used more than once per instruction\n");
685 break;
686 }
687
688 LLVM_DEBUG(dbgs() << " combining: " << OrigMI);
689 if (Use == Src0) {
690 if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
691 OldOpndValue, CombBCZ, IsShrinkable)) {
692 DPPMIs.push_back(DPPInst);
693 Rollback = false;
694 }
695 } else {
696 assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]
697 auto *BB = OrigMI.getParent();
698 auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI);
699 BB->insert(OrigMI, NewMI);
700 if (TII->commuteInstruction(*NewMI)) {
701 LLVM_DEBUG(dbgs() << " commuted: " << *NewMI);
702 if (auto *DPPInst =
703 createDPPInst(*NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,
704 IsShrinkable)) {
705 DPPMIs.push_back(DPPInst);
706 Rollback = false;
707 }
708 } else
709 LLVM_DEBUG(dbgs() << " failed: cannot be commuted\n");
710 NewMI->eraseFromParent();
711 }
712 if (Rollback)
713 break;
714 OrigMIs.push_back(&OrigMI);
715 }
716
717 Rollback |= !Uses.empty();
718
719 for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))
720 MI->eraseFromParent();
721
722 if (!Rollback) {
723 for (auto &S : RegSeqWithOpNos) {
724 if (MRI->use_nodbg_empty(S.first->getOperand(0).getReg())) {
725 S.first->eraseFromParent();
726 continue;
727 }
728 while (!S.second.empty())
729 S.first->getOperand(S.second.pop_back_val()).setIsUndef();
730 }
731 }
732
733 return !Rollback;
734}
735
736bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
738 if (!ST->hasDPP() || skipFunction(MF.getFunction()))
739 return false;
740
741 MRI = &MF.getRegInfo();
742 TII = ST->getInstrInfo();
743
744 bool Changed = false;
745 for (auto &MBB : MF) {
747 if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) {
748 Changed = true;
749 ++NumDPPMovsCombined;
750 } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
751 MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
752 if (ST->hasDPALU_DPP() && combineDPPMov(MI)) {
753 Changed = true;
754 ++NumDPPMovsCombined;
755 } else {
756 auto Split = TII->expandMovDPP64(MI);
757 for (auto *M : {Split.first, Split.second}) {
758 if (M && combineDPPMov(*M))
759 ++NumDPPMovsCombined;
760 }
761 Changed = true;
762 }
763 }
764 }
765 }
766 return Changed;
767}
unsigned const MachineRegisterInfo * MRI
#define Fail
MachineBasicBlock & MBB
Provides AMDGPU specific target descriptions.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd)
static unsigned getOperandSize(MachineInstr &MI, unsigned Idx, MachineRegisterInfo &MRI)
#define DEBUG_TYPE
Rewrite Partial Register Uses
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
TargetInstrInfo::RegSubRegPair RegSubRegPair
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:269
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
unsigned getSize(const MachineInstr &MI) const
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:543
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:326
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:546
bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register.
bool isCommutable(QueryType Type=IgnoreBundle) const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z,...
void insert(mop_iterator InsertBefore, ArrayRef< MachineOperand > Ops)
Inserts Ops BEFORE It. Can untie/retie tied operands.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:472
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
uint32_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:371
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
LLVM_READONLY int getVOPe32(uint16_t Opcode)
LLVM_READNONE bool isLegalDPALU_DPPControl(unsigned DC)
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
LLVM_READONLY int getDPPOp32(uint16_t Opcode)
bool isTrue16Inst(unsigned Opc)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
LLVM_READONLY int getDPPOp64(uint16_t Opcode)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ Undef
Value of the register doesn't matter.
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O)
Create RegSubRegPair from a register MachineOperand.
Definition: SIInstrInfo.h:1300
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:665
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
MachineInstr * getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, MachineRegisterInfo &MRI)
Return the defining instruction for a given reg:subreg pair skipping copy like instructions and subre...
char & GCNDPPCombineID
bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P, const TargetRegisterClass &TRC, MachineRegisterInfo &MRI)
Returns true if a reg:subreg pair P has a TRC class.
Definition: SIInstrInfo.h:1288
void initializeGCNDPPCombinePass(PassRegistry &)
FunctionPass * createGCNDPPCombinePass()
bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI)
Return false if EXEC is not changed between the def of VReg at DefMI and all its uses.
A pair composed of a register and a sub-register index.