LLVM 24.0.0git
X86InstrInfo.cpp
Go to the documentation of this file.
1//===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86InstrInfo.h"
14#include "X86.h"
15#include "X86InstrBuilder.h"
16#include "X86InstrFoldTables.h"
18#include "X86Subtarget.h"
19#include "X86TargetMachine.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Sequence.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/InstrTypes.h"
36#include "llvm/IR/Module.h"
37#include "llvm/MC/MCAsmInfo.h"
38#include "llvm/MC/MCExpr.h"
39#include "llvm/MC/MCInst.h"
40#include "llvm/Support/Debug.h"
45#include <optional>
46
47using namespace llvm;
48
49#define DEBUG_TYPE "x86-instr-info"
50
51#define GET_INSTRINFO_CTOR_DTOR
52#include "X86GenInstrInfo.inc"
53
54// Pin the vtable to this file.
55void X86InstrInfo::anchor() {}
56
58 : X86GenInstrInfo(STI, RI,
59 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64
60 : X86::ADJCALLSTACKDOWN32),
61 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
62 : X86::ADJCALLSTACKUP32),
63 X86::CATCHRET, (STI.is64Bit() ? X86::RET64 : X86::RET32)),
64 Subtarget(STI), RI(STI.getTargetTriple()) {}
65
67 unsigned OpNum) const {
68 auto *RC = TargetInstrInfo::getRegClass(MCID, OpNum);
69 // If the target does not have egpr, then r16-r31 will be resereved for all
70 // instructions.
71 if (!RC || !Subtarget.hasEGPR())
72 return RC;
73
75 return RC;
76
77 const X86RegisterInfo *RI = Subtarget.getRegisterInfo();
78 return RI->constrainRegClassToNonRex2(RC);
79}
80
83 if (Subtarget.isTarget64BitLP64())
84 return &X86::GR64RegClass;
85 // If the target is 64bit but we have been told to use 32bit addresses, we can
86 // still use 64-bit register as long as we know the high bits are zeros.
87 // Reflect that in the returned register class.
88 return Subtarget.is64Bit() ? &X86::LOW32_ADDR_ACCESSRegClass
89 : &X86::GR32RegClass;
90}
91
93 Register &SrcReg, Register &DstReg,
94 unsigned &SubIdx) const {
95 switch (MI.getOpcode()) {
96 default:
97 break;
98 case X86::MOVSX16rr8:
99 case X86::MOVZX16rr8:
100 case X86::MOVSX32rr8:
101 case X86::MOVZX32rr8:
102 case X86::MOVSX64rr8:
103 if (!Subtarget.is64Bit())
104 // It's not always legal to reference the low 8-bit of the larger
105 // register in 32-bit mode.
106 return false;
107 [[fallthrough]];
108 case X86::MOVSX32rr16:
109 case X86::MOVZX32rr16:
110 case X86::MOVSX64rr16:
111 case X86::MOVSX64rr32: {
112 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
113 // Be conservative.
114 return false;
115 SrcReg = MI.getOperand(1).getReg();
116 DstReg = MI.getOperand(0).getReg();
117 switch (MI.getOpcode()) {
118 default:
119 llvm_unreachable("Unreachable!");
120 case X86::MOVSX16rr8:
121 case X86::MOVZX16rr8:
122 case X86::MOVSX32rr8:
123 case X86::MOVZX32rr8:
124 case X86::MOVSX64rr8:
125 SubIdx = X86::sub_8bit;
126 break;
127 case X86::MOVSX32rr16:
128 case X86::MOVZX32rr16:
129 case X86::MOVSX64rr16:
130 SubIdx = X86::sub_16bit;
131 break;
132 case X86::MOVSX64rr32:
133 SubIdx = X86::sub_32bit;
134 break;
135 }
136 return true;
137 }
138 }
139 return false;
140}
141
143 if (MI.mayLoad() || MI.mayStore())
144 return false;
145
146 // Some target-independent operations that trivially lower to data-invariant
147 // instructions.
148 if (MI.isCopyLike() || MI.isInsertSubreg())
149 return true;
150
151 unsigned Opcode = MI.getOpcode();
152 using namespace X86;
153 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
154 // However, they set flags and are perhaps the most surprisingly constant
155 // time operations so we call them out here separately.
156 if (isIMUL(Opcode))
157 return true;
158 // Bit scanning and counting instructions that are somewhat surprisingly
159 // constant time as they scan across bits and do other fairly complex
160 // operations like popcnt, but are believed to be constant time on x86.
161 // However, these set flags.
162 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
163 isTZCNT(Opcode))
164 return true;
165 // Bit manipulation instructions are effectively combinations of basic
166 // arithmetic ops, and should still execute in constant time. These also
167 // set flags.
168 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
169 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
170 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
171 isTZMSK(Opcode))
172 return true;
173 // Bit extracting and clearing instructions should execute in constant time,
174 // and set flags.
175 if (isBEXTR(Opcode) || isBZHI(Opcode))
176 return true;
177 // Shift and rotate.
178 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
179 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
180 return true;
181 // Basic arithmetic is constant time on the input but does set flags.
182 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
183 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
184 return true;
185 // Arithmetic with just 32-bit and 64-bit variants and no immediates.
186 if (isANDN(Opcode))
187 return true;
188 // Unary arithmetic operations.
189 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
190 return true;
191 // Unlike other arithmetic, NOT doesn't set EFLAGS.
192 if (isNOT(Opcode))
193 return true;
194 // Various move instructions used to zero or sign extend things. Note that we
195 // intentionally don't support the _NOREX variants as we can't handle that
196 // register constraint anyways.
197 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
198 return true;
199 // Arithmetic instructions that are both constant time and don't set flags.
200 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
201 return true;
202 // LEA doesn't actually access memory, and its arithmetic is constant time.
203 if (isLEA(Opcode))
204 return true;
205 // By default, assume that the instruction is not data invariant.
206 return false;
207}
208
210 switch (MI.getOpcode()) {
211 default:
212 // By default, assume that the load will immediately leak.
213 return false;
214
215 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
216 // However, they set flags and are perhaps the most surprisingly constant
217 // time operations so we call them out here separately.
218 case X86::IMUL16rm:
219 case X86::IMUL16rmi:
220 case X86::IMUL32rm:
221 case X86::IMUL32rmi:
222 case X86::IMUL64rm:
223 case X86::IMUL64rmi32:
224
225 // Bit scanning and counting instructions that are somewhat surprisingly
226 // constant time as they scan across bits and do other fairly complex
227 // operations like popcnt, but are believed to be constant time on x86.
228 // However, these set flags.
229 case X86::BSF16rm:
230 case X86::BSF32rm:
231 case X86::BSF64rm:
232 case X86::BSR16rm:
233 case X86::BSR32rm:
234 case X86::BSR64rm:
235 case X86::LZCNT16rm:
236 case X86::LZCNT32rm:
237 case X86::LZCNT64rm:
238 case X86::POPCNT16rm:
239 case X86::POPCNT32rm:
240 case X86::POPCNT64rm:
241 case X86::TZCNT16rm:
242 case X86::TZCNT32rm:
243 case X86::TZCNT64rm:
244
245 // Bit manipulation instructions are effectively combinations of basic
246 // arithmetic ops, and should still execute in constant time. These also
247 // set flags.
248 case X86::BLCFILL32rm:
249 case X86::BLCFILL64rm:
250 case X86::BLCI32rm:
251 case X86::BLCI64rm:
252 case X86::BLCIC32rm:
253 case X86::BLCIC64rm:
254 case X86::BLCMSK32rm:
255 case X86::BLCMSK64rm:
256 case X86::BLCS32rm:
257 case X86::BLCS64rm:
258 case X86::BLSFILL32rm:
259 case X86::BLSFILL64rm:
260 case X86::BLSI32rm:
261 case X86::BLSI64rm:
262 case X86::BLSIC32rm:
263 case X86::BLSIC64rm:
264 case X86::BLSMSK32rm:
265 case X86::BLSMSK64rm:
266 case X86::BLSR32rm:
267 case X86::BLSR64rm:
268 case X86::TZMSK32rm:
269 case X86::TZMSK64rm:
270
271 // Bit extracting and clearing instructions should execute in constant time,
272 // and set flags.
273 case X86::BEXTR32rm:
274 case X86::BEXTR64rm:
275 case X86::BEXTRI32mi:
276 case X86::BEXTRI64mi:
277 case X86::BZHI32rm:
278 case X86::BZHI64rm:
279
280 // Basic arithmetic is constant time on the input but does set flags.
281 case X86::ADC8rm:
282 case X86::ADC16rm:
283 case X86::ADC32rm:
284 case X86::ADC64rm:
285 case X86::ADD8rm:
286 case X86::ADD16rm:
287 case X86::ADD32rm:
288 case X86::ADD64rm:
289 case X86::AND8rm:
290 case X86::AND16rm:
291 case X86::AND32rm:
292 case X86::AND64rm:
293 case X86::ANDN32rm:
294 case X86::ANDN64rm:
295 case X86::OR8rm:
296 case X86::OR16rm:
297 case X86::OR32rm:
298 case X86::OR64rm:
299 case X86::SBB8rm:
300 case X86::SBB16rm:
301 case X86::SBB32rm:
302 case X86::SBB64rm:
303 case X86::SUB8rm:
304 case X86::SUB16rm:
305 case X86::SUB32rm:
306 case X86::SUB64rm:
307 case X86::XOR8rm:
308 case X86::XOR16rm:
309 case X86::XOR32rm:
310 case X86::XOR64rm:
311
312 // Integer multiply w/o affecting flags is still believed to be constant
313 // time on x86. Called out separately as this is among the most surprising
314 // instructions to exhibit that behavior.
315 case X86::MULX32rm:
316 case X86::MULX64rm:
317
318 // Arithmetic instructions that are both constant time and don't set flags.
319 case X86::RORX32mi:
320 case X86::RORX64mi:
321 case X86::SARX32rm:
322 case X86::SARX64rm:
323 case X86::SHLX32rm:
324 case X86::SHLX64rm:
325 case X86::SHRX32rm:
326 case X86::SHRX64rm:
327
328 // Conversions are believed to be constant time and don't set flags.
329 case X86::CVTTSD2SI64rm:
330 case X86::VCVTTSD2SI64rm:
331 case X86::VCVTTSD2SI64Zrm:
332 case X86::CVTTSD2SIrm:
333 case X86::VCVTTSD2SIrm:
334 case X86::VCVTTSD2SIZrm:
335 case X86::CVTTSS2SI64rm:
336 case X86::VCVTTSS2SI64rm:
337 case X86::VCVTTSS2SI64Zrm:
338 case X86::CVTTSS2SIrm:
339 case X86::VCVTTSS2SIrm:
340 case X86::VCVTTSS2SIZrm:
341 case X86::CVTSI2SDrm:
342 case X86::VCVTSI2SDrm:
343 case X86::VCVTSI2SDZrm:
344 case X86::CVTSI2SSrm:
345 case X86::VCVTSI2SSrm:
346 case X86::VCVTSI2SSZrm:
347 case X86::CVTSI642SDrm:
348 case X86::VCVTSI642SDrm:
349 case X86::VCVTSI642SDZrm:
350 case X86::CVTSI642SSrm:
351 case X86::VCVTSI642SSrm:
352 case X86::VCVTSI642SSZrm:
353 case X86::CVTSS2SDrm:
354 case X86::VCVTSS2SDrm:
355 case X86::VCVTSS2SDZrm:
356 case X86::CVTSD2SSrm:
357 case X86::VCVTSD2SSrm:
358 case X86::VCVTSD2SSZrm:
359 // AVX512 added unsigned integer conversions.
360 case X86::VCVTTSD2USI64Zrm:
361 case X86::VCVTTSD2USIZrm:
362 case X86::VCVTTSS2USI64Zrm:
363 case X86::VCVTTSS2USIZrm:
364 case X86::VCVTUSI2SDZrm:
365 case X86::VCVTUSI642SDZrm:
366 case X86::VCVTUSI2SSZrm:
367 case X86::VCVTUSI642SSZrm:
368
369 // Loads to register don't set flags.
370 case X86::MOV8rm:
371 case X86::MOV8rm_NOREX:
372 case X86::MOV16rm:
373 case X86::MOV32rm:
374 case X86::MOV64rm:
375 case X86::MOVSX16rm8:
376 case X86::MOVSX32rm16:
377 case X86::MOVSX32rm8:
378 case X86::MOVSX32rm8_NOREX:
379 case X86::MOVSX64rm16:
380 case X86::MOVSX64rm32:
381 case X86::MOVSX64rm8:
382 case X86::MOVZX16rm8:
383 case X86::MOVZX32rm16:
384 case X86::MOVZX32rm8:
385 case X86::MOVZX32rm8_NOREX:
386 case X86::MOVZX64rm16:
387 case X86::MOVZX64rm8:
388 return true;
389 }
390}
391
393 const MachineFunction *MF = MI.getParent()->getParent();
395
396 if (isFrameInstr(MI)) {
397 int SPAdj = alignTo(getFrameSize(MI), TFI->getStackAlign());
398 SPAdj -= getFrameAdjustment(MI);
399 if (!isFrameSetup(MI))
400 SPAdj = -SPAdj;
401 return SPAdj;
402 }
403
404 // To know whether a call adjusts the stack, we need information
405 // that is bound to the following ADJCALLSTACKUP pseudo.
406 // Look for the next ADJCALLSTACKUP that follows the call.
407 if (MI.isCall()) {
408 const MachineBasicBlock *MBB = MI.getParent();
410 for (auto E = MBB->end(); I != E; ++I) {
411 if (I->getOpcode() == getCallFrameDestroyOpcode() || I->isCall())
412 break;
413 }
414
415 // If we could not find a frame destroy opcode, then it has already
416 // been simplified, so we don't care.
417 if (I->getOpcode() != getCallFrameDestroyOpcode())
418 return 0;
419
420 return -(I->getOperand(1).getImm());
421 }
422
423 // Currently handle only PUSHes we can reasonably expect to see
424 // in call sequences
425 switch (MI.getOpcode()) {
426 default:
427 return 0;
428 case X86::PUSH32r:
429 case X86::PUSH32rmm:
430 case X86::PUSH32rmr:
431 case X86::PUSH32i:
432 return 4;
433 case X86::PUSH64r:
434 case X86::PUSH64rmm:
435 case X86::PUSH64rmr:
436 case X86::PUSH64i32:
437 return 8;
438 }
439}
440
441/// Return true and the FrameIndex if the specified
442/// operand and follow operands form a reference to the stack frame.
443bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op,
444 int &FrameIndex) const {
445 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() &&
446 MI.getOperand(Op + X86::AddrScaleAmt).isImm() &&
447 MI.getOperand(Op + X86::AddrIndexReg).isReg() &&
448 MI.getOperand(Op + X86::AddrDisp).isImm() &&
449 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 &&
450 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 &&
451 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) {
452 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex();
453 return true;
454 }
455 return false;
456}
457
458static bool isFrameLoadOpcode(int Opcode, TypeSize &MemBytes) {
459 switch (Opcode) {
460 default:
461 return false;
462 case X86::MOV8rm:
463 case X86::KMOVBkm:
464 case X86::KMOVBkm_EVEX:
465 MemBytes = TypeSize::getFixed(1);
466 return true;
467 case X86::MOV16rm:
468 case X86::KMOVWkm:
469 case X86::KMOVWkm_EVEX:
470 case X86::VMOVSHZrm:
471 case X86::VMOVSHZrm_alt:
472 MemBytes = TypeSize::getFixed(2);
473 return true;
474 case X86::MOV32rm:
475 case X86::MOVSSrm:
476 case X86::MOVSSrm_alt:
477 case X86::VMOVSSrm:
478 case X86::VMOVSSrm_alt:
479 case X86::VMOVSSZrm:
480 case X86::VMOVSSZrm_alt:
481 case X86::KMOVDkm:
482 case X86::KMOVDkm_EVEX:
483 MemBytes = TypeSize::getFixed(4);
484 return true;
485 case X86::MOV64rm:
486 case X86::LD_Fp64m:
487 case X86::MOVSDrm:
488 case X86::MOVSDrm_alt:
489 case X86::VMOVSDrm:
490 case X86::VMOVSDrm_alt:
491 case X86::VMOVSDZrm:
492 case X86::VMOVSDZrm_alt:
493 case X86::MMX_MOVD64rm:
494 case X86::MMX_MOVQ64rm:
495 case X86::KMOVQkm:
496 case X86::KMOVQkm_EVEX:
497 MemBytes = TypeSize::getFixed(8);
498 return true;
499 case X86::MOVAPSrm:
500 case X86::MOVUPSrm:
501 case X86::MOVAPDrm:
502 case X86::MOVUPDrm:
503 case X86::MOVDQArm:
504 case X86::MOVDQUrm:
505 case X86::VMOVAPSrm:
506 case X86::VMOVUPSrm:
507 case X86::VMOVAPDrm:
508 case X86::VMOVUPDrm:
509 case X86::VMOVDQArm:
510 case X86::VMOVDQUrm:
511 case X86::VMOVAPSZ128rm:
512 case X86::VMOVUPSZ128rm:
513 case X86::VMOVAPSZ128rm_NOVLX:
514 case X86::VMOVUPSZ128rm_NOVLX:
515 case X86::VMOVAPDZ128rm:
516 case X86::VMOVUPDZ128rm:
517 case X86::VMOVDQU8Z128rm:
518 case X86::VMOVDQU16Z128rm:
519 case X86::VMOVDQA32Z128rm:
520 case X86::VMOVDQU32Z128rm:
521 case X86::VMOVDQA64Z128rm:
522 case X86::VMOVDQU64Z128rm:
523 MemBytes = TypeSize::getFixed(16);
524 return true;
525 case X86::VMOVAPSYrm:
526 case X86::VMOVUPSYrm:
527 case X86::VMOVAPDYrm:
528 case X86::VMOVUPDYrm:
529 case X86::VMOVDQAYrm:
530 case X86::VMOVDQUYrm:
531 case X86::VMOVAPSZ256rm:
532 case X86::VMOVUPSZ256rm:
533 case X86::VMOVAPSZ256rm_NOVLX:
534 case X86::VMOVUPSZ256rm_NOVLX:
535 case X86::VMOVAPDZ256rm:
536 case X86::VMOVUPDZ256rm:
537 case X86::VMOVDQU8Z256rm:
538 case X86::VMOVDQU16Z256rm:
539 case X86::VMOVDQA32Z256rm:
540 case X86::VMOVDQU32Z256rm:
541 case X86::VMOVDQA64Z256rm:
542 case X86::VMOVDQU64Z256rm:
543 MemBytes = TypeSize::getFixed(32);
544 return true;
545 case X86::VMOVAPSZrm:
546 case X86::VMOVUPSZrm:
547 case X86::VMOVAPDZrm:
548 case X86::VMOVUPDZrm:
549 case X86::VMOVDQU8Zrm:
550 case X86::VMOVDQU16Zrm:
551 case X86::VMOVDQA32Zrm:
552 case X86::VMOVDQU32Zrm:
553 case X86::VMOVDQA64Zrm:
554 case X86::VMOVDQU64Zrm:
555 MemBytes = TypeSize::getFixed(64);
556 return true;
557 }
558}
559
560static bool isFrameStoreOpcode(int Opcode, TypeSize &MemBytes) {
561 switch (Opcode) {
562 default:
563 return false;
564 case X86::MOV8mr:
565 case X86::KMOVBmk:
566 case X86::KMOVBmk_EVEX:
567 MemBytes = TypeSize::getFixed(1);
568 return true;
569 case X86::MOV16mr:
570 case X86::KMOVWmk:
571 case X86::KMOVWmk_EVEX:
572 case X86::VMOVSHZmr:
573 MemBytes = TypeSize::getFixed(2);
574 return true;
575 case X86::MOV32mr:
576 case X86::MOVSSmr:
577 case X86::VMOVSSmr:
578 case X86::VMOVSSZmr:
579 case X86::KMOVDmk:
580 case X86::KMOVDmk_EVEX:
581 MemBytes = TypeSize::getFixed(4);
582 return true;
583 case X86::MOV64mr:
584 case X86::ST_FpP64m:
585 case X86::MOVSDmr:
586 case X86::VMOVSDmr:
587 case X86::VMOVSDZmr:
588 case X86::MMX_MOVD64mr:
589 case X86::MMX_MOVQ64mr:
590 case X86::MMX_MOVNTQmr:
591 case X86::KMOVQmk:
592 case X86::KMOVQmk_EVEX:
593 MemBytes = TypeSize::getFixed(8);
594 return true;
595 case X86::MOVAPSmr:
596 case X86::MOVUPSmr:
597 case X86::MOVAPDmr:
598 case X86::MOVUPDmr:
599 case X86::MOVDQAmr:
600 case X86::MOVDQUmr:
601 case X86::VMOVAPSmr:
602 case X86::VMOVUPSmr:
603 case X86::VMOVAPDmr:
604 case X86::VMOVUPDmr:
605 case X86::VMOVDQAmr:
606 case X86::VMOVDQUmr:
607 case X86::VMOVUPSZ128mr:
608 case X86::VMOVAPSZ128mr:
609 case X86::VMOVUPSZ128mr_NOVLX:
610 case X86::VMOVAPSZ128mr_NOVLX:
611 case X86::VMOVUPDZ128mr:
612 case X86::VMOVAPDZ128mr:
613 case X86::VMOVDQA32Z128mr:
614 case X86::VMOVDQU32Z128mr:
615 case X86::VMOVDQA64Z128mr:
616 case X86::VMOVDQU64Z128mr:
617 case X86::VMOVDQU8Z128mr:
618 case X86::VMOVDQU16Z128mr:
619 MemBytes = TypeSize::getFixed(16);
620 return true;
621 case X86::VMOVUPSYmr:
622 case X86::VMOVAPSYmr:
623 case X86::VMOVUPDYmr:
624 case X86::VMOVAPDYmr:
625 case X86::VMOVDQUYmr:
626 case X86::VMOVDQAYmr:
627 case X86::VMOVUPSZ256mr:
628 case X86::VMOVAPSZ256mr:
629 case X86::VMOVUPSZ256mr_NOVLX:
630 case X86::VMOVAPSZ256mr_NOVLX:
631 case X86::VMOVUPDZ256mr:
632 case X86::VMOVAPDZ256mr:
633 case X86::VMOVDQU8Z256mr:
634 case X86::VMOVDQU16Z256mr:
635 case X86::VMOVDQA32Z256mr:
636 case X86::VMOVDQU32Z256mr:
637 case X86::VMOVDQA64Z256mr:
638 case X86::VMOVDQU64Z256mr:
639 MemBytes = TypeSize::getFixed(32);
640 return true;
641 case X86::VMOVUPSZmr:
642 case X86::VMOVAPSZmr:
643 case X86::VMOVUPDZmr:
644 case X86::VMOVAPDZmr:
645 case X86::VMOVDQU8Zmr:
646 case X86::VMOVDQU16Zmr:
647 case X86::VMOVDQA32Zmr:
648 case X86::VMOVDQU32Zmr:
649 case X86::VMOVDQA64Zmr:
650 case X86::VMOVDQU64Zmr:
651 MemBytes = TypeSize::getFixed(64);
652 return true;
653 }
654 return false;
655}
656
658 int &FrameIndex) const {
659 TypeSize Dummy = TypeSize::getZero();
660 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy);
661}
662
664 int &FrameIndex,
665 TypeSize &MemBytes) const {
666 if (isFrameLoadOpcode(MI.getOpcode(), MemBytes))
667 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex))
668 return MI.getOperand(0).getReg();
669 return Register();
670}
671
673 int &FrameIndex) const {
674 TypeSize Dummy = TypeSize::getZero();
675 if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) {
676 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
677 return Reg;
678 // Check for post-frame index elimination operations
680 if (hasLoadFromStackSlot(MI, Accesses)) {
681 FrameIndex =
682 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
683 ->getFrameIndex();
684 return MI.getOperand(0).getReg();
685 }
686 }
687 return Register();
688}
689
691 int &FrameIndex) const {
692 TypeSize Dummy = TypeSize::getZero();
693 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy);
694}
695
697 int &FrameIndex,
698 TypeSize &MemBytes) const {
699 if (isFrameStoreOpcode(MI.getOpcode(), MemBytes))
700 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 &&
701 isFrameOperand(MI, 0, FrameIndex))
702 return MI.getOperand(X86::AddrNumOperands).getReg();
703 return Register();
704}
705
707 int &FrameIndex) const {
708 TypeSize Dummy = TypeSize::getZero();
709 if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) {
710 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
711 return Reg;
712 // Check for post-frame index elimination operations
714 if (hasStoreToStackSlot(MI, Accesses)) {
715 FrameIndex =
716 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
717 ->getFrameIndex();
718 return MI.getOperand(X86::AddrNumOperands).getReg();
719 }
720 }
721 return Register();
722}
723
724/// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
725static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) {
726 // Don't waste compile time scanning use-def chains of physregs.
727 if (!BaseReg.isVirtual())
728 return false;
729 bool isPICBase = false;
730 for (const MachineInstr &DefMI : MRI.def_instructions(BaseReg)) {
731 if (DefMI.getOpcode() != X86::MOVPC32r)
732 return false;
733 assert(!isPICBase && "More than one PIC base?");
734 isPICBase = true;
735 }
736 return isPICBase;
737}
738
740 const MachineInstr &MI) const {
741 switch (MI.getOpcode()) {
742 default:
743 // This function should only be called for opcodes with the ReMaterializable
744 // flag set.
745 llvm_unreachable("Unknown rematerializable operation!");
746 break;
747 case X86::IMPLICIT_DEF:
748 // Defer to generic logic.
749 break;
750 case X86::LOAD_STACK_GUARD:
751 case X86::LD_Fp032:
752 case X86::LD_Fp064:
753 case X86::LD_Fp080:
754 case X86::LD_Fp132:
755 case X86::LD_Fp164:
756 case X86::LD_Fp180:
757 case X86::AVX1_SETALLONES:
758 case X86::AVX2_SETALLONES:
759 case X86::AVX512_128_SET0:
760 case X86::AVX512_128_SETALLONES:
761 case X86::AVX512_256_SETALLONES:
762 case X86::AVX512_512_SETALLONES:
763 case X86::AVX512_FsFLD0SD:
764 case X86::AVX512_FsFLD0SH:
765 case X86::AVX512_FsFLD0SS:
766 case X86::AVX512_FsFLD0F128:
767 case X86::FsFLD0SD:
768 case X86::FsFLD0SS:
769 case X86::FsFLD0SH:
770 case X86::FsFLD0F128:
771 case X86::KSET0B:
772 case X86::KSET0D:
773 case X86::KSET0Q:
774 case X86::KSET0W:
775 case X86::KSET1B:
776 case X86::KSET1D:
777 case X86::KSET1Q:
778 case X86::KSET1W:
779 case X86::MMX_SET0:
780 case X86::MOV32ImmSExti8:
781 case X86::MOV32r0:
782 case X86::MOV32r1:
783 case X86::MOV32r_1:
784 case X86::MOV32ri64:
785 case X86::MOV64ImmSExti8:
786 case X86::V_SET0:
787 case X86::V_SETALLONES:
788 case X86::MOV16ri:
789 case X86::MOV32ri:
790 case X86::MOV64ri:
791 case X86::MOV64ri32:
792 case X86::MOV8ri:
793 case X86::PTILEZEROV:
794 return true;
795
796 case X86::MOV8rm:
797 case X86::MOV8rm_NOREX:
798 case X86::MOV16rm:
799 case X86::MOV32rm:
800 case X86::MOV64rm:
801 case X86::MOVSSrm:
802 case X86::MOVSSrm_alt:
803 case X86::MOVSDrm:
804 case X86::MOVSDrm_alt:
805 case X86::MOVAPSrm:
806 case X86::MOVUPSrm:
807 case X86::MOVAPDrm:
808 case X86::MOVUPDrm:
809 case X86::MOVDQArm:
810 case X86::MOVDQUrm:
811 case X86::VMOVSSrm:
812 case X86::VMOVSSrm_alt:
813 case X86::VMOVSDrm:
814 case X86::VMOVSDrm_alt:
815 case X86::VMOVAPSrm:
816 case X86::VMOVUPSrm:
817 case X86::VMOVAPDrm:
818 case X86::VMOVUPDrm:
819 case X86::VMOVDQArm:
820 case X86::VMOVDQUrm:
821 case X86::VMOVAPSYrm:
822 case X86::VMOVUPSYrm:
823 case X86::VMOVAPDYrm:
824 case X86::VMOVUPDYrm:
825 case X86::VMOVDQAYrm:
826 case X86::VMOVDQUYrm:
827 case X86::MMX_MOVD64rm:
828 case X86::MMX_MOVQ64rm:
829 case X86::VBROADCASTSSrm:
830 case X86::VBROADCASTSSYrm:
831 case X86::VBROADCASTSDYrm:
832 // AVX-512
833 case X86::VPBROADCASTBZ128rm:
834 case X86::VPBROADCASTBZ256rm:
835 case X86::VPBROADCASTBZrm:
836 case X86::VBROADCASTF32X2Z256rm:
837 case X86::VBROADCASTF32X2Zrm:
838 case X86::VBROADCASTI32X2Z128rm:
839 case X86::VBROADCASTI32X2Z256rm:
840 case X86::VBROADCASTI32X2Zrm:
841 case X86::VPBROADCASTWZ128rm:
842 case X86::VPBROADCASTWZ256rm:
843 case X86::VPBROADCASTWZrm:
844 case X86::VPBROADCASTDZ128rm:
845 case X86::VPBROADCASTDZ256rm:
846 case X86::VPBROADCASTDZrm:
847 case X86::VBROADCASTSSZ128rm:
848 case X86::VBROADCASTSSZ256rm:
849 case X86::VBROADCASTSSZrm:
850 case X86::VPBROADCASTQZ128rm:
851 case X86::VPBROADCASTQZ256rm:
852 case X86::VPBROADCASTQZrm:
853 case X86::VBROADCASTSDZ256rm:
854 case X86::VBROADCASTSDZrm:
855 case X86::VMOVSSZrm:
856 case X86::VMOVSSZrm_alt:
857 case X86::VMOVSDZrm:
858 case X86::VMOVSDZrm_alt:
859 case X86::VMOVSHZrm:
860 case X86::VMOVSHZrm_alt:
861 case X86::VMOVAPDZ128rm:
862 case X86::VMOVAPDZ256rm:
863 case X86::VMOVAPDZrm:
864 case X86::VMOVAPSZ128rm:
865 case X86::VMOVAPSZ256rm:
866 case X86::VMOVAPSZ128rm_NOVLX:
867 case X86::VMOVAPSZ256rm_NOVLX:
868 case X86::VMOVAPSZrm:
869 case X86::VMOVDQA32Z128rm:
870 case X86::VMOVDQA32Z256rm:
871 case X86::VMOVDQA32Zrm:
872 case X86::VMOVDQA64Z128rm:
873 case X86::VMOVDQA64Z256rm:
874 case X86::VMOVDQA64Zrm:
875 case X86::VMOVDQU16Z128rm:
876 case X86::VMOVDQU16Z256rm:
877 case X86::VMOVDQU16Zrm:
878 case X86::VMOVDQU32Z128rm:
879 case X86::VMOVDQU32Z256rm:
880 case X86::VMOVDQU32Zrm:
881 case X86::VMOVDQU64Z128rm:
882 case X86::VMOVDQU64Z256rm:
883 case X86::VMOVDQU64Zrm:
884 case X86::VMOVDQU8Z128rm:
885 case X86::VMOVDQU8Z256rm:
886 case X86::VMOVDQU8Zrm:
887 case X86::VMOVUPDZ128rm:
888 case X86::VMOVUPDZ256rm:
889 case X86::VMOVUPDZrm:
890 case X86::VMOVUPSZ128rm:
891 case X86::VMOVUPSZ256rm:
892 case X86::VMOVUPSZ128rm_NOVLX:
893 case X86::VMOVUPSZ256rm_NOVLX:
894 case X86::VMOVUPSZrm: {
895 // Loads from constant pools are trivially rematerializable.
896 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() &&
897 MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
898 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
899 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
900 MI.isDereferenceableInvariantLoad()) {
901 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
902 if (BaseReg == 0 || BaseReg == X86::RIP)
903 return true;
904 // Allow re-materialization of PIC load.
905 if (!(!Subtarget.getCLOpts().remat_pic_stub_load &&
906 MI.getOperand(1 + X86::AddrDisp).isGlobal())) {
907 const MachineFunction &MF = *MI.getParent()->getParent();
908 const MachineRegisterInfo &MRI = MF.getRegInfo();
909 if (regIsPICBase(BaseReg, MRI))
910 return true;
911 }
912 }
913 break;
914 }
915
916 case X86::LEA32r:
917 case X86::LEA64r: {
918 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
919 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
920 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
921 !MI.getOperand(1 + X86::AddrDisp).isReg()) {
922 // lea fi#, lea GV, etc. are all rematerializable.
923 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg())
924 return true;
925 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
926 if (BaseReg == 0)
927 return true;
928 // Allow re-materialization of lea PICBase + x.
929 const MachineFunction &MF = *MI.getParent()->getParent();
930 const MachineRegisterInfo &MRI = MF.getRegInfo();
931 if (regIsPICBase(BaseReg, MRI))
932 return true;
933 }
934 break;
935 }
936 }
938}
939
942 Register DestReg, unsigned SubIdx,
943 const MachineInstr &Orig,
944 LaneBitmask UsedLanes) const {
945 bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI);
946 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) !=
948 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side
949 // effects.
950 int Value;
951 switch (Orig.getOpcode()) {
952 case X86::MOV32r0:
953 Value = 0;
954 break;
955 case X86::MOV32r1:
956 Value = 1;
957 break;
958 case X86::MOV32r_1:
959 Value = -1;
960 break;
961 default:
962 llvm_unreachable("Unexpected instruction!");
963 }
964
965 const DebugLoc &DL = Orig.getDebugLoc();
966 BuildMI(MBB, I, DL, get(X86::MOV32ri))
967 .add(Orig.getOperand(0))
968 .addImm(Value);
969 } else {
970 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
971 MBB.insert(I, MI);
972 }
973
974 MachineInstr &NewMI = *std::prev(I);
975 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
976}
977
978/// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead.
980 for (const MachineOperand &MO : MI.operands()) {
981 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS &&
982 !MO.isDead()) {
983 return true;
984 }
985 }
986 return false;
987}
988
989/// Check whether the shift count for a machine operand is non-zero.
990inline static unsigned getTruncatedShiftCount(const MachineInstr &MI,
991 unsigned ShiftAmtOperandIdx) {
992 // The shift count is six bits with the REX.W prefix and five bits without.
993 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31;
994 unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm();
995 return Imm & ShiftCountMask;
996}
997
998/// Check whether the given shift count is appropriate
999/// can be represented by a LEA instruction.
1000inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) {
1001 // Left shift instructions can be transformed into load-effective-address
1002 // instructions if we can encode them appropriately.
1003 // A LEA instruction utilizes a SIB byte to encode its scale factor.
1004 // The SIB.scale field is two bits wide which means that we can encode any
1005 // shift amount less than 4.
1006 return ShAmt < 4 && ShAmt > 0;
1007}
1008
1009static bool
1011 const MachineRegisterInfo *MRI, MachineInstr **AndInstr,
1012 const TargetRegisterInfo *TRI, const X86Subtarget &ST,
1013 bool &NoSignFlag, bool &ClearsOverflowFlag) {
1014 if (!(CmpValDefInstr.getOpcode() == X86::SUBREG_TO_REG &&
1015 CmpInstr.getOpcode() == X86::TEST64rr) &&
1016 !(CmpValDefInstr.getOpcode() == X86::COPY &&
1017 CmpInstr.getOpcode() == X86::TEST16rr))
1018 return false;
1019
1020 // CmpInstr is a TEST16rr/TEST64rr instruction, and
1021 // `X86InstrInfo::analyzeCompare` guarantees that it's analyzable only if two
1022 // registers are identical.
1023 assert((CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) &&
1024 "CmpInstr is an analyzable TEST16rr/TEST64rr, and "
1025 "`X86InstrInfo::analyzeCompare` requires two reg operands are the"
1026 "same.");
1027
1028 // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that
1029 // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case
1030 // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is
1031 // redundant.
1032 assert(
1033 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) &&
1034 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG or TEST16rr "
1035 "is a user of COPY sub16bit.");
1036 MachineInstr *VregDefInstr = nullptr;
1037 if (CmpInstr.getOpcode() == X86::TEST16rr) {
1038 if (!CmpValDefInstr.getOperand(1).getReg().isVirtual())
1039 return false;
1040 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(1).getReg());
1041 if (!VregDefInstr)
1042 return false;
1043 // We can only remove test when AND32ri or AND64ri32 whose imm can fit 16bit
1044 // size, others 32/64 bit ops would test higher bits which test16rr don't
1045 // want to.
1046 if (!((VregDefInstr->getOpcode() == X86::AND32ri ||
1047 VregDefInstr->getOpcode() == X86::AND64ri32) &&
1048 isUInt<16>(VregDefInstr->getOperand(2).getImm())))
1049 return false;
1050 }
1051
1052 if (CmpInstr.getOpcode() == X86::TEST64rr) {
1053 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically
1054 // sub_32bit or sub_xmm.
1055 if (CmpValDefInstr.getOperand(2).getImm() != X86::sub_32bit)
1056 return false;
1057
1058 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(1).getReg());
1059 }
1060
1061 assert(VregDefInstr && "Must have a definition (SSA)");
1062
1063 // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB
1064 // to simplify the subsequent analysis.
1065 //
1066 // FIXME: If `VregDefInstr->getParent()` is the only predecessor of
1067 // `CmpValDefInstr.getParent()`, this could be handled.
1068 if (VregDefInstr->getParent() != CmpValDefInstr.getParent())
1069 return false;
1070
1071 if (X86::isAND(VregDefInstr->getOpcode()) &&
1072 (!ST.hasNF() || VregDefInstr->modifiesRegister(X86::EFLAGS, TRI))) {
1073 // Get a sequence of instructions like
1074 // %reg = and* ... // Set EFLAGS
1075 // ... // EFLAGS not changed
1076 // %extended_reg = subreg_to_reg %reg, %subreg.sub_32bit
1077 // test64rr %extended_reg, %extended_reg, implicit-def $eflags
1078 // or
1079 // %reg = and32* ...
1080 // ... // EFLAGS not changed.
1081 // %src_reg = copy %reg.sub_16bit:gr32
1082 // test16rr %src_reg, %src_reg, implicit-def $eflags
1083 //
1084 // If subsequent readers use a subset of bits that don't change
1085 // after `and*` instructions, it's likely that the test64rr could
1086 // be optimized away.
1087 for (const MachineInstr &Instr :
1088 make_range(std::next(MachineBasicBlock::iterator(VregDefInstr)),
1089 MachineBasicBlock::iterator(CmpValDefInstr))) {
1090 // There are instructions between 'VregDefInstr' and
1091 // 'CmpValDefInstr' that modifies EFLAGS.
1092 if (Instr.modifiesRegister(X86::EFLAGS, TRI))
1093 return false;
1094 }
1095
1096 *AndInstr = VregDefInstr;
1097
1098 // AND instruction will essentially update SF and clear OF, so
1099 // NoSignFlag should be false in the sense that SF is modified by `AND`.
1100 //
1101 // However, the implementation artifically sets `NoSignFlag` to true
1102 // to poison the SF bit; that is to say, if SF is looked at later, the
1103 // optimization (to erase TEST64rr) will be disabled.
1104 //
1105 // The reason to poison SF bit is that SF bit value could be different
1106 // in the `AND` and `TEST` operation; signed bit is not known for `AND`,
1107 // and is known to be 0 as a result of `TEST64rr`.
1108 //
1109 // FIXME: As opposed to poisoning the SF bit directly, consider peeking into
1110 // the AND instruction and using the static information to guide peephole
1111 // optimization if possible. For example, it's possible to fold a
1112 // conditional move into a copy if the relevant EFLAG bits could be deduced
1113 // from an immediate operand of and operation.
1114 //
1115 NoSignFlag = true;
1116 // ClearsOverflowFlag is true for AND operation (no surprise).
1117 ClearsOverflowFlag = true;
1118 return true;
1119 }
1120 return false;
1121}
1122
1124 unsigned Opc, bool AllowSP, Register &NewSrc,
1125 unsigned &NewSrcSubReg, bool &isKill,
1126 MachineOperand &ImplicitOp,
1127 LiveIntervals *LIS) const {
1128 MachineFunction &MF = *MI.getParent()->getParent();
1129 const TargetRegisterClass *RC;
1130 if (AllowSP) {
1131 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1132 } else {
1133 RC = Opc != X86::LEA32r ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1134 }
1135 Register SrcReg = Src.getReg();
1136 unsigned SubReg = Src.getSubReg();
1137 isKill = MI.killsRegister(SrcReg, /*TRI=*/nullptr);
1138
1139 NewSrcSubReg = X86::NoSubRegister;
1140
1141 // For both LEA64 and LEA32 the register already has essentially the right
1142 // type (32-bit or 64-bit) we may just need to forbid SP.
1143 if (Opc != X86::LEA64_32r) {
1144 NewSrc = SrcReg;
1145 NewSrcSubReg = SubReg;
1146 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1147
1148 if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(NewSrc, RC))
1149 return false;
1150
1151 return true;
1152 }
1153
1154 // This is for an LEA64_32r and incoming registers are 32-bit. One way or
1155 // another we need to add 64-bit registers to the final MI.
1156 if (SrcReg.isPhysical()) {
1157 ImplicitOp = Src;
1158 ImplicitOp.setImplicit();
1159
1160 NewSrc = getX86SubSuperRegister(SrcReg, 64);
1161 assert(!SubReg && "no superregister for source");
1162 assert(NewSrc.isValid() && "Invalid Operand");
1163 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1164 } else {
1165 // Virtual register of the wrong class, we have to create a temporary 64-bit
1166 // vreg to feed into the LEA.
1167 NewSrc = MF.getRegInfo().createVirtualRegister(RC);
1168 NewSrcSubReg = X86::NoSubRegister;
1169 MachineInstr *Copy =
1170 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1171 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit)
1172 .addReg(SrcReg, getKillRegState(isKill), SubReg);
1173
1174 isKill = false;
1175
1176 if (LIS) {
1177 SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(*Copy);
1178 SlotIndex Idx = LIS->getInstructionIndex(MI);
1179 LiveInterval &LI = LIS->getInterval(SrcReg);
1181 if (S->end.getBaseIndex() == Idx)
1182 S->end = CopyIdx.getRegSlot();
1183 }
1184 }
1185
1186 // We've set all the parameters without issue.
1187 return true;
1188}
1189
1190MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
1192 LiveIntervals *LIS,
1193 bool Is8BitOp) const {
1194 // We handle 8-bit adds and various 16-bit opcodes in the switch below.
1195 MachineBasicBlock &MBB = *MI.getParent();
1196 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
1197 assert((Is8BitOp ||
1198 RegInfo.getTargetRegisterInfo()->getRegSizeInBits(
1199 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) &&
1200 "Unexpected type for LEA transform");
1201
1202 // TODO: For a 32-bit target, we need to adjust the LEA variables with
1203 // something like this:
1204 // Opcode = X86::LEA32r;
1205 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1206 // OutRegLEA =
1207 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass)
1208 // : RegInfo.createVirtualRegister(&X86::GR32RegClass);
1209 if (!Subtarget.is64Bit())
1210 return nullptr;
1211
1212 unsigned Opcode = X86::LEA64_32r;
1213 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1214 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass);
1215 Register InRegLEA2;
1216
1217 // Build and insert into an implicit UNDEF value. This is OK because
1218 // we will be shifting and then extracting the lower 8/16-bits.
1219 // This has the potential to cause partial register stall. e.g.
1220 // movw (%rbp,%rcx,2), %dx
1221 // leal -65(%rdx), %esi
1222 // But testing has shown this *does* help performance in 64-bit mode (at
1223 // least on modern x86 machines).
1224 MachineBasicBlock::iterator MBBI = MI.getIterator();
1225 Register Dest = MI.getOperand(0).getReg();
1226 Register Src = MI.getOperand(1).getReg();
1227 unsigned SrcSubReg = MI.getOperand(1).getSubReg();
1228 Register Src2;
1229 unsigned Src2SubReg;
1230 bool IsDead = MI.getOperand(0).isDead();
1231 bool IsKill = MI.getOperand(1).isKill();
1232 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1233 assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization");
1234 MachineInstr *ImpDef =
1235 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA);
1236 MachineInstr *InsMI =
1237 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1238 .addReg(InRegLEA, RegState::Define, SubReg)
1239 .addReg(Src, getKillRegState(IsKill), SrcSubReg);
1240 MachineInstr *ImpDef2 = nullptr;
1241 MachineInstr *InsMI2 = nullptr;
1242
1244 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA);
1245#define CASE_NF(OP) \
1246 case X86::OP: \
1247 case X86::OP##_NF:
1248 switch (MIOpc) {
1249 default:
1250 llvm_unreachable("Unreachable!");
1251 CASE_NF(SHL8ri)
1252 CASE_NF(SHL16ri) {
1253 unsigned ShAmt = MI.getOperand(2).getImm();
1254 MIB.addReg(0)
1255 .addImm(1LL << ShAmt)
1256 .addReg(InRegLEA)
1257 .addImm(0)
1258 .addReg(0);
1259 break;
1260 }
1261 CASE_NF(INC8r)
1262 CASE_NF(INC16r)
1263 addRegOffset(MIB, InRegLEA, 1);
1264 break;
1265 CASE_NF(DEC8r)
1266 CASE_NF(DEC16r)
1267 addRegOffset(MIB, InRegLEA, -1);
1268 break;
1269 CASE_NF(ADD8ri)
1270 CASE_NF(ADD16ri)
1271 case X86::ADD8ri_DB:
1272 case X86::ADD16ri_DB:
1273 addRegOffset(MIB, InRegLEA, MI.getOperand(2).getImm());
1274 break;
1275 CASE_NF(ADD8rr)
1276 CASE_NF(ADD16rr)
1277 case X86::ADD8rr_DB:
1278 case X86::ADD16rr_DB: {
1279 Src2 = MI.getOperand(2).getReg();
1280 Src2SubReg = MI.getOperand(2).getSubReg();
1281 bool IsKill2 = MI.getOperand(2).isKill();
1282 assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization");
1283 if (Src == Src2) {
1284 // ADD8rr/ADD16rr killed %reg1028, %reg1028
1285 // just a single insert_subreg.
1286 addRegReg(MIB, InRegLEA, false, X86::NoSubRegister, InRegLEA, false,
1287 X86::NoSubRegister);
1288 } else {
1289 if (Subtarget.is64Bit())
1290 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1291 else
1292 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1293 // Build and insert into an implicit UNDEF value. This is OK because
1294 // we will be shifting and then extracting the lower 8/16-bits.
1295 ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF),
1296 InRegLEA2);
1297 InsMI2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY))
1298 .addReg(InRegLEA2, RegState::Define, SubReg)
1299 .addReg(Src2, getKillRegState(IsKill2), Src2SubReg);
1300 addRegReg(MIB, InRegLEA, false, X86::NoSubRegister, InRegLEA2, false,
1301 X86::NoSubRegister);
1302 }
1303 break;
1304 }
1305 }
1306
1307 MachineInstr *NewMI = MIB;
1308 MachineInstr *ExtMI =
1309 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1311 .addReg(OutRegLEA, {}, SubReg);
1312
1313 if (LIS) {
1314 LIS->InsertMachineInstrInMaps(*ImpDef);
1315 SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(*InsMI);
1316 if (ImpDef2)
1317 LIS->InsertMachineInstrInMaps(*ImpDef2);
1318 SlotIndex Ins2Idx;
1319 if (InsMI2)
1320 Ins2Idx = LIS->InsertMachineInstrInMaps(*InsMI2);
1321 SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
1322 SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(*ExtMI);
1323
1324 // Drop the dead EFLAGS def MI had; the replacement does not define EFLAGS.
1325 if (MI.definesRegister(X86::EFLAGS, &RI))
1326 LIS->removePhysRegDefAt(X86::EFLAGS, NewIdx.getRegSlot());
1327
1328 LIS->getInterval(InRegLEA);
1329 LIS->getInterval(OutRegLEA);
1330 if (InRegLEA2)
1331 LIS->getInterval(InRegLEA2);
1332
1333 // Move the use of Src up to InsMI.
1334 LiveInterval &SrcLI = LIS->getInterval(Src);
1335 LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(NewIdx);
1336 if (SrcSeg->end == NewIdx.getRegSlot())
1337 SrcSeg->end = InsIdx.getRegSlot();
1338
1339 if (InsMI2) {
1340 // Move the use of Src2 up to InsMI2.
1341 LiveInterval &Src2LI = LIS->getInterval(Src2);
1342 LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(NewIdx);
1343 if (Src2Seg->end == NewIdx.getRegSlot())
1344 Src2Seg->end = Ins2Idx.getRegSlot();
1345 }
1346
1347 // Move the definition of Dest down to ExtMI.
1348 LiveInterval &DestLI = LIS->getInterval(Dest);
1349 LiveRange::Segment *DestSeg =
1350 DestLI.getSegmentContaining(NewIdx.getRegSlot());
1351 assert(DestSeg->start == NewIdx.getRegSlot() &&
1352 DestSeg->valno->def == NewIdx.getRegSlot());
1353 DestSeg->start = ExtIdx.getRegSlot();
1354 DestSeg->valno->def = ExtIdx.getRegSlot();
1355 }
1356
1357 return ExtMI;
1358}
1359
1360/// This method must be implemented by targets that
1361/// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target
1362/// may be able to convert a two-address instruction into a true
1363/// three-address instruction on demand. This allows the X86 target (for
1364/// example) to convert ADD and SHL instructions into LEA instructions if they
1365/// would require register copies due to two-addressness.
1366///
1367/// This method returns a null pointer if the transformation cannot be
1368/// performed, otherwise it returns the new instruction.
1369///
1371 LiveIntervals *LIS) const {
1372 // The following opcodes also sets the condition code register(s). Only
1373 // convert them to equivalent lea if the condition code register def's
1374 // are dead!
1376 return nullptr;
1377
1378 MachineFunction &MF = *MI.getParent()->getParent();
1379 // All instructions input are two-addr instructions. Get the known operands.
1380 const MachineOperand &Dest = MI.getOperand(0);
1381 const MachineOperand &Src = MI.getOperand(1);
1382
1383 // Ideally, operations with undef should be folded before we get here, but we
1384 // can't guarantee it. Bail out because optimizing undefs is a waste of time.
1385 // Without this, we have to forward undef state to new register operands to
1386 // avoid machine verifier errors.
1387 if (Src.isUndef())
1388 return nullptr;
1389 if (MI.getNumOperands() > 2)
1390 if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef())
1391 return nullptr;
1392
1393 MachineInstr *NewMI = nullptr;
1394 Register SrcReg, SrcReg2;
1395 unsigned SrcSubReg, SrcSubReg2;
1396 bool Is64Bit = Subtarget.is64Bit();
1397
1398 bool Is8BitOp = false;
1399 unsigned MIOpc = MI.getOpcode();
1400 switch (MIOpc) {
1401 default:
1402 llvm_unreachable("Unreachable!");
1403 CASE_NF(SHL64ri) {
1404 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1405 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1406 if (!isTruncatedShiftCountForLEA(ShAmt))
1407 return nullptr;
1408
1409 // LEA can't handle RSP.
1410 if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass(
1411 Src.getReg(), &X86::GR64_NOSPRegClass))
1412 return nullptr;
1413
1414 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
1415 .add(Dest)
1416 .addReg(0)
1417 .addImm(1LL << ShAmt)
1418 .add(Src)
1419 .addImm(0)
1420 .addReg(0);
1421 break;
1422 }
1423 CASE_NF(SHL32ri) {
1424 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1425 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1426 if (!isTruncatedShiftCountForLEA(ShAmt))
1427 return nullptr;
1428
1429 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1430
1431 // LEA can't handle ESP.
1432 bool isKill;
1433 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1434 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
1435 isKill, ImplicitOp, LIS))
1436 return nullptr;
1437
1439 BuildMI(MF, MI.getDebugLoc(), get(Opc))
1440 .add(Dest)
1441 .addReg(0)
1442 .addImm(1LL << ShAmt)
1443 .addReg(SrcReg, getKillRegState(isKill), SrcSubReg)
1444 .addImm(0)
1445 .addReg(0);
1446 if (ImplicitOp.getReg() != 0)
1447 MIB.add(ImplicitOp);
1448 NewMI = MIB;
1449
1450 break;
1451 }
1452 CASE_NF(SHL8ri)
1453 Is8BitOp = true;
1454 [[fallthrough]];
1455 CASE_NF(SHL16ri) {
1456 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1457 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1458 if (!isTruncatedShiftCountForLEA(ShAmt))
1459 return nullptr;
1460 return convertToThreeAddressWithLEA(MIOpc, MI, LIS, Is8BitOp);
1461 }
1462 CASE_NF(INC64r)
1463 CASE_NF(INC32r) {
1464 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
1465 unsigned Opc = (MIOpc == X86::INC64r || MIOpc == X86::INC64r_NF)
1466 ? X86::LEA64r
1467 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1468 bool isKill;
1469 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1470 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
1471 isKill, ImplicitOp, LIS))
1472 return nullptr;
1473
1474 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1475 .add(Dest)
1476 .addReg(SrcReg, getKillRegState(isKill));
1477 if (ImplicitOp.getReg() != 0)
1478 MIB.add(ImplicitOp);
1479
1480 NewMI = addOffset(MIB, 1);
1481
1482 break;
1483 }
1484 CASE_NF(DEC64r)
1485 CASE_NF(DEC32r) {
1486 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
1487 unsigned Opc = (MIOpc == X86::DEC64r || MIOpc == X86::DEC64r_NF)
1488 ? X86::LEA64r
1489 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1490
1491 bool isKill;
1492 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1493 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
1494 isKill, ImplicitOp, LIS))
1495 return nullptr;
1496
1497 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1498 .add(Dest)
1499 .addReg(SrcReg, getKillRegState(isKill));
1500 if (ImplicitOp.getReg() != 0)
1501 MIB.add(ImplicitOp);
1502
1503 NewMI = addOffset(MIB, -1);
1504
1505 break;
1506 }
1507 CASE_NF(DEC8r)
1508 CASE_NF(INC8r)
1509 Is8BitOp = true;
1510 [[fallthrough]];
1511 CASE_NF(DEC16r)
1512 CASE_NF(INC16r)
1513 return convertToThreeAddressWithLEA(MIOpc, MI, LIS, Is8BitOp);
1514 CASE_NF(ADD64rr)
1515 CASE_NF(ADD32rr)
1516 case X86::ADD64rr_DB:
1517 case X86::ADD32rr_DB: {
1518 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1519 unsigned Opc;
1520 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_NF ||
1521 MIOpc == X86::ADD64rr_DB)
1522 Opc = X86::LEA64r;
1523 else
1524 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1525
1526 const MachineOperand &Src2 = MI.getOperand(2);
1527 bool isKill2;
1528 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false);
1529 if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/false, SrcReg2, SrcSubReg2,
1530 isKill2, ImplicitOp2, LIS))
1531 return nullptr;
1532
1533 bool isKill;
1534 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1535 if (Src.getReg() == Src2.getReg()) {
1536 // Don't call classify LEAReg a second time on the same register, in case
1537 // the first call inserted a COPY from Src2 and marked it as killed.
1538 isKill = isKill2;
1539 SrcReg = SrcReg2;
1540 SrcSubReg = SrcSubReg2;
1541 } else {
1542 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, SrcSubReg,
1543 isKill, ImplicitOp, LIS))
1544 return nullptr;
1545 }
1546
1547 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest);
1548 if (ImplicitOp.getReg() != 0)
1549 MIB.add(ImplicitOp);
1550 if (ImplicitOp2.getReg() != 0)
1551 MIB.add(ImplicitOp2);
1552
1553 NewMI =
1554 addRegReg(MIB, SrcReg, isKill, SrcSubReg, SrcReg2, isKill2, SrcSubReg2);
1555
1556 break;
1557 }
1558 CASE_NF(ADD8rr)
1559 case X86::ADD8rr_DB:
1560 Is8BitOp = true;
1561 [[fallthrough]];
1562 CASE_NF(ADD16rr)
1563 case X86::ADD16rr_DB:
1564 return convertToThreeAddressWithLEA(MIOpc, MI, LIS, Is8BitOp);
1565 CASE_NF(ADD64ri32)
1566 case X86::ADD64ri32_DB:
1567 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1568 NewMI = addOffset(
1569 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src),
1570 MI.getOperand(2));
1571 break;
1572 CASE_NF(ADD32ri)
1573 case X86::ADD32ri_DB: {
1574 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1575 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1576
1577 bool isKill;
1578 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1579 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, SrcSubReg,
1580 isKill, ImplicitOp, LIS))
1581 return nullptr;
1582
1584 BuildMI(MF, MI.getDebugLoc(), get(Opc))
1585 .add(Dest)
1586 .addReg(SrcReg, getKillRegState(isKill), SrcSubReg);
1587 if (ImplicitOp.getReg() != 0)
1588 MIB.add(ImplicitOp);
1589
1590 NewMI = addOffset(MIB, MI.getOperand(2));
1591
1592 break;
1593 }
1594 CASE_NF(ADD8ri)
1595 case X86::ADD8ri_DB:
1596 Is8BitOp = true;
1597 [[fallthrough]];
1598 CASE_NF(ADD16ri)
1599 case X86::ADD16ri_DB:
1600 return convertToThreeAddressWithLEA(MIOpc, MI, LIS, Is8BitOp);
1601 CASE_NF(SUB8ri)
1602 CASE_NF(SUB16ri)
1603 /// FIXME: Support these similar to ADD8ri/ADD16ri*.
1604 return nullptr;
1605 CASE_NF(SUB32ri) {
1606 if (!MI.getOperand(2).isImm())
1607 return nullptr;
1608 int64_t Imm = MI.getOperand(2).getImm();
1609 if (!isInt<32>(-Imm))
1610 return nullptr;
1611
1612 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1613 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1614
1615 bool isKill;
1616 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1617 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, SrcSubReg,
1618 isKill, ImplicitOp, LIS))
1619 return nullptr;
1620
1622 BuildMI(MF, MI.getDebugLoc(), get(Opc))
1623 .add(Dest)
1624 .addReg(SrcReg, getKillRegState(isKill), SrcSubReg);
1625 if (ImplicitOp.getReg() != 0)
1626 MIB.add(ImplicitOp);
1627
1628 NewMI = addOffset(MIB, -Imm);
1629
1630 break;
1631 }
1632
1633 CASE_NF(SUB64ri32) {
1634 if (!MI.getOperand(2).isImm())
1635 return nullptr;
1636 int64_t Imm = MI.getOperand(2).getImm();
1637 if (!isInt<32>(-Imm))
1638 return nullptr;
1639
1640 assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
1641
1643 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
1644 NewMI = addOffset(MIB, -Imm);
1645 break;
1646 }
1647
1648 case X86::VMOVDQU8Z128rmk:
1649 case X86::VMOVDQU8Z256rmk:
1650 case X86::VMOVDQU8Zrmk:
1651 case X86::VMOVDQU16Z128rmk:
1652 case X86::VMOVDQU16Z256rmk:
1653 case X86::VMOVDQU16Zrmk:
1654 case X86::VMOVDQU32Z128rmk:
1655 case X86::VMOVDQA32Z128rmk:
1656 case X86::VMOVDQU32Z256rmk:
1657 case X86::VMOVDQA32Z256rmk:
1658 case X86::VMOVDQU32Zrmk:
1659 case X86::VMOVDQA32Zrmk:
1660 case X86::VMOVDQU64Z128rmk:
1661 case X86::VMOVDQA64Z128rmk:
1662 case X86::VMOVDQU64Z256rmk:
1663 case X86::VMOVDQA64Z256rmk:
1664 case X86::VMOVDQU64Zrmk:
1665 case X86::VMOVDQA64Zrmk:
1666 case X86::VMOVUPDZ128rmk:
1667 case X86::VMOVAPDZ128rmk:
1668 case X86::VMOVUPDZ256rmk:
1669 case X86::VMOVAPDZ256rmk:
1670 case X86::VMOVUPDZrmk:
1671 case X86::VMOVAPDZrmk:
1672 case X86::VMOVUPSZ128rmk:
1673 case X86::VMOVAPSZ128rmk:
1674 case X86::VMOVUPSZ256rmk:
1675 case X86::VMOVAPSZ256rmk:
1676 case X86::VMOVUPSZrmk:
1677 case X86::VMOVAPSZrmk:
1678 case X86::VBROADCASTSDZ256rmk:
1679 case X86::VBROADCASTSDZrmk:
1680 case X86::VBROADCASTSSZ128rmk:
1681 case X86::VBROADCASTSSZ256rmk:
1682 case X86::VBROADCASTSSZrmk:
1683 case X86::VPBROADCASTDZ128rmk:
1684 case X86::VPBROADCASTDZ256rmk:
1685 case X86::VPBROADCASTDZrmk:
1686 case X86::VPBROADCASTQZ128rmk:
1687 case X86::VPBROADCASTQZ256rmk:
1688 case X86::VPBROADCASTQZrmk: {
1689 unsigned Opc;
1690 switch (MIOpc) {
1691 default:
1692 llvm_unreachable("Unreachable!");
1693 case X86::VMOVDQU8Z128rmk:
1694 Opc = X86::VPBLENDMBZ128rmk;
1695 break;
1696 case X86::VMOVDQU8Z256rmk:
1697 Opc = X86::VPBLENDMBZ256rmk;
1698 break;
1699 case X86::VMOVDQU8Zrmk:
1700 Opc = X86::VPBLENDMBZrmk;
1701 break;
1702 case X86::VMOVDQU16Z128rmk:
1703 Opc = X86::VPBLENDMWZ128rmk;
1704 break;
1705 case X86::VMOVDQU16Z256rmk:
1706 Opc = X86::VPBLENDMWZ256rmk;
1707 break;
1708 case X86::VMOVDQU16Zrmk:
1709 Opc = X86::VPBLENDMWZrmk;
1710 break;
1711 case X86::VMOVDQU32Z128rmk:
1712 Opc = X86::VPBLENDMDZ128rmk;
1713 break;
1714 case X86::VMOVDQU32Z256rmk:
1715 Opc = X86::VPBLENDMDZ256rmk;
1716 break;
1717 case X86::VMOVDQU32Zrmk:
1718 Opc = X86::VPBLENDMDZrmk;
1719 break;
1720 case X86::VMOVDQU64Z128rmk:
1721 Opc = X86::VPBLENDMQZ128rmk;
1722 break;
1723 case X86::VMOVDQU64Z256rmk:
1724 Opc = X86::VPBLENDMQZ256rmk;
1725 break;
1726 case X86::VMOVDQU64Zrmk:
1727 Opc = X86::VPBLENDMQZrmk;
1728 break;
1729 case X86::VMOVUPDZ128rmk:
1730 Opc = X86::VBLENDMPDZ128rmk;
1731 break;
1732 case X86::VMOVUPDZ256rmk:
1733 Opc = X86::VBLENDMPDZ256rmk;
1734 break;
1735 case X86::VMOVUPDZrmk:
1736 Opc = X86::VBLENDMPDZrmk;
1737 break;
1738 case X86::VMOVUPSZ128rmk:
1739 Opc = X86::VBLENDMPSZ128rmk;
1740 break;
1741 case X86::VMOVUPSZ256rmk:
1742 Opc = X86::VBLENDMPSZ256rmk;
1743 break;
1744 case X86::VMOVUPSZrmk:
1745 Opc = X86::VBLENDMPSZrmk;
1746 break;
1747 case X86::VMOVDQA32Z128rmk:
1748 Opc = X86::VPBLENDMDZ128rmk;
1749 break;
1750 case X86::VMOVDQA32Z256rmk:
1751 Opc = X86::VPBLENDMDZ256rmk;
1752 break;
1753 case X86::VMOVDQA32Zrmk:
1754 Opc = X86::VPBLENDMDZrmk;
1755 break;
1756 case X86::VMOVDQA64Z128rmk:
1757 Opc = X86::VPBLENDMQZ128rmk;
1758 break;
1759 case X86::VMOVDQA64Z256rmk:
1760 Opc = X86::VPBLENDMQZ256rmk;
1761 break;
1762 case X86::VMOVDQA64Zrmk:
1763 Opc = X86::VPBLENDMQZrmk;
1764 break;
1765 case X86::VMOVAPDZ128rmk:
1766 Opc = X86::VBLENDMPDZ128rmk;
1767 break;
1768 case X86::VMOVAPDZ256rmk:
1769 Opc = X86::VBLENDMPDZ256rmk;
1770 break;
1771 case X86::VMOVAPDZrmk:
1772 Opc = X86::VBLENDMPDZrmk;
1773 break;
1774 case X86::VMOVAPSZ128rmk:
1775 Opc = X86::VBLENDMPSZ128rmk;
1776 break;
1777 case X86::VMOVAPSZ256rmk:
1778 Opc = X86::VBLENDMPSZ256rmk;
1779 break;
1780 case X86::VMOVAPSZrmk:
1781 Opc = X86::VBLENDMPSZrmk;
1782 break;
1783 case X86::VBROADCASTSDZ256rmk:
1784 Opc = X86::VBLENDMPDZ256rmbk;
1785 break;
1786 case X86::VBROADCASTSDZrmk:
1787 Opc = X86::VBLENDMPDZrmbk;
1788 break;
1789 case X86::VBROADCASTSSZ128rmk:
1790 Opc = X86::VBLENDMPSZ128rmbk;
1791 break;
1792 case X86::VBROADCASTSSZ256rmk:
1793 Opc = X86::VBLENDMPSZ256rmbk;
1794 break;
1795 case X86::VBROADCASTSSZrmk:
1796 Opc = X86::VBLENDMPSZrmbk;
1797 break;
1798 case X86::VPBROADCASTDZ128rmk:
1799 Opc = X86::VPBLENDMDZ128rmbk;
1800 break;
1801 case X86::VPBROADCASTDZ256rmk:
1802 Opc = X86::VPBLENDMDZ256rmbk;
1803 break;
1804 case X86::VPBROADCASTDZrmk:
1805 Opc = X86::VPBLENDMDZrmbk;
1806 break;
1807 case X86::VPBROADCASTQZ128rmk:
1808 Opc = X86::VPBLENDMQZ128rmbk;
1809 break;
1810 case X86::VPBROADCASTQZ256rmk:
1811 Opc = X86::VPBLENDMQZ256rmbk;
1812 break;
1813 case X86::VPBROADCASTQZrmk:
1814 Opc = X86::VPBLENDMQZrmbk;
1815 break;
1816 }
1817
1818 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1819 .add(Dest)
1820 .add(MI.getOperand(2))
1821 .add(Src)
1822 .add(MI.getOperand(3))
1823 .add(MI.getOperand(4))
1824 .add(MI.getOperand(5))
1825 .add(MI.getOperand(6))
1826 .add(MI.getOperand(7));
1827 break;
1828 }
1829
1830 case X86::VMOVDQU8Z128rrk:
1831 case X86::VMOVDQU8Z256rrk:
1832 case X86::VMOVDQU8Zrrk:
1833 case X86::VMOVDQU16Z128rrk:
1834 case X86::VMOVDQU16Z256rrk:
1835 case X86::VMOVDQU16Zrrk:
1836 case X86::VMOVDQU32Z128rrk:
1837 case X86::VMOVDQA32Z128rrk:
1838 case X86::VMOVDQU32Z256rrk:
1839 case X86::VMOVDQA32Z256rrk:
1840 case X86::VMOVDQU32Zrrk:
1841 case X86::VMOVDQA32Zrrk:
1842 case X86::VMOVDQU64Z128rrk:
1843 case X86::VMOVDQA64Z128rrk:
1844 case X86::VMOVDQU64Z256rrk:
1845 case X86::VMOVDQA64Z256rrk:
1846 case X86::VMOVDQU64Zrrk:
1847 case X86::VMOVDQA64Zrrk:
1848 case X86::VMOVUPDZ128rrk:
1849 case X86::VMOVAPDZ128rrk:
1850 case X86::VMOVUPDZ256rrk:
1851 case X86::VMOVAPDZ256rrk:
1852 case X86::VMOVUPDZrrk:
1853 case X86::VMOVAPDZrrk:
1854 case X86::VMOVUPSZ128rrk:
1855 case X86::VMOVAPSZ128rrk:
1856 case X86::VMOVUPSZ256rrk:
1857 case X86::VMOVAPSZ256rrk:
1858 case X86::VMOVUPSZrrk:
1859 case X86::VMOVAPSZrrk: {
1860 unsigned Opc;
1861 switch (MIOpc) {
1862 default:
1863 llvm_unreachable("Unreachable!");
1864 case X86::VMOVDQU8Z128rrk:
1865 Opc = X86::VPBLENDMBZ128rrk;
1866 break;
1867 case X86::VMOVDQU8Z256rrk:
1868 Opc = X86::VPBLENDMBZ256rrk;
1869 break;
1870 case X86::VMOVDQU8Zrrk:
1871 Opc = X86::VPBLENDMBZrrk;
1872 break;
1873 case X86::VMOVDQU16Z128rrk:
1874 Opc = X86::VPBLENDMWZ128rrk;
1875 break;
1876 case X86::VMOVDQU16Z256rrk:
1877 Opc = X86::VPBLENDMWZ256rrk;
1878 break;
1879 case X86::VMOVDQU16Zrrk:
1880 Opc = X86::VPBLENDMWZrrk;
1881 break;
1882 case X86::VMOVDQU32Z128rrk:
1883 Opc = X86::VPBLENDMDZ128rrk;
1884 break;
1885 case X86::VMOVDQU32Z256rrk:
1886 Opc = X86::VPBLENDMDZ256rrk;
1887 break;
1888 case X86::VMOVDQU32Zrrk:
1889 Opc = X86::VPBLENDMDZrrk;
1890 break;
1891 case X86::VMOVDQU64Z128rrk:
1892 Opc = X86::VPBLENDMQZ128rrk;
1893 break;
1894 case X86::VMOVDQU64Z256rrk:
1895 Opc = X86::VPBLENDMQZ256rrk;
1896 break;
1897 case X86::VMOVDQU64Zrrk:
1898 Opc = X86::VPBLENDMQZrrk;
1899 break;
1900 case X86::VMOVUPDZ128rrk:
1901 Opc = X86::VBLENDMPDZ128rrk;
1902 break;
1903 case X86::VMOVUPDZ256rrk:
1904 Opc = X86::VBLENDMPDZ256rrk;
1905 break;
1906 case X86::VMOVUPDZrrk:
1907 Opc = X86::VBLENDMPDZrrk;
1908 break;
1909 case X86::VMOVUPSZ128rrk:
1910 Opc = X86::VBLENDMPSZ128rrk;
1911 break;
1912 case X86::VMOVUPSZ256rrk:
1913 Opc = X86::VBLENDMPSZ256rrk;
1914 break;
1915 case X86::VMOVUPSZrrk:
1916 Opc = X86::VBLENDMPSZrrk;
1917 break;
1918 case X86::VMOVDQA32Z128rrk:
1919 Opc = X86::VPBLENDMDZ128rrk;
1920 break;
1921 case X86::VMOVDQA32Z256rrk:
1922 Opc = X86::VPBLENDMDZ256rrk;
1923 break;
1924 case X86::VMOVDQA32Zrrk:
1925 Opc = X86::VPBLENDMDZrrk;
1926 break;
1927 case X86::VMOVDQA64Z128rrk:
1928 Opc = X86::VPBLENDMQZ128rrk;
1929 break;
1930 case X86::VMOVDQA64Z256rrk:
1931 Opc = X86::VPBLENDMQZ256rrk;
1932 break;
1933 case X86::VMOVDQA64Zrrk:
1934 Opc = X86::VPBLENDMQZrrk;
1935 break;
1936 case X86::VMOVAPDZ128rrk:
1937 Opc = X86::VBLENDMPDZ128rrk;
1938 break;
1939 case X86::VMOVAPDZ256rrk:
1940 Opc = X86::VBLENDMPDZ256rrk;
1941 break;
1942 case X86::VMOVAPDZrrk:
1943 Opc = X86::VBLENDMPDZrrk;
1944 break;
1945 case X86::VMOVAPSZ128rrk:
1946 Opc = X86::VBLENDMPSZ128rrk;
1947 break;
1948 case X86::VMOVAPSZ256rrk:
1949 Opc = X86::VBLENDMPSZ256rrk;
1950 break;
1951 case X86::VMOVAPSZrrk:
1952 Opc = X86::VBLENDMPSZrrk;
1953 break;
1954 }
1955
1956 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1957 .add(Dest)
1958 .add(MI.getOperand(2))
1959 .add(Src)
1960 .add(MI.getOperand(3));
1961 break;
1962 }
1963 }
1964#undef CASE_NF
1965
1966 if (!NewMI)
1967 return nullptr;
1968
1969 MachineBasicBlock &MBB = *MI.getParent();
1970 MBB.insert(MI.getIterator(), NewMI); // Insert the new inst
1971
1972 if (LIS) {
1973 // The replacement does not define EFLAGS; drop the dead EFLAGS def MI had.
1974 SlotIndex Idx = LIS->getInstructionIndex(MI);
1975 LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
1976
1977 if (MI.definesRegister(X86::EFLAGS, &RI))
1978 LIS->removePhysRegDefAt(X86::EFLAGS, Idx.getRegSlot());
1979 if (SrcReg)
1980 LIS->getInterval(SrcReg);
1981 if (SrcReg2)
1982 LIS->getInterval(SrcReg2);
1983 }
1984
1985 return NewMI;
1986}
1987
1988/// This determines which of three possible cases of a three source commute
1989/// the source indexes correspond to taking into account any mask operands.
1990/// All prevents commuting a passthru operand. Returns -1 if the commute isn't
1991/// possible.
1992/// Case 0 - Possible to commute the first and second operands.
1993/// Case 1 - Possible to commute the first and third operands.
1994/// Case 2 - Possible to commute the second and third operands.
1995static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1,
1996 unsigned SrcOpIdx2) {
1997 // Put the lowest index to SrcOpIdx1 to simplify the checks below.
1998 if (SrcOpIdx1 > SrcOpIdx2)
1999 std::swap(SrcOpIdx1, SrcOpIdx2);
2000
2001 unsigned Op1 = 1, Op2 = 2, Op3 = 3;
2002 if (X86II::isKMasked(TSFlags)) {
2003 Op2++;
2004 Op3++;
2005 }
2006
2007 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
2008 return 0;
2009 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
2010 return 1;
2011 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
2012 return 2;
2013 llvm_unreachable("Unknown three src commute case.");
2014}
2015
2017 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2,
2018 const X86InstrFMA3Group &FMA3Group) const {
2019
2020 unsigned Opc = MI.getOpcode();
2021
2022 // TODO: Commuting the 1st operand of FMA*_Int requires some additional
2023 // analysis. The commute optimization is legal only if all users of FMA*_Int
2024 // use only the lowest element of the FMA*_Int instruction. Such analysis are
2025 // not implemented yet. So, just return 0 in that case.
2026 // When such analysis are available this place will be the right place for
2027 // calling it.
2028 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) &&
2029 "Intrinsic instructions can't commute operand 1");
2030
2031 // Determine which case this commute is or if it can't be done.
2032 unsigned Case =
2033 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2034 assert(Case < 3 && "Unexpected case number!");
2035
2036 // Define the FMA forms mapping array that helps to map input FMA form
2037 // to output FMA form to preserve the operation semantics after
2038 // commuting the operands.
2039 const unsigned Form132Index = 0;
2040 const unsigned Form213Index = 1;
2041 const unsigned Form231Index = 2;
2042 static const unsigned FormMapping[][3] = {
2043 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2;
2044 // FMA132 A, C, b; ==> FMA231 C, A, b;
2045 // FMA213 B, A, c; ==> FMA213 A, B, c;
2046 // FMA231 C, A, b; ==> FMA132 A, C, b;
2047 {Form231Index, Form213Index, Form132Index},
2048 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3;
2049 // FMA132 A, c, B; ==> FMA132 B, c, A;
2050 // FMA213 B, a, C; ==> FMA231 C, a, B;
2051 // FMA231 C, a, B; ==> FMA213 B, a, C;
2052 {Form132Index, Form231Index, Form213Index},
2053 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3;
2054 // FMA132 a, C, B; ==> FMA213 a, B, C;
2055 // FMA213 b, A, C; ==> FMA132 b, C, A;
2056 // FMA231 c, A, B; ==> FMA231 c, B, A;
2057 {Form213Index, Form132Index, Form231Index}};
2058
2059 unsigned FMAForms[3];
2060 FMAForms[0] = FMA3Group.get132Opcode();
2061 FMAForms[1] = FMA3Group.get213Opcode();
2062 FMAForms[2] = FMA3Group.get231Opcode();
2063
2064 // Everything is ready, just adjust the FMA opcode and return it.
2065 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
2066 if (Opc == FMAForms[FormIndex])
2067 return FMAForms[FormMapping[Case][FormIndex]];
2068
2069 llvm_unreachable("Illegal FMA3 format");
2070}
2071
2072static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1,
2073 unsigned SrcOpIdx2) {
2074 // Determine which case this commute is or if it can't be done.
2075 unsigned Case =
2076 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2077 assert(Case < 3 && "Unexpected case value!");
2078
2079 // For each case we need to swap two pairs of bits in the final immediate.
2080 static const uint8_t SwapMasks[3][4] = {
2081 {0x04, 0x10, 0x08, 0x20}, // Swap bits 2/4 and 3/5.
2082 {0x02, 0x10, 0x08, 0x40}, // Swap bits 1/4 and 3/6.
2083 {0x02, 0x04, 0x20, 0x40}, // Swap bits 1/2 and 5/6.
2084 };
2085
2086 uint8_t Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
2087 // Clear out the bits we are swapping.
2088 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
2089 SwapMasks[Case][2] | SwapMasks[Case][3]);
2090 // If the immediate had a bit of the pair set, then set the opposite bit.
2091 if (Imm & SwapMasks[Case][0])
2092 NewImm |= SwapMasks[Case][1];
2093 if (Imm & SwapMasks[Case][1])
2094 NewImm |= SwapMasks[Case][0];
2095 if (Imm & SwapMasks[Case][2])
2096 NewImm |= SwapMasks[Case][3];
2097 if (Imm & SwapMasks[Case][3])
2098 NewImm |= SwapMasks[Case][2];
2099 MI.getOperand(MI.getNumOperands() - 1).setImm(NewImm);
2100}
2101
2102// Returns true if this is a VPERMI2 or VPERMT2 instruction that can be
2103// commuted.
2104static bool isCommutableVPERMV3Instruction(unsigned Opcode) {
2105#define VPERM_CASES(Suffix) \
2106 case X86::VPERMI2##Suffix##Z128rr: \
2107 case X86::VPERMT2##Suffix##Z128rr: \
2108 case X86::VPERMI2##Suffix##Z256rr: \
2109 case X86::VPERMT2##Suffix##Z256rr: \
2110 case X86::VPERMI2##Suffix##Zrr: \
2111 case X86::VPERMT2##Suffix##Zrr: \
2112 case X86::VPERMI2##Suffix##Z128rm: \
2113 case X86::VPERMT2##Suffix##Z128rm: \
2114 case X86::VPERMI2##Suffix##Z256rm: \
2115 case X86::VPERMT2##Suffix##Z256rm: \
2116 case X86::VPERMI2##Suffix##Zrm: \
2117 case X86::VPERMT2##Suffix##Zrm: \
2118 case X86::VPERMI2##Suffix##Z128rrkz: \
2119 case X86::VPERMT2##Suffix##Z128rrkz: \
2120 case X86::VPERMI2##Suffix##Z256rrkz: \
2121 case X86::VPERMT2##Suffix##Z256rrkz: \
2122 case X86::VPERMI2##Suffix##Zrrkz: \
2123 case X86::VPERMT2##Suffix##Zrrkz: \
2124 case X86::VPERMI2##Suffix##Z128rmkz: \
2125 case X86::VPERMT2##Suffix##Z128rmkz: \
2126 case X86::VPERMI2##Suffix##Z256rmkz: \
2127 case X86::VPERMT2##Suffix##Z256rmkz: \
2128 case X86::VPERMI2##Suffix##Zrmkz: \
2129 case X86::VPERMT2##Suffix##Zrmkz:
2130
2131#define VPERM_CASES_BROADCAST(Suffix) \
2132 VPERM_CASES(Suffix) \
2133 case X86::VPERMI2##Suffix##Z128rmb: \
2134 case X86::VPERMT2##Suffix##Z128rmb: \
2135 case X86::VPERMI2##Suffix##Z256rmb: \
2136 case X86::VPERMT2##Suffix##Z256rmb: \
2137 case X86::VPERMI2##Suffix##Zrmb: \
2138 case X86::VPERMT2##Suffix##Zrmb: \
2139 case X86::VPERMI2##Suffix##Z128rmbkz: \
2140 case X86::VPERMT2##Suffix##Z128rmbkz: \
2141 case X86::VPERMI2##Suffix##Z256rmbkz: \
2142 case X86::VPERMT2##Suffix##Z256rmbkz: \
2143 case X86::VPERMI2##Suffix##Zrmbkz: \
2144 case X86::VPERMT2##Suffix##Zrmbkz:
2145
2146 switch (Opcode) {
2147 default:
2148 return false;
2149 VPERM_CASES(B)
2154 VPERM_CASES(W)
2155 return true;
2156 }
2157#undef VPERM_CASES_BROADCAST
2158#undef VPERM_CASES
2159}
2160
2161// Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching
2162// from the I opcode to the T opcode and vice versa.
2163static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) {
2164#define VPERM_CASES(Orig, New) \
2165 case X86::Orig##Z128rr: \
2166 return X86::New##Z128rr; \
2167 case X86::Orig##Z128rrkz: \
2168 return X86::New##Z128rrkz; \
2169 case X86::Orig##Z128rm: \
2170 return X86::New##Z128rm; \
2171 case X86::Orig##Z128rmkz: \
2172 return X86::New##Z128rmkz; \
2173 case X86::Orig##Z256rr: \
2174 return X86::New##Z256rr; \
2175 case X86::Orig##Z256rrkz: \
2176 return X86::New##Z256rrkz; \
2177 case X86::Orig##Z256rm: \
2178 return X86::New##Z256rm; \
2179 case X86::Orig##Z256rmkz: \
2180 return X86::New##Z256rmkz; \
2181 case X86::Orig##Zrr: \
2182 return X86::New##Zrr; \
2183 case X86::Orig##Zrrkz: \
2184 return X86::New##Zrrkz; \
2185 case X86::Orig##Zrm: \
2186 return X86::New##Zrm; \
2187 case X86::Orig##Zrmkz: \
2188 return X86::New##Zrmkz;
2189
2190#define VPERM_CASES_BROADCAST(Orig, New) \
2191 VPERM_CASES(Orig, New) \
2192 case X86::Orig##Z128rmb: \
2193 return X86::New##Z128rmb; \
2194 case X86::Orig##Z128rmbkz: \
2195 return X86::New##Z128rmbkz; \
2196 case X86::Orig##Z256rmb: \
2197 return X86::New##Z256rmb; \
2198 case X86::Orig##Z256rmbkz: \
2199 return X86::New##Z256rmbkz; \
2200 case X86::Orig##Zrmb: \
2201 return X86::New##Zrmb; \
2202 case X86::Orig##Zrmbkz: \
2203 return X86::New##Zrmbkz;
2204
2205 switch (Opcode) {
2206 VPERM_CASES(VPERMI2B, VPERMT2B)
2207 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D)
2208 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD)
2209 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS)
2210 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q)
2211 VPERM_CASES(VPERMI2W, VPERMT2W)
2212 VPERM_CASES(VPERMT2B, VPERMI2B)
2213 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D)
2214 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD)
2215 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS)
2216 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q)
2217 VPERM_CASES(VPERMT2W, VPERMI2W)
2218 }
2219
2220 llvm_unreachable("Unreachable!");
2221#undef VPERM_CASES_BROADCAST
2222#undef VPERM_CASES
2223}
2224
2226 unsigned OpIdx1,
2227 unsigned OpIdx2) const {
2228 auto CloneIfNew = [&](MachineInstr &MI) {
2229 return std::exchange(NewMI, false)
2230 ? MI.getParent()->getParent()->CloneMachineInstr(&MI)
2231 : &MI;
2232 };
2233 MachineInstr *WorkingMI = nullptr;
2234 unsigned Opc = MI.getOpcode();
2235
2236#define CASE_ND(OP) \
2237 case X86::OP: \
2238 case X86::OP##_ND:
2239
2240#define CASE_SHXD(OP) \
2241 CASE_ND(OP) \
2242 case X86::OP##_NF: \
2243 case X86::OP##_NF_ND: \
2244 case X86::OP##_EVEX:
2245
2246 // clang-format off
2247 switch (Opc) {
2248 // SHLD B, C, I <-> SHRD C, B, (BitWidth - I). All rri8 variants (legacy,
2249 // _ND, _NF, _NF_ND, _EVEX) are commutable; swapping only the sources would
2250 // change the result, so the opcode and immediate must be updated too.
2251 CASE_SHXD(SHRD16rri8)
2252 CASE_SHXD(SHLD16rri8)
2253 CASE_SHXD(SHRD32rri8)
2254 CASE_SHXD(SHLD32rri8)
2255 CASE_SHXD(SHRD64rri8)
2256 CASE_SHXD(SHLD64rri8) {
2257 unsigned Size;
2258 switch (Opc) {
2259 default:
2260 llvm_unreachable("Unreachable!");
2261#define FROM_TO_SUFFIX(A, B, S, SFX) \
2262 case X86::A##SFX: \
2263 Opc = X86::B##SFX; \
2264 Size = S; \
2265 break; \
2266 case X86::B##SFX: \
2267 Opc = X86::A##SFX; \
2268 Size = S; \
2269 break;
2270#define FROM_TO_SIZE(A, B, S) \
2271 FROM_TO_SUFFIX(A, B, S, ) \
2272 FROM_TO_SUFFIX(A, B, S, _ND) \
2273 FROM_TO_SUFFIX(A, B, S, _NF) \
2274 FROM_TO_SUFFIX(A, B, S, _NF_ND) \
2275 FROM_TO_SUFFIX(A, B, S, _EVEX)
2276
2277 FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
2278 FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
2279 FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
2280#undef FROM_TO_SIZE
2281#undef FROM_TO_SUFFIX
2282#undef CASE_SHXD
2283 }
2284 WorkingMI = CloneIfNew(MI);
2285 WorkingMI->setDesc(get(Opc));
2286 WorkingMI->getOperand(3).setImm(Size - MI.getOperand(3).getImm());
2287 break;
2288 }
2289 // clang-format on
2290 case X86::PFSUBrr:
2291 case X86::PFSUBRrr:
2292 // PFSUB x, y: x = x - y
2293 // PFSUBR x, y: x = y - x
2294 WorkingMI = CloneIfNew(MI);
2295 WorkingMI->setDesc(
2296 get(X86::PFSUBRrr == Opc ? X86::PFSUBrr : X86::PFSUBRrr));
2297 break;
2298 case X86::BLENDPDrri:
2299 case X86::BLENDPSrri:
2300 case X86::PBLENDWrri:
2301 case X86::VBLENDPDrri:
2302 case X86::VBLENDPSrri:
2303 case X86::VBLENDPDYrri:
2304 case X86::VBLENDPSYrri:
2305 case X86::VPBLENDDrri:
2306 case X86::VPBLENDWrri:
2307 case X86::VPBLENDDYrri:
2308 case X86::VPBLENDWYrri: {
2309 int8_t Mask;
2310 switch (Opc) {
2311 default:
2312 llvm_unreachable("Unreachable!");
2313 case X86::BLENDPDrri:
2314 Mask = (int8_t)0x03;
2315 break;
2316 case X86::BLENDPSrri:
2317 Mask = (int8_t)0x0F;
2318 break;
2319 case X86::PBLENDWrri:
2320 Mask = (int8_t)0xFF;
2321 break;
2322 case X86::VBLENDPDrri:
2323 Mask = (int8_t)0x03;
2324 break;
2325 case X86::VBLENDPSrri:
2326 Mask = (int8_t)0x0F;
2327 break;
2328 case X86::VBLENDPDYrri:
2329 Mask = (int8_t)0x0F;
2330 break;
2331 case X86::VBLENDPSYrri:
2332 Mask = (int8_t)0xFF;
2333 break;
2334 case X86::VPBLENDDrri:
2335 Mask = (int8_t)0x0F;
2336 break;
2337 case X86::VPBLENDWrri:
2338 Mask = (int8_t)0xFF;
2339 break;
2340 case X86::VPBLENDDYrri:
2341 Mask = (int8_t)0xFF;
2342 break;
2343 case X86::VPBLENDWYrri:
2344 Mask = (int8_t)0xFF;
2345 break;
2346 }
2347 // Only the least significant bits of Imm are used.
2348 // Using int8_t to ensure it will be sign extended to the int64_t that
2349 // setImm takes in order to match isel behavior.
2350 int8_t Imm = MI.getOperand(3).getImm() & Mask;
2351 WorkingMI = CloneIfNew(MI);
2352 WorkingMI->getOperand(3).setImm(Mask ^ Imm);
2353 break;
2354 }
2355 case X86::INSERTPSrri:
2356 case X86::VINSERTPSrri:
2357 case X86::VINSERTPSZrri: {
2358 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
2359 unsigned ZMask = Imm & 15;
2360 unsigned DstIdx = (Imm >> 4) & 3;
2361 unsigned SrcIdx = (Imm >> 6) & 3;
2362
2363 // We can commute insertps if we zero 2 of the elements, the insertion is
2364 // "inline" and we don't override the insertion with a zero.
2365 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2366 llvm::popcount(ZMask) == 2) {
2367 unsigned AltIdx = llvm::countr_zero((ZMask | (1 << DstIdx)) ^ 15);
2368 assert(AltIdx < 4 && "Illegal insertion index");
2369 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
2370 WorkingMI = CloneIfNew(MI);
2371 WorkingMI->getOperand(MI.getNumOperands() - 1).setImm(AltImm);
2372 break;
2373 }
2374 return nullptr;
2375 }
2376 case X86::MOVSDrr:
2377 case X86::MOVSSrr:
2378 case X86::VMOVSDrr:
2379 case X86::VMOVSSrr: {
2380 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD.
2381 if (Subtarget.hasSSE41()) {
2382 unsigned Mask;
2383 switch (Opc) {
2384 default:
2385 llvm_unreachable("Unreachable!");
2386 case X86::MOVSDrr:
2387 Opc = X86::BLENDPDrri;
2388 Mask = 0x02;
2389 break;
2390 case X86::MOVSSrr:
2391 Opc = X86::BLENDPSrri;
2392 Mask = 0x0E;
2393 break;
2394 case X86::VMOVSDrr:
2395 Opc = X86::VBLENDPDrri;
2396 Mask = 0x02;
2397 break;
2398 case X86::VMOVSSrr:
2399 Opc = X86::VBLENDPSrri;
2400 Mask = 0x0E;
2401 break;
2402 }
2403
2404 WorkingMI = CloneIfNew(MI);
2405 WorkingMI->setDesc(get(Opc));
2406 WorkingMI->addOperand(MachineOperand::CreateImm(Mask));
2407 break;
2408 }
2409
2410 assert(Opc == X86::MOVSDrr && "Only MOVSD can commute to SHUFPD");
2411 WorkingMI = CloneIfNew(MI);
2412 WorkingMI->setDesc(get(X86::SHUFPDrri));
2413 WorkingMI->addOperand(MachineOperand::CreateImm(0x02));
2414 break;
2415 }
2416 case X86::SHUFPDrri: {
2417 // Commute to MOVSD.
2418 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!");
2419 WorkingMI = CloneIfNew(MI);
2420 WorkingMI->setDesc(get(X86::MOVSDrr));
2421 WorkingMI->removeOperand(3);
2422 break;
2423 }
2424 case X86::PCLMULQDQrri:
2425 case X86::VPCLMULQDQrri:
2426 case X86::VPCLMULQDQYrri:
2427 case X86::VPCLMULQDQZrri:
2428 case X86::VPCLMULQDQZ128rri:
2429 case X86::VPCLMULQDQZ256rri: {
2430 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0]
2431 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0]
2432 unsigned Imm = MI.getOperand(3).getImm();
2433 unsigned Src1Hi = Imm & 0x01;
2434 unsigned Src2Hi = Imm & 0x10;
2435 WorkingMI = CloneIfNew(MI);
2436 WorkingMI->getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4));
2437 break;
2438 }
2439 case X86::VPCMPBZ128rri:
2440 case X86::VPCMPUBZ128rri:
2441 case X86::VPCMPBZ256rri:
2442 case X86::VPCMPUBZ256rri:
2443 case X86::VPCMPBZrri:
2444 case X86::VPCMPUBZrri:
2445 case X86::VPCMPDZ128rri:
2446 case X86::VPCMPUDZ128rri:
2447 case X86::VPCMPDZ256rri:
2448 case X86::VPCMPUDZ256rri:
2449 case X86::VPCMPDZrri:
2450 case X86::VPCMPUDZrri:
2451 case X86::VPCMPQZ128rri:
2452 case X86::VPCMPUQZ128rri:
2453 case X86::VPCMPQZ256rri:
2454 case X86::VPCMPUQZ256rri:
2455 case X86::VPCMPQZrri:
2456 case X86::VPCMPUQZrri:
2457 case X86::VPCMPWZ128rri:
2458 case X86::VPCMPUWZ128rri:
2459 case X86::VPCMPWZ256rri:
2460 case X86::VPCMPUWZ256rri:
2461 case X86::VPCMPWZrri:
2462 case X86::VPCMPUWZrri:
2463 case X86::VPCMPBZ128rrik:
2464 case X86::VPCMPUBZ128rrik:
2465 case X86::VPCMPBZ256rrik:
2466 case X86::VPCMPUBZ256rrik:
2467 case X86::VPCMPBZrrik:
2468 case X86::VPCMPUBZrrik:
2469 case X86::VPCMPDZ128rrik:
2470 case X86::VPCMPUDZ128rrik:
2471 case X86::VPCMPDZ256rrik:
2472 case X86::VPCMPUDZ256rrik:
2473 case X86::VPCMPDZrrik:
2474 case X86::VPCMPUDZrrik:
2475 case X86::VPCMPQZ128rrik:
2476 case X86::VPCMPUQZ128rrik:
2477 case X86::VPCMPQZ256rrik:
2478 case X86::VPCMPUQZ256rrik:
2479 case X86::VPCMPQZrrik:
2480 case X86::VPCMPUQZrrik:
2481 case X86::VPCMPWZ128rrik:
2482 case X86::VPCMPUWZ128rrik:
2483 case X86::VPCMPWZ256rrik:
2484 case X86::VPCMPUWZ256rrik:
2485 case X86::VPCMPWZrrik:
2486 case X86::VPCMPUWZrrik:
2487 WorkingMI = CloneIfNew(MI);
2488 // Flip comparison mode immediate (if necessary).
2489 WorkingMI->getOperand(MI.getNumOperands() - 1)
2491 MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7));
2492 break;
2493 case X86::VPCOMBri:
2494 case X86::VPCOMUBri:
2495 case X86::VPCOMDri:
2496 case X86::VPCOMUDri:
2497 case X86::VPCOMQri:
2498 case X86::VPCOMUQri:
2499 case X86::VPCOMWri:
2500 case X86::VPCOMUWri:
2501 WorkingMI = CloneIfNew(MI);
2502 // Flip comparison mode immediate (if necessary).
2503 WorkingMI->getOperand(3).setImm(
2504 X86::getSwappedVPCOMImm(MI.getOperand(3).getImm() & 0x7));
2505 break;
2506 case X86::VCMPSDZrri:
2507 case X86::VCMPSSZrri:
2508 case X86::VCMPPDZrri:
2509 case X86::VCMPPSZrri:
2510 case X86::VCMPSHZrri:
2511 case X86::VCMPPHZrri:
2512 case X86::VCMPPHZ128rri:
2513 case X86::VCMPPHZ256rri:
2514 case X86::VCMPPDZ128rri:
2515 case X86::VCMPPSZ128rri:
2516 case X86::VCMPPDZ256rri:
2517 case X86::VCMPPSZ256rri:
2518 case X86::VCMPPDZrrik:
2519 case X86::VCMPPSZrrik:
2520 case X86::VCMPPHZrrik:
2521 case X86::VCMPPDZ128rrik:
2522 case X86::VCMPPSZ128rrik:
2523 case X86::VCMPPHZ128rrik:
2524 case X86::VCMPPDZ256rrik:
2525 case X86::VCMPPSZ256rrik:
2526 case X86::VCMPPHZ256rrik:
2527 WorkingMI = CloneIfNew(MI);
2528 WorkingMI->getOperand(MI.getNumExplicitOperands() - 1)
2530 MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f));
2531 break;
2532 case X86::VPERM2F128rri:
2533 case X86::VPERM2I128rri:
2534 // Flip permute source immediate.
2535 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi.
2536 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi.
2537 WorkingMI = CloneIfNew(MI);
2538 WorkingMI->getOperand(3).setImm((MI.getOperand(3).getImm() & 0xFF) ^ 0x22);
2539 break;
2540 case X86::MOVHLPSrr:
2541 case X86::UNPCKHPDrr:
2542 case X86::VMOVHLPSrr:
2543 case X86::VUNPCKHPDrr:
2544 case X86::VMOVHLPSZrr:
2545 case X86::VUNPCKHPDZ128rr:
2546 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!");
2547
2548 switch (Opc) {
2549 default:
2550 llvm_unreachable("Unreachable!");
2551 case X86::MOVHLPSrr:
2552 Opc = X86::UNPCKHPDrr;
2553 break;
2554 case X86::UNPCKHPDrr:
2555 Opc = X86::MOVHLPSrr;
2556 break;
2557 case X86::VMOVHLPSrr:
2558 Opc = X86::VUNPCKHPDrr;
2559 break;
2560 case X86::VUNPCKHPDrr:
2561 Opc = X86::VMOVHLPSrr;
2562 break;
2563 case X86::VMOVHLPSZrr:
2564 Opc = X86::VUNPCKHPDZ128rr;
2565 break;
2566 case X86::VUNPCKHPDZ128rr:
2567 Opc = X86::VMOVHLPSZrr;
2568 break;
2569 }
2570 WorkingMI = CloneIfNew(MI);
2571 WorkingMI->setDesc(get(Opc));
2572 break;
2573 CASE_ND(CMOV16rr)
2574 CASE_ND(CMOV32rr)
2575 CASE_ND(CMOV64rr) {
2576 WorkingMI = CloneIfNew(MI);
2577 unsigned OpNo = MI.getDesc().getNumOperands() - 1;
2578 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
2580 break;
2581 }
2582 case X86::VPTERNLOGDZrri:
2583 case X86::VPTERNLOGDZrmi:
2584 case X86::VPTERNLOGDZ128rri:
2585 case X86::VPTERNLOGDZ128rmi:
2586 case X86::VPTERNLOGDZ256rri:
2587 case X86::VPTERNLOGDZ256rmi:
2588 case X86::VPTERNLOGQZrri:
2589 case X86::VPTERNLOGQZrmi:
2590 case X86::VPTERNLOGQZ128rri:
2591 case X86::VPTERNLOGQZ128rmi:
2592 case X86::VPTERNLOGQZ256rri:
2593 case X86::VPTERNLOGQZ256rmi:
2594 case X86::VPTERNLOGDZrrik:
2595 case X86::VPTERNLOGDZ128rrik:
2596 case X86::VPTERNLOGDZ256rrik:
2597 case X86::VPTERNLOGQZrrik:
2598 case X86::VPTERNLOGQZ128rrik:
2599 case X86::VPTERNLOGQZ256rrik:
2600 case X86::VPTERNLOGDZrrikz:
2601 case X86::VPTERNLOGDZrmikz:
2602 case X86::VPTERNLOGDZ128rrikz:
2603 case X86::VPTERNLOGDZ128rmikz:
2604 case X86::VPTERNLOGDZ256rrikz:
2605 case X86::VPTERNLOGDZ256rmikz:
2606 case X86::VPTERNLOGQZrrikz:
2607 case X86::VPTERNLOGQZrmikz:
2608 case X86::VPTERNLOGQZ128rrikz:
2609 case X86::VPTERNLOGQZ128rmikz:
2610 case X86::VPTERNLOGQZ256rrikz:
2611 case X86::VPTERNLOGQZ256rmikz:
2612 case X86::VPTERNLOGDZ128rmbi:
2613 case X86::VPTERNLOGDZ256rmbi:
2614 case X86::VPTERNLOGDZrmbi:
2615 case X86::VPTERNLOGQZ128rmbi:
2616 case X86::VPTERNLOGQZ256rmbi:
2617 case X86::VPTERNLOGQZrmbi:
2618 case X86::VPTERNLOGDZ128rmbikz:
2619 case X86::VPTERNLOGDZ256rmbikz:
2620 case X86::VPTERNLOGDZrmbikz:
2621 case X86::VPTERNLOGQZ128rmbikz:
2622 case X86::VPTERNLOGQZ256rmbikz:
2623 case X86::VPTERNLOGQZrmbikz: {
2624 WorkingMI = CloneIfNew(MI);
2625 commuteVPTERNLOG(*WorkingMI, OpIdx1, OpIdx2);
2626 break;
2627 }
2628 default:
2630 WorkingMI = CloneIfNew(MI);
2632 break;
2633 }
2634
2635 if (auto *FMA3Group = getFMA3Group(Opc, MI.getDesc().TSFlags)) {
2636 WorkingMI = CloneIfNew(MI);
2637 WorkingMI->setDesc(
2638 get(getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group)));
2639 break;
2640 }
2641 }
2642 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2643}
2644
2645bool X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI,
2646 unsigned &SrcOpIdx1,
2647 unsigned &SrcOpIdx2,
2648 bool IsIntrinsic) const {
2649 uint64_t TSFlags = MI.getDesc().TSFlags;
2650
2651 unsigned FirstCommutableVecOp = 1;
2652 unsigned LastCommutableVecOp = 3;
2653 unsigned KMaskOp = -1U;
2654 if (X86II::isKMasked(TSFlags)) {
2655 // For k-zero-masked operations it is Ok to commute the first vector
2656 // operand. Unless this is an intrinsic instruction.
2657 // For regular k-masked operations a conservative choice is done as the
2658 // elements of the first vector operand, for which the corresponding bit
2659 // in the k-mask operand is set to 0, are copied to the result of the
2660 // instruction.
2661 // TODO/FIXME: The commute still may be legal if it is known that the
2662 // k-mask operand is set to either all ones or all zeroes.
2663 // It is also Ok to commute the 1st operand if all users of MI use only
2664 // the elements enabled by the k-mask operand. For example,
2665 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i]
2666 // : v1[i];
2667 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 ->
2668 // // Ok, to commute v1 in FMADD213PSZrk.
2669
2670 // The k-mask operand has index = 2 for masked and zero-masked operations.
2671 KMaskOp = 2;
2672
2673 // The operand with index = 1 is used as a source for those elements for
2674 // which the corresponding bit in the k-mask is set to 0.
2675 if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic)
2676 FirstCommutableVecOp = 3;
2677
2678 LastCommutableVecOp++;
2679 } else if (IsIntrinsic) {
2680 // Commuting the first operand of an intrinsic instruction isn't possible
2681 // unless we can prove that only the lowest element of the result is used.
2682 FirstCommutableVecOp = 2;
2683 }
2684
2685 if (isMem(MI, LastCommutableVecOp))
2686 LastCommutableVecOp--;
2687
2688 // Only the first RegOpsNum operands are commutable.
2689 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means
2690 // that the operand is not specified/fixed.
2691 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2692 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
2693 SrcOpIdx1 == KMaskOp))
2694 return false;
2695 if (SrcOpIdx2 != CommuteAnyOperandIndex &&
2696 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
2697 SrcOpIdx2 == KMaskOp))
2698 return false;
2699
2700 // Look for two different register operands assumed to be commutable
2701 // regardless of the FMA opcode. The FMA opcode is adjusted later.
2702 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2703 SrcOpIdx2 == CommuteAnyOperandIndex) {
2704 unsigned CommutableOpIdx2 = SrcOpIdx2;
2705
2706 // At least one of operands to be commuted is not specified and
2707 // this method is free to choose appropriate commutable operands.
2708 if (SrcOpIdx1 == SrcOpIdx2)
2709 // Both of operands are not fixed. By default set one of commutable
2710 // operands to the last register operand of the instruction.
2711 CommutableOpIdx2 = LastCommutableVecOp;
2712 else if (SrcOpIdx2 == CommuteAnyOperandIndex)
2713 // Only one of operands is not fixed.
2714 CommutableOpIdx2 = SrcOpIdx1;
2715
2716 // CommutableOpIdx2 is well defined now. Let's choose another commutable
2717 // operand and assign its index to CommutableOpIdx1.
2718 Register Op2Reg = MI.getOperand(CommutableOpIdx2).getReg();
2719
2720 unsigned CommutableOpIdx1;
2721 for (CommutableOpIdx1 = LastCommutableVecOp;
2722 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
2723 // Just ignore and skip the k-mask operand.
2724 if (CommutableOpIdx1 == KMaskOp)
2725 continue;
2726
2727 // The commuted operands must have different registers.
2728 // Otherwise, the commute transformation does not change anything and
2729 // is useless then.
2730 if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg())
2731 break;
2732 }
2733
2734 // No appropriate commutable operands were found.
2735 if (CommutableOpIdx1 < FirstCommutableVecOp)
2736 return false;
2737
2738 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2
2739 // to return those values.
2740 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2741 CommutableOpIdx2))
2742 return false;
2743 }
2744
2745 return true;
2746}
2747
2749 unsigned &SrcOpIdx1,
2750 unsigned &SrcOpIdx2) const {
2751 const MCInstrDesc &Desc = MI.getDesc();
2752 if (!Desc.isCommutable())
2753 return false;
2754
2755 switch (MI.getOpcode()) {
2756 case X86::CMPSDrri:
2757 case X86::CMPSSrri:
2758 case X86::CMPPDrri:
2759 case X86::CMPPSrri:
2760 case X86::VCMPSDrri:
2761 case X86::VCMPSSrri:
2762 case X86::VCMPPDrri:
2763 case X86::VCMPPSrri:
2764 case X86::VCMPPDYrri:
2765 case X86::VCMPPSYrri:
2766 case X86::VCMPSDZrri:
2767 case X86::VCMPSSZrri:
2768 case X86::VCMPPDZrri:
2769 case X86::VCMPPSZrri:
2770 case X86::VCMPSHZrri:
2771 case X86::VCMPPHZrri:
2772 case X86::VCMPPHZ128rri:
2773 case X86::VCMPPHZ256rri:
2774 case X86::VCMPPDZ128rri:
2775 case X86::VCMPPSZ128rri:
2776 case X86::VCMPPDZ256rri:
2777 case X86::VCMPPSZ256rri:
2778 case X86::VCMPPDZrrik:
2779 case X86::VCMPPSZrrik:
2780 case X86::VCMPPHZrrik:
2781 case X86::VCMPPDZ128rrik:
2782 case X86::VCMPPSZ128rrik:
2783 case X86::VCMPPHZ128rrik:
2784 case X86::VCMPPDZ256rrik:
2785 case X86::VCMPPSZ256rrik:
2786 case X86::VCMPPHZ256rrik: {
2787 unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0;
2788
2789 // Float comparison can be safely commuted for
2790 // Ordered/Unordered/Equal/NotEqual tests
2791 unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7;
2792 switch (Imm) {
2793 default:
2794 // EVEX versions can be commuted.
2795 if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX)
2796 break;
2797 return false;
2798 case 0x00: // EQUAL
2799 case 0x03: // UNORDERED
2800 case 0x04: // NOT EQUAL
2801 case 0x07: // ORDERED
2802 break;
2803 }
2804
2805 // The indices of the commutable operands are 1 and 2 (or 2 and 3
2806 // when masked).
2807 // Assign them to the returned operand indices here.
2808 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
2809 2 + OpOffset);
2810 }
2811 case X86::MOVSSrr:
2812 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can
2813 // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since
2814 // AVX implies sse4.1.
2815 if (Subtarget.hasSSE41())
2816 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2817 return false;
2818 case X86::SHUFPDrri:
2819 // We can commute this to MOVSD.
2820 if (MI.getOperand(3).getImm() == 0x02)
2821 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2822 return false;
2823 case X86::MOVHLPSrr:
2824 case X86::UNPCKHPDrr:
2825 case X86::VMOVHLPSrr:
2826 case X86::VUNPCKHPDrr:
2827 case X86::VMOVHLPSZrr:
2828 case X86::VUNPCKHPDZ128rr:
2829 if (Subtarget.hasSSE2())
2830 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2831 return false;
2832 case X86::VPTERNLOGDZrri:
2833 case X86::VPTERNLOGDZrmi:
2834 case X86::VPTERNLOGDZ128rri:
2835 case X86::VPTERNLOGDZ128rmi:
2836 case X86::VPTERNLOGDZ256rri:
2837 case X86::VPTERNLOGDZ256rmi:
2838 case X86::VPTERNLOGQZrri:
2839 case X86::VPTERNLOGQZrmi:
2840 case X86::VPTERNLOGQZ128rri:
2841 case X86::VPTERNLOGQZ128rmi:
2842 case X86::VPTERNLOGQZ256rri:
2843 case X86::VPTERNLOGQZ256rmi:
2844 case X86::VPTERNLOGDZrrik:
2845 case X86::VPTERNLOGDZ128rrik:
2846 case X86::VPTERNLOGDZ256rrik:
2847 case X86::VPTERNLOGQZrrik:
2848 case X86::VPTERNLOGQZ128rrik:
2849 case X86::VPTERNLOGQZ256rrik:
2850 case X86::VPTERNLOGDZrrikz:
2851 case X86::VPTERNLOGDZrmikz:
2852 case X86::VPTERNLOGDZ128rrikz:
2853 case X86::VPTERNLOGDZ128rmikz:
2854 case X86::VPTERNLOGDZ256rrikz:
2855 case X86::VPTERNLOGDZ256rmikz:
2856 case X86::VPTERNLOGQZrrikz:
2857 case X86::VPTERNLOGQZrmikz:
2858 case X86::VPTERNLOGQZ128rrikz:
2859 case X86::VPTERNLOGQZ128rmikz:
2860 case X86::VPTERNLOGQZ256rrikz:
2861 case X86::VPTERNLOGQZ256rmikz:
2862 case X86::VPTERNLOGDZ128rmbi:
2863 case X86::VPTERNLOGDZ256rmbi:
2864 case X86::VPTERNLOGDZrmbi:
2865 case X86::VPTERNLOGQZ128rmbi:
2866 case X86::VPTERNLOGQZ256rmbi:
2867 case X86::VPTERNLOGQZrmbi:
2868 case X86::VPTERNLOGDZ128rmbikz:
2869 case X86::VPTERNLOGDZ256rmbikz:
2870 case X86::VPTERNLOGDZrmbikz:
2871 case X86::VPTERNLOGQZ128rmbikz:
2872 case X86::VPTERNLOGQZ256rmbikz:
2873 case X86::VPTERNLOGQZrmbikz:
2874 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2875 case X86::VPDPWSSDYrr:
2876 case X86::VPDPWSSDrr:
2877 case X86::VPDPWSSDSYrr:
2878 case X86::VPDPWSSDSrr:
2879 case X86::VPDPWUUDrr:
2880 case X86::VPDPWUUDYrr:
2881 case X86::VPDPWUUDSrr:
2882 case X86::VPDPWUUDSYrr:
2883 case X86::VPDPBSSDSrr:
2884 case X86::VPDPBSSDSYrr:
2885 case X86::VPDPBSSDrr:
2886 case X86::VPDPBSSDYrr:
2887 case X86::VPDPBUUDSrr:
2888 case X86::VPDPBUUDSYrr:
2889 case X86::VPDPBUUDrr:
2890 case X86::VPDPBUUDYrr:
2891 case X86::VPDPBSSDSZ128rr:
2892 case X86::VPDPBSSDSZ128rrk:
2893 case X86::VPDPBSSDSZ128rrkz:
2894 case X86::VPDPBSSDSZ256rr:
2895 case X86::VPDPBSSDSZ256rrk:
2896 case X86::VPDPBSSDSZ256rrkz:
2897 case X86::VPDPBSSDSZrr:
2898 case X86::VPDPBSSDSZrrk:
2899 case X86::VPDPBSSDSZrrkz:
2900 case X86::VPDPBSSDZ128rr:
2901 case X86::VPDPBSSDZ128rrk:
2902 case X86::VPDPBSSDZ128rrkz:
2903 case X86::VPDPBSSDZ256rr:
2904 case X86::VPDPBSSDZ256rrk:
2905 case X86::VPDPBSSDZ256rrkz:
2906 case X86::VPDPBSSDZrr:
2907 case X86::VPDPBSSDZrrk:
2908 case X86::VPDPBSSDZrrkz:
2909 case X86::VPDPBUUDSZ128rr:
2910 case X86::VPDPBUUDSZ128rrk:
2911 case X86::VPDPBUUDSZ128rrkz:
2912 case X86::VPDPBUUDSZ256rr:
2913 case X86::VPDPBUUDSZ256rrk:
2914 case X86::VPDPBUUDSZ256rrkz:
2915 case X86::VPDPBUUDSZrr:
2916 case X86::VPDPBUUDSZrrk:
2917 case X86::VPDPBUUDSZrrkz:
2918 case X86::VPDPBUUDZ128rr:
2919 case X86::VPDPBUUDZ128rrk:
2920 case X86::VPDPBUUDZ128rrkz:
2921 case X86::VPDPBUUDZ256rr:
2922 case X86::VPDPBUUDZ256rrk:
2923 case X86::VPDPBUUDZ256rrkz:
2924 case X86::VPDPBUUDZrr:
2925 case X86::VPDPBUUDZrrk:
2926 case X86::VPDPBUUDZrrkz:
2927 case X86::VPDPWSSDZ128rr:
2928 case X86::VPDPWSSDZ128rrk:
2929 case X86::VPDPWSSDZ128rrkz:
2930 case X86::VPDPWSSDZ256rr:
2931 case X86::VPDPWSSDZ256rrk:
2932 case X86::VPDPWSSDZ256rrkz:
2933 case X86::VPDPWSSDZrr:
2934 case X86::VPDPWSSDZrrk:
2935 case X86::VPDPWSSDZrrkz:
2936 case X86::VPDPWSSDSZ128rr:
2937 case X86::VPDPWSSDSZ128rrk:
2938 case X86::VPDPWSSDSZ128rrkz:
2939 case X86::VPDPWSSDSZ256rr:
2940 case X86::VPDPWSSDSZ256rrk:
2941 case X86::VPDPWSSDSZ256rrkz:
2942 case X86::VPDPWSSDSZrr:
2943 case X86::VPDPWSSDSZrrk:
2944 case X86::VPDPWSSDSZrrkz:
2945 case X86::VPDPWUUDZ128rr:
2946 case X86::VPDPWUUDZ128rrk:
2947 case X86::VPDPWUUDZ128rrkz:
2948 case X86::VPDPWUUDZ256rr:
2949 case X86::VPDPWUUDZ256rrk:
2950 case X86::VPDPWUUDZ256rrkz:
2951 case X86::VPDPWUUDZrr:
2952 case X86::VPDPWUUDZrrk:
2953 case X86::VPDPWUUDZrrkz:
2954 case X86::VPDPWUUDSZ128rr:
2955 case X86::VPDPWUUDSZ128rrk:
2956 case X86::VPDPWUUDSZ128rrkz:
2957 case X86::VPDPWUUDSZ256rr:
2958 case X86::VPDPWUUDSZ256rrk:
2959 case X86::VPDPWUUDSZ256rrkz:
2960 case X86::VPDPWUUDSZrr:
2961 case X86::VPDPWUUDSZrrk:
2962 case X86::VPDPWUUDSZrrkz:
2963 case X86::VPMADD52HUQrr:
2964 case X86::VPMADD52HUQYrr:
2965 case X86::VPMADD52HUQZ128r:
2966 case X86::VPMADD52HUQZ128rk:
2967 case X86::VPMADD52HUQZ128rkz:
2968 case X86::VPMADD52HUQZ256r:
2969 case X86::VPMADD52HUQZ256rk:
2970 case X86::VPMADD52HUQZ256rkz:
2971 case X86::VPMADD52HUQZr:
2972 case X86::VPMADD52HUQZrk:
2973 case X86::VPMADD52HUQZrkz:
2974 case X86::VPMADD52LUQrr:
2975 case X86::VPMADD52LUQYrr:
2976 case X86::VPMADD52LUQZ128r:
2977 case X86::VPMADD52LUQZ128rk:
2978 case X86::VPMADD52LUQZ128rkz:
2979 case X86::VPMADD52LUQZ256r:
2980 case X86::VPMADD52LUQZ256rk:
2981 case X86::VPMADD52LUQZ256rkz:
2982 case X86::VPMADD52LUQZr:
2983 case X86::VPMADD52LUQZrk:
2984 case X86::VPMADD52LUQZrkz:
2985 case X86::VFMADDCPHZr:
2986 case X86::VFMADDCPHZrk:
2987 case X86::VFMADDCPHZrkz:
2988 case X86::VFMADDCPHZ128r:
2989 case X86::VFMADDCPHZ128rk:
2990 case X86::VFMADDCPHZ128rkz:
2991 case X86::VFMADDCPHZ256r:
2992 case X86::VFMADDCPHZ256rk:
2993 case X86::VFMADDCPHZ256rkz:
2994 case X86::VFMADDCSHZr:
2995 case X86::VFMADDCSHZrk:
2996 case X86::VFMADDCSHZrkz: {
2997 unsigned CommutableOpIdx1 = 2;
2998 unsigned CommutableOpIdx2 = 3;
2999 if (X86II::isKMasked(Desc.TSFlags)) {
3000 // Skip the mask register.
3001 ++CommutableOpIdx1;
3002 ++CommutableOpIdx2;
3003 }
3004 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3005 CommutableOpIdx2))
3006 return false;
3007 if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg())
3008 // No idea.
3009 return false;
3010 return true;
3011 }
3012
3013 default:
3014 const X86InstrFMA3Group *FMA3Group =
3015 getFMA3Group(MI.getOpcode(), MI.getDesc().TSFlags);
3016 if (FMA3Group)
3017 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2,
3018 FMA3Group->isIntrinsic());
3019
3020 // Handled masked instructions since we need to skip over the mask input
3021 // and the preserved input.
3022 if (X86II::isKMasked(Desc.TSFlags)) {
3023 // First assume that the first input is the mask operand and skip past it.
3024 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1;
3025 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2;
3026 // Check if the first input is tied. If there isn't one then we only
3027 // need to skip the mask operand which we did above.
3028 if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(),
3029 MCOI::TIED_TO) != -1)) {
3030 // If this is zero masking instruction with a tied operand, we need to
3031 // move the first index back to the first input since this must
3032 // be a 3 input instruction and we want the first two non-mask inputs.
3033 // Otherwise this is a 2 input instruction with a preserved input and
3034 // mask, so we need to move the indices to skip one more input.
3035 if (X86II::isKMergeMasked(Desc.TSFlags)) {
3036 ++CommutableOpIdx1;
3037 ++CommutableOpIdx2;
3038 } else {
3039 --CommutableOpIdx1;
3040 }
3041 }
3042
3043 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3044 CommutableOpIdx2))
3045 return false;
3046
3047 if (!MI.getOperand(SrcOpIdx1).isReg() ||
3048 !MI.getOperand(SrcOpIdx2).isReg())
3049 // No idea.
3050 return false;
3051 return true;
3052 }
3053
3054 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
3055 }
3056 return false;
3057}
3058
3060 unsigned Opcode = MI->getOpcode();
3061 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
3062 Opcode != X86::LEA64_32r)
3063 return false;
3064
3065 const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt);
3066 const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp);
3067 const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg);
3068
3069 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 ||
3070 Scale.getImm() > 1)
3071 return false;
3072
3073 return true;
3074}
3075
3077 // Currently we're interested in following sequence only.
3078 // r3 = lea r1, r2
3079 // r5 = add r3, r4
3080 // Both r3 and r4 are killed in add, we hope the add instruction has the
3081 // operand order
3082 // r5 = add r4, r3
3083 // So later in X86FixupLEAs the lea instruction can be rewritten as add.
3084 unsigned Opcode = MI.getOpcode();
3085 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
3086 return false;
3087
3088 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3089 Register Reg1 = MI.getOperand(1).getReg();
3090 Register Reg2 = MI.getOperand(2).getReg();
3091
3092 // Check if Reg1 comes from LEA in the same MBB.
3093 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg1)) {
3094 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
3095 Commute = true;
3096 return true;
3097 }
3098 }
3099
3100 // Check if Reg2 comes from LEA in the same MBB.
3101 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg2)) {
3102 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
3103 Commute = false;
3104 return true;
3105 }
3106 }
3107
3108 return false;
3109}
3110
3112 unsigned Opcode = MCID.getOpcode();
3113 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isSETZUCC(Opcode) ||
3114 X86::isCMOVCC(Opcode) || X86::isCFCMOVCC(Opcode) ||
3115 X86::isCCMPCC(Opcode) || X86::isCTESTCC(Opcode)))
3116 return -1;
3117 // Assume that condition code is always the last use operand.
3118 unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs();
3119 return NumUses - 1;
3120}
3121
3123 const MCInstrDesc &MCID = MI.getDesc();
3124 int CondNo = getCondSrcNoFromDesc(MCID);
3125 if (CondNo < 0)
3126 return X86::COND_INVALID;
3127 CondNo += MCID.getNumDefs();
3128 return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm());
3129}
3130
3132 return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3134}
3135
3137 return X86::isSETCC(MI.getOpcode()) || X86::isSETZUCC(MI.getOpcode())
3140}
3141
3143 return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3145}
3146
3148 return X86::isCFCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3150}
3151
3153 return X86::isCCMPCC(MI.getOpcode()) || X86::isCTESTCC(MI.getOpcode())
3156}
3157
3159 // CCMP/CTEST has two conditional operands:
3160 // - SCC: source conditonal code (same as CMOV)
3161 // - DCF: destination conditional flags, which has 4 valid bits
3162 //
3163 // +----+----+----+----+
3164 // | OF | SF | ZF | CF |
3165 // +----+----+----+----+
3166 //
3167 // If SCC(source conditional code) evaluates to false, CCMP/CTEST will updates
3168 // the conditional flags by as follows:
3169 //
3170 // OF = DCF.OF
3171 // SF = DCF.SF
3172 // ZF = DCF.ZF
3173 // CF = DCF.CF
3174 // PF = DCF.CF
3175 // AF = 0 (Auxiliary Carry Flag)
3176 //
3177 // Otherwise, the CMP or TEST is executed and it updates the
3178 // CSPAZO flags normally.
3179 //
3180 // NOTE:
3181 // If SCC = P, then SCC evaluates to true regardless of the CSPAZO value.
3182 // If SCC = NP, then SCC evaluates to false regardless of the CSPAZO value.
3183
3184 enum { CF = 1, ZF = 2, SF = 4, OF = 8, PF = CF };
3185
3186 switch (CC) {
3187 default:
3188 llvm_unreachable("Illegal condition code!");
3189 case X86::COND_NO:
3190 case X86::COND_NE:
3191 case X86::COND_GE:
3192 case X86::COND_G:
3193 case X86::COND_AE:
3194 case X86::COND_A:
3195 case X86::COND_NS:
3196 case X86::COND_NP:
3197 return 0;
3198 case X86::COND_O:
3199 return OF;
3200 case X86::COND_B:
3201 case X86::COND_BE:
3202 return CF;
3203 break;
3204 case X86::COND_E:
3205 case X86::COND_LE:
3206 return ZF;
3207 case X86::COND_S:
3208 case X86::COND_L:
3209 return SF;
3210 case X86::COND_P:
3211 return PF;
3212 }
3213}
3214
3215#define GET_X86_NF_TRANSFORM_TABLE
3216#define GET_X86_ND2NONND_TABLE
3217#include "X86GenInstrMapping.inc"
3218
3220 unsigned Opc) {
3221 const auto I = llvm::lower_bound(Table, Opc);
3222 return (I == Table.end() || I->OldOpc != Opc) ? 0U : I->NewOpc;
3223}
3224unsigned X86::getNFVariant(unsigned Opc) {
3225#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
3226 // Make sure the tables are sorted.
3227 static std::atomic<bool> NFTableChecked(false);
3228 if (!NFTableChecked.load(std::memory_order_relaxed)) {
3229 assert(llvm::is_sorted(X86NFTransformTable) &&
3230 "X86NFTransformTable is not sorted!");
3231 NFTableChecked.store(true, std::memory_order_relaxed);
3232 }
3233#endif
3234 return getNewOpcFromTable(X86NFTransformTable, Opc);
3235}
3236
3238 const TargetRegisterInfo *TRI) {
3239 if (!MI.registerDefIsDead(X86::EFLAGS, TRI))
3240 return 0;
3241 // For the instructions are ADDrm/ADDmr with relocation, we'll skip the
3242 // optimization for replacing non-NF with NF. This is to keep backward
3243 // compatiblity with old version of linkers without APX relocation type
3244 // support on Linux OS.
3245 const X86Subtarget &ST = MI.getMF()->getSubtarget<X86Subtarget>();
3246 if (!ST.getCLOpts().enable_apx_for_relocation &&
3248 return 0;
3249 return X86::getNFVariant(MI.getOpcode());
3250}
3251
3252unsigned X86::getNonNDVariant(unsigned Opc) {
3253#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
3254 // Make sure the tables are sorted.
3255 static std::atomic<bool> NDTableChecked(false);
3256 if (!NDTableChecked.load(std::memory_order_relaxed)) {
3257 assert(llvm::is_sorted(X86ND2NonNDTable) &&
3258 "X86ND2NonNDTableis not sorted!");
3259 NDTableChecked.store(true, std::memory_order_relaxed);
3260 }
3261#endif
3262 return getNewOpcFromTable(X86ND2NonNDTable, Opc);
3263}
3264
3265/// Return the inverse of the specified condition,
3266/// e.g. turning COND_E to COND_NE.
3268 switch (CC) {
3269 default:
3270 llvm_unreachable("Illegal condition code!");
3271 case X86::COND_E:
3272 return X86::COND_NE;
3273 case X86::COND_NE:
3274 return X86::COND_E;
3275 case X86::COND_L:
3276 return X86::COND_GE;
3277 case X86::COND_LE:
3278 return X86::COND_G;
3279 case X86::COND_G:
3280 return X86::COND_LE;
3281 case X86::COND_GE:
3282 return X86::COND_L;
3283 case X86::COND_B:
3284 return X86::COND_AE;
3285 case X86::COND_BE:
3286 return X86::COND_A;
3287 case X86::COND_A:
3288 return X86::COND_BE;
3289 case X86::COND_AE:
3290 return X86::COND_B;
3291 case X86::COND_S:
3292 return X86::COND_NS;
3293 case X86::COND_NS:
3294 return X86::COND_S;
3295 case X86::COND_P:
3296 return X86::COND_NP;
3297 case X86::COND_NP:
3298 return X86::COND_P;
3299 case X86::COND_O:
3300 return X86::COND_NO;
3301 case X86::COND_NO:
3302 return X86::COND_O;
3303 case X86::COND_NE_OR_P:
3304 return X86::COND_E_AND_NP;
3305 case X86::COND_E_AND_NP:
3306 return X86::COND_NE_OR_P;
3307 }
3308}
3309
3310/// Assuming the flags are set by MI(a,b), return the condition code if we
3311/// modify the instructions such that flags are set by MI(b,a).
3313 switch (CC) {
3314 default:
3315 return X86::COND_INVALID;
3316 case X86::COND_E:
3317 return X86::COND_E;
3318 case X86::COND_NE:
3319 return X86::COND_NE;
3320 case X86::COND_L:
3321 return X86::COND_G;
3322 case X86::COND_LE:
3323 return X86::COND_GE;
3324 case X86::COND_G:
3325 return X86::COND_L;
3326 case X86::COND_GE:
3327 return X86::COND_LE;
3328 case X86::COND_B:
3329 return X86::COND_A;
3330 case X86::COND_BE:
3331 return X86::COND_AE;
3332 case X86::COND_A:
3333 return X86::COND_B;
3334 case X86::COND_AE:
3335 return X86::COND_BE;
3336 }
3337}
3338
3339std::pair<X86::CondCode, bool>
3342 bool NeedSwap = false;
3343 switch (Predicate) {
3344 default:
3345 break;
3346 // Floating-point Predicates
3347 case CmpInst::FCMP_UEQ:
3348 CC = X86::COND_E;
3349 break;
3350 case CmpInst::FCMP_OLT:
3351 NeedSwap = true;
3352 [[fallthrough]];
3353 case CmpInst::FCMP_OGT:
3354 CC = X86::COND_A;
3355 break;
3356 case CmpInst::FCMP_OLE:
3357 NeedSwap = true;
3358 [[fallthrough]];
3359 case CmpInst::FCMP_OGE:
3360 CC = X86::COND_AE;
3361 break;
3362 case CmpInst::FCMP_UGT:
3363 NeedSwap = true;
3364 [[fallthrough]];
3365 case CmpInst::FCMP_ULT:
3366 CC = X86::COND_B;
3367 break;
3368 case CmpInst::FCMP_UGE:
3369 NeedSwap = true;
3370 [[fallthrough]];
3371 case CmpInst::FCMP_ULE:
3372 CC = X86::COND_BE;
3373 break;
3374 case CmpInst::FCMP_ONE:
3375 CC = X86::COND_NE;
3376 break;
3377 case CmpInst::FCMP_UNO:
3378 CC = X86::COND_P;
3379 break;
3380 case CmpInst::FCMP_ORD:
3381 CC = X86::COND_NP;
3382 break;
3383 case CmpInst::FCMP_OEQ:
3384 [[fallthrough]];
3385 case CmpInst::FCMP_UNE:
3387 break;
3388
3389 // Integer Predicates
3390 case CmpInst::ICMP_EQ:
3391 CC = X86::COND_E;
3392 break;
3393 case CmpInst::ICMP_NE:
3394 CC = X86::COND_NE;
3395 break;
3396 case CmpInst::ICMP_UGT:
3397 CC = X86::COND_A;
3398 break;
3399 case CmpInst::ICMP_UGE:
3400 CC = X86::COND_AE;
3401 break;
3402 case CmpInst::ICMP_ULT:
3403 CC = X86::COND_B;
3404 break;
3405 case CmpInst::ICMP_ULE:
3406 CC = X86::COND_BE;
3407 break;
3408 case CmpInst::ICMP_SGT:
3409 CC = X86::COND_G;
3410 break;
3411 case CmpInst::ICMP_SGE:
3412 CC = X86::COND_GE;
3413 break;
3414 case CmpInst::ICMP_SLT:
3415 CC = X86::COND_L;
3416 break;
3417 case CmpInst::ICMP_SLE:
3418 CC = X86::COND_LE;
3419 break;
3420 }
3421
3422 return std::make_pair(CC, NeedSwap);
3423}
3424
3425/// Return a cmov opcode for the given register size in bytes, and operand type.
3426unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand,
3427 bool HasNDD) {
3428 switch (RegBytes) {
3429 default:
3430 llvm_unreachable("Illegal register size!");
3431#define GET_ND_IF_ENABLED(OPC) (HasNDD ? OPC##_ND : OPC)
3432 case 2:
3433 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV16rm)
3434 : GET_ND_IF_ENABLED(X86::CMOV16rr);
3435 case 4:
3436 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV32rm)
3437 : GET_ND_IF_ENABLED(X86::CMOV32rr);
3438 case 8:
3439 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV64rm)
3440 : GET_ND_IF_ENABLED(X86::CMOV64rr);
3441 }
3442}
3443
3444unsigned X86::getMOVriOpcode(bool Use64BitReg, int64_t Imm) {
3445 if (!Use64BitReg)
3446 return X86::MOV32ri;
3447
3448 if (isUInt<32>(Imm))
3449 return X86::MOV32ri64;
3450 if (isInt<32>(Imm))
3451 return X86::MOV64ri32;
3452 return X86::MOV64ri;
3453}
3454
3455/// Get the VPCMP immediate for the given condition.
3457 switch (CC) {
3458 default:
3459 llvm_unreachable("Unexpected SETCC condition");
3460 case ISD::SETNE:
3461 return 4;
3462 case ISD::SETEQ:
3463 return 0;
3464 case ISD::SETULT:
3465 case ISD::SETLT:
3466 return 1;
3467 case ISD::SETUGT:
3468 case ISD::SETGT:
3469 return 6;
3470 case ISD::SETUGE:
3471 case ISD::SETGE:
3472 return 5;
3473 case ISD::SETULE:
3474 case ISD::SETLE:
3475 return 2;
3476 }
3477}
3478
3479/// Get the VPCMP immediate if the operands are swapped.
3480unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
3481 switch (Imm) {
3482 default:
3483 llvm_unreachable("Unreachable!");
3484 case 0x01:
3485 Imm = 0x06;
3486 break; // LT -> NLE
3487 case 0x02:
3488 Imm = 0x05;
3489 break; // LE -> NLT
3490 case 0x05:
3491 Imm = 0x02;
3492 break; // NLT -> LE
3493 case 0x06:
3494 Imm = 0x01;
3495 break; // NLE -> LT
3496 case 0x00: // EQ
3497 case 0x03: // FALSE
3498 case 0x04: // NE
3499 case 0x07: // TRUE
3500 break;
3501 }
3502
3503 return Imm;
3504}
3505
3506/// Get the VPCOM immediate if the operands are swapped.
3507unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
3508 switch (Imm) {
3509 default:
3510 llvm_unreachable("Unreachable!");
3511 case 0x00:
3512 Imm = 0x02;
3513 break; // LT -> GT
3514 case 0x01:
3515 Imm = 0x03;
3516 break; // LE -> GE
3517 case 0x02:
3518 Imm = 0x00;
3519 break; // GT -> LT
3520 case 0x03:
3521 Imm = 0x01;
3522 break; // GE -> LE
3523 case 0x04: // EQ
3524 case 0x05: // NE
3525 case 0x06: // FALSE
3526 case 0x07: // TRUE
3527 break;
3528 }
3529
3530 return Imm;
3531}
3532
3533/// Get the VCMP immediate if the operands are swapped.
3534unsigned X86::getSwappedVCMPImm(unsigned Imm) {
3535 // Only need the lower 2 bits to distinquish.
3536 switch (Imm & 0x3) {
3537 default:
3538 llvm_unreachable("Unreachable!");
3539 case 0x00:
3540 case 0x03:
3541 // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted.
3542 break;
3543 case 0x01:
3544 case 0x02:
3545 // Need to toggle bits 3:0. Bit 4 stays the same.
3546 Imm ^= 0xf;
3547 break;
3548 }
3549
3550 return Imm;
3551}
3552
3554 if (Info.RegClass == X86::VR128RegClassID ||
3555 Info.RegClass == X86::VR128XRegClassID)
3556 return 128;
3557 if (Info.RegClass == X86::VR256RegClassID ||
3558 Info.RegClass == X86::VR256XRegClassID)
3559 return 256;
3560 if (Info.RegClass == X86::VR512RegClassID)
3561 return 512;
3562 llvm_unreachable("Unknown register class!");
3563}
3564
3565/// Return true if the Reg is X87 register.
3566static bool isX87Reg(Register Reg) {
3567 return (Reg == X86::FPCW || Reg == X86::FPSW ||
3568 (Reg >= X86::ST0 && Reg <= X86::ST7));
3569}
3570
3571/// check if the instruction is X87 instruction
3573 // Call and inlineasm defs X87 register, so we special case it here because
3574 // otherwise calls are incorrectly flagged as x87 instructions
3575 // as a result.
3576 if (MI.isCall() || MI.isInlineAsm())
3577 return false;
3578 for (const MachineOperand &MO : MI.operands()) {
3579 if (!MO.isReg())
3580 continue;
3581 if (isX87Reg(MO.getReg()))
3582 return true;
3583 }
3584 return false;
3585}
3586
3588 auto IsMemOp = [](const MCOperandInfo &OpInfo) {
3589 return OpInfo.OperandType == MCOI::OPERAND_MEMORY;
3590 };
3591
3592 const MCInstrDesc &Desc = MI.getDesc();
3593
3594 // Directly invoke the MC-layer routine for real (i.e., non-pseudo)
3595 // instructions (fast case).
3596 if (!X86II::isPseudo(Desc.TSFlags)) {
3597 int MemRefIdx = X86II::getMemoryOperandIdx(Desc);
3598 if (MemRefIdx >= 0)
3599 return MemRefIdx;
3600#ifdef EXPENSIVE_CHECKS
3601 assert(none_of(Desc.operands(), IsMemOp) &&
3602 "Got false negative from X86II::getMemoryOperandIdx()!");
3603#endif
3604 return -1;
3605 }
3606
3607 // Otherwise, handle pseudo instructions by examining the type of their
3608 // operands (slow case). An instruction cannot have a memory reference if it
3609 // has fewer than AddrNumOperands (= 5) explicit operands.
3610 unsigned NumOps = Desc.getNumOperands();
3612#ifdef EXPENSIVE_CHECKS
3613 assert(none_of(Desc.operands(), IsMemOp) &&
3614 "Expected no operands to have OPERAND_MEMORY type!");
3615#endif
3616 return -1;
3617 }
3618
3619 // The first operand with type OPERAND_MEMORY indicates the start of a memory
3620 // reference. We expect the following AddrNumOperand-1 operands to also have
3621 // OPERAND_MEMORY type.
3622 for (unsigned I = 0, E = NumOps - X86::AddrNumOperands; I != E; ++I) {
3623 if (IsMemOp(Desc.operands()[I])) {
3624#ifdef EXPENSIVE_CHECKS
3625 assert(std::all_of(Desc.operands().begin() + I,
3626 Desc.operands().begin() + I + X86::AddrNumOperands,
3627 IsMemOp) &&
3628 "Expected all five operands in the memory reference to have "
3629 "OPERAND_MEMORY type!");
3630#endif
3631 return I;
3632 }
3633 }
3634
3635 return -1;
3636}
3637
3639 unsigned OpNo) {
3640 assert(MI.getNumOperands() >= (OpNo + X86::AddrNumOperands) &&
3641 "Unexpected number of operands!");
3642
3643 const MachineOperand &Index = MI.getOperand(OpNo + X86::AddrIndexReg);
3644 if (!Index.isReg() || Index.getReg() != X86::NoRegister)
3645 return nullptr;
3646
3647 const MachineOperand &Disp = MI.getOperand(OpNo + X86::AddrDisp);
3648 if (!Disp.isCPI() || Disp.getOffset() != 0)
3649 return nullptr;
3650
3652 MI.getParent()->getParent()->getConstantPool()->getConstants();
3653 const MachineConstantPoolEntry &ConstantEntry = Constants[Disp.getIndex()];
3654
3655 // Bail if this is a machine constant pool entry, we won't be able to dig out
3656 // anything useful.
3657 if (ConstantEntry.isMachineConstantPoolEntry())
3658 return nullptr;
3659
3660 return ConstantEntry.Val.ConstVal;
3661}
3662
3664 switch (MI.getOpcode()) {
3665 case X86::TCRETURNdi:
3666 case X86::TCRETURNri:
3667 case X86::TCRETURNmi:
3668 case X86::TCRETURNdi64:
3669 case X86::TCRETURNri64:
3670 case X86::TCRETURNri64_ImpCall:
3671 case X86::TCRETURNmi64:
3672 return true;
3673 default:
3674 return false;
3675 }
3676}
3677
3680 const MachineInstr &TailCall) const {
3681
3682 const MachineFunction *MF = TailCall.getMF();
3683
3684 if (MF->getTarget().getCodeModel() == CodeModel::Kernel) {
3685 // Kernel patches thunk calls in runtime, these should never be conditional.
3686 const MachineOperand &Target = TailCall.getOperand(0);
3687 if (Target.isSymbol()) {
3688 StringRef Symbol(Target.getSymbolName());
3689 // this is currently only relevant to r11/kernel indirect thunk.
3690 if (Symbol == "__x86_indirect_thunk_r11")
3691 return false;
3692 }
3693 }
3694
3695 if (TailCall.getOpcode() != X86::TCRETURNdi &&
3696 TailCall.getOpcode() != X86::TCRETURNdi64) {
3697 // Only direct calls can be done with a conditional branch.
3698 return false;
3699 }
3700
3701 if (TailCall.getCFIType()) {
3702 // An indirect tail call that was lowered to a direct jump to an indirect
3703 // thunk (e.g., __x86_indirect_thunk_r11 for retpolines) still carries a
3704 // KCFI type, and must be preceded by a KCFI check of the call target. The
3705 // check is emitted later, and cannot be made conditional, so folding the
3706 // tail call into a conditional branch would silently drop it.
3707 return false;
3708 }
3709
3710 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) {
3711 // Conditional tail calls confuse the Win64 unwinder.
3712 return false;
3713 }
3714
3715 assert(BranchCond.size() == 1);
3716 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) {
3717 // Can't make a conditional tail call with this condition.
3718 return false;
3719 }
3720
3722 if (X86FI->getTCReturnAddrDelta() != 0 ||
3723 TailCall.getOperand(1).getImm() != 0) {
3724 // A conditional tail call cannot do any stack adjustment.
3725 return false;
3726 }
3727
3728 return true;
3729}
3730
3733 const MachineInstr &TailCall) const {
3734 assert(canMakeTailCallConditional(BranchCond, TailCall));
3735
3737 while (I != MBB.begin()) {
3738 --I;
3739 if (I->isDebugInstr())
3740 continue;
3741 if (!I->isBranch())
3742 assert(0 && "Can't find the branch to replace!");
3743
3745 assert(BranchCond.size() == 1);
3746 if (CC != BranchCond[0].getImm())
3747 continue;
3748
3749 break;
3750 }
3751
3752 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
3753 : X86::TCRETURNdi64cc;
3754
3755 auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc));
3756 MIB->addOperand(TailCall.getOperand(0)); // Destination.
3757 MIB.addImm(0); // Stack offset (not used).
3758 MIB->addOperand(BranchCond[0]); // Condition.
3759 MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters.
3760
3761 // Add implicit uses and defs of all live regs potentially clobbered by the
3762 // call. This way they still appear live across the call.
3764 LiveRegs.addLiveOuts(MBB);
3766 LiveRegs.stepForward(*MIB, Clobbers);
3767 for (const auto &C : Clobbers) {
3768 MIB.addReg(C.first, RegState::Implicit);
3770 }
3771
3772 I->eraseFromParent();
3773}
3774
3775// Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may
3776// not be a fallthrough MBB now due to layout changes). Return nullptr if the
3777// fallthrough MBB cannot be identified.
3780 // Look for non-EHPad successors other than TBB. If we find exactly one, it
3781 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB
3782 // and fallthrough MBB. If we find more than one, we cannot identify the
3783 // fallthrough MBB and should return nullptr.
3784 MachineBasicBlock *FallthroughBB = nullptr;
3785 for (MachineBasicBlock *Succ : MBB->successors()) {
3786 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB))
3787 continue;
3788 // Return a nullptr if we found more than one fallthrough successor.
3789 if (FallthroughBB && FallthroughBB != TBB)
3790 return nullptr;
3791 FallthroughBB = Succ;
3792 }
3793 return FallthroughBB;
3794}
3795
3796bool X86InstrInfo::analyzeBranchImpl(
3799 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const {
3800
3801 // Start from the bottom of the block and work up, examining the
3802 // terminator instructions.
3804 MachineBasicBlock::iterator UnCondBrIter = MBB.end();
3805 while (I != MBB.begin()) {
3806 --I;
3807 if (I->isDebugInstr())
3808 continue;
3809
3810 // Working from the bottom, when we see a non-terminator instruction, we're
3811 // done.
3812 if (!isUnpredicatedTerminator(*I))
3813 break;
3814
3815 // A terminator that isn't a branch can't easily be handled by this
3816 // analysis.
3817 if (!I->isBranch())
3818 return true;
3819
3820 // Handle unconditional branches.
3821 if (I->getOpcode() == X86::JMP_1) {
3822 UnCondBrIter = I;
3823
3824 if (!AllowModify) {
3825 TBB = I->getOperand(0).getMBB();
3826 continue;
3827 }
3828
3829 // If the block has any instructions after a JMP, delete them.
3830 MBB.erase(std::next(I), MBB.end());
3831
3832 Cond.clear();
3833 FBB = nullptr;
3834
3835 // Delete the JMP if it's equivalent to a fall-through.
3836 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
3837 TBB = nullptr;
3838 I->eraseFromParent();
3839 I = MBB.end();
3840 UnCondBrIter = MBB.end();
3841 continue;
3842 }
3843
3844 // TBB is used to indicate the unconditional destination.
3845 TBB = I->getOperand(0).getMBB();
3846 continue;
3847 }
3848
3849 // Handle conditional branches.
3850 X86::CondCode BranchCode = X86::getCondFromBranch(*I);
3851 if (BranchCode == X86::COND_INVALID)
3852 return true; // Can't handle indirect branch.
3853
3854 // In practice we should never have an undef eflags operand, if we do
3855 // abort here as we are not prepared to preserve the flag.
3856 if (I->findRegisterUseOperand(X86::EFLAGS, /*TRI=*/nullptr)->isUndef())
3857 return true;
3858
3859 // Working from the bottom, handle the first conditional branch.
3860 if (Cond.empty()) {
3861 FBB = TBB;
3862 TBB = I->getOperand(0).getMBB();
3864 CondBranches.push_back(&*I);
3865 continue;
3866 }
3867
3868 // Handle subsequent conditional branches. Only handle the case where all
3869 // conditional branches branch to the same destination and their condition
3870 // opcodes fit one of the special multi-branch idioms.
3871 assert(Cond.size() == 1);
3872 assert(TBB);
3873
3874 // If the conditions are the same, we can leave them alone.
3875 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm();
3876 auto NewTBB = I->getOperand(0).getMBB();
3877 if (OldBranchCode == BranchCode && TBB == NewTBB)
3878 continue;
3879
3880 // If they differ, see if they fit one of the known patterns. Theoretically,
3881 // we could handle more patterns here, but we shouldn't expect to see them
3882 // if instruction selection has done a reasonable job.
3883 if (TBB == NewTBB &&
3884 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) ||
3885 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) {
3886 BranchCode = X86::COND_NE_OR_P;
3887 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) ||
3888 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) {
3889 if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB)))
3890 return true;
3891
3892 // X86::COND_E_AND_NP usually has two different branch destinations.
3893 //
3894 // JP B1
3895 // JE B2
3896 // JMP B1
3897 // B1:
3898 // B2:
3899 //
3900 // Here this condition branches to B2 only if NP && E. It has another
3901 // equivalent form:
3902 //
3903 // JNE B1
3904 // JNP B2
3905 // JMP B1
3906 // B1:
3907 // B2:
3908 //
3909 // Similarly it branches to B2 only if E && NP. That is why this condition
3910 // is named with COND_E_AND_NP.
3911 BranchCode = X86::COND_E_AND_NP;
3912 } else
3913 return true;
3914
3915 // Update the MachineOperand.
3916 Cond[0].setImm(BranchCode);
3917 CondBranches.push_back(&*I);
3918 }
3919
3920 return false;
3921}
3922
3925 MachineBasicBlock *&FBB,
3927 bool AllowModify) const {
3928 SmallVector<MachineInstr *, 4> CondBranches;
3929 return analyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify);
3930}
3931
3933 int MemRefBegin = X86II::getMemoryOperandIdx(MI.getDesc());
3934 assert(MemRefBegin >= 0 && "Expected a memory operand");
3935
3936 const MachineOperand &MO = MI.getOperand(MemRefBegin + X86::AddrDisp);
3937 if (!MO.isJTI())
3938 return -1;
3939
3940 return MO.getIndex();
3941}
3942
3944 Register Reg) {
3945 if (!Reg.isVirtual())
3946 return -1;
3947 MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
3948 if (MI == nullptr)
3949 return -1;
3950 unsigned Opcode = MI->getOpcode();
3951 if (Opcode != X86::LEA64r && Opcode != X86::LEA32r)
3952 return -1;
3954}
3955
3957 unsigned Opcode = MI.getOpcode();
3958 // Switch-jump pattern for non-PIC code looks like:
3959 // JMP64m $noreg, 8, %X, %jump-table.X, $noreg
3960 if (Opcode == X86::JMP64m || Opcode == X86::JMP32m) {
3962 }
3963 // The pattern for PIC code looks like:
3964 // %0 = LEA64r $rip, 1, $noreg, %jump-table.X
3965 // %1 = MOVSX64rm32 %0, 4, XX, 0, $noreg
3966 // %2 = ADD64rr %1, %0
3967 // JMP64r %2
3968 if (Opcode == X86::JMP64r || Opcode == X86::JMP32r) {
3969 Register Reg = MI.getOperand(0).getReg();
3970 if (!Reg.isVirtual())
3971 return -1;
3972 const MachineFunction &MF = *MI.getParent()->getParent();
3973 const MachineRegisterInfo &MRI = MF.getRegInfo();
3974 MachineInstr *Add = MRI.getUniqueVRegDef(Reg);
3975 if (Add == nullptr)
3976 return -1;
3977 if (Add->getOpcode() != X86::ADD64rr && Add->getOpcode() != X86::ADD32rr)
3978 return -1;
3979 int JTI1 = getJumpTableIndexFromReg(MRI, Add->getOperand(1).getReg());
3980 if (JTI1 >= 0)
3981 return JTI1;
3982 int JTI2 = getJumpTableIndexFromReg(MRI, Add->getOperand(2).getReg());
3983 if (JTI2 >= 0)
3984 return JTI2;
3985 }
3986 return -1;
3987}
3988
3990 MachineBranchPredicate &MBP,
3991 bool AllowModify) const {
3992 using namespace std::placeholders;
3993
3995 SmallVector<MachineInstr *, 4> CondBranches;
3996 if (analyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches,
3997 AllowModify))
3998 return true;
3999
4000 if (Cond.size() != 1)
4001 return true;
4002
4003 assert(MBP.TrueDest && "expected!");
4004
4005 if (!MBP.FalseDest)
4006 MBP.FalseDest = MBB.getNextNode();
4007
4009
4010 MachineInstr *ConditionDef = nullptr;
4011 bool SingleUseCondition = true;
4012
4014 if (MI.modifiesRegister(X86::EFLAGS, TRI)) {
4015 ConditionDef = &MI;
4016 break;
4017 }
4018
4019 if (MI.readsRegister(X86::EFLAGS, TRI))
4020 SingleUseCondition = false;
4021 }
4022
4023 if (!ConditionDef)
4024 return true;
4025
4026 if (SingleUseCondition) {
4027 for (auto *Succ : MBB.successors())
4028 if (Succ->isLiveIn(X86::EFLAGS))
4029 SingleUseCondition = false;
4030 }
4031
4032 MBP.ConditionDef = ConditionDef;
4033 MBP.SingleUseCondition = SingleUseCondition;
4034
4035 // Currently we only recognize the simple pattern:
4036 //
4037 // test %reg, %reg
4038 // je %label
4039 //
4040 const unsigned TestOpcode =
4041 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
4042
4043 if (ConditionDef->getOpcode() == TestOpcode &&
4044 ConditionDef->getNumOperands() == 3 &&
4045 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) &&
4046 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) {
4047 MBP.LHS = ConditionDef->getOperand(0);
4048 MBP.RHS = MachineOperand::CreateImm(0);
4049 MBP.Predicate = Cond[0].getImm() == X86::COND_NE
4050 ? MachineBranchPredicate::PRED_NE
4051 : MachineBranchPredicate::PRED_EQ;
4052 return false;
4053 }
4054
4055 return true;
4056}
4057
4059 int *BytesRemoved) const {
4060 assert(!BytesRemoved && "code size not handled");
4061
4063 unsigned Count = 0;
4064
4065 while (I != MBB.begin()) {
4066 --I;
4067 if (I->isDebugInstr())
4068 continue;
4069 if (I->getOpcode() != X86::JMP_1 &&
4071 break;
4072 // Remove the branch.
4073 I->eraseFromParent();
4074 I = MBB.end();
4075 ++Count;
4076 }
4077
4078 return Count;
4079}
4080
4083 MachineBasicBlock *FBB,
4085 const DebugLoc &DL, int *BytesAdded) const {
4086 // Shouldn't be a fall through.
4087 assert(TBB && "insertBranch must not be told to insert a fallthrough");
4088 assert((Cond.size() == 1 || Cond.size() == 0) &&
4089 "X86 branch conditions have one component!");
4090 assert(!BytesAdded && "code size not handled");
4091
4092 if (Cond.empty()) {
4093 // Unconditional branch?
4094 assert(!FBB && "Unconditional branch with multiple successors!");
4095 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB);
4096 return 1;
4097 }
4098
4099 // If FBB is null, it is implied to be a fall-through block.
4100 bool FallThru = FBB == nullptr;
4101
4102 // Conditional branch.
4103 unsigned Count = 0;
4105 switch (CC) {
4106 case X86::COND_NE_OR_P:
4107 // Synthesize NE_OR_P with two branches.
4108 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE);
4109 ++Count;
4110 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P);
4111 ++Count;
4112 break;
4113 case X86::COND_E_AND_NP:
4114 // Use the next block of MBB as FBB if it is null.
4115 if (FBB == nullptr) {
4116 FBB = getFallThroughMBB(&MBB, TBB);
4117 assert(FBB && "MBB cannot be the last block in function when the false "
4118 "body is a fall-through.");
4119 }
4120 // Synthesize COND_E_AND_NP with two branches.
4121 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE);
4122 ++Count;
4123 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP);
4124 ++Count;
4125 break;
4126 default: {
4127 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC);
4128 ++Count;
4129 }
4130 }
4131 if (!FallThru) {
4132 // Two-way Conditional branch. Insert the second branch.
4133 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB);
4134 ++Count;
4135 }
4136 return Count;
4137}
4138
4141 Register DstReg, Register TrueReg,
4142 Register FalseReg,
4143 SelectExpansion &Exp) const {
4144 // Not all subtargets have cmov instructions.
4145 if (!Subtarget.canUseCMOV())
4146 return false;
4147 if (Cond.size() != 1)
4148 return false;
4149 // We cannot do the composite conditions, at least not in SSA form.
4151 return false;
4152
4153 // Check register classes.
4154 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4155 const TargetRegisterClass *RC =
4156 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
4157 if (!RC)
4158 return false;
4159
4160 // We have cmov instructions for 16, 32, and 64 bit general purpose registers.
4161 if (X86::GR16RegClass.hasSubClassEq(RC) ||
4162 X86::GR32RegClass.hasSubClassEq(RC) ||
4163 X86::GR64RegClass.hasSubClassEq(RC)) {
4164 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy
4165 // Bridge. Probably Ivy Bridge as well.
4166 Exp.CondCycles = 2;
4167 Exp.TrueCycles = 2;
4168 Exp.FalseCycles = 2;
4169 return true;
4170 }
4171
4172 // Can't do vectors.
4173 return false;
4174}
4175
4178 const DebugLoc &DL, Register DstReg,
4180 Register FalseReg) const {
4181 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4182 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
4183 const TargetRegisterClass &RC = *MRI.getRegClass(DstReg);
4184 assert(Cond.size() == 1 && "Invalid Cond array");
4185 unsigned Opc =
4186 X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8,
4187 false /*HasMemoryOperand*/, Subtarget.hasNDD());
4188 BuildMI(MBB, I, DL, get(Opc), DstReg)
4189 .addReg(FalseReg)
4190 .addReg(TrueReg)
4191 .addImm(Cond[0].getImm());
4192}
4193
4194/// Test if the given register is a physical h register.
4195static bool isHReg(MCRegister Reg) {
4196 return X86::GR8_ABCD_HRegClass.contains(Reg);
4197}
4198
4199// Try and copy between VR128/VR64 and GR64 registers.
4200static unsigned CopyToFromAsymmetricReg(Register DestReg, Register SrcReg,
4201 const X86Subtarget &Subtarget) {
4202 bool HasAVX = Subtarget.hasAVX();
4203 bool HasAVX512 = Subtarget.hasAVX512();
4204 bool HasEGPR = Subtarget.hasEGPR();
4205
4206 // SrcReg(MaskReg) -> DestReg(GR64)
4207 // SrcReg(MaskReg) -> DestReg(GR32)
4208
4209 // All KMASK RegClasses hold the same k registers, can be tested against
4210 // anyone.
4211 if (X86::VK16RegClass.contains(SrcReg)) {
4212 if (X86::GR64RegClass.contains(DestReg)) {
4213 assert(Subtarget.hasBWI());
4214 return HasEGPR ? X86::KMOVQrk_EVEX : X86::KMOVQrk;
4215 }
4216 if (X86::GR32RegClass.contains(DestReg))
4217 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDrk_EVEX : X86::KMOVDrk)
4218 : (HasEGPR ? X86::KMOVWrk_EVEX : X86::KMOVWrk);
4219 }
4220
4221 // SrcReg(GR64) -> DestReg(MaskReg)
4222 // SrcReg(GR32) -> DestReg(MaskReg)
4223
4224 // All KMASK RegClasses hold the same k registers, can be tested against
4225 // anyone.
4226 if (X86::VK16RegClass.contains(DestReg)) {
4227 if (X86::GR64RegClass.contains(SrcReg)) {
4228 assert(Subtarget.hasBWI());
4229 return HasEGPR ? X86::KMOVQkr_EVEX : X86::KMOVQkr;
4230 }
4231 if (X86::GR32RegClass.contains(SrcReg))
4232 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDkr_EVEX : X86::KMOVDkr)
4233 : (HasEGPR ? X86::KMOVWkr_EVEX : X86::KMOVWkr);
4234 }
4235
4236 // SrcReg(VR128) -> DestReg(GR64)
4237 // SrcReg(VR64) -> DestReg(GR64)
4238 // SrcReg(GR64) -> DestReg(VR128)
4239 // SrcReg(GR64) -> DestReg(VR64)
4240
4241 if (X86::GR64RegClass.contains(DestReg)) {
4242 if (X86::VR128XRegClass.contains(SrcReg))
4243 // Copy from a VR128 register to a GR64 register.
4244 return HasAVX512 ? X86::VMOVPQIto64Zrr
4245 : HasAVX ? X86::VMOVPQIto64rr
4246 : X86::MOVPQIto64rr;
4247 if (X86::VR64RegClass.contains(SrcReg))
4248 // Copy from a VR64 register to a GR64 register.
4249 return X86::MMX_MOVD64from64rr;
4250 } else if (X86::GR64RegClass.contains(SrcReg)) {
4251 // Copy from a GR64 register to a VR128 register.
4252 if (X86::VR128XRegClass.contains(DestReg))
4253 return HasAVX512 ? X86::VMOV64toPQIZrr
4254 : HasAVX ? X86::VMOV64toPQIrr
4255 : X86::MOV64toPQIrr;
4256 // Copy from a GR64 register to a VR64 register.
4257 if (X86::VR64RegClass.contains(DestReg))
4258 return X86::MMX_MOVD64to64rr;
4259 }
4260
4261 // SrcReg(VR128) -> DestReg(GR32)
4262 // SrcReg(GR32) -> DestReg(VR128)
4263
4264 if (X86::GR32RegClass.contains(DestReg) &&
4265 X86::VR128XRegClass.contains(SrcReg))
4266 // Copy from a VR128 register to a GR32 register.
4267 return HasAVX512 ? X86::VMOVPDI2DIZrr
4268 : HasAVX ? X86::VMOVPDI2DIrr
4269 : X86::MOVPDI2DIrr;
4270
4271 if (X86::VR128XRegClass.contains(DestReg) &&
4272 X86::GR32RegClass.contains(SrcReg))
4273 // Copy from a GR32 register to a VR128 register.
4274 return HasAVX512 ? X86::VMOVDI2PDIZrr
4275 : HasAVX ? X86::VMOVDI2PDIrr
4276 : X86::MOVDI2PDIrr;
4277
4278 return 0;
4279}
4280
4283 const DebugLoc &DL, Register DestReg,
4284 Register SrcReg, bool KillSrc,
4285 bool RenamableDest, bool RenamableSrc) const {
4286 // First deal with the normal symmetric copies.
4287 bool HasAVX = Subtarget.hasAVX();
4288 bool HasVLX = Subtarget.hasVLX();
4289 bool HasEGPR = Subtarget.hasEGPR();
4290 unsigned Opc = 0;
4291 if (X86::GR64RegClass.contains(DestReg, SrcReg))
4292 Opc = X86::MOV64rr;
4293 else if (X86::GR32RegClass.contains(DestReg, SrcReg))
4294 Opc = X86::MOV32rr;
4295 else if (X86::GR16RegClass.contains(DestReg, SrcReg))
4296 Opc = X86::MOV16rr;
4297 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) {
4298 // Copying to or from a physical H register on x86-64 requires a NOREX
4299 // move. Otherwise use a normal move.
4300 if ((isHReg(DestReg) || isHReg(SrcReg)) && Subtarget.is64Bit()) {
4301 Opc = X86::MOV8rr_NOREX;
4302 // Both operands must be encodable without an REX prefix.
4303 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
4304 "8-bit H register can not be copied outside GR8_NOREX");
4305 } else
4306 Opc = X86::MOV8rr;
4307 } else if (X86::VR64RegClass.contains(DestReg, SrcReg))
4308 Opc = X86::MMX_MOVQ64rr;
4309 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) {
4310 if (HasVLX)
4311 Opc = X86::VMOVAPSZ128rr;
4312 else if (X86::VR128RegClass.contains(DestReg, SrcReg))
4313 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
4314 else {
4315 // If this an extended register and we don't have VLX we need to use a
4316 // 512-bit move.
4317 Opc = X86::VMOVAPSZrr;
4319 DestReg =
4320 TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, &X86::VR512RegClass);
4321 SrcReg =
4322 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
4323 }
4324 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) {
4325 if (HasVLX)
4326 Opc = X86::VMOVAPSZ256rr;
4327 else if (X86::VR256RegClass.contains(DestReg, SrcReg))
4328 Opc = X86::VMOVAPSYrr;
4329 else {
4330 // If this an extended register and we don't have VLX we need to use a
4331 // 512-bit move.
4332 Opc = X86::VMOVAPSZrr;
4334 DestReg =
4335 TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, &X86::VR512RegClass);
4336 SrcReg =
4337 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
4338 }
4339 } else if (X86::VR512RegClass.contains(DestReg, SrcReg))
4340 Opc = X86::VMOVAPSZrr;
4341 // All KMASK RegClasses hold the same k registers, can be tested against
4342 // anyone.
4343 else if (X86::VK16RegClass.contains(DestReg, SrcReg))
4344 Opc = Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk)
4345 : (HasEGPR ? X86::KMOVWkk_EVEX : X86::KMOVWkk);
4346
4347 if (!Opc)
4348 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget);
4349
4350 if (Opc) {
4351 BuildMI(MBB, MI, DL, get(Opc), DestReg)
4352 .addReg(SrcReg, getKillRegState(KillSrc));
4353 return;
4354 }
4355
4356 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
4357 // FIXME: We use a fatal error here because historically LLVM has tried
4358 // lower some of these physreg copies and we want to ensure we get
4359 // reasonable bug reports if someone encounters a case no other testing
4360 // found. This path should be removed after the LLVM 7 release.
4361 report_fatal_error("Unable to copy EFLAGS physical register!");
4362 }
4363
4364 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to "
4365 << RI.getName(DestReg) << '\n');
4366 report_fatal_error("Cannot emit physreg copy instruction");
4367}
4368
4369std::optional<DestSourcePair>
4371 if (MI.isMoveReg()) {
4372 // FIXME: Dirty hack for apparent invariant that doesn't hold when
4373 // subreg_to_reg is coalesced with ordinary copies, such that the bits that
4374 // were asserted as 0 are now undef.
4375 if (MI.getOperand(0).isUndef() && MI.getOperand(0).getSubReg())
4376 return std::nullopt;
4377
4378 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
4379 }
4380 return std::nullopt;
4381}
4382
4383static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI) {
4384 if (STI.hasFP16())
4385 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
4386 if (Load)
4387 return X86::MOVSHPrm;
4388 return X86::MOVSHPmr;
4389}
4390
4392 const TargetRegisterClass *RC,
4393 bool IsStackAligned,
4394 const X86Subtarget &STI, bool Load) {
4395 bool HasAVX = STI.hasAVX();
4396 bool HasAVX512 = STI.hasAVX512();
4397 bool HasVLX = STI.hasVLX();
4398 bool HasEGPR = STI.hasEGPR();
4399
4400 assert(RC != nullptr && "Invalid target register class");
4401 switch (STI.getRegisterInfo()->getSpillSize(*RC)) {
4402 default:
4403 llvm_unreachable("Unknown spill size");
4404 case 1:
4405 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass");
4406 if (STI.is64Bit())
4407 // Copying to or from a physical H register on x86-64 requires a NOREX
4408 // move. Otherwise use a normal move.
4409 if ((Reg.isPhysical() && isHReg(Reg)) ||
4410 X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
4411 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
4412 return Load ? X86::MOV8rm : X86::MOV8mr;
4413 case 2:
4414 if (X86::VK16RegClass.hasSubClassEq(RC))
4415 return Load ? (HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)
4416 : (HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk);
4417 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
4418 return Load ? X86::MOV16rm : X86::MOV16mr;
4419 case 4:
4420 if (X86::GR32RegClass.hasSubClassEq(RC))
4421 return Load ? X86::MOV32rm : X86::MOV32mr;
4422 if (X86::FR32XRegClass.hasSubClassEq(RC))
4423 return Load ? (HasAVX512 ? X86::VMOVSSZrm_alt
4424 : HasAVX ? X86::VMOVSSrm_alt
4425 : X86::MOVSSrm_alt)
4426 : (HasAVX512 ? X86::VMOVSSZmr
4427 : HasAVX ? X86::VMOVSSmr
4428 : X86::MOVSSmr);
4429 if (X86::RFP32RegClass.hasSubClassEq(RC))
4430 return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
4431 if (X86::VK32RegClass.hasSubClassEq(RC)) {
4432 assert(STI.hasBWI() && "KMOVD requires BWI");
4433 return Load ? (HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm)
4434 : (HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk);
4435 }
4436 // All of these mask pair classes have the same spill size, the same kind
4437 // of kmov instructions can be used with all of them.
4438 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
4439 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
4440 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
4441 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
4442 X86::VK16PAIRRegClass.hasSubClassEq(RC))
4443 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
4444 if (X86::FR16RegClass.hasSubClassEq(RC) ||
4445 X86::FR16XRegClass.hasSubClassEq(RC))
4446 return getLoadStoreOpcodeForFP16(Load, STI);
4447 llvm_unreachable("Unknown 4-byte regclass");
4448 case 8:
4449 if (X86::GR64RegClass.hasSubClassEq(RC))
4450 return Load ? X86::MOV64rm : X86::MOV64mr;
4451 if (X86::FR64XRegClass.hasSubClassEq(RC))
4452 return Load ? (HasAVX512 ? X86::VMOVSDZrm_alt
4453 : HasAVX ? X86::VMOVSDrm_alt
4454 : X86::MOVSDrm_alt)
4455 : (HasAVX512 ? X86::VMOVSDZmr
4456 : HasAVX ? X86::VMOVSDmr
4457 : X86::MOVSDmr);
4458 if (X86::VR64RegClass.hasSubClassEq(RC))
4459 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
4460 if (X86::RFP64RegClass.hasSubClassEq(RC))
4461 return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
4462 if (X86::VK64RegClass.hasSubClassEq(RC)) {
4463 assert(STI.hasBWI() && "KMOVQ requires BWI");
4464 return Load ? (HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm)
4465 : (HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk);
4466 }
4467 llvm_unreachable("Unknown 8-byte regclass");
4468 case 10:
4469 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
4470 return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
4471 case 16: {
4472 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
4473 // If stack is realigned we can use aligned stores.
4474 if (IsStackAligned)
4475 return Load ? (HasVLX ? X86::VMOVAPSZ128rm
4476 : HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX
4477 : HasAVX ? X86::VMOVAPSrm
4478 : X86::MOVAPSrm)
4479 : (HasVLX ? X86::VMOVAPSZ128mr
4480 : HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX
4481 : HasAVX ? X86::VMOVAPSmr
4482 : X86::MOVAPSmr);
4483 else
4484 return Load ? (HasVLX ? X86::VMOVUPSZ128rm
4485 : HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX
4486 : HasAVX ? X86::VMOVUPSrm
4487 : X86::MOVUPSrm)
4488 : (HasVLX ? X86::VMOVUPSZ128mr
4489 : HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX
4490 : HasAVX ? X86::VMOVUPSmr
4491 : X86::MOVUPSmr);
4492 }
4493 llvm_unreachable("Unknown 16-byte regclass");
4494 }
4495 case 32:
4496 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass");
4497 // If stack is realigned we can use aligned stores.
4498 if (IsStackAligned)
4499 return Load ? (HasVLX ? X86::VMOVAPSZ256rm
4500 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
4501 : X86::VMOVAPSYrm)
4502 : (HasVLX ? X86::VMOVAPSZ256mr
4503 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
4504 : X86::VMOVAPSYmr);
4505 else
4506 return Load ? (HasVLX ? X86::VMOVUPSZ256rm
4507 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
4508 : X86::VMOVUPSYrm)
4509 : (HasVLX ? X86::VMOVUPSZ256mr
4510 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
4511 : X86::VMOVUPSYmr);
4512 case 64:
4513 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass");
4514 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512");
4515 if (IsStackAligned)
4516 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
4517 else
4518 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
4519 case 1024:
4520 assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
4521 assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE");
4522#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
4523 return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
4524 : GET_EGPR_IF_ENABLED(X86::TILESTORED);
4525#undef GET_EGPR_IF_ENABLED
4526 }
4527}
4528
4529std::optional<ExtAddrMode>
4531 int MemRefBegin = X86II::getMemoryOperandIdx(MemI.getDesc());
4532 if (MemRefBegin < 0)
4533 return std::nullopt;
4534
4535 auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg);
4536 if (!BaseOp.isReg()) // Can be an MO_FrameIndex
4537 return std::nullopt;
4538
4539 const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp);
4540 // Displacement can be symbolic
4541 if (!DispMO.isImm())
4542 return std::nullopt;
4543
4544 ExtAddrMode AM;
4545 AM.BaseReg = BaseOp.getReg();
4546 AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg();
4547 AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm();
4548 AM.Displacement = DispMO.getImm();
4549 return AM;
4550}
4551
4553 StringRef &ErrInfo) const {
4554 std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI);
4555 if (!AMOrNone)
4556 return true;
4557
4558 ExtAddrMode AM = *AMOrNone;
4560 if (AM.ScaledReg != X86::NoRegister) {
4561 switch (AM.Scale) {
4562 case 1:
4563 case 2:
4564 case 4:
4565 case 8:
4566 break;
4567 default:
4568 ErrInfo = "Scale factor in address must be 1, 2, 4 or 8";
4569 return false;
4570 }
4571 }
4572 if (!isInt<32>(AM.Displacement)) {
4573 ErrInfo = "Displacement in address must fit into 32-bit signed "
4574 "integer";
4575 return false;
4576 }
4577
4578 return true;
4579}
4580
4582 const Register Reg,
4583 int64_t &ImmVal) const {
4584 Register MovReg = Reg;
4585 const MachineInstr *MovMI = &MI;
4586
4587 // Follow use-def for SUBREG_TO_REG to find the real move immediate
4588 // instruction. It is quite common for x86-64.
4589 if (MI.isSubregToReg()) {
4590 // We use following pattern to setup 64b immediate.
4591 // %8:gr32 = MOV32r0 implicit-def dead $eflags
4592 // %6:gr64 = SUBREG_TO_REG killed %8:gr32, %subreg.sub_32bit
4593 unsigned SubIdx = MI.getOperand(2).getImm();
4594 MovReg = MI.getOperand(1).getReg();
4595 if (SubIdx != X86::sub_32bit)
4596 return false;
4597 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4598 MovMI = MRI.getUniqueVRegDef(MovReg);
4599 if (!MovMI)
4600 return false;
4601 }
4602
4603 if (MovMI->getOpcode() == X86::MOV32r0 &&
4604 MovMI->getOperand(0).getReg() == MovReg) {
4605 ImmVal = 0;
4606 return true;
4607 }
4608
4609 if (MovMI->getOpcode() != X86::MOV32ri &&
4610 MovMI->getOpcode() != X86::MOV64ri &&
4611 MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri)
4612 return false;
4613 // Mov Src can be a global address.
4614 if (!MovMI->getOperand(1).isImm() || MovMI->getOperand(0).getReg() != MovReg)
4615 return false;
4616 ImmVal = MovMI->getOperand(1).getImm();
4617 return true;
4618}
4619
4621 const Register NullValueReg) const {
4622 if (!MI->modifiesRegister(NullValueReg, &RI))
4623 return true;
4624 switch (MI->getOpcode()) {
4625 // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax
4626 // X.
4627 case X86::SHR64ri:
4628 case X86::SHR32ri:
4629 case X86::SHL64ri:
4630 case X86::SHL32ri:
4631 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() &&
4632 "expected for shift opcode!");
4633 return MI->getOperand(0).getReg() == NullValueReg &&
4634 MI->getOperand(1).getReg() == NullValueReg;
4635 // Zero extend of a sub-reg of NullValueReg into itself does not change the
4636 // null value.
4637 case X86::MOV32rr:
4638 return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) {
4639 return RI.isSubRegisterEq(NullValueReg, MO.getReg());
4640 });
4641 default:
4642 return false;
4643 }
4644 llvm_unreachable("Should be handled above!");
4645}
4646
4649 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
4650 int MemRefBegin = X86II::getMemoryOperandIdx(MemOp.getDesc());
4651 if (MemRefBegin < 0)
4652 return false;
4653
4654 const MachineOperand *BaseOp =
4655 &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg);
4656 if (!BaseOp->isReg()) // Can be an MO_FrameIndex
4657 return false;
4658
4659 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1)
4660 return false;
4661
4662 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() !=
4663 X86::NoRegister)
4664 return false;
4665
4666 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp);
4667
4668 // Displacement can be symbolic
4669 if (!DispMO.isImm())
4670 return false;
4671
4672 Offset = DispMO.getImm();
4673
4674 if (!BaseOp->isReg())
4675 return false;
4676
4677 OffsetIsScalable = false;
4678 // FIXME: Relying on memoperands() may not be right thing to do here. Check
4679 // with X86 maintainers, and fix it accordingly. For now, it is ok, since
4680 // there is no use of `Width` for X86 back-end at the moment.
4681 Width = !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize()
4683 BaseOps.push_back(BaseOp);
4684 return true;
4685}
4686
4687static unsigned getStoreRegOpcode(Register SrcReg,
4688 const TargetRegisterClass *RC,
4689 bool IsStackAligned,
4690 const X86Subtarget &STI) {
4691 return getLoadStoreRegOpcode(SrcReg, RC, IsStackAligned, STI, false);
4692}
4693
4694static unsigned getLoadRegOpcode(Register DestReg,
4695 const TargetRegisterClass *RC,
4696 bool IsStackAligned, const X86Subtarget &STI) {
4697 return getLoadStoreRegOpcode(DestReg, RC, IsStackAligned, STI, true);
4698}
4699
4700static bool isAMXOpcode(unsigned Opc) {
4701 switch (Opc) {
4702 default:
4703 return false;
4704 case X86::TILELOADD:
4705 case X86::TILESTORED:
4706 case X86::TILELOADD_EVEX:
4707 case X86::TILESTORED_EVEX:
4708 return true;
4709 }
4710}
4711
4714 unsigned Opc, Register Reg, int FrameIdx,
4715 bool isKill) const {
4716 switch (Opc) {
4717 default:
4718 llvm_unreachable("Unexpected special opcode!");
4719 case X86::TILESTORED:
4720 case X86::TILESTORED_EVEX: {
4721 // tilestored %tmm, (%sp, %idx)
4722 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
4723 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4724 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4725 MachineInstr *NewMI =
4726 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
4727 .addReg(Reg, getKillRegState(isKill));
4729 MO.setReg(VirtReg);
4730 break;
4731 }
4732 case X86::TILELOADD:
4733 case X86::TILELOADD_EVEX: {
4734 // tileloadd (%sp, %idx), %tmm
4735 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
4736 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4737 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4739 BuildMI(MBB, MI, DebugLoc(), get(Opc), Reg), FrameIdx);
4741 MO.setReg(VirtReg);
4742 break;
4743 }
4744 }
4745}
4746
4749 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
4750
4751 Register VReg, MachineInstr::MIFlag Flags) const {
4752 const MachineFunction &MF = *MBB.getParent();
4753 const MachineFrameInfo &MFI = MF.getFrameInfo();
4754 assert(MFI.getObjectSize(FrameIdx) >= RI.getSpillSize(*RC) &&
4755 "Stack slot too small for store");
4756
4757 unsigned Alignment = std::max<uint32_t>(RI.getSpillSize(*RC), 16);
4758 bool isAligned =
4759 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4760 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
4761
4762 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget);
4763 if (isAMXOpcode(Opc))
4764 loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill);
4765 else
4766 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
4767 .addReg(SrcReg, getKillRegState(isKill))
4768 .setMIFlag(Flags);
4769}
4770
4773 Register DestReg, int FrameIdx,
4774 const TargetRegisterClass *RC,
4775 Register VReg, unsigned SubReg,
4776 MachineInstr::MIFlag Flags) const {
4777 const MachineFunction &MF = *MBB.getParent();
4778 const MachineFrameInfo &MFI = MF.getFrameInfo();
4779 assert(MFI.getObjectSize(FrameIdx) >= RI.getSpillSize(*RC) &&
4780 "Load size exceeds stack slot");
4781 unsigned Alignment = std::max<uint32_t>(RI.getSpillSize(*RC), 16);
4782 bool isAligned =
4783 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4784 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
4785
4786 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget);
4787 if (isAMXOpcode(Opc))
4788 loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx);
4789 else
4790 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), FrameIdx)
4791 .setMIFlag(Flags);
4792}
4793
4795 Register &SrcReg2, int64_t &CmpMask,
4796 int64_t &CmpValue) const {
4797 switch (MI.getOpcode()) {
4798 default:
4799 break;
4800 case X86::CMP64ri32:
4801 case X86::CMP32ri:
4802 case X86::CMP16ri:
4803 case X86::CMP8ri:
4804 SrcReg = MI.getOperand(0).getReg();
4805 SrcReg2 = 0;
4806 if (MI.getOperand(1).isImm()) {
4807 CmpMask = ~0;
4808 CmpValue = MI.getOperand(1).getImm();
4809 } else {
4810 CmpMask = CmpValue = 0;
4811 }
4812 return true;
4813 // A SUB can be used to perform comparison.
4814 CASE_ND(SUB64rm)
4815 CASE_ND(SUB32rm)
4816 CASE_ND(SUB16rm)
4817 CASE_ND(SUB8rm)
4818 SrcReg = MI.getOperand(1).getReg();
4819 SrcReg2 = 0;
4820 CmpMask = 0;
4821 CmpValue = 0;
4822 return true;
4823 CASE_ND(SUB64rr)
4824 CASE_ND(SUB32rr)
4825 CASE_ND(SUB16rr)
4826 CASE_ND(SUB8rr)
4827 SrcReg = MI.getOperand(1).getReg();
4828 SrcReg2 = MI.getOperand(2).getReg();
4829 CmpMask = 0;
4830 CmpValue = 0;
4831 return true;
4832 CASE_ND(SUB64ri32)
4833 CASE_ND(SUB32ri)
4834 CASE_ND(SUB16ri)
4835 CASE_ND(SUB8ri)
4836 SrcReg = MI.getOperand(1).getReg();
4837 SrcReg2 = 0;
4838 if (MI.getOperand(2).isImm()) {
4839 CmpMask = ~0;
4840 CmpValue = MI.getOperand(2).getImm();
4841 } else {
4842 CmpMask = CmpValue = 0;
4843 }
4844 return true;
4845 case X86::CMP64rr:
4846 case X86::CMP32rr:
4847 case X86::CMP16rr:
4848 case X86::CMP8rr:
4849 SrcReg = MI.getOperand(0).getReg();
4850 SrcReg2 = MI.getOperand(1).getReg();
4851 CmpMask = 0;
4852 CmpValue = 0;
4853 return true;
4854 case X86::TEST8rr:
4855 case X86::TEST16rr:
4856 case X86::TEST32rr:
4857 case X86::TEST64rr:
4858 SrcReg = MI.getOperand(0).getReg();
4859 if (MI.getOperand(1).getReg() != SrcReg)
4860 return false;
4861 // Compare against zero.
4862 SrcReg2 = 0;
4863 CmpMask = ~0;
4864 CmpValue = 0;
4865 return true;
4866 case X86::TEST64ri32:
4867 case X86::TEST32ri:
4868 case X86::TEST16ri:
4869 case X86::TEST8ri:
4870 SrcReg = MI.getOperand(0).getReg();
4871 SrcReg2 = 0;
4872 // Force identical compare.
4873 CmpMask = 0;
4874 CmpValue = 0;
4875 return true;
4876 }
4877 return false;
4878}
4879
4880bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
4881 Register SrcReg, Register SrcReg2,
4882 int64_t ImmMask, int64_t ImmValue,
4883 const MachineInstr &OI, bool *IsSwapped,
4884 int64_t *ImmDelta) const {
4885 switch (OI.getOpcode()) {
4886 case X86::CMP64rr:
4887 case X86::CMP32rr:
4888 case X86::CMP16rr:
4889 case X86::CMP8rr:
4890 CASE_ND(SUB64rr)
4891 CASE_ND(SUB32rr)
4892 CASE_ND(SUB16rr)
4893 CASE_ND(SUB8rr) {
4894 Register OISrcReg;
4895 Register OISrcReg2;
4896 int64_t OIMask;
4897 int64_t OIValue;
4898 if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
4899 OIMask != ImmMask || OIValue != ImmValue)
4900 return false;
4901 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
4902 *IsSwapped = false;
4903 return true;
4904 }
4905 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
4906 *IsSwapped = true;
4907 return true;
4908 }
4909 return false;
4910 }
4911 case X86::CMP64ri32:
4912 case X86::CMP32ri:
4913 case X86::CMP16ri:
4914 case X86::CMP8ri:
4915 case X86::TEST64ri32:
4916 case X86::TEST32ri:
4917 case X86::TEST16ri:
4918 case X86::TEST8ri:
4919 CASE_ND(SUB64ri32)
4920 CASE_ND(SUB32ri)
4921 CASE_ND(SUB16ri)
4922 CASE_ND(SUB8ri)
4923 case X86::TEST64rr:
4924 case X86::TEST32rr:
4925 case X86::TEST16rr:
4926 case X86::TEST8rr: {
4927 if (ImmMask != 0) {
4928 Register OISrcReg;
4929 Register OISrcReg2;
4930 int64_t OIMask;
4931 int64_t OIValue;
4932 if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) &&
4933 SrcReg == OISrcReg && ImmMask == OIMask) {
4934 if (OIValue == ImmValue) {
4935 *ImmDelta = 0;
4936 return true;
4937 } else if (static_cast<uint64_t>(ImmValue) ==
4938 static_cast<uint64_t>(OIValue) - 1) {
4939 *ImmDelta = -1;
4940 return true;
4941 } else if (static_cast<uint64_t>(ImmValue) ==
4942 static_cast<uint64_t>(OIValue) + 1) {
4943 *ImmDelta = 1;
4944 return true;
4945 } else {
4946 return false;
4947 }
4948 }
4949 }
4950 return FlagI.isIdenticalTo(OI);
4951 }
4952 default:
4953 return false;
4954 }
4955}
4956
4957inline static bool isCmpRedundantAfterLTZCNT(Register SrcReg, Register SrcReg2,
4958 int64_t ImmMask, int64_t ImmValue,
4959 const MachineInstr &OI) {
4960 switch (OI.getOpcode()) {
4961 default:
4962 return false;
4963 case X86::LZCNT16rr:
4964 case X86::LZCNT32rr:
4965 case X86::LZCNT64rr:
4966 case X86::TZCNT16rr:
4967 case X86::TZCNT32rr:
4968 case X86::TZCNT64rr: {
4969 if (ImmMask != 0 && !SrcReg2.isValid() && ImmValue == 1 &&
4970 OI.getOperand(1).isReg() && SrcReg == OI.getOperand(1).getReg()) {
4971 return true;
4972 }
4973 return false;
4974 }
4975 }
4976}
4977
4978#define CASE_EVEX(OP) \
4979 case X86::OP: \
4980 case X86::OP##_EVEX:
4981
4982/// Check whether the definition can be converted
4983/// to remove a comparison against zero.
4984inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
4985 bool &ClearsOverflowFlag) {
4986 NoSignFlag = false;
4987 ClearsOverflowFlag = false;
4988
4989 // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and
4990 // i386 GOTNTPOFF/INDNTPOFF relocations can convert an ADD to a LEA during
4991 // Initial Exec to Local Exec relaxation. In these cases, we must not depend
4992 // on the EFLAGS modification of ADD actually happening in the final binary.
4993 if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) {
4994 unsigned Flags = MI.getOperand(5).getTargetFlags();
4995 if (Flags == X86II::MO_GOTTPOFF || Flags == X86II::MO_INDNTPOFF ||
4996 Flags == X86II::MO_GOTNTPOFF)
4997 return false;
4998 }
4999
5000 switch (MI.getOpcode()) {
5001 default:
5002 return false;
5003
5004 // The shift instructions only modify ZF if their shift count is non-zero.
5005 // N.B.: The processor truncates the shift count depending on the encoding.
5006 CASE_ND(SAR8ri)
5007 CASE_ND(SAR16ri)
5008 CASE_ND(SAR32ri)
5009 CASE_ND(SAR64ri)
5010 CASE_ND(SHR8ri)
5011 CASE_ND(SHR16ri)
5012 CASE_ND(SHR32ri)
5013 CASE_ND(SHR64ri)
5014 return getTruncatedShiftCount(MI, 2) != 0;
5015
5016 // Some left shift instructions can be turned into LEA instructions but only
5017 // if their flags aren't used. Avoid transforming such instructions.
5018 CASE_ND(SHL8ri)
5019 CASE_ND(SHL16ri)
5020 CASE_ND(SHL32ri)
5021 CASE_ND(SHL64ri) {
5022 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
5023 // Converting to LEA only pays off when the shifted operand stays live,
5024 // since it spares a register copy; when the shift is the operand's only
5025 // user, reusing the flags is strictly better.
5026 if (isTruncatedShiftCountForLEA(ShAmt)) {
5027 Register SrcReg = MI.getOperand(1).getReg();
5028 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
5029 if (!SrcReg.isVirtual() || !MRI.hasOneNonDBGUse(SrcReg))
5030 return false;
5031 }
5032 return ShAmt != 0;
5033 }
5034
5035 CASE_ND(SHRD16rri8)
5036 CASE_ND(SHRD32rri8)
5037 CASE_ND(SHRD64rri8)
5038 CASE_ND(SHLD16rri8)
5039 CASE_ND(SHLD32rri8)
5040 CASE_ND(SHLD64rri8)
5041 return getTruncatedShiftCount(MI, 3) != 0;
5042
5043 CASE_ND(SUB64ri32)
5044 CASE_ND(SUB32ri)
5045 CASE_ND(SUB16ri)
5046 CASE_ND(SUB8ri)
5047 CASE_ND(SUB64rr)
5048 CASE_ND(SUB32rr)
5049 CASE_ND(SUB16rr)
5050 CASE_ND(SUB8rr)
5051 CASE_ND(SUB64rm)
5052 CASE_ND(SUB32rm)
5053 CASE_ND(SUB16rm)
5054 CASE_ND(SUB8rm)
5055 CASE_ND(DEC64r)
5056 CASE_ND(DEC32r)
5057 CASE_ND(DEC16r)
5058 CASE_ND(DEC8r)
5059 CASE_ND(ADD64ri32)
5060 CASE_ND(ADD32ri)
5061 CASE_ND(ADD16ri)
5062 CASE_ND(ADD8ri)
5063 CASE_ND(ADD64rr)
5064 CASE_ND(ADD32rr)
5065 CASE_ND(ADD16rr)
5066 CASE_ND(ADD8rr)
5067 CASE_ND(ADD64rm)
5068 CASE_ND(ADD32rm)
5069 CASE_ND(ADD16rm)
5070 CASE_ND(ADD8rm)
5071 CASE_ND(INC64r)
5072 CASE_ND(INC32r)
5073 CASE_ND(INC16r)
5074 CASE_ND(INC8r)
5075 CASE_ND(ADC64ri32)
5076 CASE_ND(ADC32ri)
5077 CASE_ND(ADC16ri)
5078 CASE_ND(ADC8ri)
5079 CASE_ND(ADC64rr)
5080 CASE_ND(ADC32rr)
5081 CASE_ND(ADC16rr)
5082 CASE_ND(ADC8rr)
5083 CASE_ND(ADC64rm)
5084 CASE_ND(ADC32rm)
5085 CASE_ND(ADC16rm)
5086 CASE_ND(ADC8rm)
5087 CASE_ND(SBB64ri32)
5088 CASE_ND(SBB32ri)
5089 CASE_ND(SBB16ri)
5090 CASE_ND(SBB8ri)
5091 CASE_ND(SBB64rr)
5092 CASE_ND(SBB32rr)
5093 CASE_ND(SBB16rr)
5094 CASE_ND(SBB8rr)
5095 CASE_ND(SBB64rm)
5096 CASE_ND(SBB32rm)
5097 CASE_ND(SBB16rm)
5098 CASE_ND(SBB8rm)
5099 CASE_ND(NEG8r)
5100 CASE_ND(NEG16r)
5101 CASE_ND(NEG32r)
5102 CASE_ND(NEG64r)
5103 case X86::LZCNT16rr:
5104 case X86::LZCNT16rm:
5105 case X86::LZCNT32rr:
5106 case X86::LZCNT32rm:
5107 case X86::LZCNT64rr:
5108 case X86::LZCNT64rm:
5109 case X86::POPCNT16rr:
5110 case X86::POPCNT16rm:
5111 case X86::POPCNT32rr:
5112 case X86::POPCNT32rm:
5113 case X86::POPCNT64rr:
5114 case X86::POPCNT64rm:
5115 case X86::TZCNT16rr:
5116 case X86::TZCNT16rm:
5117 case X86::TZCNT32rr:
5118 case X86::TZCNT32rm:
5119 case X86::TZCNT64rr:
5120 case X86::TZCNT64rm:
5121 return true;
5122 CASE_ND(AND64ri32)
5123 CASE_ND(AND32ri)
5124 CASE_ND(AND16ri)
5125 CASE_ND(AND8ri)
5126 CASE_ND(AND64rr)
5127 CASE_ND(AND32rr)
5128 CASE_ND(AND16rr)
5129 CASE_ND(AND8rr)
5130 CASE_ND(AND64rm)
5131 CASE_ND(AND32rm)
5132 CASE_ND(AND16rm)
5133 CASE_ND(AND8rm)
5134 CASE_ND(XOR64ri32)
5135 CASE_ND(XOR32ri)
5136 CASE_ND(XOR16ri)
5137 CASE_ND(XOR8ri)
5138 CASE_ND(XOR64rr)
5139 CASE_ND(XOR32rr)
5140 CASE_ND(XOR16rr)
5141 CASE_ND(XOR8rr)
5142 CASE_ND(XOR64rm)
5143 CASE_ND(XOR32rm)
5144 CASE_ND(XOR16rm)
5145 CASE_ND(XOR8rm)
5146 CASE_ND(OR64ri32)
5147 CASE_ND(OR32ri)
5148 CASE_ND(OR16ri)
5149 CASE_ND(OR8ri)
5150 CASE_ND(OR64rr)
5151 CASE_ND(OR32rr)
5152 CASE_ND(OR16rr)
5153 CASE_ND(OR8rr)
5154 CASE_ND(OR64rm)
5155 CASE_ND(OR32rm)
5156 CASE_ND(OR16rm)
5157 CASE_ND(OR8rm)
5158 CASE_EVEX(ANDN32rr)
5159 CASE_EVEX(ANDN32rm)
5160 CASE_EVEX(ANDN64rr)
5161 CASE_EVEX(ANDN64rm)
5162 CASE_EVEX(BLSI32rr)
5163 CASE_EVEX(BLSI32rm)
5164 CASE_EVEX(BLSI64rr)
5165 CASE_EVEX(BLSI64rm)
5166 CASE_EVEX(BLSMSK32rr)
5167 CASE_EVEX(BLSMSK32rm)
5168 CASE_EVEX(BLSMSK64rr)
5169 CASE_EVEX(BLSMSK64rm)
5170 CASE_EVEX(BLSR32rr)
5171 CASE_EVEX(BLSR32rm)
5172 CASE_EVEX(BLSR64rr)
5173 CASE_EVEX(BLSR64rm)
5174 case X86::BLCFILL32rr:
5175 case X86::BLCFILL32rm:
5176 case X86::BLCFILL64rr:
5177 case X86::BLCFILL64rm:
5178 case X86::BLCI32rr:
5179 case X86::BLCI32rm:
5180 case X86::BLCI64rr:
5181 case X86::BLCI64rm:
5182 case X86::BLCIC32rr:
5183 case X86::BLCIC32rm:
5184 case X86::BLCIC64rr:
5185 case X86::BLCIC64rm:
5186 case X86::BLCMSK32rr:
5187 case X86::BLCMSK32rm:
5188 case X86::BLCMSK64rr:
5189 case X86::BLCMSK64rm:
5190 case X86::BLCS32rr:
5191 case X86::BLCS32rm:
5192 case X86::BLCS64rr:
5193 case X86::BLCS64rm:
5194 case X86::BLSFILL32rr:
5195 case X86::BLSFILL32rm:
5196 case X86::BLSFILL64rr:
5197 case X86::BLSFILL64rm:
5198 case X86::BLSIC32rr:
5199 case X86::BLSIC32rm:
5200 case X86::BLSIC64rr:
5201 case X86::BLSIC64rm:
5202 CASE_EVEX(BZHI32rr)
5203 CASE_EVEX(BZHI32rm)
5204 CASE_EVEX(BZHI64rr)
5205 CASE_EVEX(BZHI64rm)
5206 case X86::T1MSKC32rr:
5207 case X86::T1MSKC32rm:
5208 case X86::T1MSKC64rr:
5209 case X86::T1MSKC64rm:
5210 case X86::TZMSK32rr:
5211 case X86::TZMSK32rm:
5212 case X86::TZMSK64rr:
5213 case X86::TZMSK64rm:
5214 // These instructions clear the overflow flag just like TEST.
5215 // FIXME: These are not the only instructions in this switch that clear the
5216 // overflow flag.
5217 ClearsOverflowFlag = true;
5218 return true;
5219 CASE_EVEX(BEXTR32rr)
5220 CASE_EVEX(BEXTR64rr)
5221 CASE_EVEX(BEXTR32rm)
5222 CASE_EVEX(BEXTR64rm)
5223 case X86::BEXTRI32ri:
5224 case X86::BEXTRI32mi:
5225 case X86::BEXTRI64ri:
5226 case X86::BEXTRI64mi:
5227 // BEXTR doesn't update the sign flag so we can't use it. It does clear
5228 // the overflow flag, but that's not useful without the sign flag.
5229 NoSignFlag = true;
5230 return true;
5231 }
5232}
5233
5234/// Check whether the use can be converted to remove a comparison against zero.
5235/// Returns the EFLAGS condition and the operand that we are comparing against zero.
5236static std::pair<X86::CondCode, unsigned> isUseDefConvertible(const MachineInstr &MI) {
5237 switch (MI.getOpcode()) {
5238 default:
5239 return std::make_pair(X86::COND_INVALID, ~0U);
5240 CASE_ND(NEG8r)
5241 CASE_ND(NEG16r)
5242 CASE_ND(NEG32r)
5243 CASE_ND(NEG64r)
5244 return std::make_pair(X86::COND_AE, 1U);
5245 case X86::LZCNT16rr:
5246 case X86::LZCNT32rr:
5247 case X86::LZCNT64rr:
5248 return std::make_pair(X86::COND_B, 1U);
5249 case X86::POPCNT16rr:
5250 case X86::POPCNT32rr:
5251 case X86::POPCNT64rr:
5252 return std::make_pair(X86::COND_E, 1U);
5253 case X86::TZCNT16rr:
5254 case X86::TZCNT32rr:
5255 case X86::TZCNT64rr:
5256 return std::make_pair(X86::COND_B, 1U);
5257 case X86::BSF16rr:
5258 case X86::BSF32rr:
5259 case X86::BSF64rr:
5260 case X86::BSR16rr:
5261 case X86::BSR32rr:
5262 case X86::BSR64rr:
5263 return std::make_pair(X86::COND_E, 2U);
5264 CASE_EVEX(BLSI32rr)
5265 CASE_EVEX(BLSI64rr)
5266 return std::make_pair(X86::COND_AE, 1U);
5267 CASE_EVEX(BLSR32rr)
5268 CASE_EVEX(BLSR64rr)
5269 CASE_EVEX(BLSMSK32rr)
5270 CASE_EVEX(BLSMSK64rr)
5271 return std::make_pair(X86::COND_B, 1U);
5272 // TODO: TBM instructions.
5273 }
5274}
5275#undef CASE_EVEX
5276
5277MachineInstr *X86InstrInfo::findDominatingRedundantFlagInstr(
5278 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
5279 int64_t CmpValue, MachineBasicBlock *MultiPredMBB, bool &IsSwapped,
5280 int64_t &ImmDelta,
5281 SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate) const {
5282 assert(Subtarget.hasNF() && "NF feature required");
5283 const TargetRegisterInfo *TRI = &getRegisterInfo();
5284 const unsigned MaxNFConversions =
5285 Subtarget.getCLOpts().max_nf_conversions_for_cmp_reuse;
5286
5287 // The caller already scanned MultiPredMBB without finding the producer, so it
5288 // must live in a block that strictly dominates MultiPredMBB. Walk
5289 // predecessors backward to find it and prove dominance, avoiding a
5290 // whole-function MachineDominatorTree that would be rebuilt in O(function
5291 // size) per compare.
5292 //
5293 // The producer's block dominates MultiPredMBB iff every backward path funnels
5294 // through it before a function-entry block, so expand predecessors but stop
5295 // at a block holding the producer. Bail if a predecessor-less block is
5296 // reached without the producer (a path bypasses it) or the producer is found
5297 // in two blocks (neither dominates alone). Within a block, scan backward,
5298 // collecting the NF-convertible EFLAGS clobbers above the producer and
5299 // bailing on any other clobber (it would shadow the producer's flags from
5300 // CmpInstr).
5301 //
5302 // Clobbers are staged in Pending and committed only on success. Visited
5303 // is seeded with the caller's single-predecessor chain (CmpMBB through
5304 // MultiPredMBB) so the walk doesn't re-scan blocks the caller already
5305 // staged. The walk doubles as a cycle detector: a predecessor equal to
5306 // CmpMBB is a back-edge from CmpMBB's successors into the walked region,
5307 // which means CmpMBB is on a CFG cycle. In that case the region below
5308 // CmpInstr executes on the back-edge before the next iteration's CmpInstr
5309 // and must be checked too: bail on any non-NF-convertible EFLAGS clobber,
5310 // stage NF-convertible ones.
5311 //
5312 // Each NF conversion trades a compact legacy/EVEX-compressed encoding for a
5313 // wider EVEX (often NDD three-operand) one, growing code size, while the
5314 // reuse only removes a single compare. Cap the total number of conversions
5315 // (caller chain + predecessor walk + below-scan) so the reuse cannot bloat
5316 // code just to delete one compare.
5317 MachineInstr *Sub = nullptr;
5318 MachineBasicBlock *SubMBB = nullptr;
5320
5321 MachineBasicBlock *CmpMBB = CmpInstr.getParent();
5322 SmallPtrSet<MachineBasicBlock *, 8> Visited;
5324 for (MachineBasicBlock *MBB = CmpMBB; MBB != MultiPredMBB;
5326 Visited.insert(MBB);
5327 Visited.insert(MultiPredMBB);
5328
5329 bool CmpMBBOnCycle = false;
5330 auto TryPush = [&](MachineBasicBlock *Pred) {
5331 if (Pred == CmpMBB)
5332 CmpMBBOnCycle = true;
5333 if (Visited.insert(Pred).second)
5334 Worklist.push_back(Pred);
5335 };
5336
5337 for (MachineBasicBlock *Pred : MultiPredMBB->predecessors())
5338 TryPush(Pred);
5339 while (!Worklist.empty()) {
5340 MachineBasicBlock *MBB = Worklist.pop_back_val();
5341 MachineInstr *Producer = nullptr;
5342 for (MachineInstr &Inst : reverse(*MBB)) {
5343 if (!Inst.modifiesRegister(X86::EFLAGS, TRI))
5344 continue;
5345 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
5346 Inst, &IsSwapped, &ImmDelta)) {
5347 Producer = &Inst;
5348 break;
5349 }
5350 unsigned NewOpc = X86::getNFVariantIfClobberRemovable(Inst, TRI);
5351 if (!NewOpc)
5352 return nullptr;
5353 if (InstsToUpdate.size() + Pending.size() >= MaxNFConversions)
5354 return nullptr;
5355 Pending.push_back(std::make_pair(&Inst, NewOpc));
5356 }
5357 if (Producer) {
5358 // A producer in a second block means neither dominates alone.
5359 if (Sub && SubMBB != MBB)
5360 return nullptr;
5361 Sub = Producer;
5362 SubMBB = MBB;
5363 continue;
5364 }
5365 // Entry reached without the producer: some path bypasses it.
5366 if (MBB->pred_empty())
5367 return nullptr;
5368 for (MachineBasicBlock *Pred : MBB->predecessors())
5369 TryPush(Pred);
5370 }
5371 if (!Sub)
5372 return nullptr;
5373
5374 // The forward condition-code fixup in the caller (OpsToUpdate) only rewrites
5375 // EFLAGS users within CmpMBB. When the producer's flags require a condition
5376 // swap or an immediate adjustment, EFLAGS users elsewhere in the dominated
5377 // region or in CmpMBB's successors (when EFLAGS is live-out) would also need
5378 // rewriting, which is not handled here. Restrict the multi-predecessor case
5379 // to producers that yield identical flags.
5380 if (IsSwapped || ImmDelta != 0)
5381 return nullptr;
5382
5383 // If CmpMBB is on a CFG cycle, its below-CmpInstr region is on the back-edge
5384 // path and must also be free of non-NF-convertible EFLAGS clobbers.
5385 if (CmpMBBOnCycle) {
5386 for (MachineInstr &Inst : make_range(
5387 std::next(MachineBasicBlock::iterator(CmpInstr)), CmpMBB->end())) {
5388 if (!Inst.modifiesRegister(X86::EFLAGS, TRI))
5389 continue;
5390 unsigned NewOpc = X86::getNFVariantIfClobberRemovable(Inst, TRI);
5391 if (!NewOpc)
5392 return nullptr;
5393 if (InstsToUpdate.size() + Pending.size() >= MaxNFConversions)
5394 return nullptr;
5395 Pending.push_back(std::make_pair(&Inst, NewOpc));
5396 }
5397 }
5398
5399 InstsToUpdate.append(Pending.begin(), Pending.end());
5400 return Sub;
5401}
5402
5403/// Check if there exists an earlier instruction that
5404/// operates on the same source operands and sets flags in the same way as
5405/// Compare; remove Compare if possible.
5407 Register SrcReg2, int64_t CmpMask,
5408 int64_t CmpValue,
5409 const MachineRegisterInfo *MRI) const {
5410 // Check whether we can replace SUB with CMP.
5411 switch (CmpInstr.getOpcode()) {
5412 default:
5413 break;
5414 CASE_ND(SUB64ri32)
5415 CASE_ND(SUB32ri)
5416 CASE_ND(SUB16ri)
5417 CASE_ND(SUB8ri)
5418 CASE_ND(SUB64rm)
5419 CASE_ND(SUB32rm)
5420 CASE_ND(SUB16rm)
5421 CASE_ND(SUB8rm)
5422 CASE_ND(SUB64rr)
5423 CASE_ND(SUB32rr)
5424 CASE_ND(SUB16rr)
5425 CASE_ND(SUB8rr) {
5426 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
5427 return false;
5428 // There is no use of the destination register, we can replace SUB with CMP.
5429 unsigned NewOpcode = 0;
5430#define FROM_TO(A, B) \
5431 CASE_ND(A) NewOpcode = X86::B; \
5432 break;
5433 switch (CmpInstr.getOpcode()) {
5434 default:
5435 llvm_unreachable("Unreachable!");
5436 FROM_TO(SUB64rm, CMP64rm)
5437 FROM_TO(SUB32rm, CMP32rm)
5438 FROM_TO(SUB16rm, CMP16rm)
5439 FROM_TO(SUB8rm, CMP8rm)
5440 FROM_TO(SUB64rr, CMP64rr)
5441 FROM_TO(SUB32rr, CMP32rr)
5442 FROM_TO(SUB16rr, CMP16rr)
5443 FROM_TO(SUB8rr, CMP8rr)
5444 FROM_TO(SUB64ri32, CMP64ri32)
5445 FROM_TO(SUB32ri, CMP32ri)
5446 FROM_TO(SUB16ri, CMP16ri)
5447 FROM_TO(SUB8ri, CMP8ri)
5448 }
5449#undef FROM_TO
5450 CmpInstr.setDesc(get(NewOpcode));
5451 CmpInstr.removeOperand(0);
5452 // Mutating this instruction invalidates any debug data associated with it.
5453 CmpInstr.dropDebugNumber();
5454 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
5455 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
5456 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
5457 return false;
5458 }
5459 }
5460
5461 // The following code tries to remove the comparison by re-using EFLAGS
5462 // from earlier instructions.
5463
5464 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
5465
5466 // Transformation currently requires SSA values.
5467 if (SrcReg2.isPhysical())
5468 return false;
5469 MachineInstr *SrcRegDef = MRI->getVRegDef(SrcReg);
5470 if (!SrcRegDef)
5471 return false;
5472
5473 MachineInstr *MI = nullptr;
5474 MachineInstr *Sub = nullptr;
5475 MachineInstr *Movr0Inst = nullptr;
5476 MachineInstr *LTZCNTInst = nullptr;
5478 bool NoSignFlag = false;
5479 bool ClearsOverflowFlag = false;
5480 bool ShouldUpdateCC = false;
5481 bool IsSwapped = false;
5482 bool HasNF = Subtarget.hasNF();
5483 unsigned OpNo = 0;
5485 int64_t ImmDelta = 0;
5486
5487 // Search backward from CmpInstr for the next instruction defining EFLAGS.
5489 MachineBasicBlock &CmpMBB = *CmpInstr.getParent();
5491 std::next(MachineBasicBlock::reverse_iterator(CmpInstr));
5492 for (MachineBasicBlock *MBB = &CmpMBB;;) {
5493 for (MachineInstr &Inst : make_range(From, MBB->rend())) {
5494 // Try to use EFLAGS from the instruction defining %SrcReg. Example:
5495 // %eax = addl ...
5496 // ... // EFLAGS not changed
5497 // testl %eax, %eax // <-- can be removed
5498 if (&Inst == SrcRegDef) {
5499 if (IsCmpZero &&
5500 isDefConvertible(Inst, NoSignFlag, ClearsOverflowFlag)) {
5501 MI = &Inst;
5502 break;
5503 }
5504
5505 // Look back for the following pattern, in which case the
5506 // test16rr/test64rr instruction could be erased.
5507 //
5508 // Example for test16rr:
5509 // %reg = and32ri %in_reg, 5
5510 // ... // EFLAGS not changed.
5511 // %src_reg = copy %reg.sub_16bit:gr32
5512 // test16rr %src_reg, %src_reg, implicit-def $eflags
5513 // Example for test64rr:
5514 // %reg = and32ri %in_reg, 5
5515 // ... // EFLAGS not changed.
5516 // %src_reg = subreg_to_reg %reg, %subreg.sub_index
5517 // test64rr %src_reg, %src_reg, implicit-def $eflags
5518 MachineInstr *AndInstr = nullptr;
5519 if (IsCmpZero &&
5520 findRedundantFlagInstr(CmpInstr, Inst, MRI, &AndInstr, TRI,
5521 Subtarget, NoSignFlag, ClearsOverflowFlag)) {
5522 assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode()));
5523 MI = AndInstr;
5524 break;
5525 }
5526 // Cannot find other candidates before definition of SrcReg.
5527 return false;
5528 }
5529
5530 if (Inst.modifiesRegister(X86::EFLAGS, TRI)) {
5531 // Try to use EFLAGS produced by an instruction reading %SrcReg.
5532 // Example:
5533 // %eax = ...
5534 // ...
5535 // popcntl %eax
5536 // ... // EFLAGS not changed
5537 // testl %eax, %eax // <-- can be removed
5538 if (IsCmpZero) {
5539 std::tie(NewCC, OpNo) = isUseDefConvertible(Inst);
5540 if (NewCC != X86::COND_INVALID && Inst.getOperand(OpNo).isReg() &&
5541 Inst.getOperand(OpNo).getReg() == SrcReg) {
5542 ShouldUpdateCC = true;
5543 MI = &Inst;
5544 break;
5545 }
5546 }
5547
5548 // Try to use EFLAGS from an instruction with similar flag results.
5549 // Example:
5550 // sub x, y or cmp x, y
5551 // ... // EFLAGS not changed
5552 // cmp x, y // <-- can be removed
5553 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
5554 Inst, &IsSwapped, &ImmDelta)) {
5555 Sub = &Inst;
5556 break;
5557 }
5558
5559 // Try to use CF produced by an LZCNT/TZCNT reading %SrcReg: it and
5560 // "cmp $1, %SrcReg" both set CF iff %SrcReg is zero. The other flags
5561 // differ, so all EFLAGS users need to read CF only (ADC/SBB/RCL/RCR).
5562 // Example:
5563 // lzcntq %rdi, %rax
5564 // ... // EFLAGS not changed
5565 // cmpq $1, %rdi // <-- can be removed
5566 // adcq $0, %rax // reads CF only
5567 if (isCmpRedundantAfterLTZCNT(SrcReg, SrcReg2, CmpMask, CmpValue,
5568 Inst)) {
5569 LTZCNTInst = &Inst;
5570 break;
5571 }
5572
5573 // MOV32r0 is implemented with xor which clobbers condition code. It is
5574 // safe to move up, if the definition to EFLAGS is dead and earlier
5575 // instructions do not read or write EFLAGS.
5576 if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 &&
5577 Inst.registerDefIsDead(X86::EFLAGS, TRI)) {
5578 Movr0Inst = &Inst;
5579 continue;
5580 }
5581
5582 // Try to replace non-NF with NF instructions.
5583 if (HasNF) {
5584 unsigned NewOp = X86::getNFVariantIfClobberRemovable(Inst, TRI);
5585 if (!NewOp)
5586 return false;
5587
5588 InstsToUpdate.push_back(std::make_pair(&Inst, NewOp));
5589 continue;
5590 }
5591
5592 // Cannot do anything for any other EFLAG changes.
5593 return false;
5594 }
5595 }
5596
5597 if (MI || Sub || LTZCNTInst)
5598 break;
5599
5600 // Reached the begin of the basic block. If it has exactly one predecessor,
5601 // continue the backward scan there. Otherwise (multiple predecessors), try
5602 // to reuse EFLAGS from a dominating producer (handled below).
5603 if (MBB->pred_size() != 1) {
5604 // The block has multiple predecessors. We can still reuse EFLAGS from an
5605 // equivalent flag producer that dominates CmpInstr, provided every path
5606 // from that producer to CmpInstr only clobbers EFLAGS via instructions
5607 // that have an NF (no-flags) variant (which requires APX). This handles
5608 // patterns like (CMP duplicated by CodeGenPrepare across a diamond):
5609 // entry: cmp %x, C ; br
5610 // bb1: imul ... ; clobbers EFLAGS -> {nf} imul
5611 // bb2: ...
5612 // bb3: cmp %x, C ; <-- redundant, reuse EFLAGS from entry
5613 // cmovcc ...
5614 // The helper caps the total number of NF conversions so this cannot grow
5615 // code size without bound just to delete one compare.
5616 if (HasNF)
5617 Sub = findDominatingRedundantFlagInstr(
5618 CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, MBB, IsSwapped,
5619 ImmDelta, InstsToUpdate);
5620 if (!Sub)
5621 return false;
5622 break;
5623 }
5624 MBB = *MBB->pred_begin();
5625 From = MBB->rbegin();
5626 }
5627
5628 // Scan forward from the instruction after CmpInstr for uses of EFLAGS.
5629 // It is safe to remove CmpInstr if EFLAGS is redefined or killed.
5630 // If we are done with the basic block, we need to check whether EFLAGS is
5631 // live-out.
5632 bool FlagsMayLiveOut = true;
5634 MachineBasicBlock::iterator AfterCmpInstr =
5635 std::next(MachineBasicBlock::iterator(CmpInstr));
5636 for (MachineInstr &Instr : make_range(AfterCmpInstr, CmpMBB.end())) {
5637 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI);
5638 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI);
5639 // We should check the usage if this instruction uses and updates EFLAGS.
5640 if (!UseEFLAGS && ModifyEFLAGS) {
5641 // It is safe to remove CmpInstr if EFLAGS is updated again.
5642 FlagsMayLiveOut = false;
5643 break;
5644 }
5645 if (!UseEFLAGS && !ModifyEFLAGS)
5646 continue;
5647
5648 // EFLAGS is used by this instruction.
5649 X86::CondCode OldCC = X86::getCondFromMI(Instr);
5650 if ((MI || IsSwapped || ImmDelta != 0) && OldCC == X86::COND_INVALID)
5651 return false;
5652
5653 X86::CondCode ReplacementCC = X86::COND_INVALID;
5654 if (MI) {
5655 switch (OldCC) {
5656 default:
5657 break;
5658 case X86::COND_A:
5659 case X86::COND_AE:
5660 case X86::COND_B:
5661 case X86::COND_BE:
5662 // CF is used, we can't perform this optimization.
5663 return false;
5664 case X86::COND_G:
5665 case X86::COND_GE:
5666 case X86::COND_L:
5667 case X86::COND_LE:
5668 // If SF is used, but the instruction doesn't update the SF, then we
5669 // can't do the optimization.
5670 if (NoSignFlag)
5671 return false;
5672 [[fallthrough]];
5673 case X86::COND_O:
5674 case X86::COND_NO:
5675 // If OF is used, the instruction needs to clear it like CmpZero does.
5676 if (!ClearsOverflowFlag)
5677 return false;
5678 break;
5679 case X86::COND_S:
5680 case X86::COND_NS:
5681 // If SF is used, but the instruction doesn't update the SF, then we
5682 // can't do the optimization.
5683 if (NoSignFlag)
5684 return false;
5685 break;
5686 }
5687
5688 // If we're updating the condition code check if we have to reverse the
5689 // condition.
5690 if (ShouldUpdateCC)
5691 switch (OldCC) {
5692 default:
5693 return false;
5694 case X86::COND_E:
5695 ReplacementCC = NewCC;
5696 break;
5697 case X86::COND_NE:
5698 ReplacementCC = GetOppositeBranchCondition(NewCC);
5699 break;
5700 }
5701 } else if (IsSwapped) {
5702 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs
5703 // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
5704 // We swap the condition code and synthesize the new opcode.
5705 ReplacementCC = getSwappedCondition(OldCC);
5706 if (ReplacementCC == X86::COND_INVALID)
5707 return false;
5708 ShouldUpdateCC = true;
5709 } else if (ImmDelta != 0) {
5710 unsigned BitWidth = RI.getRegSizeInBits(*MRI->getRegClass(SrcReg));
5711 // Shift amount for min/max constants to adjust for 8/16/32 instruction
5712 // sizes.
5713 switch (OldCC) {
5714 case X86::COND_L: // x <s (C + 1) --> x <=s C
5715 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue)
5716 return false;
5717 ReplacementCC = X86::COND_LE;
5718 break;
5719 case X86::COND_B: // x <u (C + 1) --> x <=u C
5720 if (ImmDelta != 1 || CmpValue == 0)
5721 return false;
5722 ReplacementCC = X86::COND_BE;
5723 break;
5724 case X86::COND_GE: // x >=s (C + 1) --> x >s C
5725 if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue)
5726 return false;
5727 ReplacementCC = X86::COND_G;
5728 break;
5729 case X86::COND_AE: // x >=u (C + 1) --> x >u C
5730 if (ImmDelta != 1 || CmpValue == 0)
5731 return false;
5732 ReplacementCC = X86::COND_A;
5733 break;
5734 case X86::COND_G: // x >s (C - 1) --> x >=s C
5735 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue)
5736 return false;
5737 ReplacementCC = X86::COND_GE;
5738 break;
5739 case X86::COND_A: // x >u (C - 1) --> x >=u C
5740 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue)
5741 return false;
5742 ReplacementCC = X86::COND_AE;
5743 break;
5744 case X86::COND_LE: // x <=s (C - 1) --> x <s C
5745 if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue)
5746 return false;
5747 ReplacementCC = X86::COND_L;
5748 break;
5749 case X86::COND_BE: // x <=u (C - 1) --> x <u C
5750 if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue)
5751 return false;
5752 ReplacementCC = X86::COND_B;
5753 break;
5754 default:
5755 return false;
5756 }
5757 ShouldUpdateCC = true;
5758 }
5759
5760 if (LTZCNTInst) {
5761 unsigned InstCode = Instr.getOpcode();
5762 if (!X86::isADC(InstCode) && !X86::isSBB(InstCode) &&
5763 !X86::isRCL(InstCode) && !X86::isRCR(InstCode))
5764 return false;
5765
5766 MI = LTZCNTInst;
5767 }
5768
5769 if (ShouldUpdateCC && ReplacementCC != OldCC) {
5770 // Push the MachineInstr to OpsToUpdate.
5771 // If it is safe to remove CmpInstr, the condition code of these
5772 // instructions will be modified.
5773 OpsToUpdate.push_back(std::make_pair(&Instr, ReplacementCC));
5774 }
5775 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) {
5776 // It is safe to remove CmpInstr if EFLAGS is updated again or killed.
5777 FlagsMayLiveOut = false;
5778 break;
5779 }
5780 }
5781
5782 if (LTZCNTInst && !MI)
5783 return false;
5784
5785 // If we have to update users but EFLAGS is live-out abort, since we cannot
5786 // easily find all of the users.
5787 if ((MI != nullptr || ShouldUpdateCC) && FlagsMayLiveOut) {
5788 for (MachineBasicBlock *Successor : CmpMBB.successors())
5789 if (Successor->isLiveIn(X86::EFLAGS))
5790 return false;
5791 }
5792
5793 // The instruction to be updated is either Sub or MI.
5794 assert((MI == nullptr || Sub == nullptr) && "Should not have Sub and MI set");
5795 Sub = MI != nullptr ? MI : Sub;
5796 MachineBasicBlock *SubBB = Sub->getParent();
5797 // Move Movr0Inst to the appropriate place before Sub.
5798 if (Movr0Inst) {
5799 // Only move within the same block so we don't accidentally move to a
5800 // block with higher execution frequency.
5801 if (&CmpMBB != SubBB)
5802 return false;
5803 // Look backwards until we find a def that doesn't use the current EFLAGS.
5805 InsertE = Sub->getParent()->rend();
5806 for (; InsertI != InsertE; ++InsertI) {
5807 MachineInstr *Instr = &*InsertI;
5808 if (!Instr->readsRegister(X86::EFLAGS, TRI) &&
5809 Instr->modifiesRegister(X86::EFLAGS, TRI)) {
5810 Movr0Inst->getParent()->remove(Movr0Inst);
5811 Instr->getParent()->insert(MachineBasicBlock::iterator(Instr),
5812 Movr0Inst);
5813 break;
5814 }
5815 }
5816 if (InsertI == InsertE)
5817 return false;
5818 }
5819
5820 // Replace non-NF with NF instructions.
5821 for (auto &Inst : InstsToUpdate) {
5822 Inst.first->setDesc(get(Inst.second));
5823 Inst.first->removeOperand(
5824 Inst.first->findRegisterDefOperandIdx(X86::EFLAGS, /*TRI=*/nullptr));
5825 }
5826
5827 // Make sure Sub instruction defines EFLAGS and mark the def live.
5828 MachineOperand *FlagDef =
5829 Sub->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
5830 assert(FlagDef && "Unable to locate a def EFLAGS operand");
5831 FlagDef->setIsDead(false);
5832
5833 CmpInstr.eraseFromParent();
5834
5835 // Modify the condition code of instructions in OpsToUpdate.
5836 for (auto &Op : OpsToUpdate) {
5837 Op.first->getOperand(Op.first->getDesc().getNumOperands() - 1)
5838 .setImm(Op.second);
5839 }
5840 // Add EFLAGS to block live-ins between CmpBB and block of flags producer.
5841 // Walk the CFG backward from CmpMBB up to (but excluding) SubBB, marking
5842 // EFLAGS live-in on every block in between. SubBB dominates CmpMBB (whether
5843 // the producer was found by the single-predecessor backward walk or the
5844 // multi-predecessor dominator search), so the walk reaches SubBB on every
5845 // path and never escapes above it. A single-predecessor chain is just the
5846 // degenerate case where every block has exactly one predecessor.
5848 SmallVector<MachineBasicBlock *, 8> Worklist(1, &CmpMBB);
5849 Visited.insert(&CmpMBB);
5850 while (!Worklist.empty()) {
5851 MachineBasicBlock *MBB = Worklist.pop_back_val();
5852 // EFLAGS is produced inside SubBB, so it is not live-in there.
5853 if (MBB == SubBB)
5854 continue;
5855 if (!MBB->isLiveIn(X86::EFLAGS))
5856 MBB->addLiveIn(X86::EFLAGS);
5857 for (MachineBasicBlock *Pred : MBB->predecessors())
5858 if (Visited.insert(Pred).second)
5859 Worklist.push_back(Pred);
5860 }
5861 return true;
5862}
5863
5864/// \returns true if the instruction can be changed to COPY when imm is 0.
5865static bool canConvert2Copy(unsigned Opc) {
5866 switch (Opc) {
5867 default:
5868 return false;
5869 CASE_ND(ADD64ri32)
5870 CASE_ND(SUB64ri32)
5871 CASE_ND(OR64ri32)
5872 CASE_ND(XOR64ri32)
5873 CASE_ND(ADD32ri)
5874 CASE_ND(SUB32ri)
5875 CASE_ND(OR32ri)
5876 CASE_ND(XOR32ri)
5877 return true;
5878 }
5879}
5880
5881/// Convert an ALUrr opcode to corresponding ALUri opcode. Such as
5882/// ADD32rr ==> ADD32ri
5883static unsigned convertALUrr2ALUri(unsigned Opc) {
5884 switch (Opc) {
5885 default:
5886 return 0;
5887#define FROM_TO(FROM, TO) \
5888 case X86::FROM: \
5889 return X86::TO; \
5890 case X86::FROM##_ND: \
5891 return X86::TO##_ND;
5892 FROM_TO(ADC64rr, ADC64ri32)
5893 FROM_TO(SBB64rr, SBB64ri32)
5894 FROM_TO(AND64rr, AND64ri32)
5895 FROM_TO(OR64rr, OR64ri32)
5896 FROM_TO(XOR64rr, XOR64ri32)
5897 FROM_TO(SHR64rCL, SHR64ri)
5898 FROM_TO(SHL64rCL, SHL64ri)
5899 FROM_TO(SAR64rCL, SAR64ri)
5900 FROM_TO(ROL64rCL, ROL64ri)
5901 FROM_TO(ROR64rCL, ROR64ri)
5902 FROM_TO(RCL64rCL, RCL64ri)
5903 FROM_TO(RCR64rCL, RCR64ri)
5904 FROM_TO(ADD32rr, ADD32ri)
5905 FROM_TO(ADC32rr, ADC32ri)
5906 FROM_TO(SUB32rr, SUB32ri)
5907 FROM_TO(SBB32rr, SBB32ri)
5908 FROM_TO(AND32rr, AND32ri)
5909 FROM_TO(OR32rr, OR32ri)
5910 FROM_TO(XOR32rr, XOR32ri)
5911 FROM_TO(SHR32rCL, SHR32ri)
5912 FROM_TO(SHL32rCL, SHL32ri)
5913 FROM_TO(SAR32rCL, SAR32ri)
5914 FROM_TO(ROL32rCL, ROL32ri)
5915 FROM_TO(ROR32rCL, ROR32ri)
5916 FROM_TO(RCL32rCL, RCL32ri)
5917 FROM_TO(RCR32rCL, RCR32ri)
5918#undef FROM_TO
5919#define FROM_TO(FROM, TO) \
5920 case X86::FROM: \
5921 return X86::TO;
5922 FROM_TO(ADD64rr, ADD64ri32)
5923 FROM_TO(SUB64rr, SUB64ri32)
5924 FROM_TO(TEST64rr, TEST64ri32)
5925 FROM_TO(CTEST64rr, CTEST64ri32)
5926 FROM_TO(CMP64rr, CMP64ri32)
5927 FROM_TO(CCMP64rr, CCMP64ri32)
5928 FROM_TO(TEST32rr, TEST32ri)
5929 FROM_TO(CTEST32rr, CTEST32ri)
5930 FROM_TO(CMP32rr, CMP32ri)
5931 FROM_TO(CCMP32rr, CCMP32ri)
5932#undef FROM_TO
5933 case X86::ADD64rr_ND:
5934 return X86::ADD64ri32_ND;
5935 case X86::SUB64rr_ND:
5936 return X86::SUB64ri32_ND;
5937 }
5938}
5939
5940/// Reg is assigned ImmVal in DefMI, and is used in UseMI.
5941/// If MakeChange is true, this function tries to replace Reg by ImmVal in
5942/// UseMI. If MakeChange is false, just check if folding is possible.
5943//
5944/// \returns true if folding is successful or possible.
5945bool X86InstrInfo::foldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI,
5946 Register Reg, int64_t ImmVal,
5948 bool MakeChange) const {
5949 bool Modified = false;
5950
5951 // 64 bit operations accept sign extended 32 bit immediates.
5952 // 32 bit operations accept all 32 bit immediates, so we don't need to check
5953 // them.
5954 const TargetRegisterClass *RC = nullptr;
5955 if (Reg.isVirtual())
5956 RC = MRI->getRegClass(Reg);
5957 if ((Reg.isPhysical() && X86::GR64RegClass.contains(Reg)) ||
5958 (Reg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC))) {
5959 if (!isInt<32>(ImmVal))
5960 return false;
5961 }
5962
5963 if (UseMI.findRegisterUseOperand(Reg, /*TRI=*/nullptr)->getSubReg())
5964 return false;
5965 // Immediate has larger code size than register. So avoid folding the
5966 // immediate if it has more than 1 use and we are optimizing for size.
5967 if (UseMI.getMF()->getFunction().hasOptSize() && Reg.isVirtual() &&
5969 return false;
5970
5971 unsigned Opc = UseMI.getOpcode();
5972 unsigned NewOpc;
5973 if (Opc == TargetOpcode::COPY) {
5974 Register ToReg = UseMI.getOperand(0).getReg();
5975 const TargetRegisterClass *RC = nullptr;
5976 if (ToReg.isVirtual())
5977 RC = MRI->getRegClass(ToReg);
5978 bool GR32Reg = (ToReg.isVirtual() && X86::GR32RegClass.hasSubClassEq(RC)) ||
5979 (ToReg.isPhysical() && X86::GR32RegClass.contains(ToReg));
5980 bool GR64Reg = (ToReg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC)) ||
5981 (ToReg.isPhysical() && X86::GR64RegClass.contains(ToReg));
5982 bool GR8Reg = (ToReg.isVirtual() && X86::GR8RegClass.hasSubClassEq(RC)) ||
5983 (ToReg.isPhysical() && X86::GR8RegClass.contains(ToReg));
5984
5985 if (ImmVal == 0) {
5986 // We have MOV32r0 only.
5987 if (!GR32Reg)
5988 return false;
5989 }
5990
5991 if (GR64Reg) {
5992 if (isUInt<32>(ImmVal))
5993 NewOpc = X86::MOV32ri64;
5994 else
5995 NewOpc = X86::MOV64ri;
5996 } else if (GR32Reg) {
5997 NewOpc = X86::MOV32ri;
5998 if (ImmVal == 0) {
5999 // MOV32r0 clobbers EFLAGS.
6000 const TargetRegisterInfo *TRI = &getRegisterInfo();
6001 if (UseMI.getParent()->computeRegisterLiveness(
6002 TRI, X86::EFLAGS, UseMI) != MachineBasicBlock::LQR_Dead)
6003 return false;
6004
6005 // MOV32r0 is different than other cases because it doesn't encode the
6006 // immediate in the instruction. So we directly modify it here.
6007 if (!MakeChange)
6008 return true;
6009 UseMI.setDesc(get(X86::MOV32r0));
6010 UseMI.removeOperand(
6011 UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr));
6012 UseMI.addOperand(MachineOperand::CreateReg(X86::EFLAGS, /*isDef=*/true,
6013 /*isImp=*/true,
6014 /*isKill=*/false,
6015 /*isDead=*/true));
6016 Modified = true;
6017 }
6018 } else if (GR8Reg)
6019 NewOpc = X86::MOV8ri;
6020 else
6021 return false;
6022 } else
6023 NewOpc = convertALUrr2ALUri(Opc);
6024
6025 if (!NewOpc)
6026 return false;
6027
6028 // For SUB instructions the immediate can only be the second source operand.
6029 if ((NewOpc == X86::SUB64ri32 || NewOpc == X86::SUB32ri ||
6030 NewOpc == X86::SBB64ri32 || NewOpc == X86::SBB32ri ||
6031 NewOpc == X86::SUB64ri32_ND || NewOpc == X86::SUB32ri_ND ||
6032 NewOpc == X86::SBB64ri32_ND || NewOpc == X86::SBB32ri_ND) &&
6033 UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr) != 2)
6034 return false;
6035 // For CMP instructions the immediate can only be at index 1.
6036 if (((NewOpc == X86::CMP64ri32 || NewOpc == X86::CMP32ri) ||
6037 (NewOpc == X86::CCMP64ri32 || NewOpc == X86::CCMP32ri)) &&
6038 UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr) != 1)
6039 return false;
6040
6041 using namespace X86;
6042 if (isSHL(Opc) || isSHR(Opc) || isSAR(Opc) || isROL(Opc) || isROR(Opc) ||
6043 isRCL(Opc) || isRCR(Opc)) {
6044 unsigned RegIdx = UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr);
6045 if (RegIdx < 2)
6046 return false;
6047 if (!isInt<8>(ImmVal))
6048 return false;
6049 assert(Reg == X86::CL);
6050
6051 if (!MakeChange)
6052 return true;
6053 UseMI.setDesc(get(NewOpc));
6054 UseMI.removeOperand(RegIdx);
6055 UseMI.addOperand(MachineOperand::CreateImm(ImmVal));
6056 // Reg is physical register $cl, so we don't know if DefMI is dead through
6057 // MRI. Let the caller handle it, or pass dead-mi-elimination can delete
6058 // the dead physical register define instruction.
6059 return true;
6060 }
6061
6062 if (!MakeChange)
6063 return true;
6064
6065 if (!Modified) {
6066 // Modify the instruction.
6067 if (ImmVal == 0 && canConvert2Copy(NewOpc) &&
6068 UseMI.registerDefIsDead(X86::EFLAGS, /*TRI=*/nullptr)) {
6069 // %100 = add %101, 0
6070 // ==>
6071 // %100 = COPY %101
6072 UseMI.setDesc(get(TargetOpcode::COPY));
6073 UseMI.removeOperand(
6074 UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr));
6075 UseMI.removeOperand(
6076 UseMI.findRegisterDefOperandIdx(X86::EFLAGS, /*TRI=*/nullptr));
6077 UseMI.untieRegOperand(0);
6080 } else {
6081 unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex;
6082 unsigned ImmOpNum = 2;
6083 if (!UseMI.getOperand(0).isDef()) {
6084 Op1 = 0; // TEST, CMP, CTEST, CCMP
6085 ImmOpNum = 1;
6086 }
6087 if (Opc == TargetOpcode::COPY)
6088 ImmOpNum = 1;
6089 if (findCommutedOpIndices(UseMI, Op1, Op2) &&
6090 UseMI.getOperand(Op1).getReg() == Reg)
6091 commuteInstruction(UseMI);
6092
6093 assert(UseMI.getOperand(ImmOpNum).getReg() == Reg);
6094 UseMI.setDesc(get(NewOpc));
6095 UseMI.getOperand(ImmOpNum).ChangeToImmediate(ImmVal);
6096 }
6097 }
6098
6099 if (Reg.isVirtual() && MRI->use_nodbg_empty(Reg))
6101
6102 return true;
6103}
6104
6105/// foldImmediate - 'Reg' is known to be defined by a move immediate
6106/// instruction, try to fold the immediate into the use instruction.
6108 Register Reg, MachineRegisterInfo *MRI) const {
6109 int64_t ImmVal;
6110 if (!getConstValDefinedInReg(DefMI, Reg, ImmVal))
6111 return false;
6112
6113 return foldImmediateImpl(UseMI, &DefMI, Reg, ImmVal, MRI, true);
6114}
6115
6116/// Expand a single-def pseudo instruction to a two-addr
6117/// instruction with two undef reads of the register being defined.
6118/// This is used for mapping:
6119/// %xmm4 = V_SET0
6120/// to:
6121/// %xmm4 = PXORrr undef %xmm4, undef %xmm4
6122///
6124 const MCInstrDesc &Desc) {
6125 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
6126 Register Reg = MIB.getReg(0);
6127 MIB->setDesc(Desc);
6128
6129 // MachineInstr::addOperand() will insert explicit operands before any
6130 // implicit operands.
6132 // But we don't trust that.
6133 assert(MIB.getReg(1) == Reg && MIB.getReg(2) == Reg && "Misplaced operand");
6134 return true;
6135}
6136
6137/// Expand a single-def pseudo instruction to a two-addr
6138/// instruction with two %k0 reads.
6139/// This is used for mapping:
6140/// %k4 = K_SET1
6141/// to:
6142/// %k4 = KXNORrr %k0, %k0
6144 Register Reg) {
6145 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
6146 MIB->setDesc(Desc);
6148 return true;
6149}
6150
6152 bool MinusOne) {
6153 MachineBasicBlock &MBB = *MIB->getParent();
6154 const DebugLoc &DL = MIB->getDebugLoc();
6155 Register Reg = MIB.getReg(0);
6156
6157 // Insert the XOR.
6158 BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg)
6161
6162 // Turn the pseudo into an INC or DEC.
6163 MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r));
6164 MIB.addReg(Reg);
6165
6166 return true;
6167}
6168
6170 const TargetInstrInfo &TII,
6171 const X86Subtarget &Subtarget) {
6172 MachineBasicBlock &MBB = *MIB->getParent();
6173 const DebugLoc &DL = MIB->getDebugLoc();
6174 int64_t Imm = MIB->getOperand(1).getImm();
6175 assert(Imm != 0 && "Using push/pop for 0 is not efficient.");
6177
6178 int StackAdjustment;
6179
6180 if (Subtarget.is64Bit()) {
6181 assert(MIB->getOpcode() == X86::MOV64ImmSExti8 ||
6182 MIB->getOpcode() == X86::MOV32ImmSExti8);
6183
6184 // Can't use push/pop lowering if the function might write to the red zone.
6185 X86MachineFunctionInfo *X86FI =
6186 MBB.getParent()->getInfo<X86MachineFunctionInfo>();
6187 if (X86FI->getUsesRedZone()) {
6188 MIB->setDesc(TII.get(MIB->getOpcode() == X86::MOV32ImmSExti8
6189 ? X86::MOV32ri
6190 : X86::MOV64ri));
6191 return true;
6192 }
6193
6194 // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and
6195 // widen the register if necessary.
6196 StackAdjustment = 8;
6197 BuildMI(MBB, I, DL, TII.get(X86::PUSH64i32)).addImm(Imm);
6198 MIB->setDesc(TII.get(X86::POP64r));
6199 MIB->getOperand(0).setReg(getX86SubSuperRegister(MIB.getReg(0), 64));
6200 } else {
6201 assert(MIB->getOpcode() == X86::MOV32ImmSExti8);
6202 StackAdjustment = 4;
6203 BuildMI(MBB, I, DL, TII.get(X86::PUSH32i)).addImm(Imm);
6204 MIB->setDesc(TII.get(X86::POP32r));
6205 }
6206 MIB->removeOperand(1);
6207 MIB->addImplicitDefUseOperands(*MBB.getParent());
6208
6209 // Build CFI if necessary.
6210 MachineFunction &MF = *MBB.getParent();
6211 const X86FrameLowering *TFL = Subtarget.getFrameLowering();
6212 bool IsWin64Prologue = MF.usesWindowsUnwindInfo();
6213 bool NeedsDwarfCFI = !IsWin64Prologue && MF.needsFrameMoves();
6214 bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI;
6215 if (EmitCFI) {
6216 TFL->BuildCFI(
6217 MBB, I, DL,
6218 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment));
6219 TFL->BuildCFI(
6220 MBB, std::next(I), DL,
6221 MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment));
6222 }
6223
6224 return true;
6225}
6226
6227// LoadStackGuard has so far only been implemented for 64-bit MachO. Different
6228// code sequence is needed for other targets.
6230 const TargetInstrInfo &TII) {
6231 MachineBasicBlock &MBB = *MIB->getParent();
6232 const DebugLoc &DL = MIB->getDebugLoc();
6233 Register Reg = MIB.getReg(0);
6234 const GlobalValue *GV =
6235 cast<GlobalValue>((*MIB->memoperands_begin())->getValue());
6236 auto Flags = MachineMemOperand::MOLoad |
6239 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
6240 MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, Align(8));
6242
6243 BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg)
6244 .addReg(X86::RIP)
6245 .addImm(1)
6246 .addReg(0)
6248 .addReg(0)
6249 .addMemOperand(MMO);
6250 MIB->setDebugLoc(DL);
6251 MIB->setDesc(TII.get(X86::MOV64rm));
6253}
6254
6256 MachineBasicBlock &MBB = *MIB->getParent();
6257 MachineFunction &MF = *MBB.getParent();
6259 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
6260 unsigned XorOp =
6261 MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr;
6262 MIB->setDesc(TII.get(XorOp));
6263 MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef);
6264 return true;
6265}
6266
6267// This is used to handle spills for 128/256-bit registers when we have AVX512,
6268// but not VLX. If it uses an extended register we need to use an instruction
6269// that loads the lower 128/256-bit, but is available with only AVX512F.
6271 const TargetRegisterInfo *TRI,
6272 const MCInstrDesc &LoadDesc,
6273 const MCInstrDesc &BroadcastDesc, unsigned SubIdx) {
6274 Register DestReg = MIB.getReg(0);
6275 // Check if DestReg is XMM16-31 or YMM16-31.
6276 if (TRI->getEncodingValue(DestReg) < 16) {
6277 // We can use a normal VEX encoded load.
6278 MIB->setDesc(LoadDesc);
6279 } else {
6280 // Use a 128/256-bit VBROADCAST instruction.
6281 MIB->setDesc(BroadcastDesc);
6282 // Change the destination to a 512-bit register.
6283 DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass);
6284 MIB->getOperand(0).setReg(DestReg);
6285 }
6286 return true;
6287}
6288
6289// This is used to handle spills for 128/256-bit registers when we have AVX512,
6290// but not VLX. If it uses an extended register we need to use an instruction
6291// that stores the lower 128/256-bit, but is available with only AVX512F.
6293 const TargetRegisterInfo *TRI,
6294 const MCInstrDesc &StoreDesc,
6295 const MCInstrDesc &ExtractDesc, unsigned SubIdx) {
6296 Register SrcReg = MIB.getReg(X86::AddrNumOperands);
6297 // Check if DestReg is XMM16-31 or YMM16-31.
6298 if (TRI->getEncodingValue(SrcReg) < 16) {
6299 // We can use a normal VEX encoded store.
6300 MIB->setDesc(StoreDesc);
6301 } else {
6302 // Use a VEXTRACTF instruction.
6303 MIB->setDesc(ExtractDesc);
6304 // Change the destination to a 512-bit register.
6305 SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass);
6307 MIB.addImm(0x0); // Append immediate to extract from the lower bits.
6308 }
6309
6310 return true;
6311}
6312
6314 MIB->setDesc(Desc);
6315 int64_t ShiftAmt = MIB->getOperand(2).getImm();
6316 // Temporarily remove the immediate so we can add another source register.
6317 MIB->removeOperand(2);
6318 // Add the register. Don't copy the kill flag if there is one.
6319 MIB.addReg(MIB.getReg(1), getUndefRegState(MIB->getOperand(1).isUndef()));
6320 // Add back the immediate.
6321 MIB.addImm(ShiftAmt);
6322 return true;
6323}
6324
6326 const TargetInstrInfo &TII, bool HasAVX) {
6327 unsigned NewOpc;
6328 if (MI.getOpcode() == X86::MOVSHPrm) {
6329 NewOpc = HasAVX ? X86::VMOVSSrm : X86::MOVSSrm;
6330 Register Reg = MI.getOperand(0).getReg();
6331 if (Reg > X86::XMM15)
6332 NewOpc = X86::VMOVSSZrm;
6333 } else {
6334 NewOpc = HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
6335 Register Reg = MI.getOperand(5).getReg();
6336 if (Reg > X86::XMM15)
6337 NewOpc = X86::VMOVSSZmr;
6338 }
6339
6340 MIB->setDesc(TII.get(NewOpc));
6341 return true;
6342}
6343
6345 bool HasAVX = Subtarget.hasAVX();
6346 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
6347 switch (MI.getOpcode()) {
6348 case X86::MOV32r0:
6349 return Expand2AddrUndef(MIB, get(X86::XOR32rr));
6350 case X86::MOV32r1:
6351 return expandMOV32r1(MIB, *this, /*MinusOne=*/false);
6352 case X86::MOV32r_1:
6353 return expandMOV32r1(MIB, *this, /*MinusOne=*/true);
6354 case X86::MOV32ImmSExti8:
6355 case X86::MOV64ImmSExti8:
6356 return ExpandMOVImmSExti8(MIB, *this, Subtarget);
6357 case X86::SETB_C32r:
6358 return Expand2AddrUndef(MIB, get(X86::SBB32rr));
6359 case X86::SETB_C64r:
6360 return Expand2AddrUndef(MIB, get(X86::SBB64rr));
6361 case X86::MMX_SET0:
6362 return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr));
6363 case X86::V_SET0:
6364 case X86::FsFLD0SS:
6365 case X86::FsFLD0SD:
6366 case X86::FsFLD0SH:
6367 case X86::FsFLD0F128:
6368 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr));
6369 case X86::AVX512_128_SET0:
6370 case X86::AVX512_FsFLD0SH:
6371 case X86::AVX512_FsFLD0SS:
6372 case X86::AVX512_FsFLD0SD:
6373 case X86::AVX512_FsFLD0F128: {
6374 bool HasVLX = Subtarget.hasVLX();
6375 Register SrcReg = MIB.getReg(0);
6377 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16)
6378 return Expand2AddrUndef(MIB,
6379 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
6380 // Extended register without VLX. Use a larger XOR.
6381 SrcReg =
6382 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
6383 MIB->getOperand(0).setReg(SrcReg);
6384 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
6385 }
6386 case X86::MOVSHPmr:
6387 case X86::MOVSHPrm:
6388 return expandMOVSHP(MIB, MI, *this, Subtarget.hasAVX());
6389 case X86::V_SETALLONES:
6390 return Expand2AddrUndef(MIB,
6391 get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr));
6392 case X86::AVX2_SETALLONES:
6393 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr));
6394 case X86::AVX1_SETALLONES: {
6395 Register Reg = MIB.getReg(0);
6396 // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS.
6397 MIB->setDesc(get(X86::VCMPPSYrri));
6398 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf);
6399 return true;
6400 }
6401 case X86::AVX512_128_SETALLONES:
6402 case X86::AVX512_256_SETALLONES:
6403 case X86::AVX512_512_SETALLONES: {
6404 Register Reg = MIB.getReg(0);
6405 unsigned Opc;
6406 switch (MI.getOpcode()) {
6407 case X86::AVX512_128_SETALLONES: {
6408 if (X86::VR128RegClass.contains(Reg))
6409 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDrr));
6410
6411 Opc = X86::VPTERNLOGDZ128rri;
6412 break;
6413 }
6414 case X86::AVX512_256_SETALLONES: {
6415 if (X86::VR256RegClass.contains(Reg))
6416 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr));
6417
6418 Opc = X86::VPTERNLOGDZ256rri;
6419 break;
6420 }
6421 case X86::AVX512_512_SETALLONES:
6422 Opc = X86::VPTERNLOGDZrri;
6423 break;
6424 }
6425 MIB->setDesc(get(Opc));
6426 // VPTERNLOGD needs 3 register inputs and an immediate.
6427 // 0xff will return 1s for any input.
6428 MIB.addReg(Reg, RegState::Undef)
6429 .addReg(Reg, RegState::Undef)
6430 .addReg(Reg, RegState::Undef)
6431 .addImm(0xff);
6432 return true;
6433 }
6434 case X86::AVX512_512_SEXT_MASK_32:
6435 case X86::AVX512_512_SEXT_MASK_64: {
6436 Register Reg = MIB.getReg(0);
6437 Register MaskReg = MIB.getReg(1);
6438 RegState MaskState = getRegState(MIB->getOperand(1));
6439 unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64)
6440 ? X86::VPTERNLOGQZrrikz
6441 : X86::VPTERNLOGDZrrikz;
6442 MI.removeOperand(1);
6443 MIB->setDesc(get(Opc));
6444 // VPTERNLOG needs 3 register inputs and an immediate.
6445 // 0xff will return 1s for any input.
6446 MIB.addReg(Reg, RegState::Undef)
6447 .addReg(MaskReg, MaskState)
6448 .addReg(Reg, RegState::Undef)
6449 .addReg(Reg, RegState::Undef)
6450 .addImm(0xff);
6451 return true;
6452 }
6453 case X86::VMOVAPSZ128rm_NOVLX:
6454 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm),
6455 get(X86::VBROADCASTF32X4Zrm), X86::sub_xmm);
6456 case X86::VMOVUPSZ128rm_NOVLX:
6457 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm),
6458 get(X86::VBROADCASTF32X4Zrm), X86::sub_xmm);
6459 case X86::VMOVAPSZ256rm_NOVLX:
6460 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm),
6461 get(X86::VBROADCASTF64X4Zrm), X86::sub_ymm);
6462 case X86::VMOVUPSZ256rm_NOVLX:
6463 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm),
6464 get(X86::VBROADCASTF64X4Zrm), X86::sub_ymm);
6465 case X86::VMOVAPSZ128mr_NOVLX:
6466 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr),
6467 get(X86::VEXTRACTF32X4Zmri), X86::sub_xmm);
6468 case X86::VMOVUPSZ128mr_NOVLX:
6469 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr),
6470 get(X86::VEXTRACTF32X4Zmri), X86::sub_xmm);
6471 case X86::VMOVAPSZ256mr_NOVLX:
6472 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr),
6473 get(X86::VEXTRACTF64X4Zmri), X86::sub_ymm);
6474 case X86::VMOVUPSZ256mr_NOVLX:
6475 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr),
6476 get(X86::VEXTRACTF64X4Zmri), X86::sub_ymm);
6477 case X86::MOV32ri64: {
6478 MachineOperand &DefMO = MIB->getOperand(0);
6479 Register Reg32 = RI.getSubReg(DefMO.getReg(), X86::sub_32bit);
6480 MI.setDesc(get(X86::MOV32ri));
6481 DefMO.setReg(Reg32);
6482 DefMO.setIsDead(none_of(MI.all_defs(), [=](const MachineOperand &MO) {
6483 return MO.getReg() == Reg32 && !MO.isDead();
6484 }));
6485 return true;
6486 }
6487
6488 case X86::RDFLAGS32:
6489 case X86::RDFLAGS64: {
6490 unsigned Is64Bit = MI.getOpcode() == X86::RDFLAGS64;
6491 MachineBasicBlock &MBB = *MIB->getParent();
6492
6493 MachineInstr *NewMI = BuildMI(MBB, MI, MIB->getDebugLoc(),
6494 get(Is64Bit ? X86::PUSHF64 : X86::PUSHF32))
6495 .getInstr();
6496
6497 // Permit reads of the EFLAGS and DF registers without them being defined.
6498 // This intrinsic exists to read external processor state in flags, such as
6499 // the trap flag, interrupt flag, and direction flag, none of which are
6500 // modeled by the backend.
6501 assert(NewMI->getOperand(2).getReg() == X86::EFLAGS &&
6502 "Unexpected register in operand! Should be EFLAGS.");
6503 NewMI->getOperand(2).setIsUndef();
6504 assert(NewMI->getOperand(3).getReg() == X86::DF &&
6505 "Unexpected register in operand! Should be DF.");
6506 NewMI->getOperand(3).setIsUndef();
6507
6508 MIB->setDesc(get(Is64Bit ? X86::POP64r : X86::POP32r));
6509 return true;
6510 }
6511
6512 case X86::WRFLAGS32:
6513 case X86::WRFLAGS64: {
6514 unsigned Is64Bit = MI.getOpcode() == X86::WRFLAGS64;
6515 MachineBasicBlock &MBB = *MIB->getParent();
6516
6517 BuildMI(MBB, MI, MIB->getDebugLoc(),
6518 get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
6519 .addReg(MI.getOperand(0).getReg());
6520 BuildMI(MBB, MI, MIB->getDebugLoc(),
6521 get(Is64Bit ? X86::POPF64 : X86::POPF32));
6522 MI.eraseFromParent();
6523 return true;
6524 }
6525
6526 // KNL does not recognize dependency-breaking idioms for mask registers,
6527 // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1.
6528 // Using %k0 as the undef input register is a performance heuristic based
6529 // on the assumption that %k0 is used less frequently than the other mask
6530 // registers, since it is not usable as a write mask.
6531 // FIXME: A more advanced approach would be to choose the best input mask
6532 // register based on context.
6533 case X86::KSET0B:
6534 return Expand2AddrKreg(MIB, get(X86::KXORBkk), X86::K0);
6535 case X86::KSET0W:
6536 return Expand2AddrKreg(MIB, get(X86::KXORWkk), X86::K0);
6537 case X86::KSET0D:
6538 return Expand2AddrKreg(MIB, get(X86::KXORDkk), X86::K0);
6539 case X86::KSET0Q:
6540 return Expand2AddrKreg(MIB, get(X86::KXORQkk), X86::K0);
6541 case X86::KSET1B:
6542 return Expand2AddrKreg(MIB, get(X86::KXNORBkk), X86::K0);
6543 case X86::KSET1W:
6544 return Expand2AddrKreg(MIB, get(X86::KXNORWkk), X86::K0);
6545 case X86::KSET1D:
6546 return Expand2AddrKreg(MIB, get(X86::KXNORDkk), X86::K0);
6547 case X86::KSET1Q:
6548 return Expand2AddrKreg(MIB, get(X86::KXNORQkk), X86::K0);
6549 case TargetOpcode::LOAD_STACK_GUARD:
6550 expandLoadStackGuard(MIB, *this);
6551 return true;
6552 case X86::XOR64_FP:
6553 case X86::XOR32_FP:
6554 return expandXorFP(MIB, *this);
6555 case X86::SHLDROT32ri:
6556 return expandSHXDROT(MIB, get(X86::SHLD32rri8));
6557 case X86::SHLDROT64ri:
6558 return expandSHXDROT(MIB, get(X86::SHLD64rri8));
6559 case X86::SHRDROT32ri:
6560 return expandSHXDROT(MIB, get(X86::SHRD32rri8));
6561 case X86::SHRDROT64ri:
6562 return expandSHXDROT(MIB, get(X86::SHRD64rri8));
6563 case X86::ADD8rr_DB:
6564 MIB->setDesc(get(X86::OR8rr));
6565 break;
6566 case X86::ADD16rr_DB:
6567 MIB->setDesc(get(X86::OR16rr));
6568 break;
6569 case X86::ADD32rr_DB:
6570 MIB->setDesc(get(X86::OR32rr));
6571 break;
6572 case X86::ADD64rr_DB:
6573 MIB->setDesc(get(X86::OR64rr));
6574 break;
6575 case X86::ADD8ri_DB:
6576 MIB->setDesc(get(X86::OR8ri));
6577 break;
6578 case X86::ADD16ri_DB:
6579 MIB->setDesc(get(X86::OR16ri));
6580 break;
6581 case X86::ADD32ri_DB:
6582 MIB->setDesc(get(X86::OR32ri));
6583 break;
6584 case X86::ADD64ri32_DB:
6585 MIB->setDesc(get(X86::OR64ri32));
6586 break;
6587 }
6588 return false;
6589}
6590
6591/// Return true for all instructions that only update
6592/// the first 32 or 64-bits of the destination register and leave the rest
6593/// unmodified. This can be used to avoid folding loads if the instructions
6594/// only update part of the destination register, and the non-updated part is
6595/// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these
6596/// instructions breaks the partial register dependency and it can improve
6597/// performance. e.g.:
6598///
6599/// movss (%rdi), %xmm0
6600/// cvtss2sd %xmm0, %xmm0
6601///
6602/// Instead of
6603/// cvtss2sd (%rdi), %xmm0
6604///
6605/// FIXME: This should be turned into a TSFlags.
6606///
6607static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget,
6608 bool ForLoadFold = false) {
6609 switch (Opcode) {
6610 case X86::CVTSI2SSrr:
6611 case X86::CVTSI2SSrm:
6612 case X86::CVTSI642SSrr:
6613 case X86::CVTSI642SSrm:
6614 case X86::CVTSI2SDrr:
6615 case X86::CVTSI2SDrm:
6616 case X86::CVTSI642SDrr:
6617 case X86::CVTSI642SDrm:
6618 // Load folding won't effect the undef register update since the input is
6619 // a GPR.
6620 return !ForLoadFold;
6621 case X86::CVTSD2SSrr:
6622 case X86::CVTSD2SSrm:
6623 case X86::CVTSS2SDrr:
6624 case X86::CVTSS2SDrm:
6625 case X86::MOVHPDrm:
6626 case X86::MOVHPSrm:
6627 case X86::MOVLPDrm:
6628 case X86::MOVLPSrm:
6629 case X86::RCPSSr:
6630 case X86::RCPSSm:
6631 case X86::RCPSSr_Int:
6632 case X86::RCPSSm_Int:
6633 case X86::ROUNDSDri:
6634 case X86::ROUNDSDmi:
6635 case X86::ROUNDSSri:
6636 case X86::ROUNDSSmi:
6637 case X86::RSQRTSSr:
6638 case X86::RSQRTSSm:
6639 case X86::RSQRTSSr_Int:
6640 case X86::RSQRTSSm_Int:
6641 case X86::SQRTSSr:
6642 case X86::SQRTSSm:
6643 case X86::SQRTSSr_Int:
6644 case X86::SQRTSSm_Int:
6645 case X86::SQRTSDr:
6646 case X86::SQRTSDm:
6647 case X86::SQRTSDr_Int:
6648 case X86::SQRTSDm_Int:
6649 return true;
6650 case X86::VFCMULCPHZ128rm:
6651 case X86::VFCMULCPHZ128rmb:
6652 case X86::VFCMULCPHZ128rmbkz:
6653 case X86::VFCMULCPHZ128rmkz:
6654 case X86::VFCMULCPHZ128rr:
6655 case X86::VFCMULCPHZ128rrkz:
6656 case X86::VFCMULCPHZ256rm:
6657 case X86::VFCMULCPHZ256rmb:
6658 case X86::VFCMULCPHZ256rmbkz:
6659 case X86::VFCMULCPHZ256rmkz:
6660 case X86::VFCMULCPHZ256rr:
6661 case X86::VFCMULCPHZ256rrkz:
6662 case X86::VFCMULCPHZrm:
6663 case X86::VFCMULCPHZrmb:
6664 case X86::VFCMULCPHZrmbkz:
6665 case X86::VFCMULCPHZrmkz:
6666 case X86::VFCMULCPHZrr:
6667 case X86::VFCMULCPHZrrb:
6668 case X86::VFCMULCPHZrrbkz:
6669 case X86::VFCMULCPHZrrkz:
6670 case X86::VFMULCPHZ128rm:
6671 case X86::VFMULCPHZ128rmb:
6672 case X86::VFMULCPHZ128rmbkz:
6673 case X86::VFMULCPHZ128rmkz:
6674 case X86::VFMULCPHZ128rr:
6675 case X86::VFMULCPHZ128rrkz:
6676 case X86::VFMULCPHZ256rm:
6677 case X86::VFMULCPHZ256rmb:
6678 case X86::VFMULCPHZ256rmbkz:
6679 case X86::VFMULCPHZ256rmkz:
6680 case X86::VFMULCPHZ256rr:
6681 case X86::VFMULCPHZ256rrkz:
6682 case X86::VFMULCPHZrm:
6683 case X86::VFMULCPHZrmb:
6684 case X86::VFMULCPHZrmbkz:
6685 case X86::VFMULCPHZrmkz:
6686 case X86::VFMULCPHZrr:
6687 case X86::VFMULCPHZrrb:
6688 case X86::VFMULCPHZrrbkz:
6689 case X86::VFMULCPHZrrkz:
6690 case X86::VFCMULCSHZrm:
6691 case X86::VFCMULCSHZrmkz:
6692 case X86::VFCMULCSHZrr:
6693 case X86::VFCMULCSHZrrb:
6694 case X86::VFCMULCSHZrrbkz:
6695 case X86::VFCMULCSHZrrkz:
6696 case X86::VFMULCSHZrm:
6697 case X86::VFMULCSHZrmkz:
6698 case X86::VFMULCSHZrr:
6699 case X86::VFMULCSHZrrb:
6700 case X86::VFMULCSHZrrbkz:
6701 case X86::VFMULCSHZrrkz:
6702 return Subtarget.hasMULCFalseDeps();
6703 case X86::VPERMDYrm:
6704 case X86::VPERMDYrr:
6705 case X86::VPERMQYmi:
6706 case X86::VPERMQYri:
6707 case X86::VPERMPSYrm:
6708 case X86::VPERMPSYrr:
6709 case X86::VPERMPDYmi:
6710 case X86::VPERMPDYri:
6711 case X86::VPERMDZ256rm:
6712 case X86::VPERMDZ256rmb:
6713 case X86::VPERMDZ256rmbkz:
6714 case X86::VPERMDZ256rmkz:
6715 case X86::VPERMDZ256rr:
6716 case X86::VPERMDZ256rrkz:
6717 case X86::VPERMDZrm:
6718 case X86::VPERMDZrmb:
6719 case X86::VPERMDZrmbkz:
6720 case X86::VPERMDZrmkz:
6721 case X86::VPERMDZrr:
6722 case X86::VPERMDZrrkz:
6723 case X86::VPERMQZ256mbi:
6724 case X86::VPERMQZ256mbikz:
6725 case X86::VPERMQZ256mi:
6726 case X86::VPERMQZ256mikz:
6727 case X86::VPERMQZ256ri:
6728 case X86::VPERMQZ256rikz:
6729 case X86::VPERMQZ256rm:
6730 case X86::VPERMQZ256rmb:
6731 case X86::VPERMQZ256rmbkz:
6732 case X86::VPERMQZ256rmkz:
6733 case X86::VPERMQZ256rr:
6734 case X86::VPERMQZ256rrkz:
6735 case X86::VPERMQZmbi:
6736 case X86::VPERMQZmbikz:
6737 case X86::VPERMQZmi:
6738 case X86::VPERMQZmikz:
6739 case X86::VPERMQZri:
6740 case X86::VPERMQZrikz:
6741 case X86::VPERMQZrm:
6742 case X86::VPERMQZrmb:
6743 case X86::VPERMQZrmbkz:
6744 case X86::VPERMQZrmkz:
6745 case X86::VPERMQZrr:
6746 case X86::VPERMQZrrkz:
6747 case X86::VPERMPSZ256rm:
6748 case X86::VPERMPSZ256rmb:
6749 case X86::VPERMPSZ256rmbkz:
6750 case X86::VPERMPSZ256rmkz:
6751 case X86::VPERMPSZ256rr:
6752 case X86::VPERMPSZ256rrkz:
6753 case X86::VPERMPSZrm:
6754 case X86::VPERMPSZrmb:
6755 case X86::VPERMPSZrmbkz:
6756 case X86::VPERMPSZrmkz:
6757 case X86::VPERMPSZrr:
6758 case X86::VPERMPSZrrkz:
6759 case X86::VPERMPDZ256mbi:
6760 case X86::VPERMPDZ256mbikz:
6761 case X86::VPERMPDZ256mi:
6762 case X86::VPERMPDZ256mikz:
6763 case X86::VPERMPDZ256ri:
6764 case X86::VPERMPDZ256rikz:
6765 case X86::VPERMPDZ256rm:
6766 case X86::VPERMPDZ256rmb:
6767 case X86::VPERMPDZ256rmbkz:
6768 case X86::VPERMPDZ256rmkz:
6769 case X86::VPERMPDZ256rr:
6770 case X86::VPERMPDZ256rrkz:
6771 case X86::VPERMPDZmbi:
6772 case X86::VPERMPDZmbikz:
6773 case X86::VPERMPDZmi:
6774 case X86::VPERMPDZmikz:
6775 case X86::VPERMPDZri:
6776 case X86::VPERMPDZrikz:
6777 case X86::VPERMPDZrm:
6778 case X86::VPERMPDZrmb:
6779 case X86::VPERMPDZrmbkz:
6780 case X86::VPERMPDZrmkz:
6781 case X86::VPERMPDZrr:
6782 case X86::VPERMPDZrrkz:
6783 return Subtarget.hasPERMFalseDeps();
6784 case X86::VRANGEPDZ128rmbi:
6785 case X86::VRANGEPDZ128rmbikz:
6786 case X86::VRANGEPDZ128rmi:
6787 case X86::VRANGEPDZ128rmikz:
6788 case X86::VRANGEPDZ128rri:
6789 case X86::VRANGEPDZ128rrikz:
6790 case X86::VRANGEPDZ256rmbi:
6791 case X86::VRANGEPDZ256rmbikz:
6792 case X86::VRANGEPDZ256rmi:
6793 case X86::VRANGEPDZ256rmikz:
6794 case X86::VRANGEPDZ256rri:
6795 case X86::VRANGEPDZ256rrikz:
6796 case X86::VRANGEPDZrmbi:
6797 case X86::VRANGEPDZrmbikz:
6798 case X86::VRANGEPDZrmi:
6799 case X86::VRANGEPDZrmikz:
6800 case X86::VRANGEPDZrri:
6801 case X86::VRANGEPDZrrib:
6802 case X86::VRANGEPDZrribkz:
6803 case X86::VRANGEPDZrrikz:
6804 case X86::VRANGEPSZ128rmbi:
6805 case X86::VRANGEPSZ128rmbikz:
6806 case X86::VRANGEPSZ128rmi:
6807 case X86::VRANGEPSZ128rmikz:
6808 case X86::VRANGEPSZ128rri:
6809 case X86::VRANGEPSZ128rrikz:
6810 case X86::VRANGEPSZ256rmbi:
6811 case X86::VRANGEPSZ256rmbikz:
6812 case X86::VRANGEPSZ256rmi:
6813 case X86::VRANGEPSZ256rmikz:
6814 case X86::VRANGEPSZ256rri:
6815 case X86::VRANGEPSZ256rrikz:
6816 case X86::VRANGEPSZrmbi:
6817 case X86::VRANGEPSZrmbikz:
6818 case X86::VRANGEPSZrmi:
6819 case X86::VRANGEPSZrmikz:
6820 case X86::VRANGEPSZrri:
6821 case X86::VRANGEPSZrrib:
6822 case X86::VRANGEPSZrribkz:
6823 case X86::VRANGEPSZrrikz:
6824 case X86::VRANGESDZrmi:
6825 case X86::VRANGESDZrmikz:
6826 case X86::VRANGESDZrri:
6827 case X86::VRANGESDZrrib:
6828 case X86::VRANGESDZrribkz:
6829 case X86::VRANGESDZrrikz:
6830 case X86::VRANGESSZrmi:
6831 case X86::VRANGESSZrmikz:
6832 case X86::VRANGESSZrri:
6833 case X86::VRANGESSZrrib:
6834 case X86::VRANGESSZrribkz:
6835 case X86::VRANGESSZrrikz:
6836 return Subtarget.hasRANGEFalseDeps();
6837 case X86::VGETMANTSSZrmi:
6838 case X86::VGETMANTSSZrmikz:
6839 case X86::VGETMANTSSZrri:
6840 case X86::VGETMANTSSZrrib:
6841 case X86::VGETMANTSSZrribkz:
6842 case X86::VGETMANTSSZrrikz:
6843 case X86::VGETMANTSDZrmi:
6844 case X86::VGETMANTSDZrmikz:
6845 case X86::VGETMANTSDZrri:
6846 case X86::VGETMANTSDZrrib:
6847 case X86::VGETMANTSDZrribkz:
6848 case X86::VGETMANTSDZrrikz:
6849 case X86::VGETMANTSHZrmi:
6850 case X86::VGETMANTSHZrmikz:
6851 case X86::VGETMANTSHZrri:
6852 case X86::VGETMANTSHZrrib:
6853 case X86::VGETMANTSHZrribkz:
6854 case X86::VGETMANTSHZrrikz:
6855 case X86::VGETMANTPSZ128rmbi:
6856 case X86::VGETMANTPSZ128rmbikz:
6857 case X86::VGETMANTPSZ128rmi:
6858 case X86::VGETMANTPSZ128rmikz:
6859 case X86::VGETMANTPSZ256rmbi:
6860 case X86::VGETMANTPSZ256rmbikz:
6861 case X86::VGETMANTPSZ256rmi:
6862 case X86::VGETMANTPSZ256rmikz:
6863 case X86::VGETMANTPSZrmbi:
6864 case X86::VGETMANTPSZrmbikz:
6865 case X86::VGETMANTPSZrmi:
6866 case X86::VGETMANTPSZrmikz:
6867 case X86::VGETMANTPDZ128rmbi:
6868 case X86::VGETMANTPDZ128rmbikz:
6869 case X86::VGETMANTPDZ128rmi:
6870 case X86::VGETMANTPDZ128rmikz:
6871 case X86::VGETMANTPDZ256rmbi:
6872 case X86::VGETMANTPDZ256rmbikz:
6873 case X86::VGETMANTPDZ256rmi:
6874 case X86::VGETMANTPDZ256rmikz:
6875 case X86::VGETMANTPDZrmbi:
6876 case X86::VGETMANTPDZrmbikz:
6877 case X86::VGETMANTPDZrmi:
6878 case X86::VGETMANTPDZrmikz:
6879 return Subtarget.hasGETMANTFalseDeps();
6880 case X86::VPMULLQZ128rm:
6881 case X86::VPMULLQZ128rmb:
6882 case X86::VPMULLQZ128rmbkz:
6883 case X86::VPMULLQZ128rmkz:
6884 case X86::VPMULLQZ128rr:
6885 case X86::VPMULLQZ128rrkz:
6886 case X86::VPMULLQZ256rm:
6887 case X86::VPMULLQZ256rmb:
6888 case X86::VPMULLQZ256rmbkz:
6889 case X86::VPMULLQZ256rmkz:
6890 case X86::VPMULLQZ256rr:
6891 case X86::VPMULLQZ256rrkz:
6892 case X86::VPMULLQZrm:
6893 case X86::VPMULLQZrmb:
6894 case X86::VPMULLQZrmbkz:
6895 case X86::VPMULLQZrmkz:
6896 case X86::VPMULLQZrr:
6897 case X86::VPMULLQZrrkz:
6898 return Subtarget.hasMULLQFalseDeps();
6899 case X86::VPCOMPRESSBZ128rrkz:
6900 case X86::VPCOMPRESSBZ256rrkz:
6901 case X86::VPCOMPRESSBZrrkz:
6902 case X86::VPCOMPRESSWZ128rrkz:
6903 case X86::VPCOMPRESSWZ256rrkz:
6904 case X86::VPCOMPRESSWZrrkz:
6905 case X86::VPCOMPRESSDZ128rrkz:
6906 case X86::VPCOMPRESSDZ256rrkz:
6907 case X86::VPCOMPRESSDZrrkz:
6908 case X86::VPCOMPRESSQZ128rrkz:
6909 case X86::VPCOMPRESSQZ256rrkz:
6910 case X86::VPCOMPRESSQZrrkz:
6911 case X86::VCOMPRESSPSZ128rrkz:
6912 case X86::VCOMPRESSPSZ256rrkz:
6913 case X86::VCOMPRESSPSZrrkz:
6914 case X86::VCOMPRESSPDZ128rrkz:
6915 case X86::VCOMPRESSPDZ256rrkz:
6916 case X86::VCOMPRESSPDZrrkz:
6917 return Subtarget.hasCOMPRESSFalseDeps();
6918 case X86::VPEXPANDBZ128rmkz:
6919 case X86::VPEXPANDBZ128rrkz:
6920 case X86::VPEXPANDBZ256rmkz:
6921 case X86::VPEXPANDBZ256rrkz:
6922 case X86::VPEXPANDBZrmkz:
6923 case X86::VPEXPANDBZrrkz:
6924 case X86::VPEXPANDWZ128rmkz:
6925 case X86::VPEXPANDWZ128rrkz:
6926 case X86::VPEXPANDWZ256rmkz:
6927 case X86::VPEXPANDWZ256rrkz:
6928 case X86::VPEXPANDWZrmkz:
6929 case X86::VPEXPANDWZrrkz:
6930 case X86::VPEXPANDDZ128rmkz:
6931 case X86::VPEXPANDDZ128rrkz:
6932 case X86::VPEXPANDDZ256rmkz:
6933 case X86::VPEXPANDDZ256rrkz:
6934 case X86::VPEXPANDDZrmkz:
6935 case X86::VPEXPANDDZrrkz:
6936 case X86::VPEXPANDQZ128rmkz:
6937 case X86::VPEXPANDQZ128rrkz:
6938 case X86::VPEXPANDQZ256rmkz:
6939 case X86::VPEXPANDQZ256rrkz:
6940 case X86::VPEXPANDQZrmkz:
6941 case X86::VPEXPANDQZrrkz:
6942 case X86::VEXPANDPSZ128rmkz:
6943 case X86::VEXPANDPSZ128rrkz:
6944 case X86::VEXPANDPSZ256rmkz:
6945 case X86::VEXPANDPSZ256rrkz:
6946 case X86::VEXPANDPSZrmkz:
6947 case X86::VEXPANDPSZrrkz:
6948 case X86::VEXPANDPDZ128rmkz:
6949 case X86::VEXPANDPDZ128rrkz:
6950 case X86::VEXPANDPDZ256rmkz:
6951 case X86::VEXPANDPDZ256rrkz:
6952 case X86::VEXPANDPDZrmkz:
6953 case X86::VEXPANDPDZrrkz:
6954 return Subtarget.hasEXPANDFalseDeps();
6955 // GPR
6956 case X86::POPCNT32rm:
6957 case X86::POPCNT32rr:
6958 case X86::POPCNT64rm:
6959 case X86::POPCNT64rr:
6960 return Subtarget.hasPOPCNTFalseDeps();
6961 case X86::LZCNT32rm:
6962 case X86::LZCNT32rr:
6963 case X86::LZCNT64rm:
6964 case X86::LZCNT64rr:
6965 return Subtarget.hasLZCNTFalseDeps();
6966 case X86::TZCNT32rm:
6967 case X86::TZCNT32rr:
6968 case X86::TZCNT64rm:
6969 case X86::TZCNT64rr:
6970 return Subtarget.hasTZCNTFalseDeps();
6971 case X86::BLSR32rr:
6972 case X86::BLSR32rm:
6973 case X86::BLSR64rr:
6974 case X86::BLSR64rm:
6975 case X86::BLSI32rr:
6976 case X86::BLSI32rm:
6977 case X86::BLSI64rr:
6978 case X86::BLSI64rm:
6979 case X86::BLSMSK32rr:
6980 case X86::BLSMSK32rm:
6981 case X86::BLSMSK64rr:
6982 case X86::BLSMSK64rm:
6983 return Subtarget.hasBLSFalseDeps() && !ForLoadFold; // Preserve load folding
6984 }
6985
6986 return false;
6987}
6988
6989/// Inform the BreakFalseDeps pass how many idle
6990/// instructions we would like before a partial register update.
6992 unsigned OpNum) const {
6993
6994 if (OpNum != 0)
6995 return 0;
6996
6997 // NDD ops with 8/16b results may appear to be partial register
6998 // updates after register allocation.
6999 bool HasNDDPartialWrite = false;
7000 if (X86II::hasNewDataDest(MI.getDesc().TSFlags)) {
7001 Register Reg = MI.getOperand(0).getReg();
7002 if (!Reg.isVirtual())
7003 HasNDDPartialWrite =
7004 X86::GR8RegClass.contains(Reg) || X86::GR16RegClass.contains(Reg);
7005 }
7006
7007 if (!(HasNDDPartialWrite || hasPartialRegUpdate(MI.getOpcode(), Subtarget)))
7008 return 0;
7009
7010 // Check if the result register is also used as a source.
7011 // For non-NDD ops, this means a partial update is wanted, hence we return 0.
7012 // For NDD ops, this means it is possible to compress the instruction
7013 // to a legacy form in CompressEVEX, which would create an unwanted partial
7014 // update, so we return the clearance.
7015 const MachineOperand &MO = MI.getOperand(0);
7016 Register Reg = MO.getReg();
7017 bool ReadsReg = false;
7018 if (Reg.isVirtual())
7019 ReadsReg = (MO.readsReg() || MI.readsVirtualRegister(Reg));
7020 else
7021 ReadsReg = MI.readsRegister(Reg, &RI);
7022 if (ReadsReg != HasNDDPartialWrite)
7023 return 0;
7024
7025 // If any instructions in the clearance range are reading Reg, insert a
7026 // dependency breaking instruction, which is inexpensive and is likely to
7027 // be hidden in other instruction's cycles.
7028 return Subtarget.getCLOpts().partial_reg_update_clearance;
7029}
7030
7031// Return true for any instruction the copies the high bits of the first source
7032// operand into the unused high bits of the destination operand.
7033// Also returns true for instructions that have two inputs where one may
7034// be undef and we want it to use the same register as the other input.
7035static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum,
7036 bool ForLoadFold = false) {
7037 // Set the OpNum parameter to the first source operand.
7038 switch (Opcode) {
7039 case X86::MMX_PUNPCKHBWrr:
7040 case X86::MMX_PUNPCKHWDrr:
7041 case X86::MMX_PUNPCKHDQrr:
7042 case X86::MMX_PUNPCKLBWrr:
7043 case X86::MMX_PUNPCKLWDrr:
7044 case X86::MMX_PUNPCKLDQrr:
7045 case X86::MOVHLPSrr:
7046 case X86::PACKSSWBrr:
7047 case X86::PACKUSWBrr:
7048 case X86::PACKSSDWrr:
7049 case X86::PACKUSDWrr:
7050 case X86::PUNPCKHBWrr:
7051 case X86::PUNPCKLBWrr:
7052 case X86::PUNPCKHWDrr:
7053 case X86::PUNPCKLWDrr:
7054 case X86::PUNPCKHDQrr:
7055 case X86::PUNPCKLDQrr:
7056 case X86::PUNPCKHQDQrr:
7057 case X86::PUNPCKLQDQrr:
7058 case X86::SHUFPDrri:
7059 case X86::SHUFPSrri:
7060 // These instructions are sometimes used with an undef first or second
7061 // source. Return true here so BreakFalseDeps will assign this source to the
7062 // same register as the first source to avoid a false dependency.
7063 // Operand 1 of these instructions is tied so they're separate from their
7064 // VEX counterparts.
7065 return OpNum == 2 && !ForLoadFold;
7066
7067 case X86::VMOVLHPSrr:
7068 case X86::VMOVLHPSZrr:
7069 case X86::VPACKSSWBrr:
7070 case X86::VPACKUSWBrr:
7071 case X86::VPACKSSDWrr:
7072 case X86::VPACKUSDWrr:
7073 case X86::VPACKSSWBZ128rr:
7074 case X86::VPACKUSWBZ128rr:
7075 case X86::VPACKSSDWZ128rr:
7076 case X86::VPACKUSDWZ128rr:
7077 case X86::VPERM2F128rri:
7078 case X86::VPERM2I128rri:
7079 case X86::VSHUFF32X4Z256rri:
7080 case X86::VSHUFF32X4Zrri:
7081 case X86::VSHUFF64X2Z256rri:
7082 case X86::VSHUFF64X2Zrri:
7083 case X86::VSHUFI32X4Z256rri:
7084 case X86::VSHUFI32X4Zrri:
7085 case X86::VSHUFI64X2Z256rri:
7086 case X86::VSHUFI64X2Zrri:
7087 case X86::VPUNPCKHBWrr:
7088 case X86::VPUNPCKLBWrr:
7089 case X86::VPUNPCKHBWYrr:
7090 case X86::VPUNPCKLBWYrr:
7091 case X86::VPUNPCKHBWZ128rr:
7092 case X86::VPUNPCKLBWZ128rr:
7093 case X86::VPUNPCKHBWZ256rr:
7094 case X86::VPUNPCKLBWZ256rr:
7095 case X86::VPUNPCKHBWZrr:
7096 case X86::VPUNPCKLBWZrr:
7097 case X86::VPUNPCKHWDrr:
7098 case X86::VPUNPCKLWDrr:
7099 case X86::VPUNPCKHWDYrr:
7100 case X86::VPUNPCKLWDYrr:
7101 case X86::VPUNPCKHWDZ128rr:
7102 case X86::VPUNPCKLWDZ128rr:
7103 case X86::VPUNPCKHWDZ256rr:
7104 case X86::VPUNPCKLWDZ256rr:
7105 case X86::VPUNPCKHWDZrr:
7106 case X86::VPUNPCKLWDZrr:
7107 case X86::VPUNPCKHDQrr:
7108 case X86::VPUNPCKLDQrr:
7109 case X86::VPUNPCKHDQYrr:
7110 case X86::VPUNPCKLDQYrr:
7111 case X86::VPUNPCKHDQZ128rr:
7112 case X86::VPUNPCKLDQZ128rr:
7113 case X86::VPUNPCKHDQZ256rr:
7114 case X86::VPUNPCKLDQZ256rr:
7115 case X86::VPUNPCKHDQZrr:
7116 case X86::VPUNPCKLDQZrr:
7117 case X86::VPUNPCKHQDQrr:
7118 case X86::VPUNPCKLQDQrr:
7119 case X86::VPUNPCKHQDQYrr:
7120 case X86::VPUNPCKLQDQYrr:
7121 case X86::VPUNPCKHQDQZ128rr:
7122 case X86::VPUNPCKLQDQZ128rr:
7123 case X86::VPUNPCKHQDQZ256rr:
7124 case X86::VPUNPCKLQDQZ256rr:
7125 case X86::VPUNPCKHQDQZrr:
7126 case X86::VPUNPCKLQDQZrr:
7127 // These instructions are sometimes used with an undef first or second
7128 // source. Return true here so BreakFalseDeps will assign this source to the
7129 // same register as the first source to avoid a false dependency.
7130 return (OpNum == 1 || OpNum == 2) && !ForLoadFold;
7131
7132 case X86::VCVTSI2SSrr:
7133 case X86::VCVTSI2SSrm:
7134 case X86::VCVTSI2SSrr_Int:
7135 case X86::VCVTSI2SSrm_Int:
7136 case X86::VCVTSI642SSrr:
7137 case X86::VCVTSI642SSrm:
7138 case X86::VCVTSI642SSrr_Int:
7139 case X86::VCVTSI642SSrm_Int:
7140 case X86::VCVTSI2SDrr:
7141 case X86::VCVTSI2SDrm:
7142 case X86::VCVTSI2SDrr_Int:
7143 case X86::VCVTSI2SDrm_Int:
7144 case X86::VCVTSI642SDrr:
7145 case X86::VCVTSI642SDrm:
7146 case X86::VCVTSI642SDrr_Int:
7147 case X86::VCVTSI642SDrm_Int:
7148 // AVX-512
7149 case X86::VCVTSI2SSZrr:
7150 case X86::VCVTSI2SSZrm:
7151 case X86::VCVTSI2SSZrr_Int:
7152 case X86::VCVTSI2SSZrrb_Int:
7153 case X86::VCVTSI2SSZrm_Int:
7154 case X86::VCVTSI642SSZrr:
7155 case X86::VCVTSI642SSZrm:
7156 case X86::VCVTSI642SSZrr_Int:
7157 case X86::VCVTSI642SSZrrb_Int:
7158 case X86::VCVTSI642SSZrm_Int:
7159 case X86::VCVTSI2SDZrr:
7160 case X86::VCVTSI2SDZrm:
7161 case X86::VCVTSI2SDZrr_Int:
7162 case X86::VCVTSI2SDZrm_Int:
7163 case X86::VCVTSI642SDZrr:
7164 case X86::VCVTSI642SDZrm:
7165 case X86::VCVTSI642SDZrr_Int:
7166 case X86::VCVTSI642SDZrrb_Int:
7167 case X86::VCVTSI642SDZrm_Int:
7168 case X86::VCVTUSI2SSZrr:
7169 case X86::VCVTUSI2SSZrm:
7170 case X86::VCVTUSI2SSZrr_Int:
7171 case X86::VCVTUSI2SSZrrb_Int:
7172 case X86::VCVTUSI2SSZrm_Int:
7173 case X86::VCVTUSI642SSZrr:
7174 case X86::VCVTUSI642SSZrm:
7175 case X86::VCVTUSI642SSZrr_Int:
7176 case X86::VCVTUSI642SSZrrb_Int:
7177 case X86::VCVTUSI642SSZrm_Int:
7178 case X86::VCVTUSI2SDZrr:
7179 case X86::VCVTUSI2SDZrm:
7180 case X86::VCVTUSI2SDZrr_Int:
7181 case X86::VCVTUSI2SDZrm_Int:
7182 case X86::VCVTUSI642SDZrr:
7183 case X86::VCVTUSI642SDZrm:
7184 case X86::VCVTUSI642SDZrr_Int:
7185 case X86::VCVTUSI642SDZrrb_Int:
7186 case X86::VCVTUSI642SDZrm_Int:
7187 case X86::VCVTSI2SHZrr:
7188 case X86::VCVTSI2SHZrm:
7189 case X86::VCVTSI2SHZrr_Int:
7190 case X86::VCVTSI2SHZrrb_Int:
7191 case X86::VCVTSI2SHZrm_Int:
7192 case X86::VCVTSI642SHZrr:
7193 case X86::VCVTSI642SHZrm:
7194 case X86::VCVTSI642SHZrr_Int:
7195 case X86::VCVTSI642SHZrrb_Int:
7196 case X86::VCVTSI642SHZrm_Int:
7197 case X86::VCVTUSI2SHZrr:
7198 case X86::VCVTUSI2SHZrm:
7199 case X86::VCVTUSI2SHZrr_Int:
7200 case X86::VCVTUSI2SHZrrb_Int:
7201 case X86::VCVTUSI2SHZrm_Int:
7202 case X86::VCVTUSI642SHZrr:
7203 case X86::VCVTUSI642SHZrm:
7204 case X86::VCVTUSI642SHZrr_Int:
7205 case X86::VCVTUSI642SHZrrb_Int:
7206 case X86::VCVTUSI642SHZrm_Int:
7207 // Load folding won't effect the undef register update since the input is
7208 // a GPR.
7209 return OpNum == 1 && !ForLoadFold;
7210 case X86::VCVTSD2SSrr:
7211 case X86::VCVTSD2SSrm:
7212 case X86::VCVTSD2SSrr_Int:
7213 case X86::VCVTSD2SSrm_Int:
7214 case X86::VCVTSS2SDrr:
7215 case X86::VCVTSS2SDrm:
7216 case X86::VCVTSS2SDrr_Int:
7217 case X86::VCVTSS2SDrm_Int:
7218 case X86::VRCPSSr:
7219 case X86::VRCPSSr_Int:
7220 case X86::VRCPSSm:
7221 case X86::VRCPSSm_Int:
7222 case X86::VROUNDSDri:
7223 case X86::VROUNDSDmi:
7224 case X86::VROUNDSDri_Int:
7225 case X86::VROUNDSDmi_Int:
7226 case X86::VROUNDSSri:
7227 case X86::VROUNDSSmi:
7228 case X86::VROUNDSSri_Int:
7229 case X86::VROUNDSSmi_Int:
7230 case X86::VRSQRTSSr:
7231 case X86::VRSQRTSSr_Int:
7232 case X86::VRSQRTSSm:
7233 case X86::VRSQRTSSm_Int:
7234 case X86::VSQRTSSr:
7235 case X86::VSQRTSSr_Int:
7236 case X86::VSQRTSSm:
7237 case X86::VSQRTSSm_Int:
7238 case X86::VSQRTSDr:
7239 case X86::VSQRTSDr_Int:
7240 case X86::VSQRTSDm:
7241 case X86::VSQRTSDm_Int:
7242 // AVX-512
7243 case X86::VCVTSD2SSZrr:
7244 case X86::VCVTSD2SSZrr_Int:
7245 case X86::VCVTSD2SSZrrb_Int:
7246 case X86::VCVTSD2SSZrm:
7247 case X86::VCVTSD2SSZrm_Int:
7248 case X86::VCVTSS2SDZrr:
7249 case X86::VCVTSS2SDZrr_Int:
7250 case X86::VCVTSS2SDZrrb_Int:
7251 case X86::VCVTSS2SDZrm:
7252 case X86::VCVTSS2SDZrm_Int:
7253 case X86::VGETEXPSDZr:
7254 case X86::VGETEXPSDZrb:
7255 case X86::VGETEXPSDZm:
7256 case X86::VGETEXPSSZr:
7257 case X86::VGETEXPSSZrb:
7258 case X86::VGETEXPSSZm:
7259 case X86::VGETMANTSDZrri:
7260 case X86::VGETMANTSDZrrib:
7261 case X86::VGETMANTSDZrmi:
7262 case X86::VGETMANTSSZrri:
7263 case X86::VGETMANTSSZrrib:
7264 case X86::VGETMANTSSZrmi:
7265 case X86::VRNDSCALESDZrri:
7266 case X86::VRNDSCALESDZrri_Int:
7267 case X86::VRNDSCALESDZrrib_Int:
7268 case X86::VRNDSCALESDZrmi:
7269 case X86::VRNDSCALESDZrmi_Int:
7270 case X86::VRNDSCALESSZrri:
7271 case X86::VRNDSCALESSZrri_Int:
7272 case X86::VRNDSCALESSZrrib_Int:
7273 case X86::VRNDSCALESSZrmi:
7274 case X86::VRNDSCALESSZrmi_Int:
7275 case X86::VRCP14SDZrr:
7276 case X86::VRCP14SDZrm:
7277 case X86::VRCP14SSZrr:
7278 case X86::VRCP14SSZrm:
7279 case X86::VRCPSHZrr:
7280 case X86::VRCPSHZrm:
7281 case X86::VRSQRTSHZrr:
7282 case X86::VRSQRTSHZrm:
7283 case X86::VREDUCESHZrmi:
7284 case X86::VREDUCESHZrri:
7285 case X86::VREDUCESHZrrib:
7286 case X86::VGETEXPSHZr:
7287 case X86::VGETEXPSHZrb:
7288 case X86::VGETEXPSHZm:
7289 case X86::VGETMANTSHZrri:
7290 case X86::VGETMANTSHZrrib:
7291 case X86::VGETMANTSHZrmi:
7292 case X86::VRNDSCALESHZrri:
7293 case X86::VRNDSCALESHZrri_Int:
7294 case X86::VRNDSCALESHZrrib_Int:
7295 case X86::VRNDSCALESHZrmi:
7296 case X86::VRNDSCALESHZrmi_Int:
7297 case X86::VSQRTSHZr:
7298 case X86::VSQRTSHZr_Int:
7299 case X86::VSQRTSHZrb_Int:
7300 case X86::VSQRTSHZm:
7301 case X86::VSQRTSHZm_Int:
7302 case X86::VRCP28SDZr:
7303 case X86::VRCP28SDZrb:
7304 case X86::VRCP28SDZm:
7305 case X86::VRCP28SSZr:
7306 case X86::VRCP28SSZrb:
7307 case X86::VRCP28SSZm:
7308 case X86::VREDUCESSZrmi:
7309 case X86::VREDUCESSZrri:
7310 case X86::VREDUCESSZrrib:
7311 case X86::VRSQRT14SDZrr:
7312 case X86::VRSQRT14SDZrm:
7313 case X86::VRSQRT14SSZrr:
7314 case X86::VRSQRT14SSZrm:
7315 case X86::VRSQRT28SDZr:
7316 case X86::VRSQRT28SDZrb:
7317 case X86::VRSQRT28SDZm:
7318 case X86::VRSQRT28SSZr:
7319 case X86::VRSQRT28SSZrb:
7320 case X86::VRSQRT28SSZm:
7321 case X86::VSQRTSSZr:
7322 case X86::VSQRTSSZr_Int:
7323 case X86::VSQRTSSZrb_Int:
7324 case X86::VSQRTSSZm:
7325 case X86::VSQRTSSZm_Int:
7326 case X86::VSQRTSDZr:
7327 case X86::VSQRTSDZr_Int:
7328 case X86::VSQRTSDZrb_Int:
7329 case X86::VSQRTSDZm:
7330 case X86::VSQRTSDZm_Int:
7331 case X86::VCVTSD2SHZrr:
7332 case X86::VCVTSD2SHZrr_Int:
7333 case X86::VCVTSD2SHZrrb_Int:
7334 case X86::VCVTSD2SHZrm:
7335 case X86::VCVTSD2SHZrm_Int:
7336 case X86::VCVTSS2SHZrr:
7337 case X86::VCVTSS2SHZrr_Int:
7338 case X86::VCVTSS2SHZrrb_Int:
7339 case X86::VCVTSS2SHZrm:
7340 case X86::VCVTSS2SHZrm_Int:
7341 case X86::VCVTSH2SDZrr:
7342 case X86::VCVTSH2SDZrr_Int:
7343 case X86::VCVTSH2SDZrrb_Int:
7344 case X86::VCVTSH2SDZrm:
7345 case X86::VCVTSH2SDZrm_Int:
7346 case X86::VCVTSH2SSZrr:
7347 case X86::VCVTSH2SSZrr_Int:
7348 case X86::VCVTSH2SSZrrb_Int:
7349 case X86::VCVTSH2SSZrm:
7350 case X86::VCVTSH2SSZrm_Int:
7351 return OpNum == 1;
7352 case X86::VMOVSSZrrk:
7353 case X86::VMOVSDZrrk:
7354 return OpNum == 3 && !ForLoadFold;
7355 case X86::VMOVSSZrrkz:
7356 case X86::VMOVSDZrrkz:
7357 return OpNum == 2 && !ForLoadFold;
7358 }
7359
7360 return false;
7361}
7362
7363/// Inform the BreakFalseDeps pass how many idle instructions we would like
7364/// before certain undef register reads.
7365///
7366/// This catches the VCVTSI2SD family of instructions:
7367///
7368/// vcvtsi2sdq %rax, undef %xmm0, %xmm14
7369///
7370/// We should to be careful *not* to catch VXOR idioms which are presumably
7371/// handled specially in the pipeline:
7372///
7373/// vxorps undef %xmm1, undef %xmm1, %xmm1
7374///
7375/// Like getPartialRegUpdateClearance, this makes a strong assumption that the
7376/// high bits that are passed-through are not live.
7378 unsigned OpNum) const {
7379 const MachineOperand &MO = MI.getOperand(OpNum);
7380 if (MO.getReg().isPhysical() && hasUndefRegUpdate(MI.getOpcode(), OpNum))
7381 return Subtarget.getCLOpts().undef_reg_clearance;
7382
7383 return 0;
7384}
7385
7387 unsigned OpNum) const {
7388 Register Reg = MI.getOperand(OpNum).getReg();
7389 // If MI kills this register, the false dependence is already broken.
7390 if (MI.killsRegister(Reg, &RI))
7391 return;
7392
7393 if (X86::VR128RegClass.contains(Reg)) {
7394 // These instructions are all floating point domain, so xorps is the best
7395 // choice.
7396 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr;
7397 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg)
7398 .addReg(Reg, RegState::Undef)
7399 .addReg(Reg, RegState::Undef);
7400 MI.addRegisterKilled(Reg, &RI, true);
7401 } else if (X86::VR256RegClass.contains(Reg)) {
7402 // Use vxorps to clear the full ymm register.
7403 // It wants to read and write the xmm sub-register.
7404 Register XReg = RI.getSubReg(Reg, X86::sub_xmm);
7405 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg)
7406 .addReg(XReg, RegState::Undef)
7407 .addReg(XReg, RegState::Undef)
7409 MI.addRegisterKilled(Reg, &RI, true);
7410 } else if (X86::VR128XRegClass.contains(Reg)) {
7411 // Only handle VLX targets.
7412 if (!Subtarget.hasVLX())
7413 return;
7414 // Since vxorps requires AVX512DQ, vpxord should be the best choice.
7415 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), Reg)
7416 .addReg(Reg, RegState::Undef)
7417 .addReg(Reg, RegState::Undef);
7418 MI.addRegisterKilled(Reg, &RI, true);
7419 } else if (X86::VR256XRegClass.contains(Reg) ||
7420 X86::VR512RegClass.contains(Reg)) {
7421 // Only handle VLX targets.
7422 if (!Subtarget.hasVLX())
7423 return;
7424 // Use vpxord to clear the full ymm/zmm register.
7425 // It wants to read and write the xmm sub-register.
7426 Register XReg = RI.getSubReg(Reg, X86::sub_xmm);
7427 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), XReg)
7428 .addReg(XReg, RegState::Undef)
7429 .addReg(XReg, RegState::Undef)
7431 MI.addRegisterKilled(Reg, &RI, true);
7432 } else if (X86::GR64RegClass.contains(Reg)) {
7433 // Using XOR32rr because it has shorter encoding and zeros up the upper bits
7434 // as well.
7435 Register XReg = RI.getSubReg(Reg, X86::sub_32bit);
7436 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg)
7437 .addReg(XReg, RegState::Undef)
7438 .addReg(XReg, RegState::Undef)
7440 MI.addRegisterKilled(Reg, &RI, true);
7441 } else if (X86::GR32RegClass.contains(Reg)) {
7442 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg)
7443 .addReg(Reg, RegState::Undef)
7444 .addReg(Reg, RegState::Undef);
7445 MI.addRegisterKilled(Reg, &RI, true);
7446 } else if ((X86::GR16RegClass.contains(Reg) ||
7447 X86::GR8RegClass.contains(Reg)) &&
7448 X86II::hasNewDataDest(MI.getDesc().TSFlags)) {
7449 // This case is only expected for NDD ops which appear to be partial
7450 // writes, but are not due to the zeroing of the upper part. Here
7451 // we add an implicit def of the superegister, which prevents
7452 // CompressEVEX from converting this to a legacy form.
7453 Register SuperReg = getX86SubSuperRegister(Reg, 64);
7454 MachineInstrBuilder BuildMI(*MI.getParent()->getParent(), &MI);
7455 if (!MI.definesRegister(SuperReg, /*TRI=*/nullptr))
7456 BuildMI.addReg(SuperReg, RegState::ImplicitDefine);
7457 }
7458}
7459
7461 int PtrOffset = 0) {
7462 unsigned NumAddrOps = MOs.size();
7463
7464 if (NumAddrOps < 4) {
7465 // FrameIndex only - add an immediate offset (whether its zero or not).
7466 for (unsigned i = 0; i != NumAddrOps; ++i)
7467 MIB.add(MOs[i]);
7468 addOffset(MIB, PtrOffset);
7469 } else {
7470 // General Memory Addressing - we need to add any offset to an existing
7471 // offset.
7472 assert(MOs.size() == 5 && "Unexpected memory operand list length");
7473 for (unsigned i = 0; i != NumAddrOps; ++i) {
7474 const MachineOperand &MO = MOs[i];
7475 if (i == 3 && PtrOffset != 0) {
7476 MIB.addDisp(MO, PtrOffset);
7477 } else {
7478 MIB.add(MO);
7479 }
7480 }
7481 }
7482}
7483
7485 MachineInstr &NewMI,
7486 const TargetInstrInfo &TII) {
7488
7489 for (int Idx : llvm::seq<int>(0, NewMI.getNumOperands())) {
7490 MachineOperand &MO = NewMI.getOperand(Idx);
7491 // We only need to update constraints on virtual register operands.
7492 if (!MO.isReg())
7493 continue;
7494 Register Reg = MO.getReg();
7495 if (!Reg.isVirtual())
7496 continue;
7497
7498 auto *NewRC =
7499 MRI.constrainRegClass(Reg, TII.getRegClass(NewMI.getDesc(), Idx));
7500 if (!NewRC) {
7501 LLVM_DEBUG(
7502 dbgs() << "WARNING: Unable to update register constraint for operand "
7503 << Idx << " of instruction:\n";
7504 NewMI.dump(); dbgs() << "\n");
7505 }
7506 }
7507}
7508
7509static MachineInstr *fuseTwoAddrInst(MachineFunction &MF, unsigned Opcode,
7513 const TargetInstrInfo &TII) {
7514 // Create the base instruction with the memory operand as the first part.
7515 // Omit the implicit operands, something BuildMI can't do.
7516 MachineInstr *NewMI =
7517 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true);
7518 MachineInstrBuilder MIB(MF, NewMI);
7519 addOperands(MIB, MOs);
7520
7521 // Loop over the rest of the ri operands, converting them over.
7522 unsigned NumOps = MI.getDesc().getNumOperands() - 2;
7523 for (unsigned i = 0; i != NumOps; ++i) {
7524 MachineOperand &MO = MI.getOperand(i + 2);
7525 MIB.add(MO);
7526 }
7527 for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), NumOps + 2))
7528 MIB.add(MO);
7529
7530 updateOperandRegConstraints(MF, *NewMI, TII);
7531
7532 MachineBasicBlock *MBB = InsertPt->getParent();
7533 MBB->insert(InsertPt, NewMI);
7534
7535 return MIB;
7536}
7537
7538static MachineInstr *fuseInst(MachineFunction &MF, unsigned Opcode,
7539 unsigned OpNo, ArrayRef<MachineOperand> MOs,
7542 int PtrOffset = 0) {
7543 // Omit the implicit operands, something BuildMI can't do.
7544 MachineInstr *NewMI =
7545 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true);
7546 MachineInstrBuilder MIB(MF, NewMI);
7547
7548 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
7549 MachineOperand &MO = MI.getOperand(i);
7550 if (i == OpNo) {
7551 assert(MO.isReg() && "Expected to fold into reg operand!");
7552 addOperands(MIB, MOs, PtrOffset);
7553 } else {
7554 MIB.add(MO);
7555 }
7556 }
7557
7558 updateOperandRegConstraints(MF, *NewMI, TII);
7559
7560 // Copy the NoFPExcept flag from the instruction we're fusing.
7563
7564 MachineBasicBlock *MBB = InsertPt->getParent();
7565 MBB->insert(InsertPt, NewMI);
7566
7567 return MIB;
7568}
7569
7570static MachineInstr *makeM0Inst(const TargetInstrInfo &TII, unsigned Opcode,
7573 MachineInstr &MI) {
7574 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt,
7575 MI.getDebugLoc(), TII.get(Opcode));
7576 addOperands(MIB, MOs);
7577 return MIB.addImm(0);
7578}
7579
7580MachineInstr *X86InstrInfo::foldMemoryOperandCustom(
7581 MachineFunction &MF, MachineInstr &MI, unsigned OpNum,
7583 unsigned Size, Align Alignment) const {
7584 switch (MI.getOpcode()) {
7585 case X86::INSERTPSrri:
7586 case X86::VINSERTPSrri:
7587 case X86::VINSERTPSZrri:
7588 // Attempt to convert the load of inserted vector into a fold load
7589 // of a single float.
7590 if (OpNum == 2) {
7591 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
7592 unsigned ZMask = Imm & 15;
7593 unsigned DstIdx = (Imm >> 4) & 3;
7594 unsigned SrcIdx = (Imm >> 6) & 3;
7595
7596 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7597 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum);
7598 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8;
7599 if ((Size == 0 || Size >= 16) && RCSize >= 16 &&
7600 (MI.getOpcode() != X86::INSERTPSrri || Alignment >= Align(4))) {
7601 int PtrOffset = SrcIdx * 4;
7602 unsigned NewImm = (DstIdx << 4) | ZMask;
7603 unsigned NewOpCode =
7604 (MI.getOpcode() == X86::VINSERTPSZrri) ? X86::VINSERTPSZrmi
7605 : (MI.getOpcode() == X86::VINSERTPSrri) ? X86::VINSERTPSrmi
7606 : X86::INSERTPSrmi;
7607 MachineInstr *NewMI =
7608 fuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, PtrOffset);
7609 NewMI->getOperand(NewMI->getNumOperands() - 1).setImm(NewImm);
7610 return NewMI;
7611 }
7612 }
7613 break;
7614 case X86::MOVHLPSrr:
7615 case X86::VMOVHLPSrr:
7616 case X86::VMOVHLPSZrr:
7617 // Move the upper 64-bits of the second operand to the lower 64-bits.
7618 // To fold the load, adjust the pointer to the upper and use (V)MOVLPS.
7619 // TODO: In most cases AVX doesn't have a 8-byte alignment requirement.
7620 if (OpNum == 2) {
7621 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7622 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum);
7623 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8;
7624 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(8)) {
7625 unsigned NewOpCode =
7626 (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm
7627 : (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm
7628 : X86::MOVLPSrm;
7629 MachineInstr *NewMI =
7630 fuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, 8);
7631 return NewMI;
7632 }
7633 }
7634 break;
7635 case X86::UNPCKLPDrr:
7636 // If we won't be able to fold this to the memory form of UNPCKL, use
7637 // MOVHPD instead. Done as custom because we can't have this in the load
7638 // table twice.
7639 if (OpNum == 2) {
7640 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7641 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum);
7642 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8;
7643 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment < Align(16)) {
7644 MachineInstr *NewMI =
7645 fuseInst(MF, X86::MOVHPDrm, OpNum, MOs, InsertPt, MI, *this);
7646 return NewMI;
7647 }
7648 }
7649 break;
7650 case X86::MOV32r0:
7651 if (auto *NewMI =
7652 makeM0Inst(*this, (Size == 4) ? X86::MOV32mi : X86::MOV64mi32, MOs,
7653 InsertPt, MI))
7654 return NewMI;
7655 break;
7656 }
7657
7658 return nullptr;
7659}
7660
7662 MachineInstr &MI) {
7663 if (!hasUndefRegUpdate(MI.getOpcode(), 1, /*ForLoadFold*/ true) ||
7664 !MI.getOperand(1).isReg())
7665 return false;
7666
7667 // The are two cases we need to handle depending on where in the pipeline
7668 // the folding attempt is being made.
7669 // -Register has the undef flag set.
7670 // -Register is produced by the IMPLICIT_DEF instruction.
7671
7672 if (MI.getOperand(1).isUndef())
7673 return true;
7674
7676 MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(MI.getOperand(1).getReg());
7677 return VRegDef && VRegDef->isImplicitDef();
7678}
7679
7680unsigned X86InstrInfo::commuteOperandsForFold(MachineInstr &MI,
7681 unsigned Idx1) const {
7682 unsigned Idx2 = CommuteAnyOperandIndex;
7683 if (!findCommutedOpIndices(MI, Idx1, Idx2))
7684 return Idx1;
7685
7686 bool HasDef = MI.getDesc().getNumDefs();
7687 Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register();
7688 Register Reg1 = MI.getOperand(Idx1).getReg();
7689 Register Reg2 = MI.getOperand(Idx2).getReg();
7690 bool Tied1 = 0 == MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO);
7691 bool Tied2 = 0 == MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO);
7692
7693 // If either of the commutable operands are tied to the destination
7694 // then we can not commute + fold.
7695 if ((HasDef && Reg0 == Reg1 && Tied1) || (HasDef && Reg0 == Reg2 && Tied2))
7696 return Idx1;
7697
7698 return commuteInstruction(MI, false, Idx1, Idx2) ? Idx2 : Idx1;
7699}
7700
7701static void printFailMsgforFold(const MachineInstr &MI, unsigned Idx) {
7702 const X86Subtarget &ST = MI.getMF()->getSubtarget<X86Subtarget>();
7703 if (ST.getCLOpts().print_failed_fuse_candidates && !MI.isCopy())
7704 dbgs() << "We failed to fuse operand " << Idx << " in " << MI;
7705}
7706
7708 MachineFunction &MF, MachineInstr &MI, unsigned OpNum,
7710 unsigned Size, Align Alignment, bool AllowCommute, MachineInstr *&CopyMI,
7711 VirtRegMap *VRM) const {
7712 bool isSlowTwoMemOps = Subtarget.slowTwoMemOps();
7713 bool isSlowIndirectCall = Subtarget.slowIndirectCall();
7714 unsigned Opc = MI.getOpcode();
7715
7716 // For CPUs that favor the register form of a call,
7717 // do not fold loads into calls, unless optimizing for size aggressively.
7718 if ((isSlowTwoMemOps || isSlowIndirectCall) &&
7719 !MF.getFunction().hasMinSize() &&
7720 (Opc == X86::CALL32r || Opc == X86::CALL64r ||
7721 Opc == X86::CALL64r_ImpCall))
7722 return nullptr;
7723
7724 // For CPUs that favor the register form of a push,
7725 // do not fold loads into pushes, unless optimizing for size aggressively.
7726 if (isSlowTwoMemOps && !MF.getFunction().hasMinSize() &&
7727 (Opc == X86::PUSH16r || Opc == X86::PUSH32r || Opc == X86::PUSH64r))
7728 return nullptr;
7729
7730 // Avoid partial and undef register update stalls unless optimizing for size.
7731 if (!MF.getFunction().hasOptSize() &&
7732 (hasPartialRegUpdate(Opc, Subtarget, /*ForLoadFold*/ true) ||
7734 return nullptr;
7735
7736 unsigned NumOps = MI.getDesc().getNumOperands();
7737 bool IsTwoAddr = NumOps > 1 && OpNum < 2 && MI.getOperand(0).isReg() &&
7738 MI.getOperand(1).isReg() &&
7739 MI.getOperand(0).getReg() == MI.getOperand(1).getReg();
7740
7741 // FIXME: AsmPrinter doesn't know how to handle
7742 // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding.
7743 if (Opc == X86::ADD32ri &&
7744 MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS)
7745 return nullptr;
7746
7747 // GOTTPOFF relocation loads can only be folded into add instructions.
7748 // FIXME: Need to exclude other relocations that only support specific
7749 // instructions.
7750 if (MOs.size() == X86::AddrNumOperands &&
7751 MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF &&
7752 Opc != X86::ADD64rr)
7753 return nullptr;
7754
7755 // Don't fold loads into indirect calls that need a KCFI check as we'll
7756 // have to unfold these in X86TargetLowering::EmitKCFICheck anyway.
7757 if (MI.isCall() && MI.getCFIType())
7758 return nullptr;
7759
7760 // Attempt to fold any custom cases we have.
7761 if (auto *CustomMI = foldMemoryOperandCustom(MF, MI, OpNum, MOs, InsertPt,
7762 Size, Alignment))
7763 return CustomMI;
7764
7765 // Folding a memory location into the two-address part of a two-address
7766 // instruction is different than folding it other places. It requires
7767 // replacing the *two* registers with the memory location.
7768 //
7769 // Utilize the mapping NonNDD -> RMW for the NDD variant.
7770 unsigned NonNDOpc = Subtarget.hasNDD() ? X86::getNonNDVariant(Opc) : 0U;
7771 // Utilize the mapping NonNDD if NDD memory variant is not preferred.
7772 bool NoNDDM = NonNDOpc && !Subtarget.hasNDDM();
7773
7775 if (NoNDDM && !IsTwoAddr && !MRI.isSSA()) {
7776 // Bail out if dst has subreg. It happens during register-coalescer from
7777 // 704B %19:gr32 = SUB32rr_ND killed %0:gr32, killed %7:gr32, ...
7778 // 752B undef %23.sub_32bit:gr64 = COPY killed %19:gr32
7779 // 768B %25:gr32 = LEA64_32r killed %23:gr64, 1, killed %21:gr64_nosp, ...
7780 // to
7781 // 704B undef %23.sub_32bit:gr64_with_sub_8bit = SUB32rr_ND %0:gr32, ...
7782 // 768B %25:gr32 = LEA64_32r %23:gr64_with_sub_8bit, 1, %21:gr64_nosp, ...
7783 // Machine verifier fails if we try to tie %23 to the source.
7784 if (MI.getOperand(0).getSubReg())
7785 return nullptr;
7786
7787 // Bail out if dst has been assigned a physical register. Otherwise, we
7788 // cannot update LiveRegMatrix properly.
7789 Register Dst = MI.getOperand(0).getReg();
7790 if (VRM && Dst != MI.getOperand(1).getReg() &&
7791 (!Dst.isVirtual() || VRM->getPhys(Dst)))
7792 return nullptr;
7793 }
7794
7795 const X86FoldTableEntry *I =
7796 IsTwoAddr ? lookupTwoAddrFoldTable(NonNDOpc ? NonNDOpc : Opc)
7797 : lookupFoldTable(NoNDDM ? NonNDOpc : Opc, OpNum);
7798
7799 MachineInstr *NewMI = nullptr;
7800 if (I) {
7801 unsigned Opcode = I->DstOp;
7802 if (Alignment <
7804 return nullptr;
7805 bool NarrowToMOV32rm = false;
7806 if (Size) {
7808 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum);
7809 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8;
7810 // Check if it's safe to fold the load. If the size of the object is
7811 // narrower than the load width, then it's not.
7812 // FIXME: Allow scalar intrinsic instructions like ADDSSrm_Int.
7813 if ((I->Flags & TB_FOLDED_LOAD) && Size < RCSize) {
7814 // If this is a 64-bit load, but the spill slot is 32, then we can do
7815 // a 32-bit load which is implicitly zero-extended. This likely is
7816 // due to live interval analysis remat'ing a load from stack slot.
7817 if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4)
7818 return nullptr;
7819 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
7820 return nullptr;
7821 Opcode = X86::MOV32rm;
7822 NarrowToMOV32rm = true;
7823 }
7824 // For stores, make sure the size of the object is equal to the size of
7825 // the store. If the object is larger, the extra bits would be garbage. If
7826 // the object is smaller we might overwrite another object or fault.
7827 if ((I->Flags & TB_FOLDED_STORE) && Size != RCSize)
7828 return nullptr;
7829 }
7830
7831 NewMI = IsTwoAddr ? fuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this)
7832 : fuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this);
7833
7834 if (NarrowToMOV32rm) {
7835 // If this is the special case where we use a MOV32rm to load a 32-bit
7836 // value and zero-extend the top bits. Change the destination register
7837 // to a 32-bit one.
7838 Register DstReg = NewMI->getOperand(0).getReg();
7839 if (DstReg.isPhysical())
7840 NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit));
7841 else
7842 NewMI->getOperand(0).setSubReg(X86::sub_32bit);
7843 }
7844
7845 if (NoNDDM && !IsTwoAddr) {
7846 Register SrcReg = MI.getOperand(1).getReg();
7847 unsigned SrcSub = MI.getOperand(1).getSubReg();
7848 if (MI.killsRegister(SrcReg, /*TRI=*/nullptr) ||
7849 MI.getOperand(0).getReg() == SrcReg)
7850 return NewMI;
7851
7852 Register NewSrc = MI.getOperand(0).getReg();
7853 if (MRI.isSSA())
7854 NewSrc = MRI.createVirtualRegister(getRegClass(NewMI->getDesc(), 1));
7855
7856 CopyMI = BuildMI(*NewMI->getParent(), *NewMI, MI.getDebugLoc(),
7857 get(TargetOpcode::COPY))
7858 .addDef(NewSrc)
7859 .addReg(SrcReg, {}, SrcSub);
7860 NewMI->getOperand(1).setReg(NewSrc);
7861 NewMI->getOperand(1).setSubReg(0);
7862 }
7863 return NewMI;
7864 }
7865
7866 if (AllowCommute) {
7867 // If the instruction and target operand are commutable, commute the
7868 // instruction and try again.
7869 unsigned CommuteOpIdx2 = commuteOperandsForFold(MI, OpNum);
7870 if (CommuteOpIdx2 == OpNum) {
7871 printFailMsgforFold(MI, OpNum);
7872 return nullptr;
7873 }
7874 // Attempt to fold with the commuted version of the instruction.
7875 NewMI = foldMemoryOperandImpl(MF, MI, CommuteOpIdx2, MOs, InsertPt, Size,
7876 Alignment, /*AllowCommute=*/false, CopyMI);
7877 if (NewMI)
7878 return NewMI;
7879 // Folding failed again - undo the commute before returning.
7880 commuteInstruction(MI, false, OpNum, CommuteOpIdx2);
7881 }
7882
7883 printFailMsgforFold(MI, OpNum);
7884 return nullptr;
7885}
7886
7889 ArrayRef<unsigned> Ops, int FrameIndex,
7890 MachineInstr *&CopyMI, LiveIntervals *LIS,
7891 VirtRegMap *VRM) const {
7893 // Check switch flag
7894 if (Subtarget.getCLOpts().disable_spill_fusing)
7895 return nullptr;
7896
7897 // Avoid partial and undef register update stalls unless optimizing for size.
7898 if (!MF.getFunction().hasOptSize() &&
7899 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/ true) ||
7901 return nullptr;
7902
7903 // Don't fold subreg spills, or reloads that use a high subreg.
7904 for (auto Op : Ops) {
7905 MachineOperand &MO = MI.getOperand(Op);
7906 auto SubReg = MO.getSubReg();
7907 // MOV32r0 is special b/c it's used to clear a 64-bit register too.
7908 // (See patterns for MOV32r0 in TD files).
7909 if (MI.getOpcode() == X86::MOV32r0 && SubReg == X86::sub_32bit)
7910 continue;
7911 if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi))
7912 return nullptr;
7913 }
7914
7915 const MachineFrameInfo &MFI = MF.getFrameInfo();
7916 unsigned Size = MFI.getObjectSize(FrameIndex);
7917 Align Alignment = MFI.getObjectAlign(FrameIndex);
7918 // If the function stack isn't realigned we don't want to fold instructions
7919 // that need increased alignment.
7920 if (!RI.hasStackRealignment(MF))
7921 Alignment =
7922 std::min(Alignment, Subtarget.getFrameLowering()->getStackAlign());
7923
7924 auto Impl = [&]() {
7925 return foldMemoryOperandImpl(
7926 MF, MI, Ops[0], MachineOperand::CreateFI(FrameIndex), InsertPt, Size,
7927 Alignment, /*AllowCommute=*/true, CopyMI, VRM);
7928 };
7929 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
7930 unsigned NewOpc = 0;
7931 unsigned RCSize = 0;
7932 unsigned Opc = MI.getOpcode();
7933 switch (Opc) {
7934 default:
7935 // NDD can be folded into RMW though its Op0 and Op1 are not tied.
7936 return (Subtarget.hasNDD() ? X86::getNonNDVariant(Opc) : 0U) ? Impl()
7937 : nullptr;
7938 case X86::TEST8rr:
7939 NewOpc = X86::CMP8ri;
7940 RCSize = 1;
7941 break;
7942 case X86::TEST16rr:
7943 NewOpc = X86::CMP16ri;
7944 RCSize = 2;
7945 break;
7946 case X86::TEST32rr:
7947 NewOpc = X86::CMP32ri;
7948 RCSize = 4;
7949 break;
7950 case X86::TEST64rr:
7951 NewOpc = X86::CMP64ri32;
7952 RCSize = 8;
7953 break;
7954 }
7955 // Check if it's safe to fold the load. If the size of the object is
7956 // narrower than the load width, then it's not.
7957 if (Size < RCSize)
7958 return nullptr;
7959 // Change to CMPXXri r, 0 first.
7960 MI.setDesc(get(NewOpc));
7961 MI.getOperand(1).ChangeToImmediate(0);
7962 } else if (Ops.size() != 1)
7963 return nullptr;
7964
7965 return Impl();
7966}
7967
7968/// Check if \p LoadMI is a partial register load that we can't fold into \p MI
7969/// because the latter uses contents that wouldn't be defined in the folded
7970/// version. For instance, this transformation isn't legal:
7971/// movss (%rdi), %xmm0
7972/// addps %xmm0, %xmm0
7973/// ->
7974/// addps (%rdi), %xmm0
7975///
7976/// But this one is:
7977/// movss (%rdi), %xmm0
7978/// addss %xmm0, %xmm0
7979/// ->
7980/// addss (%rdi), %xmm0
7981///
7983 const MachineInstr &UserMI,
7984 const MachineFunction &MF) {
7985 unsigned Opc = LoadMI.getOpcode();
7986 unsigned UserOpc = UserMI.getOpcode();
7988 const TargetRegisterClass *RC =
7989 MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg());
7990 unsigned RegSize = TRI.getRegSizeInBits(*RC);
7991
7992 if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm ||
7993 Opc == X86::MOVSSrm_alt || Opc == X86::VMOVSSrm_alt ||
7994 Opc == X86::VMOVSSZrm_alt) &&
7995 RegSize > 32) {
7996 // These instructions only load 32 bits, we can't fold them if the
7997 // destination register is wider than 32 bits (4 bytes), and its user
7998 // instruction isn't scalar (SS).
7999 switch (UserOpc) {
8000 case X86::CVTSS2SDrr_Int:
8001 case X86::VCVTSS2SDrr_Int:
8002 case X86::VCVTSS2SDZrr_Int:
8003 case X86::VCVTSS2SDZrrk_Int:
8004 case X86::VCVTSS2SDZrrkz_Int:
8005 case X86::CVTSS2SIrr_Int:
8006 case X86::CVTSS2SI64rr_Int:
8007 case X86::VCVTSS2SIrr_Int:
8008 case X86::VCVTSS2SI64rr_Int:
8009 case X86::VCVTSS2SIZrr_Int:
8010 case X86::VCVTSS2SI64Zrr_Int:
8011 case X86::CVTTSS2SIrr_Int:
8012 case X86::CVTTSS2SI64rr_Int:
8013 case X86::VCVTTSS2SIrr_Int:
8014 case X86::VCVTTSS2SI64rr_Int:
8015 case X86::VCVTTSS2SIZrr_Int:
8016 case X86::VCVTTSS2SI64Zrr_Int:
8017 case X86::VCVTSS2USIZrr_Int:
8018 case X86::VCVTSS2USI64Zrr_Int:
8019 case X86::VCVTTSS2USIZrr_Int:
8020 case X86::VCVTTSS2USI64Zrr_Int:
8021 case X86::RCPSSr_Int:
8022 case X86::VRCPSSr_Int:
8023 case X86::RSQRTSSr_Int:
8024 case X86::VRSQRTSSr_Int:
8025 case X86::ROUNDSSri_Int:
8026 case X86::VROUNDSSri_Int:
8027 case X86::COMISSrr_Int:
8028 case X86::VCOMISSrr_Int:
8029 case X86::VCOMISSZrr_Int:
8030 case X86::UCOMISSrr_Int:
8031 case X86::VUCOMISSrr_Int:
8032 case X86::VUCOMISSZrr_Int:
8033 case X86::ADDSSrr_Int:
8034 case X86::VADDSSrr_Int:
8035 case X86::VADDSSZrr_Int:
8036 case X86::CMPSSrri_Int:
8037 case X86::VCMPSSrri_Int:
8038 case X86::VCMPSSZrri_Int:
8039 case X86::DIVSSrr_Int:
8040 case X86::VDIVSSrr_Int:
8041 case X86::VDIVSSZrr_Int:
8042 case X86::MAXSSrr_Int:
8043 case X86::VMAXSSrr_Int:
8044 case X86::VMAXSSZrr_Int:
8045 case X86::MINSSrr_Int:
8046 case X86::VMINSSrr_Int:
8047 case X86::VMINSSZrr_Int:
8048 case X86::MULSSrr_Int:
8049 case X86::VMULSSrr_Int:
8050 case X86::VMULSSZrr_Int:
8051 case X86::SQRTSSr_Int:
8052 case X86::VSQRTSSr_Int:
8053 case X86::VSQRTSSZr_Int:
8054 case X86::SUBSSrr_Int:
8055 case X86::VSUBSSrr_Int:
8056 case X86::VSUBSSZrr_Int:
8057 case X86::VADDSSZrrk_Int:
8058 case X86::VADDSSZrrkz_Int:
8059 case X86::VCMPSSZrrik_Int:
8060 case X86::VDIVSSZrrk_Int:
8061 case X86::VDIVSSZrrkz_Int:
8062 case X86::VMAXSSZrrk_Int:
8063 case X86::VMAXSSZrrkz_Int:
8064 case X86::VMINSSZrrk_Int:
8065 case X86::VMINSSZrrkz_Int:
8066 case X86::VMULSSZrrk_Int:
8067 case X86::VMULSSZrrkz_Int:
8068 case X86::VSQRTSSZrk_Int:
8069 case X86::VSQRTSSZrkz_Int:
8070 case X86::VSUBSSZrrk_Int:
8071 case X86::VSUBSSZrrkz_Int:
8072 case X86::VFMADDSS4rr_Int:
8073 case X86::VFNMADDSS4rr_Int:
8074 case X86::VFMSUBSS4rr_Int:
8075 case X86::VFNMSUBSS4rr_Int:
8076 case X86::VFMADD132SSr_Int:
8077 case X86::VFNMADD132SSr_Int:
8078 case X86::VFMADD213SSr_Int:
8079 case X86::VFNMADD213SSr_Int:
8080 case X86::VFMADD231SSr_Int:
8081 case X86::VFNMADD231SSr_Int:
8082 case X86::VFMSUB132SSr_Int:
8083 case X86::VFNMSUB132SSr_Int:
8084 case X86::VFMSUB213SSr_Int:
8085 case X86::VFNMSUB213SSr_Int:
8086 case X86::VFMSUB231SSr_Int:
8087 case X86::VFNMSUB231SSr_Int:
8088 case X86::VFMADD132SSZr_Int:
8089 case X86::VFNMADD132SSZr_Int:
8090 case X86::VFMADD213SSZr_Int:
8091 case X86::VFNMADD213SSZr_Int:
8092 case X86::VFMADD231SSZr_Int:
8093 case X86::VFNMADD231SSZr_Int:
8094 case X86::VFMSUB132SSZr_Int:
8095 case X86::VFNMSUB132SSZr_Int:
8096 case X86::VFMSUB213SSZr_Int:
8097 case X86::VFNMSUB213SSZr_Int:
8098 case X86::VFMSUB231SSZr_Int:
8099 case X86::VFNMSUB231SSZr_Int:
8100 case X86::VFMADD132SSZrk_Int:
8101 case X86::VFNMADD132SSZrk_Int:
8102 case X86::VFMADD213SSZrk_Int:
8103 case X86::VFNMADD213SSZrk_Int:
8104 case X86::VFMADD231SSZrk_Int:
8105 case X86::VFNMADD231SSZrk_Int:
8106 case X86::VFMSUB132SSZrk_Int:
8107 case X86::VFNMSUB132SSZrk_Int:
8108 case X86::VFMSUB213SSZrk_Int:
8109 case X86::VFNMSUB213SSZrk_Int:
8110 case X86::VFMSUB231SSZrk_Int:
8111 case X86::VFNMSUB231SSZrk_Int:
8112 case X86::VFMADD132SSZrkz_Int:
8113 case X86::VFNMADD132SSZrkz_Int:
8114 case X86::VFMADD213SSZrkz_Int:
8115 case X86::VFNMADD213SSZrkz_Int:
8116 case X86::VFMADD231SSZrkz_Int:
8117 case X86::VFNMADD231SSZrkz_Int:
8118 case X86::VFMSUB132SSZrkz_Int:
8119 case X86::VFNMSUB132SSZrkz_Int:
8120 case X86::VFMSUB213SSZrkz_Int:
8121 case X86::VFNMSUB213SSZrkz_Int:
8122 case X86::VFMSUB231SSZrkz_Int:
8123 case X86::VFNMSUB231SSZrkz_Int:
8124 case X86::VFIXUPIMMSSZrri:
8125 case X86::VFIXUPIMMSSZrrik:
8126 case X86::VFIXUPIMMSSZrrikz:
8127 case X86::VFPCLASSSSZri:
8128 case X86::VFPCLASSSSZrik:
8129 case X86::VGETEXPSSZr:
8130 case X86::VGETEXPSSZrk:
8131 case X86::VGETEXPSSZrkz:
8132 case X86::VGETMANTSSZrri:
8133 case X86::VGETMANTSSZrrik:
8134 case X86::VGETMANTSSZrrikz:
8135 case X86::VRANGESSZrri:
8136 case X86::VRANGESSZrrik:
8137 case X86::VRANGESSZrrikz:
8138 case X86::VRCP14SSZrr:
8139 case X86::VRCP14SSZrrk:
8140 case X86::VRCP14SSZrrkz:
8141 case X86::VRCP28SSZr:
8142 case X86::VRCP28SSZrk:
8143 case X86::VRCP28SSZrkz:
8144 case X86::VREDUCESSZrri:
8145 case X86::VREDUCESSZrrik:
8146 case X86::VREDUCESSZrrikz:
8147 case X86::VRNDSCALESSZrri_Int:
8148 case X86::VRNDSCALESSZrrik_Int:
8149 case X86::VRNDSCALESSZrrikz_Int:
8150 case X86::VRSQRT14SSZrr:
8151 case X86::VRSQRT14SSZrrk:
8152 case X86::VRSQRT14SSZrrkz:
8153 case X86::VRSQRT28SSZr:
8154 case X86::VRSQRT28SSZrk:
8155 case X86::VRSQRT28SSZrkz:
8156 case X86::VSCALEFSSZrr:
8157 case X86::VSCALEFSSZrrk:
8158 case X86::VSCALEFSSZrrkz:
8159 return false;
8160 default:
8161 return true;
8162 }
8163 }
8164
8165 if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm ||
8166 Opc == X86::MOVSDrm_alt || Opc == X86::VMOVSDrm_alt ||
8167 Opc == X86::VMOVSDZrm_alt) &&
8168 RegSize > 64) {
8169 // These instructions only load 64 bits, we can't fold them if the
8170 // destination register is wider than 64 bits (8 bytes), and its user
8171 // instruction isn't scalar (SD).
8172 switch (UserOpc) {
8173 case X86::CVTSD2SSrr_Int:
8174 case X86::VCVTSD2SSrr_Int:
8175 case X86::VCVTSD2SSZrr_Int:
8176 case X86::VCVTSD2SSZrrk_Int:
8177 case X86::VCVTSD2SSZrrkz_Int:
8178 case X86::CVTSD2SIrr_Int:
8179 case X86::CVTSD2SI64rr_Int:
8180 case X86::VCVTSD2SIrr_Int:
8181 case X86::VCVTSD2SI64rr_Int:
8182 case X86::VCVTSD2SIZrr_Int:
8183 case X86::VCVTSD2SI64Zrr_Int:
8184 case X86::CVTTSD2SIrr_Int:
8185 case X86::CVTTSD2SI64rr_Int:
8186 case X86::VCVTTSD2SIrr_Int:
8187 case X86::VCVTTSD2SI64rr_Int:
8188 case X86::VCVTTSD2SIZrr_Int:
8189 case X86::VCVTTSD2SI64Zrr_Int:
8190 case X86::VCVTSD2USIZrr_Int:
8191 case X86::VCVTSD2USI64Zrr_Int:
8192 case X86::VCVTTSD2USIZrr_Int:
8193 case X86::VCVTTSD2USI64Zrr_Int:
8194 case X86::ROUNDSDri_Int:
8195 case X86::VROUNDSDri_Int:
8196 case X86::COMISDrr_Int:
8197 case X86::VCOMISDrr_Int:
8198 case X86::VCOMISDZrr_Int:
8199 case X86::UCOMISDrr_Int:
8200 case X86::VUCOMISDrr_Int:
8201 case X86::VUCOMISDZrr_Int:
8202 case X86::ADDSDrr_Int:
8203 case X86::VADDSDrr_Int:
8204 case X86::VADDSDZrr_Int:
8205 case X86::CMPSDrri_Int:
8206 case X86::VCMPSDrri_Int:
8207 case X86::VCMPSDZrri_Int:
8208 case X86::DIVSDrr_Int:
8209 case X86::VDIVSDrr_Int:
8210 case X86::VDIVSDZrr_Int:
8211 case X86::MAXSDrr_Int:
8212 case X86::VMAXSDrr_Int:
8213 case X86::VMAXSDZrr_Int:
8214 case X86::MINSDrr_Int:
8215 case X86::VMINSDrr_Int:
8216 case X86::VMINSDZrr_Int:
8217 case X86::MULSDrr_Int:
8218 case X86::VMULSDrr_Int:
8219 case X86::VMULSDZrr_Int:
8220 case X86::SQRTSDr_Int:
8221 case X86::VSQRTSDr_Int:
8222 case X86::VSQRTSDZr_Int:
8223 case X86::SUBSDrr_Int:
8224 case X86::VSUBSDrr_Int:
8225 case X86::VSUBSDZrr_Int:
8226 case X86::VADDSDZrrk_Int:
8227 case X86::VADDSDZrrkz_Int:
8228 case X86::VCMPSDZrrik_Int:
8229 case X86::VDIVSDZrrk_Int:
8230 case X86::VDIVSDZrrkz_Int:
8231 case X86::VMAXSDZrrk_Int:
8232 case X86::VMAXSDZrrkz_Int:
8233 case X86::VMINSDZrrk_Int:
8234 case X86::VMINSDZrrkz_Int:
8235 case X86::VMULSDZrrk_Int:
8236 case X86::VMULSDZrrkz_Int:
8237 case X86::VSQRTSDZrk_Int:
8238 case X86::VSQRTSDZrkz_Int:
8239 case X86::VSUBSDZrrk_Int:
8240 case X86::VSUBSDZrrkz_Int:
8241 case X86::VFMADDSD4rr_Int:
8242 case X86::VFNMADDSD4rr_Int:
8243 case X86::VFMSUBSD4rr_Int:
8244 case X86::VFNMSUBSD4rr_Int:
8245 case X86::VFMADD132SDr_Int:
8246 case X86::VFNMADD132SDr_Int:
8247 case X86::VFMADD213SDr_Int:
8248 case X86::VFNMADD213SDr_Int:
8249 case X86::VFMADD231SDr_Int:
8250 case X86::VFNMADD231SDr_Int:
8251 case X86::VFMSUB132SDr_Int:
8252 case X86::VFNMSUB132SDr_Int:
8253 case X86::VFMSUB213SDr_Int:
8254 case X86::VFNMSUB213SDr_Int:
8255 case X86::VFMSUB231SDr_Int:
8256 case X86::VFNMSUB231SDr_Int:
8257 case X86::VFMADD132SDZr_Int:
8258 case X86::VFNMADD132SDZr_Int:
8259 case X86::VFMADD213SDZr_Int:
8260 case X86::VFNMADD213SDZr_Int:
8261 case X86::VFMADD231SDZr_Int:
8262 case X86::VFNMADD231SDZr_Int:
8263 case X86::VFMSUB132SDZr_Int:
8264 case X86::VFNMSUB132SDZr_Int:
8265 case X86::VFMSUB213SDZr_Int:
8266 case X86::VFNMSUB213SDZr_Int:
8267 case X86::VFMSUB231SDZr_Int:
8268 case X86::VFNMSUB231SDZr_Int:
8269 case X86::VFMADD132SDZrk_Int:
8270 case X86::VFNMADD132SDZrk_Int:
8271 case X86::VFMADD213SDZrk_Int:
8272 case X86::VFNMADD213SDZrk_Int:
8273 case X86::VFMADD231SDZrk_Int:
8274 case X86::VFNMADD231SDZrk_Int:
8275 case X86::VFMSUB132SDZrk_Int:
8276 case X86::VFNMSUB132SDZrk_Int:
8277 case X86::VFMSUB213SDZrk_Int:
8278 case X86::VFNMSUB213SDZrk_Int:
8279 case X86::VFMSUB231SDZrk_Int:
8280 case X86::VFNMSUB231SDZrk_Int:
8281 case X86::VFMADD132SDZrkz_Int:
8282 case X86::VFNMADD132SDZrkz_Int:
8283 case X86::VFMADD213SDZrkz_Int:
8284 case X86::VFNMADD213SDZrkz_Int:
8285 case X86::VFMADD231SDZrkz_Int:
8286 case X86::VFNMADD231SDZrkz_Int:
8287 case X86::VFMSUB132SDZrkz_Int:
8288 case X86::VFNMSUB132SDZrkz_Int:
8289 case X86::VFMSUB213SDZrkz_Int:
8290 case X86::VFNMSUB213SDZrkz_Int:
8291 case X86::VFMSUB231SDZrkz_Int:
8292 case X86::VFNMSUB231SDZrkz_Int:
8293 case X86::VFIXUPIMMSDZrri:
8294 case X86::VFIXUPIMMSDZrrik:
8295 case X86::VFIXUPIMMSDZrrikz:
8296 case X86::VFPCLASSSDZri:
8297 case X86::VFPCLASSSDZrik:
8298 case X86::VGETEXPSDZr:
8299 case X86::VGETEXPSDZrk:
8300 case X86::VGETEXPSDZrkz:
8301 case X86::VGETMANTSDZrri:
8302 case X86::VGETMANTSDZrrik:
8303 case X86::VGETMANTSDZrrikz:
8304 case X86::VRANGESDZrri:
8305 case X86::VRANGESDZrrik:
8306 case X86::VRANGESDZrrikz:
8307 case X86::VRCP14SDZrr:
8308 case X86::VRCP14SDZrrk:
8309 case X86::VRCP14SDZrrkz:
8310 case X86::VRCP28SDZr:
8311 case X86::VRCP28SDZrk:
8312 case X86::VRCP28SDZrkz:
8313 case X86::VREDUCESDZrri:
8314 case X86::VREDUCESDZrrik:
8315 case X86::VREDUCESDZrrikz:
8316 case X86::VRNDSCALESDZrri_Int:
8317 case X86::VRNDSCALESDZrrik_Int:
8318 case X86::VRNDSCALESDZrrikz_Int:
8319 case X86::VRSQRT14SDZrr:
8320 case X86::VRSQRT14SDZrrk:
8321 case X86::VRSQRT14SDZrrkz:
8322 case X86::VRSQRT28SDZr:
8323 case X86::VRSQRT28SDZrk:
8324 case X86::VRSQRT28SDZrkz:
8325 case X86::VSCALEFSDZrr:
8326 case X86::VSCALEFSDZrrk:
8327 case X86::VSCALEFSDZrrkz:
8328 return false;
8329 default:
8330 return true;
8331 }
8332 }
8333
8334 if ((Opc == X86::VMOVSHZrm || Opc == X86::VMOVSHZrm_alt) && RegSize > 16) {
8335 // These instructions only load 16 bits, we can't fold them if the
8336 // destination register is wider than 16 bits (2 bytes), and its user
8337 // instruction isn't scalar (SH).
8338 switch (UserOpc) {
8339 case X86::VADDSHZrr_Int:
8340 case X86::VCMPSHZrri_Int:
8341 case X86::VDIVSHZrr_Int:
8342 case X86::VMAXSHZrr_Int:
8343 case X86::VMINSHZrr_Int:
8344 case X86::VMULSHZrr_Int:
8345 case X86::VSUBSHZrr_Int:
8346 case X86::VADDSHZrrk_Int:
8347 case X86::VADDSHZrrkz_Int:
8348 case X86::VCMPSHZrrik_Int:
8349 case X86::VDIVSHZrrk_Int:
8350 case X86::VDIVSHZrrkz_Int:
8351 case X86::VMAXSHZrrk_Int:
8352 case X86::VMAXSHZrrkz_Int:
8353 case X86::VMINSHZrrk_Int:
8354 case X86::VMINSHZrrkz_Int:
8355 case X86::VMULSHZrrk_Int:
8356 case X86::VMULSHZrrkz_Int:
8357 case X86::VSUBSHZrrk_Int:
8358 case X86::VSUBSHZrrkz_Int:
8359 case X86::VFMADD132SHZr_Int:
8360 case X86::VFNMADD132SHZr_Int:
8361 case X86::VFMADD213SHZr_Int:
8362 case X86::VFNMADD213SHZr_Int:
8363 case X86::VFMADD231SHZr_Int:
8364 case X86::VFNMADD231SHZr_Int:
8365 case X86::VFMSUB132SHZr_Int:
8366 case X86::VFNMSUB132SHZr_Int:
8367 case X86::VFMSUB213SHZr_Int:
8368 case X86::VFNMSUB213SHZr_Int:
8369 case X86::VFMSUB231SHZr_Int:
8370 case X86::VFNMSUB231SHZr_Int:
8371 case X86::VFMADD132SHZrk_Int:
8372 case X86::VFNMADD132SHZrk_Int:
8373 case X86::VFMADD213SHZrk_Int:
8374 case X86::VFNMADD213SHZrk_Int:
8375 case X86::VFMADD231SHZrk_Int:
8376 case X86::VFNMADD231SHZrk_Int:
8377 case X86::VFMSUB132SHZrk_Int:
8378 case X86::VFNMSUB132SHZrk_Int:
8379 case X86::VFMSUB213SHZrk_Int:
8380 case X86::VFNMSUB213SHZrk_Int:
8381 case X86::VFMSUB231SHZrk_Int:
8382 case X86::VFNMSUB231SHZrk_Int:
8383 case X86::VFMADD132SHZrkz_Int:
8384 case X86::VFNMADD132SHZrkz_Int:
8385 case X86::VFMADD213SHZrkz_Int:
8386 case X86::VFNMADD213SHZrkz_Int:
8387 case X86::VFMADD231SHZrkz_Int:
8388 case X86::VFNMADD231SHZrkz_Int:
8389 case X86::VFMSUB132SHZrkz_Int:
8390 case X86::VFNMSUB132SHZrkz_Int:
8391 case X86::VFMSUB213SHZrkz_Int:
8392 case X86::VFNMSUB213SHZrkz_Int:
8393 case X86::VFMSUB231SHZrkz_Int:
8394 case X86::VFNMSUB231SHZrkz_Int:
8395 return false;
8396 default:
8397 return true;
8398 }
8399 }
8400
8401 return false;
8402}
8403
8407 MachineInstr &LoadMI, MachineInstr *&CopyMI,
8408 LiveIntervals *LIS, VirtRegMap *VRM) const {
8410
8411 // If LoadMI is a masked load, check MI having the same mask.
8412 const MCInstrDesc &MCID = get(LoadMI.getOpcode());
8413 unsigned NumOps = MCID.getNumOperands();
8414 if (NumOps >= 3) {
8415 Register MaskReg;
8416 const MachineOperand &Op1 = LoadMI.getOperand(1);
8417 const MachineOperand &Op2 = LoadMI.getOperand(2);
8418
8419 auto IsVKWMClass = [](const TargetRegisterClass *RC) {
8420 return RC == &X86::VK2WMRegClass || RC == &X86::VK4WMRegClass ||
8421 RC == &X86::VK8WMRegClass || RC == &X86::VK16WMRegClass ||
8422 RC == &X86::VK32WMRegClass || RC == &X86::VK64WMRegClass;
8423 };
8424
8425 if (Op1.isReg() && IsVKWMClass(getRegClass(MCID, 1)))
8426 MaskReg = Op1.getReg();
8427 else if (Op2.isReg() && IsVKWMClass(getRegClass(MCID, 2)))
8428 MaskReg = Op2.getReg();
8429
8430 if (MaskReg) {
8431 // Some instructions are invalid to fold into even with the same mask.
8432 // Folding is unsafe if an active destination element may read from a
8433 // source element that is masked off.
8434 if (isNonFoldableWithSameMask(MI.getOpcode()))
8435 return nullptr;
8436 bool HasSameMask = false;
8437 for (unsigned I = 1, E = MI.getDesc().getNumOperands(); I < E; ++I) {
8438 const MachineOperand &Op = MI.getOperand(I);
8439 if (Op.isReg() && Op.getReg() == MaskReg) {
8440 HasSameMask = true;
8441 break;
8442 }
8443 }
8444 if (!HasSameMask)
8445 return nullptr;
8446 }
8447 }
8448
8449 // TODO: Support the case where LoadMI loads a wide register, but MI
8450 // only uses a subreg.
8451 for (auto Op : Ops) {
8452 if (MI.getOperand(Op).getSubReg())
8453 return nullptr;
8454 }
8455
8456 // If loading from a FrameIndex, fold directly from the FrameIndex.
8457 int FrameIndex;
8458 if (isLoadFromStackSlot(LoadMI, FrameIndex)) {
8459 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF))
8460 return nullptr;
8461 return foldMemoryOperandImpl(MF, MI, Ops, FrameIndex, CopyMI, LIS, VRM);
8462 }
8463
8464 // Check switch flag
8465 if (Subtarget.getCLOpts().disable_spill_fusing)
8466 return nullptr;
8467
8468 // Avoid partial and undef register update stalls unless optimizing for size.
8469 if (!MF.getFunction().hasOptSize() &&
8470 (hasPartialRegUpdate(MI.getOpcode(), Subtarget, /*ForLoadFold*/ true) ||
8472 return nullptr;
8473
8474 // Do not fold a NDD instruction and a memory instruction with relocation to
8475 // avoid emit APX relocation when the flag is disabled for backward
8476 // compatibility.
8477 uint64_t TSFlags = MI.getDesc().TSFlags;
8478 if (!Subtarget.getCLOpts().enable_apx_for_relocation &&
8480 return nullptr;
8481
8482 // Determine the alignment of the load.
8483 Align Alignment;
8484 unsigned LoadOpc = LoadMI.getOpcode();
8485 if (LoadMI.hasOneMemOperand())
8486 Alignment = (*LoadMI.memoperands_begin())->getAlign();
8487 else
8488 switch (LoadOpc) {
8489 case X86::AVX512_512_SETALLONES:
8490 Alignment = Align(64);
8491 break;
8492 case X86::AVX2_SETALLONES:
8493 case X86::AVX1_SETALLONES:
8494 case X86::AVX512_256_SETALLONES:
8495 Alignment = Align(32);
8496 break;
8497 case X86::V_SET0:
8498 case X86::V_SETALLONES:
8499 case X86::AVX512_128_SET0:
8500 case X86::FsFLD0F128:
8501 case X86::AVX512_FsFLD0F128:
8502 case X86::AVX512_128_SETALLONES:
8503 Alignment = Align(16);
8504 break;
8505 case X86::MMX_SET0:
8506 case X86::FsFLD0SD:
8507 case X86::AVX512_FsFLD0SD:
8508 Alignment = Align(8);
8509 break;
8510 case X86::FsFLD0SS:
8511 case X86::AVX512_FsFLD0SS:
8512 Alignment = Align(4);
8513 break;
8514 case X86::FsFLD0SH:
8515 case X86::AVX512_FsFLD0SH:
8516 Alignment = Align(2);
8517 break;
8518 default:
8519 return nullptr;
8520 }
8521 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
8522 unsigned NewOpc = 0;
8523 switch (MI.getOpcode()) {
8524 default:
8525 return nullptr;
8526 case X86::TEST8rr:
8527 NewOpc = X86::CMP8ri;
8528 break;
8529 case X86::TEST16rr:
8530 NewOpc = X86::CMP16ri;
8531 break;
8532 case X86::TEST32rr:
8533 NewOpc = X86::CMP32ri;
8534 break;
8535 case X86::TEST64rr:
8536 NewOpc = X86::CMP64ri32;
8537 break;
8538 }
8539 // Change to CMPXXri r, 0 first.
8540 MI.setDesc(get(NewOpc));
8541 MI.getOperand(1).ChangeToImmediate(0);
8542 } else if (Ops.size() != 1)
8543 return nullptr;
8544
8545 // Make sure the subregisters match.
8546 // Otherwise we risk changing the size of the load.
8547 if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg())
8548 return nullptr;
8549
8551 switch (LoadOpc) {
8552 case X86::MMX_SET0:
8553 case X86::V_SET0:
8554 case X86::V_SETALLONES:
8555 case X86::AVX2_SETALLONES:
8556 case X86::AVX1_SETALLONES:
8557 case X86::AVX512_128_SET0:
8558 case X86::AVX512_128_SETALLONES:
8559 case X86::AVX512_256_SETALLONES:
8560 case X86::AVX512_512_SETALLONES:
8561 case X86::FsFLD0SH:
8562 case X86::AVX512_FsFLD0SH:
8563 case X86::FsFLD0SD:
8564 case X86::AVX512_FsFLD0SD:
8565 case X86::FsFLD0SS:
8566 case X86::AVX512_FsFLD0SS:
8567 case X86::FsFLD0F128:
8568 case X86::AVX512_FsFLD0F128: {
8569 // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure.
8570 // Create a constant-pool entry and operands to load from it.
8571
8572 // Large code model can't fold loads this way.
8574 return nullptr;
8575
8576 // x86-32 PIC requires a PIC base register for constant pools.
8577 unsigned PICBase = 0;
8578 // Since we're using Small or Kernel code model, we can always use
8579 // RIP-relative addressing for a smaller encoding.
8580 if (Subtarget.is64Bit()) {
8581 PICBase = X86::RIP;
8582 } else if (MF.getTarget().isPositionIndependent()) {
8583 // FIXME: PICBase = getGlobalBaseReg(&MF);
8584 // This doesn't work for several reasons.
8585 // 1. GlobalBaseReg may have been spilled.
8586 // 2. It may not be live at MI.
8587 return nullptr;
8588 }
8589
8590 // Create a constant-pool entry.
8592 Type *Ty;
8593 bool IsAllOnes = false;
8594 switch (LoadOpc) {
8595 case X86::FsFLD0SS:
8596 case X86::AVX512_FsFLD0SS:
8598 break;
8599 case X86::FsFLD0SD:
8600 case X86::AVX512_FsFLD0SD:
8602 break;
8603 case X86::FsFLD0F128:
8604 case X86::AVX512_FsFLD0F128:
8606 break;
8607 case X86::FsFLD0SH:
8608 case X86::AVX512_FsFLD0SH:
8610 break;
8611 case X86::AVX512_512_SETALLONES:
8612 IsAllOnes = true;
8614 16);
8615 break;
8616 case X86::AVX1_SETALLONES:
8617 case X86::AVX2_SETALLONES:
8618 case X86::AVX512_256_SETALLONES:
8619 IsAllOnes = true;
8621 8);
8622
8623 break;
8624 case X86::MMX_SET0:
8626 2);
8627 break;
8628 case X86::V_SETALLONES:
8629 case X86::AVX512_128_SETALLONES:
8630 IsAllOnes = true;
8631 [[fallthrough]];
8632 case X86::V_SET0:
8633 case X86::AVX512_128_SET0:
8635 4);
8636 break;
8637 }
8638
8639 const Constant *C =
8641 unsigned CPI = MCP.getConstantPoolIndex(C, Alignment);
8642
8643 // Create operands to load from the constant pool entry.
8644 MOs.push_back(MachineOperand::CreateReg(PICBase, false));
8646 MOs.push_back(MachineOperand::CreateReg(0, false));
8648 MOs.push_back(MachineOperand::CreateReg(0, false));
8649 break;
8650 }
8651 case X86::VPBROADCASTBZ128rm:
8652 case X86::VPBROADCASTBZ256rm:
8653 case X86::VPBROADCASTBZrm:
8654 case X86::VBROADCASTF32X2Z256rm:
8655 case X86::VBROADCASTF32X2Zrm:
8656 case X86::VBROADCASTI32X2Z128rm:
8657 case X86::VBROADCASTI32X2Z256rm:
8658 case X86::VBROADCASTI32X2Zrm:
8659 // No instructions currently fuse with 8bits or 32bits x 2.
8660 return nullptr;
8661
8662#define FOLD_BROADCAST(SIZE) \
8663 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, \
8664 LoadMI.operands_begin() + NumOps); \
8665 return foldMemoryBroadcast(MF, MI, Ops[0], MOs, InsertPt, /*Size=*/SIZE, \
8666 /*AllowCommute=*/true);
8667 case X86::VPBROADCASTWZ128rm:
8668 case X86::VPBROADCASTWZ256rm:
8669 case X86::VPBROADCASTWZrm:
8670 FOLD_BROADCAST(16);
8671 case X86::VPBROADCASTDZ128rm:
8672 case X86::VPBROADCASTDZ256rm:
8673 case X86::VPBROADCASTDZrm:
8674 case X86::VBROADCASTSSZ128rm:
8675 case X86::VBROADCASTSSZ256rm:
8676 case X86::VBROADCASTSSZrm:
8677 FOLD_BROADCAST(32);
8678 case X86::VPBROADCASTQZ128rm:
8679 case X86::VPBROADCASTQZ256rm:
8680 case X86::VPBROADCASTQZrm:
8681 case X86::VBROADCASTSDZ256rm:
8682 case X86::VBROADCASTSDZrm:
8683 FOLD_BROADCAST(64);
8684 default: {
8685 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF))
8686 return nullptr;
8687
8688 // Folding a normal load. Just copy the load's address operands.
8690 LoadMI.operands_begin() + NumOps);
8691 break;
8692 }
8693 }
8694 return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt,
8695 /*Size=*/0, Alignment, /*AllowCommute=*/true,
8696 CopyMI, VRM);
8697}
8698
8700X86InstrInfo::foldMemoryBroadcast(MachineFunction &MF, MachineInstr &MI,
8701 unsigned OpNum, ArrayRef<MachineOperand> MOs,
8703 unsigned BitsSize, bool AllowCommute) const {
8704
8705 if (auto *I = lookupBroadcastFoldTable(MI.getOpcode(), OpNum))
8706 return matchBroadcastSize(*I, BitsSize)
8707 ? fuseInst(MF, I->DstOp, OpNum, MOs, InsertPt, MI, *this)
8708 : nullptr;
8709
8710 if (AllowCommute) {
8711 // If the instruction and target operand are commutable, commute the
8712 // instruction and try again.
8713 unsigned CommuteOpIdx2 = commuteOperandsForFold(MI, OpNum);
8714 if (CommuteOpIdx2 == OpNum) {
8715 printFailMsgforFold(MI, OpNum);
8716 return nullptr;
8717 }
8718 MachineInstr *NewMI =
8719 foldMemoryBroadcast(MF, MI, CommuteOpIdx2, MOs, InsertPt, BitsSize,
8720 /*AllowCommute=*/false);
8721 if (NewMI)
8722 return NewMI;
8723 // Folding failed again - undo the commute before returning.
8724 commuteInstruction(MI, false, OpNum, CommuteOpIdx2);
8725 }
8726
8727 printFailMsgforFold(MI, OpNum);
8728 return nullptr;
8729}
8730
8734
8735 for (MachineMemOperand *MMO : MMOs) {
8736 if (!MMO->isLoad())
8737 continue;
8738
8739 if (!MMO->isStore()) {
8740 // Reuse the MMO.
8741 LoadMMOs.push_back(MMO);
8742 } else {
8743 // Clone the MMO and unset the store flag.
8744 LoadMMOs.push_back(MF.getMachineMemOperand(
8745 MMO, MMO->getFlags() & ~MachineMemOperand::MOStore));
8746 }
8747 }
8748
8749 return LoadMMOs;
8750}
8751
8755
8756 for (MachineMemOperand *MMO : MMOs) {
8757 if (!MMO->isStore())
8758 continue;
8759
8760 if (!MMO->isLoad()) {
8761 // Reuse the MMO.
8762 StoreMMOs.push_back(MMO);
8763 } else {
8764 // Clone the MMO and unset the load flag.
8765 StoreMMOs.push_back(MF.getMachineMemOperand(
8766 MMO, MMO->getFlags() & ~MachineMemOperand::MOLoad));
8767 }
8768 }
8769
8770 return StoreMMOs;
8771}
8772
8774 const TargetRegisterClass *RC,
8775 const X86Subtarget &STI) {
8776 assert(STI.hasAVX512() && "Expected at least AVX512!");
8777 unsigned SpillSize = STI.getRegisterInfo()->getSpillSize(*RC);
8778 assert((SpillSize == 64 || STI.hasVLX()) &&
8779 "Can't broadcast less than 64 bytes without AVX512VL!");
8780
8781#define CASE_BCAST_TYPE_OPC(TYPE, OP16, OP32, OP64) \
8782 case TYPE: \
8783 switch (SpillSize) { \
8784 default: \
8785 llvm_unreachable("Unknown spill size"); \
8786 case 16: \
8787 return X86::OP16; \
8788 case 32: \
8789 return X86::OP32; \
8790 case 64: \
8791 return X86::OP64; \
8792 } \
8793 break;
8794
8795 switch (I->Flags & TB_BCAST_MASK) {
8796 default:
8797 llvm_unreachable("Unexpected broadcast type!");
8798 CASE_BCAST_TYPE_OPC(TB_BCAST_W, VPBROADCASTWZ128rm, VPBROADCASTWZ256rm,
8799 VPBROADCASTWZrm)
8800 CASE_BCAST_TYPE_OPC(TB_BCAST_D, VPBROADCASTDZ128rm, VPBROADCASTDZ256rm,
8801 VPBROADCASTDZrm)
8802 CASE_BCAST_TYPE_OPC(TB_BCAST_Q, VPBROADCASTQZ128rm, VPBROADCASTQZ256rm,
8803 VPBROADCASTQZrm)
8804 CASE_BCAST_TYPE_OPC(TB_BCAST_SH, VPBROADCASTWZ128rm, VPBROADCASTWZ256rm,
8805 VPBROADCASTWZrm)
8806 CASE_BCAST_TYPE_OPC(TB_BCAST_SS, VBROADCASTSSZ128rm, VBROADCASTSSZ256rm,
8807 VBROADCASTSSZrm)
8808 CASE_BCAST_TYPE_OPC(TB_BCAST_SD, VMOVDDUPZ128rm, VBROADCASTSDZ256rm,
8809 VBROADCASTSDZrm)
8810 }
8811}
8812
8814 MachineFunction &MF, MachineInstr &MI, Register Reg, bool UnfoldLoad,
8815 bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const {
8816 const X86FoldTableEntry *I = lookupUnfoldTable(MI.getOpcode());
8817 if (I == nullptr)
8818 return false;
8819 unsigned Opc = I->DstOp;
8820 unsigned Index = I->Flags & TB_INDEX_MASK;
8821 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD;
8822 bool FoldedStore = I->Flags & TB_FOLDED_STORE;
8823 if (UnfoldLoad && !FoldedLoad)
8824 return false;
8825 UnfoldLoad &= FoldedLoad;
8826 if (UnfoldStore && !FoldedStore)
8827 return false;
8828 UnfoldStore &= FoldedStore;
8829
8830 const MCInstrDesc &MCID = get(Opc);
8831
8832 const TargetRegisterClass *RC = getRegClass(MCID, Index);
8834 // TODO: Check if 32-byte or greater accesses are slow too?
8835 if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass &&
8836 Subtarget.isUnalignedMem16Slow())
8837 // Without memoperands, storeRegToStackSlot will conservatively
8838 // assume the address is unaligned. That's bad for performance.
8839 return false;
8844 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
8845 MachineOperand &Op = MI.getOperand(i);
8846 if (i >= Index && i < Index + X86::AddrNumOperands)
8847 AddrOps.push_back(Op);
8848 else if (Op.isReg() && Op.isImplicit())
8849 ImpOps.push_back(Op);
8850 else if (i < Index)
8851 BeforeOps.push_back(Op);
8852 else if (i > Index)
8853 AfterOps.push_back(Op);
8854 }
8855
8856 // Emit the load or broadcast instruction.
8857 if (UnfoldLoad) {
8858 auto MMOs = extractLoadMMOs(MI.memoperands(), MF);
8859
8860 unsigned Opc;
8861 if (I->Flags & TB_BCAST_MASK) {
8862 Opc = getBroadcastOpcode(I, RC, Subtarget);
8863 } else {
8864 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16);
8865 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8866 Opc = getLoadRegOpcode(Reg, RC, isAligned, Subtarget);
8867 }
8868
8869 DebugLoc DL;
8870 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), Reg);
8871 for (const MachineOperand &AddrOp : AddrOps)
8872 MIB.add(AddrOp);
8873 MIB.setMemRefs(MMOs);
8874 NewMIs.push_back(MIB);
8875
8876 if (UnfoldStore) {
8877 // Address operands cannot be marked isKill.
8878 for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) {
8879 MachineOperand &MO = NewMIs[0]->getOperand(i);
8880 if (MO.isReg())
8881 MO.setIsKill(false);
8882 }
8883 }
8884 }
8885
8886 // Emit the data processing instruction.
8887 MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI.getDebugLoc(), true);
8888 MachineInstrBuilder MIB(MF, DataMI);
8889
8890 if (FoldedStore)
8891 MIB.addReg(Reg, RegState::Define);
8892 for (MachineOperand &BeforeOp : BeforeOps)
8893 MIB.add(BeforeOp);
8894 if (FoldedLoad)
8895 MIB.addReg(Reg);
8896 for (MachineOperand &AfterOp : AfterOps)
8897 MIB.add(AfterOp);
8898 for (MachineOperand &ImpOp : ImpOps) {
8899 MIB.addReg(ImpOp.getReg(), getDefRegState(ImpOp.isDef()) |
8901 getKillRegState(ImpOp.isKill()) |
8902 getDeadRegState(ImpOp.isDead()) |
8903 getUndefRegState(ImpOp.isUndef()));
8904 }
8905 // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
8906 switch (DataMI->getOpcode()) {
8907 default:
8908 break;
8909 case X86::CMP64ri32:
8910 case X86::CMP32ri:
8911 case X86::CMP16ri:
8912 case X86::CMP8ri: {
8913 MachineOperand &MO0 = DataMI->getOperand(0);
8914 MachineOperand &MO1 = DataMI->getOperand(1);
8915 if (MO1.isImm() && MO1.getImm() == 0) {
8916 unsigned NewOpc;
8917 switch (DataMI->getOpcode()) {
8918 default:
8919 llvm_unreachable("Unreachable!");
8920 case X86::CMP64ri32:
8921 NewOpc = X86::TEST64rr;
8922 break;
8923 case X86::CMP32ri:
8924 NewOpc = X86::TEST32rr;
8925 break;
8926 case X86::CMP16ri:
8927 NewOpc = X86::TEST16rr;
8928 break;
8929 case X86::CMP8ri:
8930 NewOpc = X86::TEST8rr;
8931 break;
8932 }
8933 DataMI->setDesc(get(NewOpc));
8934 MO1.ChangeToRegister(MO0.getReg(), false);
8935 }
8936 }
8937 }
8938 NewMIs.push_back(DataMI);
8939
8940 // Emit the store instruction.
8941 if (UnfoldStore) {
8942 const TargetRegisterClass *DstRC = getRegClass(MCID, 0);
8943 auto MMOs = extractStoreMMOs(MI.memoperands(), MF);
8944 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*DstRC), 16);
8945 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8946 unsigned Opc = getStoreRegOpcode(Reg, DstRC, isAligned, Subtarget);
8947 DebugLoc DL;
8948 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc));
8949 for (const MachineOperand &AddrOp : AddrOps)
8950 MIB.add(AddrOp);
8951 MIB.addReg(Reg, RegState::Kill);
8952 MIB.setMemRefs(MMOs);
8953 NewMIs.push_back(MIB);
8954 }
8955
8956 return true;
8957}
8958
8960 SelectionDAG &DAG, SDNode *N, SmallVectorImpl<SDNode *> &NewNodes) const {
8961 if (!N->isMachineOpcode())
8962 return false;
8963
8964 const X86FoldTableEntry *I = lookupUnfoldTable(N->getMachineOpcode());
8965 if (I == nullptr)
8966 return false;
8967 unsigned Opc = I->DstOp;
8968 unsigned Index = I->Flags & TB_INDEX_MASK;
8969 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD;
8970 bool FoldedStore = I->Flags & TB_FOLDED_STORE;
8971 const MCInstrDesc &MCID = get(Opc);
8974 const TargetRegisterClass *RC = getRegClass(MCID, Index);
8975 unsigned NumDefs = MCID.NumDefs;
8976 std::vector<SDValue> AddrOps;
8977 std::vector<SDValue> BeforeOps;
8978 std::vector<SDValue> AfterOps;
8979 SDLoc dl(N);
8980 unsigned NumOps = N->getNumOperands();
8981 for (unsigned i = 0; i != NumOps - 1; ++i) {
8982 SDValue Op = N->getOperand(i);
8983 if (i >= Index - NumDefs && i < Index - NumDefs + X86::AddrNumOperands)
8984 AddrOps.push_back(Op);
8985 else if (i < Index - NumDefs)
8986 BeforeOps.push_back(Op);
8987 else if (i > Index - NumDefs)
8988 AfterOps.push_back(Op);
8989 }
8990 SDValue Chain = N->getOperand(NumOps - 1);
8991 AddrOps.push_back(Chain);
8992
8993 // Emit the load instruction.
8994 SDNode *Load = nullptr;
8995 if (FoldedLoad) {
8996 EVT VT = *TRI.legalclasstypes_begin(*RC);
8997 auto MMOs = extractLoadMMOs(cast<MachineSDNode>(N)->memoperands(), MF);
8998 if (MMOs.empty() && RC == &X86::VR128RegClass &&
8999 Subtarget.isUnalignedMem16Slow())
9000 // Do not introduce a slow unaligned load.
9001 return false;
9002 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte
9003 // memory access is slow above.
9004
9005 unsigned Opc;
9006 if (I->Flags & TB_BCAST_MASK) {
9007 Opc = getBroadcastOpcode(I, RC, Subtarget);
9008 } else {
9009 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16);
9010 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
9011 Opc = getLoadRegOpcode(0, RC, isAligned, Subtarget);
9012 }
9013
9014 Load = DAG.getMachineNode(Opc, dl, VT, MVT::Other, AddrOps);
9015 NewNodes.push_back(Load);
9016
9017 // Preserve memory reference information.
9019 }
9020
9021 // Emit the data processing instruction.
9022 std::vector<EVT> VTs;
9023 const TargetRegisterClass *DstRC = nullptr;
9024 if (MCID.getNumDefs() > 0) {
9025 DstRC = getRegClass(MCID, 0);
9026 VTs.push_back(*TRI.legalclasstypes_begin(*DstRC));
9027 }
9028 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
9029 EVT VT = N->getValueType(i);
9030 if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs())
9031 VTs.push_back(VT);
9032 }
9033 if (Load)
9034 BeforeOps.push_back(SDValue(Load, 0));
9035 llvm::append_range(BeforeOps, AfterOps);
9036 // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
9037 switch (Opc) {
9038 default:
9039 break;
9040 case X86::CMP64ri32:
9041 case X86::CMP32ri:
9042 case X86::CMP16ri:
9043 case X86::CMP8ri:
9044 if (isNullConstant(BeforeOps[1])) {
9045 switch (Opc) {
9046 default:
9047 llvm_unreachable("Unreachable!");
9048 case X86::CMP64ri32:
9049 Opc = X86::TEST64rr;
9050 break;
9051 case X86::CMP32ri:
9052 Opc = X86::TEST32rr;
9053 break;
9054 case X86::CMP16ri:
9055 Opc = X86::TEST16rr;
9056 break;
9057 case X86::CMP8ri:
9058 Opc = X86::TEST8rr;
9059 break;
9060 }
9061 BeforeOps[1] = BeforeOps[0];
9062 }
9063 }
9064 SDNode *NewNode = DAG.getMachineNode(Opc, dl, VTs, BeforeOps);
9065 NewNodes.push_back(NewNode);
9066
9067 // Emit the store instruction.
9068 if (FoldedStore) {
9069 AddrOps.pop_back();
9070 AddrOps.push_back(SDValue(NewNode, 0));
9071 AddrOps.push_back(Chain);
9072 auto MMOs = extractStoreMMOs(cast<MachineSDNode>(N)->memoperands(), MF);
9073 if (MMOs.empty() && RC == &X86::VR128RegClass &&
9074 Subtarget.isUnalignedMem16Slow())
9075 // Do not introduce a slow unaligned store.
9076 return false;
9077 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte
9078 // memory access is slow above.
9079 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16);
9080 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
9081 SDNode *Store =
9082 DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget),
9083 dl, MVT::Other, AddrOps);
9084 NewNodes.push_back(Store);
9085
9086 // Preserve memory reference information.
9088 }
9089
9090 return true;
9091}
9092
9093unsigned
9095 bool UnfoldStore,
9096 unsigned *LoadRegIndex) const {
9098 if (I == nullptr)
9099 return 0;
9100 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD;
9101 bool FoldedStore = I->Flags & TB_FOLDED_STORE;
9102 if (UnfoldLoad && !FoldedLoad)
9103 return 0;
9104 if (UnfoldStore && !FoldedStore)
9105 return 0;
9106 if (LoadRegIndex)
9107 *LoadRegIndex = I->Flags & TB_INDEX_MASK;
9108 return I->DstOp;
9109}
9110
9112 int64_t &Offset1,
9113 int64_t &Offset2) const {
9114 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
9115 return false;
9116
9117 auto IsLoadOpcode = [&](unsigned Opcode) {
9118 switch (Opcode) {
9119 default:
9120 return false;
9121 case X86::MOV8rm:
9122 case X86::MOV16rm:
9123 case X86::MOV32rm:
9124 case X86::MOV64rm:
9125 case X86::LD_Fp32m:
9126 case X86::LD_Fp64m:
9127 case X86::LD_Fp80m:
9128 case X86::MOVSSrm:
9129 case X86::MOVSSrm_alt:
9130 case X86::MOVSDrm:
9131 case X86::MOVSDrm_alt:
9132 case X86::MMX_MOVD64rm:
9133 case X86::MMX_MOVQ64rm:
9134 case X86::MOVAPSrm:
9135 case X86::MOVUPSrm:
9136 case X86::MOVAPDrm:
9137 case X86::MOVUPDrm:
9138 case X86::MOVDQArm:
9139 case X86::MOVDQUrm:
9140 // AVX load instructions
9141 case X86::VMOVSSrm:
9142 case X86::VMOVSSrm_alt:
9143 case X86::VMOVSDrm:
9144 case X86::VMOVSDrm_alt:
9145 case X86::VMOVAPSrm:
9146 case X86::VMOVUPSrm:
9147 case X86::VMOVAPDrm:
9148 case X86::VMOVUPDrm:
9149 case X86::VMOVDQArm:
9150 case X86::VMOVDQUrm:
9151 case X86::VMOVAPSYrm:
9152 case X86::VMOVUPSYrm:
9153 case X86::VMOVAPDYrm:
9154 case X86::VMOVUPDYrm:
9155 case X86::VMOVDQAYrm:
9156 case X86::VMOVDQUYrm:
9157 // AVX512 load instructions
9158 case X86::VMOVSSZrm:
9159 case X86::VMOVSSZrm_alt:
9160 case X86::VMOVSDZrm:
9161 case X86::VMOVSDZrm_alt:
9162 case X86::VMOVAPSZ128rm:
9163 case X86::VMOVUPSZ128rm:
9164 case X86::VMOVAPSZ128rm_NOVLX:
9165 case X86::VMOVUPSZ128rm_NOVLX:
9166 case X86::VMOVAPDZ128rm:
9167 case X86::VMOVUPDZ128rm:
9168 case X86::VMOVDQU8Z128rm:
9169 case X86::VMOVDQU16Z128rm:
9170 case X86::VMOVDQA32Z128rm:
9171 case X86::VMOVDQU32Z128rm:
9172 case X86::VMOVDQA64Z128rm:
9173 case X86::VMOVDQU64Z128rm:
9174 case X86::VMOVAPSZ256rm:
9175 case X86::VMOVUPSZ256rm:
9176 case X86::VMOVAPSZ256rm_NOVLX:
9177 case X86::VMOVUPSZ256rm_NOVLX:
9178 case X86::VMOVAPDZ256rm:
9179 case X86::VMOVUPDZ256rm:
9180 case X86::VMOVDQU8Z256rm:
9181 case X86::VMOVDQU16Z256rm:
9182 case X86::VMOVDQA32Z256rm:
9183 case X86::VMOVDQU32Z256rm:
9184 case X86::VMOVDQA64Z256rm:
9185 case X86::VMOVDQU64Z256rm:
9186 case X86::VMOVAPSZrm:
9187 case X86::VMOVUPSZrm:
9188 case X86::VMOVAPDZrm:
9189 case X86::VMOVUPDZrm:
9190 case X86::VMOVDQU8Zrm:
9191 case X86::VMOVDQU16Zrm:
9192 case X86::VMOVDQA32Zrm:
9193 case X86::VMOVDQU32Zrm:
9194 case X86::VMOVDQA64Zrm:
9195 case X86::VMOVDQU64Zrm:
9196 case X86::KMOVBkm:
9197 case X86::KMOVBkm_EVEX:
9198 case X86::KMOVWkm:
9199 case X86::KMOVWkm_EVEX:
9200 case X86::KMOVDkm:
9201 case X86::KMOVDkm_EVEX:
9202 case X86::KMOVQkm:
9203 case X86::KMOVQkm_EVEX:
9204 return true;
9205 }
9206 };
9207
9208 if (!IsLoadOpcode(Load1->getMachineOpcode()) ||
9209 !IsLoadOpcode(Load2->getMachineOpcode()))
9210 return false;
9211
9212 // Lambda to check if both the loads have the same value for an operand index.
9213 auto HasSameOp = [&](int I) {
9214 return Load1->getOperand(I) == Load2->getOperand(I);
9215 };
9216
9217 // All operands except the displacement should match.
9218 if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) ||
9219 !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg))
9220 return false;
9221
9222 // Chain Operand must be the same.
9223 if (!HasSameOp(5))
9224 return false;
9225
9226 // Now let's examine if the displacements are constants.
9229 if (!Disp1 || !Disp2)
9230 return false;
9231
9232 Offset1 = Disp1->getSExtValue();
9233 Offset2 = Disp2->getSExtValue();
9234 return true;
9235}
9236
9238 int64_t Offset1, int64_t Offset2,
9239 unsigned NumLoads) const {
9240 assert(Offset2 > Offset1);
9241 if ((Offset2 - Offset1) / 8 > 64)
9242 return false;
9243
9244 unsigned Opc1 = Load1->getMachineOpcode();
9245 unsigned Opc2 = Load2->getMachineOpcode();
9246 if (Opc1 != Opc2)
9247 return false; // FIXME: overly conservative?
9248
9249 switch (Opc1) {
9250 default:
9251 break;
9252 case X86::LD_Fp32m:
9253 case X86::LD_Fp64m:
9254 case X86::LD_Fp80m:
9255 case X86::MMX_MOVD64rm:
9256 case X86::MMX_MOVQ64rm:
9257 return false;
9258 }
9259
9260 EVT VT = Load1->getValueType(0);
9261 switch (VT.getSimpleVT().SimpleTy) {
9262 default:
9263 // XMM registers. In 64-bit mode we can be a bit more aggressive since we
9264 // have 16 of them to play with.
9265 if (Subtarget.is64Bit()) {
9266 if (NumLoads >= 3)
9267 return false;
9268 } else if (NumLoads) {
9269 return false;
9270 }
9271 break;
9272 case MVT::i8:
9273 case MVT::i16:
9274 case MVT::i32:
9275 case MVT::i64:
9276 case MVT::f32:
9277 case MVT::f64:
9278 if (NumLoads)
9279 return false;
9280 break;
9281 }
9282
9283 return true;
9284}
9285
9287 const MachineBasicBlock *MBB,
9288 const MachineFunction &MF) const {
9289
9290 // ENDBR instructions should not be scheduled around.
9291 unsigned Opcode = MI.getOpcode();
9292 if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32 ||
9293 Opcode == X86::PLDTILECFGV)
9294 return true;
9295
9296 // Frame setup and destroy can't be scheduled around.
9297 if (MI.getFlag(MachineInstr::FrameSetup) ||
9299 return true;
9300
9302}
9303
9306 assert(Cond.size() == 1 && "Invalid X86 branch condition!");
9307 X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm());
9309 return false;
9310}
9311
9313 const TargetRegisterClass *RC) const {
9314 // FIXME: Return false for x87 stack register classes for now. We can't
9315 // allow any loads of these registers before FpGet_ST0_80.
9316 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass ||
9317 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass ||
9318 RC == &X86::RFP80RegClass);
9319}
9320
9321/// Return a virtual register initialized with the
9322/// the global base register value. Output instructions required to
9323/// initialize the register in the function entry block, if necessary.
9324///
9325/// TODO: Eliminate this and move the code to X86MachineFunctionInfo.
9326///
9329 Register GlobalBaseReg = X86FI->getGlobalBaseReg();
9330 if (GlobalBaseReg)
9331 return GlobalBaseReg;
9332
9333 // Create the register. The code to initialize it is inserted
9334 // later, by the CGBR pass (below).
9335 MachineRegisterInfo &RegInfo = MF->getRegInfo();
9336 GlobalBaseReg = RegInfo.createVirtualRegister(
9337 Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass);
9338 X86FI->setGlobalBaseReg(GlobalBaseReg);
9339 return GlobalBaseReg;
9340}
9341
9342// FIXME: Some shuffle and unpack instructions have equivalents in different
9343// domains, but they require a bit more work than just switching opcodes.
9344
9345static const uint16_t *lookup(unsigned opcode, unsigned domain,
9346 ArrayRef<uint16_t[3]> Table) {
9347 for (const uint16_t(&Row)[3] : Table)
9348 if (Row[domain - 1] == opcode)
9349 return Row;
9350 return nullptr;
9351}
9352
9353static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain,
9354 ArrayRef<uint16_t[4]> Table) {
9355 // If this is the integer domain make sure to check both integer columns.
9356 for (const uint16_t(&Row)[4] : Table)
9357 if (Row[domain - 1] == opcode || (domain == 3 && Row[3] == opcode))
9358 return Row;
9359 return nullptr;
9360}
9361
9362// Helper to attempt to widen/narrow blend masks.
9363static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth,
9364 unsigned NewWidth, unsigned *pNewMask = nullptr) {
9365 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) &&
9366 "Illegal blend mask scale");
9367 unsigned NewMask = 0;
9368
9369 if ((OldWidth % NewWidth) == 0) {
9370 unsigned Scale = OldWidth / NewWidth;
9371 unsigned SubMask = (1u << Scale) - 1;
9372 for (unsigned i = 0; i != NewWidth; ++i) {
9373 unsigned Sub = (OldMask >> (i * Scale)) & SubMask;
9374 if (Sub == SubMask)
9375 NewMask |= (1u << i);
9376 else if (Sub != 0x0)
9377 return false;
9378 }
9379 } else {
9380 unsigned Scale = NewWidth / OldWidth;
9381 unsigned SubMask = (1u << Scale) - 1;
9382 for (unsigned i = 0; i != OldWidth; ++i) {
9383 if (OldMask & (1 << i)) {
9384 NewMask |= (SubMask << (i * Scale));
9385 }
9386 }
9387 }
9388
9389 if (pNewMask)
9390 *pNewMask = NewMask;
9391 return true;
9392}
9393
9395 unsigned Opcode = MI.getOpcode();
9396 unsigned NumOperands = MI.getDesc().getNumOperands();
9397
9398 auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) {
9399 uint16_t validDomains = 0;
9400 if (MI.getOperand(NumOperands - 1).isImm()) {
9401 unsigned Imm = MI.getOperand(NumOperands - 1).getImm();
9402 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4))
9403 validDomains |= 0x2; // PackedSingle
9404 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2))
9405 validDomains |= 0x4; // PackedDouble
9406 if (!Is256 || Subtarget.hasAVX2())
9407 validDomains |= 0x8; // PackedInt
9408 }
9409 return validDomains;
9410 };
9411
9412 switch (Opcode) {
9413 case X86::BLENDPDrmi:
9414 case X86::BLENDPDrri:
9415 case X86::VBLENDPDrmi:
9416 case X86::VBLENDPDrri:
9417 return GetBlendDomains(2, false);
9418 case X86::VBLENDPDYrmi:
9419 case X86::VBLENDPDYrri:
9420 return GetBlendDomains(4, true);
9421 case X86::BLENDPSrmi:
9422 case X86::BLENDPSrri:
9423 case X86::VBLENDPSrmi:
9424 case X86::VBLENDPSrri:
9425 case X86::VPBLENDDrmi:
9426 case X86::VPBLENDDrri:
9427 return GetBlendDomains(4, false);
9428 case X86::VBLENDPSYrmi:
9429 case X86::VBLENDPSYrri:
9430 case X86::VPBLENDDYrmi:
9431 case X86::VPBLENDDYrri:
9432 return GetBlendDomains(8, true);
9433 case X86::PBLENDWrmi:
9434 case X86::PBLENDWrri:
9435 case X86::VPBLENDWrmi:
9436 case X86::VPBLENDWrri:
9437 // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks.
9438 case X86::VPBLENDWYrmi:
9439 case X86::VPBLENDWYrri:
9440 return GetBlendDomains(8, false);
9441 case X86::VPANDDZ128rr:
9442 case X86::VPANDDZ128rm:
9443 case X86::VPANDDZ256rr:
9444 case X86::VPANDDZ256rm:
9445 case X86::VPANDQZ128rr:
9446 case X86::VPANDQZ128rm:
9447 case X86::VPANDQZ256rr:
9448 case X86::VPANDQZ256rm:
9449 case X86::VPANDNDZ128rr:
9450 case X86::VPANDNDZ128rm:
9451 case X86::VPANDNDZ256rr:
9452 case X86::VPANDNDZ256rm:
9453 case X86::VPANDNQZ128rr:
9454 case X86::VPANDNQZ128rm:
9455 case X86::VPANDNQZ256rr:
9456 case X86::VPANDNQZ256rm:
9457 case X86::VPORDZ128rr:
9458 case X86::VPORDZ128rm:
9459 case X86::VPORDZ256rr:
9460 case X86::VPORDZ256rm:
9461 case X86::VPORQZ128rr:
9462 case X86::VPORQZ128rm:
9463 case X86::VPORQZ256rr:
9464 case X86::VPORQZ256rm:
9465 case X86::VPXORDZ128rr:
9466 case X86::VPXORDZ128rm:
9467 case X86::VPXORDZ256rr:
9468 case X86::VPXORDZ256rm:
9469 case X86::VPXORQZ128rr:
9470 case X86::VPXORQZ128rm:
9471 case X86::VPXORQZ256rr:
9472 case X86::VPXORQZ256rm:
9473 // If we don't have DQI see if we can still switch from an EVEX integer
9474 // instruction to a VEX floating point instruction.
9475 if (Subtarget.hasDQI())
9476 return 0;
9477
9478 if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16)
9479 return 0;
9480 if (RI.getEncodingValue(MI.getOperand(1).getReg()) >= 16)
9481 return 0;
9482 // Register forms will have 3 operands. Memory form will have more.
9483 if (NumOperands == 3 &&
9484 RI.getEncodingValue(MI.getOperand(2).getReg()) >= 16)
9485 return 0;
9486
9487 // All domains are valid.
9488 return 0xe;
9489 case X86::MOVHLPSrr:
9490 // We can swap domains when both inputs are the same register.
9491 // FIXME: This doesn't catch all the cases we would like. If the input
9492 // register isn't KILLed by the instruction, the two address instruction
9493 // pass puts a COPY on one input. The other input uses the original
9494 // register. This prevents the same physical register from being used by
9495 // both inputs.
9496 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg() &&
9497 MI.getOperand(0).getSubReg() == 0 &&
9498 MI.getOperand(1).getSubReg() == 0 && MI.getOperand(2).getSubReg() == 0)
9499 return 0x6;
9500 return 0;
9501 case X86::SHUFPDrri:
9502 return 0x6;
9503 }
9504 return 0;
9505}
9506
9507#include "X86ReplaceableInstrs.def"
9508
9510 unsigned Domain) const {
9511 assert(Domain > 0 && Domain < 4 && "Invalid execution domain");
9512 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
9513 assert(dom && "Not an SSE instruction");
9514
9515 unsigned Opcode = MI.getOpcode();
9516 unsigned NumOperands = MI.getDesc().getNumOperands();
9517
9518 auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) {
9519 if (MI.getOperand(NumOperands - 1).isImm()) {
9520 unsigned Imm = MI.getOperand(NumOperands - 1).getImm() & 255;
9521 Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm);
9522 unsigned NewImm = Imm;
9523
9524 const uint16_t *table = lookup(Opcode, dom, ReplaceableBlendInstrs);
9525 if (!table)
9526 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs);
9527
9528 if (Domain == 1) { // PackedSingle
9529 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm);
9530 } else if (Domain == 2) { // PackedDouble
9531 AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2, &NewImm);
9532 } else if (Domain == 3) { // PackedInt
9533 if (Subtarget.hasAVX2()) {
9534 // If we are already VPBLENDW use that, else use VPBLENDD.
9535 if ((ImmWidth / (Is256 ? 2 : 1)) != 8) {
9536 table = lookup(Opcode, dom, ReplaceableBlendAVX2Instrs);
9537 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm);
9538 }
9539 } else {
9540 assert(!Is256 && "128-bit vector expected");
9541 AdjustBlendMask(Imm, ImmWidth, 8, &NewImm);
9542 }
9543 }
9544
9545 assert(table && table[Domain - 1] && "Unknown domain op");
9546 MI.setDesc(get(table[Domain - 1]));
9547 MI.getOperand(NumOperands - 1).setImm(NewImm & 255);
9548 }
9549 return true;
9550 };
9551
9552 switch (Opcode) {
9553 case X86::BLENDPDrmi:
9554 case X86::BLENDPDrri:
9555 case X86::VBLENDPDrmi:
9556 case X86::VBLENDPDrri:
9557 return SetBlendDomain(2, false);
9558 case X86::VBLENDPDYrmi:
9559 case X86::VBLENDPDYrri:
9560 return SetBlendDomain(4, true);
9561 case X86::BLENDPSrmi:
9562 case X86::BLENDPSrri:
9563 case X86::VBLENDPSrmi:
9564 case X86::VBLENDPSrri:
9565 case X86::VPBLENDDrmi:
9566 case X86::VPBLENDDrri:
9567 return SetBlendDomain(4, false);
9568 case X86::VBLENDPSYrmi:
9569 case X86::VBLENDPSYrri:
9570 case X86::VPBLENDDYrmi:
9571 case X86::VPBLENDDYrri:
9572 return SetBlendDomain(8, true);
9573 case X86::PBLENDWrmi:
9574 case X86::PBLENDWrri:
9575 case X86::VPBLENDWrmi:
9576 case X86::VPBLENDWrri:
9577 return SetBlendDomain(8, false);
9578 case X86::VPBLENDWYrmi:
9579 case X86::VPBLENDWYrri:
9580 return SetBlendDomain(16, true);
9581 case X86::VPANDDZ128rr:
9582 case X86::VPANDDZ128rm:
9583 case X86::VPANDDZ256rr:
9584 case X86::VPANDDZ256rm:
9585 case X86::VPANDQZ128rr:
9586 case X86::VPANDQZ128rm:
9587 case X86::VPANDQZ256rr:
9588 case X86::VPANDQZ256rm:
9589 case X86::VPANDNDZ128rr:
9590 case X86::VPANDNDZ128rm:
9591 case X86::VPANDNDZ256rr:
9592 case X86::VPANDNDZ256rm:
9593 case X86::VPANDNQZ128rr:
9594 case X86::VPANDNQZ128rm:
9595 case X86::VPANDNQZ256rr:
9596 case X86::VPANDNQZ256rm:
9597 case X86::VPORDZ128rr:
9598 case X86::VPORDZ128rm:
9599 case X86::VPORDZ256rr:
9600 case X86::VPORDZ256rm:
9601 case X86::VPORQZ128rr:
9602 case X86::VPORQZ128rm:
9603 case X86::VPORQZ256rr:
9604 case X86::VPORQZ256rm:
9605 case X86::VPXORDZ128rr:
9606 case X86::VPXORDZ128rm:
9607 case X86::VPXORDZ256rr:
9608 case X86::VPXORDZ256rm:
9609 case X86::VPXORQZ128rr:
9610 case X86::VPXORQZ128rm:
9611 case X86::VPXORQZ256rr:
9612 case X86::VPXORQZ256rm: {
9613 // Without DQI, convert EVEX instructions to VEX instructions.
9614 if (Subtarget.hasDQI())
9615 return false;
9616
9617 const uint16_t *table =
9618 lookupAVX512(MI.getOpcode(), dom, ReplaceableCustomAVX512LogicInstrs);
9619 assert(table && "Instruction not found in table?");
9620 // Don't change integer Q instructions to D instructions and
9621 // use D instructions if we started with a PS instruction.
9622 if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode()))
9623 Domain = 4;
9624 MI.setDesc(get(table[Domain - 1]));
9625 return true;
9626 }
9627 case X86::UNPCKHPDrr:
9628 case X86::MOVHLPSrr:
9629 // We just need to commute the instruction which will switch the domains.
9630 if (Domain != dom && Domain != 3 &&
9631 MI.getOperand(1).getReg() == MI.getOperand(2).getReg() &&
9632 MI.getOperand(0).getSubReg() == 0 &&
9633 MI.getOperand(1).getSubReg() == 0 &&
9634 MI.getOperand(2).getSubReg() == 0) {
9635 commuteInstruction(MI, false);
9636 return true;
9637 }
9638 // We must always return true for MOVHLPSrr.
9639 if (Opcode == X86::MOVHLPSrr)
9640 return true;
9641 break;
9642 case X86::SHUFPDrri: {
9643 if (Domain == 1) {
9644 unsigned Imm = MI.getOperand(3).getImm();
9645 unsigned NewImm = 0x44;
9646 if (Imm & 1)
9647 NewImm |= 0x0a;
9648 if (Imm & 2)
9649 NewImm |= 0xa0;
9650 MI.getOperand(3).setImm(NewImm);
9651 MI.setDesc(get(X86::SHUFPSrri));
9652 }
9653 return true;
9654 }
9655 }
9656 return false;
9657}
9658
9659std::pair<uint16_t, uint16_t>
9661 uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
9662 unsigned opcode = MI.getOpcode();
9663 uint16_t validDomains = 0;
9664 if (domain) {
9665 // Attempt to match for custom instructions.
9666 validDomains = getExecutionDomainCustom(MI);
9667 if (validDomains)
9668 return std::make_pair(domain, validDomains);
9669
9670 if (lookup(opcode, domain, ReplaceableInstrs)) {
9671 validDomains = 0xe;
9672 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2)) {
9673 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6;
9674 } else if (lookup(opcode, domain, ReplaceableInstrsFP)) {
9675 validDomains = 0x6;
9676 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) {
9677 // Insert/extract instructions should only effect domain if AVX2
9678 // is enabled.
9679 if (!Subtarget.hasAVX2())
9680 return std::make_pair(0, 0);
9681 validDomains = 0xe;
9682 } else if (lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) {
9683 validDomains = 0xe;
9684 } else if (Subtarget.hasDQI() &&
9685 lookupAVX512(opcode, domain, ReplaceableInstrsAVX512DQ)) {
9686 validDomains = 0xe;
9687 } else if (Subtarget.hasDQI()) {
9688 if (const uint16_t *table =
9689 lookupAVX512(opcode, domain, ReplaceableInstrsAVX512DQMasked)) {
9690 if (domain == 1 || (domain == 3 && table[3] == opcode))
9691 validDomains = 0xa;
9692 else
9693 validDomains = 0xc;
9694 }
9695 }
9696 }
9697 return std::make_pair(domain, validDomains);
9698}
9699
9701 assert(Domain > 0 && Domain < 4 && "Invalid execution domain");
9702 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
9703 assert(dom && "Not an SSE instruction");
9704
9705 // Attempt to match for custom instructions.
9707 return;
9708
9709 const uint16_t *table = lookup(MI.getOpcode(), dom, ReplaceableInstrs);
9710 if (!table) { // try the other table
9711 assert((Subtarget.hasAVX2() || Domain < 3) &&
9712 "256-bit vector operations only available in AVX2");
9713 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2);
9714 }
9715 if (!table) { // try the FP table
9716 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsFP);
9717 assert((!table || Domain < 3) &&
9718 "Can only select PackedSingle or PackedDouble");
9719 }
9720 if (!table) { // try the other table
9721 assert(Subtarget.hasAVX2() &&
9722 "256-bit insert/extract only available in AVX2");
9723 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract);
9724 }
9725 if (!table) { // try the AVX512 table
9726 assert(Subtarget.hasAVX512() && "Requires AVX-512");
9727 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512);
9728 // Don't change integer Q instructions to D instructions.
9729 if (table && Domain == 3 && table[3] == MI.getOpcode())
9730 Domain = 4;
9731 }
9732 if (!table) { // try the AVX512DQ table
9733 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ");
9734 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ);
9735 // Don't change integer Q instructions to D instructions and
9736 // use D instructions if we started with a PS instruction.
9737 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode()))
9738 Domain = 4;
9739 }
9740 if (!table) { // try the AVX512DQMasked table
9741 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ");
9742 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked);
9743 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode()))
9744 Domain = 4;
9745 }
9746 assert(table && "Cannot change domain");
9747 MI.setDesc(get(table[Domain - 1]));
9748}
9749
9755
9756/// Return the noop instruction to use for a noop.
9758 MCInst Nop;
9759 Nop.setOpcode(X86::NOOP);
9760 return Nop;
9761}
9762
9764 switch (opc) {
9765 default:
9766 return false;
9767 case X86::DIVPDrm:
9768 case X86::DIVPDrr:
9769 case X86::DIVPSrm:
9770 case X86::DIVPSrr:
9771 case X86::DIVSDrm:
9772 case X86::DIVSDrm_Int:
9773 case X86::DIVSDrr:
9774 case X86::DIVSDrr_Int:
9775 case X86::DIVSSrm:
9776 case X86::DIVSSrm_Int:
9777 case X86::DIVSSrr:
9778 case X86::DIVSSrr_Int:
9779 case X86::SQRTPDm:
9780 case X86::SQRTPDr:
9781 case X86::SQRTPSm:
9782 case X86::SQRTPSr:
9783 case X86::SQRTSDm:
9784 case X86::SQRTSDm_Int:
9785 case X86::SQRTSDr:
9786 case X86::SQRTSDr_Int:
9787 case X86::SQRTSSm:
9788 case X86::SQRTSSm_Int:
9789 case X86::SQRTSSr:
9790 case X86::SQRTSSr_Int:
9791 // AVX instructions with high latency
9792 case X86::VDIVPDrm:
9793 case X86::VDIVPDrr:
9794 case X86::VDIVPDYrm:
9795 case X86::VDIVPDYrr:
9796 case X86::VDIVPSrm:
9797 case X86::VDIVPSrr:
9798 case X86::VDIVPSYrm:
9799 case X86::VDIVPSYrr:
9800 case X86::VDIVSDrm:
9801 case X86::VDIVSDrm_Int:
9802 case X86::VDIVSDrr:
9803 case X86::VDIVSDrr_Int:
9804 case X86::VDIVSSrm:
9805 case X86::VDIVSSrm_Int:
9806 case X86::VDIVSSrr:
9807 case X86::VDIVSSrr_Int:
9808 case X86::VSQRTPDm:
9809 case X86::VSQRTPDr:
9810 case X86::VSQRTPDYm:
9811 case X86::VSQRTPDYr:
9812 case X86::VSQRTPSm:
9813 case X86::VSQRTPSr:
9814 case X86::VSQRTPSYm:
9815 case X86::VSQRTPSYr:
9816 case X86::VSQRTSDm:
9817 case X86::VSQRTSDm_Int:
9818 case X86::VSQRTSDr:
9819 case X86::VSQRTSDr_Int:
9820 case X86::VSQRTSSm:
9821 case X86::VSQRTSSm_Int:
9822 case X86::VSQRTSSr:
9823 case X86::VSQRTSSr_Int:
9824 // AVX512 instructions with high latency
9825 case X86::VDIVPDZ128rm:
9826 case X86::VDIVPDZ128rmb:
9827 case X86::VDIVPDZ128rmbk:
9828 case X86::VDIVPDZ128rmbkz:
9829 case X86::VDIVPDZ128rmk:
9830 case X86::VDIVPDZ128rmkz:
9831 case X86::VDIVPDZ128rr:
9832 case X86::VDIVPDZ128rrk:
9833 case X86::VDIVPDZ128rrkz:
9834 case X86::VDIVPDZ256rm:
9835 case X86::VDIVPDZ256rmb:
9836 case X86::VDIVPDZ256rmbk:
9837 case X86::VDIVPDZ256rmbkz:
9838 case X86::VDIVPDZ256rmk:
9839 case X86::VDIVPDZ256rmkz:
9840 case X86::VDIVPDZ256rr:
9841 case X86::VDIVPDZ256rrk:
9842 case X86::VDIVPDZ256rrkz:
9843 case X86::VDIVPDZrrb:
9844 case X86::VDIVPDZrrbk:
9845 case X86::VDIVPDZrrbkz:
9846 case X86::VDIVPDZrm:
9847 case X86::VDIVPDZrmb:
9848 case X86::VDIVPDZrmbk:
9849 case X86::VDIVPDZrmbkz:
9850 case X86::VDIVPDZrmk:
9851 case X86::VDIVPDZrmkz:
9852 case X86::VDIVPDZrr:
9853 case X86::VDIVPDZrrk:
9854 case X86::VDIVPDZrrkz:
9855 case X86::VDIVPSZ128rm:
9856 case X86::VDIVPSZ128rmb:
9857 case X86::VDIVPSZ128rmbk:
9858 case X86::VDIVPSZ128rmbkz:
9859 case X86::VDIVPSZ128rmk:
9860 case X86::VDIVPSZ128rmkz:
9861 case X86::VDIVPSZ128rr:
9862 case X86::VDIVPSZ128rrk:
9863 case X86::VDIVPSZ128rrkz:
9864 case X86::VDIVPSZ256rm:
9865 case X86::VDIVPSZ256rmb:
9866 case X86::VDIVPSZ256rmbk:
9867 case X86::VDIVPSZ256rmbkz:
9868 case X86::VDIVPSZ256rmk:
9869 case X86::VDIVPSZ256rmkz:
9870 case X86::VDIVPSZ256rr:
9871 case X86::VDIVPSZ256rrk:
9872 case X86::VDIVPSZ256rrkz:
9873 case X86::VDIVPSZrrb:
9874 case X86::VDIVPSZrrbk:
9875 case X86::VDIVPSZrrbkz:
9876 case X86::VDIVPSZrm:
9877 case X86::VDIVPSZrmb:
9878 case X86::VDIVPSZrmbk:
9879 case X86::VDIVPSZrmbkz:
9880 case X86::VDIVPSZrmk:
9881 case X86::VDIVPSZrmkz:
9882 case X86::VDIVPSZrr:
9883 case X86::VDIVPSZrrk:
9884 case X86::VDIVPSZrrkz:
9885 case X86::VDIVSDZrm:
9886 case X86::VDIVSDZrr:
9887 case X86::VDIVSDZrm_Int:
9888 case X86::VDIVSDZrmk_Int:
9889 case X86::VDIVSDZrmkz_Int:
9890 case X86::VDIVSDZrr_Int:
9891 case X86::VDIVSDZrrk_Int:
9892 case X86::VDIVSDZrrkz_Int:
9893 case X86::VDIVSDZrrb_Int:
9894 case X86::VDIVSDZrrbk_Int:
9895 case X86::VDIVSDZrrbkz_Int:
9896 case X86::VDIVSSZrm:
9897 case X86::VDIVSSZrr:
9898 case X86::VDIVSSZrm_Int:
9899 case X86::VDIVSSZrmk_Int:
9900 case X86::VDIVSSZrmkz_Int:
9901 case X86::VDIVSSZrr_Int:
9902 case X86::VDIVSSZrrk_Int:
9903 case X86::VDIVSSZrrkz_Int:
9904 case X86::VDIVSSZrrb_Int:
9905 case X86::VDIVSSZrrbk_Int:
9906 case X86::VDIVSSZrrbkz_Int:
9907 case X86::VSQRTPDZ128m:
9908 case X86::VSQRTPDZ128mb:
9909 case X86::VSQRTPDZ128mbk:
9910 case X86::VSQRTPDZ128mbkz:
9911 case X86::VSQRTPDZ128mk:
9912 case X86::VSQRTPDZ128mkz:
9913 case X86::VSQRTPDZ128r:
9914 case X86::VSQRTPDZ128rk:
9915 case X86::VSQRTPDZ128rkz:
9916 case X86::VSQRTPDZ256m:
9917 case X86::VSQRTPDZ256mb:
9918 case X86::VSQRTPDZ256mbk:
9919 case X86::VSQRTPDZ256mbkz:
9920 case X86::VSQRTPDZ256mk:
9921 case X86::VSQRTPDZ256mkz:
9922 case X86::VSQRTPDZ256r:
9923 case X86::VSQRTPDZ256rk:
9924 case X86::VSQRTPDZ256rkz:
9925 case X86::VSQRTPDZm:
9926 case X86::VSQRTPDZmb:
9927 case X86::VSQRTPDZmbk:
9928 case X86::VSQRTPDZmbkz:
9929 case X86::VSQRTPDZmk:
9930 case X86::VSQRTPDZmkz:
9931 case X86::VSQRTPDZr:
9932 case X86::VSQRTPDZrb:
9933 case X86::VSQRTPDZrbk:
9934 case X86::VSQRTPDZrbkz:
9935 case X86::VSQRTPDZrk:
9936 case X86::VSQRTPDZrkz:
9937 case X86::VSQRTPSZ128m:
9938 case X86::VSQRTPSZ128mb:
9939 case X86::VSQRTPSZ128mbk:
9940 case X86::VSQRTPSZ128mbkz:
9941 case X86::VSQRTPSZ128mk:
9942 case X86::VSQRTPSZ128mkz:
9943 case X86::VSQRTPSZ128r:
9944 case X86::VSQRTPSZ128rk:
9945 case X86::VSQRTPSZ128rkz:
9946 case X86::VSQRTPSZ256m:
9947 case X86::VSQRTPSZ256mb:
9948 case X86::VSQRTPSZ256mbk:
9949 case X86::VSQRTPSZ256mbkz:
9950 case X86::VSQRTPSZ256mk:
9951 case X86::VSQRTPSZ256mkz:
9952 case X86::VSQRTPSZ256r:
9953 case X86::VSQRTPSZ256rk:
9954 case X86::VSQRTPSZ256rkz:
9955 case X86::VSQRTPSZm:
9956 case X86::VSQRTPSZmb:
9957 case X86::VSQRTPSZmbk:
9958 case X86::VSQRTPSZmbkz:
9959 case X86::VSQRTPSZmk:
9960 case X86::VSQRTPSZmkz:
9961 case X86::VSQRTPSZr:
9962 case X86::VSQRTPSZrb:
9963 case X86::VSQRTPSZrbk:
9964 case X86::VSQRTPSZrbkz:
9965 case X86::VSQRTPSZrk:
9966 case X86::VSQRTPSZrkz:
9967 case X86::VSQRTSDZm:
9968 case X86::VSQRTSDZm_Int:
9969 case X86::VSQRTSDZmk_Int:
9970 case X86::VSQRTSDZmkz_Int:
9971 case X86::VSQRTSDZr:
9972 case X86::VSQRTSDZr_Int:
9973 case X86::VSQRTSDZrk_Int:
9974 case X86::VSQRTSDZrkz_Int:
9975 case X86::VSQRTSDZrb_Int:
9976 case X86::VSQRTSDZrbk_Int:
9977 case X86::VSQRTSDZrbkz_Int:
9978 case X86::VSQRTSSZm:
9979 case X86::VSQRTSSZm_Int:
9980 case X86::VSQRTSSZmk_Int:
9981 case X86::VSQRTSSZmkz_Int:
9982 case X86::VSQRTSSZr:
9983 case X86::VSQRTSSZr_Int:
9984 case X86::VSQRTSSZrk_Int:
9985 case X86::VSQRTSSZrkz_Int:
9986 case X86::VSQRTSSZrb_Int:
9987 case X86::VSQRTSSZrbk_Int:
9988 case X86::VSQRTSSZrbkz_Int:
9989
9990 case X86::VGATHERDPDYrm:
9991 case X86::VGATHERDPDZ128rm:
9992 case X86::VGATHERDPDZ256rm:
9993 case X86::VGATHERDPDZrm:
9994 case X86::VGATHERDPDrm:
9995 case X86::VGATHERDPSYrm:
9996 case X86::VGATHERDPSZ128rm:
9997 case X86::VGATHERDPSZ256rm:
9998 case X86::VGATHERDPSZrm:
9999 case X86::VGATHERDPSrm:
10000 case X86::VGATHERPF0DPDm:
10001 case X86::VGATHERPF0DPSm:
10002 case X86::VGATHERPF0QPDm:
10003 case X86::VGATHERPF0QPSm:
10004 case X86::VGATHERPF1DPDm:
10005 case X86::VGATHERPF1DPSm:
10006 case X86::VGATHERPF1QPDm:
10007 case X86::VGATHERPF1QPSm:
10008 case X86::VGATHERQPDYrm:
10009 case X86::VGATHERQPDZ128rm:
10010 case X86::VGATHERQPDZ256rm:
10011 case X86::VGATHERQPDZrm:
10012 case X86::VGATHERQPDrm:
10013 case X86::VGATHERQPSYrm:
10014 case X86::VGATHERQPSZ128rm:
10015 case X86::VGATHERQPSZ256rm:
10016 case X86::VGATHERQPSZrm:
10017 case X86::VGATHERQPSrm:
10018 case X86::VPGATHERDDYrm:
10019 case X86::VPGATHERDDZ128rm:
10020 case X86::VPGATHERDDZ256rm:
10021 case X86::VPGATHERDDZrm:
10022 case X86::VPGATHERDDrm:
10023 case X86::VPGATHERDQYrm:
10024 case X86::VPGATHERDQZ128rm:
10025 case X86::VPGATHERDQZ256rm:
10026 case X86::VPGATHERDQZrm:
10027 case X86::VPGATHERDQrm:
10028 case X86::VPGATHERQDYrm:
10029 case X86::VPGATHERQDZ128rm:
10030 case X86::VPGATHERQDZ256rm:
10031 case X86::VPGATHERQDZrm:
10032 case X86::VPGATHERQDrm:
10033 case X86::VPGATHERQQYrm:
10034 case X86::VPGATHERQQZ128rm:
10035 case X86::VPGATHERQQZ256rm:
10036 case X86::VPGATHERQQZrm:
10037 case X86::VPGATHERQQrm:
10038 case X86::VSCATTERDPDZ128mr:
10039 case X86::VSCATTERDPDZ256mr:
10040 case X86::VSCATTERDPDZmr:
10041 case X86::VSCATTERDPSZ128mr:
10042 case X86::VSCATTERDPSZ256mr:
10043 case X86::VSCATTERDPSZmr:
10044 case X86::VSCATTERPF0DPDm:
10045 case X86::VSCATTERPF0DPSm:
10046 case X86::VSCATTERPF0QPDm:
10047 case X86::VSCATTERPF0QPSm:
10048 case X86::VSCATTERPF1DPDm:
10049 case X86::VSCATTERPF1DPSm:
10050 case X86::VSCATTERPF1QPDm:
10051 case X86::VSCATTERPF1QPSm:
10052 case X86::VSCATTERQPDZ128mr:
10053 case X86::VSCATTERQPDZ256mr:
10054 case X86::VSCATTERQPDZmr:
10055 case X86::VSCATTERQPSZ128mr:
10056 case X86::VSCATTERQPSZ256mr:
10057 case X86::VSCATTERQPSZmr:
10058 case X86::VPSCATTERDDZ128mr:
10059 case X86::VPSCATTERDDZ256mr:
10060 case X86::VPSCATTERDDZmr:
10061 case X86::VPSCATTERDQZ128mr:
10062 case X86::VPSCATTERDQZ256mr:
10063 case X86::VPSCATTERDQZmr:
10064 case X86::VPSCATTERQDZ128mr:
10065 case X86::VPSCATTERQDZ256mr:
10066 case X86::VPSCATTERQDZmr:
10067 case X86::VPSCATTERQQZ128mr:
10068 case X86::VPSCATTERQQZ256mr:
10069 case X86::VPSCATTERQQZmr:
10070 return true;
10071 }
10072}
10073
10075 const MachineRegisterInfo *MRI,
10076 const MachineInstr &DefMI,
10077 unsigned DefIdx,
10078 const MachineInstr &UseMI,
10079 unsigned UseIdx) const {
10080 return isHighLatencyDef(DefMI.getOpcode());
10081}
10082
10084 const MachineBasicBlock *MBB) const {
10085 assert(Inst.getNumExplicitOperands() == 3 && Inst.getNumExplicitDefs() == 1 &&
10086 Inst.getNumDefs() <= 2 && "Reassociation needs binary operators");
10087
10088 // Integer binary math/logic instructions have a third source operand:
10089 // the EFLAGS register. That operand must be both defined here and never
10090 // used; ie, it must be dead. If the EFLAGS operand is live, then we can
10091 // not change anything because rearranging the operands could affect other
10092 // instructions that depend on the exact status flags (zero, sign, etc.)
10093 // that are set by using these particular operands with this operation.
10094 const MachineOperand *FlagDef =
10095 Inst.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10096 assert((Inst.getNumDefs() == 1 || FlagDef) && "Implicit def isn't flags?");
10097 if (FlagDef && !FlagDef->isDead())
10098 return false;
10099
10101}
10102
10103// TODO: There are many more machine instruction opcodes to match:
10104// 1. Other data types (integer, vectors)
10105// 2. Other math / logic operations (xor, or)
10106// 3. Other forms of the same operation (intrinsics and other variants)
10108 bool Invert) const {
10109 if (Invert)
10110 return false;
10111 switch (Inst.getOpcode()) {
10112 CASE_ND(ADD8rr)
10113 CASE_ND(ADD16rr)
10114 CASE_ND(ADD32rr)
10115 CASE_ND(ADD64rr)
10116 CASE_ND(AND8rr)
10117 CASE_ND(AND16rr)
10118 CASE_ND(AND32rr)
10119 CASE_ND(AND64rr)
10120 CASE_ND(OR8rr)
10121 CASE_ND(OR16rr)
10122 CASE_ND(OR32rr)
10123 CASE_ND(OR64rr)
10124 CASE_ND(XOR8rr)
10125 CASE_ND(XOR16rr)
10126 CASE_ND(XOR32rr)
10127 CASE_ND(XOR64rr)
10128 CASE_ND(IMUL16rr)
10129 CASE_ND(IMUL32rr)
10130 CASE_ND(IMUL64rr)
10131 case X86::PANDrr:
10132 case X86::PORrr:
10133 case X86::PXORrr:
10134 case X86::ANDPDrr:
10135 case X86::ANDPSrr:
10136 case X86::ORPDrr:
10137 case X86::ORPSrr:
10138 case X86::XORPDrr:
10139 case X86::XORPSrr:
10140 case X86::PADDBrr:
10141 case X86::PADDWrr:
10142 case X86::PADDDrr:
10143 case X86::PADDQrr:
10144 case X86::PMULLWrr:
10145 case X86::PMULLDrr:
10146 case X86::PMAXSBrr:
10147 case X86::PMAXSDrr:
10148 case X86::PMAXSWrr:
10149 case X86::PMAXUBrr:
10150 case X86::PMAXUDrr:
10151 case X86::PMAXUWrr:
10152 case X86::PMINSBrr:
10153 case X86::PMINSDrr:
10154 case X86::PMINSWrr:
10155 case X86::PMINUBrr:
10156 case X86::PMINUDrr:
10157 case X86::PMINUWrr:
10158 case X86::VPANDrr:
10159 case X86::VPANDYrr:
10160 case X86::VPANDDZ128rr:
10161 case X86::VPANDDZ256rr:
10162 case X86::VPANDDZrr:
10163 case X86::VPANDQZ128rr:
10164 case X86::VPANDQZ256rr:
10165 case X86::VPANDQZrr:
10166 case X86::VPORrr:
10167 case X86::VPORYrr:
10168 case X86::VPORDZ128rr:
10169 case X86::VPORDZ256rr:
10170 case X86::VPORDZrr:
10171 case X86::VPORQZ128rr:
10172 case X86::VPORQZ256rr:
10173 case X86::VPORQZrr:
10174 case X86::VPXORrr:
10175 case X86::VPXORYrr:
10176 case X86::VPXORDZ128rr:
10177 case X86::VPXORDZ256rr:
10178 case X86::VPXORDZrr:
10179 case X86::VPXORQZ128rr:
10180 case X86::VPXORQZ256rr:
10181 case X86::VPXORQZrr:
10182 case X86::VANDPDrr:
10183 case X86::VANDPSrr:
10184 case X86::VANDPDYrr:
10185 case X86::VANDPSYrr:
10186 case X86::VANDPDZ128rr:
10187 case X86::VANDPSZ128rr:
10188 case X86::VANDPDZ256rr:
10189 case X86::VANDPSZ256rr:
10190 case X86::VANDPDZrr:
10191 case X86::VANDPSZrr:
10192 case X86::VORPDrr:
10193 case X86::VORPSrr:
10194 case X86::VORPDYrr:
10195 case X86::VORPSYrr:
10196 case X86::VORPDZ128rr:
10197 case X86::VORPSZ128rr:
10198 case X86::VORPDZ256rr:
10199 case X86::VORPSZ256rr:
10200 case X86::VORPDZrr:
10201 case X86::VORPSZrr:
10202 case X86::VXORPDrr:
10203 case X86::VXORPSrr:
10204 case X86::VXORPDYrr:
10205 case X86::VXORPSYrr:
10206 case X86::VXORPDZ128rr:
10207 case X86::VXORPSZ128rr:
10208 case X86::VXORPDZ256rr:
10209 case X86::VXORPSZ256rr:
10210 case X86::VXORPDZrr:
10211 case X86::VXORPSZrr:
10212 case X86::KADDBkk:
10213 case X86::KADDWkk:
10214 case X86::KADDDkk:
10215 case X86::KADDQkk:
10216 case X86::KANDBkk:
10217 case X86::KANDWkk:
10218 case X86::KANDDkk:
10219 case X86::KANDQkk:
10220 case X86::KORBkk:
10221 case X86::KORWkk:
10222 case X86::KORDkk:
10223 case X86::KORQkk:
10224 case X86::KXORBkk:
10225 case X86::KXORWkk:
10226 case X86::KXORDkk:
10227 case X86::KXORQkk:
10228 case X86::VPADDBrr:
10229 case X86::VPADDWrr:
10230 case X86::VPADDDrr:
10231 case X86::VPADDQrr:
10232 case X86::VPADDBYrr:
10233 case X86::VPADDWYrr:
10234 case X86::VPADDDYrr:
10235 case X86::VPADDQYrr:
10236 case X86::VPADDBZ128rr:
10237 case X86::VPADDWZ128rr:
10238 case X86::VPADDDZ128rr:
10239 case X86::VPADDQZ128rr:
10240 case X86::VPADDBZ256rr:
10241 case X86::VPADDWZ256rr:
10242 case X86::VPADDDZ256rr:
10243 case X86::VPADDQZ256rr:
10244 case X86::VPADDBZrr:
10245 case X86::VPADDWZrr:
10246 case X86::VPADDDZrr:
10247 case X86::VPADDQZrr:
10248 case X86::VPMULLWrr:
10249 case X86::VPMULLWYrr:
10250 case X86::VPMULLWZ128rr:
10251 case X86::VPMULLWZ256rr:
10252 case X86::VPMULLWZrr:
10253 case X86::VPMULLDrr:
10254 case X86::VPMULLDYrr:
10255 case X86::VPMULLDZ128rr:
10256 case X86::VPMULLDZ256rr:
10257 case X86::VPMULLDZrr:
10258 case X86::VPMULLQZ128rr:
10259 case X86::VPMULLQZ256rr:
10260 case X86::VPMULLQZrr:
10261 case X86::VPMAXSBrr:
10262 case X86::VPMAXSBYrr:
10263 case X86::VPMAXSBZ128rr:
10264 case X86::VPMAXSBZ256rr:
10265 case X86::VPMAXSBZrr:
10266 case X86::VPMAXSDrr:
10267 case X86::VPMAXSDYrr:
10268 case X86::VPMAXSDZ128rr:
10269 case X86::VPMAXSDZ256rr:
10270 case X86::VPMAXSDZrr:
10271 case X86::VPMAXSQZ128rr:
10272 case X86::VPMAXSQZ256rr:
10273 case X86::VPMAXSQZrr:
10274 case X86::VPMAXSWrr:
10275 case X86::VPMAXSWYrr:
10276 case X86::VPMAXSWZ128rr:
10277 case X86::VPMAXSWZ256rr:
10278 case X86::VPMAXSWZrr:
10279 case X86::VPMAXUBrr:
10280 case X86::VPMAXUBYrr:
10281 case X86::VPMAXUBZ128rr:
10282 case X86::VPMAXUBZ256rr:
10283 case X86::VPMAXUBZrr:
10284 case X86::VPMAXUDrr:
10285 case X86::VPMAXUDYrr:
10286 case X86::VPMAXUDZ128rr:
10287 case X86::VPMAXUDZ256rr:
10288 case X86::VPMAXUDZrr:
10289 case X86::VPMAXUQZ128rr:
10290 case X86::VPMAXUQZ256rr:
10291 case X86::VPMAXUQZrr:
10292 case X86::VPMAXUWrr:
10293 case X86::VPMAXUWYrr:
10294 case X86::VPMAXUWZ128rr:
10295 case X86::VPMAXUWZ256rr:
10296 case X86::VPMAXUWZrr:
10297 case X86::VPMINSBrr:
10298 case X86::VPMINSBYrr:
10299 case X86::VPMINSBZ128rr:
10300 case X86::VPMINSBZ256rr:
10301 case X86::VPMINSBZrr:
10302 case X86::VPMINSDrr:
10303 case X86::VPMINSDYrr:
10304 case X86::VPMINSDZ128rr:
10305 case X86::VPMINSDZ256rr:
10306 case X86::VPMINSDZrr:
10307 case X86::VPMINSQZ128rr:
10308 case X86::VPMINSQZ256rr:
10309 case X86::VPMINSQZrr:
10310 case X86::VPMINSWrr:
10311 case X86::VPMINSWYrr:
10312 case X86::VPMINSWZ128rr:
10313 case X86::VPMINSWZ256rr:
10314 case X86::VPMINSWZrr:
10315 case X86::VPMINUBrr:
10316 case X86::VPMINUBYrr:
10317 case X86::VPMINUBZ128rr:
10318 case X86::VPMINUBZ256rr:
10319 case X86::VPMINUBZrr:
10320 case X86::VPMINUDrr:
10321 case X86::VPMINUDYrr:
10322 case X86::VPMINUDZ128rr:
10323 case X86::VPMINUDZ256rr:
10324 case X86::VPMINUDZrr:
10325 case X86::VPMINUQZ128rr:
10326 case X86::VPMINUQZ256rr:
10327 case X86::VPMINUQZrr:
10328 case X86::VPMINUWrr:
10329 case X86::VPMINUWYrr:
10330 case X86::VPMINUWZ128rr:
10331 case X86::VPMINUWZ256rr:
10332 case X86::VPMINUWZrr:
10333 // Normal min/max instructions are not commutative because of NaN and signed
10334 // zero semantics, but these are. Thus, there's no need to check for global
10335 // relaxed math; the instructions themselves have the properties we need.
10336 case X86::MAXCPDrr:
10337 case X86::MAXCPSrr:
10338 case X86::MAXCSDrr:
10339 case X86::MAXCSSrr:
10340 case X86::MINCPDrr:
10341 case X86::MINCPSrr:
10342 case X86::MINCSDrr:
10343 case X86::MINCSSrr:
10344 case X86::VMAXCPDrr:
10345 case X86::VMAXCPSrr:
10346 case X86::VMAXCPDYrr:
10347 case X86::VMAXCPSYrr:
10348 case X86::VMAXCPDZ128rr:
10349 case X86::VMAXCPSZ128rr:
10350 case X86::VMAXCPDZ256rr:
10351 case X86::VMAXCPSZ256rr:
10352 case X86::VMAXCPDZrr:
10353 case X86::VMAXCPSZrr:
10354 case X86::VMAXCSDrr:
10355 case X86::VMAXCSSrr:
10356 case X86::VMAXCSDZrr:
10357 case X86::VMAXCSSZrr:
10358 case X86::VMINCPDrr:
10359 case X86::VMINCPSrr:
10360 case X86::VMINCPDYrr:
10361 case X86::VMINCPSYrr:
10362 case X86::VMINCPDZ128rr:
10363 case X86::VMINCPSZ128rr:
10364 case X86::VMINCPDZ256rr:
10365 case X86::VMINCPSZ256rr:
10366 case X86::VMINCPDZrr:
10367 case X86::VMINCPSZrr:
10368 case X86::VMINCSDrr:
10369 case X86::VMINCSSrr:
10370 case X86::VMINCSDZrr:
10371 case X86::VMINCSSZrr:
10372 case X86::VMAXCPHZ128rr:
10373 case X86::VMAXCPHZ256rr:
10374 case X86::VMAXCPHZrr:
10375 case X86::VMAXCSHZrr:
10376 case X86::VMINCPHZ128rr:
10377 case X86::VMINCPHZ256rr:
10378 case X86::VMINCPHZrr:
10379 case X86::VMINCSHZrr:
10380 return true;
10381 case X86::ADDPDrr:
10382 case X86::ADDPSrr:
10383 case X86::ADDSDrr:
10384 case X86::ADDSSrr:
10385 case X86::MULPDrr:
10386 case X86::MULPSrr:
10387 case X86::MULSDrr:
10388 case X86::MULSSrr:
10389 case X86::VADDPDrr:
10390 case X86::VADDPSrr:
10391 case X86::VADDPDYrr:
10392 case X86::VADDPSYrr:
10393 case X86::VADDPDZ128rr:
10394 case X86::VADDPSZ128rr:
10395 case X86::VADDPDZ256rr:
10396 case X86::VADDPSZ256rr:
10397 case X86::VADDPDZrr:
10398 case X86::VADDPSZrr:
10399 case X86::VADDSDrr:
10400 case X86::VADDSSrr:
10401 case X86::VADDSDZrr:
10402 case X86::VADDSSZrr:
10403 case X86::VMULPDrr:
10404 case X86::VMULPSrr:
10405 case X86::VMULPDYrr:
10406 case X86::VMULPSYrr:
10407 case X86::VMULPDZ128rr:
10408 case X86::VMULPSZ128rr:
10409 case X86::VMULPDZ256rr:
10410 case X86::VMULPSZ256rr:
10411 case X86::VMULPDZrr:
10412 case X86::VMULPSZrr:
10413 case X86::VMULSDrr:
10414 case X86::VMULSSrr:
10415 case X86::VMULSDZrr:
10416 case X86::VMULSSZrr:
10417 case X86::VADDPHZ128rr:
10418 case X86::VADDPHZ256rr:
10419 case X86::VADDPHZrr:
10420 case X86::VADDSHZrr:
10421 case X86::VMULPHZ128rr:
10422 case X86::VMULPHZ256rr:
10423 case X86::VMULPHZrr:
10424 case X86::VMULSHZrr:
10427 default:
10428 return false;
10429 }
10430}
10431
10432/// If \p DescribedReg overlaps with the MOVrr instruction's destination
10433/// register then, if possible, describe the value in terms of the source
10434/// register.
10435static std::optional<ParamLoadedValue>
10437 const TargetRegisterInfo *TRI) {
10438 Register DestReg = MI.getOperand(0).getReg();
10439 Register SrcReg = MI.getOperand(1).getReg();
10440
10441 auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {});
10442
10443 // If the described register is the destination, just return the source.
10444 if (DestReg == DescribedReg)
10445 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
10446
10447 // If the described register is a sub-register of the destination register,
10448 // then pick out the source register's corresponding sub-register.
10449 if (unsigned SubRegIdx = TRI->getSubRegIndex(DestReg, DescribedReg)) {
10450 Register SrcSubReg = TRI->getSubReg(SrcReg, SubRegIdx);
10451 return ParamLoadedValue(MachineOperand::CreateReg(SrcSubReg, false), Expr);
10452 }
10453
10454 // The remaining case to consider is when the described register is a
10455 // super-register of the destination register. MOV8rr and MOV16rr does not
10456 // write to any of the other bytes in the register, meaning that we'd have to
10457 // describe the value using a combination of the source register and the
10458 // non-overlapping bits in the described register, which is not currently
10459 // possible.
10460 if (MI.getOpcode() == X86::MOV8rr || MI.getOpcode() == X86::MOV16rr ||
10461 !TRI->isSuperRegister(DestReg, DescribedReg))
10462 return std::nullopt;
10463
10464 assert(MI.getOpcode() == X86::MOV32rr && "Unexpected super-register case");
10465 return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr);
10466}
10467
10468std::optional<ParamLoadedValue>
10470 const MachineOperand *Op = nullptr;
10471 DIExpression *Expr = nullptr;
10472
10474
10475 switch (MI.getOpcode()) {
10476 case X86::LEA32r:
10477 case X86::LEA64r:
10478 case X86::LEA64_32r: {
10479 // We may need to describe a 64-bit parameter with a 32-bit LEA.
10480 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
10481 return std::nullopt;
10482
10483 // Operand 4 could be global address. For now we do not support
10484 // such situation.
10485 if (!MI.getOperand(4).isImm() || !MI.getOperand(2).isImm())
10486 return std::nullopt;
10487
10488 const MachineOperand &Op1 = MI.getOperand(1);
10489 const MachineOperand &Op2 = MI.getOperand(3);
10490 assert(Op2.isReg() &&
10491 (Op2.getReg() == X86::NoRegister || Op2.getReg().isPhysical()));
10492
10493 // Omit situations like:
10494 // %rsi = lea %rsi, 4, ...
10495 if ((Op1.isReg() && Op1.getReg() == MI.getOperand(0).getReg()) ||
10496 Op2.getReg() == MI.getOperand(0).getReg())
10497 return std::nullopt;
10498 else if ((Op1.isReg() && Op1.getReg() != X86::NoRegister &&
10499 TRI->regsOverlap(Op1.getReg(), MI.getOperand(0).getReg())) ||
10500 (Op2.getReg() != X86::NoRegister &&
10501 TRI->regsOverlap(Op2.getReg(), MI.getOperand(0).getReg())))
10502 return std::nullopt;
10503
10504 int64_t Coef = MI.getOperand(2).getImm();
10505 int64_t Offset = MI.getOperand(4).getImm();
10507
10508 if ((Op1.isReg() && Op1.getReg() != X86::NoRegister)) {
10509 Op = &Op1;
10510 } else if (Op1.isFI())
10511 Op = &Op1;
10512
10513 if (Op && Op->isReg() && Op->getReg() == Op2.getReg() && Coef > 0) {
10514 Ops.push_back(dwarf::DW_OP_constu);
10515 Ops.push_back(Coef + 1);
10516 Ops.push_back(dwarf::DW_OP_mul);
10517 } else {
10518 if (Op && Op2.getReg() != X86::NoRegister) {
10519 int dwarfReg = TRI->getDwarfRegNum(Op2.getReg(), false);
10520 if (dwarfReg < 0)
10521 return std::nullopt;
10522 else if (dwarfReg < 32) {
10523 Ops.push_back(dwarf::DW_OP_breg0 + dwarfReg);
10524 Ops.push_back(0);
10525 } else {
10526 Ops.push_back(dwarf::DW_OP_bregx);
10527 Ops.push_back(dwarfReg);
10528 Ops.push_back(0);
10529 }
10530 } else if (!Op) {
10531 assert(Op2.getReg() != X86::NoRegister);
10532 Op = &Op2;
10533 }
10534
10535 if (Coef > 1) {
10536 assert(Op2.getReg() != X86::NoRegister);
10537 Ops.push_back(dwarf::DW_OP_constu);
10538 Ops.push_back(Coef);
10539 Ops.push_back(dwarf::DW_OP_mul);
10540 }
10541
10542 if (((Op1.isReg() && Op1.getReg() != X86::NoRegister) || Op1.isFI()) &&
10543 Op2.getReg() != X86::NoRegister) {
10544 Ops.push_back(dwarf::DW_OP_plus);
10545 }
10546 }
10547
10549 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), Ops);
10550
10551 return ParamLoadedValue(*Op, Expr);
10552 }
10553 case X86::MOV8ri:
10554 case X86::MOV16ri:
10555 // TODO: Handle MOV8ri and MOV16ri.
10556 return std::nullopt;
10557 case X86::MOV32ri:
10558 case X86::MOV64ri:
10559 case X86::MOV64ri32:
10560 // MOV32ri may be used for producing zero-extended 32-bit immediates in
10561 // 64-bit parameters, so we need to consider super-registers.
10562 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
10563 return std::nullopt;
10564 return ParamLoadedValue(MI.getOperand(1), Expr);
10565 case X86::MOV8rr:
10566 case X86::MOV16rr:
10567 case X86::MOV32rr:
10568 case X86::MOV64rr:
10569 return describeMOVrrLoadedValue(MI, Reg, TRI);
10570 case X86::XOR32rr: {
10571 // 64-bit parameters are zero-materialized using XOR32rr, so also consider
10572 // super-registers.
10573 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
10574 return std::nullopt;
10575 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
10577 return std::nullopt;
10578 }
10579 case X86::MOVSX64rr32: {
10580 // We may need to describe the lower 32 bits of the MOVSX; for example, in
10581 // cases like this:
10582 //
10583 // $ebx = [...]
10584 // $rdi = MOVSX64rr32 $ebx
10585 // $esi = MOV32rr $edi
10586 if (!TRI->isSubRegisterEq(MI.getOperand(0).getReg(), Reg))
10587 return std::nullopt;
10588
10589 Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {});
10590
10591 // If the described register is the destination register we need to
10592 // sign-extend the source register from 32 bits. The other case we handle
10593 // is when the described register is the 32-bit sub-register of the
10594 // destination register, in case we just need to return the source
10595 // register.
10596 if (Reg == MI.getOperand(0).getReg())
10597 Expr = DIExpression::appendExt(Expr, 32, 64, true);
10598 else
10599 assert(getX86MCRegisterClass(X86::GR32RegClassID).contains(Reg) &&
10600 "Unhandled sub-register case for MOVSX64rr32");
10601
10602 return ParamLoadedValue(MI.getOperand(1), Expr);
10603 }
10604 default:
10605 assert(!MI.isMoveImmediate() && "Unexpected MoveImm instruction");
10607 }
10608}
10609
10610/// This is an architecture-specific helper function of reassociateOps.
10611/// Set special operand attributes for new instructions after reassociation.
10613 MachineInstr &OldMI2,
10614 MachineInstr &NewMI1,
10615 MachineInstr &NewMI2) const {
10616 // Integer instructions may define an implicit EFLAGS dest register operand.
10617 MachineOperand *OldFlagDef1 =
10618 OldMI1.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10619 MachineOperand *OldFlagDef2 =
10620 OldMI2.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10621
10622 assert(!OldFlagDef1 == !OldFlagDef2 &&
10623 "Unexpected instruction type for reassociation");
10624
10625 if (!OldFlagDef1 || !OldFlagDef2)
10626 return;
10627
10628 assert(OldFlagDef1->isDead() && OldFlagDef2->isDead() &&
10629 "Must have dead EFLAGS operand in reassociable instruction");
10630
10631 MachineOperand *NewFlagDef1 =
10632 NewMI1.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10633 MachineOperand *NewFlagDef2 =
10634 NewMI2.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10635
10636 assert(NewFlagDef1 && NewFlagDef2 &&
10637 "Unexpected operand in reassociable instruction");
10638
10639 // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations
10640 // of this pass or other passes. The EFLAGS operands must be dead in these new
10641 // instructions because the EFLAGS operands in the original instructions must
10642 // be dead in order for reassociation to occur.
10643 NewFlagDef1->setIsDead();
10644 NewFlagDef2->setIsDead();
10645}
10646
10647std::pair<unsigned, unsigned>
10649 return std::make_pair(TF, 0u);
10650}
10651
10654 using namespace X86II;
10655 static const std::pair<unsigned, const char *> TargetFlags[] = {
10656 {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"},
10657 {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"},
10658 {MO_GOT, "x86-got"},
10659 {MO_GOTOFF, "x86-gotoff"},
10660 {MO_GOTPCREL, "x86-gotpcrel"},
10661 {MO_GOTPCREL_NORELAX, "x86-gotpcrel-norelax"},
10662 {MO_PLT, "x86-plt"},
10663 {MO_TLSGD, "x86-tlsgd"},
10664 {MO_TLSLD, "x86-tlsld"},
10665 {MO_TLSLDM, "x86-tlsldm"},
10666 {MO_GOTTPOFF, "x86-gottpoff"},
10667 {MO_INDNTPOFF, "x86-indntpoff"},
10668 {MO_TPOFF, "x86-tpoff"},
10669 {MO_DTPOFF, "x86-dtpoff"},
10670 {MO_NTPOFF, "x86-ntpoff"},
10671 {MO_GOTNTPOFF, "x86-gotntpoff"},
10672 {MO_DLLIMPORT, "x86-dllimport"},
10673 {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"},
10674 {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"},
10675 {MO_TLVP, "x86-tlvp"},
10676 {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"},
10677 {MO_SECREL, "x86-secrel"},
10678 {MO_COFFSTUB, "x86-coffstub"}};
10679 return ArrayRef(TargetFlags);
10680}
10681
10682/// Constants defining how certain sequences should be outlined.
10683///
10684/// \p MachineOutlinerDefault implies that the function is called with a call
10685/// instruction, and a return must be emitted for the outlined function frame.
10686///
10687/// That is,
10688///
10689/// I1 OUTLINED_FUNCTION:
10690/// I2 --> call OUTLINED_FUNCTION I1
10691/// I3 I2
10692/// I3
10693/// ret
10694///
10695/// * Call construction overhead: 1 (call instruction)
10696/// * Frame construction overhead: 1 (return instruction)
10697///
10698/// \p MachineOutlinerTailCall implies that the function is being tail called.
10699/// A jump is emitted instead of a call, and the return is already present in
10700/// the outlined sequence. That is,
10701///
10702/// I1 OUTLINED_FUNCTION:
10703/// I2 --> jmp OUTLINED_FUNCTION I1
10704/// ret I2
10705/// ret
10706///
10707/// * Call construction overhead: 1 (jump instruction)
10708/// * Frame construction overhead: 0 (don't need to return)
10709///
10711
10712std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10714 const MachineModuleInfo &MMI,
10715 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10716 unsigned MinRepeats) const {
10717 unsigned SequenceSize = 0;
10718 for (auto &MI : RepeatedSequenceLocs[0]) {
10719 // FIXME: x86 doesn't implement getInstSizeInBytes, so
10720 // we can't tell the cost. Just assume each instruction
10721 // is one byte.
10722 if (MI.isDebugInstr() || MI.isKill())
10723 continue;
10724 SequenceSize += 1;
10725 }
10726
10727 // We check to see if CFI Instructions are present, and if they are
10728 // we find the number of CFI Instructions in the candidates.
10729 unsigned CFICount = 0;
10730 for (auto &I : RepeatedSequenceLocs[0]) {
10731 if (I.isCFIInstruction())
10732 CFICount++;
10733 }
10734
10735 // We compare the number of found CFI Instructions to the number of CFI
10736 // instructions in the parent function for each candidate. We must check this
10737 // since if we outline one of the CFI instructions in a function, we have to
10738 // outline them all for correctness. If we do not, the address offsets will be
10739 // incorrect between the two sections of the program.
10740 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10741 std::vector<MCCFIInstruction> CFIInstructions =
10742 C.getMF()->getFrameInstructions();
10743
10744 if (CFICount > 0 && CFICount != CFIInstructions.size())
10745 return std::nullopt;
10746 }
10747
10748 // FIXME: Use real size in bytes for call and ret instructions.
10749 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10750 for (outliner::Candidate &C : RepeatedSequenceLocs)
10751 C.setCallInfo(MachineOutlinerTailCall, 1);
10752
10753 return std::make_unique<outliner::OutlinedFunction>(
10754 RepeatedSequenceLocs, SequenceSize,
10755 0, // Number of bytes to emit frame.
10756 MachineOutlinerTailCall // Type of frame.
10757 );
10758 }
10759
10760 if (CFICount > 0)
10761 return std::nullopt;
10762
10763 for (outliner::Candidate &C : RepeatedSequenceLocs)
10764 C.setCallInfo(MachineOutlinerDefault, 1);
10765
10766 return std::make_unique<outliner::OutlinedFunction>(
10767 RepeatedSequenceLocs, SequenceSize, 1, MachineOutlinerDefault);
10768}
10769
10771 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
10772 const Function &F = MF.getFunction();
10773
10774 // Does the function use a red zone? If it does, then we can't risk messing
10775 // with the stack.
10776 if (Subtarget.getFrameLowering()->has128ByteRedZone(MF)) {
10777 // It could have a red zone. If it does, then we don't want to touch it.
10779 if (!X86FI || X86FI->getUsesRedZone())
10780 return false;
10781 }
10782
10783 // If we *don't* want to outline from things that could potentially be deduped
10784 // then return false.
10785 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
10786 return false;
10787
10788 // This function is viable for outlining, so return true.
10789 return true;
10790}
10791
10795 unsigned Flags) const {
10796 MachineInstr &MI = *MIT;
10797
10798 // Is this a terminator for a basic block?
10799 if (MI.isTerminator())
10800 // TargetInstrInfo::getOutliningType has already filtered out anything
10801 // that would break this, so we can allow it here.
10803
10804 // Don't outline anything that modifies or reads from the stack pointer.
10805 //
10806 // FIXME: There are instructions which are being manually built without
10807 // explicit uses/defs so we also have to check the MCInstrDesc. We should be
10808 // able to remove the extra checks once those are fixed up. For example,
10809 // sometimes we might get something like %rax = POP64r 1. This won't be
10810 // caught by modifiesRegister or readsRegister even though the instruction
10811 // really ought to be formed so that modifiesRegister/readsRegister would
10812 // catch it.
10813 if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) ||
10814 MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) ||
10815 MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP))
10817
10818 // Outlined calls change the instruction pointer, so don't read from it.
10819 if (MI.readsRegister(X86::RIP, &RI) ||
10820 MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) ||
10821 MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP))
10823
10824 // Don't outline CFI instructions.
10825 if (MI.isCFIInstruction())
10827
10829}
10830
10833 const outliner::OutlinedFunction &OF) const {
10834 // If we're a tail call, we already have a return, so don't do anything.
10835 if (OF.FrameConstructionID == MachineOutlinerTailCall)
10836 return;
10837
10838 // We're a normal call, so our sequence doesn't have a return instruction.
10839 // Add it in.
10840 MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RET64));
10841 MBB.insert(MBB.end(), retq);
10842}
10843
10847 // Is it a tail call?
10848 if (C.CallConstructionID == MachineOutlinerTailCall) {
10849 // Yes, just insert a JMP.
10850 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(X86::TAILJMPd64))
10851 .addGlobalAddress(M.getNamedValue(MF.getName())));
10852 } else {
10853 // No, insert a call.
10854 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32))
10855 .addGlobalAddress(M.getNamedValue(MF.getName())));
10856 }
10857
10858 return It;
10859}
10860
10863 DebugLoc &DL,
10864 bool AllowSideEffects) const {
10865 const MachineFunction &MF = *MBB.getParent();
10866 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
10868
10869 if (ST.hasMMX() && X86::VR64RegClass.contains(Reg))
10870 // FIXME: Should we ignore MMX registers?
10871 return;
10872
10873 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
10874 // Convert register to the 32-bit version. Both 'movl' and 'xorl' clear the
10875 // upper bits of a 64-bit register automagically.
10876 Reg = getX86SubSuperRegister(Reg, 32);
10877
10878 if (!AllowSideEffects)
10879 // XOR affects flags, so use a MOV instead.
10880 BuildMI(MBB, Iter, DL, get(X86::MOV32ri), Reg).addImm(0);
10881 else
10882 BuildMI(MBB, Iter, DL, get(X86::XOR32rr), Reg)
10883 .addReg(Reg, RegState::Undef)
10884 .addReg(Reg, RegState::Undef);
10885 } else if (X86::VR128RegClass.contains(Reg)) {
10886 // XMM#
10887 if (!ST.hasSSE1())
10888 return;
10889
10890 BuildMI(MBB, Iter, DL, get(X86::V_SET0), Reg);
10891 } else if (X86::VR256RegClass.contains(Reg)) {
10892 // YMM#
10893 if (!ST.hasAVX())
10894 return;
10895
10896 BuildMI(MBB, Iter, DL, get(X86::V_SET0), TRI.getSubReg(Reg, X86::sub_xmm));
10897 } else if (X86::VR512RegClass.contains(Reg)) {
10898 // ZMM#
10899 if (!ST.hasAVX512())
10900 return;
10901
10902 BuildMI(MBB, Iter, DL, get(X86::AVX512_128_SET0),
10903 TRI.getSubReg(Reg, X86::sub_xmm));
10904 } else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) ||
10905 X86::VK4RegClass.contains(Reg) || X86::VK8RegClass.contains(Reg) ||
10906 X86::VK16RegClass.contains(Reg)) {
10907 if (!ST.hasAVX512())
10908 return;
10909
10910 unsigned Op = ST.hasBWI() ? X86::KSET0Q : X86::KSET0W;
10911 BuildMI(MBB, Iter, DL, get(Op), Reg);
10912 }
10913}
10914
10916 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
10917 bool DoRegPressureReduce) const {
10918 unsigned Opc = Root.getOpcode();
10919 switch (Opc) {
10920 case X86::VPDPWSSDrr:
10921 case X86::VPDPWSSDrm:
10922 case X86::VPDPWSSDYrr:
10923 case X86::VPDPWSSDYrm: {
10924 if (!Subtarget.hasFastDPWSSD()) {
10926 return true;
10927 }
10928 break;
10929 }
10930 case X86::VPDPWSSDZ128rr:
10931 case X86::VPDPWSSDZ128rm:
10932 case X86::VPDPWSSDZ256rr:
10933 case X86::VPDPWSSDZ256rm:
10934 case X86::VPDPWSSDZrr:
10935 case X86::VPDPWSSDZrm: {
10936 if (Subtarget.hasBWI() && !Subtarget.hasFastDPWSSD()) {
10938 return true;
10939 }
10940 break;
10941 }
10942 }
10944 Patterns, DoRegPressureReduce);
10945}
10946
10947static void
10951 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
10952 MachineFunction *MF = Root.getMF();
10954
10955 unsigned Opc = Root.getOpcode();
10956 unsigned AddOpc = 0;
10957 unsigned MaddOpc = 0;
10958 switch (Opc) {
10959 default:
10960 assert(false && "It should not reach here");
10961 break;
10962 // vpdpwssd xmm2,xmm3,xmm1
10963 // -->
10964 // vpmaddwd xmm3,xmm3,xmm1
10965 // vpaddd xmm2,xmm2,xmm3
10966 case X86::VPDPWSSDrr:
10967 MaddOpc = X86::VPMADDWDrr;
10968 AddOpc = X86::VPADDDrr;
10969 break;
10970 case X86::VPDPWSSDrm:
10971 MaddOpc = X86::VPMADDWDrm;
10972 AddOpc = X86::VPADDDrr;
10973 break;
10974 case X86::VPDPWSSDZ128rr:
10975 MaddOpc = X86::VPMADDWDZ128rr;
10976 AddOpc = X86::VPADDDZ128rr;
10977 break;
10978 case X86::VPDPWSSDZ128rm:
10979 MaddOpc = X86::VPMADDWDZ128rm;
10980 AddOpc = X86::VPADDDZ128rr;
10981 break;
10982 // vpdpwssd ymm2,ymm3,ymm1
10983 // -->
10984 // vpmaddwd ymm3,ymm3,ymm1
10985 // vpaddd ymm2,ymm2,ymm3
10986 case X86::VPDPWSSDYrr:
10987 MaddOpc = X86::VPMADDWDYrr;
10988 AddOpc = X86::VPADDDYrr;
10989 break;
10990 case X86::VPDPWSSDYrm:
10991 MaddOpc = X86::VPMADDWDYrm;
10992 AddOpc = X86::VPADDDYrr;
10993 break;
10994 case X86::VPDPWSSDZ256rr:
10995 MaddOpc = X86::VPMADDWDZ256rr;
10996 AddOpc = X86::VPADDDZ256rr;
10997 break;
10998 case X86::VPDPWSSDZ256rm:
10999 MaddOpc = X86::VPMADDWDZ256rm;
11000 AddOpc = X86::VPADDDZ256rr;
11001 break;
11002 // vpdpwssd zmm2,zmm3,zmm1
11003 // -->
11004 // vpmaddwd zmm3,zmm3,zmm1
11005 // vpaddd zmm2,zmm2,zmm3
11006 case X86::VPDPWSSDZrr:
11007 MaddOpc = X86::VPMADDWDZrr;
11008 AddOpc = X86::VPADDDZrr;
11009 break;
11010 case X86::VPDPWSSDZrm:
11011 MaddOpc = X86::VPMADDWDZrm;
11012 AddOpc = X86::VPADDDZrr;
11013 break;
11014 }
11015 // Create vpmaddwd.
11016 const TargetRegisterClass *RC =
11017 RegInfo.getRegClass(Root.getOperand(0).getReg());
11018 Register NewReg = RegInfo.createVirtualRegister(RC);
11019 MachineInstr *Madd = Root.getMF()->CloneMachineInstr(&Root);
11020 Madd->setDesc(TII.get(MaddOpc));
11021 Madd->untieRegOperand(1);
11022 Madd->removeOperand(1);
11023 Madd->getOperand(0).setReg(NewReg);
11024 InstrIdxForVirtReg.insert(std::make_pair(NewReg, 0));
11025 // Create vpaddd.
11026 Register DstReg = Root.getOperand(0).getReg();
11027 MachineInstr *Add = BuildMI(*MF, MIMetadata(Root), TII.get(AddOpc), DstReg)
11028 .addReg(Root.getOperand(1).getReg())
11029 .addReg(Madd->getOperand(0).getReg());
11030 InsInstrs.push_back(Madd);
11031 InsInstrs.push_back(Add);
11032 DelInstrs.push_back(&Root);
11033}
11034
11036 MachineInstr &Root, unsigned Pattern,
11039 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
11040 switch (Pattern) {
11041 default:
11042 // Reassociate instructions.
11044 DelInstrs, InstrIdxForVirtReg);
11045 return;
11047 genAlternativeDpCodeSequence(Root, *this, InsInstrs, DelInstrs,
11048 InstrIdxForVirtReg);
11049 return;
11050 }
11051}
11052
11053// See also: X86DAGToDAGISel::SelectInlineAsmMemoryOperand().
11055 int FI) const {
11058 M.Base.FrameIndex = FI;
11059 M.getFullAddress(Ops);
11060}
11061
11063X86InstrInfo::insertCodePrefetchInstr(MachineBasicBlock &MBB,
11064 MachineBasicBlock::iterator InsertBefore,
11065 const GlobalValue *GV) const {
11066 MachineFunction &MF = *MBB.getParent();
11067 MachineInstr *PrefetchInstr = MF.CreateMachineInstr(
11068 get(X86::PREFETCHIT1),
11069 InsertBefore == MBB.instr_end() ? MBB.findPrevDebugLoc(InsertBefore)
11070 : InsertBefore->getDebugLoc(),
11071 true);
11072 MachineInstrBuilder MIB(MF, PrefetchInstr);
11075 /*base_alignment=*/llvm::Align(1)));
11076 MIB.addReg(X86::RIP).addImm(1).addReg(X86::NoRegister);
11077 MIB.addGlobalAddress(GV);
11078 MIB.addReg(X86::NoRegister);
11079 MBB.insert(InsertBefore, PrefetchInstr);
11080 return PrefetchInstr;
11081}
11082
11083#define GET_INSTRINFO_HELPERS
11084#include "X86GenInstrInfo.inc"
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
return SDValue()
const AArch64Subtarget & Subtarget
SDValue Pred
static bool isFrameStoreOpcode(int Opcode)
static bool isFrameLoadOpcode(int Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerDefault
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo MachineBasicBlock::iterator InsertPt
const AArch64InstrInfo & TII
uint64_t uint64_t
uint64_t Imm
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
DXIL Forward Handle Accesses
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
static bool lookup(const GsymReader &GR, GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr, uint64_t Addr, SourceLocations &SrcLocs, llvm::Error &Err)
A Lookup helper functions.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static SDValue isNOT(SDValue V, SelectionDAG &DAG)
static int getJumpTableIndexFromReg(const MachineRegisterInfo &MRI, Register Reg)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
bool IsDead
This file contains some templates that are useful if you are working with the STL at all.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
Provides some synthesis utilities to produce sequences of values.
static SPCC::CondCodes GetOppositeBranchCondition(SPCC::CondCodes CC)
#define LLVM_DEBUG(...)
Definition Debug.h:119
#define FROM_TO(FROM, TO)
static bool is64Bit(const char *name)
#define GET_EGPR_IF_ENABLED(OPC)
static bool isLEA(unsigned Opcode)
static void addOperands(MachineInstrBuilder &MIB, ArrayRef< MachineOperand > MOs, int PtrOffset=0)
static std::optional< ParamLoadedValue > describeMOVrrLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetRegisterInfo *TRI)
If DescribedReg overlaps with the MOVrr instruction's destination register then, if possible,...
static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, MachineInstr &MI)
static unsigned CopyToFromAsymmetricReg(Register DestReg, Register SrcReg, const X86Subtarget &Subtarget)
static bool isConvertibleLEA(MachineInstr *MI)
static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, const X86Subtarget &Subtarget)
static bool isAMXOpcode(unsigned Opc)
static void updateOperandRegConstraints(MachineFunction &MF, MachineInstr &NewMI, const TargetInstrInfo &TII)
static int getJumpTableIndexFromAddr(const MachineInstr &MI)
static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, unsigned NewWidth, unsigned *pNewMask=nullptr)
static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, bool MinusOne)
static unsigned getNewOpcFromTable(ArrayRef< X86TableEntry > Table, unsigned Opc)
static unsigned getStoreRegOpcode(Register SrcReg, const TargetRegisterClass *RC, bool IsStackAligned, const X86Subtarget &STI)
#define FOLD_BROADCAST(SIZE)
#define CASE_BCAST_TYPE_OPC(TYPE, OP16, OP32, OP64)
static bool isTruncatedShiftCountForLEA(unsigned ShAmt)
Check whether the given shift count is appropriate can be represented by a LEA instruction.
static SmallVector< MachineMemOperand *, 2 > extractLoadMMOs(ArrayRef< MachineMemOperand * > MMOs, MachineFunction &MF)
static MachineInstr * fuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, ArrayRef< MachineOperand > MOs, MachineBasicBlock::iterator InsertPt, MachineInstr &MI, const TargetInstrInfo &TII)
static void printFailMsgforFold(const MachineInstr &MI, unsigned Idx)
static bool canConvert2Copy(unsigned Opc)
static bool expandNOVLXStore(MachineInstrBuilder &MIB, const TargetRegisterInfo *TRI, const MCInstrDesc &StoreDesc, const MCInstrDesc &ExtractDesc, unsigned SubIdx)
static bool isX87Reg(Register Reg)
Return true if the Reg is X87 register.
static bool Expand2AddrKreg(MachineInstrBuilder &MIB, const MCInstrDesc &Desc, Register Reg)
Expand a single-def pseudo instruction to a two-addr instruction with two k0 reads.
#define VPERM_CASES_BROADCAST(Suffix)
static std::pair< X86::CondCode, unsigned > isUseDefConvertible(const MachineInstr &MI)
Check whether the use can be converted to remove a comparison against zero.
static bool findRedundantFlagInstr(MachineInstr &CmpInstr, MachineInstr &CmpValDefInstr, const MachineRegisterInfo *MRI, MachineInstr **AndInstr, const TargetRegisterInfo *TRI, const X86Subtarget &ST, bool &NoSignFlag, bool &ClearsOverflowFlag)
static bool expandSHXDROT(MachineInstrBuilder &MIB, const MCInstrDesc &Desc)
static unsigned getLoadRegOpcode(Register DestReg, const TargetRegisterClass *RC, bool IsStackAligned, const X86Subtarget &STI)
static void expandLoadStackGuard(MachineInstrBuilder &MIB, const TargetInstrInfo &TII)
static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum, bool ForLoadFold=false)
static MachineInstr * makeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, ArrayRef< MachineOperand > MOs, MachineBasicBlock::iterator InsertPt, MachineInstr &MI)
static bool isHReg(MCRegister Reg)
Test if the given register is a physical h register.
#define GET_ND_IF_ENABLED(OPC)
static bool expandMOVSHP(MachineInstrBuilder &MIB, MachineInstr &MI, const TargetInstrInfo &TII, bool HasAVX)
static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget, bool ForLoadFold=false)
Return true for all instructions that only update the first 32 or 64-bits of the destination register...
#define CASE_NF(OP)
#define CASE_SHXD(OP)
static const uint16_t * lookupAVX512(unsigned opcode, unsigned domain, ArrayRef< uint16_t[4]> Table)
static unsigned getLoadStoreRegOpcode(Register Reg, const TargetRegisterClass *RC, bool IsStackAligned, const X86Subtarget &STI, bool Load)
#define VPERM_CASES(Suffix)
#define FROM_TO_SIZE(A, B, S)
static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2)
static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag, bool &ClearsOverflowFlag)
Check whether the definition can be converted to remove a comparison against zero.
static MachineInstr * fuseInst(MachineFunction &MF, unsigned Opcode, unsigned OpNo, ArrayRef< MachineOperand > MOs, MachineBasicBlock::iterator InsertPt, MachineInstr &MI, const TargetInstrInfo &TII, int PtrOffset=0)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static unsigned getCommutedVPERMV3Opcode(unsigned Opcode)
static bool isCmpRedundantAfterLTZCNT(Register SrcReg, Register SrcReg2, int64_t ImmMask, int64_t ImmValue, const MachineInstr &OI)
static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII)
static MachineBasicBlock * getFallThroughMBB(MachineBasicBlock *MBB, MachineBasicBlock *TBB)
static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, const MachineInstr &UserMI, const MachineFunction &MF)
Check if LoadMI is a partial register load that we can't fold into MI because the latter uses content...
static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI)
static bool expandNOVLXLoad(MachineInstrBuilder &MIB, const TargetRegisterInfo *TRI, const MCInstrDesc &LoadDesc, const MCInstrDesc &BroadcastDesc, unsigned SubIdx)
#define CASE_EVEX(OP)
static void genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
#define CASE_ND(OP)
static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, unsigned SrcOpIdx2)
This determines which of three possible cases of a three source commute the source indexes correspond...
static unsigned getTruncatedShiftCount(const MachineInstr &MI, unsigned ShiftAmtOperandIdx)
Check whether the shift count for a machine operand is non-zero.
static SmallVector< MachineMemOperand *, 2 > extractStoreMMOs(ArrayRef< MachineMemOperand * > MMOs, MachineFunction &MF)
static unsigned getBroadcastOpcode(const X86FoldTableEntry *I, const TargetRegisterClass *RC, const X86Subtarget &STI)
static unsigned convertALUrr2ALUri(unsigned Opc)
Convert an ALUrr opcode to corresponding ALUri opcode.
static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI)
Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
static bool Expand2AddrUndef(MachineInstrBuilder &MIB, const MCInstrDesc &Desc)
Expand a single-def pseudo instruction to a two-addr instruction with two undef reads of the register...
static bool isCommutableVPERMV3Instruction(unsigned Opcode)
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DWARF expression.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static LLVM_ABI DIExpression * appendExt(const DIExpression *Expr, unsigned FromSize, unsigned ToSize, bool Signed)
Append a zero- or sign-extension to Expr.
A debug info location.
Definition DebugLoc.h:126
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition Function.h:708
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
LiveInterval - This class represents the liveness of a register, or stack slot.
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveInterval & getInterval(Register Reg)
LLVM_ABI void removePhysRegDefAt(MCRegister Reg, SlotIndex Pos)
Remove value numbers and related live segments starting at position Pos that are part of any liverang...
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
A set of physical registers with utility functions to track liveness when walking backward/forward th...
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
static LocationSize precise(uint64_t Value)
static MCCFIInstruction createAdjustCfaOffset(MCSymbol *L, int64_t Adjustment, SMLoc Loc={})
.cfi_adjust_cfa_offset Same as .cfi_def_cfa_offset, but Offset is a relative value that is added/subt...
Definition MCDwarf.h:651
Instances of this class represent a single low-level machine instruction.
Definition MCInst.h:188
void setOpcode(unsigned Op)
Definition MCInst.h:201
Describe properties that are true of each instruction in the target description file.
This holds information about one operand of a machine instruction, indicating the register class for ...
Definition MCInstrDesc.h:88
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1582
Set of metadata that should be preserved when using BuildMI().
SimpleValueType SimpleTy
MachineInstrBundleIterator< const MachineInstr > const_iterator
LLVM_ABI const MachineBasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor.
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
LLVM_ABI bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
MachineInstrBundleIterator< MachineInstr > iterator
@ LQR_Dead
Register is known to be fully dead.
This class is a data container for one entry in a MachineConstantPool.
union llvm::MachineConstantPoolEntry::@004270020304201266316354007027341142157160323045 Val
The constant itself.
bool isMachineConstantPoolEntry() const
isMachineConstantPoolEntry - Return true if the MachineConstantPoolEntry is indeed a target specific ...
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
LLVM_ABI unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool needsFrameMoves() const
True if this function needs frame moves for debug or exceptions.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
bool usesWindowsUnwindInfo() const
True if this function describes its unwind information with Windows SEH unwind opcodes (....
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDisp(const MachineOperand &Disp, int64_t off, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
mop_iterator operands_begin()
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isImplicitDef() const
const MachineBasicBlock * getParent() const
void dropDebugNumber()
Drop any variable location debugging information associated with this instruction.
uint32_t getCFIType() const
Helper to extract a CFI type hash if one has been added.
LLVM_ABI void addImplicitDefUseOperands(MachineFunction &MF)
Add all implicit def and use operands to this instruction.
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void untieRegOperand(unsigned OpIdx)
Break any tie involving OpIdx.
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
LLVM_ABI void eraseFromBundle()
Unlink 'this' from its basic block and delete it.
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
LLVM_ABI void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo)
Replace all occurrences of FromReg with ToReg:SubIdx, properly composing subreg indices where necessa...
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
LLVM_ABI bool isIdenticalTo(const MachineInstr &Other, MICheckType Check=CheckDefs) const
Return true if this instruction is identical to Other.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
void setFlag(MIFlag Flag)
Set a MI flag.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
LLVM_ABI void dump() const
const MachineOperand & getOperand(unsigned i) const
unsigned getNumDefs() const
Returns the total number of definitions.
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void setDebugLoc(DebugLoc DL)
Replace current source information with new such.
MachineOperand * findRegisterDefOperand(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImplicit(bool Val=true)
void setImm(int64_t immVal)
int64_t getImm() const
bool readsReg() const
readsReg - Returns true if this operand reads the previous value of its register.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
bool isJTI() const
isJTI - Tests if this is a MO_JumpTableIndex operand.
LLVM_ABI void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value.
static MachineOperand CreateImm(int64_t Val)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateCPI(unsigned Idx, int Offset, unsigned TargetFlags=0)
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
int64_t getOffset() const
Return the offset from the symbol in this operand.
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
MachineFunction & getMachineFunction() const
SlotIndex - An opaque wrapper around machine indexes.
Definition SlotIndexes.h:70
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Information about stack frame layout on the target.
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum) const
Given a machine instruction descriptor, returns the register class constraint for OpNum,...
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isReMaterializableImpl(const MachineInstr &MI) const
For instructions with opcodes for which the M_REMATERIALIZABLE flag is set, this hook lets the target...
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
bool isPositionIndependent() const
CodeModel::Model getCodeModel() const
Returns the code model.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Provide an instruction scheduling machine model to CodeGen passes.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
static constexpr TypeSize getZero()
Definition TypeSize.h:358
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
static LLVM_ABI Type * getFP128Ty(LLVMContext &C)
Definition Type.cpp:276
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
Definition Type.cpp:272
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:271
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:269
SlotIndex def
The index of the defining instruction.
LLVM Value Representation.
Definition Value.h:75
MCRegister getPhys(Register virtReg) const
returns the physical register mapped to the specified virtual register
Definition VirtRegMap.h:91
void BuildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCCFIInstruction &CFIInst, MachineInstr::MIFlag Flag=MachineInstr::NoFlags) const
Wraps up getting a CFI index and building a MachineInstr for it.
void getFrameIndexOperands(SmallVectorImpl< MachineOperand > &Ops, int FI) const override
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
Check if there exists an earlier instruction that operates on the same source operands and sets eflag...
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DestReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
Overrides the isSchedulingBoundary from Codegen/TargetInstrInfo.cpp to make it capable of identifying...
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI) const override
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
void replaceBranchWithTailCall(MachineBasicBlock &MBB, SmallVectorImpl< MachineOperand > &Cond, const MachineInstr &TailCall) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, unsigned *LoadRegIndex=nullptr) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
Returns true iff the routine could find two commutable operands in the given machine instruction.
bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1, int64_t &Offset2) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, Register VReg, unsigned SubReg=0, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
X86InstrInfo(const X86Subtarget &STI)
static bool isDataInvariantLoad(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value l...
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned CommuteOpIdx1, unsigned CommuteOpIdx2) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
const X86RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
bool hasCommutePreference(MachineInstr &MI, bool &Commute) const override
Returns true if we have preference on the operands order in MI, the commute decision is returned in C...
bool hasLiveCondCodeDef(MachineInstr &MI) const
True if MI has a condition code def, e.g.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool canMakeTailCallConditional(SmallVectorImpl< MachineOperand > &Cond, const MachineInstr &TailCall) const override
bool unfoldMemoryOperand(MachineFunction &MF, MachineInstr &MI, Register Reg, bool UnfoldLoad, bool UnfoldStore, SmallVectorImpl< MachineInstr * > &NewMIs) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
bool preservesZeroValueInReg(const MachineInstr *MI, const Register NullValueReg) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool expandPostRAPseudo(MachineInstr &MI) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
MCInst getNop() const override
Return the noop instruction to use for a noop.
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MIT, unsigned Flags) const override
const TargetRegisterClass * getInlineAsmMemoryOperandRegClass(InlineAsm::ConstraintCode C) const override
bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, int64_t Offset1, int64_t Offset2, unsigned NumLoads) const override
This is a used by the pre-regalloc scheduler to determine (in conjunction with areLoadsFromSameBasePt...
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg, int64_t &ImmVal) const override
Register isStoreToStackSlotPostFE(const MachineInstr &MI, int &FrameIndex) const override
isStoreToStackSlotPostFE - Check for post-frame ptr elimination stack locations as well.
const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum) const override
Given a machine instruction descriptor, returns the register class constraint for OpNum,...
bool isUnconditionalTailCall(const MachineInstr &MI) const override
void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, LaneBitmask UsedLanes=LaneBitmask::getAll()) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &LdSt, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const override
Register isLoadFromStackSlotPostFE(const MachineInstr &MI, int &FrameIndex) const override
isLoadFromStackSlotPostFE - Check for post-frame ptr elimination stack locations as well.
void setExecutionDomain(MachineInstr &MI, unsigned Domain) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool setExecutionDomainCustom(MachineInstr &MI, unsigned Domain) const
unsigned getPartialRegUpdateClearance(const MachineInstr &MI, unsigned OpNum) const override
Inform the BreakFalseDeps pass how many idle instructions we would like before a partial register upd...
int getSPAdjust(const MachineInstr &MI) const override
getSPAdjust - This returns the stack pointer adjustment made by this instruction.
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, SelectExpansion &) const override
bool isReMaterializableImpl(const MachineInstr &MI) const override
Register getGlobalBaseReg(MachineFunction *MF) const
getGlobalBaseReg - Return a virtual register initialized with the the global base register value.
int getJumpTableIndex(const MachineInstr &MI) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, MachineInstr &NewMI1, MachineInstr &NewMI2) const override
This is an architecture-specific helper function of reassociateOps.
std::pair< uint16_t, uint16_t > getExecutionDomain(const MachineInstr &MI) const override
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
isCoalescableExtInstr - Return true if the instruction is a "coalescable" extension instruction.
void loadStoreTileReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned Opc, Register Reg, int FrameIdx, bool isKill=false) const
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds potential patterns, this function generates the instructions ...
bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, TargetInstrInfo::MachineBranchPredicate &MBP, bool AllowModify=false) const override
static bool isDataInvariant(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value o...
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
Fold a load or store of the specified stack slot into the specified machine instruction for the speci...
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
unsigned getUndefRegClearance(const MachineInstr &MI, unsigned OpNum) const override
Inform the BreakFalseDeps pass how many idle instructions we would like before certain undef register...
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
int64_t getFrameAdjustment(const MachineInstr &I) const
Returns the stack pointer adjustment that happens inside the frame setup..destroy sequence (e....
void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum) const override
bool hasHighOperandLatency(const TargetSchedModel &SchedModel, const MachineRegisterInfo *MRI, const MachineInstr &DefMI, unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const override
bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const override
bool classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, unsigned LEAOpcode, bool AllowSP, Register &NewSrc, unsigned &NewSrcSubReg, bool &isKill, MachineOperand &ImplicitOp, LiveIntervals *LIS) const
Given an operand within a MachineInstr, insert preceding code to put it into the right format for a p...
uint16_t getExecutionDomainCustom(const MachineInstr &MI) const
bool isHighLatencyDef(int opc) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const override
foldImmediate - 'Reg' is known to be defined by a move immediate instruction, try to fold the immedia...
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveIntervals *LIS) const override
convertToThreeAddress - This method must be implemented by targets that set the M_CONVERTIBLE_TO_3_AD...
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
unsigned getFMA3OpcodeToCommuteOperands(const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, const X86InstrFMA3Group &FMA3Group) const
Returns an adjusted FMA opcode that must be used in FMA instruction that performs the same computatio...
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
const TargetRegisterClass * constrainRegClassToNonRex2(const TargetRegisterClass *RC) const
bool hasAVX512() const
const X86RegisterInfo * getRegisterInfo() const override
bool hasAVX() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
@ X86
Windows x64, Windows Itanium (IA-64)
Definition MCAsmInfo.h:53
X86II - This namespace holds all of the target specific flags that instruction info tracks.
bool isKMergeMasked(uint64_t TSFlags)
bool hasNewDataDest(uint64_t TSFlags)
@ MO_GOT_ABSOLUTE_ADDRESS
MO_GOT_ABSOLUTE_ADDRESS - On a symbol operand, this represents a relocation of: SYMBOL_LABEL + [.
@ MO_INDNTPOFF
MO_INDNTPOFF - On a symbol operand this indicates that the immediate is the absolute address of the G...
@ MO_GOTNTPOFF
MO_GOTNTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry w...
@ MO_GOTTPOFF
MO_GOTTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry wi...
@ MO_GOTPCREL
MO_GOTPCREL - On a symbol operand this indicates that the immediate is offset to the GOT entry for th...
int getMemoryOperandIdx(const MCInstrDesc &Desc)
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ SSEDomainShift
Execution domain for SSE instructions.
bool canUseApxExtendedReg(const MCInstrDesc &Desc)
bool isPseudo(uint64_t TSFlags)
bool isKMasked(uint64_t TSFlags)
Define some predicates that are used for node matching.
CondCode getCondFromBranch(const MachineInstr &MI)
CondCode getCondFromCFCMov(const MachineInstr &MI)
@ LAST_VALID_COND
Definition X86BaseInfo.h:95
CondCode getCondFromMI(const MachineInstr &MI)
Return the condition code of the instruction.
int getFirstAddrOperandIdx(const MachineInstr &MI)
Return the index of the instruction's first address operand, if it has a memory reference,...
@ AddrNumOperands
Definition X86BaseInfo.h:37
unsigned getSwappedVCMPImm(unsigned Imm)
Get the VCMP immediate if the opcodes are swapped.
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
unsigned getSwappedVPCOMImm(unsigned Imm)
Get the VPCOM immediate if the opcodes are swapped.
bool isX87Instruction(MachineInstr &MI)
Check if the instruction is X87 instruction.
unsigned getNonNDVariant(unsigned Opc)
unsigned getVPCMPImmForCond(ISD::CondCode CC)
Get the VPCMP immediate for the given condition.
std::pair< CondCode, bool > getX86ConditionCode(CmpInst::Predicate Predicate)
Return a pair of condition code for the given predicate and whether the instruction operands should b...
CondCode getCondFromSETCC(const MachineInstr &MI)
unsigned getSwappedVPCMPImm(unsigned Imm)
Get the VPCMP immediate if the opcodes are swapped.
CondCode getCondFromCCMP(const MachineInstr &MI)
int getCCMPCondFlagsFromCondCode(CondCode CC)
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
const Constant * getConstantFromPool(const MachineInstr &MI, unsigned OpNo)
Find any constant pool entry associated with a specific instruction operand.
unsigned getNFVariantIfClobberRemovable(const MachineInstr &MI, const TargetRegisterInfo *TRI=nullptr)
unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm)
Return a MOVri opcode for materializing Imm into a 32- or 64-bit GPR.
unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false, bool HasNDD=false)
Return a cmov opcode for the given register size in bytes, and operand type.
unsigned getNFVariant(unsigned Opc)
unsigned getVectorRegisterWidth(const MCOperandInfo &Info)
Get the width of the vector register operand.
CondCode getCondFromCMov(const MachineInstr &MI)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
static bool isAddMemInstrWithRelocation(const MachineInstr &MI)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
RegState
Flags to represent properties of register accesses.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ Define
Register definition.
static bool isMem(const MachineInstr &MI, unsigned Op)
constexpr RegState getKillRegState(bool B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
Definition Alignment.h:139
MCRegister getX86SubSuperRegister(MCRegister Reg, unsigned Size, bool High=false)
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
static const MachineInstrBuilder & addRegReg(const MachineInstrBuilder &MIB, Register Reg1, bool isKill1, unsigned SubReg1, Register Reg2, bool isKill2, unsigned SubReg2)
addRegReg - This function is used to add a memory reference of the form: [Reg + Reg].
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
addFrameReference - This function is used to add a reference to the base of an abstract object on the...
constexpr RegState getDeadRegState(bool B)
Op::Description Desc
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
bool isNonFoldableWithSameMask(unsigned RegOp)
const X86FoldTableEntry * lookupBroadcastFoldTable(unsigned RegOp, unsigned OpNum)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
const X86InstrFMA3Group * getFMA3Group(unsigned Opcode, uint64_t TSFlags)
Returns a reference to a group of FMA3 opcodes to where the given Opcode is included.
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
const X86FoldTableEntry * lookupTwoAddrFoldTable(unsigned RegOp)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
bool is_sorted(R &&Range, Compare C)
Wrapper function around std::is_sorted to check if elements in a range R are sorted with respect to a...
Definition STLExtras.h:1986
constexpr RegState getDefRegState(bool B)
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
RegState getRegState(const MachineOperand &RegOp)
Get all register state flags from machine operand RegOp.
static bool isMemInstrWithGOTPCREL(const MachineInstr &MI)
static const MachineInstrBuilder & addOffset(const MachineInstrBuilder &MIB, int Offset)
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition STLExtras.h:2068
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
const X86FoldTableEntry * lookupUnfoldTable(unsigned MemOp)
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
MaybeAlign getStackAlign(const Function &F, unsigned Index)
bool matchBroadcastSize(const X86FoldTableEntry &Entry, unsigned BroadcastBits)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
const X86FoldTableEntry * lookupFoldTable(unsigned RegOp, unsigned OpNum)
constexpr RegState getUndefRegState(bool B)
static const MachineInstrBuilder & addRegOffset(const MachineInstrBuilder &MIB, Register Reg, int Offset)
addRegOffset - This function is used to add a memory reference of the form [Reg + Offset],...
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
static constexpr Align fromLog2(unsigned Log2Value)
Returns an alignment of 1 << Log2Value bytes.
Definition Alignment.h:73
Extended Value Type.
Definition ValueTypes.h:35
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
This represents a simple continuous liveness interval for a value.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
X86AddressMode - This struct holds a generalized full x86 address mode.
enum llvm::X86AddressMode::@202116273335065351270200035056227005202106004277 BaseType
This class is used to group {132, 213, 231} forms of FMA opcodes together.
unsigned get213Opcode() const
Returns the 213 form of FMA opcode.
unsigned get231Opcode() const
Returns the 231 form of FMA opcode.
bool isIntrinsic() const
Returns true iff the group of FMA opcodes holds intrinsic opcodes.
unsigned get132Opcode() const
Returns the 132 form of FMA opcode.
An individual sequence of instructions to be replaced with a call to an outlined function.
The information necessary to create an outlined function for some class of candidate.