File: | lib/Target/X86/X86InstrInfo.cpp |
Warning: | line 3772, column 37 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===// | |||
2 | // | |||
3 | // The LLVM Compiler Infrastructure | |||
4 | // | |||
5 | // This file is distributed under the University of Illinois Open Source | |||
6 | // License. See LICENSE.TXT for details. | |||
7 | // | |||
8 | //===----------------------------------------------------------------------===// | |||
9 | // | |||
10 | // This file contains the X86 implementation of the TargetInstrInfo class. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "X86InstrInfo.h" | |||
15 | #include "X86.h" | |||
16 | #include "X86InstrBuilder.h" | |||
17 | #include "X86InstrFoldTables.h" | |||
18 | #include "X86MachineFunctionInfo.h" | |||
19 | #include "X86Subtarget.h" | |||
20 | #include "X86TargetMachine.h" | |||
21 | #include "llvm/ADT/STLExtras.h" | |||
22 | #include "llvm/ADT/Sequence.h" | |||
23 | #include "llvm/CodeGen/LivePhysRegs.h" | |||
24 | #include "llvm/CodeGen/LiveVariables.h" | |||
25 | #include "llvm/CodeGen/MachineConstantPool.h" | |||
26 | #include "llvm/CodeGen/MachineDominators.h" | |||
27 | #include "llvm/CodeGen/MachineFrameInfo.h" | |||
28 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
29 | #include "llvm/CodeGen/MachineModuleInfo.h" | |||
30 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
31 | #include "llvm/CodeGen/StackMaps.h" | |||
32 | #include "llvm/IR/DerivedTypes.h" | |||
33 | #include "llvm/IR/Function.h" | |||
34 | #include "llvm/IR/LLVMContext.h" | |||
35 | #include "llvm/MC/MCAsmInfo.h" | |||
36 | #include "llvm/MC/MCExpr.h" | |||
37 | #include "llvm/MC/MCInst.h" | |||
38 | #include "llvm/Support/CommandLine.h" | |||
39 | #include "llvm/Support/Debug.h" | |||
40 | #include "llvm/Support/ErrorHandling.h" | |||
41 | #include "llvm/Support/raw_ostream.h" | |||
42 | #include "llvm/Target/TargetOptions.h" | |||
43 | ||||
44 | using namespace llvm; | |||
45 | ||||
46 | #define DEBUG_TYPE"x86-instr-info" "x86-instr-info" | |||
47 | ||||
48 | #define GET_INSTRINFO_CTOR_DTOR | |||
49 | #include "X86GenInstrInfo.inc" | |||
50 | ||||
51 | static cl::opt<bool> | |||
52 | NoFusing("disable-spill-fusing", | |||
53 | cl::desc("Disable fusing of spill code into instructions"), | |||
54 | cl::Hidden); | |||
55 | static cl::opt<bool> | |||
56 | PrintFailedFusing("print-failed-fuse-candidates", | |||
57 | cl::desc("Print instructions that the allocator wants to" | |||
58 | " fuse, but the X86 backend currently can't"), | |||
59 | cl::Hidden); | |||
60 | static cl::opt<bool> | |||
61 | ReMatPICStubLoad("remat-pic-stub-load", | |||
62 | cl::desc("Re-materialize load from stub in PIC mode"), | |||
63 | cl::init(false), cl::Hidden); | |||
64 | static cl::opt<unsigned> | |||
65 | PartialRegUpdateClearance("partial-reg-update-clearance", | |||
66 | cl::desc("Clearance between two register writes " | |||
67 | "for inserting XOR to avoid partial " | |||
68 | "register update"), | |||
69 | cl::init(64), cl::Hidden); | |||
70 | static cl::opt<unsigned> | |||
71 | UndefRegClearance("undef-reg-clearance", | |||
72 | cl::desc("How many idle instructions we would like before " | |||
73 | "certain undef register reads"), | |||
74 | cl::init(128), cl::Hidden); | |||
75 | ||||
76 | ||||
77 | // Pin the vtable to this file. | |||
78 | void X86InstrInfo::anchor() {} | |||
79 | ||||
80 | X86InstrInfo::X86InstrInfo(X86Subtarget &STI) | |||
81 | : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64 | |||
82 | : X86::ADJCALLSTACKDOWN32), | |||
83 | (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64 | |||
84 | : X86::ADJCALLSTACKUP32), | |||
85 | X86::CATCHRET, | |||
86 | (STI.is64Bit() ? X86::RETQ : X86::RETL)), | |||
87 | Subtarget(STI), RI(STI.getTargetTriple()) { | |||
88 | } | |||
89 | ||||
90 | bool | |||
91 | X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, | |||
92 | unsigned &SrcReg, unsigned &DstReg, | |||
93 | unsigned &SubIdx) const { | |||
94 | switch (MI.getOpcode()) { | |||
95 | default: break; | |||
96 | case X86::MOVSX16rr8: | |||
97 | case X86::MOVZX16rr8: | |||
98 | case X86::MOVSX32rr8: | |||
99 | case X86::MOVZX32rr8: | |||
100 | case X86::MOVSX64rr8: | |||
101 | if (!Subtarget.is64Bit()) | |||
102 | // It's not always legal to reference the low 8-bit of the larger | |||
103 | // register in 32-bit mode. | |||
104 | return false; | |||
105 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
106 | case X86::MOVSX32rr16: | |||
107 | case X86::MOVZX32rr16: | |||
108 | case X86::MOVSX64rr16: | |||
109 | case X86::MOVSX64rr32: { | |||
110 | if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) | |||
111 | // Be conservative. | |||
112 | return false; | |||
113 | SrcReg = MI.getOperand(1).getReg(); | |||
114 | DstReg = MI.getOperand(0).getReg(); | |||
115 | switch (MI.getOpcode()) { | |||
116 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 116); | |||
117 | case X86::MOVSX16rr8: | |||
118 | case X86::MOVZX16rr8: | |||
119 | case X86::MOVSX32rr8: | |||
120 | case X86::MOVZX32rr8: | |||
121 | case X86::MOVSX64rr8: | |||
122 | SubIdx = X86::sub_8bit; | |||
123 | break; | |||
124 | case X86::MOVSX32rr16: | |||
125 | case X86::MOVZX32rr16: | |||
126 | case X86::MOVSX64rr16: | |||
127 | SubIdx = X86::sub_16bit; | |||
128 | break; | |||
129 | case X86::MOVSX64rr32: | |||
130 | SubIdx = X86::sub_32bit; | |||
131 | break; | |||
132 | } | |||
133 | return true; | |||
134 | } | |||
135 | } | |||
136 | return false; | |||
137 | } | |||
138 | ||||
139 | int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const { | |||
140 | const MachineFunction *MF = MI.getParent()->getParent(); | |||
141 | const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); | |||
142 | ||||
143 | if (isFrameInstr(MI)) { | |||
144 | unsigned StackAlign = TFI->getStackAlignment(); | |||
145 | int SPAdj = alignTo(getFrameSize(MI), StackAlign); | |||
146 | SPAdj -= getFrameAdjustment(MI); | |||
147 | if (!isFrameSetup(MI)) | |||
148 | SPAdj = -SPAdj; | |||
149 | return SPAdj; | |||
150 | } | |||
151 | ||||
152 | // To know whether a call adjusts the stack, we need information | |||
153 | // that is bound to the following ADJCALLSTACKUP pseudo. | |||
154 | // Look for the next ADJCALLSTACKUP that follows the call. | |||
155 | if (MI.isCall()) { | |||
156 | const MachineBasicBlock *MBB = MI.getParent(); | |||
157 | auto I = ++MachineBasicBlock::const_iterator(MI); | |||
158 | for (auto E = MBB->end(); I != E; ++I) { | |||
159 | if (I->getOpcode() == getCallFrameDestroyOpcode() || | |||
160 | I->isCall()) | |||
161 | break; | |||
162 | } | |||
163 | ||||
164 | // If we could not find a frame destroy opcode, then it has already | |||
165 | // been simplified, so we don't care. | |||
166 | if (I->getOpcode() != getCallFrameDestroyOpcode()) | |||
167 | return 0; | |||
168 | ||||
169 | return -(I->getOperand(1).getImm()); | |||
170 | } | |||
171 | ||||
172 | // Currently handle only PUSHes we can reasonably expect to see | |||
173 | // in call sequences | |||
174 | switch (MI.getOpcode()) { | |||
175 | default: | |||
176 | return 0; | |||
177 | case X86::PUSH32i8: | |||
178 | case X86::PUSH32r: | |||
179 | case X86::PUSH32rmm: | |||
180 | case X86::PUSH32rmr: | |||
181 | case X86::PUSHi32: | |||
182 | return 4; | |||
183 | case X86::PUSH64i8: | |||
184 | case X86::PUSH64r: | |||
185 | case X86::PUSH64rmm: | |||
186 | case X86::PUSH64rmr: | |||
187 | case X86::PUSH64i32: | |||
188 | return 8; | |||
189 | } | |||
190 | } | |||
191 | ||||
192 | /// Return true and the FrameIndex if the specified | |||
193 | /// operand and follow operands form a reference to the stack frame. | |||
194 | bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op, | |||
195 | int &FrameIndex) const { | |||
196 | if (MI.getOperand(Op + X86::AddrBaseReg).isFI() && | |||
197 | MI.getOperand(Op + X86::AddrScaleAmt).isImm() && | |||
198 | MI.getOperand(Op + X86::AddrIndexReg).isReg() && | |||
199 | MI.getOperand(Op + X86::AddrDisp).isImm() && | |||
200 | MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 && | |||
201 | MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 && | |||
202 | MI.getOperand(Op + X86::AddrDisp).getImm() == 0) { | |||
203 | FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex(); | |||
204 | return true; | |||
205 | } | |||
206 | return false; | |||
207 | } | |||
208 | ||||
209 | static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) { | |||
210 | switch (Opcode) { | |||
211 | default: | |||
212 | return false; | |||
213 | case X86::MOV8rm: | |||
214 | case X86::KMOVBkm: | |||
215 | MemBytes = 1; | |||
216 | return true; | |||
217 | case X86::MOV16rm: | |||
218 | case X86::KMOVWkm: | |||
219 | MemBytes = 2; | |||
220 | return true; | |||
221 | case X86::MOV32rm: | |||
222 | case X86::MOVSSrm: | |||
223 | case X86::VMOVSSZrm: | |||
224 | case X86::VMOVSSrm: | |||
225 | case X86::KMOVDkm: | |||
226 | MemBytes = 4; | |||
227 | return true; | |||
228 | case X86::MOV64rm: | |||
229 | case X86::LD_Fp64m: | |||
230 | case X86::MOVSDrm: | |||
231 | case X86::VMOVSDrm: | |||
232 | case X86::VMOVSDZrm: | |||
233 | case X86::MMX_MOVD64rm: | |||
234 | case X86::MMX_MOVQ64rm: | |||
235 | case X86::KMOVQkm: | |||
236 | MemBytes = 8; | |||
237 | return true; | |||
238 | case X86::MOVAPSrm: | |||
239 | case X86::MOVUPSrm: | |||
240 | case X86::MOVAPDrm: | |||
241 | case X86::MOVUPDrm: | |||
242 | case X86::MOVDQArm: | |||
243 | case X86::MOVDQUrm: | |||
244 | case X86::VMOVAPSrm: | |||
245 | case X86::VMOVUPSrm: | |||
246 | case X86::VMOVAPDrm: | |||
247 | case X86::VMOVUPDrm: | |||
248 | case X86::VMOVDQArm: | |||
249 | case X86::VMOVDQUrm: | |||
250 | case X86::VMOVAPSZ128rm: | |||
251 | case X86::VMOVUPSZ128rm: | |||
252 | case X86::VMOVAPSZ128rm_NOVLX: | |||
253 | case X86::VMOVUPSZ128rm_NOVLX: | |||
254 | case X86::VMOVAPDZ128rm: | |||
255 | case X86::VMOVUPDZ128rm: | |||
256 | case X86::VMOVDQU8Z128rm: | |||
257 | case X86::VMOVDQU16Z128rm: | |||
258 | case X86::VMOVDQA32Z128rm: | |||
259 | case X86::VMOVDQU32Z128rm: | |||
260 | case X86::VMOVDQA64Z128rm: | |||
261 | case X86::VMOVDQU64Z128rm: | |||
262 | MemBytes = 16; | |||
263 | return true; | |||
264 | case X86::VMOVAPSYrm: | |||
265 | case X86::VMOVUPSYrm: | |||
266 | case X86::VMOVAPDYrm: | |||
267 | case X86::VMOVUPDYrm: | |||
268 | case X86::VMOVDQAYrm: | |||
269 | case X86::VMOVDQUYrm: | |||
270 | case X86::VMOVAPSZ256rm: | |||
271 | case X86::VMOVUPSZ256rm: | |||
272 | case X86::VMOVAPSZ256rm_NOVLX: | |||
273 | case X86::VMOVUPSZ256rm_NOVLX: | |||
274 | case X86::VMOVAPDZ256rm: | |||
275 | case X86::VMOVUPDZ256rm: | |||
276 | case X86::VMOVDQU8Z256rm: | |||
277 | case X86::VMOVDQU16Z256rm: | |||
278 | case X86::VMOVDQA32Z256rm: | |||
279 | case X86::VMOVDQU32Z256rm: | |||
280 | case X86::VMOVDQA64Z256rm: | |||
281 | case X86::VMOVDQU64Z256rm: | |||
282 | MemBytes = 32; | |||
283 | return true; | |||
284 | case X86::VMOVAPSZrm: | |||
285 | case X86::VMOVUPSZrm: | |||
286 | case X86::VMOVAPDZrm: | |||
287 | case X86::VMOVUPDZrm: | |||
288 | case X86::VMOVDQU8Zrm: | |||
289 | case X86::VMOVDQU16Zrm: | |||
290 | case X86::VMOVDQA32Zrm: | |||
291 | case X86::VMOVDQU32Zrm: | |||
292 | case X86::VMOVDQA64Zrm: | |||
293 | case X86::VMOVDQU64Zrm: | |||
294 | MemBytes = 64; | |||
295 | return true; | |||
296 | } | |||
297 | } | |||
298 | ||||
299 | static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) { | |||
300 | switch (Opcode) { | |||
301 | default: | |||
302 | return false; | |||
303 | case X86::MOV8mr: | |||
304 | case X86::KMOVBmk: | |||
305 | MemBytes = 1; | |||
306 | return true; | |||
307 | case X86::MOV16mr: | |||
308 | case X86::KMOVWmk: | |||
309 | MemBytes = 2; | |||
310 | return true; | |||
311 | case X86::MOV32mr: | |||
312 | case X86::MOVSSmr: | |||
313 | case X86::VMOVSSmr: | |||
314 | case X86::VMOVSSZmr: | |||
315 | case X86::KMOVDmk: | |||
316 | MemBytes = 4; | |||
317 | return true; | |||
318 | case X86::MOV64mr: | |||
319 | case X86::ST_FpP64m: | |||
320 | case X86::MOVSDmr: | |||
321 | case X86::VMOVSDmr: | |||
322 | case X86::VMOVSDZmr: | |||
323 | case X86::MMX_MOVD64mr: | |||
324 | case X86::MMX_MOVQ64mr: | |||
325 | case X86::MMX_MOVNTQmr: | |||
326 | case X86::KMOVQmk: | |||
327 | MemBytes = 8; | |||
328 | return true; | |||
329 | case X86::MOVAPSmr: | |||
330 | case X86::MOVUPSmr: | |||
331 | case X86::MOVAPDmr: | |||
332 | case X86::MOVUPDmr: | |||
333 | case X86::MOVDQAmr: | |||
334 | case X86::MOVDQUmr: | |||
335 | case X86::VMOVAPSmr: | |||
336 | case X86::VMOVUPSmr: | |||
337 | case X86::VMOVAPDmr: | |||
338 | case X86::VMOVUPDmr: | |||
339 | case X86::VMOVDQAmr: | |||
340 | case X86::VMOVDQUmr: | |||
341 | case X86::VMOVUPSZ128mr: | |||
342 | case X86::VMOVAPSZ128mr: | |||
343 | case X86::VMOVUPSZ128mr_NOVLX: | |||
344 | case X86::VMOVAPSZ128mr_NOVLX: | |||
345 | case X86::VMOVUPDZ128mr: | |||
346 | case X86::VMOVAPDZ128mr: | |||
347 | case X86::VMOVDQA32Z128mr: | |||
348 | case X86::VMOVDQU32Z128mr: | |||
349 | case X86::VMOVDQA64Z128mr: | |||
350 | case X86::VMOVDQU64Z128mr: | |||
351 | case X86::VMOVDQU8Z128mr: | |||
352 | case X86::VMOVDQU16Z128mr: | |||
353 | MemBytes = 16; | |||
354 | return true; | |||
355 | case X86::VMOVUPSYmr: | |||
356 | case X86::VMOVAPSYmr: | |||
357 | case X86::VMOVUPDYmr: | |||
358 | case X86::VMOVAPDYmr: | |||
359 | case X86::VMOVDQUYmr: | |||
360 | case X86::VMOVDQAYmr: | |||
361 | case X86::VMOVUPSZ256mr: | |||
362 | case X86::VMOVAPSZ256mr: | |||
363 | case X86::VMOVUPSZ256mr_NOVLX: | |||
364 | case X86::VMOVAPSZ256mr_NOVLX: | |||
365 | case X86::VMOVUPDZ256mr: | |||
366 | case X86::VMOVAPDZ256mr: | |||
367 | case X86::VMOVDQU8Z256mr: | |||
368 | case X86::VMOVDQU16Z256mr: | |||
369 | case X86::VMOVDQA32Z256mr: | |||
370 | case X86::VMOVDQU32Z256mr: | |||
371 | case X86::VMOVDQA64Z256mr: | |||
372 | case X86::VMOVDQU64Z256mr: | |||
373 | MemBytes = 32; | |||
374 | return true; | |||
375 | case X86::VMOVUPSZmr: | |||
376 | case X86::VMOVAPSZmr: | |||
377 | case X86::VMOVUPDZmr: | |||
378 | case X86::VMOVAPDZmr: | |||
379 | case X86::VMOVDQU8Zmr: | |||
380 | case X86::VMOVDQU16Zmr: | |||
381 | case X86::VMOVDQA32Zmr: | |||
382 | case X86::VMOVDQU32Zmr: | |||
383 | case X86::VMOVDQA64Zmr: | |||
384 | case X86::VMOVDQU64Zmr: | |||
385 | MemBytes = 64; | |||
386 | return true; | |||
387 | } | |||
388 | return false; | |||
389 | } | |||
390 | ||||
391 | unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, | |||
392 | int &FrameIndex) const { | |||
393 | unsigned Dummy; | |||
394 | return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy); | |||
395 | } | |||
396 | ||||
397 | unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, | |||
398 | int &FrameIndex, | |||
399 | unsigned &MemBytes) const { | |||
400 | if (isFrameLoadOpcode(MI.getOpcode(), MemBytes)) | |||
401 | if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex)) | |||
402 | return MI.getOperand(0).getReg(); | |||
403 | return 0; | |||
404 | } | |||
405 | ||||
406 | unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI, | |||
407 | int &FrameIndex) const { | |||
408 | unsigned Dummy; | |||
409 | if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) { | |||
410 | unsigned Reg; | |||
411 | if ((Reg = isLoadFromStackSlot(MI, FrameIndex))) | |||
412 | return Reg; | |||
413 | // Check for post-frame index elimination operations | |||
414 | SmallVector<const MachineMemOperand *, 1> Accesses; | |||
415 | if (hasLoadFromStackSlot(MI, Accesses)) { | |||
416 | FrameIndex = | |||
417 | cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) | |||
418 | ->getFrameIndex(); | |||
419 | return 1; | |||
420 | } | |||
421 | } | |||
422 | return 0; | |||
423 | } | |||
424 | ||||
425 | unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, | |||
426 | int &FrameIndex) const { | |||
427 | unsigned Dummy; | |||
428 | return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy); | |||
429 | } | |||
430 | ||||
431 | unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, | |||
432 | int &FrameIndex, | |||
433 | unsigned &MemBytes) const { | |||
434 | if (isFrameStoreOpcode(MI.getOpcode(), MemBytes)) | |||
435 | if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 && | |||
436 | isFrameOperand(MI, 0, FrameIndex)) | |||
437 | return MI.getOperand(X86::AddrNumOperands).getReg(); | |||
438 | return 0; | |||
439 | } | |||
440 | ||||
441 | unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI, | |||
442 | int &FrameIndex) const { | |||
443 | unsigned Dummy; | |||
444 | if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) { | |||
445 | unsigned Reg; | |||
446 | if ((Reg = isStoreToStackSlot(MI, FrameIndex))) | |||
447 | return Reg; | |||
448 | // Check for post-frame index elimination operations | |||
449 | SmallVector<const MachineMemOperand *, 1> Accesses; | |||
450 | if (hasStoreToStackSlot(MI, Accesses)) { | |||
451 | FrameIndex = | |||
452 | cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) | |||
453 | ->getFrameIndex(); | |||
454 | return 1; | |||
455 | } | |||
456 | } | |||
457 | return 0; | |||
458 | } | |||
459 | ||||
460 | /// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r. | |||
461 | static bool regIsPICBase(unsigned BaseReg, const MachineRegisterInfo &MRI) { | |||
462 | // Don't waste compile time scanning use-def chains of physregs. | |||
463 | if (!TargetRegisterInfo::isVirtualRegister(BaseReg)) | |||
464 | return false; | |||
465 | bool isPICBase = false; | |||
466 | for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg), | |||
467 | E = MRI.def_instr_end(); I != E; ++I) { | |||
468 | MachineInstr *DefMI = &*I; | |||
469 | if (DefMI->getOpcode() != X86::MOVPC32r) | |||
470 | return false; | |||
471 | assert(!isPICBase && "More than one PIC base?")((!isPICBase && "More than one PIC base?") ? static_cast <void> (0) : __assert_fail ("!isPICBase && \"More than one PIC base?\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 471, __PRETTY_FUNCTION__)); | |||
472 | isPICBase = true; | |||
473 | } | |||
474 | return isPICBase; | |||
475 | } | |||
476 | ||||
477 | bool X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, | |||
478 | AliasAnalysis *AA) const { | |||
479 | switch (MI.getOpcode()) { | |||
480 | default: break; | |||
481 | case X86::MOV8rm: | |||
482 | case X86::MOV8rm_NOREX: | |||
483 | case X86::MOV16rm: | |||
484 | case X86::MOV32rm: | |||
485 | case X86::MOV64rm: | |||
486 | case X86::LD_Fp64m: | |||
487 | case X86::MOVSSrm: | |||
488 | case X86::MOVSDrm: | |||
489 | case X86::MOVAPSrm: | |||
490 | case X86::MOVUPSrm: | |||
491 | case X86::MOVAPDrm: | |||
492 | case X86::MOVUPDrm: | |||
493 | case X86::MOVDQArm: | |||
494 | case X86::MOVDQUrm: | |||
495 | case X86::VMOVSSrm: | |||
496 | case X86::VMOVSDrm: | |||
497 | case X86::VMOVAPSrm: | |||
498 | case X86::VMOVUPSrm: | |||
499 | case X86::VMOVAPDrm: | |||
500 | case X86::VMOVUPDrm: | |||
501 | case X86::VMOVDQArm: | |||
502 | case X86::VMOVDQUrm: | |||
503 | case X86::VMOVAPSYrm: | |||
504 | case X86::VMOVUPSYrm: | |||
505 | case X86::VMOVAPDYrm: | |||
506 | case X86::VMOVUPDYrm: | |||
507 | case X86::VMOVDQAYrm: | |||
508 | case X86::VMOVDQUYrm: | |||
509 | case X86::MMX_MOVD64rm: | |||
510 | case X86::MMX_MOVQ64rm: | |||
511 | // AVX-512 | |||
512 | case X86::VMOVSSZrm: | |||
513 | case X86::VMOVSDZrm: | |||
514 | case X86::VMOVAPDZ128rm: | |||
515 | case X86::VMOVAPDZ256rm: | |||
516 | case X86::VMOVAPDZrm: | |||
517 | case X86::VMOVAPSZ128rm: | |||
518 | case X86::VMOVAPSZ256rm: | |||
519 | case X86::VMOVAPSZ128rm_NOVLX: | |||
520 | case X86::VMOVAPSZ256rm_NOVLX: | |||
521 | case X86::VMOVAPSZrm: | |||
522 | case X86::VMOVDQA32Z128rm: | |||
523 | case X86::VMOVDQA32Z256rm: | |||
524 | case X86::VMOVDQA32Zrm: | |||
525 | case X86::VMOVDQA64Z128rm: | |||
526 | case X86::VMOVDQA64Z256rm: | |||
527 | case X86::VMOVDQA64Zrm: | |||
528 | case X86::VMOVDQU16Z128rm: | |||
529 | case X86::VMOVDQU16Z256rm: | |||
530 | case X86::VMOVDQU16Zrm: | |||
531 | case X86::VMOVDQU32Z128rm: | |||
532 | case X86::VMOVDQU32Z256rm: | |||
533 | case X86::VMOVDQU32Zrm: | |||
534 | case X86::VMOVDQU64Z128rm: | |||
535 | case X86::VMOVDQU64Z256rm: | |||
536 | case X86::VMOVDQU64Zrm: | |||
537 | case X86::VMOVDQU8Z128rm: | |||
538 | case X86::VMOVDQU8Z256rm: | |||
539 | case X86::VMOVDQU8Zrm: | |||
540 | case X86::VMOVUPDZ128rm: | |||
541 | case X86::VMOVUPDZ256rm: | |||
542 | case X86::VMOVUPDZrm: | |||
543 | case X86::VMOVUPSZ128rm: | |||
544 | case X86::VMOVUPSZ256rm: | |||
545 | case X86::VMOVUPSZ128rm_NOVLX: | |||
546 | case X86::VMOVUPSZ256rm_NOVLX: | |||
547 | case X86::VMOVUPSZrm: { | |||
548 | // Loads from constant pools are trivially rematerializable. | |||
549 | if (MI.getOperand(1 + X86::AddrBaseReg).isReg() && | |||
550 | MI.getOperand(1 + X86::AddrScaleAmt).isImm() && | |||
551 | MI.getOperand(1 + X86::AddrIndexReg).isReg() && | |||
552 | MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && | |||
553 | MI.isDereferenceableInvariantLoad(AA)) { | |||
554 | unsigned BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); | |||
555 | if (BaseReg == 0 || BaseReg == X86::RIP) | |||
556 | return true; | |||
557 | // Allow re-materialization of PIC load. | |||
558 | if (!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal()) | |||
559 | return false; | |||
560 | const MachineFunction &MF = *MI.getParent()->getParent(); | |||
561 | const MachineRegisterInfo &MRI = MF.getRegInfo(); | |||
562 | return regIsPICBase(BaseReg, MRI); | |||
563 | } | |||
564 | return false; | |||
565 | } | |||
566 | ||||
567 | case X86::LEA32r: | |||
568 | case X86::LEA64r: { | |||
569 | if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() && | |||
570 | MI.getOperand(1 + X86::AddrIndexReg).isReg() && | |||
571 | MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && | |||
572 | !MI.getOperand(1 + X86::AddrDisp).isReg()) { | |||
573 | // lea fi#, lea GV, etc. are all rematerializable. | |||
574 | if (!MI.getOperand(1 + X86::AddrBaseReg).isReg()) | |||
575 | return true; | |||
576 | unsigned BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); | |||
577 | if (BaseReg == 0) | |||
578 | return true; | |||
579 | // Allow re-materialization of lea PICBase + x. | |||
580 | const MachineFunction &MF = *MI.getParent()->getParent(); | |||
581 | const MachineRegisterInfo &MRI = MF.getRegInfo(); | |||
582 | return regIsPICBase(BaseReg, MRI); | |||
583 | } | |||
584 | return false; | |||
585 | } | |||
586 | } | |||
587 | ||||
588 | // All other instructions marked M_REMATERIALIZABLE are always trivially | |||
589 | // rematerializable. | |||
590 | return true; | |||
591 | } | |||
592 | ||||
593 | bool X86InstrInfo::isSafeToClobberEFLAGS(MachineBasicBlock &MBB, | |||
594 | MachineBasicBlock::iterator I) const { | |||
595 | MachineBasicBlock::iterator E = MBB.end(); | |||
596 | ||||
597 | // For compile time consideration, if we are not able to determine the | |||
598 | // safety after visiting 4 instructions in each direction, we will assume | |||
599 | // it's not safe. | |||
600 | MachineBasicBlock::iterator Iter = I; | |||
601 | for (unsigned i = 0; Iter != E && i < 4; ++i) { | |||
602 | bool SeenDef = false; | |||
603 | for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) { | |||
604 | MachineOperand &MO = Iter->getOperand(j); | |||
605 | if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS)) | |||
606 | SeenDef = true; | |||
607 | if (!MO.isReg()) | |||
608 | continue; | |||
609 | if (MO.getReg() == X86::EFLAGS) { | |||
610 | if (MO.isUse()) | |||
611 | return false; | |||
612 | SeenDef = true; | |||
613 | } | |||
614 | } | |||
615 | ||||
616 | if (SeenDef) | |||
617 | // This instruction defines EFLAGS, no need to look any further. | |||
618 | return true; | |||
619 | ++Iter; | |||
620 | // Skip over debug instructions. | |||
621 | while (Iter != E && Iter->isDebugInstr()) | |||
622 | ++Iter; | |||
623 | } | |||
624 | ||||
625 | // It is safe to clobber EFLAGS at the end of a block of no successor has it | |||
626 | // live in. | |||
627 | if (Iter == E) { | |||
628 | for (MachineBasicBlock *S : MBB.successors()) | |||
629 | if (S->isLiveIn(X86::EFLAGS)) | |||
630 | return false; | |||
631 | return true; | |||
632 | } | |||
633 | ||||
634 | MachineBasicBlock::iterator B = MBB.begin(); | |||
635 | Iter = I; | |||
636 | for (unsigned i = 0; i < 4; ++i) { | |||
637 | // If we make it to the beginning of the block, it's safe to clobber | |||
638 | // EFLAGS iff EFLAGS is not live-in. | |||
639 | if (Iter == B) | |||
640 | return !MBB.isLiveIn(X86::EFLAGS); | |||
641 | ||||
642 | --Iter; | |||
643 | // Skip over debug instructions. | |||
644 | while (Iter != B && Iter->isDebugInstr()) | |||
645 | --Iter; | |||
646 | ||||
647 | bool SawKill = false; | |||
648 | for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) { | |||
649 | MachineOperand &MO = Iter->getOperand(j); | |||
650 | // A register mask may clobber EFLAGS, but we should still look for a | |||
651 | // live EFLAGS def. | |||
652 | if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS)) | |||
653 | SawKill = true; | |||
654 | if (MO.isReg() && MO.getReg() == X86::EFLAGS) { | |||
655 | if (MO.isDef()) return MO.isDead(); | |||
656 | if (MO.isKill()) SawKill = true; | |||
657 | } | |||
658 | } | |||
659 | ||||
660 | if (SawKill) | |||
661 | // This instruction kills EFLAGS and doesn't redefine it, so | |||
662 | // there's no need to look further. | |||
663 | return true; | |||
664 | } | |||
665 | ||||
666 | // Conservative answer. | |||
667 | return false; | |||
668 | } | |||
669 | ||||
670 | void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB, | |||
671 | MachineBasicBlock::iterator I, | |||
672 | unsigned DestReg, unsigned SubIdx, | |||
673 | const MachineInstr &Orig, | |||
674 | const TargetRegisterInfo &TRI) const { | |||
675 | bool ClobbersEFLAGS = false; | |||
676 | for (const MachineOperand &MO : Orig.operands()) { | |||
677 | if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) { | |||
678 | ClobbersEFLAGS = true; | |||
679 | break; | |||
680 | } | |||
681 | } | |||
682 | ||||
683 | if (ClobbersEFLAGS && !isSafeToClobberEFLAGS(MBB, I)) { | |||
684 | // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side | |||
685 | // effects. | |||
686 | unsigned NewOpc = X86::MOV32ri; | |||
687 | int Value; | |||
688 | switch (Orig.getOpcode()) { | |||
689 | case X86::MOV64r0: NewOpc = X86::MOV32ri64; Value = 0; break; | |||
690 | case X86::MOV32r0: Value = 0; break; | |||
691 | case X86::MOV32r1: Value = 1; break; | |||
692 | case X86::MOV32r_1: Value = -1; break; | |||
693 | default: | |||
694 | llvm_unreachable("Unexpected instruction!")::llvm::llvm_unreachable_internal("Unexpected instruction!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 694); | |||
695 | } | |||
696 | ||||
697 | const DebugLoc &DL = Orig.getDebugLoc(); | |||
698 | BuildMI(MBB, I, DL, get(NewOpc)) | |||
699 | .add(Orig.getOperand(0)) | |||
700 | .addImm(Value); | |||
701 | } else { | |||
702 | MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); | |||
703 | MBB.insert(I, MI); | |||
704 | } | |||
705 | ||||
706 | MachineInstr &NewMI = *std::prev(I); | |||
707 | NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI); | |||
708 | } | |||
709 | ||||
710 | /// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead. | |||
711 | bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const { | |||
712 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
713 | MachineOperand &MO = MI.getOperand(i); | |||
714 | if (MO.isReg() && MO.isDef() && | |||
715 | MO.getReg() == X86::EFLAGS && !MO.isDead()) { | |||
716 | return true; | |||
717 | } | |||
718 | } | |||
719 | return false; | |||
720 | } | |||
721 | ||||
722 | /// Check whether the shift count for a machine operand is non-zero. | |||
723 | inline static unsigned getTruncatedShiftCount(MachineInstr &MI, | |||
724 | unsigned ShiftAmtOperandIdx) { | |||
725 | // The shift count is six bits with the REX.W prefix and five bits without. | |||
726 | unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31; | |||
727 | unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm(); | |||
728 | return Imm & ShiftCountMask; | |||
729 | } | |||
730 | ||||
731 | /// Check whether the given shift count is appropriate | |||
732 | /// can be represented by a LEA instruction. | |||
733 | inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) { | |||
734 | // Left shift instructions can be transformed into load-effective-address | |||
735 | // instructions if we can encode them appropriately. | |||
736 | // A LEA instruction utilizes a SIB byte to encode its scale factor. | |||
737 | // The SIB.scale field is two bits wide which means that we can encode any | |||
738 | // shift amount less than 4. | |||
739 | return ShAmt < 4 && ShAmt > 0; | |||
740 | } | |||
741 | ||||
742 | bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, | |||
743 | unsigned Opc, bool AllowSP, unsigned &NewSrc, | |||
744 | bool &isKill, bool &isUndef, | |||
745 | MachineOperand &ImplicitOp, | |||
746 | LiveVariables *LV) const { | |||
747 | MachineFunction &MF = *MI.getParent()->getParent(); | |||
748 | const TargetRegisterClass *RC; | |||
749 | if (AllowSP) { | |||
750 | RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass; | |||
751 | } else { | |||
752 | RC = Opc != X86::LEA32r ? | |||
753 | &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass; | |||
754 | } | |||
755 | unsigned SrcReg = Src.getReg(); | |||
756 | ||||
757 | // For both LEA64 and LEA32 the register already has essentially the right | |||
758 | // type (32-bit or 64-bit) we may just need to forbid SP. | |||
759 | if (Opc != X86::LEA64_32r) { | |||
760 | NewSrc = SrcReg; | |||
761 | isKill = Src.isKill(); | |||
762 | isUndef = Src.isUndef(); | |||
763 | ||||
764 | if (TargetRegisterInfo::isVirtualRegister(NewSrc) && | |||
765 | !MF.getRegInfo().constrainRegClass(NewSrc, RC)) | |||
766 | return false; | |||
767 | ||||
768 | return true; | |||
769 | } | |||
770 | ||||
771 | // This is for an LEA64_32r and incoming registers are 32-bit. One way or | |||
772 | // another we need to add 64-bit registers to the final MI. | |||
773 | if (TargetRegisterInfo::isPhysicalRegister(SrcReg)) { | |||
774 | ImplicitOp = Src; | |||
775 | ImplicitOp.setImplicit(); | |||
776 | ||||
777 | NewSrc = getX86SubSuperRegister(Src.getReg(), 64); | |||
778 | isKill = Src.isKill(); | |||
779 | isUndef = Src.isUndef(); | |||
780 | } else { | |||
781 | // Virtual register of the wrong class, we have to create a temporary 64-bit | |||
782 | // vreg to feed into the LEA. | |||
783 | NewSrc = MF.getRegInfo().createVirtualRegister(RC); | |||
784 | MachineInstr *Copy = | |||
785 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
786 | .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit) | |||
787 | .add(Src); | |||
788 | ||||
789 | // Which is obviously going to be dead after we're done with it. | |||
790 | isKill = true; | |||
791 | isUndef = false; | |||
792 | ||||
793 | if (LV) | |||
794 | LV->replaceKillInstruction(SrcReg, MI, *Copy); | |||
795 | } | |||
796 | ||||
797 | // We've set all the parameters without issue. | |||
798 | return true; | |||
799 | } | |||
800 | ||||
801 | /// Helper for convertToThreeAddress when 16-bit LEA is disabled, use 32-bit | |||
802 | /// LEA to form 3-address code by promoting to a 32-bit superregister and then | |||
803 | /// truncating back down to a 16-bit subregister. | |||
804 | MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA( | |||
805 | unsigned MIOpc, MachineFunction::iterator &MFI, MachineInstr &MI, | |||
806 | LiveVariables *LV) const { | |||
807 | MachineBasicBlock::iterator MBBI = MI.getIterator(); | |||
808 | unsigned Dest = MI.getOperand(0).getReg(); | |||
809 | unsigned Src = MI.getOperand(1).getReg(); | |||
810 | bool isDead = MI.getOperand(0).isDead(); | |||
811 | bool isKill = MI.getOperand(1).isKill(); | |||
812 | ||||
813 | MachineRegisterInfo &RegInfo = MFI->getParent()->getRegInfo(); | |||
814 | unsigned leaOutReg = RegInfo.createVirtualRegister(&X86::GR32RegClass); | |||
815 | unsigned Opc, leaInReg; | |||
816 | if (Subtarget.is64Bit()) { | |||
817 | Opc = X86::LEA64_32r; | |||
818 | leaInReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); | |||
819 | } else { | |||
820 | Opc = X86::LEA32r; | |||
821 | leaInReg = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); | |||
822 | } | |||
823 | ||||
824 | // Build and insert into an implicit UNDEF value. This is OK because | |||
825 | // well be shifting and then extracting the lower 16-bits. | |||
826 | // This has the potential to cause partial register stall. e.g. | |||
827 | // movw (%rbp,%rcx,2), %dx | |||
828 | // leal -65(%rdx), %esi | |||
829 | // But testing has shown this *does* help performance in 64-bit mode (at | |||
830 | // least on modern x86 machines). | |||
831 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), leaInReg); | |||
832 | MachineInstr *InsMI = | |||
833 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
834 | .addReg(leaInReg, RegState::Define, X86::sub_16bit) | |||
835 | .addReg(Src, getKillRegState(isKill)); | |||
836 | ||||
837 | MachineInstrBuilder MIB = | |||
838 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(Opc), leaOutReg); | |||
839 | switch (MIOpc) { | |||
840 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 840); | |||
841 | case X86::SHL16ri: { | |||
842 | unsigned ShAmt = MI.getOperand(2).getImm(); | |||
843 | MIB.addReg(0).addImm(1ULL << ShAmt) | |||
844 | .addReg(leaInReg, RegState::Kill).addImm(0).addReg(0); | |||
845 | break; | |||
846 | } | |||
847 | case X86::INC16r: | |||
848 | addRegOffset(MIB, leaInReg, true, 1); | |||
849 | break; | |||
850 | case X86::DEC16r: | |||
851 | addRegOffset(MIB, leaInReg, true, -1); | |||
852 | break; | |||
853 | case X86::ADD16ri: | |||
854 | case X86::ADD16ri8: | |||
855 | case X86::ADD16ri_DB: | |||
856 | case X86::ADD16ri8_DB: | |||
857 | addRegOffset(MIB, leaInReg, true, MI.getOperand(2).getImm()); | |||
858 | break; | |||
859 | case X86::ADD16rr: | |||
860 | case X86::ADD16rr_DB: { | |||
861 | unsigned Src2 = MI.getOperand(2).getReg(); | |||
862 | bool isKill2 = MI.getOperand(2).isKill(); | |||
863 | unsigned leaInReg2 = 0; | |||
864 | MachineInstr *InsMI2 = nullptr; | |||
865 | if (Src == Src2) { | |||
866 | // ADD16rr killed %reg1028, %reg1028 | |||
867 | // just a single insert_subreg. | |||
868 | addRegReg(MIB, leaInReg, true, leaInReg, false); | |||
869 | } else { | |||
870 | if (Subtarget.is64Bit()) | |||
871 | leaInReg2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); | |||
872 | else | |||
873 | leaInReg2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); | |||
874 | // Build and insert into an implicit UNDEF value. This is OK because | |||
875 | // well be shifting and then extracting the lower 16-bits. | |||
876 | BuildMI(*MFI, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), leaInReg2); | |||
877 | InsMI2 = BuildMI(*MFI, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
878 | .addReg(leaInReg2, RegState::Define, X86::sub_16bit) | |||
879 | .addReg(Src2, getKillRegState(isKill2)); | |||
880 | addRegReg(MIB, leaInReg, true, leaInReg2, true); | |||
881 | } | |||
882 | if (LV && isKill2 && InsMI2) | |||
883 | LV->replaceKillInstruction(Src2, MI, *InsMI2); | |||
884 | break; | |||
885 | } | |||
886 | } | |||
887 | ||||
888 | MachineInstr *NewMI = MIB; | |||
889 | MachineInstr *ExtMI = | |||
890 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
891 | .addReg(Dest, RegState::Define | getDeadRegState(isDead)) | |||
892 | .addReg(leaOutReg, RegState::Kill, X86::sub_16bit); | |||
893 | ||||
894 | if (LV) { | |||
895 | // Update live variables | |||
896 | LV->getVarInfo(leaInReg).Kills.push_back(NewMI); | |||
897 | LV->getVarInfo(leaOutReg).Kills.push_back(ExtMI); | |||
898 | if (isKill) | |||
899 | LV->replaceKillInstruction(Src, MI, *InsMI); | |||
900 | if (isDead) | |||
901 | LV->replaceKillInstruction(Dest, MI, *ExtMI); | |||
902 | } | |||
903 | ||||
904 | return ExtMI; | |||
905 | } | |||
906 | ||||
907 | /// This method must be implemented by targets that | |||
908 | /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target | |||
909 | /// may be able to convert a two-address instruction into a true | |||
910 | /// three-address instruction on demand. This allows the X86 target (for | |||
911 | /// example) to convert ADD and SHL instructions into LEA instructions if they | |||
912 | /// would require register copies due to two-addressness. | |||
913 | /// | |||
914 | /// This method returns a null pointer if the transformation cannot be | |||
915 | /// performed, otherwise it returns the new instruction. | |||
916 | /// | |||
917 | MachineInstr * | |||
918 | X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI, | |||
919 | MachineInstr &MI, LiveVariables *LV) const { | |||
920 | // The following opcodes also sets the condition code register(s). Only | |||
921 | // convert them to equivalent lea if the condition code register def's | |||
922 | // are dead! | |||
923 | if (hasLiveCondCodeDef(MI)) | |||
924 | return nullptr; | |||
925 | ||||
926 | MachineFunction &MF = *MI.getParent()->getParent(); | |||
927 | // All instructions input are two-addr instructions. Get the known operands. | |||
928 | const MachineOperand &Dest = MI.getOperand(0); | |||
929 | const MachineOperand &Src = MI.getOperand(1); | |||
930 | ||||
931 | MachineInstr *NewMI = nullptr; | |||
932 | // FIXME: 16-bit LEA's are really slow on Athlons, but not bad on P4's. When | |||
933 | // we have better subtarget support, enable the 16-bit LEA generation here. | |||
934 | // 16-bit LEA is also slow on Core2. | |||
935 | bool DisableLEA16 = true; | |||
936 | bool is64Bit = Subtarget.is64Bit(); | |||
937 | ||||
938 | unsigned MIOpc = MI.getOpcode(); | |||
939 | switch (MIOpc) { | |||
940 | default: return nullptr; | |||
941 | case X86::SHL64ri: { | |||
942 | assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!")((MI.getNumOperands() >= 3 && "Unknown shift instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown shift instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 942, __PRETTY_FUNCTION__)); | |||
943 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
944 | if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; | |||
945 | ||||
946 | // LEA can't handle RSP. | |||
947 | if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) && | |||
948 | !MF.getRegInfo().constrainRegClass(Src.getReg(), | |||
949 | &X86::GR64_NOSPRegClass)) | |||
950 | return nullptr; | |||
951 | ||||
952 | NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)) | |||
953 | .add(Dest) | |||
954 | .addReg(0) | |||
955 | .addImm(1ULL << ShAmt) | |||
956 | .add(Src) | |||
957 | .addImm(0) | |||
958 | .addReg(0); | |||
959 | break; | |||
960 | } | |||
961 | case X86::SHL32ri: { | |||
962 | assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!")((MI.getNumOperands() >= 3 && "Unknown shift instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown shift instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 962, __PRETTY_FUNCTION__)); | |||
963 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
964 | if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; | |||
965 | ||||
966 | unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r; | |||
967 | ||||
968 | // LEA can't handle ESP. | |||
969 | bool isKill, isUndef; | |||
970 | unsigned SrcReg; | |||
971 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
972 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, | |||
973 | SrcReg, isKill, isUndef, ImplicitOp, LV)) | |||
974 | return nullptr; | |||
975 | ||||
976 | MachineInstrBuilder MIB = | |||
977 | BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
978 | .add(Dest) | |||
979 | .addReg(0) | |||
980 | .addImm(1ULL << ShAmt) | |||
981 | .addReg(SrcReg, getKillRegState(isKill) | getUndefRegState(isUndef)) | |||
982 | .addImm(0) | |||
983 | .addReg(0); | |||
984 | if (ImplicitOp.getReg() != 0) | |||
985 | MIB.add(ImplicitOp); | |||
986 | NewMI = MIB; | |||
987 | ||||
988 | break; | |||
989 | } | |||
990 | case X86::SHL16ri: { | |||
991 | assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!")((MI.getNumOperands() >= 3 && "Unknown shift instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown shift instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 991, __PRETTY_FUNCTION__)); | |||
992 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
993 | if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; | |||
994 | ||||
995 | if (DisableLEA16) | |||
996 | return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) | |||
997 | : nullptr; | |||
998 | NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)) | |||
999 | .add(Dest) | |||
1000 | .addReg(0) | |||
1001 | .addImm(1ULL << ShAmt) | |||
1002 | .add(Src) | |||
1003 | .addImm(0) | |||
1004 | .addReg(0); | |||
1005 | break; | |||
1006 | } | |||
1007 | case X86::INC64r: | |||
1008 | case X86::INC32r: { | |||
1009 | assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!")((MI.getNumOperands() >= 2 && "Unknown inc instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 2 && \"Unknown inc instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1009, __PRETTY_FUNCTION__)); | |||
1010 | unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r | |||
1011 | : (is64Bit ? X86::LEA64_32r : X86::LEA32r); | |||
1012 | bool isKill, isUndef; | |||
1013 | unsigned SrcReg; | |||
1014 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1015 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, | |||
1016 | SrcReg, isKill, isUndef, ImplicitOp, LV)) | |||
1017 | return nullptr; | |||
1018 | ||||
1019 | MachineInstrBuilder MIB = | |||
1020 | BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1021 | .add(Dest) | |||
1022 | .addReg(SrcReg, | |||
1023 | getKillRegState(isKill) | getUndefRegState(isUndef)); | |||
1024 | if (ImplicitOp.getReg() != 0) | |||
1025 | MIB.add(ImplicitOp); | |||
1026 | ||||
1027 | NewMI = addOffset(MIB, 1); | |||
1028 | break; | |||
1029 | } | |||
1030 | case X86::INC16r: | |||
1031 | if (DisableLEA16) | |||
1032 | return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) | |||
1033 | : nullptr; | |||
1034 | assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!")((MI.getNumOperands() >= 2 && "Unknown inc instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 2 && \"Unknown inc instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1034, __PRETTY_FUNCTION__)); | |||
1035 | NewMI = addOffset( | |||
1036 | BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest).add(Src), 1); | |||
1037 | break; | |||
1038 | case X86::DEC64r: | |||
1039 | case X86::DEC32r: { | |||
1040 | assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!")((MI.getNumOperands() >= 2 && "Unknown dec instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 2 && \"Unknown dec instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1040, __PRETTY_FUNCTION__)); | |||
1041 | unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r | |||
1042 | : (is64Bit ? X86::LEA64_32r : X86::LEA32r); | |||
1043 | ||||
1044 | bool isKill, isUndef; | |||
1045 | unsigned SrcReg; | |||
1046 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1047 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, | |||
1048 | SrcReg, isKill, isUndef, ImplicitOp, LV)) | |||
1049 | return nullptr; | |||
1050 | ||||
1051 | MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1052 | .add(Dest) | |||
1053 | .addReg(SrcReg, getUndefRegState(isUndef) | | |||
1054 | getKillRegState(isKill)); | |||
1055 | if (ImplicitOp.getReg() != 0) | |||
1056 | MIB.add(ImplicitOp); | |||
1057 | ||||
1058 | NewMI = addOffset(MIB, -1); | |||
1059 | ||||
1060 | break; | |||
1061 | } | |||
1062 | case X86::DEC16r: | |||
1063 | if (DisableLEA16) | |||
1064 | return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) | |||
1065 | : nullptr; | |||
1066 | assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!")((MI.getNumOperands() >= 2 && "Unknown dec instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 2 && \"Unknown dec instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1066, __PRETTY_FUNCTION__)); | |||
1067 | NewMI = addOffset( | |||
1068 | BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest).add(Src), -1); | |||
1069 | break; | |||
1070 | case X86::ADD64rr: | |||
1071 | case X86::ADD64rr_DB: | |||
1072 | case X86::ADD32rr: | |||
1073 | case X86::ADD32rr_DB: { | |||
1074 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1074, __PRETTY_FUNCTION__)); | |||
1075 | unsigned Opc; | |||
1076 | if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB) | |||
1077 | Opc = X86::LEA64r; | |||
1078 | else | |||
1079 | Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r; | |||
1080 | ||||
1081 | bool isKill, isUndef; | |||
1082 | unsigned SrcReg; | |||
1083 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1084 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true, | |||
1085 | SrcReg, isKill, isUndef, ImplicitOp, LV)) | |||
1086 | return nullptr; | |||
1087 | ||||
1088 | const MachineOperand &Src2 = MI.getOperand(2); | |||
1089 | bool isKill2, isUndef2; | |||
1090 | unsigned SrcReg2; | |||
1091 | MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false); | |||
1092 | if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/ false, | |||
1093 | SrcReg2, isKill2, isUndef2, ImplicitOp2, LV)) | |||
1094 | return nullptr; | |||
1095 | ||||
1096 | MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest); | |||
1097 | if (ImplicitOp.getReg() != 0) | |||
1098 | MIB.add(ImplicitOp); | |||
1099 | if (ImplicitOp2.getReg() != 0) | |||
1100 | MIB.add(ImplicitOp2); | |||
1101 | ||||
1102 | NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2); | |||
1103 | ||||
1104 | // Preserve undefness of the operands. | |||
1105 | NewMI->getOperand(1).setIsUndef(isUndef); | |||
1106 | NewMI->getOperand(3).setIsUndef(isUndef2); | |||
1107 | ||||
1108 | if (LV && Src2.isKill()) | |||
1109 | LV->replaceKillInstruction(SrcReg2, MI, *NewMI); | |||
1110 | break; | |||
1111 | } | |||
1112 | case X86::ADD16rr: | |||
1113 | case X86::ADD16rr_DB: { | |||
1114 | if (DisableLEA16) | |||
1115 | return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) | |||
1116 | : nullptr; | |||
1117 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1117, __PRETTY_FUNCTION__)); | |||
1118 | unsigned Src2 = MI.getOperand(2).getReg(); | |||
1119 | bool isKill2 = MI.getOperand(2).isKill(); | |||
1120 | NewMI = addRegReg(BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest), | |||
1121 | Src.getReg(), Src.isKill(), Src2, isKill2); | |||
1122 | ||||
1123 | // Preserve undefness of the operands. | |||
1124 | bool isUndef = MI.getOperand(1).isUndef(); | |||
1125 | bool isUndef2 = MI.getOperand(2).isUndef(); | |||
1126 | NewMI->getOperand(1).setIsUndef(isUndef); | |||
1127 | NewMI->getOperand(3).setIsUndef(isUndef2); | |||
1128 | ||||
1129 | if (LV && isKill2) | |||
1130 | LV->replaceKillInstruction(Src2, MI, *NewMI); | |||
1131 | break; | |||
1132 | } | |||
1133 | case X86::ADD64ri32: | |||
1134 | case X86::ADD64ri8: | |||
1135 | case X86::ADD64ri32_DB: | |||
1136 | case X86::ADD64ri8_DB: | |||
1137 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1137, __PRETTY_FUNCTION__)); | |||
1138 | NewMI = addOffset( | |||
1139 | BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src), | |||
1140 | MI.getOperand(2)); | |||
1141 | break; | |||
1142 | case X86::ADD32ri: | |||
1143 | case X86::ADD32ri8: | |||
1144 | case X86::ADD32ri_DB: | |||
1145 | case X86::ADD32ri8_DB: { | |||
1146 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1146, __PRETTY_FUNCTION__)); | |||
1147 | unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r; | |||
1148 | ||||
1149 | bool isKill, isUndef; | |||
1150 | unsigned SrcReg; | |||
1151 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1152 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true, | |||
1153 | SrcReg, isKill, isUndef, ImplicitOp, LV)) | |||
1154 | return nullptr; | |||
1155 | ||||
1156 | MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1157 | .add(Dest) | |||
1158 | .addReg(SrcReg, getUndefRegState(isUndef) | | |||
1159 | getKillRegState(isKill)); | |||
1160 | if (ImplicitOp.getReg() != 0) | |||
1161 | MIB.add(ImplicitOp); | |||
1162 | ||||
1163 | NewMI = addOffset(MIB, MI.getOperand(2)); | |||
1164 | break; | |||
1165 | } | |||
1166 | case X86::ADD16ri: | |||
1167 | case X86::ADD16ri8: | |||
1168 | case X86::ADD16ri_DB: | |||
1169 | case X86::ADD16ri8_DB: | |||
1170 | if (DisableLEA16) | |||
1171 | return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) | |||
1172 | : nullptr; | |||
1173 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1173, __PRETTY_FUNCTION__)); | |||
1174 | NewMI = addOffset( | |||
1175 | BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest).add(Src), | |||
1176 | MI.getOperand(2)); | |||
1177 | break; | |||
1178 | ||||
1179 | case X86::VMOVDQU8Z128rmk: | |||
1180 | case X86::VMOVDQU8Z256rmk: | |||
1181 | case X86::VMOVDQU8Zrmk: | |||
1182 | case X86::VMOVDQU16Z128rmk: | |||
1183 | case X86::VMOVDQU16Z256rmk: | |||
1184 | case X86::VMOVDQU16Zrmk: | |||
1185 | case X86::VMOVDQU32Z128rmk: case X86::VMOVDQA32Z128rmk: | |||
1186 | case X86::VMOVDQU32Z256rmk: case X86::VMOVDQA32Z256rmk: | |||
1187 | case X86::VMOVDQU32Zrmk: case X86::VMOVDQA32Zrmk: | |||
1188 | case X86::VMOVDQU64Z128rmk: case X86::VMOVDQA64Z128rmk: | |||
1189 | case X86::VMOVDQU64Z256rmk: case X86::VMOVDQA64Z256rmk: | |||
1190 | case X86::VMOVDQU64Zrmk: case X86::VMOVDQA64Zrmk: | |||
1191 | case X86::VMOVUPDZ128rmk: case X86::VMOVAPDZ128rmk: | |||
1192 | case X86::VMOVUPDZ256rmk: case X86::VMOVAPDZ256rmk: | |||
1193 | case X86::VMOVUPDZrmk: case X86::VMOVAPDZrmk: | |||
1194 | case X86::VMOVUPSZ128rmk: case X86::VMOVAPSZ128rmk: | |||
1195 | case X86::VMOVUPSZ256rmk: case X86::VMOVAPSZ256rmk: | |||
1196 | case X86::VMOVUPSZrmk: case X86::VMOVAPSZrmk: { | |||
1197 | unsigned Opc; | |||
1198 | switch (MIOpc) { | |||
1199 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1199); | |||
1200 | case X86::VMOVDQU8Z128rmk: Opc = X86::VPBLENDMBZ128rmk; break; | |||
1201 | case X86::VMOVDQU8Z256rmk: Opc = X86::VPBLENDMBZ256rmk; break; | |||
1202 | case X86::VMOVDQU8Zrmk: Opc = X86::VPBLENDMBZrmk; break; | |||
1203 | case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk; break; | |||
1204 | case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk; break; | |||
1205 | case X86::VMOVDQU16Zrmk: Opc = X86::VPBLENDMWZrmk; break; | |||
1206 | case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; | |||
1207 | case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; | |||
1208 | case X86::VMOVDQU32Zrmk: Opc = X86::VPBLENDMDZrmk; break; | |||
1209 | case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; | |||
1210 | case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; | |||
1211 | case X86::VMOVDQU64Zrmk: Opc = X86::VPBLENDMQZrmk; break; | |||
1212 | case X86::VMOVUPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; | |||
1213 | case X86::VMOVUPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; | |||
1214 | case X86::VMOVUPDZrmk: Opc = X86::VBLENDMPDZrmk; break; | |||
1215 | case X86::VMOVUPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; | |||
1216 | case X86::VMOVUPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; | |||
1217 | case X86::VMOVUPSZrmk: Opc = X86::VBLENDMPSZrmk; break; | |||
1218 | case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; | |||
1219 | case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; | |||
1220 | case X86::VMOVDQA32Zrmk: Opc = X86::VPBLENDMDZrmk; break; | |||
1221 | case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; | |||
1222 | case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; | |||
1223 | case X86::VMOVDQA64Zrmk: Opc = X86::VPBLENDMQZrmk; break; | |||
1224 | case X86::VMOVAPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; | |||
1225 | case X86::VMOVAPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; | |||
1226 | case X86::VMOVAPDZrmk: Opc = X86::VBLENDMPDZrmk; break; | |||
1227 | case X86::VMOVAPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; | |||
1228 | case X86::VMOVAPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; | |||
1229 | case X86::VMOVAPSZrmk: Opc = X86::VBLENDMPSZrmk; break; | |||
1230 | } | |||
1231 | ||||
1232 | NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1233 | .add(Dest) | |||
1234 | .add(MI.getOperand(2)) | |||
1235 | .add(Src) | |||
1236 | .add(MI.getOperand(3)) | |||
1237 | .add(MI.getOperand(4)) | |||
1238 | .add(MI.getOperand(5)) | |||
1239 | .add(MI.getOperand(6)) | |||
1240 | .add(MI.getOperand(7)); | |||
1241 | break; | |||
1242 | } | |||
1243 | case X86::VMOVDQU8Z128rrk: | |||
1244 | case X86::VMOVDQU8Z256rrk: | |||
1245 | case X86::VMOVDQU8Zrrk: | |||
1246 | case X86::VMOVDQU16Z128rrk: | |||
1247 | case X86::VMOVDQU16Z256rrk: | |||
1248 | case X86::VMOVDQU16Zrrk: | |||
1249 | case X86::VMOVDQU32Z128rrk: case X86::VMOVDQA32Z128rrk: | |||
1250 | case X86::VMOVDQU32Z256rrk: case X86::VMOVDQA32Z256rrk: | |||
1251 | case X86::VMOVDQU32Zrrk: case X86::VMOVDQA32Zrrk: | |||
1252 | case X86::VMOVDQU64Z128rrk: case X86::VMOVDQA64Z128rrk: | |||
1253 | case X86::VMOVDQU64Z256rrk: case X86::VMOVDQA64Z256rrk: | |||
1254 | case X86::VMOVDQU64Zrrk: case X86::VMOVDQA64Zrrk: | |||
1255 | case X86::VMOVUPDZ128rrk: case X86::VMOVAPDZ128rrk: | |||
1256 | case X86::VMOVUPDZ256rrk: case X86::VMOVAPDZ256rrk: | |||
1257 | case X86::VMOVUPDZrrk: case X86::VMOVAPDZrrk: | |||
1258 | case X86::VMOVUPSZ128rrk: case X86::VMOVAPSZ128rrk: | |||
1259 | case X86::VMOVUPSZ256rrk: case X86::VMOVAPSZ256rrk: | |||
1260 | case X86::VMOVUPSZrrk: case X86::VMOVAPSZrrk: { | |||
1261 | unsigned Opc; | |||
1262 | switch (MIOpc) { | |||
1263 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1263); | |||
1264 | case X86::VMOVDQU8Z128rrk: Opc = X86::VPBLENDMBZ128rrk; break; | |||
1265 | case X86::VMOVDQU8Z256rrk: Opc = X86::VPBLENDMBZ256rrk; break; | |||
1266 | case X86::VMOVDQU8Zrrk: Opc = X86::VPBLENDMBZrrk; break; | |||
1267 | case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk; break; | |||
1268 | case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk; break; | |||
1269 | case X86::VMOVDQU16Zrrk: Opc = X86::VPBLENDMWZrrk; break; | |||
1270 | case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; | |||
1271 | case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; | |||
1272 | case X86::VMOVDQU32Zrrk: Opc = X86::VPBLENDMDZrrk; break; | |||
1273 | case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; | |||
1274 | case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; | |||
1275 | case X86::VMOVDQU64Zrrk: Opc = X86::VPBLENDMQZrrk; break; | |||
1276 | case X86::VMOVUPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; | |||
1277 | case X86::VMOVUPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; | |||
1278 | case X86::VMOVUPDZrrk: Opc = X86::VBLENDMPDZrrk; break; | |||
1279 | case X86::VMOVUPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; | |||
1280 | case X86::VMOVUPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; | |||
1281 | case X86::VMOVUPSZrrk: Opc = X86::VBLENDMPSZrrk; break; | |||
1282 | case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; | |||
1283 | case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; | |||
1284 | case X86::VMOVDQA32Zrrk: Opc = X86::VPBLENDMDZrrk; break; | |||
1285 | case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; | |||
1286 | case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; | |||
1287 | case X86::VMOVDQA64Zrrk: Opc = X86::VPBLENDMQZrrk; break; | |||
1288 | case X86::VMOVAPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; | |||
1289 | case X86::VMOVAPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; | |||
1290 | case X86::VMOVAPDZrrk: Opc = X86::VBLENDMPDZrrk; break; | |||
1291 | case X86::VMOVAPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; | |||
1292 | case X86::VMOVAPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; | |||
1293 | case X86::VMOVAPSZrrk: Opc = X86::VBLENDMPSZrrk; break; | |||
1294 | } | |||
1295 | ||||
1296 | NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1297 | .add(Dest) | |||
1298 | .add(MI.getOperand(2)) | |||
1299 | .add(Src) | |||
1300 | .add(MI.getOperand(3)); | |||
1301 | break; | |||
1302 | } | |||
1303 | } | |||
1304 | ||||
1305 | if (!NewMI) return nullptr; | |||
1306 | ||||
1307 | if (LV) { // Update live variables | |||
1308 | if (Src.isKill()) | |||
1309 | LV->replaceKillInstruction(Src.getReg(), MI, *NewMI); | |||
1310 | if (Dest.isDead()) | |||
1311 | LV->replaceKillInstruction(Dest.getReg(), MI, *NewMI); | |||
1312 | } | |||
1313 | ||||
1314 | MFI->insert(MI.getIterator(), NewMI); // Insert the new inst | |||
1315 | return NewMI; | |||
1316 | } | |||
1317 | ||||
1318 | /// This determines which of three possible cases of a three source commute | |||
1319 | /// the source indexes correspond to taking into account any mask operands. | |||
1320 | /// All prevents commuting a passthru operand. Returns -1 if the commute isn't | |||
1321 | /// possible. | |||
1322 | /// Case 0 - Possible to commute the first and second operands. | |||
1323 | /// Case 1 - Possible to commute the first and third operands. | |||
1324 | /// Case 2 - Possible to commute the second and third operands. | |||
1325 | static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, | |||
1326 | unsigned SrcOpIdx2) { | |||
1327 | // Put the lowest index to SrcOpIdx1 to simplify the checks below. | |||
1328 | if (SrcOpIdx1 > SrcOpIdx2) | |||
1329 | std::swap(SrcOpIdx1, SrcOpIdx2); | |||
1330 | ||||
1331 | unsigned Op1 = 1, Op2 = 2, Op3 = 3; | |||
1332 | if (X86II::isKMasked(TSFlags)) { | |||
1333 | Op2++; | |||
1334 | Op3++; | |||
1335 | } | |||
1336 | ||||
1337 | if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2) | |||
1338 | return 0; | |||
1339 | if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3) | |||
1340 | return 1; | |||
1341 | if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3) | |||
1342 | return 2; | |||
1343 | llvm_unreachable("Unknown three src commute case.")::llvm::llvm_unreachable_internal("Unknown three src commute case." , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1343); | |||
1344 | } | |||
1345 | ||||
1346 | unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands( | |||
1347 | const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, | |||
1348 | const X86InstrFMA3Group &FMA3Group) const { | |||
1349 | ||||
1350 | unsigned Opc = MI.getOpcode(); | |||
1351 | ||||
1352 | // TODO: Commuting the 1st operand of FMA*_Int requires some additional | |||
1353 | // analysis. The commute optimization is legal only if all users of FMA*_Int | |||
1354 | // use only the lowest element of the FMA*_Int instruction. Such analysis are | |||
1355 | // not implemented yet. So, just return 0 in that case. | |||
1356 | // When such analysis are available this place will be the right place for | |||
1357 | // calling it. | |||
1358 | assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) &&((!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && "Intrinsic instructions can't commute operand 1" ) ? static_cast<void> (0) : __assert_fail ("!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && \"Intrinsic instructions can't commute operand 1\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1359, __PRETTY_FUNCTION__)) | |||
1359 | "Intrinsic instructions can't commute operand 1")((!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && "Intrinsic instructions can't commute operand 1" ) ? static_cast<void> (0) : __assert_fail ("!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && \"Intrinsic instructions can't commute operand 1\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1359, __PRETTY_FUNCTION__)); | |||
1360 | ||||
1361 | // Determine which case this commute is or if it can't be done. | |||
1362 | unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, | |||
1363 | SrcOpIdx2); | |||
1364 | assert(Case < 3 && "Unexpected case number!")((Case < 3 && "Unexpected case number!") ? static_cast <void> (0) : __assert_fail ("Case < 3 && \"Unexpected case number!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1364, __PRETTY_FUNCTION__)); | |||
1365 | ||||
1366 | // Define the FMA forms mapping array that helps to map input FMA form | |||
1367 | // to output FMA form to preserve the operation semantics after | |||
1368 | // commuting the operands. | |||
1369 | const unsigned Form132Index = 0; | |||
1370 | const unsigned Form213Index = 1; | |||
1371 | const unsigned Form231Index = 2; | |||
1372 | static const unsigned FormMapping[][3] = { | |||
1373 | // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2; | |||
1374 | // FMA132 A, C, b; ==> FMA231 C, A, b; | |||
1375 | // FMA213 B, A, c; ==> FMA213 A, B, c; | |||
1376 | // FMA231 C, A, b; ==> FMA132 A, C, b; | |||
1377 | { Form231Index, Form213Index, Form132Index }, | |||
1378 | // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3; | |||
1379 | // FMA132 A, c, B; ==> FMA132 B, c, A; | |||
1380 | // FMA213 B, a, C; ==> FMA231 C, a, B; | |||
1381 | // FMA231 C, a, B; ==> FMA213 B, a, C; | |||
1382 | { Form132Index, Form231Index, Form213Index }, | |||
1383 | // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3; | |||
1384 | // FMA132 a, C, B; ==> FMA213 a, B, C; | |||
1385 | // FMA213 b, A, C; ==> FMA132 b, C, A; | |||
1386 | // FMA231 c, A, B; ==> FMA231 c, B, A; | |||
1387 | { Form213Index, Form132Index, Form231Index } | |||
1388 | }; | |||
1389 | ||||
1390 | unsigned FMAForms[3]; | |||
1391 | FMAForms[0] = FMA3Group.get132Opcode(); | |||
1392 | FMAForms[1] = FMA3Group.get213Opcode(); | |||
1393 | FMAForms[2] = FMA3Group.get231Opcode(); | |||
1394 | unsigned FormIndex; | |||
1395 | for (FormIndex = 0; FormIndex < 3; FormIndex++) | |||
1396 | if (Opc == FMAForms[FormIndex]) | |||
1397 | break; | |||
1398 | ||||
1399 | // Everything is ready, just adjust the FMA opcode and return it. | |||
1400 | FormIndex = FormMapping[Case][FormIndex]; | |||
1401 | return FMAForms[FormIndex]; | |||
1402 | } | |||
1403 | ||||
1404 | static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, | |||
1405 | unsigned SrcOpIdx2) { | |||
1406 | // Determine which case this commute is or if it can't be done. | |||
1407 | unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, | |||
1408 | SrcOpIdx2); | |||
1409 | assert(Case < 3 && "Unexpected case value!")((Case < 3 && "Unexpected case value!") ? static_cast <void> (0) : __assert_fail ("Case < 3 && \"Unexpected case value!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1409, __PRETTY_FUNCTION__)); | |||
1410 | ||||
1411 | // For each case we need to swap two pairs of bits in the final immediate. | |||
1412 | static const uint8_t SwapMasks[3][4] = { | |||
1413 | { 0x04, 0x10, 0x08, 0x20 }, // Swap bits 2/4 and 3/5. | |||
1414 | { 0x02, 0x10, 0x08, 0x40 }, // Swap bits 1/4 and 3/6. | |||
1415 | { 0x02, 0x04, 0x20, 0x40 }, // Swap bits 1/2 and 5/6. | |||
1416 | }; | |||
1417 | ||||
1418 | uint8_t Imm = MI.getOperand(MI.getNumOperands()-1).getImm(); | |||
1419 | // Clear out the bits we are swapping. | |||
1420 | uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] | | |||
1421 | SwapMasks[Case][2] | SwapMasks[Case][3]); | |||
1422 | // If the immediate had a bit of the pair set, then set the opposite bit. | |||
1423 | if (Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1]; | |||
1424 | if (Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0]; | |||
1425 | if (Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3]; | |||
1426 | if (Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2]; | |||
1427 | MI.getOperand(MI.getNumOperands()-1).setImm(NewImm); | |||
1428 | } | |||
1429 | ||||
1430 | // Returns true if this is a VPERMI2 or VPERMT2 instruction that can be | |||
1431 | // commuted. | |||
1432 | static bool isCommutableVPERMV3Instruction(unsigned Opcode) { | |||
1433 | #define VPERM_CASES(Suffix) \ | |||
1434 | case X86::VPERMI2##Suffix##128rr: case X86::VPERMT2##Suffix##128rr: \ | |||
1435 | case X86::VPERMI2##Suffix##256rr: case X86::VPERMT2##Suffix##256rr: \ | |||
1436 | case X86::VPERMI2##Suffix##rr: case X86::VPERMT2##Suffix##rr: \ | |||
1437 | case X86::VPERMI2##Suffix##128rm: case X86::VPERMT2##Suffix##128rm: \ | |||
1438 | case X86::VPERMI2##Suffix##256rm: case X86::VPERMT2##Suffix##256rm: \ | |||
1439 | case X86::VPERMI2##Suffix##rm: case X86::VPERMT2##Suffix##rm: \ | |||
1440 | case X86::VPERMI2##Suffix##128rrkz: case X86::VPERMT2##Suffix##128rrkz: \ | |||
1441 | case X86::VPERMI2##Suffix##256rrkz: case X86::VPERMT2##Suffix##256rrkz: \ | |||
1442 | case X86::VPERMI2##Suffix##rrkz: case X86::VPERMT2##Suffix##rrkz: \ | |||
1443 | case X86::VPERMI2##Suffix##128rmkz: case X86::VPERMT2##Suffix##128rmkz: \ | |||
1444 | case X86::VPERMI2##Suffix##256rmkz: case X86::VPERMT2##Suffix##256rmkz: \ | |||
1445 | case X86::VPERMI2##Suffix##rmkz: case X86::VPERMT2##Suffix##rmkz: | |||
1446 | ||||
1447 | #define VPERM_CASES_BROADCAST(Suffix) \ | |||
1448 | VPERM_CASES(Suffix) \ | |||
1449 | case X86::VPERMI2##Suffix##128rmb: case X86::VPERMT2##Suffix##128rmb: \ | |||
1450 | case X86::VPERMI2##Suffix##256rmb: case X86::VPERMT2##Suffix##256rmb: \ | |||
1451 | case X86::VPERMI2##Suffix##rmb: case X86::VPERMT2##Suffix##rmb: \ | |||
1452 | case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \ | |||
1453 | case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \ | |||
1454 | case X86::VPERMI2##Suffix##rmbkz: case X86::VPERMT2##Suffix##rmbkz: | |||
1455 | ||||
1456 | switch (Opcode) { | |||
1457 | default: return false; | |||
1458 | VPERM_CASES(B) | |||
1459 | VPERM_CASES_BROADCAST(D) | |||
1460 | VPERM_CASES_BROADCAST(PD) | |||
1461 | VPERM_CASES_BROADCAST(PS) | |||
1462 | VPERM_CASES_BROADCAST(Q) | |||
1463 | VPERM_CASES(W) | |||
1464 | return true; | |||
1465 | } | |||
1466 | #undef VPERM_CASES_BROADCAST | |||
1467 | #undef VPERM_CASES | |||
1468 | } | |||
1469 | ||||
1470 | // Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching | |||
1471 | // from the I opcode to the T opcode and vice versa. | |||
1472 | static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) { | |||
1473 | #define VPERM_CASES(Orig, New) \ | |||
1474 | case X86::Orig##128rr: return X86::New##128rr; \ | |||
1475 | case X86::Orig##128rrkz: return X86::New##128rrkz; \ | |||
1476 | case X86::Orig##128rm: return X86::New##128rm; \ | |||
1477 | case X86::Orig##128rmkz: return X86::New##128rmkz; \ | |||
1478 | case X86::Orig##256rr: return X86::New##256rr; \ | |||
1479 | case X86::Orig##256rrkz: return X86::New##256rrkz; \ | |||
1480 | case X86::Orig##256rm: return X86::New##256rm; \ | |||
1481 | case X86::Orig##256rmkz: return X86::New##256rmkz; \ | |||
1482 | case X86::Orig##rr: return X86::New##rr; \ | |||
1483 | case X86::Orig##rrkz: return X86::New##rrkz; \ | |||
1484 | case X86::Orig##rm: return X86::New##rm; \ | |||
1485 | case X86::Orig##rmkz: return X86::New##rmkz; | |||
1486 | ||||
1487 | #define VPERM_CASES_BROADCAST(Orig, New) \ | |||
1488 | VPERM_CASES(Orig, New) \ | |||
1489 | case X86::Orig##128rmb: return X86::New##128rmb; \ | |||
1490 | case X86::Orig##128rmbkz: return X86::New##128rmbkz; \ | |||
1491 | case X86::Orig##256rmb: return X86::New##256rmb; \ | |||
1492 | case X86::Orig##256rmbkz: return X86::New##256rmbkz; \ | |||
1493 | case X86::Orig##rmb: return X86::New##rmb; \ | |||
1494 | case X86::Orig##rmbkz: return X86::New##rmbkz; | |||
1495 | ||||
1496 | switch (Opcode) { | |||
1497 | VPERM_CASES(VPERMI2B, VPERMT2B) | |||
1498 | VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D) | |||
1499 | VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD) | |||
1500 | VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS) | |||
1501 | VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q) | |||
1502 | VPERM_CASES(VPERMI2W, VPERMT2W) | |||
1503 | VPERM_CASES(VPERMT2B, VPERMI2B) | |||
1504 | VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D) | |||
1505 | VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD) | |||
1506 | VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS) | |||
1507 | VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q) | |||
1508 | VPERM_CASES(VPERMT2W, VPERMI2W) | |||
1509 | } | |||
1510 | ||||
1511 | llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1511); | |||
1512 | #undef VPERM_CASES_BROADCAST | |||
1513 | #undef VPERM_CASES | |||
1514 | } | |||
1515 | ||||
1516 | MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, | |||
1517 | unsigned OpIdx1, | |||
1518 | unsigned OpIdx2) const { | |||
1519 | auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { | |||
1520 | if (NewMI) | |||
1521 | return *MI.getParent()->getParent()->CloneMachineInstr(&MI); | |||
1522 | return MI; | |||
1523 | }; | |||
1524 | ||||
1525 | switch (MI.getOpcode()) { | |||
1526 | case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I) | |||
1527 | case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I) | |||
1528 | case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I) | |||
1529 | case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I) | |||
1530 | case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I) | |||
1531 | case X86::SHLD64rri8:{// A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, (64-I) | |||
1532 | unsigned Opc; | |||
1533 | unsigned Size; | |||
1534 | switch (MI.getOpcode()) { | |||
1535 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1535); | |||
1536 | case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8; break; | |||
1537 | case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8; break; | |||
1538 | case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8; break; | |||
1539 | case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8; break; | |||
1540 | case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8; break; | |||
1541 | case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8; break; | |||
1542 | } | |||
1543 | unsigned Amt = MI.getOperand(3).getImm(); | |||
1544 | auto &WorkingMI = cloneIfNew(MI); | |||
1545 | WorkingMI.setDesc(get(Opc)); | |||
1546 | WorkingMI.getOperand(3).setImm(Size - Amt); | |||
1547 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1548 | OpIdx1, OpIdx2); | |||
1549 | } | |||
1550 | case X86::PFSUBrr: | |||
1551 | case X86::PFSUBRrr: { | |||
1552 | // PFSUB x, y: x = x - y | |||
1553 | // PFSUBR x, y: x = y - x | |||
1554 | unsigned Opc = | |||
1555 | (X86::PFSUBRrr == MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr); | |||
1556 | auto &WorkingMI = cloneIfNew(MI); | |||
1557 | WorkingMI.setDesc(get(Opc)); | |||
1558 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1559 | OpIdx1, OpIdx2); | |||
1560 | } | |||
1561 | case X86::BLENDPDrri: | |||
1562 | case X86::BLENDPSrri: | |||
1563 | case X86::VBLENDPDrri: | |||
1564 | case X86::VBLENDPSrri: | |||
1565 | // If we're optimizing for size, try to use MOVSD/MOVSS. | |||
1566 | if (MI.getParent()->getParent()->getFunction().optForSize()) { | |||
1567 | unsigned Mask, Opc; | |||
1568 | switch (MI.getOpcode()) { | |||
1569 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1569); | |||
1570 | case X86::BLENDPDrri: Opc = X86::MOVSDrr; Mask = 0x03; break; | |||
1571 | case X86::BLENDPSrri: Opc = X86::MOVSSrr; Mask = 0x0F; break; | |||
1572 | case X86::VBLENDPDrri: Opc = X86::VMOVSDrr; Mask = 0x03; break; | |||
1573 | case X86::VBLENDPSrri: Opc = X86::VMOVSSrr; Mask = 0x0F; break; | |||
1574 | } | |||
1575 | if ((MI.getOperand(3).getImm() ^ Mask) == 1) { | |||
1576 | auto &WorkingMI = cloneIfNew(MI); | |||
1577 | WorkingMI.setDesc(get(Opc)); | |||
1578 | WorkingMI.RemoveOperand(3); | |||
1579 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, | |||
1580 | /*NewMI=*/false, | |||
1581 | OpIdx1, OpIdx2); | |||
1582 | } | |||
1583 | } | |||
1584 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
1585 | case X86::PBLENDWrri: | |||
1586 | case X86::VBLENDPDYrri: | |||
1587 | case X86::VBLENDPSYrri: | |||
1588 | case X86::VPBLENDDrri: | |||
1589 | case X86::VPBLENDWrri: | |||
1590 | case X86::VPBLENDDYrri: | |||
1591 | case X86::VPBLENDWYrri:{ | |||
1592 | unsigned Mask; | |||
1593 | switch (MI.getOpcode()) { | |||
1594 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1594); | |||
1595 | case X86::BLENDPDrri: Mask = 0x03; break; | |||
1596 | case X86::BLENDPSrri: Mask = 0x0F; break; | |||
1597 | case X86::PBLENDWrri: Mask = 0xFF; break; | |||
1598 | case X86::VBLENDPDrri: Mask = 0x03; break; | |||
1599 | case X86::VBLENDPSrri: Mask = 0x0F; break; | |||
1600 | case X86::VBLENDPDYrri: Mask = 0x0F; break; | |||
1601 | case X86::VBLENDPSYrri: Mask = 0xFF; break; | |||
1602 | case X86::VPBLENDDrri: Mask = 0x0F; break; | |||
1603 | case X86::VPBLENDWrri: Mask = 0xFF; break; | |||
1604 | case X86::VPBLENDDYrri: Mask = 0xFF; break; | |||
1605 | case X86::VPBLENDWYrri: Mask = 0xFF; break; | |||
1606 | } | |||
1607 | // Only the least significant bits of Imm are used. | |||
1608 | unsigned Imm = MI.getOperand(3).getImm() & Mask; | |||
1609 | auto &WorkingMI = cloneIfNew(MI); | |||
1610 | WorkingMI.getOperand(3).setImm(Mask ^ Imm); | |||
1611 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1612 | OpIdx1, OpIdx2); | |||
1613 | } | |||
1614 | case X86::MOVSDrr: | |||
1615 | case X86::MOVSSrr: | |||
1616 | case X86::VMOVSDrr: | |||
1617 | case X86::VMOVSSrr:{ | |||
1618 | // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD. | |||
1619 | assert(Subtarget.hasSSE41() && "Commuting MOVSD/MOVSS requires SSE41!")((Subtarget.hasSSE41() && "Commuting MOVSD/MOVSS requires SSE41!" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE41() && \"Commuting MOVSD/MOVSS requires SSE41!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1619, __PRETTY_FUNCTION__)); | |||
1620 | ||||
1621 | unsigned Mask, Opc; | |||
1622 | switch (MI.getOpcode()) { | |||
1623 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1623); | |||
1624 | case X86::MOVSDrr: Opc = X86::BLENDPDrri; Mask = 0x02; break; | |||
1625 | case X86::MOVSSrr: Opc = X86::BLENDPSrri; Mask = 0x0E; break; | |||
1626 | case X86::VMOVSDrr: Opc = X86::VBLENDPDrri; Mask = 0x02; break; | |||
1627 | case X86::VMOVSSrr: Opc = X86::VBLENDPSrri; Mask = 0x0E; break; | |||
1628 | } | |||
1629 | ||||
1630 | auto &WorkingMI = cloneIfNew(MI); | |||
1631 | WorkingMI.setDesc(get(Opc)); | |||
1632 | WorkingMI.addOperand(MachineOperand::CreateImm(Mask)); | |||
1633 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1634 | OpIdx1, OpIdx2); | |||
1635 | } | |||
1636 | case X86::PCLMULQDQrr: | |||
1637 | case X86::VPCLMULQDQrr: | |||
1638 | case X86::VPCLMULQDQYrr: | |||
1639 | case X86::VPCLMULQDQZrr: | |||
1640 | case X86::VPCLMULQDQZ128rr: | |||
1641 | case X86::VPCLMULQDQZ256rr: { | |||
1642 | // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0] | |||
1643 | // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0] | |||
1644 | unsigned Imm = MI.getOperand(3).getImm(); | |||
1645 | unsigned Src1Hi = Imm & 0x01; | |||
1646 | unsigned Src2Hi = Imm & 0x10; | |||
1647 | auto &WorkingMI = cloneIfNew(MI); | |||
1648 | WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4)); | |||
1649 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1650 | OpIdx1, OpIdx2); | |||
1651 | } | |||
1652 | case X86::VPCMPBZ128rri: case X86::VPCMPUBZ128rri: | |||
1653 | case X86::VPCMPBZ256rri: case X86::VPCMPUBZ256rri: | |||
1654 | case X86::VPCMPBZrri: case X86::VPCMPUBZrri: | |||
1655 | case X86::VPCMPDZ128rri: case X86::VPCMPUDZ128rri: | |||
1656 | case X86::VPCMPDZ256rri: case X86::VPCMPUDZ256rri: | |||
1657 | case X86::VPCMPDZrri: case X86::VPCMPUDZrri: | |||
1658 | case X86::VPCMPQZ128rri: case X86::VPCMPUQZ128rri: | |||
1659 | case X86::VPCMPQZ256rri: case X86::VPCMPUQZ256rri: | |||
1660 | case X86::VPCMPQZrri: case X86::VPCMPUQZrri: | |||
1661 | case X86::VPCMPWZ128rri: case X86::VPCMPUWZ128rri: | |||
1662 | case X86::VPCMPWZ256rri: case X86::VPCMPUWZ256rri: | |||
1663 | case X86::VPCMPWZrri: case X86::VPCMPUWZrri: | |||
1664 | case X86::VPCMPBZ128rrik: case X86::VPCMPUBZ128rrik: | |||
1665 | case X86::VPCMPBZ256rrik: case X86::VPCMPUBZ256rrik: | |||
1666 | case X86::VPCMPBZrrik: case X86::VPCMPUBZrrik: | |||
1667 | case X86::VPCMPDZ128rrik: case X86::VPCMPUDZ128rrik: | |||
1668 | case X86::VPCMPDZ256rrik: case X86::VPCMPUDZ256rrik: | |||
1669 | case X86::VPCMPDZrrik: case X86::VPCMPUDZrrik: | |||
1670 | case X86::VPCMPQZ128rrik: case X86::VPCMPUQZ128rrik: | |||
1671 | case X86::VPCMPQZ256rrik: case X86::VPCMPUQZ256rrik: | |||
1672 | case X86::VPCMPQZrrik: case X86::VPCMPUQZrrik: | |||
1673 | case X86::VPCMPWZ128rrik: case X86::VPCMPUWZ128rrik: | |||
1674 | case X86::VPCMPWZ256rrik: case X86::VPCMPUWZ256rrik: | |||
1675 | case X86::VPCMPWZrrik: case X86::VPCMPUWZrrik: { | |||
1676 | // Flip comparison mode immediate (if necessary). | |||
1677 | unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7; | |||
1678 | Imm = X86::getSwappedVPCMPImm(Imm); | |||
1679 | auto &WorkingMI = cloneIfNew(MI); | |||
1680 | WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm); | |||
1681 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1682 | OpIdx1, OpIdx2); | |||
1683 | } | |||
1684 | case X86::VPCOMBri: case X86::VPCOMUBri: | |||
1685 | case X86::VPCOMDri: case X86::VPCOMUDri: | |||
1686 | case X86::VPCOMQri: case X86::VPCOMUQri: | |||
1687 | case X86::VPCOMWri: case X86::VPCOMUWri: { | |||
1688 | // Flip comparison mode immediate (if necessary). | |||
1689 | unsigned Imm = MI.getOperand(3).getImm() & 0x7; | |||
1690 | Imm = X86::getSwappedVPCOMImm(Imm); | |||
1691 | auto &WorkingMI = cloneIfNew(MI); | |||
1692 | WorkingMI.getOperand(3).setImm(Imm); | |||
1693 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1694 | OpIdx1, OpIdx2); | |||
1695 | } | |||
1696 | case X86::VPERM2F128rr: | |||
1697 | case X86::VPERM2I128rr: { | |||
1698 | // Flip permute source immediate. | |||
1699 | // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi. | |||
1700 | // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi. | |||
1701 | unsigned Imm = MI.getOperand(3).getImm() & 0xFF; | |||
1702 | auto &WorkingMI = cloneIfNew(MI); | |||
1703 | WorkingMI.getOperand(3).setImm(Imm ^ 0x22); | |||
1704 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1705 | OpIdx1, OpIdx2); | |||
1706 | } | |||
1707 | case X86::MOVHLPSrr: | |||
1708 | case X86::UNPCKHPDrr: | |||
1709 | case X86::VMOVHLPSrr: | |||
1710 | case X86::VUNPCKHPDrr: | |||
1711 | case X86::VMOVHLPSZrr: | |||
1712 | case X86::VUNPCKHPDZ128rr: { | |||
1713 | assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!")((Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Commuting MOVHLP/UNPCKHPD requires SSE2!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1713, __PRETTY_FUNCTION__)); | |||
1714 | ||||
1715 | unsigned Opc = MI.getOpcode(); | |||
1716 | switch (Opc) { | |||
1717 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1717); | |||
1718 | case X86::MOVHLPSrr: Opc = X86::UNPCKHPDrr; break; | |||
1719 | case X86::UNPCKHPDrr: Opc = X86::MOVHLPSrr; break; | |||
1720 | case X86::VMOVHLPSrr: Opc = X86::VUNPCKHPDrr; break; | |||
1721 | case X86::VUNPCKHPDrr: Opc = X86::VMOVHLPSrr; break; | |||
1722 | case X86::VMOVHLPSZrr: Opc = X86::VUNPCKHPDZ128rr; break; | |||
1723 | case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr; break; | |||
1724 | } | |||
1725 | auto &WorkingMI = cloneIfNew(MI); | |||
1726 | WorkingMI.setDesc(get(Opc)); | |||
1727 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1728 | OpIdx1, OpIdx2); | |||
1729 | } | |||
1730 | case X86::CMOVB16rr: case X86::CMOVB32rr: case X86::CMOVB64rr: | |||
1731 | case X86::CMOVAE16rr: case X86::CMOVAE32rr: case X86::CMOVAE64rr: | |||
1732 | case X86::CMOVE16rr: case X86::CMOVE32rr: case X86::CMOVE64rr: | |||
1733 | case X86::CMOVNE16rr: case X86::CMOVNE32rr: case X86::CMOVNE64rr: | |||
1734 | case X86::CMOVBE16rr: case X86::CMOVBE32rr: case X86::CMOVBE64rr: | |||
1735 | case X86::CMOVA16rr: case X86::CMOVA32rr: case X86::CMOVA64rr: | |||
1736 | case X86::CMOVL16rr: case X86::CMOVL32rr: case X86::CMOVL64rr: | |||
1737 | case X86::CMOVGE16rr: case X86::CMOVGE32rr: case X86::CMOVGE64rr: | |||
1738 | case X86::CMOVLE16rr: case X86::CMOVLE32rr: case X86::CMOVLE64rr: | |||
1739 | case X86::CMOVG16rr: case X86::CMOVG32rr: case X86::CMOVG64rr: | |||
1740 | case X86::CMOVS16rr: case X86::CMOVS32rr: case X86::CMOVS64rr: | |||
1741 | case X86::CMOVNS16rr: case X86::CMOVNS32rr: case X86::CMOVNS64rr: | |||
1742 | case X86::CMOVP16rr: case X86::CMOVP32rr: case X86::CMOVP64rr: | |||
1743 | case X86::CMOVNP16rr: case X86::CMOVNP32rr: case X86::CMOVNP64rr: | |||
1744 | case X86::CMOVO16rr: case X86::CMOVO32rr: case X86::CMOVO64rr: | |||
1745 | case X86::CMOVNO16rr: case X86::CMOVNO32rr: case X86::CMOVNO64rr: { | |||
1746 | unsigned Opc; | |||
1747 | switch (MI.getOpcode()) { | |||
1748 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 1748); | |||
1749 | case X86::CMOVB16rr: Opc = X86::CMOVAE16rr; break; | |||
1750 | case X86::CMOVB32rr: Opc = X86::CMOVAE32rr; break; | |||
1751 | case X86::CMOVB64rr: Opc = X86::CMOVAE64rr; break; | |||
1752 | case X86::CMOVAE16rr: Opc = X86::CMOVB16rr; break; | |||
1753 | case X86::CMOVAE32rr: Opc = X86::CMOVB32rr; break; | |||
1754 | case X86::CMOVAE64rr: Opc = X86::CMOVB64rr; break; | |||
1755 | case X86::CMOVE16rr: Opc = X86::CMOVNE16rr; break; | |||
1756 | case X86::CMOVE32rr: Opc = X86::CMOVNE32rr; break; | |||
1757 | case X86::CMOVE64rr: Opc = X86::CMOVNE64rr; break; | |||
1758 | case X86::CMOVNE16rr: Opc = X86::CMOVE16rr; break; | |||
1759 | case X86::CMOVNE32rr: Opc = X86::CMOVE32rr; break; | |||
1760 | case X86::CMOVNE64rr: Opc = X86::CMOVE64rr; break; | |||
1761 | case X86::CMOVBE16rr: Opc = X86::CMOVA16rr; break; | |||
1762 | case X86::CMOVBE32rr: Opc = X86::CMOVA32rr; break; | |||
1763 | case X86::CMOVBE64rr: Opc = X86::CMOVA64rr; break; | |||
1764 | case X86::CMOVA16rr: Opc = X86::CMOVBE16rr; break; | |||
1765 | case X86::CMOVA32rr: Opc = X86::CMOVBE32rr; break; | |||
1766 | case X86::CMOVA64rr: Opc = X86::CMOVBE64rr; break; | |||
1767 | case X86::CMOVL16rr: Opc = X86::CMOVGE16rr; break; | |||
1768 | case X86::CMOVL32rr: Opc = X86::CMOVGE32rr; break; | |||
1769 | case X86::CMOVL64rr: Opc = X86::CMOVGE64rr; break; | |||
1770 | case X86::CMOVGE16rr: Opc = X86::CMOVL16rr; break; | |||
1771 | case X86::CMOVGE32rr: Opc = X86::CMOVL32rr; break; | |||
1772 | case X86::CMOVGE64rr: Opc = X86::CMOVL64rr; break; | |||
1773 | case X86::CMOVLE16rr: Opc = X86::CMOVG16rr; break; | |||
1774 | case X86::CMOVLE32rr: Opc = X86::CMOVG32rr; break; | |||
1775 | case X86::CMOVLE64rr: Opc = X86::CMOVG64rr; break; | |||
1776 | case X86::CMOVG16rr: Opc = X86::CMOVLE16rr; break; | |||
1777 | case X86::CMOVG32rr: Opc = X86::CMOVLE32rr; break; | |||
1778 | case X86::CMOVG64rr: Opc = X86::CMOVLE64rr; break; | |||
1779 | case X86::CMOVS16rr: Opc = X86::CMOVNS16rr; break; | |||
1780 | case X86::CMOVS32rr: Opc = X86::CMOVNS32rr; break; | |||
1781 | case X86::CMOVS64rr: Opc = X86::CMOVNS64rr; break; | |||
1782 | case X86::CMOVNS16rr: Opc = X86::CMOVS16rr; break; | |||
1783 | case X86::CMOVNS32rr: Opc = X86::CMOVS32rr; break; | |||
1784 | case X86::CMOVNS64rr: Opc = X86::CMOVS64rr; break; | |||
1785 | case X86::CMOVP16rr: Opc = X86::CMOVNP16rr; break; | |||
1786 | case X86::CMOVP32rr: Opc = X86::CMOVNP32rr; break; | |||
1787 | case X86::CMOVP64rr: Opc = X86::CMOVNP64rr; break; | |||
1788 | case X86::CMOVNP16rr: Opc = X86::CMOVP16rr; break; | |||
1789 | case X86::CMOVNP32rr: Opc = X86::CMOVP32rr; break; | |||
1790 | case X86::CMOVNP64rr: Opc = X86::CMOVP64rr; break; | |||
1791 | case X86::CMOVO16rr: Opc = X86::CMOVNO16rr; break; | |||
1792 | case X86::CMOVO32rr: Opc = X86::CMOVNO32rr; break; | |||
1793 | case X86::CMOVO64rr: Opc = X86::CMOVNO64rr; break; | |||
1794 | case X86::CMOVNO16rr: Opc = X86::CMOVO16rr; break; | |||
1795 | case X86::CMOVNO32rr: Opc = X86::CMOVO32rr; break; | |||
1796 | case X86::CMOVNO64rr: Opc = X86::CMOVO64rr; break; | |||
1797 | } | |||
1798 | auto &WorkingMI = cloneIfNew(MI); | |||
1799 | WorkingMI.setDesc(get(Opc)); | |||
1800 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1801 | OpIdx1, OpIdx2); | |||
1802 | } | |||
1803 | case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: | |||
1804 | case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: | |||
1805 | case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: | |||
1806 | case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: | |||
1807 | case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: | |||
1808 | case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: | |||
1809 | case X86::VPTERNLOGDZrrik: | |||
1810 | case X86::VPTERNLOGDZ128rrik: | |||
1811 | case X86::VPTERNLOGDZ256rrik: | |||
1812 | case X86::VPTERNLOGQZrrik: | |||
1813 | case X86::VPTERNLOGQZ128rrik: | |||
1814 | case X86::VPTERNLOGQZ256rrik: | |||
1815 | case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: | |||
1816 | case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: | |||
1817 | case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: | |||
1818 | case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: | |||
1819 | case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: | |||
1820 | case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: | |||
1821 | case X86::VPTERNLOGDZ128rmbi: | |||
1822 | case X86::VPTERNLOGDZ256rmbi: | |||
1823 | case X86::VPTERNLOGDZrmbi: | |||
1824 | case X86::VPTERNLOGQZ128rmbi: | |||
1825 | case X86::VPTERNLOGQZ256rmbi: | |||
1826 | case X86::VPTERNLOGQZrmbi: | |||
1827 | case X86::VPTERNLOGDZ128rmbikz: | |||
1828 | case X86::VPTERNLOGDZ256rmbikz: | |||
1829 | case X86::VPTERNLOGDZrmbikz: | |||
1830 | case X86::VPTERNLOGQZ128rmbikz: | |||
1831 | case X86::VPTERNLOGQZ256rmbikz: | |||
1832 | case X86::VPTERNLOGQZrmbikz: { | |||
1833 | auto &WorkingMI = cloneIfNew(MI); | |||
1834 | commuteVPTERNLOG(WorkingMI, OpIdx1, OpIdx2); | |||
1835 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1836 | OpIdx1, OpIdx2); | |||
1837 | } | |||
1838 | default: { | |||
1839 | if (isCommutableVPERMV3Instruction(MI.getOpcode())) { | |||
1840 | unsigned Opc = getCommutedVPERMV3Opcode(MI.getOpcode()); | |||
1841 | auto &WorkingMI = cloneIfNew(MI); | |||
1842 | WorkingMI.setDesc(get(Opc)); | |||
1843 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1844 | OpIdx1, OpIdx2); | |||
1845 | } | |||
1846 | ||||
1847 | const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), | |||
1848 | MI.getDesc().TSFlags); | |||
1849 | if (FMA3Group) { | |||
1850 | unsigned Opc = | |||
1851 | getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group); | |||
1852 | auto &WorkingMI = cloneIfNew(MI); | |||
1853 | WorkingMI.setDesc(get(Opc)); | |||
1854 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1855 | OpIdx1, OpIdx2); | |||
1856 | } | |||
1857 | ||||
1858 | return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); | |||
1859 | } | |||
1860 | } | |||
1861 | } | |||
1862 | ||||
1863 | bool | |||
1864 | X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI, | |||
1865 | unsigned &SrcOpIdx1, | |||
1866 | unsigned &SrcOpIdx2, | |||
1867 | bool IsIntrinsic) const { | |||
1868 | uint64_t TSFlags = MI.getDesc().TSFlags; | |||
1869 | ||||
1870 | unsigned FirstCommutableVecOp = 1; | |||
1871 | unsigned LastCommutableVecOp = 3; | |||
1872 | unsigned KMaskOp = -1U; | |||
1873 | if (X86II::isKMasked(TSFlags)) { | |||
1874 | // For k-zero-masked operations it is Ok to commute the first vector | |||
1875 | // operand. | |||
1876 | // For regular k-masked operations a conservative choice is done as the | |||
1877 | // elements of the first vector operand, for which the corresponding bit | |||
1878 | // in the k-mask operand is set to 0, are copied to the result of the | |||
1879 | // instruction. | |||
1880 | // TODO/FIXME: The commute still may be legal if it is known that the | |||
1881 | // k-mask operand is set to either all ones or all zeroes. | |||
1882 | // It is also Ok to commute the 1st operand if all users of MI use only | |||
1883 | // the elements enabled by the k-mask operand. For example, | |||
1884 | // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i] | |||
1885 | // : v1[i]; | |||
1886 | // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 -> | |||
1887 | // // Ok, to commute v1 in FMADD213PSZrk. | |||
1888 | ||||
1889 | // The k-mask operand has index = 2 for masked and zero-masked operations. | |||
1890 | KMaskOp = 2; | |||
1891 | ||||
1892 | // The operand with index = 1 is used as a source for those elements for | |||
1893 | // which the corresponding bit in the k-mask is set to 0. | |||
1894 | if (X86II::isKMergeMasked(TSFlags)) | |||
1895 | FirstCommutableVecOp = 3; | |||
1896 | ||||
1897 | LastCommutableVecOp++; | |||
1898 | } else if (IsIntrinsic) { | |||
1899 | // Commuting the first operand of an intrinsic instruction isn't possible | |||
1900 | // unless we can prove that only the lowest element of the result is used. | |||
1901 | FirstCommutableVecOp = 2; | |||
1902 | } | |||
1903 | ||||
1904 | if (isMem(MI, LastCommutableVecOp)) | |||
1905 | LastCommutableVecOp--; | |||
1906 | ||||
1907 | // Only the first RegOpsNum operands are commutable. | |||
1908 | // Also, the value 'CommuteAnyOperandIndex' is valid here as it means | |||
1909 | // that the operand is not specified/fixed. | |||
1910 | if (SrcOpIdx1 != CommuteAnyOperandIndex && | |||
1911 | (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp || | |||
1912 | SrcOpIdx1 == KMaskOp)) | |||
1913 | return false; | |||
1914 | if (SrcOpIdx2 != CommuteAnyOperandIndex && | |||
1915 | (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp || | |||
1916 | SrcOpIdx2 == KMaskOp)) | |||
1917 | return false; | |||
1918 | ||||
1919 | // Look for two different register operands assumed to be commutable | |||
1920 | // regardless of the FMA opcode. The FMA opcode is adjusted later. | |||
1921 | if (SrcOpIdx1 == CommuteAnyOperandIndex || | |||
1922 | SrcOpIdx2 == CommuteAnyOperandIndex) { | |||
1923 | unsigned CommutableOpIdx1 = SrcOpIdx1; | |||
1924 | unsigned CommutableOpIdx2 = SrcOpIdx2; | |||
1925 | ||||
1926 | // At least one of operands to be commuted is not specified and | |||
1927 | // this method is free to choose appropriate commutable operands. | |||
1928 | if (SrcOpIdx1 == SrcOpIdx2) | |||
1929 | // Both of operands are not fixed. By default set one of commutable | |||
1930 | // operands to the last register operand of the instruction. | |||
1931 | CommutableOpIdx2 = LastCommutableVecOp; | |||
1932 | else if (SrcOpIdx2 == CommuteAnyOperandIndex) | |||
1933 | // Only one of operands is not fixed. | |||
1934 | CommutableOpIdx2 = SrcOpIdx1; | |||
1935 | ||||
1936 | // CommutableOpIdx2 is well defined now. Let's choose another commutable | |||
1937 | // operand and assign its index to CommutableOpIdx1. | |||
1938 | unsigned Op2Reg = MI.getOperand(CommutableOpIdx2).getReg(); | |||
1939 | for (CommutableOpIdx1 = LastCommutableVecOp; | |||
1940 | CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) { | |||
1941 | // Just ignore and skip the k-mask operand. | |||
1942 | if (CommutableOpIdx1 == KMaskOp) | |||
1943 | continue; | |||
1944 | ||||
1945 | // The commuted operands must have different registers. | |||
1946 | // Otherwise, the commute transformation does not change anything and | |||
1947 | // is useless then. | |||
1948 | if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg()) | |||
1949 | break; | |||
1950 | } | |||
1951 | ||||
1952 | // No appropriate commutable operands were found. | |||
1953 | if (CommutableOpIdx1 < FirstCommutableVecOp) | |||
1954 | return false; | |||
1955 | ||||
1956 | // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2 | |||
1957 | // to return those values. | |||
1958 | if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, | |||
1959 | CommutableOpIdx1, CommutableOpIdx2)) | |||
1960 | return false; | |||
1961 | } | |||
1962 | ||||
1963 | return true; | |||
1964 | } | |||
1965 | ||||
1966 | bool X86InstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1, | |||
1967 | unsigned &SrcOpIdx2) const { | |||
1968 | const MCInstrDesc &Desc = MI.getDesc(); | |||
1969 | if (!Desc.isCommutable()) | |||
1970 | return false; | |||
1971 | ||||
1972 | switch (MI.getOpcode()) { | |||
1973 | case X86::CMPSDrr: | |||
1974 | case X86::CMPSSrr: | |||
1975 | case X86::CMPPDrri: | |||
1976 | case X86::CMPPSrri: | |||
1977 | case X86::VCMPSDrr: | |||
1978 | case X86::VCMPSSrr: | |||
1979 | case X86::VCMPPDrri: | |||
1980 | case X86::VCMPPSrri: | |||
1981 | case X86::VCMPPDYrri: | |||
1982 | case X86::VCMPPSYrri: | |||
1983 | case X86::VCMPSDZrr: | |||
1984 | case X86::VCMPSSZrr: | |||
1985 | case X86::VCMPPDZrri: | |||
1986 | case X86::VCMPPSZrri: | |||
1987 | case X86::VCMPPDZ128rri: | |||
1988 | case X86::VCMPPSZ128rri: | |||
1989 | case X86::VCMPPDZ256rri: | |||
1990 | case X86::VCMPPSZ256rri: { | |||
1991 | // Float comparison can be safely commuted for | |||
1992 | // Ordered/Unordered/Equal/NotEqual tests | |||
1993 | unsigned Imm = MI.getOperand(3).getImm() & 0x7; | |||
1994 | switch (Imm) { | |||
1995 | case 0x00: // EQUAL | |||
1996 | case 0x03: // UNORDERED | |||
1997 | case 0x04: // NOT EQUAL | |||
1998 | case 0x07: // ORDERED | |||
1999 | // The indices of the commutable operands are 1 and 2. | |||
2000 | // Assign them to the returned operand indices here. | |||
2001 | return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2); | |||
2002 | } | |||
2003 | return false; | |||
2004 | } | |||
2005 | case X86::MOVSDrr: | |||
2006 | case X86::MOVSSrr: | |||
2007 | case X86::VMOVSDrr: | |||
2008 | case X86::VMOVSSrr: | |||
2009 | if (Subtarget.hasSSE41()) | |||
2010 | return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
2011 | return false; | |||
2012 | case X86::MOVHLPSrr: | |||
2013 | case X86::UNPCKHPDrr: | |||
2014 | case X86::VMOVHLPSrr: | |||
2015 | case X86::VUNPCKHPDrr: | |||
2016 | case X86::VMOVHLPSZrr: | |||
2017 | case X86::VUNPCKHPDZ128rr: | |||
2018 | if (Subtarget.hasSSE2()) | |||
2019 | return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
2020 | return false; | |||
2021 | case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: | |||
2022 | case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: | |||
2023 | case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: | |||
2024 | case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: | |||
2025 | case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: | |||
2026 | case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: | |||
2027 | case X86::VPTERNLOGDZrrik: | |||
2028 | case X86::VPTERNLOGDZ128rrik: | |||
2029 | case X86::VPTERNLOGDZ256rrik: | |||
2030 | case X86::VPTERNLOGQZrrik: | |||
2031 | case X86::VPTERNLOGQZ128rrik: | |||
2032 | case X86::VPTERNLOGQZ256rrik: | |||
2033 | case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: | |||
2034 | case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: | |||
2035 | case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: | |||
2036 | case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: | |||
2037 | case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: | |||
2038 | case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: | |||
2039 | case X86::VPTERNLOGDZ128rmbi: | |||
2040 | case X86::VPTERNLOGDZ256rmbi: | |||
2041 | case X86::VPTERNLOGDZrmbi: | |||
2042 | case X86::VPTERNLOGQZ128rmbi: | |||
2043 | case X86::VPTERNLOGQZ256rmbi: | |||
2044 | case X86::VPTERNLOGQZrmbi: | |||
2045 | case X86::VPTERNLOGDZ128rmbikz: | |||
2046 | case X86::VPTERNLOGDZ256rmbikz: | |||
2047 | case X86::VPTERNLOGDZrmbikz: | |||
2048 | case X86::VPTERNLOGQZ128rmbikz: | |||
2049 | case X86::VPTERNLOGQZ256rmbikz: | |||
2050 | case X86::VPTERNLOGQZrmbikz: | |||
2051 | return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
2052 | case X86::VPMADD52HUQZ128r: | |||
2053 | case X86::VPMADD52HUQZ128rk: | |||
2054 | case X86::VPMADD52HUQZ128rkz: | |||
2055 | case X86::VPMADD52HUQZ256r: | |||
2056 | case X86::VPMADD52HUQZ256rk: | |||
2057 | case X86::VPMADD52HUQZ256rkz: | |||
2058 | case X86::VPMADD52HUQZr: | |||
2059 | case X86::VPMADD52HUQZrk: | |||
2060 | case X86::VPMADD52HUQZrkz: | |||
2061 | case X86::VPMADD52LUQZ128r: | |||
2062 | case X86::VPMADD52LUQZ128rk: | |||
2063 | case X86::VPMADD52LUQZ128rkz: | |||
2064 | case X86::VPMADD52LUQZ256r: | |||
2065 | case X86::VPMADD52LUQZ256rk: | |||
2066 | case X86::VPMADD52LUQZ256rkz: | |||
2067 | case X86::VPMADD52LUQZr: | |||
2068 | case X86::VPMADD52LUQZrk: | |||
2069 | case X86::VPMADD52LUQZrkz: { | |||
2070 | unsigned CommutableOpIdx1 = 2; | |||
2071 | unsigned CommutableOpIdx2 = 3; | |||
2072 | if (X86II::isKMasked(Desc.TSFlags)) { | |||
2073 | // Skip the mask register. | |||
2074 | ++CommutableOpIdx1; | |||
2075 | ++CommutableOpIdx2; | |||
2076 | } | |||
2077 | if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, | |||
2078 | CommutableOpIdx1, CommutableOpIdx2)) | |||
2079 | return false; | |||
2080 | if (!MI.getOperand(SrcOpIdx1).isReg() || | |||
2081 | !MI.getOperand(SrcOpIdx2).isReg()) | |||
2082 | // No idea. | |||
2083 | return false; | |||
2084 | return true; | |||
2085 | } | |||
2086 | ||||
2087 | default: | |||
2088 | const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), | |||
2089 | MI.getDesc().TSFlags); | |||
2090 | if (FMA3Group) | |||
2091 | return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2, | |||
2092 | FMA3Group->isIntrinsic()); | |||
2093 | ||||
2094 | // Handled masked instructions since we need to skip over the mask input | |||
2095 | // and the preserved input. | |||
2096 | if (X86II::isKMasked(Desc.TSFlags)) { | |||
2097 | // First assume that the first input is the mask operand and skip past it. | |||
2098 | unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1; | |||
2099 | unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2; | |||
2100 | // Check if the first input is tied. If there isn't one then we only | |||
2101 | // need to skip the mask operand which we did above. | |||
2102 | if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(), | |||
2103 | MCOI::TIED_TO) != -1)) { | |||
2104 | // If this is zero masking instruction with a tied operand, we need to | |||
2105 | // move the first index back to the first input since this must | |||
2106 | // be a 3 input instruction and we want the first two non-mask inputs. | |||
2107 | // Otherwise this is a 2 input instruction with a preserved input and | |||
2108 | // mask, so we need to move the indices to skip one more input. | |||
2109 | if (X86II::isKMergeMasked(Desc.TSFlags)) { | |||
2110 | ++CommutableOpIdx1; | |||
2111 | ++CommutableOpIdx2; | |||
2112 | } else { | |||
2113 | --CommutableOpIdx1; | |||
2114 | } | |||
2115 | } | |||
2116 | ||||
2117 | if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, | |||
2118 | CommutableOpIdx1, CommutableOpIdx2)) | |||
2119 | return false; | |||
2120 | ||||
2121 | if (!MI.getOperand(SrcOpIdx1).isReg() || | |||
2122 | !MI.getOperand(SrcOpIdx2).isReg()) | |||
2123 | // No idea. | |||
2124 | return false; | |||
2125 | return true; | |||
2126 | } | |||
2127 | ||||
2128 | return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
2129 | } | |||
2130 | return false; | |||
2131 | } | |||
2132 | ||||
2133 | X86::CondCode X86::getCondFromBranchOpc(unsigned BrOpc) { | |||
2134 | switch (BrOpc) { | |||
2135 | default: return X86::COND_INVALID; | |||
2136 | case X86::JE_1: return X86::COND_E; | |||
2137 | case X86::JNE_1: return X86::COND_NE; | |||
2138 | case X86::JL_1: return X86::COND_L; | |||
2139 | case X86::JLE_1: return X86::COND_LE; | |||
2140 | case X86::JG_1: return X86::COND_G; | |||
2141 | case X86::JGE_1: return X86::COND_GE; | |||
2142 | case X86::JB_1: return X86::COND_B; | |||
2143 | case X86::JBE_1: return X86::COND_BE; | |||
2144 | case X86::JA_1: return X86::COND_A; | |||
2145 | case X86::JAE_1: return X86::COND_AE; | |||
2146 | case X86::JS_1: return X86::COND_S; | |||
2147 | case X86::JNS_1: return X86::COND_NS; | |||
2148 | case X86::JP_1: return X86::COND_P; | |||
2149 | case X86::JNP_1: return X86::COND_NP; | |||
2150 | case X86::JO_1: return X86::COND_O; | |||
2151 | case X86::JNO_1: return X86::COND_NO; | |||
2152 | } | |||
2153 | } | |||
2154 | ||||
2155 | /// Return condition code of a SET opcode. | |||
2156 | X86::CondCode X86::getCondFromSETOpc(unsigned Opc) { | |||
2157 | switch (Opc) { | |||
2158 | default: return X86::COND_INVALID; | |||
2159 | case X86::SETAr: case X86::SETAm: return X86::COND_A; | |||
2160 | case X86::SETAEr: case X86::SETAEm: return X86::COND_AE; | |||
2161 | case X86::SETBr: case X86::SETBm: return X86::COND_B; | |||
2162 | case X86::SETBEr: case X86::SETBEm: return X86::COND_BE; | |||
2163 | case X86::SETEr: case X86::SETEm: return X86::COND_E; | |||
2164 | case X86::SETGr: case X86::SETGm: return X86::COND_G; | |||
2165 | case X86::SETGEr: case X86::SETGEm: return X86::COND_GE; | |||
2166 | case X86::SETLr: case X86::SETLm: return X86::COND_L; | |||
2167 | case X86::SETLEr: case X86::SETLEm: return X86::COND_LE; | |||
2168 | case X86::SETNEr: case X86::SETNEm: return X86::COND_NE; | |||
2169 | case X86::SETNOr: case X86::SETNOm: return X86::COND_NO; | |||
2170 | case X86::SETNPr: case X86::SETNPm: return X86::COND_NP; | |||
2171 | case X86::SETNSr: case X86::SETNSm: return X86::COND_NS; | |||
2172 | case X86::SETOr: case X86::SETOm: return X86::COND_O; | |||
2173 | case X86::SETPr: case X86::SETPm: return X86::COND_P; | |||
2174 | case X86::SETSr: case X86::SETSm: return X86::COND_S; | |||
2175 | } | |||
2176 | } | |||
2177 | ||||
2178 | /// Return condition code of a CMov opcode. | |||
2179 | X86::CondCode X86::getCondFromCMovOpc(unsigned Opc) { | |||
2180 | switch (Opc) { | |||
2181 | default: return X86::COND_INVALID; | |||
2182 | case X86::CMOVA16rm: case X86::CMOVA16rr: case X86::CMOVA32rm: | |||
2183 | case X86::CMOVA32rr: case X86::CMOVA64rm: case X86::CMOVA64rr: | |||
2184 | return X86::COND_A; | |||
2185 | case X86::CMOVAE16rm: case X86::CMOVAE16rr: case X86::CMOVAE32rm: | |||
2186 | case X86::CMOVAE32rr: case X86::CMOVAE64rm: case X86::CMOVAE64rr: | |||
2187 | return X86::COND_AE; | |||
2188 | case X86::CMOVB16rm: case X86::CMOVB16rr: case X86::CMOVB32rm: | |||
2189 | case X86::CMOVB32rr: case X86::CMOVB64rm: case X86::CMOVB64rr: | |||
2190 | return X86::COND_B; | |||
2191 | case X86::CMOVBE16rm: case X86::CMOVBE16rr: case X86::CMOVBE32rm: | |||
2192 | case X86::CMOVBE32rr: case X86::CMOVBE64rm: case X86::CMOVBE64rr: | |||
2193 | return X86::COND_BE; | |||
2194 | case X86::CMOVE16rm: case X86::CMOVE16rr: case X86::CMOVE32rm: | |||
2195 | case X86::CMOVE32rr: case X86::CMOVE64rm: case X86::CMOVE64rr: | |||
2196 | return X86::COND_E; | |||
2197 | case X86::CMOVG16rm: case X86::CMOVG16rr: case X86::CMOVG32rm: | |||
2198 | case X86::CMOVG32rr: case X86::CMOVG64rm: case X86::CMOVG64rr: | |||
2199 | return X86::COND_G; | |||
2200 | case X86::CMOVGE16rm: case X86::CMOVGE16rr: case X86::CMOVGE32rm: | |||
2201 | case X86::CMOVGE32rr: case X86::CMOVGE64rm: case X86::CMOVGE64rr: | |||
2202 | return X86::COND_GE; | |||
2203 | case X86::CMOVL16rm: case X86::CMOVL16rr: case X86::CMOVL32rm: | |||
2204 | case X86::CMOVL32rr: case X86::CMOVL64rm: case X86::CMOVL64rr: | |||
2205 | return X86::COND_L; | |||
2206 | case X86::CMOVLE16rm: case X86::CMOVLE16rr: case X86::CMOVLE32rm: | |||
2207 | case X86::CMOVLE32rr: case X86::CMOVLE64rm: case X86::CMOVLE64rr: | |||
2208 | return X86::COND_LE; | |||
2209 | case X86::CMOVNE16rm: case X86::CMOVNE16rr: case X86::CMOVNE32rm: | |||
2210 | case X86::CMOVNE32rr: case X86::CMOVNE64rm: case X86::CMOVNE64rr: | |||
2211 | return X86::COND_NE; | |||
2212 | case X86::CMOVNO16rm: case X86::CMOVNO16rr: case X86::CMOVNO32rm: | |||
2213 | case X86::CMOVNO32rr: case X86::CMOVNO64rm: case X86::CMOVNO64rr: | |||
2214 | return X86::COND_NO; | |||
2215 | case X86::CMOVNP16rm: case X86::CMOVNP16rr: case X86::CMOVNP32rm: | |||
2216 | case X86::CMOVNP32rr: case X86::CMOVNP64rm: case X86::CMOVNP64rr: | |||
2217 | return X86::COND_NP; | |||
2218 | case X86::CMOVNS16rm: case X86::CMOVNS16rr: case X86::CMOVNS32rm: | |||
2219 | case X86::CMOVNS32rr: case X86::CMOVNS64rm: case X86::CMOVNS64rr: | |||
2220 | return X86::COND_NS; | |||
2221 | case X86::CMOVO16rm: case X86::CMOVO16rr: case X86::CMOVO32rm: | |||
2222 | case X86::CMOVO32rr: case X86::CMOVO64rm: case X86::CMOVO64rr: | |||
2223 | return X86::COND_O; | |||
2224 | case X86::CMOVP16rm: case X86::CMOVP16rr: case X86::CMOVP32rm: | |||
2225 | case X86::CMOVP32rr: case X86::CMOVP64rm: case X86::CMOVP64rr: | |||
2226 | return X86::COND_P; | |||
2227 | case X86::CMOVS16rm: case X86::CMOVS16rr: case X86::CMOVS32rm: | |||
2228 | case X86::CMOVS32rr: case X86::CMOVS64rm: case X86::CMOVS64rr: | |||
2229 | return X86::COND_S; | |||
2230 | } | |||
2231 | } | |||
2232 | ||||
2233 | unsigned X86::GetCondBranchFromCond(X86::CondCode CC) { | |||
2234 | switch (CC) { | |||
2235 | default: llvm_unreachable("Illegal condition code!")::llvm::llvm_unreachable_internal("Illegal condition code!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2235); | |||
2236 | case X86::COND_E: return X86::JE_1; | |||
2237 | case X86::COND_NE: return X86::JNE_1; | |||
2238 | case X86::COND_L: return X86::JL_1; | |||
2239 | case X86::COND_LE: return X86::JLE_1; | |||
2240 | case X86::COND_G: return X86::JG_1; | |||
2241 | case X86::COND_GE: return X86::JGE_1; | |||
2242 | case X86::COND_B: return X86::JB_1; | |||
2243 | case X86::COND_BE: return X86::JBE_1; | |||
2244 | case X86::COND_A: return X86::JA_1; | |||
2245 | case X86::COND_AE: return X86::JAE_1; | |||
2246 | case X86::COND_S: return X86::JS_1; | |||
2247 | case X86::COND_NS: return X86::JNS_1; | |||
2248 | case X86::COND_P: return X86::JP_1; | |||
2249 | case X86::COND_NP: return X86::JNP_1; | |||
2250 | case X86::COND_O: return X86::JO_1; | |||
2251 | case X86::COND_NO: return X86::JNO_1; | |||
2252 | } | |||
2253 | } | |||
2254 | ||||
2255 | /// Return the inverse of the specified condition, | |||
2256 | /// e.g. turning COND_E to COND_NE. | |||
2257 | X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) { | |||
2258 | switch (CC) { | |||
2259 | default: llvm_unreachable("Illegal condition code!")::llvm::llvm_unreachable_internal("Illegal condition code!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2259); | |||
2260 | case X86::COND_E: return X86::COND_NE; | |||
2261 | case X86::COND_NE: return X86::COND_E; | |||
2262 | case X86::COND_L: return X86::COND_GE; | |||
2263 | case X86::COND_LE: return X86::COND_G; | |||
2264 | case X86::COND_G: return X86::COND_LE; | |||
2265 | case X86::COND_GE: return X86::COND_L; | |||
2266 | case X86::COND_B: return X86::COND_AE; | |||
2267 | case X86::COND_BE: return X86::COND_A; | |||
2268 | case X86::COND_A: return X86::COND_BE; | |||
2269 | case X86::COND_AE: return X86::COND_B; | |||
2270 | case X86::COND_S: return X86::COND_NS; | |||
2271 | case X86::COND_NS: return X86::COND_S; | |||
2272 | case X86::COND_P: return X86::COND_NP; | |||
2273 | case X86::COND_NP: return X86::COND_P; | |||
2274 | case X86::COND_O: return X86::COND_NO; | |||
2275 | case X86::COND_NO: return X86::COND_O; | |||
2276 | case X86::COND_NE_OR_P: return X86::COND_E_AND_NP; | |||
2277 | case X86::COND_E_AND_NP: return X86::COND_NE_OR_P; | |||
2278 | } | |||
2279 | } | |||
2280 | ||||
2281 | /// Assuming the flags are set by MI(a,b), return the condition code if we | |||
2282 | /// modify the instructions such that flags are set by MI(b,a). | |||
2283 | static X86::CondCode getSwappedCondition(X86::CondCode CC) { | |||
2284 | switch (CC) { | |||
2285 | default: return X86::COND_INVALID; | |||
2286 | case X86::COND_E: return X86::COND_E; | |||
2287 | case X86::COND_NE: return X86::COND_NE; | |||
2288 | case X86::COND_L: return X86::COND_G; | |||
2289 | case X86::COND_LE: return X86::COND_GE; | |||
2290 | case X86::COND_G: return X86::COND_L; | |||
2291 | case X86::COND_GE: return X86::COND_LE; | |||
2292 | case X86::COND_B: return X86::COND_A; | |||
2293 | case X86::COND_BE: return X86::COND_AE; | |||
2294 | case X86::COND_A: return X86::COND_B; | |||
2295 | case X86::COND_AE: return X86::COND_BE; | |||
2296 | } | |||
2297 | } | |||
2298 | ||||
2299 | std::pair<X86::CondCode, bool> | |||
2300 | X86::getX86ConditionCode(CmpInst::Predicate Predicate) { | |||
2301 | X86::CondCode CC = X86::COND_INVALID; | |||
2302 | bool NeedSwap = false; | |||
2303 | switch (Predicate) { | |||
2304 | default: break; | |||
2305 | // Floating-point Predicates | |||
2306 | case CmpInst::FCMP_UEQ: CC = X86::COND_E; break; | |||
2307 | case CmpInst::FCMP_OLT: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2308 | case CmpInst::FCMP_OGT: CC = X86::COND_A; break; | |||
2309 | case CmpInst::FCMP_OLE: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2310 | case CmpInst::FCMP_OGE: CC = X86::COND_AE; break; | |||
2311 | case CmpInst::FCMP_UGT: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2312 | case CmpInst::FCMP_ULT: CC = X86::COND_B; break; | |||
2313 | case CmpInst::FCMP_UGE: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2314 | case CmpInst::FCMP_ULE: CC = X86::COND_BE; break; | |||
2315 | case CmpInst::FCMP_ONE: CC = X86::COND_NE; break; | |||
2316 | case CmpInst::FCMP_UNO: CC = X86::COND_P; break; | |||
2317 | case CmpInst::FCMP_ORD: CC = X86::COND_NP; break; | |||
2318 | case CmpInst::FCMP_OEQ: LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2319 | case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break; | |||
2320 | ||||
2321 | // Integer Predicates | |||
2322 | case CmpInst::ICMP_EQ: CC = X86::COND_E; break; | |||
2323 | case CmpInst::ICMP_NE: CC = X86::COND_NE; break; | |||
2324 | case CmpInst::ICMP_UGT: CC = X86::COND_A; break; | |||
2325 | case CmpInst::ICMP_UGE: CC = X86::COND_AE; break; | |||
2326 | case CmpInst::ICMP_ULT: CC = X86::COND_B; break; | |||
2327 | case CmpInst::ICMP_ULE: CC = X86::COND_BE; break; | |||
2328 | case CmpInst::ICMP_SGT: CC = X86::COND_G; break; | |||
2329 | case CmpInst::ICMP_SGE: CC = X86::COND_GE; break; | |||
2330 | case CmpInst::ICMP_SLT: CC = X86::COND_L; break; | |||
2331 | case CmpInst::ICMP_SLE: CC = X86::COND_LE; break; | |||
2332 | } | |||
2333 | ||||
2334 | return std::make_pair(CC, NeedSwap); | |||
2335 | } | |||
2336 | ||||
2337 | /// Return a set opcode for the given condition and | |||
2338 | /// whether it has memory operand. | |||
2339 | unsigned X86::getSETFromCond(CondCode CC, bool HasMemoryOperand) { | |||
2340 | static const uint16_t Opc[16][2] = { | |||
2341 | { X86::SETAr, X86::SETAm }, | |||
2342 | { X86::SETAEr, X86::SETAEm }, | |||
2343 | { X86::SETBr, X86::SETBm }, | |||
2344 | { X86::SETBEr, X86::SETBEm }, | |||
2345 | { X86::SETEr, X86::SETEm }, | |||
2346 | { X86::SETGr, X86::SETGm }, | |||
2347 | { X86::SETGEr, X86::SETGEm }, | |||
2348 | { X86::SETLr, X86::SETLm }, | |||
2349 | { X86::SETLEr, X86::SETLEm }, | |||
2350 | { X86::SETNEr, X86::SETNEm }, | |||
2351 | { X86::SETNOr, X86::SETNOm }, | |||
2352 | { X86::SETNPr, X86::SETNPm }, | |||
2353 | { X86::SETNSr, X86::SETNSm }, | |||
2354 | { X86::SETOr, X86::SETOm }, | |||
2355 | { X86::SETPr, X86::SETPm }, | |||
2356 | { X86::SETSr, X86::SETSm } | |||
2357 | }; | |||
2358 | ||||
2359 | assert(CC <= LAST_VALID_COND && "Can only handle standard cond codes")((CC <= LAST_VALID_COND && "Can only handle standard cond codes" ) ? static_cast<void> (0) : __assert_fail ("CC <= LAST_VALID_COND && \"Can only handle standard cond codes\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2359, __PRETTY_FUNCTION__)); | |||
2360 | return Opc[CC][HasMemoryOperand ? 1 : 0]; | |||
2361 | } | |||
2362 | ||||
2363 | /// Return a cmov opcode for the given condition, | |||
2364 | /// register size in bytes, and operand type. | |||
2365 | unsigned X86::getCMovFromCond(CondCode CC, unsigned RegBytes, | |||
2366 | bool HasMemoryOperand) { | |||
2367 | static const uint16_t Opc[32][3] = { | |||
2368 | { X86::CMOVA16rr, X86::CMOVA32rr, X86::CMOVA64rr }, | |||
2369 | { X86::CMOVAE16rr, X86::CMOVAE32rr, X86::CMOVAE64rr }, | |||
2370 | { X86::CMOVB16rr, X86::CMOVB32rr, X86::CMOVB64rr }, | |||
2371 | { X86::CMOVBE16rr, X86::CMOVBE32rr, X86::CMOVBE64rr }, | |||
2372 | { X86::CMOVE16rr, X86::CMOVE32rr, X86::CMOVE64rr }, | |||
2373 | { X86::CMOVG16rr, X86::CMOVG32rr, X86::CMOVG64rr }, | |||
2374 | { X86::CMOVGE16rr, X86::CMOVGE32rr, X86::CMOVGE64rr }, | |||
2375 | { X86::CMOVL16rr, X86::CMOVL32rr, X86::CMOVL64rr }, | |||
2376 | { X86::CMOVLE16rr, X86::CMOVLE32rr, X86::CMOVLE64rr }, | |||
2377 | { X86::CMOVNE16rr, X86::CMOVNE32rr, X86::CMOVNE64rr }, | |||
2378 | { X86::CMOVNO16rr, X86::CMOVNO32rr, X86::CMOVNO64rr }, | |||
2379 | { X86::CMOVNP16rr, X86::CMOVNP32rr, X86::CMOVNP64rr }, | |||
2380 | { X86::CMOVNS16rr, X86::CMOVNS32rr, X86::CMOVNS64rr }, | |||
2381 | { X86::CMOVO16rr, X86::CMOVO32rr, X86::CMOVO64rr }, | |||
2382 | { X86::CMOVP16rr, X86::CMOVP32rr, X86::CMOVP64rr }, | |||
2383 | { X86::CMOVS16rr, X86::CMOVS32rr, X86::CMOVS64rr }, | |||
2384 | { X86::CMOVA16rm, X86::CMOVA32rm, X86::CMOVA64rm }, | |||
2385 | { X86::CMOVAE16rm, X86::CMOVAE32rm, X86::CMOVAE64rm }, | |||
2386 | { X86::CMOVB16rm, X86::CMOVB32rm, X86::CMOVB64rm }, | |||
2387 | { X86::CMOVBE16rm, X86::CMOVBE32rm, X86::CMOVBE64rm }, | |||
2388 | { X86::CMOVE16rm, X86::CMOVE32rm, X86::CMOVE64rm }, | |||
2389 | { X86::CMOVG16rm, X86::CMOVG32rm, X86::CMOVG64rm }, | |||
2390 | { X86::CMOVGE16rm, X86::CMOVGE32rm, X86::CMOVGE64rm }, | |||
2391 | { X86::CMOVL16rm, X86::CMOVL32rm, X86::CMOVL64rm }, | |||
2392 | { X86::CMOVLE16rm, X86::CMOVLE32rm, X86::CMOVLE64rm }, | |||
2393 | { X86::CMOVNE16rm, X86::CMOVNE32rm, X86::CMOVNE64rm }, | |||
2394 | { X86::CMOVNO16rm, X86::CMOVNO32rm, X86::CMOVNO64rm }, | |||
2395 | { X86::CMOVNP16rm, X86::CMOVNP32rm, X86::CMOVNP64rm }, | |||
2396 | { X86::CMOVNS16rm, X86::CMOVNS32rm, X86::CMOVNS64rm }, | |||
2397 | { X86::CMOVO16rm, X86::CMOVO32rm, X86::CMOVO64rm }, | |||
2398 | { X86::CMOVP16rm, X86::CMOVP32rm, X86::CMOVP64rm }, | |||
2399 | { X86::CMOVS16rm, X86::CMOVS32rm, X86::CMOVS64rm } | |||
2400 | }; | |||
2401 | ||||
2402 | assert(CC < 16 && "Can only handle standard cond codes")((CC < 16 && "Can only handle standard cond codes" ) ? static_cast<void> (0) : __assert_fail ("CC < 16 && \"Can only handle standard cond codes\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2402, __PRETTY_FUNCTION__)); | |||
2403 | unsigned Idx = HasMemoryOperand ? 16+CC : CC; | |||
2404 | switch(RegBytes) { | |||
2405 | default: llvm_unreachable("Illegal register size!")::llvm::llvm_unreachable_internal("Illegal register size!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2405); | |||
2406 | case 2: return Opc[Idx][0]; | |||
2407 | case 4: return Opc[Idx][1]; | |||
2408 | case 8: return Opc[Idx][2]; | |||
2409 | } | |||
2410 | } | |||
2411 | ||||
2412 | /// Get the VPCMP immediate for the given condition. | |||
2413 | unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) { | |||
2414 | switch (CC) { | |||
2415 | default: llvm_unreachable("Unexpected SETCC condition")::llvm::llvm_unreachable_internal("Unexpected SETCC condition" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2415); | |||
2416 | case ISD::SETNE: return 4; | |||
2417 | case ISD::SETEQ: return 0; | |||
2418 | case ISD::SETULT: | |||
2419 | case ISD::SETLT: return 1; | |||
2420 | case ISD::SETUGT: | |||
2421 | case ISD::SETGT: return 6; | |||
2422 | case ISD::SETUGE: | |||
2423 | case ISD::SETGE: return 5; | |||
2424 | case ISD::SETULE: | |||
2425 | case ISD::SETLE: return 2; | |||
2426 | } | |||
2427 | } | |||
2428 | ||||
2429 | /// Get the VPCMP immediate if the opcodes are swapped. | |||
2430 | unsigned X86::getSwappedVPCMPImm(unsigned Imm) { | |||
2431 | switch (Imm) { | |||
2432 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2432); | |||
2433 | case 0x01: Imm = 0x06; break; // LT -> NLE | |||
2434 | case 0x02: Imm = 0x05; break; // LE -> NLT | |||
2435 | case 0x05: Imm = 0x02; break; // NLT -> LE | |||
2436 | case 0x06: Imm = 0x01; break; // NLE -> LT | |||
2437 | case 0x00: // EQ | |||
2438 | case 0x03: // FALSE | |||
2439 | case 0x04: // NE | |||
2440 | case 0x07: // TRUE | |||
2441 | break; | |||
2442 | } | |||
2443 | ||||
2444 | return Imm; | |||
2445 | } | |||
2446 | ||||
2447 | /// Get the VPCOM immediate if the opcodes are swapped. | |||
2448 | unsigned X86::getSwappedVPCOMImm(unsigned Imm) { | |||
2449 | switch (Imm) { | |||
2450 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2450); | |||
2451 | case 0x00: Imm = 0x02; break; // LT -> GT | |||
2452 | case 0x01: Imm = 0x03; break; // LE -> GE | |||
2453 | case 0x02: Imm = 0x00; break; // GT -> LT | |||
2454 | case 0x03: Imm = 0x01; break; // GE -> LE | |||
2455 | case 0x04: // EQ | |||
2456 | case 0x05: // NE | |||
2457 | case 0x06: // FALSE | |||
2458 | case 0x07: // TRUE | |||
2459 | break; | |||
2460 | } | |||
2461 | ||||
2462 | return Imm; | |||
2463 | } | |||
2464 | ||||
2465 | bool X86InstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const { | |||
2466 | if (!MI.isTerminator()) return false; | |||
2467 | ||||
2468 | // Conditional branch is a special case. | |||
2469 | if (MI.isBranch() && !MI.isBarrier()) | |||
2470 | return true; | |||
2471 | if (!MI.isPredicable()) | |||
2472 | return true; | |||
2473 | return !isPredicated(MI); | |||
2474 | } | |||
2475 | ||||
2476 | bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const { | |||
2477 | switch (MI.getOpcode()) { | |||
2478 | case X86::TCRETURNdi: | |||
2479 | case X86::TCRETURNri: | |||
2480 | case X86::TCRETURNmi: | |||
2481 | case X86::TCRETURNdi64: | |||
2482 | case X86::TCRETURNri64: | |||
2483 | case X86::TCRETURNmi64: | |||
2484 | return true; | |||
2485 | default: | |||
2486 | return false; | |||
2487 | } | |||
2488 | } | |||
2489 | ||||
2490 | bool X86InstrInfo::canMakeTailCallConditional( | |||
2491 | SmallVectorImpl<MachineOperand> &BranchCond, | |||
2492 | const MachineInstr &TailCall) const { | |||
2493 | if (TailCall.getOpcode() != X86::TCRETURNdi && | |||
2494 | TailCall.getOpcode() != X86::TCRETURNdi64) { | |||
2495 | // Only direct calls can be done with a conditional branch. | |||
2496 | return false; | |||
2497 | } | |||
2498 | ||||
2499 | const MachineFunction *MF = TailCall.getParent()->getParent(); | |||
2500 | if (Subtarget.isTargetWin64() && MF->hasWinCFI()) { | |||
2501 | // Conditional tail calls confuse the Win64 unwinder. | |||
2502 | return false; | |||
2503 | } | |||
2504 | ||||
2505 | assert(BranchCond.size() == 1)((BranchCond.size() == 1) ? static_cast<void> (0) : __assert_fail ("BranchCond.size() == 1", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2505, __PRETTY_FUNCTION__)); | |||
2506 | if (BranchCond[0].getImm() > X86::LAST_VALID_COND) { | |||
2507 | // Can't make a conditional tail call with this condition. | |||
2508 | return false; | |||
2509 | } | |||
2510 | ||||
2511 | const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); | |||
2512 | if (X86FI->getTCReturnAddrDelta() != 0 || | |||
2513 | TailCall.getOperand(1).getImm() != 0) { | |||
2514 | // A conditional tail call cannot do any stack adjustment. | |||
2515 | return false; | |||
2516 | } | |||
2517 | ||||
2518 | return true; | |||
2519 | } | |||
2520 | ||||
2521 | void X86InstrInfo::replaceBranchWithTailCall( | |||
2522 | MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond, | |||
2523 | const MachineInstr &TailCall) const { | |||
2524 | assert(canMakeTailCallConditional(BranchCond, TailCall))((canMakeTailCallConditional(BranchCond, TailCall)) ? static_cast <void> (0) : __assert_fail ("canMakeTailCallConditional(BranchCond, TailCall)" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2524, __PRETTY_FUNCTION__)); | |||
2525 | ||||
2526 | MachineBasicBlock::iterator I = MBB.end(); | |||
2527 | while (I != MBB.begin()) { | |||
2528 | --I; | |||
2529 | if (I->isDebugInstr()) | |||
2530 | continue; | |||
2531 | if (!I->isBranch()) | |||
2532 | assert(0 && "Can't find the branch to replace!")((0 && "Can't find the branch to replace!") ? static_cast <void> (0) : __assert_fail ("0 && \"Can't find the branch to replace!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2532, __PRETTY_FUNCTION__)); | |||
2533 | ||||
2534 | X86::CondCode CC = X86::getCondFromBranchOpc(I->getOpcode()); | |||
2535 | assert(BranchCond.size() == 1)((BranchCond.size() == 1) ? static_cast<void> (0) : __assert_fail ("BranchCond.size() == 1", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2535, __PRETTY_FUNCTION__)); | |||
2536 | if (CC != BranchCond[0].getImm()) | |||
2537 | continue; | |||
2538 | ||||
2539 | break; | |||
2540 | } | |||
2541 | ||||
2542 | unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc | |||
2543 | : X86::TCRETURNdi64cc; | |||
2544 | ||||
2545 | auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc)); | |||
2546 | MIB->addOperand(TailCall.getOperand(0)); // Destination. | |||
2547 | MIB.addImm(0); // Stack offset (not used). | |||
2548 | MIB->addOperand(BranchCond[0]); // Condition. | |||
2549 | MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters. | |||
2550 | ||||
2551 | // Add implicit uses and defs of all live regs potentially clobbered by the | |||
2552 | // call. This way they still appear live across the call. | |||
2553 | LivePhysRegs LiveRegs(getRegisterInfo()); | |||
2554 | LiveRegs.addLiveOuts(MBB); | |||
2555 | SmallVector<std::pair<unsigned, const MachineOperand *>, 8> Clobbers; | |||
2556 | LiveRegs.stepForward(*MIB, Clobbers); | |||
2557 | for (const auto &C : Clobbers) { | |||
2558 | MIB.addReg(C.first, RegState::Implicit); | |||
2559 | MIB.addReg(C.first, RegState::Implicit | RegState::Define); | |||
2560 | } | |||
2561 | ||||
2562 | I->eraseFromParent(); | |||
2563 | } | |||
2564 | ||||
2565 | // Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may | |||
2566 | // not be a fallthrough MBB now due to layout changes). Return nullptr if the | |||
2567 | // fallthrough MBB cannot be identified. | |||
2568 | static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB, | |||
2569 | MachineBasicBlock *TBB) { | |||
2570 | // Look for non-EHPad successors other than TBB. If we find exactly one, it | |||
2571 | // is the fallthrough MBB. If we find zero, then TBB is both the target MBB | |||
2572 | // and fallthrough MBB. If we find more than one, we cannot identify the | |||
2573 | // fallthrough MBB and should return nullptr. | |||
2574 | MachineBasicBlock *FallthroughBB = nullptr; | |||
2575 | for (auto SI = MBB->succ_begin(), SE = MBB->succ_end(); SI != SE; ++SI) { | |||
2576 | if ((*SI)->isEHPad() || (*SI == TBB && FallthroughBB)) | |||
2577 | continue; | |||
2578 | // Return a nullptr if we found more than one fallthrough successor. | |||
2579 | if (FallthroughBB && FallthroughBB != TBB) | |||
2580 | return nullptr; | |||
2581 | FallthroughBB = *SI; | |||
2582 | } | |||
2583 | return FallthroughBB; | |||
2584 | } | |||
2585 | ||||
2586 | bool X86InstrInfo::AnalyzeBranchImpl( | |||
2587 | MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, | |||
2588 | SmallVectorImpl<MachineOperand> &Cond, | |||
2589 | SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const { | |||
2590 | ||||
2591 | // Start from the bottom of the block and work up, examining the | |||
2592 | // terminator instructions. | |||
2593 | MachineBasicBlock::iterator I = MBB.end(); | |||
2594 | MachineBasicBlock::iterator UnCondBrIter = MBB.end(); | |||
2595 | while (I != MBB.begin()) { | |||
2596 | --I; | |||
2597 | if (I->isDebugInstr()) | |||
2598 | continue; | |||
2599 | ||||
2600 | // Working from the bottom, when we see a non-terminator instruction, we're | |||
2601 | // done. | |||
2602 | if (!isUnpredicatedTerminator(*I)) | |||
2603 | break; | |||
2604 | ||||
2605 | // A terminator that isn't a branch can't easily be handled by this | |||
2606 | // analysis. | |||
2607 | if (!I->isBranch()) | |||
2608 | return true; | |||
2609 | ||||
2610 | // Handle unconditional branches. | |||
2611 | if (I->getOpcode() == X86::JMP_1) { | |||
2612 | UnCondBrIter = I; | |||
2613 | ||||
2614 | if (!AllowModify) { | |||
2615 | TBB = I->getOperand(0).getMBB(); | |||
2616 | continue; | |||
2617 | } | |||
2618 | ||||
2619 | // If the block has any instructions after a JMP, delete them. | |||
2620 | while (std::next(I) != MBB.end()) | |||
2621 | std::next(I)->eraseFromParent(); | |||
2622 | ||||
2623 | Cond.clear(); | |||
2624 | FBB = nullptr; | |||
2625 | ||||
2626 | // Delete the JMP if it's equivalent to a fall-through. | |||
2627 | if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { | |||
2628 | TBB = nullptr; | |||
2629 | I->eraseFromParent(); | |||
2630 | I = MBB.end(); | |||
2631 | UnCondBrIter = MBB.end(); | |||
2632 | continue; | |||
2633 | } | |||
2634 | ||||
2635 | // TBB is used to indicate the unconditional destination. | |||
2636 | TBB = I->getOperand(0).getMBB(); | |||
2637 | continue; | |||
2638 | } | |||
2639 | ||||
2640 | // Handle conditional branches. | |||
2641 | X86::CondCode BranchCode = X86::getCondFromBranchOpc(I->getOpcode()); | |||
2642 | if (BranchCode == X86::COND_INVALID) | |||
2643 | return true; // Can't handle indirect branch. | |||
2644 | ||||
2645 | // In practice we should never have an undef eflags operand, if we do | |||
2646 | // abort here as we are not prepared to preserve the flag. | |||
2647 | if (I->getOperand(1).isUndef()) | |||
2648 | return true; | |||
2649 | ||||
2650 | // Working from the bottom, handle the first conditional branch. | |||
2651 | if (Cond.empty()) { | |||
2652 | MachineBasicBlock *TargetBB = I->getOperand(0).getMBB(); | |||
2653 | if (AllowModify && UnCondBrIter != MBB.end() && | |||
2654 | MBB.isLayoutSuccessor(TargetBB)) { | |||
2655 | // If we can modify the code and it ends in something like: | |||
2656 | // | |||
2657 | // jCC L1 | |||
2658 | // jmp L2 | |||
2659 | // L1: | |||
2660 | // ... | |||
2661 | // L2: | |||
2662 | // | |||
2663 | // Then we can change this to: | |||
2664 | // | |||
2665 | // jnCC L2 | |||
2666 | // L1: | |||
2667 | // ... | |||
2668 | // L2: | |||
2669 | // | |||
2670 | // Which is a bit more efficient. | |||
2671 | // We conditionally jump to the fall-through block. | |||
2672 | BranchCode = GetOppositeBranchCondition(BranchCode); | |||
2673 | unsigned JNCC = GetCondBranchFromCond(BranchCode); | |||
2674 | MachineBasicBlock::iterator OldInst = I; | |||
2675 | ||||
2676 | BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(JNCC)) | |||
2677 | .addMBB(UnCondBrIter->getOperand(0).getMBB()); | |||
2678 | BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(X86::JMP_1)) | |||
2679 | .addMBB(TargetBB); | |||
2680 | ||||
2681 | OldInst->eraseFromParent(); | |||
2682 | UnCondBrIter->eraseFromParent(); | |||
2683 | ||||
2684 | // Restart the analysis. | |||
2685 | UnCondBrIter = MBB.end(); | |||
2686 | I = MBB.end(); | |||
2687 | continue; | |||
2688 | } | |||
2689 | ||||
2690 | FBB = TBB; | |||
2691 | TBB = I->getOperand(0).getMBB(); | |||
2692 | Cond.push_back(MachineOperand::CreateImm(BranchCode)); | |||
2693 | CondBranches.push_back(&*I); | |||
2694 | continue; | |||
2695 | } | |||
2696 | ||||
2697 | // Handle subsequent conditional branches. Only handle the case where all | |||
2698 | // conditional branches branch to the same destination and their condition | |||
2699 | // opcodes fit one of the special multi-branch idioms. | |||
2700 | assert(Cond.size() == 1)((Cond.size() == 1) ? static_cast<void> (0) : __assert_fail ("Cond.size() == 1", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2700, __PRETTY_FUNCTION__)); | |||
2701 | assert(TBB)((TBB) ? static_cast<void> (0) : __assert_fail ("TBB", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2701, __PRETTY_FUNCTION__)); | |||
2702 | ||||
2703 | // If the conditions are the same, we can leave them alone. | |||
2704 | X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm(); | |||
2705 | auto NewTBB = I->getOperand(0).getMBB(); | |||
2706 | if (OldBranchCode == BranchCode && TBB == NewTBB) | |||
2707 | continue; | |||
2708 | ||||
2709 | // If they differ, see if they fit one of the known patterns. Theoretically, | |||
2710 | // we could handle more patterns here, but we shouldn't expect to see them | |||
2711 | // if instruction selection has done a reasonable job. | |||
2712 | if (TBB == NewTBB && | |||
2713 | ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) || | |||
2714 | (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) { | |||
2715 | BranchCode = X86::COND_NE_OR_P; | |||
2716 | } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) || | |||
2717 | (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) { | |||
2718 | if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB))) | |||
2719 | return true; | |||
2720 | ||||
2721 | // X86::COND_E_AND_NP usually has two different branch destinations. | |||
2722 | // | |||
2723 | // JP B1 | |||
2724 | // JE B2 | |||
2725 | // JMP B1 | |||
2726 | // B1: | |||
2727 | // B2: | |||
2728 | // | |||
2729 | // Here this condition branches to B2 only if NP && E. It has another | |||
2730 | // equivalent form: | |||
2731 | // | |||
2732 | // JNE B1 | |||
2733 | // JNP B2 | |||
2734 | // JMP B1 | |||
2735 | // B1: | |||
2736 | // B2: | |||
2737 | // | |||
2738 | // Similarly it branches to B2 only if E && NP. That is why this condition | |||
2739 | // is named with COND_E_AND_NP. | |||
2740 | BranchCode = X86::COND_E_AND_NP; | |||
2741 | } else | |||
2742 | return true; | |||
2743 | ||||
2744 | // Update the MachineOperand. | |||
2745 | Cond[0].setImm(BranchCode); | |||
2746 | CondBranches.push_back(&*I); | |||
2747 | } | |||
2748 | ||||
2749 | return false; | |||
2750 | } | |||
2751 | ||||
2752 | bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB, | |||
2753 | MachineBasicBlock *&TBB, | |||
2754 | MachineBasicBlock *&FBB, | |||
2755 | SmallVectorImpl<MachineOperand> &Cond, | |||
2756 | bool AllowModify) const { | |||
2757 | SmallVector<MachineInstr *, 4> CondBranches; | |||
2758 | return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify); | |||
2759 | } | |||
2760 | ||||
2761 | bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB, | |||
2762 | MachineBranchPredicate &MBP, | |||
2763 | bool AllowModify) const { | |||
2764 | using namespace std::placeholders; | |||
2765 | ||||
2766 | SmallVector<MachineOperand, 4> Cond; | |||
2767 | SmallVector<MachineInstr *, 4> CondBranches; | |||
2768 | if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches, | |||
2769 | AllowModify)) | |||
2770 | return true; | |||
2771 | ||||
2772 | if (Cond.size() != 1) | |||
2773 | return true; | |||
2774 | ||||
2775 | assert(MBP.TrueDest && "expected!")((MBP.TrueDest && "expected!") ? static_cast<void> (0) : __assert_fail ("MBP.TrueDest && \"expected!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2775, __PRETTY_FUNCTION__)); | |||
2776 | ||||
2777 | if (!MBP.FalseDest) | |||
2778 | MBP.FalseDest = MBB.getNextNode(); | |||
2779 | ||||
2780 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
2781 | ||||
2782 | MachineInstr *ConditionDef = nullptr; | |||
2783 | bool SingleUseCondition = true; | |||
2784 | ||||
2785 | for (auto I = std::next(MBB.rbegin()), E = MBB.rend(); I != E; ++I) { | |||
2786 | if (I->modifiesRegister(X86::EFLAGS, TRI)) { | |||
2787 | ConditionDef = &*I; | |||
2788 | break; | |||
2789 | } | |||
2790 | ||||
2791 | if (I->readsRegister(X86::EFLAGS, TRI)) | |||
2792 | SingleUseCondition = false; | |||
2793 | } | |||
2794 | ||||
2795 | if (!ConditionDef) | |||
2796 | return true; | |||
2797 | ||||
2798 | if (SingleUseCondition) { | |||
2799 | for (auto *Succ : MBB.successors()) | |||
2800 | if (Succ->isLiveIn(X86::EFLAGS)) | |||
2801 | SingleUseCondition = false; | |||
2802 | } | |||
2803 | ||||
2804 | MBP.ConditionDef = ConditionDef; | |||
2805 | MBP.SingleUseCondition = SingleUseCondition; | |||
2806 | ||||
2807 | // Currently we only recognize the simple pattern: | |||
2808 | // | |||
2809 | // test %reg, %reg | |||
2810 | // je %label | |||
2811 | // | |||
2812 | const unsigned TestOpcode = | |||
2813 | Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr; | |||
2814 | ||||
2815 | if (ConditionDef->getOpcode() == TestOpcode && | |||
2816 | ConditionDef->getNumOperands() == 3 && | |||
2817 | ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) && | |||
2818 | (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) { | |||
2819 | MBP.LHS = ConditionDef->getOperand(0); | |||
2820 | MBP.RHS = MachineOperand::CreateImm(0); | |||
2821 | MBP.Predicate = Cond[0].getImm() == X86::COND_NE | |||
2822 | ? MachineBranchPredicate::PRED_NE | |||
2823 | : MachineBranchPredicate::PRED_EQ; | |||
2824 | return false; | |||
2825 | } | |||
2826 | ||||
2827 | return true; | |||
2828 | } | |||
2829 | ||||
2830 | unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB, | |||
2831 | int *BytesRemoved) const { | |||
2832 | assert(!BytesRemoved && "code size not handled")((!BytesRemoved && "code size not handled") ? static_cast <void> (0) : __assert_fail ("!BytesRemoved && \"code size not handled\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2832, __PRETTY_FUNCTION__)); | |||
2833 | ||||
2834 | MachineBasicBlock::iterator I = MBB.end(); | |||
2835 | unsigned Count = 0; | |||
2836 | ||||
2837 | while (I != MBB.begin()) { | |||
2838 | --I; | |||
2839 | if (I->isDebugInstr()) | |||
2840 | continue; | |||
2841 | if (I->getOpcode() != X86::JMP_1 && | |||
2842 | X86::getCondFromBranchOpc(I->getOpcode()) == X86::COND_INVALID) | |||
2843 | break; | |||
2844 | // Remove the branch. | |||
2845 | I->eraseFromParent(); | |||
2846 | I = MBB.end(); | |||
2847 | ++Count; | |||
2848 | } | |||
2849 | ||||
2850 | return Count; | |||
2851 | } | |||
2852 | ||||
2853 | unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB, | |||
2854 | MachineBasicBlock *TBB, | |||
2855 | MachineBasicBlock *FBB, | |||
2856 | ArrayRef<MachineOperand> Cond, | |||
2857 | const DebugLoc &DL, | |||
2858 | int *BytesAdded) const { | |||
2859 | // Shouldn't be a fall through. | |||
2860 | assert(TBB && "insertBranch must not be told to insert a fallthrough")((TBB && "insertBranch must not be told to insert a fallthrough" ) ? static_cast<void> (0) : __assert_fail ("TBB && \"insertBranch must not be told to insert a fallthrough\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2860, __PRETTY_FUNCTION__)); | |||
2861 | assert((Cond.size() == 1 || Cond.size() == 0) &&(((Cond.size() == 1 || Cond.size() == 0) && "X86 branch conditions have one component!" ) ? static_cast<void> (0) : __assert_fail ("(Cond.size() == 1 || Cond.size() == 0) && \"X86 branch conditions have one component!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2862, __PRETTY_FUNCTION__)) | |||
2862 | "X86 branch conditions have one component!")(((Cond.size() == 1 || Cond.size() == 0) && "X86 branch conditions have one component!" ) ? static_cast<void> (0) : __assert_fail ("(Cond.size() == 1 || Cond.size() == 0) && \"X86 branch conditions have one component!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2862, __PRETTY_FUNCTION__)); | |||
2863 | assert(!BytesAdded && "code size not handled")((!BytesAdded && "code size not handled") ? static_cast <void> (0) : __assert_fail ("!BytesAdded && \"code size not handled\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2863, __PRETTY_FUNCTION__)); | |||
2864 | ||||
2865 | if (Cond.empty()) { | |||
2866 | // Unconditional branch? | |||
2867 | assert(!FBB && "Unconditional branch with multiple successors!")((!FBB && "Unconditional branch with multiple successors!" ) ? static_cast<void> (0) : __assert_fail ("!FBB && \"Unconditional branch with multiple successors!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2867, __PRETTY_FUNCTION__)); | |||
2868 | BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB); | |||
2869 | return 1; | |||
2870 | } | |||
2871 | ||||
2872 | // If FBB is null, it is implied to be a fall-through block. | |||
2873 | bool FallThru = FBB == nullptr; | |||
2874 | ||||
2875 | // Conditional branch. | |||
2876 | unsigned Count = 0; | |||
2877 | X86::CondCode CC = (X86::CondCode)Cond[0].getImm(); | |||
2878 | switch (CC) { | |||
2879 | case X86::COND_NE_OR_P: | |||
2880 | // Synthesize NE_OR_P with two branches. | |||
2881 | BuildMI(&MBB, DL, get(X86::JNE_1)).addMBB(TBB); | |||
2882 | ++Count; | |||
2883 | BuildMI(&MBB, DL, get(X86::JP_1)).addMBB(TBB); | |||
2884 | ++Count; | |||
2885 | break; | |||
2886 | case X86::COND_E_AND_NP: | |||
2887 | // Use the next block of MBB as FBB if it is null. | |||
2888 | if (FBB == nullptr) { | |||
2889 | FBB = getFallThroughMBB(&MBB, TBB); | |||
2890 | assert(FBB && "MBB cannot be the last block in function when the false "((FBB && "MBB cannot be the last block in function when the false " "body is a fall-through.") ? static_cast<void> (0) : __assert_fail ("FBB && \"MBB cannot be the last block in function when the false \" \"body is a fall-through.\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2891, __PRETTY_FUNCTION__)) | |||
2891 | "body is a fall-through.")((FBB && "MBB cannot be the last block in function when the false " "body is a fall-through.") ? static_cast<void> (0) : __assert_fail ("FBB && \"MBB cannot be the last block in function when the false \" \"body is a fall-through.\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2891, __PRETTY_FUNCTION__)); | |||
2892 | } | |||
2893 | // Synthesize COND_E_AND_NP with two branches. | |||
2894 | BuildMI(&MBB, DL, get(X86::JNE_1)).addMBB(FBB); | |||
2895 | ++Count; | |||
2896 | BuildMI(&MBB, DL, get(X86::JNP_1)).addMBB(TBB); | |||
2897 | ++Count; | |||
2898 | break; | |||
2899 | default: { | |||
2900 | unsigned Opc = GetCondBranchFromCond(CC); | |||
2901 | BuildMI(&MBB, DL, get(Opc)).addMBB(TBB); | |||
2902 | ++Count; | |||
2903 | } | |||
2904 | } | |||
2905 | if (!FallThru) { | |||
2906 | // Two-way Conditional branch. Insert the second branch. | |||
2907 | BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB); | |||
2908 | ++Count; | |||
2909 | } | |||
2910 | return Count; | |||
2911 | } | |||
2912 | ||||
2913 | bool X86InstrInfo:: | |||
2914 | canInsertSelect(const MachineBasicBlock &MBB, | |||
2915 | ArrayRef<MachineOperand> Cond, | |||
2916 | unsigned TrueReg, unsigned FalseReg, | |||
2917 | int &CondCycles, int &TrueCycles, int &FalseCycles) const { | |||
2918 | // Not all subtargets have cmov instructions. | |||
2919 | if (!Subtarget.hasCMov()) | |||
2920 | return false; | |||
2921 | if (Cond.size() != 1) | |||
2922 | return false; | |||
2923 | // We cannot do the composite conditions, at least not in SSA form. | |||
2924 | if ((X86::CondCode)Cond[0].getImm() > X86::COND_S) | |||
2925 | return false; | |||
2926 | ||||
2927 | // Check register classes. | |||
2928 | const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); | |||
2929 | const TargetRegisterClass *RC = | |||
2930 | RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); | |||
2931 | if (!RC) | |||
2932 | return false; | |||
2933 | ||||
2934 | // We have cmov instructions for 16, 32, and 64 bit general purpose registers. | |||
2935 | if (X86::GR16RegClass.hasSubClassEq(RC) || | |||
2936 | X86::GR32RegClass.hasSubClassEq(RC) || | |||
2937 | X86::GR64RegClass.hasSubClassEq(RC)) { | |||
2938 | // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy | |||
2939 | // Bridge. Probably Ivy Bridge as well. | |||
2940 | CondCycles = 2; | |||
2941 | TrueCycles = 2; | |||
2942 | FalseCycles = 2; | |||
2943 | return true; | |||
2944 | } | |||
2945 | ||||
2946 | // Can't do vectors. | |||
2947 | return false; | |||
2948 | } | |||
2949 | ||||
2950 | void X86InstrInfo::insertSelect(MachineBasicBlock &MBB, | |||
2951 | MachineBasicBlock::iterator I, | |||
2952 | const DebugLoc &DL, unsigned DstReg, | |||
2953 | ArrayRef<MachineOperand> Cond, unsigned TrueReg, | |||
2954 | unsigned FalseReg) const { | |||
2955 | MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); | |||
2956 | const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); | |||
2957 | const TargetRegisterClass &RC = *MRI.getRegClass(DstReg); | |||
2958 | assert(Cond.size() == 1 && "Invalid Cond array")((Cond.size() == 1 && "Invalid Cond array") ? static_cast <void> (0) : __assert_fail ("Cond.size() == 1 && \"Invalid Cond array\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2958, __PRETTY_FUNCTION__)); | |||
2959 | unsigned Opc = getCMovFromCond((X86::CondCode)Cond[0].getImm(), | |||
2960 | TRI.getRegSizeInBits(RC) / 8, | |||
2961 | false /*HasMemoryOperand*/); | |||
2962 | BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(FalseReg).addReg(TrueReg); | |||
2963 | } | |||
2964 | ||||
2965 | /// Test if the given register is a physical h register. | |||
2966 | static bool isHReg(unsigned Reg) { | |||
2967 | return X86::GR8_ABCD_HRegClass.contains(Reg); | |||
2968 | } | |||
2969 | ||||
2970 | // Try and copy between VR128/VR64 and GR64 registers. | |||
2971 | static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, | |||
2972 | const X86Subtarget &Subtarget) { | |||
2973 | bool HasAVX = Subtarget.hasAVX(); | |||
2974 | bool HasAVX512 = Subtarget.hasAVX512(); | |||
2975 | ||||
2976 | // SrcReg(MaskReg) -> DestReg(GR64) | |||
2977 | // SrcReg(MaskReg) -> DestReg(GR32) | |||
2978 | ||||
2979 | // All KMASK RegClasses hold the same k registers, can be tested against anyone. | |||
2980 | if (X86::VK16RegClass.contains(SrcReg)) { | |||
2981 | if (X86::GR64RegClass.contains(DestReg)) { | |||
2982 | assert(Subtarget.hasBWI())((Subtarget.hasBWI()) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI()", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2982, __PRETTY_FUNCTION__)); | |||
2983 | return X86::KMOVQrk; | |||
2984 | } | |||
2985 | if (X86::GR32RegClass.contains(DestReg)) | |||
2986 | return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk; | |||
2987 | } | |||
2988 | ||||
2989 | // SrcReg(GR64) -> DestReg(MaskReg) | |||
2990 | // SrcReg(GR32) -> DestReg(MaskReg) | |||
2991 | ||||
2992 | // All KMASK RegClasses hold the same k registers, can be tested against anyone. | |||
2993 | if (X86::VK16RegClass.contains(DestReg)) { | |||
2994 | if (X86::GR64RegClass.contains(SrcReg)) { | |||
2995 | assert(Subtarget.hasBWI())((Subtarget.hasBWI()) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI()", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 2995, __PRETTY_FUNCTION__)); | |||
2996 | return X86::KMOVQkr; | |||
2997 | } | |||
2998 | if (X86::GR32RegClass.contains(SrcReg)) | |||
2999 | return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr; | |||
3000 | } | |||
3001 | ||||
3002 | ||||
3003 | // SrcReg(VR128) -> DestReg(GR64) | |||
3004 | // SrcReg(VR64) -> DestReg(GR64) | |||
3005 | // SrcReg(GR64) -> DestReg(VR128) | |||
3006 | // SrcReg(GR64) -> DestReg(VR64) | |||
3007 | ||||
3008 | if (X86::GR64RegClass.contains(DestReg)) { | |||
3009 | if (X86::VR128XRegClass.contains(SrcReg)) | |||
3010 | // Copy from a VR128 register to a GR64 register. | |||
3011 | return HasAVX512 ? X86::VMOVPQIto64Zrr : | |||
3012 | HasAVX ? X86::VMOVPQIto64rr : | |||
3013 | X86::MOVPQIto64rr; | |||
3014 | if (X86::VR64RegClass.contains(SrcReg)) | |||
3015 | // Copy from a VR64 register to a GR64 register. | |||
3016 | return X86::MMX_MOVD64from64rr; | |||
3017 | } else if (X86::GR64RegClass.contains(SrcReg)) { | |||
3018 | // Copy from a GR64 register to a VR128 register. | |||
3019 | if (X86::VR128XRegClass.contains(DestReg)) | |||
3020 | return HasAVX512 ? X86::VMOV64toPQIZrr : | |||
3021 | HasAVX ? X86::VMOV64toPQIrr : | |||
3022 | X86::MOV64toPQIrr; | |||
3023 | // Copy from a GR64 register to a VR64 register. | |||
3024 | if (X86::VR64RegClass.contains(DestReg)) | |||
3025 | return X86::MMX_MOVD64to64rr; | |||
3026 | } | |||
3027 | ||||
3028 | // SrcReg(FR32) -> DestReg(GR32) | |||
3029 | // SrcReg(GR32) -> DestReg(FR32) | |||
3030 | ||||
3031 | if (X86::GR32RegClass.contains(DestReg) && | |||
3032 | X86::FR32XRegClass.contains(SrcReg)) | |||
3033 | // Copy from a FR32 register to a GR32 register. | |||
3034 | return HasAVX512 ? X86::VMOVSS2DIZrr : | |||
3035 | HasAVX ? X86::VMOVSS2DIrr : | |||
3036 | X86::MOVSS2DIrr; | |||
3037 | ||||
3038 | if (X86::FR32XRegClass.contains(DestReg) && | |||
3039 | X86::GR32RegClass.contains(SrcReg)) | |||
3040 | // Copy from a GR32 register to a FR32 register. | |||
3041 | return HasAVX512 ? X86::VMOVDI2SSZrr : | |||
3042 | HasAVX ? X86::VMOVDI2SSrr : | |||
3043 | X86::MOVDI2SSrr; | |||
3044 | return 0; | |||
3045 | } | |||
3046 | ||||
3047 | void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB, | |||
3048 | MachineBasicBlock::iterator MI, | |||
3049 | const DebugLoc &DL, unsigned DestReg, | |||
3050 | unsigned SrcReg, bool KillSrc) const { | |||
3051 | // First deal with the normal symmetric copies. | |||
3052 | bool HasAVX = Subtarget.hasAVX(); | |||
3053 | bool HasVLX = Subtarget.hasVLX(); | |||
3054 | unsigned Opc = 0; | |||
3055 | if (X86::GR64RegClass.contains(DestReg, SrcReg)) | |||
3056 | Opc = X86::MOV64rr; | |||
3057 | else if (X86::GR32RegClass.contains(DestReg, SrcReg)) | |||
3058 | Opc = X86::MOV32rr; | |||
3059 | else if (X86::GR16RegClass.contains(DestReg, SrcReg)) | |||
3060 | Opc = X86::MOV16rr; | |||
3061 | else if (X86::GR8RegClass.contains(DestReg, SrcReg)) { | |||
3062 | // Copying to or from a physical H register on x86-64 requires a NOREX | |||
3063 | // move. Otherwise use a normal move. | |||
3064 | if ((isHReg(DestReg) || isHReg(SrcReg)) && | |||
3065 | Subtarget.is64Bit()) { | |||
3066 | Opc = X86::MOV8rr_NOREX; | |||
3067 | // Both operands must be encodable without an REX prefix. | |||
3068 | assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&((X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && "8-bit H register can not be copied outside GR8_NOREX") ? static_cast <void> (0) : __assert_fail ("X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && \"8-bit H register can not be copied outside GR8_NOREX\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3069, __PRETTY_FUNCTION__)) | |||
3069 | "8-bit H register can not be copied outside GR8_NOREX")((X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && "8-bit H register can not be copied outside GR8_NOREX") ? static_cast <void> (0) : __assert_fail ("X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && \"8-bit H register can not be copied outside GR8_NOREX\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3069, __PRETTY_FUNCTION__)); | |||
3070 | } else | |||
3071 | Opc = X86::MOV8rr; | |||
3072 | } | |||
3073 | else if (X86::VR64RegClass.contains(DestReg, SrcReg)) | |||
3074 | Opc = X86::MMX_MOVQ64rr; | |||
3075 | else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) { | |||
3076 | if (HasVLX) | |||
3077 | Opc = X86::VMOVAPSZ128rr; | |||
3078 | else if (X86::VR128RegClass.contains(DestReg, SrcReg)) | |||
3079 | Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr; | |||
3080 | else { | |||
3081 | // If this an extended register and we don't have VLX we need to use a | |||
3082 | // 512-bit move. | |||
3083 | Opc = X86::VMOVAPSZrr; | |||
3084 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
3085 | DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, | |||
3086 | &X86::VR512RegClass); | |||
3087 | SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, | |||
3088 | &X86::VR512RegClass); | |||
3089 | } | |||
3090 | } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) { | |||
3091 | if (HasVLX) | |||
3092 | Opc = X86::VMOVAPSZ256rr; | |||
3093 | else if (X86::VR256RegClass.contains(DestReg, SrcReg)) | |||
3094 | Opc = X86::VMOVAPSYrr; | |||
3095 | else { | |||
3096 | // If this an extended register and we don't have VLX we need to use a | |||
3097 | // 512-bit move. | |||
3098 | Opc = X86::VMOVAPSZrr; | |||
3099 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
3100 | DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, | |||
3101 | &X86::VR512RegClass); | |||
3102 | SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, | |||
3103 | &X86::VR512RegClass); | |||
3104 | } | |||
3105 | } else if (X86::VR512RegClass.contains(DestReg, SrcReg)) | |||
3106 | Opc = X86::VMOVAPSZrr; | |||
3107 | // All KMASK RegClasses hold the same k registers, can be tested against anyone. | |||
3108 | else if (X86::VK16RegClass.contains(DestReg, SrcReg)) | |||
3109 | Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk; | |||
3110 | if (!Opc) | |||
3111 | Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget); | |||
3112 | ||||
3113 | if (Opc) { | |||
3114 | BuildMI(MBB, MI, DL, get(Opc), DestReg) | |||
3115 | .addReg(SrcReg, getKillRegState(KillSrc)); | |||
3116 | return; | |||
3117 | } | |||
3118 | ||||
3119 | if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) { | |||
3120 | // FIXME: We use a fatal error here because historically LLVM has tried | |||
3121 | // lower some of these physreg copies and we want to ensure we get | |||
3122 | // reasonable bug reports if someone encounters a case no other testing | |||
3123 | // found. This path should be removed after the LLVM 7 release. | |||
3124 | report_fatal_error("Unable to copy EFLAGS physical register!"); | |||
3125 | } | |||
3126 | ||||
3127 | LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " << RI.getName(DestReg ) << '\n'; } } while (false) | |||
3128 | << RI.getName(DestReg) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " << RI.getName(DestReg ) << '\n'; } } while (false); | |||
3129 | report_fatal_error("Cannot emit physreg copy instruction"); | |||
3130 | } | |||
3131 | ||||
3132 | bool X86InstrInfo::isCopyInstrImpl(const MachineInstr &MI, | |||
3133 | const MachineOperand *&Src, | |||
3134 | const MachineOperand *&Dest) const { | |||
3135 | if (MI.isMoveReg()) { | |||
3136 | Dest = &MI.getOperand(0); | |||
3137 | Src = &MI.getOperand(1); | |||
3138 | return true; | |||
3139 | } | |||
3140 | return false; | |||
3141 | } | |||
3142 | ||||
3143 | static unsigned getLoadStoreRegOpcode(unsigned Reg, | |||
3144 | const TargetRegisterClass *RC, | |||
3145 | bool isStackAligned, | |||
3146 | const X86Subtarget &STI, | |||
3147 | bool load) { | |||
3148 | bool HasAVX = STI.hasAVX(); | |||
3149 | bool HasAVX512 = STI.hasAVX512(); | |||
3150 | bool HasVLX = STI.hasVLX(); | |||
3151 | ||||
3152 | switch (STI.getRegisterInfo()->getSpillSize(*RC)) { | |||
3153 | default: | |||
3154 | llvm_unreachable("Unknown spill size")::llvm::llvm_unreachable_internal("Unknown spill size", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3154); | |||
3155 | case 1: | |||
3156 | assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass")((X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::GR8RegClass.hasSubClassEq(RC) && \"Unknown 1-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3156, __PRETTY_FUNCTION__)); | |||
3157 | if (STI.is64Bit()) | |||
3158 | // Copying to or from a physical H register on x86-64 requires a NOREX | |||
3159 | // move. Otherwise use a normal move. | |||
3160 | if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC)) | |||
3161 | return load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX; | |||
3162 | return load ? X86::MOV8rm : X86::MOV8mr; | |||
3163 | case 2: | |||
3164 | if (X86::VK16RegClass.hasSubClassEq(RC)) | |||
3165 | return load ? X86::KMOVWkm : X86::KMOVWmk; | |||
3166 | assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass")((X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::GR16RegClass.hasSubClassEq(RC) && \"Unknown 2-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3166, __PRETTY_FUNCTION__)); | |||
3167 | return load ? X86::MOV16rm : X86::MOV16mr; | |||
3168 | case 4: | |||
3169 | if (X86::GR32RegClass.hasSubClassEq(RC)) | |||
3170 | return load ? X86::MOV32rm : X86::MOV32mr; | |||
3171 | if (X86::FR32XRegClass.hasSubClassEq(RC)) | |||
3172 | return load ? | |||
3173 | (HasAVX512 ? X86::VMOVSSZrm : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) : | |||
3174 | (HasAVX512 ? X86::VMOVSSZmr : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr); | |||
3175 | if (X86::RFP32RegClass.hasSubClassEq(RC)) | |||
3176 | return load ? X86::LD_Fp32m : X86::ST_Fp32m; | |||
3177 | if (X86::VK32RegClass.hasSubClassEq(RC)) { | |||
3178 | assert(STI.hasBWI() && "KMOVD requires BWI")((STI.hasBWI() && "KMOVD requires BWI") ? static_cast <void> (0) : __assert_fail ("STI.hasBWI() && \"KMOVD requires BWI\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3178, __PRETTY_FUNCTION__)); | |||
3179 | return load ? X86::KMOVDkm : X86::KMOVDmk; | |||
3180 | } | |||
3181 | llvm_unreachable("Unknown 4-byte regclass")::llvm::llvm_unreachable_internal("Unknown 4-byte regclass", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3181); | |||
3182 | case 8: | |||
3183 | if (X86::GR64RegClass.hasSubClassEq(RC)) | |||
3184 | return load ? X86::MOV64rm : X86::MOV64mr; | |||
3185 | if (X86::FR64XRegClass.hasSubClassEq(RC)) | |||
3186 | return load ? | |||
3187 | (HasAVX512 ? X86::VMOVSDZrm : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) : | |||
3188 | (HasAVX512 ? X86::VMOVSDZmr : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr); | |||
3189 | if (X86::VR64RegClass.hasSubClassEq(RC)) | |||
3190 | return load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr; | |||
3191 | if (X86::RFP64RegClass.hasSubClassEq(RC)) | |||
3192 | return load ? X86::LD_Fp64m : X86::ST_Fp64m; | |||
3193 | if (X86::VK64RegClass.hasSubClassEq(RC)) { | |||
3194 | assert(STI.hasBWI() && "KMOVQ requires BWI")((STI.hasBWI() && "KMOVQ requires BWI") ? static_cast <void> (0) : __assert_fail ("STI.hasBWI() && \"KMOVQ requires BWI\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3194, __PRETTY_FUNCTION__)); | |||
3195 | return load ? X86::KMOVQkm : X86::KMOVQmk; | |||
3196 | } | |||
3197 | llvm_unreachable("Unknown 8-byte regclass")::llvm::llvm_unreachable_internal("Unknown 8-byte regclass", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3197); | |||
3198 | case 10: | |||
3199 | assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass")((X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::RFP80RegClass.hasSubClassEq(RC) && \"Unknown 10-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3199, __PRETTY_FUNCTION__)); | |||
3200 | return load ? X86::LD_Fp80m : X86::ST_FpP80m; | |||
3201 | case 16: { | |||
3202 | if (X86::VR128XRegClass.hasSubClassEq(RC)) { | |||
3203 | // If stack is realigned we can use aligned stores. | |||
3204 | if (isStackAligned) | |||
3205 | return load ? | |||
3206 | (HasVLX ? X86::VMOVAPSZ128rm : | |||
3207 | HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX : | |||
3208 | HasAVX ? X86::VMOVAPSrm : | |||
3209 | X86::MOVAPSrm): | |||
3210 | (HasVLX ? X86::VMOVAPSZ128mr : | |||
3211 | HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX : | |||
3212 | HasAVX ? X86::VMOVAPSmr : | |||
3213 | X86::MOVAPSmr); | |||
3214 | else | |||
3215 | return load ? | |||
3216 | (HasVLX ? X86::VMOVUPSZ128rm : | |||
3217 | HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX : | |||
3218 | HasAVX ? X86::VMOVUPSrm : | |||
3219 | X86::MOVUPSrm): | |||
3220 | (HasVLX ? X86::VMOVUPSZ128mr : | |||
3221 | HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX : | |||
3222 | HasAVX ? X86::VMOVUPSmr : | |||
3223 | X86::MOVUPSmr); | |||
3224 | } | |||
3225 | if (X86::BNDRRegClass.hasSubClassEq(RC)) { | |||
3226 | if (STI.is64Bit()) | |||
3227 | return load ? X86::BNDMOV64rm : X86::BNDMOV64mr; | |||
3228 | else | |||
3229 | return load ? X86::BNDMOV32rm : X86::BNDMOV32mr; | |||
3230 | } | |||
3231 | llvm_unreachable("Unknown 16-byte regclass")::llvm::llvm_unreachable_internal("Unknown 16-byte regclass", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3231); | |||
3232 | } | |||
3233 | case 32: | |||
3234 | assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass")((X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::VR256XRegClass.hasSubClassEq(RC) && \"Unknown 32-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3234, __PRETTY_FUNCTION__)); | |||
3235 | // If stack is realigned we can use aligned stores. | |||
3236 | if (isStackAligned) | |||
3237 | return load ? | |||
3238 | (HasVLX ? X86::VMOVAPSZ256rm : | |||
3239 | HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX : | |||
3240 | X86::VMOVAPSYrm) : | |||
3241 | (HasVLX ? X86::VMOVAPSZ256mr : | |||
3242 | HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX : | |||
3243 | X86::VMOVAPSYmr); | |||
3244 | else | |||
3245 | return load ? | |||
3246 | (HasVLX ? X86::VMOVUPSZ256rm : | |||
3247 | HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX : | |||
3248 | X86::VMOVUPSYrm) : | |||
3249 | (HasVLX ? X86::VMOVUPSZ256mr : | |||
3250 | HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX : | |||
3251 | X86::VMOVUPSYmr); | |||
3252 | case 64: | |||
3253 | assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass")((X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::VR512RegClass.hasSubClassEq(RC) && \"Unknown 64-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3253, __PRETTY_FUNCTION__)); | |||
3254 | assert(STI.hasAVX512() && "Using 512-bit register requires AVX512")((STI.hasAVX512() && "Using 512-bit register requires AVX512" ) ? static_cast<void> (0) : __assert_fail ("STI.hasAVX512() && \"Using 512-bit register requires AVX512\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3254, __PRETTY_FUNCTION__)); | |||
3255 | if (isStackAligned) | |||
3256 | return load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; | |||
3257 | else | |||
3258 | return load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; | |||
3259 | } | |||
3260 | } | |||
3261 | ||||
3262 | bool X86InstrInfo::getMemOpBaseRegImmOfs(MachineInstr &MemOp, unsigned &BaseReg, | |||
3263 | int64_t &Offset, | |||
3264 | const TargetRegisterInfo *TRI) const { | |||
3265 | const MCInstrDesc &Desc = MemOp.getDesc(); | |||
3266 | int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); | |||
3267 | if (MemRefBegin < 0) | |||
3268 | return false; | |||
3269 | ||||
3270 | MemRefBegin += X86II::getOperandBias(Desc); | |||
3271 | ||||
3272 | MachineOperand &BaseMO = MemOp.getOperand(MemRefBegin + X86::AddrBaseReg); | |||
3273 | if (!BaseMO.isReg()) // Can be an MO_FrameIndex | |||
3274 | return false; | |||
3275 | ||||
3276 | BaseReg = BaseMO.getReg(); | |||
3277 | if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1) | |||
3278 | return false; | |||
3279 | ||||
3280 | if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() != | |||
3281 | X86::NoRegister) | |||
3282 | return false; | |||
3283 | ||||
3284 | const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp); | |||
3285 | ||||
3286 | // Displacement can be symbolic | |||
3287 | if (!DispMO.isImm()) | |||
3288 | return false; | |||
3289 | ||||
3290 | Offset = DispMO.getImm(); | |||
3291 | ||||
3292 | return true; | |||
3293 | } | |||
3294 | ||||
3295 | static unsigned getStoreRegOpcode(unsigned SrcReg, | |||
3296 | const TargetRegisterClass *RC, | |||
3297 | bool isStackAligned, | |||
3298 | const X86Subtarget &STI) { | |||
3299 | return getLoadStoreRegOpcode(SrcReg, RC, isStackAligned, STI, false); | |||
3300 | } | |||
3301 | ||||
3302 | ||||
3303 | static unsigned getLoadRegOpcode(unsigned DestReg, | |||
3304 | const TargetRegisterClass *RC, | |||
3305 | bool isStackAligned, | |||
3306 | const X86Subtarget &STI) { | |||
3307 | return getLoadStoreRegOpcode(DestReg, RC, isStackAligned, STI, true); | |||
3308 | } | |||
3309 | ||||
3310 | void X86InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, | |||
3311 | MachineBasicBlock::iterator MI, | |||
3312 | unsigned SrcReg, bool isKill, int FrameIdx, | |||
3313 | const TargetRegisterClass *RC, | |||
3314 | const TargetRegisterInfo *TRI) const { | |||
3315 | const MachineFunction &MF = *MBB.getParent(); | |||
3316 | assert(MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&((MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize (*RC) && "Stack slot too small for store") ? static_cast <void> (0) : __assert_fail ("MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && \"Stack slot too small for store\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3317, __PRETTY_FUNCTION__)) | |||
3317 | "Stack slot too small for store")((MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize (*RC) && "Stack slot too small for store") ? static_cast <void> (0) : __assert_fail ("MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && \"Stack slot too small for store\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3317, __PRETTY_FUNCTION__)); | |||
3318 | unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); | |||
3319 | bool isAligned = | |||
3320 | (Subtarget.getFrameLowering()->getStackAlignment() >= Alignment) || | |||
3321 | RI.canRealignStack(MF); | |||
3322 | unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); | |||
3323 | addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) | |||
3324 | .addReg(SrcReg, getKillRegState(isKill)); | |||
3325 | } | |||
3326 | ||||
3327 | void X86InstrInfo::storeRegToAddr( | |||
3328 | MachineFunction &MF, unsigned SrcReg, bool isKill, | |||
3329 | SmallVectorImpl<MachineOperand> &Addr, const TargetRegisterClass *RC, | |||
3330 | ArrayRef<MachineMemOperand *> MMOs, | |||
3331 | SmallVectorImpl<MachineInstr *> &NewMIs) const { | |||
3332 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
3333 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
3334 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
3335 | unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); | |||
3336 | DebugLoc DL; | |||
3337 | MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc)); | |||
3338 | for (unsigned i = 0, e = Addr.size(); i != e; ++i) | |||
3339 | MIB.add(Addr[i]); | |||
3340 | MIB.addReg(SrcReg, getKillRegState(isKill)); | |||
3341 | MIB.setMemRefs(MMOs); | |||
3342 | NewMIs.push_back(MIB); | |||
3343 | } | |||
3344 | ||||
3345 | ||||
3346 | void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, | |||
3347 | MachineBasicBlock::iterator MI, | |||
3348 | unsigned DestReg, int FrameIdx, | |||
3349 | const TargetRegisterClass *RC, | |||
3350 | const TargetRegisterInfo *TRI) const { | |||
3351 | const MachineFunction &MF = *MBB.getParent(); | |||
3352 | unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); | |||
3353 | bool isAligned = | |||
3354 | (Subtarget.getFrameLowering()->getStackAlignment() >= Alignment) || | |||
3355 | RI.canRealignStack(MF); | |||
3356 | unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); | |||
3357 | addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), FrameIdx); | |||
3358 | } | |||
3359 | ||||
3360 | void X86InstrInfo::loadRegFromAddr( | |||
3361 | MachineFunction &MF, unsigned DestReg, | |||
3362 | SmallVectorImpl<MachineOperand> &Addr, const TargetRegisterClass *RC, | |||
3363 | ArrayRef<MachineMemOperand *> MMOs, | |||
3364 | SmallVectorImpl<MachineInstr *> &NewMIs) const { | |||
3365 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
3366 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
3367 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
3368 | unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); | |||
3369 | DebugLoc DL; | |||
3370 | MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), DestReg); | |||
3371 | for (unsigned i = 0, e = Addr.size(); i != e; ++i) | |||
3372 | MIB.add(Addr[i]); | |||
3373 | MIB.setMemRefs(MMOs); | |||
3374 | NewMIs.push_back(MIB); | |||
3375 | } | |||
3376 | ||||
3377 | bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg, | |||
3378 | unsigned &SrcReg2, int &CmpMask, | |||
3379 | int &CmpValue) const { | |||
3380 | switch (MI.getOpcode()) { | |||
3381 | default: break; | |||
3382 | case X86::CMP64ri32: | |||
3383 | case X86::CMP64ri8: | |||
3384 | case X86::CMP32ri: | |||
3385 | case X86::CMP32ri8: | |||
3386 | case X86::CMP16ri: | |||
3387 | case X86::CMP16ri8: | |||
3388 | case X86::CMP8ri: | |||
3389 | SrcReg = MI.getOperand(0).getReg(); | |||
3390 | SrcReg2 = 0; | |||
3391 | if (MI.getOperand(1).isImm()) { | |||
3392 | CmpMask = ~0; | |||
3393 | CmpValue = MI.getOperand(1).getImm(); | |||
3394 | } else { | |||
3395 | CmpMask = CmpValue = 0; | |||
3396 | } | |||
3397 | return true; | |||
3398 | // A SUB can be used to perform comparison. | |||
3399 | case X86::SUB64rm: | |||
3400 | case X86::SUB32rm: | |||
3401 | case X86::SUB16rm: | |||
3402 | case X86::SUB8rm: | |||
3403 | SrcReg = MI.getOperand(1).getReg(); | |||
3404 | SrcReg2 = 0; | |||
3405 | CmpMask = 0; | |||
3406 | CmpValue = 0; | |||
3407 | return true; | |||
3408 | case X86::SUB64rr: | |||
3409 | case X86::SUB32rr: | |||
3410 | case X86::SUB16rr: | |||
3411 | case X86::SUB8rr: | |||
3412 | SrcReg = MI.getOperand(1).getReg(); | |||
3413 | SrcReg2 = MI.getOperand(2).getReg(); | |||
3414 | CmpMask = 0; | |||
3415 | CmpValue = 0; | |||
3416 | return true; | |||
3417 | case X86::SUB64ri32: | |||
3418 | case X86::SUB64ri8: | |||
3419 | case X86::SUB32ri: | |||
3420 | case X86::SUB32ri8: | |||
3421 | case X86::SUB16ri: | |||
3422 | case X86::SUB16ri8: | |||
3423 | case X86::SUB8ri: | |||
3424 | SrcReg = MI.getOperand(1).getReg(); | |||
3425 | SrcReg2 = 0; | |||
3426 | if (MI.getOperand(2).isImm()) { | |||
3427 | CmpMask = ~0; | |||
3428 | CmpValue = MI.getOperand(2).getImm(); | |||
3429 | } else { | |||
3430 | CmpMask = CmpValue = 0; | |||
3431 | } | |||
3432 | return true; | |||
3433 | case X86::CMP64rr: | |||
3434 | case X86::CMP32rr: | |||
3435 | case X86::CMP16rr: | |||
3436 | case X86::CMP8rr: | |||
3437 | SrcReg = MI.getOperand(0).getReg(); | |||
3438 | SrcReg2 = MI.getOperand(1).getReg(); | |||
3439 | CmpMask = 0; | |||
3440 | CmpValue = 0; | |||
3441 | return true; | |||
3442 | case X86::TEST8rr: | |||
3443 | case X86::TEST16rr: | |||
3444 | case X86::TEST32rr: | |||
3445 | case X86::TEST64rr: | |||
3446 | SrcReg = MI.getOperand(0).getReg(); | |||
3447 | if (MI.getOperand(1).getReg() != SrcReg) | |||
3448 | return false; | |||
3449 | // Compare against zero. | |||
3450 | SrcReg2 = 0; | |||
3451 | CmpMask = ~0; | |||
3452 | CmpValue = 0; | |||
3453 | return true; | |||
3454 | } | |||
3455 | return false; | |||
3456 | } | |||
3457 | ||||
3458 | /// Check whether the first instruction, whose only | |||
3459 | /// purpose is to update flags, can be made redundant. | |||
3460 | /// CMPrr can be made redundant by SUBrr if the operands are the same. | |||
3461 | /// This function can be extended later on. | |||
3462 | /// SrcReg, SrcRegs: register operands for FlagI. | |||
3463 | /// ImmValue: immediate for FlagI if it takes an immediate. | |||
3464 | inline static bool isRedundantFlagInstr(MachineInstr &FlagI, unsigned SrcReg, | |||
3465 | unsigned SrcReg2, int ImmMask, | |||
3466 | int ImmValue, MachineInstr &OI) { | |||
3467 | if (((FlagI.getOpcode() == X86::CMP64rr && OI.getOpcode() == X86::SUB64rr) || | |||
3468 | (FlagI.getOpcode() == X86::CMP32rr && OI.getOpcode() == X86::SUB32rr) || | |||
3469 | (FlagI.getOpcode() == X86::CMP16rr && OI.getOpcode() == X86::SUB16rr) || | |||
3470 | (FlagI.getOpcode() == X86::CMP8rr && OI.getOpcode() == X86::SUB8rr)) && | |||
3471 | ((OI.getOperand(1).getReg() == SrcReg && | |||
3472 | OI.getOperand(2).getReg() == SrcReg2) || | |||
3473 | (OI.getOperand(1).getReg() == SrcReg2 && | |||
3474 | OI.getOperand(2).getReg() == SrcReg))) | |||
3475 | return true; | |||
3476 | ||||
3477 | if (ImmMask != 0 && | |||
3478 | ((FlagI.getOpcode() == X86::CMP64ri32 && | |||
3479 | OI.getOpcode() == X86::SUB64ri32) || | |||
3480 | (FlagI.getOpcode() == X86::CMP64ri8 && | |||
3481 | OI.getOpcode() == X86::SUB64ri8) || | |||
3482 | (FlagI.getOpcode() == X86::CMP32ri && OI.getOpcode() == X86::SUB32ri) || | |||
3483 | (FlagI.getOpcode() == X86::CMP32ri8 && | |||
3484 | OI.getOpcode() == X86::SUB32ri8) || | |||
3485 | (FlagI.getOpcode() == X86::CMP16ri && OI.getOpcode() == X86::SUB16ri) || | |||
3486 | (FlagI.getOpcode() == X86::CMP16ri8 && | |||
3487 | OI.getOpcode() == X86::SUB16ri8) || | |||
3488 | (FlagI.getOpcode() == X86::CMP8ri && OI.getOpcode() == X86::SUB8ri)) && | |||
3489 | OI.getOperand(1).getReg() == SrcReg && | |||
3490 | OI.getOperand(2).getImm() == ImmValue) | |||
3491 | return true; | |||
3492 | return false; | |||
3493 | } | |||
3494 | ||||
3495 | /// Check whether the definition can be converted | |||
3496 | /// to remove a comparison against zero. | |||
3497 | inline static bool isDefConvertible(MachineInstr &MI) { | |||
3498 | switch (MI.getOpcode()) { | |||
3499 | default: return false; | |||
3500 | ||||
3501 | // The shift instructions only modify ZF if their shift count is non-zero. | |||
3502 | // N.B.: The processor truncates the shift count depending on the encoding. | |||
3503 | case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri:case X86::SAR64ri: | |||
3504 | case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri:case X86::SHR64ri: | |||
3505 | return getTruncatedShiftCount(MI, 2) != 0; | |||
3506 | ||||
3507 | // Some left shift instructions can be turned into LEA instructions but only | |||
3508 | // if their flags aren't used. Avoid transforming such instructions. | |||
3509 | case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri:case X86::SHL64ri:{ | |||
3510 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
3511 | if (isTruncatedShiftCountForLEA(ShAmt)) return false; | |||
3512 | return ShAmt != 0; | |||
3513 | } | |||
3514 | ||||
3515 | case X86::SHRD16rri8:case X86::SHRD32rri8:case X86::SHRD64rri8: | |||
3516 | case X86::SHLD16rri8:case X86::SHLD32rri8:case X86::SHLD64rri8: | |||
3517 | return getTruncatedShiftCount(MI, 3) != 0; | |||
3518 | ||||
3519 | case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB32ri: | |||
3520 | case X86::SUB32ri8: case X86::SUB16ri: case X86::SUB16ri8: | |||
3521 | case X86::SUB8ri: case X86::SUB64rr: case X86::SUB32rr: | |||
3522 | case X86::SUB16rr: case X86::SUB8rr: case X86::SUB64rm: | |||
3523 | case X86::SUB32rm: case X86::SUB16rm: case X86::SUB8rm: | |||
3524 | case X86::DEC64r: case X86::DEC32r: case X86::DEC16r: case X86::DEC8r: | |||
3525 | case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri: | |||
3526 | case X86::ADD32ri8: case X86::ADD16ri: case X86::ADD16ri8: | |||
3527 | case X86::ADD8ri: case X86::ADD64rr: case X86::ADD32rr: | |||
3528 | case X86::ADD16rr: case X86::ADD8rr: case X86::ADD64rm: | |||
3529 | case X86::ADD32rm: case X86::ADD16rm: case X86::ADD8rm: | |||
3530 | case X86::INC64r: case X86::INC32r: case X86::INC16r: case X86::INC8r: | |||
3531 | case X86::AND64ri32: case X86::AND64ri8: case X86::AND32ri: | |||
3532 | case X86::AND32ri8: case X86::AND16ri: case X86::AND16ri8: | |||
3533 | case X86::AND8ri: case X86::AND64rr: case X86::AND32rr: | |||
3534 | case X86::AND16rr: case X86::AND8rr: case X86::AND64rm: | |||
3535 | case X86::AND32rm: case X86::AND16rm: case X86::AND8rm: | |||
3536 | case X86::XOR64ri32: case X86::XOR64ri8: case X86::XOR32ri: | |||
3537 | case X86::XOR32ri8: case X86::XOR16ri: case X86::XOR16ri8: | |||
3538 | case X86::XOR8ri: case X86::XOR64rr: case X86::XOR32rr: | |||
3539 | case X86::XOR16rr: case X86::XOR8rr: case X86::XOR64rm: | |||
3540 | case X86::XOR32rm: case X86::XOR16rm: case X86::XOR8rm: | |||
3541 | case X86::OR64ri32: case X86::OR64ri8: case X86::OR32ri: | |||
3542 | case X86::OR32ri8: case X86::OR16ri: case X86::OR16ri8: | |||
3543 | case X86::OR8ri: case X86::OR64rr: case X86::OR32rr: | |||
3544 | case X86::OR16rr: case X86::OR8rr: case X86::OR64rm: | |||
3545 | case X86::OR32rm: case X86::OR16rm: case X86::OR8rm: | |||
3546 | case X86::ADC64ri32: case X86::ADC64ri8: case X86::ADC32ri: | |||
3547 | case X86::ADC32ri8: case X86::ADC16ri: case X86::ADC16ri8: | |||
3548 | case X86::ADC8ri: case X86::ADC64rr: case X86::ADC32rr: | |||
3549 | case X86::ADC16rr: case X86::ADC8rr: case X86::ADC64rm: | |||
3550 | case X86::ADC32rm: case X86::ADC16rm: case X86::ADC8rm: | |||
3551 | case X86::SBB64ri32: case X86::SBB64ri8: case X86::SBB32ri: | |||
3552 | case X86::SBB32ri8: case X86::SBB16ri: case X86::SBB16ri8: | |||
3553 | case X86::SBB8ri: case X86::SBB64rr: case X86::SBB32rr: | |||
3554 | case X86::SBB16rr: case X86::SBB8rr: case X86::SBB64rm: | |||
3555 | case X86::SBB32rm: case X86::SBB16rm: case X86::SBB8rm: | |||
3556 | case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r: | |||
3557 | case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1:case X86::SAR64r1: | |||
3558 | case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1:case X86::SHR64r1: | |||
3559 | case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1:case X86::SHL64r1: | |||
3560 | case X86::ANDN32rr: case X86::ANDN32rm: | |||
3561 | case X86::ANDN64rr: case X86::ANDN64rm: | |||
3562 | case X86::BEXTR32rr: case X86::BEXTR64rr: | |||
3563 | case X86::BEXTR32rm: case X86::BEXTR64rm: | |||
3564 | case X86::BLSI32rr: case X86::BLSI32rm: | |||
3565 | case X86::BLSI64rr: case X86::BLSI64rm: | |||
3566 | case X86::BLSMSK32rr:case X86::BLSMSK32rm: | |||
3567 | case X86::BLSMSK64rr:case X86::BLSMSK64rm: | |||
3568 | case X86::BLSR32rr: case X86::BLSR32rm: | |||
3569 | case X86::BLSR64rr: case X86::BLSR64rm: | |||
3570 | case X86::BZHI32rr: case X86::BZHI32rm: | |||
3571 | case X86::BZHI64rr: case X86::BZHI64rm: | |||
3572 | case X86::LZCNT16rr: case X86::LZCNT16rm: | |||
3573 | case X86::LZCNT32rr: case X86::LZCNT32rm: | |||
3574 | case X86::LZCNT64rr: case X86::LZCNT64rm: | |||
3575 | case X86::POPCNT16rr:case X86::POPCNT16rm: | |||
3576 | case X86::POPCNT32rr:case X86::POPCNT32rm: | |||
3577 | case X86::POPCNT64rr:case X86::POPCNT64rm: | |||
3578 | case X86::TZCNT16rr: case X86::TZCNT16rm: | |||
3579 | case X86::TZCNT32rr: case X86::TZCNT32rm: | |||
3580 | case X86::TZCNT64rr: case X86::TZCNT64rm: | |||
3581 | case X86::BEXTRI32ri: case X86::BEXTRI32mi: | |||
3582 | case X86::BEXTRI64ri: case X86::BEXTRI64mi: | |||
3583 | case X86::BLCFILL32rr: case X86::BLCFILL32rm: | |||
3584 | case X86::BLCFILL64rr: case X86::BLCFILL64rm: | |||
3585 | case X86::BLCI32rr: case X86::BLCI32rm: | |||
3586 | case X86::BLCI64rr: case X86::BLCI64rm: | |||
3587 | case X86::BLCIC32rr: case X86::BLCIC32rm: | |||
3588 | case X86::BLCIC64rr: case X86::BLCIC64rm: | |||
3589 | case X86::BLCMSK32rr: case X86::BLCMSK32rm: | |||
3590 | case X86::BLCMSK64rr: case X86::BLCMSK64rm: | |||
3591 | case X86::BLCS32rr: case X86::BLCS32rm: | |||
3592 | case X86::BLCS64rr: case X86::BLCS64rm: | |||
3593 | case X86::BLSFILL32rr: case X86::BLSFILL32rm: | |||
3594 | case X86::BLSFILL64rr: case X86::BLSFILL64rm: | |||
3595 | case X86::BLSIC32rr: case X86::BLSIC32rm: | |||
3596 | case X86::BLSIC64rr: case X86::BLSIC64rm: | |||
3597 | return true; | |||
3598 | } | |||
3599 | } | |||
3600 | ||||
3601 | /// Check whether the use can be converted to remove a comparison against zero. | |||
3602 | static X86::CondCode isUseDefConvertible(MachineInstr &MI) { | |||
3603 | switch (MI.getOpcode()) { | |||
3604 | default: return X86::COND_INVALID; | |||
3605 | case X86::LZCNT16rr: case X86::LZCNT16rm: | |||
3606 | case X86::LZCNT32rr: case X86::LZCNT32rm: | |||
3607 | case X86::LZCNT64rr: case X86::LZCNT64rm: | |||
3608 | return X86::COND_B; | |||
3609 | case X86::POPCNT16rr:case X86::POPCNT16rm: | |||
3610 | case X86::POPCNT32rr:case X86::POPCNT32rm: | |||
3611 | case X86::POPCNT64rr:case X86::POPCNT64rm: | |||
3612 | return X86::COND_E; | |||
3613 | case X86::TZCNT16rr: case X86::TZCNT16rm: | |||
3614 | case X86::TZCNT32rr: case X86::TZCNT32rm: | |||
3615 | case X86::TZCNT64rr: case X86::TZCNT64rm: | |||
3616 | return X86::COND_B; | |||
3617 | case X86::BSF16rr: | |||
3618 | case X86::BSF16rm: | |||
3619 | case X86::BSF32rr: | |||
3620 | case X86::BSF32rm: | |||
3621 | case X86::BSF64rr: | |||
3622 | case X86::BSF64rm: | |||
3623 | return X86::COND_E; | |||
3624 | } | |||
3625 | } | |||
3626 | ||||
3627 | /// Check if there exists an earlier instruction that | |||
3628 | /// operates on the same source operands and sets flags in the same way as | |||
3629 | /// Compare; remove Compare if possible. | |||
3630 | bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg, | |||
3631 | unsigned SrcReg2, int CmpMask, | |||
3632 | int CmpValue, | |||
3633 | const MachineRegisterInfo *MRI) const { | |||
3634 | // Check whether we can replace SUB with CMP. | |||
3635 | unsigned NewOpcode = 0; | |||
3636 | switch (CmpInstr.getOpcode()) { | |||
| ||||
3637 | default: break; | |||
3638 | case X86::SUB64ri32: | |||
3639 | case X86::SUB64ri8: | |||
3640 | case X86::SUB32ri: | |||
3641 | case X86::SUB32ri8: | |||
3642 | case X86::SUB16ri: | |||
3643 | case X86::SUB16ri8: | |||
3644 | case X86::SUB8ri: | |||
3645 | case X86::SUB64rm: | |||
3646 | case X86::SUB32rm: | |||
3647 | case X86::SUB16rm: | |||
3648 | case X86::SUB8rm: | |||
3649 | case X86::SUB64rr: | |||
3650 | case X86::SUB32rr: | |||
3651 | case X86::SUB16rr: | |||
3652 | case X86::SUB8rr: { | |||
3653 | if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) | |||
3654 | return false; | |||
3655 | // There is no use of the destination register, we can replace SUB with CMP. | |||
3656 | switch (CmpInstr.getOpcode()) { | |||
3657 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3657); | |||
3658 | case X86::SUB64rm: NewOpcode = X86::CMP64rm; break; | |||
3659 | case X86::SUB32rm: NewOpcode = X86::CMP32rm; break; | |||
3660 | case X86::SUB16rm: NewOpcode = X86::CMP16rm; break; | |||
3661 | case X86::SUB8rm: NewOpcode = X86::CMP8rm; break; | |||
3662 | case X86::SUB64rr: NewOpcode = X86::CMP64rr; break; | |||
3663 | case X86::SUB32rr: NewOpcode = X86::CMP32rr; break; | |||
3664 | case X86::SUB16rr: NewOpcode = X86::CMP16rr; break; | |||
3665 | case X86::SUB8rr: NewOpcode = X86::CMP8rr; break; | |||
3666 | case X86::SUB64ri32: NewOpcode = X86::CMP64ri32; break; | |||
3667 | case X86::SUB64ri8: NewOpcode = X86::CMP64ri8; break; | |||
3668 | case X86::SUB32ri: NewOpcode = X86::CMP32ri; break; | |||
3669 | case X86::SUB32ri8: NewOpcode = X86::CMP32ri8; break; | |||
3670 | case X86::SUB16ri: NewOpcode = X86::CMP16ri; break; | |||
3671 | case X86::SUB16ri8: NewOpcode = X86::CMP16ri8; break; | |||
3672 | case X86::SUB8ri: NewOpcode = X86::CMP8ri; break; | |||
3673 | } | |||
3674 | CmpInstr.setDesc(get(NewOpcode)); | |||
3675 | CmpInstr.RemoveOperand(0); | |||
3676 | // Fall through to optimize Cmp if Cmp is CMPrr or CMPri. | |||
3677 | if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm || | |||
3678 | NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm) | |||
3679 | return false; | |||
3680 | } | |||
3681 | } | |||
3682 | ||||
3683 | // Get the unique definition of SrcReg. | |||
3684 | MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); | |||
3685 | if (!MI) return false; | |||
3686 | ||||
3687 | // CmpInstr is the first instruction of the BB. | |||
3688 | MachineBasicBlock::iterator I = CmpInstr, Def = MI; | |||
3689 | ||||
3690 | // If we are comparing against zero, check whether we can use MI to update | |||
3691 | // EFLAGS. If MI is not in the same BB as CmpInstr, do not optimize. | |||
3692 | bool IsCmpZero = (CmpMask != 0 && CmpValue == 0); | |||
3693 | if (IsCmpZero && MI->getParent() != CmpInstr.getParent()) | |||
3694 | return false; | |||
3695 | ||||
3696 | // If we have a use of the source register between the def and our compare | |||
3697 | // instruction we can eliminate the compare iff the use sets EFLAGS in the | |||
3698 | // right way. | |||
3699 | bool ShouldUpdateCC = false; | |||
3700 | X86::CondCode NewCC = X86::COND_INVALID; | |||
3701 | if (IsCmpZero && !isDefConvertible(*MI)) { | |||
3702 | // Scan forward from the use until we hit the use we're looking for or the | |||
3703 | // compare instruction. | |||
3704 | for (MachineBasicBlock::iterator J = MI;; ++J) { | |||
3705 | // Do we have a convertible instruction? | |||
3706 | NewCC = isUseDefConvertible(*J); | |||
3707 | if (NewCC != X86::COND_INVALID && J->getOperand(1).isReg() && | |||
3708 | J->getOperand(1).getReg() == SrcReg) { | |||
3709 | assert(J->definesRegister(X86::EFLAGS) && "Must be an EFLAGS def!")((J->definesRegister(X86::EFLAGS) && "Must be an EFLAGS def!" ) ? static_cast<void> (0) : __assert_fail ("J->definesRegister(X86::EFLAGS) && \"Must be an EFLAGS def!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3709, __PRETTY_FUNCTION__)); | |||
3710 | ShouldUpdateCC = true; // Update CC later on. | |||
3711 | // This is not a def of SrcReg, but still a def of EFLAGS. Keep going | |||
3712 | // with the new def. | |||
3713 | Def = J; | |||
3714 | MI = &*Def; | |||
3715 | break; | |||
3716 | } | |||
3717 | ||||
3718 | if (J == I) | |||
3719 | return false; | |||
3720 | } | |||
3721 | } | |||
3722 | ||||
3723 | // We are searching for an earlier instruction that can make CmpInstr | |||
3724 | // redundant and that instruction will be saved in Sub. | |||
3725 | MachineInstr *Sub = nullptr; | |||
3726 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
3727 | ||||
3728 | // We iterate backward, starting from the instruction before CmpInstr and | |||
3729 | // stop when reaching the definition of a source register or done with the BB. | |||
3730 | // RI points to the instruction before CmpInstr. | |||
3731 | // If the definition is in this basic block, RE points to the definition; | |||
3732 | // otherwise, RE is the rend of the basic block. | |||
3733 | MachineBasicBlock::reverse_iterator | |||
3734 | RI = ++I.getReverse(), | |||
3735 | RE = CmpInstr.getParent() == MI->getParent() | |||
3736 | ? Def.getReverse() /* points to MI */ | |||
3737 | : CmpInstr.getParent()->rend(); | |||
3738 | MachineInstr *Movr0Inst = nullptr; | |||
3739 | for (; RI != RE; ++RI) { | |||
3740 | MachineInstr &Instr = *RI; | |||
3741 | // Check whether CmpInstr can be made redundant by the current instruction. | |||
3742 | if (!IsCmpZero && isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, | |||
3743 | CmpValue, Instr)) { | |||
3744 | Sub = &Instr; | |||
3745 | break; | |||
3746 | } | |||
3747 | ||||
3748 | if (Instr.modifiesRegister(X86::EFLAGS, TRI) || | |||
3749 | Instr.readsRegister(X86::EFLAGS, TRI)) { | |||
3750 | // This instruction modifies or uses EFLAGS. | |||
3751 | ||||
3752 | // MOV32r0 etc. are implemented with xor which clobbers condition code. | |||
3753 | // They are safe to move up, if the definition to EFLAGS is dead and | |||
3754 | // earlier instructions do not read or write EFLAGS. | |||
3755 | if (!Movr0Inst && | |||
3756 | (Instr.getOpcode() == X86::MOV32r0 || | |||
3757 | Instr.getOpcode() == X86::MOV64r0) && | |||
3758 | Instr.registerDefIsDead(X86::EFLAGS, TRI)) { | |||
3759 | Movr0Inst = &Instr; | |||
3760 | continue; | |||
3761 | } | |||
3762 | ||||
3763 | // We can't remove CmpInstr. | |||
3764 | return false; | |||
3765 | } | |||
3766 | } | |||
3767 | ||||
3768 | // Return false if no candidates exist. | |||
3769 | if (!IsCmpZero && !Sub) | |||
3770 | return false; | |||
3771 | ||||
3772 | bool IsSwapped = (SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 && | |||
| ||||
3773 | Sub->getOperand(2).getReg() == SrcReg); | |||
3774 | ||||
3775 | // Scan forward from the instruction after CmpInstr for uses of EFLAGS. | |||
3776 | // It is safe to remove CmpInstr if EFLAGS is redefined or killed. | |||
3777 | // If we are done with the basic block, we need to check whether EFLAGS is | |||
3778 | // live-out. | |||
3779 | bool IsSafe = false; | |||
3780 | SmallVector<std::pair<MachineInstr*, unsigned /*NewOpc*/>, 4> OpsToUpdate; | |||
3781 | MachineBasicBlock::iterator E = CmpInstr.getParent()->end(); | |||
3782 | for (++I; I != E; ++I) { | |||
3783 | const MachineInstr &Instr = *I; | |||
3784 | bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI); | |||
3785 | bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI); | |||
3786 | // We should check the usage if this instruction uses and updates EFLAGS. | |||
3787 | if (!UseEFLAGS && ModifyEFLAGS) { | |||
3788 | // It is safe to remove CmpInstr if EFLAGS is updated again. | |||
3789 | IsSafe = true; | |||
3790 | break; | |||
3791 | } | |||
3792 | if (!UseEFLAGS && !ModifyEFLAGS) | |||
3793 | continue; | |||
3794 | ||||
3795 | // EFLAGS is used by this instruction. | |||
3796 | X86::CondCode OldCC = X86::COND_INVALID; | |||
3797 | bool OpcIsSET = false; | |||
3798 | if (IsCmpZero || IsSwapped) { | |||
3799 | // We decode the condition code from opcode. | |||
3800 | if (Instr.isBranch()) | |||
3801 | OldCC = X86::getCondFromBranchOpc(Instr.getOpcode()); | |||
3802 | else { | |||
3803 | OldCC = X86::getCondFromSETOpc(Instr.getOpcode()); | |||
3804 | if (OldCC != X86::COND_INVALID) | |||
3805 | OpcIsSET = true; | |||
3806 | else | |||
3807 | OldCC = X86::getCondFromCMovOpc(Instr.getOpcode()); | |||
3808 | } | |||
3809 | if (OldCC == X86::COND_INVALID) return false; | |||
3810 | } | |||
3811 | X86::CondCode ReplacementCC = X86::COND_INVALID; | |||
3812 | if (IsCmpZero) { | |||
3813 | switch (OldCC) { | |||
3814 | default: break; | |||
3815 | case X86::COND_A: case X86::COND_AE: | |||
3816 | case X86::COND_B: case X86::COND_BE: | |||
3817 | case X86::COND_G: case X86::COND_GE: | |||
3818 | case X86::COND_L: case X86::COND_LE: | |||
3819 | case X86::COND_O: case X86::COND_NO: | |||
3820 | // CF and OF are used, we can't perform this optimization. | |||
3821 | return false; | |||
3822 | } | |||
3823 | ||||
3824 | // If we're updating the condition code check if we have to reverse the | |||
3825 | // condition. | |||
3826 | if (ShouldUpdateCC) | |||
3827 | switch (OldCC) { | |||
3828 | default: | |||
3829 | return false; | |||
3830 | case X86::COND_E: | |||
3831 | ReplacementCC = NewCC; | |||
3832 | break; | |||
3833 | case X86::COND_NE: | |||
3834 | ReplacementCC = GetOppositeBranchCondition(NewCC); | |||
3835 | break; | |||
3836 | } | |||
3837 | } else if (IsSwapped) { | |||
3838 | // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs | |||
3839 | // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. | |||
3840 | // We swap the condition code and synthesize the new opcode. | |||
3841 | ReplacementCC = getSwappedCondition(OldCC); | |||
3842 | if (ReplacementCC == X86::COND_INVALID) return false; | |||
3843 | } | |||
3844 | ||||
3845 | if ((ShouldUpdateCC || IsSwapped) && ReplacementCC != OldCC) { | |||
3846 | // Synthesize the new opcode. | |||
3847 | bool HasMemoryOperand = Instr.hasOneMemOperand(); | |||
3848 | unsigned NewOpc; | |||
3849 | if (Instr.isBranch()) | |||
3850 | NewOpc = GetCondBranchFromCond(ReplacementCC); | |||
3851 | else if(OpcIsSET) | |||
3852 | NewOpc = getSETFromCond(ReplacementCC, HasMemoryOperand); | |||
3853 | else { | |||
3854 | unsigned DstReg = Instr.getOperand(0).getReg(); | |||
3855 | const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); | |||
3856 | NewOpc = getCMovFromCond(ReplacementCC, TRI->getRegSizeInBits(*DstRC)/8, | |||
3857 | HasMemoryOperand); | |||
3858 | } | |||
3859 | ||||
3860 | // Push the MachineInstr to OpsToUpdate. | |||
3861 | // If it is safe to remove CmpInstr, the condition code of these | |||
3862 | // instructions will be modified. | |||
3863 | OpsToUpdate.push_back(std::make_pair(&*I, NewOpc)); | |||
3864 | } | |||
3865 | if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) { | |||
3866 | // It is safe to remove CmpInstr if EFLAGS is updated again or killed. | |||
3867 | IsSafe = true; | |||
3868 | break; | |||
3869 | } | |||
3870 | } | |||
3871 | ||||
3872 | // If EFLAGS is not killed nor re-defined, we should check whether it is | |||
3873 | // live-out. If it is live-out, do not optimize. | |||
3874 | if ((IsCmpZero || IsSwapped) && !IsSafe) { | |||
3875 | MachineBasicBlock *MBB = CmpInstr.getParent(); | |||
3876 | for (MachineBasicBlock *Successor : MBB->successors()) | |||
3877 | if (Successor->isLiveIn(X86::EFLAGS)) | |||
3878 | return false; | |||
3879 | } | |||
3880 | ||||
3881 | // The instruction to be updated is either Sub or MI. | |||
3882 | Sub = IsCmpZero ? MI : Sub; | |||
3883 | // Move Movr0Inst to the appropriate place before Sub. | |||
3884 | if (Movr0Inst) { | |||
3885 | // Look backwards until we find a def that doesn't use the current EFLAGS. | |||
3886 | Def = Sub; | |||
3887 | MachineBasicBlock::reverse_iterator InsertI = Def.getReverse(), | |||
3888 | InsertE = Sub->getParent()->rend(); | |||
3889 | for (; InsertI != InsertE; ++InsertI) { | |||
3890 | MachineInstr *Instr = &*InsertI; | |||
3891 | if (!Instr->readsRegister(X86::EFLAGS, TRI) && | |||
3892 | Instr->modifiesRegister(X86::EFLAGS, TRI)) { | |||
3893 | Sub->getParent()->remove(Movr0Inst); | |||
3894 | Instr->getParent()->insert(MachineBasicBlock::iterator(Instr), | |||
3895 | Movr0Inst); | |||
3896 | break; | |||
3897 | } | |||
3898 | } | |||
3899 | if (InsertI == InsertE) | |||
3900 | return false; | |||
3901 | } | |||
3902 | ||||
3903 | // Make sure Sub instruction defines EFLAGS and mark the def live. | |||
3904 | unsigned i = 0, e = Sub->getNumOperands(); | |||
3905 | for (; i != e; ++i) { | |||
3906 | MachineOperand &MO = Sub->getOperand(i); | |||
3907 | if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) { | |||
3908 | MO.setIsDead(false); | |||
3909 | break; | |||
3910 | } | |||
3911 | } | |||
3912 | assert(i != e && "Unable to locate a def EFLAGS operand")((i != e && "Unable to locate a def EFLAGS operand") ? static_cast<void> (0) : __assert_fail ("i != e && \"Unable to locate a def EFLAGS operand\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3912, __PRETTY_FUNCTION__)); | |||
3913 | ||||
3914 | CmpInstr.eraseFromParent(); | |||
3915 | ||||
3916 | // Modify the condition code of instructions in OpsToUpdate. | |||
3917 | for (auto &Op : OpsToUpdate) | |||
3918 | Op.first->setDesc(get(Op.second)); | |||
3919 | return true; | |||
3920 | } | |||
3921 | ||||
3922 | /// Try to remove the load by folding it to a register | |||
3923 | /// operand at the use. We fold the load instructions if load defines a virtual | |||
3924 | /// register, the virtual register is used once in the same BB, and the | |||
3925 | /// instructions in-between do not load or store, and have no side effects. | |||
3926 | MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI, | |||
3927 | const MachineRegisterInfo *MRI, | |||
3928 | unsigned &FoldAsLoadDefReg, | |||
3929 | MachineInstr *&DefMI) const { | |||
3930 | // Check whether we can move DefMI here. | |||
3931 | DefMI = MRI->getVRegDef(FoldAsLoadDefReg); | |||
3932 | assert(DefMI)((DefMI) ? static_cast<void> (0) : __assert_fail ("DefMI" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3932, __PRETTY_FUNCTION__)); | |||
3933 | bool SawStore = false; | |||
3934 | if (!DefMI->isSafeToMove(nullptr, SawStore)) | |||
3935 | return nullptr; | |||
3936 | ||||
3937 | // Collect information about virtual register operands of MI. | |||
3938 | SmallVector<unsigned, 1> SrcOperandIds; | |||
3939 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
3940 | MachineOperand &MO = MI.getOperand(i); | |||
3941 | if (!MO.isReg()) | |||
3942 | continue; | |||
3943 | unsigned Reg = MO.getReg(); | |||
3944 | if (Reg != FoldAsLoadDefReg) | |||
3945 | continue; | |||
3946 | // Do not fold if we have a subreg use or a def. | |||
3947 | if (MO.getSubReg() || MO.isDef()) | |||
3948 | return nullptr; | |||
3949 | SrcOperandIds.push_back(i); | |||
3950 | } | |||
3951 | if (SrcOperandIds.empty()) | |||
3952 | return nullptr; | |||
3953 | ||||
3954 | // Check whether we can fold the def into SrcOperandId. | |||
3955 | if (MachineInstr *FoldMI = foldMemoryOperand(MI, SrcOperandIds, *DefMI)) { | |||
3956 | FoldAsLoadDefReg = 0; | |||
3957 | return FoldMI; | |||
3958 | } | |||
3959 | ||||
3960 | return nullptr; | |||
3961 | } | |||
3962 | ||||
3963 | /// Expand a single-def pseudo instruction to a two-addr | |||
3964 | /// instruction with two undef reads of the register being defined. | |||
3965 | /// This is used for mapping: | |||
3966 | /// %xmm4 = V_SET0 | |||
3967 | /// to: | |||
3968 | /// %xmm4 = PXORrr undef %xmm4, undef %xmm4 | |||
3969 | /// | |||
3970 | static bool Expand2AddrUndef(MachineInstrBuilder &MIB, | |||
3971 | const MCInstrDesc &Desc) { | |||
3972 | assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.")((Desc.getNumOperands() == 3 && "Expected two-addr instruction." ) ? static_cast<void> (0) : __assert_fail ("Desc.getNumOperands() == 3 && \"Expected two-addr instruction.\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3972, __PRETTY_FUNCTION__)); | |||
3973 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
3974 | MIB->setDesc(Desc); | |||
3975 | ||||
3976 | // MachineInstr::addOperand() will insert explicit operands before any | |||
3977 | // implicit operands. | |||
3978 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); | |||
3979 | // But we don't trust that. | |||
3980 | assert(MIB->getOperand(1).getReg() == Reg &&((MIB->getOperand(1).getReg() == Reg && MIB->getOperand (2).getReg() == Reg && "Misplaced operand") ? static_cast <void> (0) : __assert_fail ("MIB->getOperand(1).getReg() == Reg && MIB->getOperand(2).getReg() == Reg && \"Misplaced operand\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3981, __PRETTY_FUNCTION__)) | |||
3981 | MIB->getOperand(2).getReg() == Reg && "Misplaced operand")((MIB->getOperand(1).getReg() == Reg && MIB->getOperand (2).getReg() == Reg && "Misplaced operand") ? static_cast <void> (0) : __assert_fail ("MIB->getOperand(1).getReg() == Reg && MIB->getOperand(2).getReg() == Reg && \"Misplaced operand\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3981, __PRETTY_FUNCTION__)); | |||
3982 | return true; | |||
3983 | } | |||
3984 | ||||
3985 | /// Expand a single-def pseudo instruction to a two-addr | |||
3986 | /// instruction with two %k0 reads. | |||
3987 | /// This is used for mapping: | |||
3988 | /// %k4 = K_SET1 | |||
3989 | /// to: | |||
3990 | /// %k4 = KXNORrr %k0, %k0 | |||
3991 | static bool Expand2AddrKreg(MachineInstrBuilder &MIB, | |||
3992 | const MCInstrDesc &Desc, unsigned Reg) { | |||
3993 | assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.")((Desc.getNumOperands() == 3 && "Expected two-addr instruction." ) ? static_cast<void> (0) : __assert_fail ("Desc.getNumOperands() == 3 && \"Expected two-addr instruction.\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 3993, __PRETTY_FUNCTION__)); | |||
3994 | MIB->setDesc(Desc); | |||
3995 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); | |||
3996 | return true; | |||
3997 | } | |||
3998 | ||||
3999 | static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, | |||
4000 | bool MinusOne) { | |||
4001 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
4002 | DebugLoc DL = MIB->getDebugLoc(); | |||
4003 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4004 | ||||
4005 | // Insert the XOR. | |||
4006 | BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg) | |||
4007 | .addReg(Reg, RegState::Undef) | |||
4008 | .addReg(Reg, RegState::Undef); | |||
4009 | ||||
4010 | // Turn the pseudo into an INC or DEC. | |||
4011 | MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r)); | |||
4012 | MIB.addReg(Reg); | |||
4013 | ||||
4014 | return true; | |||
4015 | } | |||
4016 | ||||
4017 | static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, | |||
4018 | const TargetInstrInfo &TII, | |||
4019 | const X86Subtarget &Subtarget) { | |||
4020 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
4021 | DebugLoc DL = MIB->getDebugLoc(); | |||
4022 | int64_t Imm = MIB->getOperand(1).getImm(); | |||
4023 | assert(Imm != 0 && "Using push/pop for 0 is not efficient.")((Imm != 0 && "Using push/pop for 0 is not efficient." ) ? static_cast<void> (0) : __assert_fail ("Imm != 0 && \"Using push/pop for 0 is not efficient.\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 4023, __PRETTY_FUNCTION__)); | |||
4024 | MachineBasicBlock::iterator I = MIB.getInstr(); | |||
4025 | ||||
4026 | int StackAdjustment; | |||
4027 | ||||
4028 | if (Subtarget.is64Bit()) { | |||
4029 | assert(MIB->getOpcode() == X86::MOV64ImmSExti8 ||((MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode () == X86::MOV32ImmSExti8) ? static_cast<void> (0) : __assert_fail ("MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode() == X86::MOV32ImmSExti8" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 4030, __PRETTY_FUNCTION__)) | |||
4030 | MIB->getOpcode() == X86::MOV32ImmSExti8)((MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode () == X86::MOV32ImmSExti8) ? static_cast<void> (0) : __assert_fail ("MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode() == X86::MOV32ImmSExti8" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 4030, __PRETTY_FUNCTION__)); | |||
4031 | ||||
4032 | // Can't use push/pop lowering if the function might write to the red zone. | |||
4033 | X86MachineFunctionInfo *X86FI = | |||
4034 | MBB.getParent()->getInfo<X86MachineFunctionInfo>(); | |||
4035 | if (X86FI->getUsesRedZone()) { | |||
4036 | MIB->setDesc(TII.get(MIB->getOpcode() == | |||
4037 | X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri)); | |||
4038 | return true; | |||
4039 | } | |||
4040 | ||||
4041 | // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and | |||
4042 | // widen the register if necessary. | |||
4043 | StackAdjustment = 8; | |||
4044 | BuildMI(MBB, I, DL, TII.get(X86::PUSH64i8)).addImm(Imm); | |||
4045 | MIB->setDesc(TII.get(X86::POP64r)); | |||
4046 | MIB->getOperand(0) | |||
4047 | .setReg(getX86SubSuperRegister(MIB->getOperand(0).getReg(), 64)); | |||
4048 | } else { | |||
4049 | assert(MIB->getOpcode() == X86::MOV32ImmSExti8)((MIB->getOpcode() == X86::MOV32ImmSExti8) ? static_cast< void> (0) : __assert_fail ("MIB->getOpcode() == X86::MOV32ImmSExti8" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 4049, __PRETTY_FUNCTION__)); | |||
4050 | StackAdjustment = 4; | |||
4051 | BuildMI(MBB, I, DL, TII.get(X86::PUSH32i8)).addImm(Imm); | |||
4052 | MIB->setDesc(TII.get(X86::POP32r)); | |||
4053 | } | |||
4054 | ||||
4055 | // Build CFI if necessary. | |||
4056 | MachineFunction &MF = *MBB.getParent(); | |||
4057 | const X86FrameLowering *TFL = Subtarget.getFrameLowering(); | |||
4058 | bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); | |||
4059 | bool NeedsDwarfCFI = | |||
4060 | !IsWin64Prologue && | |||
4061 | (MF.getMMI().hasDebugInfo() || MF.getFunction().needsUnwindTableEntry()); | |||
4062 | bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI; | |||
4063 | if (EmitCFI) { | |||
4064 | TFL->BuildCFI(MBB, I, DL, | |||
4065 | MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment)); | |||
4066 | TFL->BuildCFI(MBB, std::next(I), DL, | |||
4067 | MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment)); | |||
4068 | } | |||
4069 | ||||
4070 | return true; | |||
4071 | } | |||
4072 | ||||
4073 | // LoadStackGuard has so far only been implemented for 64-bit MachO. Different | |||
4074 | // code sequence is needed for other targets. | |||
4075 | static void expandLoadStackGuard(MachineInstrBuilder &MIB, | |||
4076 | const TargetInstrInfo &TII) { | |||
4077 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
4078 | DebugLoc DL = MIB->getDebugLoc(); | |||
4079 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4080 | const GlobalValue *GV = | |||
4081 | cast<GlobalValue>((*MIB->memoperands_begin())->getValue()); | |||
4082 | auto Flags = MachineMemOperand::MOLoad | | |||
4083 | MachineMemOperand::MODereferenceable | | |||
4084 | MachineMemOperand::MOInvariant; | |||
4085 | MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( | |||
4086 | MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, 8); | |||
4087 | MachineBasicBlock::iterator I = MIB.getInstr(); | |||
4088 | ||||
4089 | BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg).addReg(X86::RIP).addImm(1) | |||
4090 | .addReg(0).addGlobalAddress(GV, 0, X86II::MO_GOTPCREL).addReg(0) | |||
4091 | .addMemOperand(MMO); | |||
4092 | MIB->setDebugLoc(DL); | |||
4093 | MIB->setDesc(TII.get(X86::MOV64rm)); | |||
4094 | MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0); | |||
4095 | } | |||
4096 | ||||
4097 | static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) { | |||
4098 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
4099 | MachineFunction &MF = *MBB.getParent(); | |||
4100 | const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>(); | |||
4101 | const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); | |||
4102 | unsigned XorOp = | |||
4103 | MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr; | |||
4104 | MIB->setDesc(TII.get(XorOp)); | |||
4105 | MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef); | |||
4106 | return true; | |||
4107 | } | |||
4108 | ||||
4109 | // This is used to handle spills for 128/256-bit registers when we have AVX512, | |||
4110 | // but not VLX. If it uses an extended register we need to use an instruction | |||
4111 | // that loads the lower 128/256-bit, but is available with only AVX512F. | |||
4112 | static bool expandNOVLXLoad(MachineInstrBuilder &MIB, | |||
4113 | const TargetRegisterInfo *TRI, | |||
4114 | const MCInstrDesc &LoadDesc, | |||
4115 | const MCInstrDesc &BroadcastDesc, | |||
4116 | unsigned SubIdx) { | |||
4117 | unsigned DestReg = MIB->getOperand(0).getReg(); | |||
4118 | // Check if DestReg is XMM16-31 or YMM16-31. | |||
4119 | if (TRI->getEncodingValue(DestReg) < 16) { | |||
4120 | // We can use a normal VEX encoded load. | |||
4121 | MIB->setDesc(LoadDesc); | |||
4122 | } else { | |||
4123 | // Use a 128/256-bit VBROADCAST instruction. | |||
4124 | MIB->setDesc(BroadcastDesc); | |||
4125 | // Change the destination to a 512-bit register. | |||
4126 | DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass); | |||
4127 | MIB->getOperand(0).setReg(DestReg); | |||
4128 | } | |||
4129 | return true; | |||
4130 | } | |||
4131 | ||||
4132 | // This is used to handle spills for 128/256-bit registers when we have AVX512, | |||
4133 | // but not VLX. If it uses an extended register we need to use an instruction | |||
4134 | // that stores the lower 128/256-bit, but is available with only AVX512F. | |||
4135 | static bool expandNOVLXStore(MachineInstrBuilder &MIB, | |||
4136 | const TargetRegisterInfo *TRI, | |||
4137 | const MCInstrDesc &StoreDesc, | |||
4138 | const MCInstrDesc &ExtractDesc, | |||
4139 | unsigned SubIdx) { | |||
4140 | unsigned SrcReg = MIB->getOperand(X86::AddrNumOperands).getReg(); | |||
4141 | // Check if DestReg is XMM16-31 or YMM16-31. | |||
4142 | if (TRI->getEncodingValue(SrcReg) < 16) { | |||
4143 | // We can use a normal VEX encoded store. | |||
4144 | MIB->setDesc(StoreDesc); | |||
4145 | } else { | |||
4146 | // Use a VEXTRACTF instruction. | |||
4147 | MIB->setDesc(ExtractDesc); | |||
4148 | // Change the destination to a 512-bit register. | |||
4149 | SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass); | |||
4150 | MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg); | |||
4151 | MIB.addImm(0x0); // Append immediate to extract from the lower bits. | |||
4152 | } | |||
4153 | ||||
4154 | return true; | |||
4155 | } | |||
4156 | bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { | |||
4157 | bool HasAVX = Subtarget.hasAVX(); | |||
4158 | MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); | |||
4159 | switch (MI.getOpcode()) { | |||
4160 | case X86::MOV32r0: | |||
4161 | return Expand2AddrUndef(MIB, get(X86::XOR32rr)); | |||
4162 | case X86::MOV64r0: { | |||
4163 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
4164 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4165 | unsigned Reg32 = TRI->getSubReg(Reg, X86::sub_32bit); | |||
4166 | MIB->getOperand(0).setReg(Reg32); | |||
4167 | Expand2AddrUndef(MIB, get(X86::XOR32rr)); | |||
4168 | MIB.addReg(Reg, RegState::ImplicitDefine); | |||
4169 | return true; | |||
4170 | } | |||
4171 | case X86::MOV32r1: | |||
4172 | return expandMOV32r1(MIB, *this, /*MinusOne=*/ false); | |||
4173 | case X86::MOV32r_1: | |||
4174 | return expandMOV32r1(MIB, *this, /*MinusOne=*/ true); | |||
4175 | case X86::MOV32ImmSExti8: | |||
4176 | case X86::MOV64ImmSExti8: | |||
4177 | return ExpandMOVImmSExti8(MIB, *this, Subtarget); | |||
4178 | case X86::SETB_C8r: | |||
4179 | return Expand2AddrUndef(MIB, get(X86::SBB8rr)); | |||
4180 | case X86::SETB_C16r: | |||
4181 | return Expand2AddrUndef(MIB, get(X86::SBB16rr)); | |||
4182 | case X86::SETB_C32r: | |||
4183 | return Expand2AddrUndef(MIB, get(X86::SBB32rr)); | |||
4184 | case X86::SETB_C64r: | |||
4185 | return Expand2AddrUndef(MIB, get(X86::SBB64rr)); | |||
4186 | case X86::MMX_SET0: | |||
4187 | return Expand2AddrUndef(MIB, get(X86::MMX_PXORirr)); | |||
4188 | case X86::V_SET0: | |||
4189 | case X86::FsFLD0SS: | |||
4190 | case X86::FsFLD0SD: | |||
4191 | return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr)); | |||
4192 | case X86::AVX_SET0: { | |||
4193 | assert(HasAVX && "AVX not supported")((HasAVX && "AVX not supported") ? static_cast<void > (0) : __assert_fail ("HasAVX && \"AVX not supported\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 4193, __PRETTY_FUNCTION__)); | |||
4194 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
4195 | unsigned SrcReg = MIB->getOperand(0).getReg(); | |||
4196 | unsigned XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); | |||
4197 | MIB->getOperand(0).setReg(XReg); | |||
4198 | Expand2AddrUndef(MIB, get(X86::VXORPSrr)); | |||
4199 | MIB.addReg(SrcReg, RegState::ImplicitDefine); | |||
4200 | return true; | |||
4201 | } | |||
4202 | case X86::AVX512_128_SET0: | |||
4203 | case X86::AVX512_FsFLD0SS: | |||
4204 | case X86::AVX512_FsFLD0SD: { | |||
4205 | bool HasVLX = Subtarget.hasVLX(); | |||
4206 | unsigned SrcReg = MIB->getOperand(0).getReg(); | |||
4207 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
4208 | if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) | |||
4209 | return Expand2AddrUndef(MIB, | |||
4210 | get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); | |||
4211 | // Extended register without VLX. Use a larger XOR. | |||
4212 | SrcReg = | |||
4213 | TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass); | |||
4214 | MIB->getOperand(0).setReg(SrcReg); | |||
4215 | return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); | |||
4216 | } | |||
4217 | case X86::AVX512_256_SET0: | |||
4218 | case X86::AVX512_512_SET0: { | |||
4219 | bool HasVLX = Subtarget.hasVLX(); | |||
4220 | unsigned SrcReg = MIB->getOperand(0).getReg(); | |||
4221 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
4222 | if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) { | |||
4223 | unsigned XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); | |||
4224 | MIB->getOperand(0).setReg(XReg); | |||
4225 | Expand2AddrUndef(MIB, | |||
4226 | get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); | |||
4227 | MIB.addReg(SrcReg, RegState::ImplicitDefine); | |||
4228 | return true; | |||
4229 | } | |||
4230 | return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); | |||
4231 | } | |||
4232 | case X86::V_SETALLONES: | |||
4233 | return Expand2AddrUndef(MIB, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr)); | |||
4234 | case X86::AVX2_SETALLONES: | |||
4235 | return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr)); | |||
4236 | case X86::AVX1_SETALLONES: { | |||
4237 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4238 | // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS. | |||
4239 | MIB->setDesc(get(X86::VCMPPSYrri)); | |||
4240 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf); | |||
4241 | return true; | |||
4242 | } | |||
4243 | case X86::AVX512_512_SETALLONES: { | |||
4244 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4245 | MIB->setDesc(get(X86::VPTERNLOGDZrri)); | |||
4246 | // VPTERNLOGD needs 3 register inputs and an immediate. | |||
4247 | // 0xff will return 1s for any input. | |||
4248 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef) | |||
4249 | .addReg(Reg, RegState::Undef).addImm(0xff); | |||
4250 | return true; | |||
4251 | } | |||
4252 | case X86::AVX512_512_SEXT_MASK_32: | |||
4253 | case X86::AVX512_512_SEXT_MASK_64: { | |||
4254 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4255 | unsigned MaskReg = MIB->getOperand(1).getReg(); | |||
4256 | unsigned MaskState = getRegState(MIB->getOperand(1)); | |||
4257 | unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64) ? | |||
4258 | X86::VPTERNLOGQZrrikz : X86::VPTERNLOGDZrrikz; | |||
4259 | MI.RemoveOperand(1); | |||
4260 | MIB->setDesc(get(Opc)); | |||
4261 | // VPTERNLOG needs 3 register inputs and an immediate. | |||
4262 | // 0xff will return 1s for any input. | |||
4263 | MIB.addReg(Reg, RegState::Undef).addReg(MaskReg, MaskState) | |||
4264 | .addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xff); | |||
4265 | return true; | |||
4266 | } | |||
4267 | case X86::VMOVAPSZ128rm_NOVLX: | |||
4268 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm), | |||
4269 | get(X86::VBROADCASTF32X4rm), X86::sub_xmm); | |||
4270 | case X86::VMOVUPSZ128rm_NOVLX: | |||
4271 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm), | |||
4272 | get(X86::VBROADCASTF32X4rm), X86::sub_xmm); | |||
4273 | case X86::VMOVAPSZ256rm_NOVLX: | |||
4274 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm), | |||
4275 | get(X86::VBROADCASTF64X4rm), X86::sub_ymm); | |||
4276 | case X86::VMOVUPSZ256rm_NOVLX: | |||
4277 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm), | |||
4278 | get(X86::VBROADCASTF64X4rm), X86::sub_ymm); | |||
4279 | case X86::VMOVAPSZ128mr_NOVLX: | |||
4280 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr), | |||
4281 | get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); | |||
4282 | case X86::VMOVUPSZ128mr_NOVLX: | |||
4283 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr), | |||
4284 | get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); | |||
4285 | case X86::VMOVAPSZ256mr_NOVLX: | |||
4286 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr), | |||
4287 | get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); | |||
4288 | case X86::VMOVUPSZ256mr_NOVLX: | |||
4289 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr), | |||
4290 | get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); | |||
4291 | case X86::MOV32ri64: { | |||
4292 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4293 | unsigned Reg32 = RI.getSubReg(Reg, X86::sub_32bit); | |||
4294 | MI.setDesc(get(X86::MOV32ri)); | |||
4295 | MIB->getOperand(0).setReg(Reg32); | |||
4296 | MIB.addReg(Reg, RegState::ImplicitDefine); | |||
4297 | return true; | |||
4298 | } | |||
4299 | ||||
4300 | // KNL does not recognize dependency-breaking idioms for mask registers, | |||
4301 | // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1. | |||
4302 | // Using %k0 as the undef input register is a performance heuristic based | |||
4303 | // on the assumption that %k0 is used less frequently than the other mask | |||
4304 | // registers, since it is not usable as a write mask. | |||
4305 | // FIXME: A more advanced approach would be to choose the best input mask | |||
4306 | // register based on context. | |||
4307 | case X86::KSET0W: return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0); | |||
4308 | case X86::KSET0D: return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0); | |||
4309 | case X86::KSET0Q: return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0); | |||
4310 | case X86::KSET1W: return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0); | |||
4311 | case X86::KSET1D: return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0); | |||
4312 | case X86::KSET1Q: return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0); | |||
4313 | case TargetOpcode::LOAD_STACK_GUARD: | |||
4314 | expandLoadStackGuard(MIB, *this); | |||
4315 | return true; | |||
4316 | case X86::XOR64_FP: | |||
4317 | case X86::XOR32_FP: | |||
4318 | return expandXorFP(MIB, *this); | |||
4319 | } | |||
4320 | return false; | |||
4321 | } | |||
4322 | ||||
4323 | /// Return true for all instructions that only update | |||
4324 | /// the first 32 or 64-bits of the destination register and leave the rest | |||
4325 | /// unmodified. This can be used to avoid folding loads if the instructions | |||
4326 | /// only update part of the destination register, and the non-updated part is | |||
4327 | /// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these | |||
4328 | /// instructions breaks the partial register dependency and it can improve | |||
4329 | /// performance. e.g.: | |||
4330 | /// | |||
4331 | /// movss (%rdi), %xmm0 | |||
4332 | /// cvtss2sd %xmm0, %xmm0 | |||
4333 | /// | |||
4334 | /// Instead of | |||
4335 | /// cvtss2sd (%rdi), %xmm0 | |||
4336 | /// | |||
4337 | /// FIXME: This should be turned into a TSFlags. | |||
4338 | /// | |||
4339 | static bool hasPartialRegUpdate(unsigned Opcode, | |||
4340 | const X86Subtarget &Subtarget) { | |||
4341 | switch (Opcode) { | |||
4342 | case X86::CVTSI2SSrr: | |||
4343 | case X86::CVTSI2SSrm: | |||
4344 | case X86::CVTSI642SSrr: | |||
4345 | case X86::CVTSI642SSrm: | |||
4346 | case X86::CVTSI2SDrr: | |||
4347 | case X86::CVTSI2SDrm: | |||
4348 | case X86::CVTSI642SDrr: | |||
4349 | case X86::CVTSI642SDrm: | |||
4350 | case X86::CVTSD2SSrr: | |||
4351 | case X86::CVTSD2SSrm: | |||
4352 | case X86::CVTSS2SDrr: | |||
4353 | case X86::CVTSS2SDrm: | |||
4354 | case X86::MOVHPDrm: | |||
4355 | case X86::MOVHPSrm: | |||
4356 | case X86::MOVLPDrm: | |||
4357 | case X86::MOVLPSrm: | |||
4358 | case X86::RCPSSr: | |||
4359 | case X86::RCPSSm: | |||
4360 | case X86::RCPSSr_Int: | |||
4361 | case X86::RCPSSm_Int: | |||
4362 | case X86::ROUNDSDr: | |||
4363 | case X86::ROUNDSDm: | |||
4364 | case X86::ROUNDSSr: | |||
4365 | case X86::ROUNDSSm: | |||
4366 | case X86::RSQRTSSr: | |||
4367 | case X86::RSQRTSSm: | |||
4368 | case X86::RSQRTSSr_Int: | |||
4369 | case X86::RSQRTSSm_Int: | |||
4370 | case X86::SQRTSSr: | |||
4371 | case X86::SQRTSSm: | |||
4372 | case X86::SQRTSSr_Int: | |||
4373 | case X86::SQRTSSm_Int: | |||
4374 | case X86::SQRTSDr: | |||
4375 | case X86::SQRTSDm: | |||
4376 | case X86::SQRTSDr_Int: | |||
4377 | case X86::SQRTSDm_Int: | |||
4378 | return true; | |||
4379 | // GPR | |||
4380 | case X86::POPCNT32rm: | |||
4381 | case X86::POPCNT32rr: | |||
4382 | case X86::POPCNT64rm: | |||
4383 | case X86::POPCNT64rr: | |||
4384 | return Subtarget.hasPOPCNTFalseDeps(); | |||
4385 | case X86::LZCNT32rm: | |||
4386 | case X86::LZCNT32rr: | |||
4387 | case X86::LZCNT64rm: | |||
4388 | case X86::LZCNT64rr: | |||
4389 | case X86::TZCNT32rm: | |||
4390 | case X86::TZCNT32rr: | |||
4391 | case X86::TZCNT64rm: | |||
4392 | case X86::TZCNT64rr: | |||
4393 | return Subtarget.hasLZCNTFalseDeps(); | |||
4394 | } | |||
4395 | ||||
4396 | return false; | |||
4397 | } | |||
4398 | ||||
4399 | /// Inform the BreakFalseDeps pass how many idle | |||
4400 | /// instructions we would like before a partial register update. | |||
4401 | unsigned X86InstrInfo::getPartialRegUpdateClearance( | |||
4402 | const MachineInstr &MI, unsigned OpNum, | |||
4403 | const TargetRegisterInfo *TRI) const { | |||
4404 | if (OpNum != 0 || !hasPartialRegUpdate(MI.getOpcode(), Subtarget)) | |||
4405 | return 0; | |||
4406 | ||||
4407 | // If MI is marked as reading Reg, the partial register update is wanted. | |||
4408 | const MachineOperand &MO = MI.getOperand(0); | |||
4409 | unsigned Reg = MO.getReg(); | |||
4410 | if (TargetRegisterInfo::isVirtualRegister(Reg)) { | |||
4411 | if (MO.readsReg() || MI.readsVirtualRegister(Reg)) | |||
4412 | return 0; | |||
4413 | } else { | |||
4414 | if (MI.readsRegister(Reg, TRI)) | |||
4415 | return 0; | |||
4416 | } | |||
4417 | ||||
4418 | // If any instructions in the clearance range are reading Reg, insert a | |||
4419 | // dependency breaking instruction, which is inexpensive and is likely to | |||
4420 | // be hidden in other instruction's cycles. | |||
4421 | return PartialRegUpdateClearance; | |||
4422 | } | |||
4423 | ||||
4424 | // Return true for any instruction the copies the high bits of the first source | |||
4425 | // operand into the unused high bits of the destination operand. | |||
4426 | static bool hasUndefRegUpdate(unsigned Opcode) { | |||
4427 | switch (Opcode) { | |||
4428 | case X86::VCVTSI2SSrr: | |||
4429 | case X86::VCVTSI2SSrm: | |||
4430 | case X86::VCVTSI2SSrr_Int: | |||
4431 | case X86::VCVTSI2SSrm_Int: | |||
4432 | case X86::VCVTSI642SSrr: | |||
4433 | case X86::VCVTSI642SSrm: | |||
4434 | case X86::VCVTSI642SSrr_Int: | |||
4435 | case X86::VCVTSI642SSrm_Int: | |||
4436 | case X86::VCVTSI2SDrr: | |||
4437 | case X86::VCVTSI2SDrm: | |||
4438 | case X86::VCVTSI2SDrr_Int: | |||
4439 | case X86::VCVTSI2SDrm_Int: | |||
4440 | case X86::VCVTSI642SDrr: | |||
4441 | case X86::VCVTSI642SDrm: | |||
4442 | case X86::VCVTSI642SDrr_Int: | |||
4443 | case X86::VCVTSI642SDrm_Int: | |||
4444 | case X86::VCVTSD2SSrr: | |||
4445 | case X86::VCVTSD2SSrm: | |||
4446 | case X86::VCVTSD2SSrr_Int: | |||
4447 | case X86::VCVTSD2SSrm_Int: | |||
4448 | case X86::VCVTSS2SDrr: | |||
4449 | case X86::VCVTSS2SDrm: | |||
4450 | case X86::VCVTSS2SDrr_Int: | |||
4451 | case X86::VCVTSS2SDrm_Int: | |||
4452 | case X86::VRCPSSr: | |||
4453 | case X86::VRCPSSr_Int: | |||
4454 | case X86::VRCPSSm: | |||
4455 | case X86::VRCPSSm_Int: | |||
4456 | case X86::VROUNDSDr: | |||
4457 | case X86::VROUNDSDm: | |||
4458 | case X86::VROUNDSDr_Int: | |||
4459 | case X86::VROUNDSDm_Int: | |||
4460 | case X86::VROUNDSSr: | |||
4461 | case X86::VROUNDSSm: | |||
4462 | case X86::VROUNDSSr_Int: | |||
4463 | case X86::VROUNDSSm_Int: | |||
4464 | case X86::VRSQRTSSr: | |||
4465 | case X86::VRSQRTSSr_Int: | |||
4466 | case X86::VRSQRTSSm: | |||
4467 | case X86::VRSQRTSSm_Int: | |||
4468 | case X86::VSQRTSSr: | |||
4469 | case X86::VSQRTSSr_Int: | |||
4470 | case X86::VSQRTSSm: | |||
4471 | case X86::VSQRTSSm_Int: | |||
4472 | case X86::VSQRTSDr: | |||
4473 | case X86::VSQRTSDr_Int: | |||
4474 | case X86::VSQRTSDm: | |||
4475 | case X86::VSQRTSDm_Int: | |||
4476 | // AVX-512 | |||
4477 | case X86::VCVTSI2SSZrr: | |||
4478 | case X86::VCVTSI2SSZrm: | |||
4479 | case X86::VCVTSI2SSZrr_Int: | |||
4480 | case X86::VCVTSI2SSZrrb_Int: | |||
4481 | case X86::VCVTSI2SSZrm_Int: | |||
4482 | case X86::VCVTSI642SSZrr: | |||
4483 | case X86::VCVTSI642SSZrm: | |||
4484 | case X86::VCVTSI642SSZrr_Int: | |||
4485 | case X86::VCVTSI642SSZrrb_Int: | |||
4486 | case X86::VCVTSI642SSZrm_Int: | |||
4487 | case X86::VCVTSI2SDZrr: | |||
4488 | case X86::VCVTSI2SDZrm: | |||
4489 | case X86::VCVTSI2SDZrr_Int: | |||
4490 | case X86::VCVTSI2SDZrrb_Int: | |||
4491 | case X86::VCVTSI2SDZrm_Int: | |||
4492 | case X86::VCVTSI642SDZrr: | |||
4493 | case X86::VCVTSI642SDZrm: | |||
4494 | case X86::VCVTSI642SDZrr_Int: | |||
4495 | case X86::VCVTSI642SDZrrb_Int: | |||
4496 | case X86::VCVTSI642SDZrm_Int: | |||
4497 | case X86::VCVTUSI2SSZrr: | |||
4498 | case X86::VCVTUSI2SSZrm: | |||
4499 | case X86::VCVTUSI2SSZrr_Int: | |||
4500 | case X86::VCVTUSI2SSZrrb_Int: | |||
4501 | case X86::VCVTUSI2SSZrm_Int: | |||
4502 | case X86::VCVTUSI642SSZrr: | |||
4503 | case X86::VCVTUSI642SSZrm: | |||
4504 | case X86::VCVTUSI642SSZrr_Int: | |||
4505 | case X86::VCVTUSI642SSZrrb_Int: | |||
4506 | case X86::VCVTUSI642SSZrm_Int: | |||
4507 | case X86::VCVTUSI2SDZrr: | |||
4508 | case X86::VCVTUSI2SDZrm: | |||
4509 | case X86::VCVTUSI2SDZrr_Int: | |||
4510 | case X86::VCVTUSI2SDZrm_Int: | |||
4511 | case X86::VCVTUSI642SDZrr: | |||
4512 | case X86::VCVTUSI642SDZrm: | |||
4513 | case X86::VCVTUSI642SDZrr_Int: | |||
4514 | case X86::VCVTUSI642SDZrrb_Int: | |||
4515 | case X86::VCVTUSI642SDZrm_Int: | |||
4516 | case X86::VCVTSD2SSZrr: | |||
4517 | case X86::VCVTSD2SSZrr_Int: | |||
4518 | case X86::VCVTSD2SSZrrb_Int: | |||
4519 | case X86::VCVTSD2SSZrm: | |||
4520 | case X86::VCVTSD2SSZrm_Int: | |||
4521 | case X86::VCVTSS2SDZrr: | |||
4522 | case X86::VCVTSS2SDZrr_Int: | |||
4523 | case X86::VCVTSS2SDZrrb_Int: | |||
4524 | case X86::VCVTSS2SDZrm: | |||
4525 | case X86::VCVTSS2SDZrm_Int: | |||
4526 | case X86::VGETEXPSDZr: | |||
4527 | case X86::VGETEXPSDZrb: | |||
4528 | case X86::VGETEXPSDZm: | |||
4529 | case X86::VGETEXPSSZr: | |||
4530 | case X86::VGETEXPSSZrb: | |||
4531 | case X86::VGETEXPSSZm: | |||
4532 | case X86::VGETMANTSDZrri: | |||
4533 | case X86::VGETMANTSDZrrib: | |||
4534 | case X86::VGETMANTSDZrmi: | |||
4535 | case X86::VGETMANTSSZrri: | |||
4536 | case X86::VGETMANTSSZrrib: | |||
4537 | case X86::VGETMANTSSZrmi: | |||
4538 | case X86::VRNDSCALESDZr: | |||
4539 | case X86::VRNDSCALESDZr_Int: | |||
4540 | case X86::VRNDSCALESDZrb_Int: | |||
4541 | case X86::VRNDSCALESDZm: | |||
4542 | case X86::VRNDSCALESDZm_Int: | |||
4543 | case X86::VRNDSCALESSZr: | |||
4544 | case X86::VRNDSCALESSZr_Int: | |||
4545 | case X86::VRNDSCALESSZrb_Int: | |||
4546 | case X86::VRNDSCALESSZm: | |||
4547 | case X86::VRNDSCALESSZm_Int: | |||
4548 | case X86::VRCP14SDZrr: | |||
4549 | case X86::VRCP14SDZrm: | |||
4550 | case X86::VRCP14SSZrr: | |||
4551 | case X86::VRCP14SSZrm: | |||
4552 | case X86::VRCP28SDZr: | |||
4553 | case X86::VRCP28SDZrb: | |||
4554 | case X86::VRCP28SDZm: | |||
4555 | case X86::VRCP28SSZr: | |||
4556 | case X86::VRCP28SSZrb: | |||
4557 | case X86::VRCP28SSZm: | |||
4558 | case X86::VREDUCESSZrmi: | |||
4559 | case X86::VREDUCESSZrri: | |||
4560 | case X86::VREDUCESSZrrib: | |||
4561 | case X86::VRSQRT14SDZrr: | |||
4562 | case X86::VRSQRT14SDZrm: | |||
4563 | case X86::VRSQRT14SSZrr: | |||
4564 | case X86::VRSQRT14SSZrm: | |||
4565 | case X86::VRSQRT28SDZr: | |||
4566 | case X86::VRSQRT28SDZrb: | |||
4567 | case X86::VRSQRT28SDZm: | |||
4568 | case X86::VRSQRT28SSZr: | |||
4569 | case X86::VRSQRT28SSZrb: | |||
4570 | case X86::VRSQRT28SSZm: | |||
4571 | case X86::VSQRTSSZr: | |||
4572 | case X86::VSQRTSSZr_Int: | |||
4573 | case X86::VSQRTSSZrb_Int: | |||
4574 | case X86::VSQRTSSZm: | |||
4575 | case X86::VSQRTSSZm_Int: | |||
4576 | case X86::VSQRTSDZr: | |||
4577 | case X86::VSQRTSDZr_Int: | |||
4578 | case X86::VSQRTSDZrb_Int: | |||
4579 | case X86::VSQRTSDZm: | |||
4580 | case X86::VSQRTSDZm_Int: | |||
4581 | return true; | |||
4582 | } | |||
4583 | ||||
4584 | return false; | |||
4585 | } | |||
4586 | ||||
4587 | /// Inform the BreakFalseDeps pass how many idle instructions we would like | |||
4588 | /// before certain undef register reads. | |||
4589 | /// | |||
4590 | /// This catches the VCVTSI2SD family of instructions: | |||
4591 | /// | |||
4592 | /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 | |||
4593 | /// | |||
4594 | /// We should to be careful *not* to catch VXOR idioms which are presumably | |||
4595 | /// handled specially in the pipeline: | |||
4596 | /// | |||
4597 | /// vxorps undef %xmm1, undef %xmm1, %xmm1 | |||
4598 | /// | |||
4599 | /// Like getPartialRegUpdateClearance, this makes a strong assumption that the | |||
4600 | /// high bits that are passed-through are not live. | |||
4601 | unsigned | |||
4602 | X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned &OpNum, | |||
4603 | const TargetRegisterInfo *TRI) const { | |||
4604 | if (!hasUndefRegUpdate(MI.getOpcode())) | |||
4605 | return 0; | |||
4606 | ||||
4607 | // Set the OpNum parameter to the first source operand. | |||
4608 | OpNum = 1; | |||
4609 | ||||
4610 | const MachineOperand &MO = MI.getOperand(OpNum); | |||
4611 | if (MO.isUndef() && TargetRegisterInfo::isPhysicalRegister(MO.getReg())) { | |||
4612 | return UndefRegClearance; | |||
4613 | } | |||
4614 | return 0; | |||
4615 | } | |||
4616 | ||||
4617 | void X86InstrInfo::breakPartialRegDependency( | |||
4618 | MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const { | |||
4619 | unsigned Reg = MI.getOperand(OpNum).getReg(); | |||
4620 | // If MI kills this register, the false dependence is already broken. | |||
4621 | if (MI.killsRegister(Reg, TRI)) | |||
4622 | return; | |||
4623 | ||||
4624 | if (X86::VR128RegClass.contains(Reg)) { | |||
4625 | // These instructions are all floating point domain, so xorps is the best | |||
4626 | // choice. | |||
4627 | unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr; | |||
4628 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg) | |||
4629 | .addReg(Reg, RegState::Undef) | |||
4630 | .addReg(Reg, RegState::Undef); | |||
4631 | MI.addRegisterKilled(Reg, TRI, true); | |||
4632 | } else if (X86::VR256RegClass.contains(Reg)) { | |||
4633 | // Use vxorps to clear the full ymm register. | |||
4634 | // It wants to read and write the xmm sub-register. | |||
4635 | unsigned XReg = TRI->getSubReg(Reg, X86::sub_xmm); | |||
4636 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg) | |||
4637 | .addReg(XReg, RegState::Undef) | |||
4638 | .addReg(XReg, RegState::Undef) | |||
4639 | .addReg(Reg, RegState::ImplicitDefine); | |||
4640 | MI.addRegisterKilled(Reg, TRI, true); | |||
4641 | } else if (X86::GR64RegClass.contains(Reg)) { | |||
4642 | // Using XOR32rr because it has shorter encoding and zeros up the upper bits | |||
4643 | // as well. | |||
4644 | unsigned XReg = TRI->getSubReg(Reg, X86::sub_32bit); | |||
4645 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg) | |||
4646 | .addReg(XReg, RegState::Undef) | |||
4647 | .addReg(XReg, RegState::Undef) | |||
4648 | .addReg(Reg, RegState::ImplicitDefine); | |||
4649 | MI.addRegisterKilled(Reg, TRI, true); | |||
4650 | } else if (X86::GR32RegClass.contains(Reg)) { | |||
4651 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg) | |||
4652 | .addReg(Reg, RegState::Undef) | |||
4653 | .addReg(Reg, RegState::Undef); | |||
4654 | MI.addRegisterKilled(Reg, TRI, true); | |||
4655 | } | |||
4656 | } | |||
4657 | ||||
4658 | static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs, | |||
4659 | int PtrOffset = 0) { | |||
4660 | unsigned NumAddrOps = MOs.size(); | |||
4661 | ||||
4662 | if (NumAddrOps < 4) { | |||
4663 | // FrameIndex only - add an immediate offset (whether its zero or not). | |||
4664 | for (unsigned i = 0; i != NumAddrOps; ++i) | |||
4665 | MIB.add(MOs[i]); | |||
4666 | addOffset(MIB, PtrOffset); | |||
4667 | } else { | |||
4668 | // General Memory Addressing - we need to add any offset to an existing | |||
4669 | // offset. | |||
4670 | assert(MOs.size() == 5 && "Unexpected memory operand list length")((MOs.size() == 5 && "Unexpected memory operand list length" ) ? static_cast<void> (0) : __assert_fail ("MOs.size() == 5 && \"Unexpected memory operand list length\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 4670, __PRETTY_FUNCTION__)); | |||
4671 | for (unsigned i = 0; i != NumAddrOps; ++i) { | |||
4672 | const MachineOperand &MO = MOs[i]; | |||
4673 | if (i == 3 && PtrOffset != 0) { | |||
4674 | MIB.addDisp(MO, PtrOffset); | |||
4675 | } else { | |||
4676 | MIB.add(MO); | |||
4677 | } | |||
4678 | } | |||
4679 | } | |||
4680 | } | |||
4681 | ||||
4682 | static void updateOperandRegConstraints(MachineFunction &MF, | |||
4683 | MachineInstr &NewMI, | |||
4684 | const TargetInstrInfo &TII) { | |||
4685 | MachineRegisterInfo &MRI = MF.getRegInfo(); | |||
4686 | const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); | |||
4687 | ||||
4688 | for (int Idx : llvm::seq<int>(0, NewMI.getNumOperands())) { | |||
4689 | MachineOperand &MO = NewMI.getOperand(Idx); | |||
4690 | // We only need to update constraints on virtual register operands. | |||
4691 | if (!MO.isReg()) | |||
4692 | continue; | |||
4693 | unsigned Reg = MO.getReg(); | |||
4694 | if (!TRI.isVirtualRegister(Reg)) | |||
4695 | continue; | |||
4696 | ||||
4697 | auto *NewRC = MRI.constrainRegClass( | |||
4698 | Reg, TII.getRegClass(NewMI.getDesc(), Idx, &TRI, MF)); | |||
4699 | if (!NewRC) { | |||
4700 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false) | |||
4701 | dbgs() << "WARNING: Unable to update register constraint for operand "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false) | |||
4702 | << Idx << " of instruction:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false) | |||
4703 | NewMI.dump(); dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false); | |||
4704 | } | |||
4705 | } | |||
4706 | } | |||
4707 | ||||
4708 | static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, | |||
4709 | ArrayRef<MachineOperand> MOs, | |||
4710 | MachineBasicBlock::iterator InsertPt, | |||
4711 | MachineInstr &MI, | |||
4712 | const TargetInstrInfo &TII) { | |||
4713 | // Create the base instruction with the memory operand as the first part. | |||
4714 | // Omit the implicit operands, something BuildMI can't do. | |||
4715 | MachineInstr *NewMI = | |||
4716 | MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); | |||
4717 | MachineInstrBuilder MIB(MF, NewMI); | |||
4718 | addOperands(MIB, MOs); | |||
4719 | ||||
4720 | // Loop over the rest of the ri operands, converting them over. | |||
4721 | unsigned NumOps = MI.getDesc().getNumOperands() - 2; | |||
4722 | for (unsigned i = 0; i != NumOps; ++i) { | |||
4723 | MachineOperand &MO = MI.getOperand(i + 2); | |||
4724 | MIB.add(MO); | |||
4725 | } | |||
4726 | for (unsigned i = NumOps + 2, e = MI.getNumOperands(); i != e; ++i) { | |||
4727 | MachineOperand &MO = MI.getOperand(i); | |||
4728 | MIB.add(MO); | |||
4729 | } | |||
4730 | ||||
4731 | updateOperandRegConstraints(MF, *NewMI, TII); | |||
4732 | ||||
4733 | MachineBasicBlock *MBB = InsertPt->getParent(); | |||
4734 | MBB->insert(InsertPt, NewMI); | |||
4735 | ||||
4736 | return MIB; | |||
4737 | } | |||
4738 | ||||
4739 | static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode, | |||
4740 | unsigned OpNo, ArrayRef<MachineOperand> MOs, | |||
4741 | MachineBasicBlock::iterator InsertPt, | |||
4742 | MachineInstr &MI, const TargetInstrInfo &TII, | |||
4743 | int PtrOffset = 0) { | |||
4744 | // Omit the implicit operands, something BuildMI can't do. | |||
4745 | MachineInstr *NewMI = | |||
4746 | MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); | |||
4747 | MachineInstrBuilder MIB(MF, NewMI); | |||
4748 | ||||
4749 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
4750 | MachineOperand &MO = MI.getOperand(i); | |||
4751 | if (i == OpNo) { | |||
4752 | assert(MO.isReg() && "Expected to fold into reg operand!")((MO.isReg() && "Expected to fold into reg operand!") ? static_cast<void> (0) : __assert_fail ("MO.isReg() && \"Expected to fold into reg operand!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 4752, __PRETTY_FUNCTION__)); | |||
4753 | addOperands(MIB, MOs, PtrOffset); | |||
4754 | } else { | |||
4755 | MIB.add(MO); | |||
4756 | } | |||
4757 | } | |||
4758 | ||||
4759 | updateOperandRegConstraints(MF, *NewMI, TII); | |||
4760 | ||||
4761 | MachineBasicBlock *MBB = InsertPt->getParent(); | |||
4762 | MBB->insert(InsertPt, NewMI); | |||
4763 | ||||
4764 | return MIB; | |||
4765 | } | |||
4766 | ||||
4767 | static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, | |||
4768 | ArrayRef<MachineOperand> MOs, | |||
4769 | MachineBasicBlock::iterator InsertPt, | |||
4770 | MachineInstr &MI) { | |||
4771 | MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, | |||
4772 | MI.getDebugLoc(), TII.get(Opcode)); | |||
4773 | addOperands(MIB, MOs); | |||
4774 | return MIB.addImm(0); | |||
4775 | } | |||
4776 | ||||
4777 | MachineInstr *X86InstrInfo::foldMemoryOperandCustom( | |||
4778 | MachineFunction &MF, MachineInstr &MI, unsigned OpNum, | |||
4779 | ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, | |||
4780 | unsigned Size, unsigned Align) const { | |||
4781 | switch (MI.getOpcode()) { | |||
4782 | case X86::INSERTPSrr: | |||
4783 | case X86::VINSERTPSrr: | |||
4784 | case X86::VINSERTPSZrr: | |||
4785 | // Attempt to convert the load of inserted vector into a fold load | |||
4786 | // of a single float. | |||
4787 | if (OpNum == 2) { | |||
4788 | unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); | |||
4789 | unsigned ZMask = Imm & 15; | |||
4790 | unsigned DstIdx = (Imm >> 4) & 3; | |||
4791 | unsigned SrcIdx = (Imm >> 6) & 3; | |||
4792 | ||||
4793 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
4794 | const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); | |||
4795 | unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; | |||
4796 | if (Size <= RCSize && 4 <= Align) { | |||
4797 | int PtrOffset = SrcIdx * 4; | |||
4798 | unsigned NewImm = (DstIdx << 4) | ZMask; | |||
4799 | unsigned NewOpCode = | |||
4800 | (MI.getOpcode() == X86::VINSERTPSZrr) ? X86::VINSERTPSZrm : | |||
4801 | (MI.getOpcode() == X86::VINSERTPSrr) ? X86::VINSERTPSrm : | |||
4802 | X86::INSERTPSrm; | |||
4803 | MachineInstr *NewMI = | |||
4804 | FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, PtrOffset); | |||
4805 | NewMI->getOperand(NewMI->getNumOperands() - 1).setImm(NewImm); | |||
4806 | return NewMI; | |||
4807 | } | |||
4808 | } | |||
4809 | break; | |||
4810 | case X86::MOVHLPSrr: | |||
4811 | case X86::VMOVHLPSrr: | |||
4812 | case X86::VMOVHLPSZrr: | |||
4813 | // Move the upper 64-bits of the second operand to the lower 64-bits. | |||
4814 | // To fold the load, adjust the pointer to the upper and use (V)MOVLPS. | |||
4815 | // TODO: In most cases AVX doesn't have a 8-byte alignment requirement. | |||
4816 | if (OpNum == 2) { | |||
4817 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
4818 | const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); | |||
4819 | unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; | |||
4820 | if (Size <= RCSize && 8 <= Align) { | |||
4821 | unsigned NewOpCode = | |||
4822 | (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm : | |||
4823 | (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm : | |||
4824 | X86::MOVLPSrm; | |||
4825 | MachineInstr *NewMI = | |||
4826 | FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, 8); | |||
4827 | return NewMI; | |||
4828 | } | |||
4829 | } | |||
4830 | break; | |||
4831 | }; | |||
4832 | ||||
4833 | return nullptr; | |||
4834 | } | |||
4835 | ||||
4836 | static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, MachineInstr &MI) { | |||
4837 | if (MF.getFunction().optForSize() || !hasUndefRegUpdate(MI.getOpcode()) || | |||
4838 | !MI.getOperand(1).isReg()) | |||
4839 | return false; | |||
4840 | ||||
4841 | // The are two cases we need to handle depending on where in the pipeline | |||
4842 | // the folding attempt is being made. | |||
4843 | // -Register has the undef flag set. | |||
4844 | // -Register is produced by the IMPLICIT_DEF instruction. | |||
4845 | ||||
4846 | if (MI.getOperand(1).isUndef()) | |||
4847 | return true; | |||
4848 | ||||
4849 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | |||
4850 | MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(MI.getOperand(1).getReg()); | |||
4851 | return VRegDef && VRegDef->isImplicitDef(); | |||
4852 | } | |||
4853 | ||||
4854 | ||||
4855 | MachineInstr *X86InstrInfo::foldMemoryOperandImpl( | |||
4856 | MachineFunction &MF, MachineInstr &MI, unsigned OpNum, | |||
4857 | ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, | |||
4858 | unsigned Size, unsigned Align, bool AllowCommute) const { | |||
4859 | bool isSlowTwoMemOps = Subtarget.slowTwoMemOps(); | |||
4860 | bool isTwoAddrFold = false; | |||
4861 | ||||
4862 | // For CPUs that favor the register form of a call or push, | |||
4863 | // do not fold loads into calls or pushes, unless optimizing for size | |||
4864 | // aggressively. | |||
4865 | if (isSlowTwoMemOps && !MF.getFunction().optForMinSize() && | |||
4866 | (MI.getOpcode() == X86::CALL32r || MI.getOpcode() == X86::CALL64r || | |||
4867 | MI.getOpcode() == X86::PUSH16r || MI.getOpcode() == X86::PUSH32r || | |||
4868 | MI.getOpcode() == X86::PUSH64r)) | |||
4869 | return nullptr; | |||
4870 | ||||
4871 | // Avoid partial and undef register update stalls unless optimizing for size. | |||
4872 | if (!MF.getFunction().optForSize() && | |||
4873 | (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || | |||
4874 | shouldPreventUndefRegUpdateMemFold(MF, MI))) | |||
4875 | return nullptr; | |||
4876 | ||||
4877 | unsigned NumOps = MI.getDesc().getNumOperands(); | |||
4878 | bool isTwoAddr = | |||
4879 | NumOps > 1 && MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1; | |||
4880 | ||||
4881 | // FIXME: AsmPrinter doesn't know how to handle | |||
4882 | // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding. | |||
4883 | if (MI.getOpcode() == X86::ADD32ri && | |||
4884 | MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS) | |||
4885 | return nullptr; | |||
4886 | ||||
4887 | // GOTTPOFF relocation loads can only be folded into add instructions. | |||
4888 | // FIXME: Need to exclude other relocations that only support specific | |||
4889 | // instructions. | |||
4890 | if (MOs.size() == X86::AddrNumOperands && | |||
4891 | MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF && | |||
4892 | MI.getOpcode() != X86::ADD64rr) | |||
4893 | return nullptr; | |||
4894 | ||||
4895 | MachineInstr *NewMI = nullptr; | |||
4896 | ||||
4897 | // Attempt to fold any custom cases we have. | |||
4898 | if (MachineInstr *CustomMI = | |||
4899 | foldMemoryOperandCustom(MF, MI, OpNum, MOs, InsertPt, Size, Align)) | |||
4900 | return CustomMI; | |||
4901 | ||||
4902 | const X86MemoryFoldTableEntry *I = nullptr; | |||
4903 | ||||
4904 | // Folding a memory location into the two-address part of a two-address | |||
4905 | // instruction is different than folding it other places. It requires | |||
4906 | // replacing the *two* registers with the memory location. | |||
4907 | if (isTwoAddr && NumOps >= 2 && OpNum < 2 && MI.getOperand(0).isReg() && | |||
4908 | MI.getOperand(1).isReg() && | |||
4909 | MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) { | |||
4910 | I = lookupTwoAddrFoldTable(MI.getOpcode()); | |||
4911 | isTwoAddrFold = true; | |||
4912 | } else { | |||
4913 | if (OpNum == 0) { | |||
4914 | if (MI.getOpcode() == X86::MOV32r0 || MI.getOpcode() == X86::MOV64r0) { | |||
4915 | unsigned NewOpc = MI.getOpcode() == X86::MOV64r0 ? X86::MOV64mi32 | |||
4916 | : X86::MOV32mi; | |||
4917 | NewMI = MakeM0Inst(*this, NewOpc, MOs, InsertPt, MI); | |||
4918 | if (NewMI) | |||
4919 | return NewMI; | |||
4920 | } | |||
4921 | } | |||
4922 | ||||
4923 | I = lookupFoldTable(MI.getOpcode(), OpNum); | |||
4924 | } | |||
4925 | ||||
4926 | if (I != nullptr) { | |||
4927 | unsigned Opcode = I->DstOp; | |||
4928 | unsigned MinAlign = (I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT; | |||
4929 | if (Align < MinAlign) | |||
4930 | return nullptr; | |||
4931 | bool NarrowToMOV32rm = false; | |||
4932 | if (Size) { | |||
4933 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
4934 | const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, | |||
4935 | &RI, MF); | |||
4936 | unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; | |||
4937 | if (Size < RCSize) { | |||
4938 | // Check if it's safe to fold the load. If the size of the object is | |||
4939 | // narrower than the load width, then it's not. | |||
4940 | if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4) | |||
4941 | return nullptr; | |||
4942 | // If this is a 64-bit load, but the spill slot is 32, then we can do | |||
4943 | // a 32-bit load which is implicitly zero-extended. This likely is | |||
4944 | // due to live interval analysis remat'ing a load from stack slot. | |||
4945 | if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) | |||
4946 | return nullptr; | |||
4947 | Opcode = X86::MOV32rm; | |||
4948 | NarrowToMOV32rm = true; | |||
4949 | } | |||
4950 | } | |||
4951 | ||||
4952 | if (isTwoAddrFold) | |||
4953 | NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this); | |||
4954 | else | |||
4955 | NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this); | |||
4956 | ||||
4957 | if (NarrowToMOV32rm) { | |||
4958 | // If this is the special case where we use a MOV32rm to load a 32-bit | |||
4959 | // value and zero-extend the top bits. Change the destination register | |||
4960 | // to a 32-bit one. | |||
4961 | unsigned DstReg = NewMI->getOperand(0).getReg(); | |||
4962 | if (TargetRegisterInfo::isPhysicalRegister(DstReg)) | |||
4963 | NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit)); | |||
4964 | else | |||
4965 | NewMI->getOperand(0).setSubReg(X86::sub_32bit); | |||
4966 | } | |||
4967 | return NewMI; | |||
4968 | } | |||
4969 | ||||
4970 | // If the instruction and target operand are commutable, commute the | |||
4971 | // instruction and try again. | |||
4972 | if (AllowCommute) { | |||
4973 | unsigned CommuteOpIdx1 = OpNum, CommuteOpIdx2 = CommuteAnyOperandIndex; | |||
4974 | if (findCommutedOpIndices(MI, CommuteOpIdx1, CommuteOpIdx2)) { | |||
4975 | bool HasDef = MI.getDesc().getNumDefs(); | |||
4976 | unsigned Reg0 = HasDef ? MI.getOperand(0).getReg() : 0; | |||
4977 | unsigned Reg1 = MI.getOperand(CommuteOpIdx1).getReg(); | |||
4978 | unsigned Reg2 = MI.getOperand(CommuteOpIdx2).getReg(); | |||
4979 | bool Tied1 = | |||
4980 | 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx1, MCOI::TIED_TO); | |||
4981 | bool Tied2 = | |||
4982 | 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx2, MCOI::TIED_TO); | |||
4983 | ||||
4984 | // If either of the commutable operands are tied to the destination | |||
4985 | // then we can not commute + fold. | |||
4986 | if ((HasDef && Reg0 == Reg1 && Tied1) || | |||
4987 | (HasDef && Reg0 == Reg2 && Tied2)) | |||
4988 | return nullptr; | |||
4989 | ||||
4990 | MachineInstr *CommutedMI = | |||
4991 | commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); | |||
4992 | if (!CommutedMI) { | |||
4993 | // Unable to commute. | |||
4994 | return nullptr; | |||
4995 | } | |||
4996 | if (CommutedMI != &MI) { | |||
4997 | // New instruction. We can't fold from this. | |||
4998 | CommutedMI->eraseFromParent(); | |||
4999 | return nullptr; | |||
5000 | } | |||
5001 | ||||
5002 | // Attempt to fold with the commuted version of the instruction. | |||
5003 | NewMI = foldMemoryOperandImpl(MF, MI, CommuteOpIdx2, MOs, InsertPt, | |||
5004 | Size, Align, /*AllowCommute=*/false); | |||
5005 | if (NewMI) | |||
5006 | return NewMI; | |||
5007 | ||||
5008 | // Folding failed again - undo the commute before returning. | |||
5009 | MachineInstr *UncommutedMI = | |||
5010 | commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); | |||
5011 | if (!UncommutedMI) { | |||
5012 | // Unable to commute. | |||
5013 | return nullptr; | |||
5014 | } | |||
5015 | if (UncommutedMI != &MI) { | |||
5016 | // New instruction. It doesn't need to be kept. | |||
5017 | UncommutedMI->eraseFromParent(); | |||
5018 | return nullptr; | |||
5019 | } | |||
5020 | ||||
5021 | // Return here to prevent duplicate fuse failure report. | |||
5022 | return nullptr; | |||
5023 | } | |||
5024 | } | |||
5025 | ||||
5026 | // No fusion | |||
5027 | if (PrintFailedFusing && !MI.isCopy()) | |||
5028 | dbgs() << "We failed to fuse operand " << OpNum << " in " << MI; | |||
5029 | return nullptr; | |||
5030 | } | |||
5031 | ||||
5032 | MachineInstr * | |||
5033 | X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, | |||
5034 | ArrayRef<unsigned> Ops, | |||
5035 | MachineBasicBlock::iterator InsertPt, | |||
5036 | int FrameIndex, LiveIntervals *LIS) const { | |||
5037 | // Check switch flag | |||
5038 | if (NoFusing) | |||
5039 | return nullptr; | |||
5040 | ||||
5041 | // Avoid partial and undef register update stalls unless optimizing for size. | |||
5042 | if (!MF.getFunction().optForSize() && | |||
5043 | (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || | |||
5044 | shouldPreventUndefRegUpdateMemFold(MF, MI))) | |||
5045 | return nullptr; | |||
5046 | ||||
5047 | // Don't fold subreg spills, or reloads that use a high subreg. | |||
5048 | for (auto Op : Ops) { | |||
5049 | MachineOperand &MO = MI.getOperand(Op); | |||
5050 | auto SubReg = MO.getSubReg(); | |||
5051 | if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi)) | |||
5052 | return nullptr; | |||
5053 | } | |||
5054 | ||||
5055 | const MachineFrameInfo &MFI = MF.getFrameInfo(); | |||
5056 | unsigned Size = MFI.getObjectSize(FrameIndex); | |||
5057 | unsigned Alignment = MFI.getObjectAlignment(FrameIndex); | |||
5058 | // If the function stack isn't realigned we don't want to fold instructions | |||
5059 | // that need increased alignment. | |||
5060 | if (!RI.needsStackRealignment(MF)) | |||
5061 | Alignment = | |||
5062 | std::min(Alignment, Subtarget.getFrameLowering()->getStackAlignment()); | |||
5063 | if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { | |||
5064 | unsigned NewOpc = 0; | |||
5065 | unsigned RCSize = 0; | |||
5066 | switch (MI.getOpcode()) { | |||
5067 | default: return nullptr; | |||
5068 | case X86::TEST8rr: NewOpc = X86::CMP8ri; RCSize = 1; break; | |||
5069 | case X86::TEST16rr: NewOpc = X86::CMP16ri8; RCSize = 2; break; | |||
5070 | case X86::TEST32rr: NewOpc = X86::CMP32ri8; RCSize = 4; break; | |||
5071 | case X86::TEST64rr: NewOpc = X86::CMP64ri8; RCSize = 8; break; | |||
5072 | } | |||
5073 | // Check if it's safe to fold the load. If the size of the object is | |||
5074 | // narrower than the load width, then it's not. | |||
5075 | if (Size < RCSize) | |||
5076 | return nullptr; | |||
5077 | // Change to CMPXXri r, 0 first. | |||
5078 | MI.setDesc(get(NewOpc)); | |||
5079 | MI.getOperand(1).ChangeToImmediate(0); | |||
5080 | } else if (Ops.size() != 1) | |||
5081 | return nullptr; | |||
5082 | ||||
5083 | return foldMemoryOperandImpl(MF, MI, Ops[0], | |||
5084 | MachineOperand::CreateFI(FrameIndex), InsertPt, | |||
5085 | Size, Alignment, /*AllowCommute=*/true); | |||
5086 | } | |||
5087 | ||||
5088 | /// Check if \p LoadMI is a partial register load that we can't fold into \p MI | |||
5089 | /// because the latter uses contents that wouldn't be defined in the folded | |||
5090 | /// version. For instance, this transformation isn't legal: | |||
5091 | /// movss (%rdi), %xmm0 | |||
5092 | /// addps %xmm0, %xmm0 | |||
5093 | /// -> | |||
5094 | /// addps (%rdi), %xmm0 | |||
5095 | /// | |||
5096 | /// But this one is: | |||
5097 | /// movss (%rdi), %xmm0 | |||
5098 | /// addss %xmm0, %xmm0 | |||
5099 | /// -> | |||
5100 | /// addss (%rdi), %xmm0 | |||
5101 | /// | |||
5102 | static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, | |||
5103 | const MachineInstr &UserMI, | |||
5104 | const MachineFunction &MF) { | |||
5105 | unsigned Opc = LoadMI.getOpcode(); | |||
5106 | unsigned UserOpc = UserMI.getOpcode(); | |||
5107 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
5108 | const TargetRegisterClass *RC = | |||
5109 | MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg()); | |||
5110 | unsigned RegSize = TRI.getRegSizeInBits(*RC); | |||
5111 | ||||
5112 | if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm) && | |||
5113 | RegSize > 32) { | |||
5114 | // These instructions only load 32 bits, we can't fold them if the | |||
5115 | // destination register is wider than 32 bits (4 bytes), and its user | |||
5116 | // instruction isn't scalar (SS). | |||
5117 | switch (UserOpc) { | |||
5118 | case X86::ADDSSrr_Int: case X86::VADDSSrr_Int: case X86::VADDSSZrr_Int: | |||
5119 | case X86::CMPSSrr_Int: case X86::VCMPSSrr_Int: case X86::VCMPSSZrr_Int: | |||
5120 | case X86::DIVSSrr_Int: case X86::VDIVSSrr_Int: case X86::VDIVSSZrr_Int: | |||
5121 | case X86::MAXSSrr_Int: case X86::VMAXSSrr_Int: case X86::VMAXSSZrr_Int: | |||
5122 | case X86::MINSSrr_Int: case X86::VMINSSrr_Int: case X86::VMINSSZrr_Int: | |||
5123 | case X86::MULSSrr_Int: case X86::VMULSSrr_Int: case X86::VMULSSZrr_Int: | |||
5124 | case X86::SUBSSrr_Int: case X86::VSUBSSrr_Int: case X86::VSUBSSZrr_Int: | |||
5125 | case X86::VADDSSZrr_Intk: case X86::VADDSSZrr_Intkz: | |||
5126 | case X86::VDIVSSZrr_Intk: case X86::VDIVSSZrr_Intkz: | |||
5127 | case X86::VMAXSSZrr_Intk: case X86::VMAXSSZrr_Intkz: | |||
5128 | case X86::VMINSSZrr_Intk: case X86::VMINSSZrr_Intkz: | |||
5129 | case X86::VMULSSZrr_Intk: case X86::VMULSSZrr_Intkz: | |||
5130 | case X86::VSUBSSZrr_Intk: case X86::VSUBSSZrr_Intkz: | |||
5131 | case X86::VFMADDSS4rr_Int: case X86::VFNMADDSS4rr_Int: | |||
5132 | case X86::VFMSUBSS4rr_Int: case X86::VFNMSUBSS4rr_Int: | |||
5133 | case X86::VFMADD132SSr_Int: case X86::VFNMADD132SSr_Int: | |||
5134 | case X86::VFMADD213SSr_Int: case X86::VFNMADD213SSr_Int: | |||
5135 | case X86::VFMADD231SSr_Int: case X86::VFNMADD231SSr_Int: | |||
5136 | case X86::VFMSUB132SSr_Int: case X86::VFNMSUB132SSr_Int: | |||
5137 | case X86::VFMSUB213SSr_Int: case X86::VFNMSUB213SSr_Int: | |||
5138 | case X86::VFMSUB231SSr_Int: case X86::VFNMSUB231SSr_Int: | |||
5139 | case X86::VFMADD132SSZr_Int: case X86::VFNMADD132SSZr_Int: | |||
5140 | case X86::VFMADD213SSZr_Int: case X86::VFNMADD213SSZr_Int: | |||
5141 | case X86::VFMADD231SSZr_Int: case X86::VFNMADD231SSZr_Int: | |||
5142 | case X86::VFMSUB132SSZr_Int: case X86::VFNMSUB132SSZr_Int: | |||
5143 | case X86::VFMSUB213SSZr_Int: case X86::VFNMSUB213SSZr_Int: | |||
5144 | case X86::VFMSUB231SSZr_Int: case X86::VFNMSUB231SSZr_Int: | |||
5145 | case X86::VFMADD132SSZr_Intk: case X86::VFNMADD132SSZr_Intk: | |||
5146 | case X86::VFMADD213SSZr_Intk: case X86::VFNMADD213SSZr_Intk: | |||
5147 | case X86::VFMADD231SSZr_Intk: case X86::VFNMADD231SSZr_Intk: | |||
5148 | case X86::VFMSUB132SSZr_Intk: case X86::VFNMSUB132SSZr_Intk: | |||
5149 | case X86::VFMSUB213SSZr_Intk: case X86::VFNMSUB213SSZr_Intk: | |||
5150 | case X86::VFMSUB231SSZr_Intk: case X86::VFNMSUB231SSZr_Intk: | |||
5151 | case X86::VFMADD132SSZr_Intkz: case X86::VFNMADD132SSZr_Intkz: | |||
5152 | case X86::VFMADD213SSZr_Intkz: case X86::VFNMADD213SSZr_Intkz: | |||
5153 | case X86::VFMADD231SSZr_Intkz: case X86::VFNMADD231SSZr_Intkz: | |||
5154 | case X86::VFMSUB132SSZr_Intkz: case X86::VFNMSUB132SSZr_Intkz: | |||
5155 | case X86::VFMSUB213SSZr_Intkz: case X86::VFNMSUB213SSZr_Intkz: | |||
5156 | case X86::VFMSUB231SSZr_Intkz: case X86::VFNMSUB231SSZr_Intkz: | |||
5157 | return false; | |||
5158 | default: | |||
5159 | return true; | |||
5160 | } | |||
5161 | } | |||
5162 | ||||
5163 | if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm) && | |||
5164 | RegSize > 64) { | |||
5165 | // These instructions only load 64 bits, we can't fold them if the | |||
5166 | // destination register is wider than 64 bits (8 bytes), and its user | |||
5167 | // instruction isn't scalar (SD). | |||
5168 | switch (UserOpc) { | |||
5169 | case X86::ADDSDrr_Int: case X86::VADDSDrr_Int: case X86::VADDSDZrr_Int: | |||
5170 | case X86::CMPSDrr_Int: case X86::VCMPSDrr_Int: case X86::VCMPSDZrr_Int: | |||
5171 | case X86::DIVSDrr_Int: case X86::VDIVSDrr_Int: case X86::VDIVSDZrr_Int: | |||
5172 | case X86::MAXSDrr_Int: case X86::VMAXSDrr_Int: case X86::VMAXSDZrr_Int: | |||
5173 | case X86::MINSDrr_Int: case X86::VMINSDrr_Int: case X86::VMINSDZrr_Int: | |||
5174 | case X86::MULSDrr_Int: case X86::VMULSDrr_Int: case X86::VMULSDZrr_Int: | |||
5175 | case X86::SUBSDrr_Int: case X86::VSUBSDrr_Int: case X86::VSUBSDZrr_Int: | |||
5176 | case X86::VADDSDZrr_Intk: case X86::VADDSDZrr_Intkz: | |||
5177 | case X86::VDIVSDZrr_Intk: case X86::VDIVSDZrr_Intkz: | |||
5178 | case X86::VMAXSDZrr_Intk: case X86::VMAXSDZrr_Intkz: | |||
5179 | case X86::VMINSDZrr_Intk: case X86::VMINSDZrr_Intkz: | |||
5180 | case X86::VMULSDZrr_Intk: case X86::VMULSDZrr_Intkz: | |||
5181 | case X86::VSUBSDZrr_Intk: case X86::VSUBSDZrr_Intkz: | |||
5182 | case X86::VFMADDSD4rr_Int: case X86::VFNMADDSD4rr_Int: | |||
5183 | case X86::VFMSUBSD4rr_Int: case X86::VFNMSUBSD4rr_Int: | |||
5184 | case X86::VFMADD132SDr_Int: case X86::VFNMADD132SDr_Int: | |||
5185 | case X86::VFMADD213SDr_Int: case X86::VFNMADD213SDr_Int: | |||
5186 | case X86::VFMADD231SDr_Int: case X86::VFNMADD231SDr_Int: | |||
5187 | case X86::VFMSUB132SDr_Int: case X86::VFNMSUB132SDr_Int: | |||
5188 | case X86::VFMSUB213SDr_Int: case X86::VFNMSUB213SDr_Int: | |||
5189 | case X86::VFMSUB231SDr_Int: case X86::VFNMSUB231SDr_Int: | |||
5190 | case X86::VFMADD132SDZr_Int: case X86::VFNMADD132SDZr_Int: | |||
5191 | case X86::VFMADD213SDZr_Int: case X86::VFNMADD213SDZr_Int: | |||
5192 | case X86::VFMADD231SDZr_Int: case X86::VFNMADD231SDZr_Int: | |||
5193 | case X86::VFMSUB132SDZr_Int: case X86::VFNMSUB132SDZr_Int: | |||
5194 | case X86::VFMSUB213SDZr_Int: case X86::VFNMSUB213SDZr_Int: | |||
5195 | case X86::VFMSUB231SDZr_Int: case X86::VFNMSUB231SDZr_Int: | |||
5196 | case X86::VFMADD132SDZr_Intk: case X86::VFNMADD132SDZr_Intk: | |||
5197 | case X86::VFMADD213SDZr_Intk: case X86::VFNMADD213SDZr_Intk: | |||
5198 | case X86::VFMADD231SDZr_Intk: case X86::VFNMADD231SDZr_Intk: | |||
5199 | case X86::VFMSUB132SDZr_Intk: case X86::VFNMSUB132SDZr_Intk: | |||
5200 | case X86::VFMSUB213SDZr_Intk: case X86::VFNMSUB213SDZr_Intk: | |||
5201 | case X86::VFMSUB231SDZr_Intk: case X86::VFNMSUB231SDZr_Intk: | |||
5202 | case X86::VFMADD132SDZr_Intkz: case X86::VFNMADD132SDZr_Intkz: | |||
5203 | case X86::VFMADD213SDZr_Intkz: case X86::VFNMADD213SDZr_Intkz: | |||
5204 | case X86::VFMADD231SDZr_Intkz: case X86::VFNMADD231SDZr_Intkz: | |||
5205 | case X86::VFMSUB132SDZr_Intkz: case X86::VFNMSUB132SDZr_Intkz: | |||
5206 | case X86::VFMSUB213SDZr_Intkz: case X86::VFNMSUB213SDZr_Intkz: | |||
5207 | case X86::VFMSUB231SDZr_Intkz: case X86::VFNMSUB231SDZr_Intkz: | |||
5208 | return false; | |||
5209 | default: | |||
5210 | return true; | |||
5211 | } | |||
5212 | } | |||
5213 | ||||
5214 | return false; | |||
5215 | } | |||
5216 | ||||
5217 | MachineInstr *X86InstrInfo::foldMemoryOperandImpl( | |||
5218 | MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, | |||
5219 | MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, | |||
5220 | LiveIntervals *LIS) const { | |||
5221 | ||||
5222 | // TODO: Support the case where LoadMI loads a wide register, but MI | |||
5223 | // only uses a subreg. | |||
5224 | for (auto Op : Ops) { | |||
5225 | if (MI.getOperand(Op).getSubReg()) | |||
5226 | return nullptr; | |||
5227 | } | |||
5228 | ||||
5229 | // If loading from a FrameIndex, fold directly from the FrameIndex. | |||
5230 | unsigned NumOps = LoadMI.getDesc().getNumOperands(); | |||
5231 | int FrameIndex; | |||
5232 | if (isLoadFromStackSlot(LoadMI, FrameIndex)) { | |||
5233 | if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) | |||
5234 | return nullptr; | |||
5235 | return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, LIS); | |||
5236 | } | |||
5237 | ||||
5238 | // Check switch flag | |||
5239 | if (NoFusing) return nullptr; | |||
5240 | ||||
5241 | // Avoid partial and undef register update stalls unless optimizing for size. | |||
5242 | if (!MF.getFunction().optForSize() && | |||
5243 | (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || | |||
5244 | shouldPreventUndefRegUpdateMemFold(MF, MI))) | |||
5245 | return nullptr; | |||
5246 | ||||
5247 | // Determine the alignment of the load. | |||
5248 | unsigned Alignment = 0; | |||
5249 | if (LoadMI.hasOneMemOperand()) | |||
5250 | Alignment = (*LoadMI.memoperands_begin())->getAlignment(); | |||
5251 | else | |||
5252 | switch (LoadMI.getOpcode()) { | |||
5253 | case X86::AVX512_512_SET0: | |||
5254 | case X86::AVX512_512_SETALLONES: | |||
5255 | Alignment = 64; | |||
5256 | break; | |||
5257 | case X86::AVX2_SETALLONES: | |||
5258 | case X86::AVX1_SETALLONES: | |||
5259 | case X86::AVX_SET0: | |||
5260 | case X86::AVX512_256_SET0: | |||
5261 | Alignment = 32; | |||
5262 | break; | |||
5263 | case X86::V_SET0: | |||
5264 | case X86::V_SETALLONES: | |||
5265 | case X86::AVX512_128_SET0: | |||
5266 | Alignment = 16; | |||
5267 | break; | |||
5268 | case X86::MMX_SET0: | |||
5269 | case X86::FsFLD0SD: | |||
5270 | case X86::AVX512_FsFLD0SD: | |||
5271 | Alignment = 8; | |||
5272 | break; | |||
5273 | case X86::FsFLD0SS: | |||
5274 | case X86::AVX512_FsFLD0SS: | |||
5275 | Alignment = 4; | |||
5276 | break; | |||
5277 | default: | |||
5278 | return nullptr; | |||
5279 | } | |||
5280 | if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { | |||
5281 | unsigned NewOpc = 0; | |||
5282 | switch (MI.getOpcode()) { | |||
5283 | default: return nullptr; | |||
5284 | case X86::TEST8rr: NewOpc = X86::CMP8ri; break; | |||
5285 | case X86::TEST16rr: NewOpc = X86::CMP16ri8; break; | |||
5286 | case X86::TEST32rr: NewOpc = X86::CMP32ri8; break; | |||
5287 | case X86::TEST64rr: NewOpc = X86::CMP64ri8; break; | |||
5288 | } | |||
5289 | // Change to CMPXXri r, 0 first. | |||
5290 | MI.setDesc(get(NewOpc)); | |||
5291 | MI.getOperand(1).ChangeToImmediate(0); | |||
5292 | } else if (Ops.size() != 1) | |||
5293 | return nullptr; | |||
5294 | ||||
5295 | // Make sure the subregisters match. | |||
5296 | // Otherwise we risk changing the size of the load. | |||
5297 | if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg()) | |||
5298 | return nullptr; | |||
5299 | ||||
5300 | SmallVector<MachineOperand,X86::AddrNumOperands> MOs; | |||
5301 | switch (LoadMI.getOpcode()) { | |||
5302 | case X86::MMX_SET0: | |||
5303 | case X86::V_SET0: | |||
5304 | case X86::V_SETALLONES: | |||
5305 | case X86::AVX2_SETALLONES: | |||
5306 | case X86::AVX1_SETALLONES: | |||
5307 | case X86::AVX_SET0: | |||
5308 | case X86::AVX512_128_SET0: | |||
5309 | case X86::AVX512_256_SET0: | |||
5310 | case X86::AVX512_512_SET0: | |||
5311 | case X86::AVX512_512_SETALLONES: | |||
5312 | case X86::FsFLD0SD: | |||
5313 | case X86::AVX512_FsFLD0SD: | |||
5314 | case X86::FsFLD0SS: | |||
5315 | case X86::AVX512_FsFLD0SS: { | |||
5316 | // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure. | |||
5317 | // Create a constant-pool entry and operands to load from it. | |||
5318 | ||||
5319 | // Medium and large mode can't fold loads this way. | |||
5320 | if (MF.getTarget().getCodeModel() != CodeModel::Small && | |||
5321 | MF.getTarget().getCodeModel() != CodeModel::Kernel) | |||
5322 | return nullptr; | |||
5323 | ||||
5324 | // x86-32 PIC requires a PIC base register for constant pools. | |||
5325 | unsigned PICBase = 0; | |||
5326 | if (MF.getTarget().isPositionIndependent()) { | |||
5327 | if (Subtarget.is64Bit()) | |||
5328 | PICBase = X86::RIP; | |||
5329 | else | |||
5330 | // FIXME: PICBase = getGlobalBaseReg(&MF); | |||
5331 | // This doesn't work for several reasons. | |||
5332 | // 1. GlobalBaseReg may have been spilled. | |||
5333 | // 2. It may not be live at MI. | |||
5334 | return nullptr; | |||
5335 | } | |||
5336 | ||||
5337 | // Create a constant-pool entry. | |||
5338 | MachineConstantPool &MCP = *MF.getConstantPool(); | |||
5339 | Type *Ty; | |||
5340 | unsigned Opc = LoadMI.getOpcode(); | |||
5341 | if (Opc == X86::FsFLD0SS || Opc == X86::AVX512_FsFLD0SS) | |||
5342 | Ty = Type::getFloatTy(MF.getFunction().getContext()); | |||
5343 | else if (Opc == X86::FsFLD0SD || Opc == X86::AVX512_FsFLD0SD) | |||
5344 | Ty = Type::getDoubleTy(MF.getFunction().getContext()); | |||
5345 | else if (Opc == X86::AVX512_512_SET0 || Opc == X86::AVX512_512_SETALLONES) | |||
5346 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()),16); | |||
5347 | else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0 || | |||
5348 | Opc == X86::AVX512_256_SET0 || Opc == X86::AVX1_SETALLONES) | |||
5349 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 8); | |||
5350 | else if (Opc == X86::MMX_SET0) | |||
5351 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 2); | |||
5352 | else | |||
5353 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 4); | |||
5354 | ||||
5355 | bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES || | |||
5356 | Opc == X86::AVX512_512_SETALLONES || | |||
5357 | Opc == X86::AVX1_SETALLONES); | |||
5358 | const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) : | |||
5359 | Constant::getNullValue(Ty); | |||
5360 | unsigned CPI = MCP.getConstantPoolIndex(C, Alignment); | |||
5361 | ||||
5362 | // Create operands to load from the constant pool entry. | |||
5363 | MOs.push_back(MachineOperand::CreateReg(PICBase, false)); | |||
5364 | MOs.push_back(MachineOperand::CreateImm(1)); | |||
5365 | MOs.push_back(MachineOperand::CreateReg(0, false)); | |||
5366 | MOs.push_back(MachineOperand::CreateCPI(CPI, 0)); | |||
5367 | MOs.push_back(MachineOperand::CreateReg(0, false)); | |||
5368 | break; | |||
5369 | } | |||
5370 | default: { | |||
5371 | if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) | |||
5372 | return nullptr; | |||
5373 | ||||
5374 | // Folding a normal load. Just copy the load's address operands. | |||
5375 | MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, | |||
5376 | LoadMI.operands_begin() + NumOps); | |||
5377 | break; | |||
5378 | } | |||
5379 | } | |||
5380 | return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt, | |||
5381 | /*Size=*/0, Alignment, /*AllowCommute=*/true); | |||
5382 | } | |||
5383 | ||||
5384 | static SmallVector<MachineMemOperand *, 2> | |||
5385 | extractLoadMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { | |||
5386 | SmallVector<MachineMemOperand *, 2> LoadMMOs; | |||
5387 | ||||
5388 | for (MachineMemOperand *MMO : MMOs) { | |||
5389 | if (!MMO->isLoad()) | |||
5390 | continue; | |||
5391 | ||||
5392 | if (!MMO->isStore()) { | |||
5393 | // Reuse the MMO. | |||
5394 | LoadMMOs.push_back(MMO); | |||
5395 | } else { | |||
5396 | // Clone the MMO and unset the store flag. | |||
5397 | LoadMMOs.push_back(MF.getMachineMemOperand( | |||
5398 | MMO->getPointerInfo(), MMO->getFlags() & ~MachineMemOperand::MOStore, | |||
5399 | MMO->getSize(), MMO->getBaseAlignment(), MMO->getAAInfo(), nullptr, | |||
5400 | MMO->getSyncScopeID(), MMO->getOrdering(), | |||
5401 | MMO->getFailureOrdering())); | |||
5402 | } | |||
5403 | } | |||
5404 | ||||
5405 | return LoadMMOs; | |||
5406 | } | |||
5407 | ||||
5408 | static SmallVector<MachineMemOperand *, 2> | |||
5409 | extractStoreMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { | |||
5410 | SmallVector<MachineMemOperand *, 2> StoreMMOs; | |||
5411 | ||||
5412 | for (MachineMemOperand *MMO : MMOs) { | |||
5413 | if (!MMO->isStore()) | |||
5414 | continue; | |||
5415 | ||||
5416 | if (!MMO->isLoad()) { | |||
5417 | // Reuse the MMO. | |||
5418 | StoreMMOs.push_back(MMO); | |||
5419 | } else { | |||
5420 | // Clone the MMO and unset the load flag. | |||
5421 | StoreMMOs.push_back(MF.getMachineMemOperand( | |||
5422 | MMO->getPointerInfo(), MMO->getFlags() & ~MachineMemOperand::MOLoad, | |||
5423 | MMO->getSize(), MMO->getBaseAlignment(), MMO->getAAInfo(), nullptr, | |||
5424 | MMO->getSyncScopeID(), MMO->getOrdering(), | |||
5425 | MMO->getFailureOrdering())); | |||
5426 | } | |||
5427 | } | |||
5428 | ||||
5429 | return StoreMMOs; | |||
5430 | } | |||
5431 | ||||
5432 | bool X86InstrInfo::unfoldMemoryOperand( | |||
5433 | MachineFunction &MF, MachineInstr &MI, unsigned Reg, bool UnfoldLoad, | |||
5434 | bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const { | |||
5435 | const X86MemoryFoldTableEntry *I = lookupUnfoldTable(MI.getOpcode()); | |||
5436 | if (I == nullptr) | |||
5437 | return false; | |||
5438 | unsigned Opc = I->DstOp; | |||
5439 | unsigned Index = I->Flags & TB_INDEX_MASK; | |||
5440 | bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; | |||
5441 | bool FoldedStore = I->Flags & TB_FOLDED_STORE; | |||
5442 | if (UnfoldLoad && !FoldedLoad) | |||
5443 | return false; | |||
5444 | UnfoldLoad &= FoldedLoad; | |||
5445 | if (UnfoldStore && !FoldedStore) | |||
5446 | return false; | |||
5447 | UnfoldStore &= FoldedStore; | |||
5448 | ||||
5449 | const MCInstrDesc &MCID = get(Opc); | |||
5450 | const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); | |||
5451 | // TODO: Check if 32-byte or greater accesses are slow too? | |||
5452 | if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass && | |||
5453 | Subtarget.isUnalignedMem16Slow()) | |||
5454 | // Without memoperands, loadRegFromAddr and storeRegToStackSlot will | |||
5455 | // conservatively assume the address is unaligned. That's bad for | |||
5456 | // performance. | |||
5457 | return false; | |||
5458 | SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps; | |||
5459 | SmallVector<MachineOperand,2> BeforeOps; | |||
5460 | SmallVector<MachineOperand,2> AfterOps; | |||
5461 | SmallVector<MachineOperand,4> ImpOps; | |||
5462 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
5463 | MachineOperand &Op = MI.getOperand(i); | |||
5464 | if (i >= Index && i < Index + X86::AddrNumOperands) | |||
5465 | AddrOps.push_back(Op); | |||
5466 | else if (Op.isReg() && Op.isImplicit()) | |||
5467 | ImpOps.push_back(Op); | |||
5468 | else if (i < Index) | |||
5469 | BeforeOps.push_back(Op); | |||
5470 | else if (i > Index) | |||
5471 | AfterOps.push_back(Op); | |||
5472 | } | |||
5473 | ||||
5474 | // Emit the load instruction. | |||
5475 | if (UnfoldLoad) { | |||
5476 | auto MMOs = extractLoadMMOs(MI.memoperands(), MF); | |||
5477 | loadRegFromAddr(MF, Reg, AddrOps, RC, MMOs, NewMIs); | |||
5478 | if (UnfoldStore) { | |||
5479 | // Address operands cannot be marked isKill. | |||
5480 | for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) { | |||
5481 | MachineOperand &MO = NewMIs[0]->getOperand(i); | |||
5482 | if (MO.isReg()) | |||
5483 | MO.setIsKill(false); | |||
5484 | } | |||
5485 | } | |||
5486 | } | |||
5487 | ||||
5488 | // Emit the data processing instruction. | |||
5489 | MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI.getDebugLoc(), true); | |||
5490 | MachineInstrBuilder MIB(MF, DataMI); | |||
5491 | ||||
5492 | if (FoldedStore) | |||
5493 | MIB.addReg(Reg, RegState::Define); | |||
5494 | for (MachineOperand &BeforeOp : BeforeOps) | |||
5495 | MIB.add(BeforeOp); | |||
5496 | if (FoldedLoad) | |||
5497 | MIB.addReg(Reg); | |||
5498 | for (MachineOperand &AfterOp : AfterOps) | |||
5499 | MIB.add(AfterOp); | |||
5500 | for (MachineOperand &ImpOp : ImpOps) { | |||
5501 | MIB.addReg(ImpOp.getReg(), | |||
5502 | getDefRegState(ImpOp.isDef()) | | |||
5503 | RegState::Implicit | | |||
5504 | getKillRegState(ImpOp.isKill()) | | |||
5505 | getDeadRegState(ImpOp.isDead()) | | |||
5506 | getUndefRegState(ImpOp.isUndef())); | |||
5507 | } | |||
5508 | // Change CMP32ri r, 0 back to TEST32rr r, r, etc. | |||
5509 | switch (DataMI->getOpcode()) { | |||
5510 | default: break; | |||
5511 | case X86::CMP64ri32: | |||
5512 | case X86::CMP64ri8: | |||
5513 | case X86::CMP32ri: | |||
5514 | case X86::CMP32ri8: | |||
5515 | case X86::CMP16ri: | |||
5516 | case X86::CMP16ri8: | |||
5517 | case X86::CMP8ri: { | |||
5518 | MachineOperand &MO0 = DataMI->getOperand(0); | |||
5519 | MachineOperand &MO1 = DataMI->getOperand(1); | |||
5520 | if (MO1.getImm() == 0) { | |||
5521 | unsigned NewOpc; | |||
5522 | switch (DataMI->getOpcode()) { | |||
5523 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5523); | |||
5524 | case X86::CMP64ri8: | |||
5525 | case X86::CMP64ri32: NewOpc = X86::TEST64rr; break; | |||
5526 | case X86::CMP32ri8: | |||
5527 | case X86::CMP32ri: NewOpc = X86::TEST32rr; break; | |||
5528 | case X86::CMP16ri8: | |||
5529 | case X86::CMP16ri: NewOpc = X86::TEST16rr; break; | |||
5530 | case X86::CMP8ri: NewOpc = X86::TEST8rr; break; | |||
5531 | } | |||
5532 | DataMI->setDesc(get(NewOpc)); | |||
5533 | MO1.ChangeToRegister(MO0.getReg(), false); | |||
5534 | } | |||
5535 | } | |||
5536 | } | |||
5537 | NewMIs.push_back(DataMI); | |||
5538 | ||||
5539 | // Emit the store instruction. | |||
5540 | if (UnfoldStore) { | |||
5541 | const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF); | |||
5542 | auto MMOs = extractStoreMMOs(MI.memoperands(), MF); | |||
5543 | storeRegToAddr(MF, Reg, true, AddrOps, DstRC, MMOs, NewMIs); | |||
5544 | } | |||
5545 | ||||
5546 | return true; | |||
5547 | } | |||
5548 | ||||
5549 | bool | |||
5550 | X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, | |||
5551 | SmallVectorImpl<SDNode*> &NewNodes) const { | |||
5552 | if (!N->isMachineOpcode()) | |||
5553 | return false; | |||
5554 | ||||
5555 | const X86MemoryFoldTableEntry *I = lookupUnfoldTable(N->getMachineOpcode()); | |||
5556 | if (I == nullptr) | |||
5557 | return false; | |||
5558 | unsigned Opc = I->DstOp; | |||
5559 | unsigned Index = I->Flags & TB_INDEX_MASK; | |||
5560 | bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; | |||
5561 | bool FoldedStore = I->Flags & TB_FOLDED_STORE; | |||
5562 | const MCInstrDesc &MCID = get(Opc); | |||
5563 | MachineFunction &MF = DAG.getMachineFunction(); | |||
5564 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
5565 | const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); | |||
5566 | unsigned NumDefs = MCID.NumDefs; | |||
5567 | std::vector<SDValue> AddrOps; | |||
5568 | std::vector<SDValue> BeforeOps; | |||
5569 | std::vector<SDValue> AfterOps; | |||
5570 | SDLoc dl(N); | |||
5571 | unsigned NumOps = N->getNumOperands(); | |||
5572 | for (unsigned i = 0; i != NumOps-1; ++i) { | |||
5573 | SDValue Op = N->getOperand(i); | |||
5574 | if (i >= Index-NumDefs && i < Index-NumDefs + X86::AddrNumOperands) | |||
5575 | AddrOps.push_back(Op); | |||
5576 | else if (i < Index-NumDefs) | |||
5577 | BeforeOps.push_back(Op); | |||
5578 | else if (i > Index-NumDefs) | |||
5579 | AfterOps.push_back(Op); | |||
5580 | } | |||
5581 | SDValue Chain = N->getOperand(NumOps-1); | |||
5582 | AddrOps.push_back(Chain); | |||
5583 | ||||
5584 | // Emit the load instruction. | |||
5585 | SDNode *Load = nullptr; | |||
5586 | if (FoldedLoad) { | |||
5587 | EVT VT = *TRI.legalclasstypes_begin(*RC); | |||
5588 | auto MMOs = extractLoadMMOs(cast<MachineSDNode>(N)->memoperands(), MF); | |||
5589 | if (MMOs.empty() && RC == &X86::VR128RegClass && | |||
5590 | Subtarget.isUnalignedMem16Slow()) | |||
5591 | // Do not introduce a slow unaligned load. | |||
5592 | return false; | |||
5593 | // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte | |||
5594 | // memory access is slow above. | |||
5595 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
5596 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
5597 | Load = DAG.getMachineNode(getLoadRegOpcode(0, RC, isAligned, Subtarget), dl, | |||
5598 | VT, MVT::Other, AddrOps); | |||
5599 | NewNodes.push_back(Load); | |||
5600 | ||||
5601 | // Preserve memory reference information. | |||
5602 | DAG.setNodeMemRefs(cast<MachineSDNode>(Load), MMOs); | |||
5603 | } | |||
5604 | ||||
5605 | // Emit the data processing instruction. | |||
5606 | std::vector<EVT> VTs; | |||
5607 | const TargetRegisterClass *DstRC = nullptr; | |||
5608 | if (MCID.getNumDefs() > 0) { | |||
5609 | DstRC = getRegClass(MCID, 0, &RI, MF); | |||
5610 | VTs.push_back(*TRI.legalclasstypes_begin(*DstRC)); | |||
5611 | } | |||
5612 | for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { | |||
5613 | EVT VT = N->getValueType(i); | |||
5614 | if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs()) | |||
5615 | VTs.push_back(VT); | |||
5616 | } | |||
5617 | if (Load) | |||
5618 | BeforeOps.push_back(SDValue(Load, 0)); | |||
5619 | BeforeOps.insert(BeforeOps.end(), AfterOps.begin(), AfterOps.end()); | |||
5620 | // Change CMP32ri r, 0 back to TEST32rr r, r, etc. | |||
5621 | switch (Opc) { | |||
5622 | default: break; | |||
5623 | case X86::CMP64ri32: | |||
5624 | case X86::CMP64ri8: | |||
5625 | case X86::CMP32ri: | |||
5626 | case X86::CMP32ri8: | |||
5627 | case X86::CMP16ri: | |||
5628 | case X86::CMP16ri8: | |||
5629 | case X86::CMP8ri: | |||
5630 | if (isNullConstant(BeforeOps[1])) { | |||
5631 | switch (Opc) { | |||
5632 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5632); | |||
5633 | case X86::CMP64ri8: | |||
5634 | case X86::CMP64ri32: Opc = X86::TEST64rr; break; | |||
5635 | case X86::CMP32ri8: | |||
5636 | case X86::CMP32ri: Opc = X86::TEST32rr; break; | |||
5637 | case X86::CMP16ri8: | |||
5638 | case X86::CMP16ri: Opc = X86::TEST16rr; break; | |||
5639 | case X86::CMP8ri: Opc = X86::TEST8rr; break; | |||
5640 | } | |||
5641 | BeforeOps[1] = BeforeOps[0]; | |||
5642 | } | |||
5643 | } | |||
5644 | SDNode *NewNode= DAG.getMachineNode(Opc, dl, VTs, BeforeOps); | |||
5645 | NewNodes.push_back(NewNode); | |||
5646 | ||||
5647 | // Emit the store instruction. | |||
5648 | if (FoldedStore) { | |||
5649 | AddrOps.pop_back(); | |||
5650 | AddrOps.push_back(SDValue(NewNode, 0)); | |||
5651 | AddrOps.push_back(Chain); | |||
5652 | auto MMOs = extractStoreMMOs(cast<MachineSDNode>(N)->memoperands(), MF); | |||
5653 | if (MMOs.empty() && RC == &X86::VR128RegClass && | |||
5654 | Subtarget.isUnalignedMem16Slow()) | |||
5655 | // Do not introduce a slow unaligned store. | |||
5656 | return false; | |||
5657 | // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte | |||
5658 | // memory access is slow above. | |||
5659 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
5660 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
5661 | SDNode *Store = | |||
5662 | DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget), | |||
5663 | dl, MVT::Other, AddrOps); | |||
5664 | NewNodes.push_back(Store); | |||
5665 | ||||
5666 | // Preserve memory reference information. | |||
5667 | DAG.setNodeMemRefs(cast<MachineSDNode>(Store), MMOs); | |||
5668 | } | |||
5669 | ||||
5670 | return true; | |||
5671 | } | |||
5672 | ||||
5673 | unsigned X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc, | |||
5674 | bool UnfoldLoad, bool UnfoldStore, | |||
5675 | unsigned *LoadRegIndex) const { | |||
5676 | const X86MemoryFoldTableEntry *I = lookupUnfoldTable(Opc); | |||
5677 | if (I == nullptr) | |||
5678 | return 0; | |||
5679 | bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; | |||
5680 | bool FoldedStore = I->Flags & TB_FOLDED_STORE; | |||
5681 | if (UnfoldLoad && !FoldedLoad) | |||
5682 | return 0; | |||
5683 | if (UnfoldStore && !FoldedStore) | |||
5684 | return 0; | |||
5685 | if (LoadRegIndex) | |||
5686 | *LoadRegIndex = I->Flags & TB_INDEX_MASK; | |||
5687 | return I->DstOp; | |||
5688 | } | |||
5689 | ||||
5690 | bool | |||
5691 | X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, | |||
5692 | int64_t &Offset1, int64_t &Offset2) const { | |||
5693 | if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) | |||
5694 | return false; | |||
5695 | unsigned Opc1 = Load1->getMachineOpcode(); | |||
5696 | unsigned Opc2 = Load2->getMachineOpcode(); | |||
5697 | switch (Opc1) { | |||
5698 | default: return false; | |||
5699 | case X86::MOV8rm: | |||
5700 | case X86::MOV16rm: | |||
5701 | case X86::MOV32rm: | |||
5702 | case X86::MOV64rm: | |||
5703 | case X86::LD_Fp32m: | |||
5704 | case X86::LD_Fp64m: | |||
5705 | case X86::LD_Fp80m: | |||
5706 | case X86::MOVSSrm: | |||
5707 | case X86::MOVSDrm: | |||
5708 | case X86::MMX_MOVD64rm: | |||
5709 | case X86::MMX_MOVQ64rm: | |||
5710 | case X86::MOVAPSrm: | |||
5711 | case X86::MOVUPSrm: | |||
5712 | case X86::MOVAPDrm: | |||
5713 | case X86::MOVUPDrm: | |||
5714 | case X86::MOVDQArm: | |||
5715 | case X86::MOVDQUrm: | |||
5716 | // AVX load instructions | |||
5717 | case X86::VMOVSSrm: | |||
5718 | case X86::VMOVSDrm: | |||
5719 | case X86::VMOVAPSrm: | |||
5720 | case X86::VMOVUPSrm: | |||
5721 | case X86::VMOVAPDrm: | |||
5722 | case X86::VMOVUPDrm: | |||
5723 | case X86::VMOVDQArm: | |||
5724 | case X86::VMOVDQUrm: | |||
5725 | case X86::VMOVAPSYrm: | |||
5726 | case X86::VMOVUPSYrm: | |||
5727 | case X86::VMOVAPDYrm: | |||
5728 | case X86::VMOVUPDYrm: | |||
5729 | case X86::VMOVDQAYrm: | |||
5730 | case X86::VMOVDQUYrm: | |||
5731 | // AVX512 load instructions | |||
5732 | case X86::VMOVSSZrm: | |||
5733 | case X86::VMOVSDZrm: | |||
5734 | case X86::VMOVAPSZ128rm: | |||
5735 | case X86::VMOVUPSZ128rm: | |||
5736 | case X86::VMOVAPSZ128rm_NOVLX: | |||
5737 | case X86::VMOVUPSZ128rm_NOVLX: | |||
5738 | case X86::VMOVAPDZ128rm: | |||
5739 | case X86::VMOVUPDZ128rm: | |||
5740 | case X86::VMOVDQU8Z128rm: | |||
5741 | case X86::VMOVDQU16Z128rm: | |||
5742 | case X86::VMOVDQA32Z128rm: | |||
5743 | case X86::VMOVDQU32Z128rm: | |||
5744 | case X86::VMOVDQA64Z128rm: | |||
5745 | case X86::VMOVDQU64Z128rm: | |||
5746 | case X86::VMOVAPSZ256rm: | |||
5747 | case X86::VMOVUPSZ256rm: | |||
5748 | case X86::VMOVAPSZ256rm_NOVLX: | |||
5749 | case X86::VMOVUPSZ256rm_NOVLX: | |||
5750 | case X86::VMOVAPDZ256rm: | |||
5751 | case X86::VMOVUPDZ256rm: | |||
5752 | case X86::VMOVDQU8Z256rm: | |||
5753 | case X86::VMOVDQU16Z256rm: | |||
5754 | case X86::VMOVDQA32Z256rm: | |||
5755 | case X86::VMOVDQU32Z256rm: | |||
5756 | case X86::VMOVDQA64Z256rm: | |||
5757 | case X86::VMOVDQU64Z256rm: | |||
5758 | case X86::VMOVAPSZrm: | |||
5759 | case X86::VMOVUPSZrm: | |||
5760 | case X86::VMOVAPDZrm: | |||
5761 | case X86::VMOVUPDZrm: | |||
5762 | case X86::VMOVDQU8Zrm: | |||
5763 | case X86::VMOVDQU16Zrm: | |||
5764 | case X86::VMOVDQA32Zrm: | |||
5765 | case X86::VMOVDQU32Zrm: | |||
5766 | case X86::VMOVDQA64Zrm: | |||
5767 | case X86::VMOVDQU64Zrm: | |||
5768 | case X86::KMOVBkm: | |||
5769 | case X86::KMOVWkm: | |||
5770 | case X86::KMOVDkm: | |||
5771 | case X86::KMOVQkm: | |||
5772 | break; | |||
5773 | } | |||
5774 | switch (Opc2) { | |||
5775 | default: return false; | |||
5776 | case X86::MOV8rm: | |||
5777 | case X86::MOV16rm: | |||
5778 | case X86::MOV32rm: | |||
5779 | case X86::MOV64rm: | |||
5780 | case X86::LD_Fp32m: | |||
5781 | case X86::LD_Fp64m: | |||
5782 | case X86::LD_Fp80m: | |||
5783 | case X86::MOVSSrm: | |||
5784 | case X86::MOVSDrm: | |||
5785 | case X86::MMX_MOVD64rm: | |||
5786 | case X86::MMX_MOVQ64rm: | |||
5787 | case X86::MOVAPSrm: | |||
5788 | case X86::MOVUPSrm: | |||
5789 | case X86::MOVAPDrm: | |||
5790 | case X86::MOVUPDrm: | |||
5791 | case X86::MOVDQArm: | |||
5792 | case X86::MOVDQUrm: | |||
5793 | // AVX load instructions | |||
5794 | case X86::VMOVSSrm: | |||
5795 | case X86::VMOVSDrm: | |||
5796 | case X86::VMOVAPSrm: | |||
5797 | case X86::VMOVUPSrm: | |||
5798 | case X86::VMOVAPDrm: | |||
5799 | case X86::VMOVUPDrm: | |||
5800 | case X86::VMOVDQArm: | |||
5801 | case X86::VMOVDQUrm: | |||
5802 | case X86::VMOVAPSYrm: | |||
5803 | case X86::VMOVUPSYrm: | |||
5804 | case X86::VMOVAPDYrm: | |||
5805 | case X86::VMOVUPDYrm: | |||
5806 | case X86::VMOVDQAYrm: | |||
5807 | case X86::VMOVDQUYrm: | |||
5808 | // AVX512 load instructions | |||
5809 | case X86::VMOVSSZrm: | |||
5810 | case X86::VMOVSDZrm: | |||
5811 | case X86::VMOVAPSZ128rm: | |||
5812 | case X86::VMOVUPSZ128rm: | |||
5813 | case X86::VMOVAPSZ128rm_NOVLX: | |||
5814 | case X86::VMOVUPSZ128rm_NOVLX: | |||
5815 | case X86::VMOVAPDZ128rm: | |||
5816 | case X86::VMOVUPDZ128rm: | |||
5817 | case X86::VMOVDQU8Z128rm: | |||
5818 | case X86::VMOVDQU16Z128rm: | |||
5819 | case X86::VMOVDQA32Z128rm: | |||
5820 | case X86::VMOVDQU32Z128rm: | |||
5821 | case X86::VMOVDQA64Z128rm: | |||
5822 | case X86::VMOVDQU64Z128rm: | |||
5823 | case X86::VMOVAPSZ256rm: | |||
5824 | case X86::VMOVUPSZ256rm: | |||
5825 | case X86::VMOVAPSZ256rm_NOVLX: | |||
5826 | case X86::VMOVUPSZ256rm_NOVLX: | |||
5827 | case X86::VMOVAPDZ256rm: | |||
5828 | case X86::VMOVUPDZ256rm: | |||
5829 | case X86::VMOVDQU8Z256rm: | |||
5830 | case X86::VMOVDQU16Z256rm: | |||
5831 | case X86::VMOVDQA32Z256rm: | |||
5832 | case X86::VMOVDQU32Z256rm: | |||
5833 | case X86::VMOVDQA64Z256rm: | |||
5834 | case X86::VMOVDQU64Z256rm: | |||
5835 | case X86::VMOVAPSZrm: | |||
5836 | case X86::VMOVUPSZrm: | |||
5837 | case X86::VMOVAPDZrm: | |||
5838 | case X86::VMOVUPDZrm: | |||
5839 | case X86::VMOVDQU8Zrm: | |||
5840 | case X86::VMOVDQU16Zrm: | |||
5841 | case X86::VMOVDQA32Zrm: | |||
5842 | case X86::VMOVDQU32Zrm: | |||
5843 | case X86::VMOVDQA64Zrm: | |||
5844 | case X86::VMOVDQU64Zrm: | |||
5845 | case X86::KMOVBkm: | |||
5846 | case X86::KMOVWkm: | |||
5847 | case X86::KMOVDkm: | |||
5848 | case X86::KMOVQkm: | |||
5849 | break; | |||
5850 | } | |||
5851 | ||||
5852 | // Lambda to check if both the loads have the same value for an operand index. | |||
5853 | auto HasSameOp = [&](int I) { | |||
5854 | return Load1->getOperand(I) == Load2->getOperand(I); | |||
5855 | }; | |||
5856 | ||||
5857 | // All operands except the displacement should match. | |||
5858 | if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) || | |||
5859 | !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg)) | |||
5860 | return false; | |||
5861 | ||||
5862 | // Chain Operand must be the same. | |||
5863 | if (!HasSameOp(5)) | |||
5864 | return false; | |||
5865 | ||||
5866 | // Now let's examine if the displacements are constants. | |||
5867 | auto Disp1 = dyn_cast<ConstantSDNode>(Load1->getOperand(X86::AddrDisp)); | |||
5868 | auto Disp2 = dyn_cast<ConstantSDNode>(Load2->getOperand(X86::AddrDisp)); | |||
5869 | if (!Disp1 || !Disp2) | |||
5870 | return false; | |||
5871 | ||||
5872 | Offset1 = Disp1->getSExtValue(); | |||
5873 | Offset2 = Disp2->getSExtValue(); | |||
5874 | return true; | |||
5875 | } | |||
5876 | ||||
5877 | bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, | |||
5878 | int64_t Offset1, int64_t Offset2, | |||
5879 | unsigned NumLoads) const { | |||
5880 | assert(Offset2 > Offset1)((Offset2 > Offset1) ? static_cast<void> (0) : __assert_fail ("Offset2 > Offset1", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5880, __PRETTY_FUNCTION__)); | |||
5881 | if ((Offset2 - Offset1) / 8 > 64) | |||
5882 | return false; | |||
5883 | ||||
5884 | unsigned Opc1 = Load1->getMachineOpcode(); | |||
5885 | unsigned Opc2 = Load2->getMachineOpcode(); | |||
5886 | if (Opc1 != Opc2) | |||
5887 | return false; // FIXME: overly conservative? | |||
5888 | ||||
5889 | switch (Opc1) { | |||
5890 | default: break; | |||
5891 | case X86::LD_Fp32m: | |||
5892 | case X86::LD_Fp64m: | |||
5893 | case X86::LD_Fp80m: | |||
5894 | case X86::MMX_MOVD64rm: | |||
5895 | case X86::MMX_MOVQ64rm: | |||
5896 | return false; | |||
5897 | } | |||
5898 | ||||
5899 | EVT VT = Load1->getValueType(0); | |||
5900 | switch (VT.getSimpleVT().SimpleTy) { | |||
5901 | default: | |||
5902 | // XMM registers. In 64-bit mode we can be a bit more aggressive since we | |||
5903 | // have 16 of them to play with. | |||
5904 | if (Subtarget.is64Bit()) { | |||
5905 | if (NumLoads >= 3) | |||
5906 | return false; | |||
5907 | } else if (NumLoads) { | |||
5908 | return false; | |||
5909 | } | |||
5910 | break; | |||
5911 | case MVT::i8: | |||
5912 | case MVT::i16: | |||
5913 | case MVT::i32: | |||
5914 | case MVT::i64: | |||
5915 | case MVT::f32: | |||
5916 | case MVT::f64: | |||
5917 | if (NumLoads) | |||
5918 | return false; | |||
5919 | break; | |||
5920 | } | |||
5921 | ||||
5922 | return true; | |||
5923 | } | |||
5924 | ||||
5925 | bool X86InstrInfo:: | |||
5926 | reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { | |||
5927 | assert(Cond.size() == 1 && "Invalid X86 branch condition!")((Cond.size() == 1 && "Invalid X86 branch condition!" ) ? static_cast<void> (0) : __assert_fail ("Cond.size() == 1 && \"Invalid X86 branch condition!\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5927, __PRETTY_FUNCTION__)); | |||
5928 | X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm()); | |||
5929 | Cond[0].setImm(GetOppositeBranchCondition(CC)); | |||
5930 | return false; | |||
5931 | } | |||
5932 | ||||
5933 | bool X86InstrInfo:: | |||
5934 | isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const { | |||
5935 | // FIXME: Return false for x87 stack register classes for now. We can't | |||
5936 | // allow any loads of these registers before FpGet_ST0_80. | |||
5937 | return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass || | |||
5938 | RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass || | |||
5939 | RC == &X86::RFP80RegClass); | |||
5940 | } | |||
5941 | ||||
5942 | /// Return a virtual register initialized with the | |||
5943 | /// the global base register value. Output instructions required to | |||
5944 | /// initialize the register in the function entry block, if necessary. | |||
5945 | /// | |||
5946 | /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. | |||
5947 | /// | |||
5948 | unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { | |||
5949 | assert((!Subtarget.is64Bit() ||(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5952, __PRETTY_FUNCTION__)) | |||
5950 | MF->getTarget().getCodeModel() == CodeModel::Medium ||(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5952, __PRETTY_FUNCTION__)) | |||
5951 | MF->getTarget().getCodeModel() == CodeModel::Large) &&(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5952, __PRETTY_FUNCTION__)) | |||
5952 | "X86-64 PIC uses RIP relative addressing")(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 5952, __PRETTY_FUNCTION__)); | |||
5953 | ||||
5954 | X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); | |||
5955 | unsigned GlobalBaseReg = X86FI->getGlobalBaseReg(); | |||
5956 | if (GlobalBaseReg != 0) | |||
5957 | return GlobalBaseReg; | |||
5958 | ||||
5959 | // Create the register. The code to initialize it is inserted | |||
5960 | // later, by the CGBR pass (below). | |||
5961 | MachineRegisterInfo &RegInfo = MF->getRegInfo(); | |||
5962 | GlobalBaseReg = RegInfo.createVirtualRegister( | |||
5963 | Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass); | |||
5964 | X86FI->setGlobalBaseReg(GlobalBaseReg); | |||
5965 | return GlobalBaseReg; | |||
5966 | } | |||
5967 | ||||
5968 | // These are the replaceable SSE instructions. Some of these have Int variants | |||
5969 | // that we don't include here. We don't want to replace instructions selected | |||
5970 | // by intrinsics. | |||
5971 | static const uint16_t ReplaceableInstrs[][3] = { | |||
5972 | //PackedSingle PackedDouble PackedInt | |||
5973 | { X86::MOVAPSmr, X86::MOVAPDmr, X86::MOVDQAmr }, | |||
5974 | { X86::MOVAPSrm, X86::MOVAPDrm, X86::MOVDQArm }, | |||
5975 | { X86::MOVAPSrr, X86::MOVAPDrr, X86::MOVDQArr }, | |||
5976 | { X86::MOVUPSmr, X86::MOVUPDmr, X86::MOVDQUmr }, | |||
5977 | { X86::MOVUPSrm, X86::MOVUPDrm, X86::MOVDQUrm }, | |||
5978 | { X86::MOVLPSmr, X86::MOVLPDmr, X86::MOVPQI2QImr }, | |||
5979 | { X86::MOVSDmr, X86::MOVSDmr, X86::MOVPQI2QImr }, | |||
5980 | { X86::MOVSSmr, X86::MOVSSmr, X86::MOVPDI2DImr }, | |||
5981 | { X86::MOVSDrm, X86::MOVSDrm, X86::MOVQI2PQIrm }, | |||
5982 | { X86::MOVSSrm, X86::MOVSSrm, X86::MOVDI2PDIrm }, | |||
5983 | { X86::MOVNTPSmr, X86::MOVNTPDmr, X86::MOVNTDQmr }, | |||
5984 | { X86::ANDNPSrm, X86::ANDNPDrm, X86::PANDNrm }, | |||
5985 | { X86::ANDNPSrr, X86::ANDNPDrr, X86::PANDNrr }, | |||
5986 | { X86::ANDPSrm, X86::ANDPDrm, X86::PANDrm }, | |||
5987 | { X86::ANDPSrr, X86::ANDPDrr, X86::PANDrr }, | |||
5988 | { X86::ORPSrm, X86::ORPDrm, X86::PORrm }, | |||
5989 | { X86::ORPSrr, X86::ORPDrr, X86::PORrr }, | |||
5990 | { X86::XORPSrm, X86::XORPDrm, X86::PXORrm }, | |||
5991 | { X86::XORPSrr, X86::XORPDrr, X86::PXORrr }, | |||
5992 | { X86::UNPCKLPDrm, X86::UNPCKLPDrm, X86::PUNPCKLQDQrm }, | |||
5993 | { X86::MOVLHPSrr, X86::UNPCKLPDrr, X86::PUNPCKLQDQrr }, | |||
5994 | { X86::UNPCKHPDrm, X86::UNPCKHPDrm, X86::PUNPCKHQDQrm }, | |||
5995 | { X86::UNPCKHPDrr, X86::UNPCKHPDrr, X86::PUNPCKHQDQrr }, | |||
5996 | { X86::UNPCKLPSrm, X86::UNPCKLPSrm, X86::PUNPCKLDQrm }, | |||
5997 | { X86::UNPCKLPSrr, X86::UNPCKLPSrr, X86::PUNPCKLDQrr }, | |||
5998 | { X86::UNPCKHPSrm, X86::UNPCKHPSrm, X86::PUNPCKHDQrm }, | |||
5999 | { X86::UNPCKHPSrr, X86::UNPCKHPSrr, X86::PUNPCKHDQrr }, | |||
6000 | { X86::EXTRACTPSmr, X86::EXTRACTPSmr, X86::PEXTRDmr }, | |||
6001 | { X86::EXTRACTPSrr, X86::EXTRACTPSrr, X86::PEXTRDrr }, | |||
6002 | // AVX 128-bit support | |||
6003 | { X86::VMOVAPSmr, X86::VMOVAPDmr, X86::VMOVDQAmr }, | |||
6004 | { X86::VMOVAPSrm, X86::VMOVAPDrm, X86::VMOVDQArm }, | |||
6005 | { X86::VMOVAPSrr, X86::VMOVAPDrr, X86::VMOVDQArr }, | |||
6006 | { X86::VMOVUPSmr, X86::VMOVUPDmr, X86::VMOVDQUmr }, | |||
6007 | { X86::VMOVUPSrm, X86::VMOVUPDrm, X86::VMOVDQUrm }, | |||
6008 | { X86::VMOVLPSmr, X86::VMOVLPDmr, X86::VMOVPQI2QImr }, | |||
6009 | { X86::VMOVSDmr, X86::VMOVSDmr, X86::VMOVPQI2QImr }, | |||
6010 | { X86::VMOVSSmr, X86::VMOVSSmr, X86::VMOVPDI2DImr }, | |||
6011 | { X86::VMOVSDrm, X86::VMOVSDrm, X86::VMOVQI2PQIrm }, | |||
6012 | { X86::VMOVSSrm, X86::VMOVSSrm, X86::VMOVDI2PDIrm }, | |||
6013 | { X86::VMOVNTPSmr, X86::VMOVNTPDmr, X86::VMOVNTDQmr }, | |||
6014 | { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNrm }, | |||
6015 | { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNrr }, | |||
6016 | { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDrm }, | |||
6017 | { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDrr }, | |||
6018 | { X86::VORPSrm, X86::VORPDrm, X86::VPORrm }, | |||
6019 | { X86::VORPSrr, X86::VORPDrr, X86::VPORrr }, | |||
6020 | { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORrm }, | |||
6021 | { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORrr }, | |||
6022 | { X86::VUNPCKLPDrm, X86::VUNPCKLPDrm, X86::VPUNPCKLQDQrm }, | |||
6023 | { X86::VMOVLHPSrr, X86::VUNPCKLPDrr, X86::VPUNPCKLQDQrr }, | |||
6024 | { X86::VUNPCKHPDrm, X86::VUNPCKHPDrm, X86::VPUNPCKHQDQrm }, | |||
6025 | { X86::VUNPCKHPDrr, X86::VUNPCKHPDrr, X86::VPUNPCKHQDQrr }, | |||
6026 | { X86::VUNPCKLPSrm, X86::VUNPCKLPSrm, X86::VPUNPCKLDQrm }, | |||
6027 | { X86::VUNPCKLPSrr, X86::VUNPCKLPSrr, X86::VPUNPCKLDQrr }, | |||
6028 | { X86::VUNPCKHPSrm, X86::VUNPCKHPSrm, X86::VPUNPCKHDQrm }, | |||
6029 | { X86::VUNPCKHPSrr, X86::VUNPCKHPSrr, X86::VPUNPCKHDQrr }, | |||
6030 | { X86::VEXTRACTPSmr, X86::VEXTRACTPSmr, X86::VPEXTRDmr }, | |||
6031 | { X86::VEXTRACTPSrr, X86::VEXTRACTPSrr, X86::VPEXTRDrr }, | |||
6032 | // AVX 256-bit support | |||
6033 | { X86::VMOVAPSYmr, X86::VMOVAPDYmr, X86::VMOVDQAYmr }, | |||
6034 | { X86::VMOVAPSYrm, X86::VMOVAPDYrm, X86::VMOVDQAYrm }, | |||
6035 | { X86::VMOVAPSYrr, X86::VMOVAPDYrr, X86::VMOVDQAYrr }, | |||
6036 | { X86::VMOVUPSYmr, X86::VMOVUPDYmr, X86::VMOVDQUYmr }, | |||
6037 | { X86::VMOVUPSYrm, X86::VMOVUPDYrm, X86::VMOVDQUYrm }, | |||
6038 | { X86::VMOVNTPSYmr, X86::VMOVNTPDYmr, X86::VMOVNTDQYmr }, | |||
6039 | { X86::VPERMPSYrm, X86::VPERMPSYrm, X86::VPERMDYrm }, | |||
6040 | { X86::VPERMPSYrr, X86::VPERMPSYrr, X86::VPERMDYrr }, | |||
6041 | { X86::VPERMPDYmi, X86::VPERMPDYmi, X86::VPERMQYmi }, | |||
6042 | { X86::VPERMPDYri, X86::VPERMPDYri, X86::VPERMQYri }, | |||
6043 | // AVX512 support | |||
6044 | { X86::VMOVLPSZ128mr, X86::VMOVLPDZ128mr, X86::VMOVPQI2QIZmr }, | |||
6045 | { X86::VMOVNTPSZ128mr, X86::VMOVNTPDZ128mr, X86::VMOVNTDQZ128mr }, | |||
6046 | { X86::VMOVNTPSZ256mr, X86::VMOVNTPDZ256mr, X86::VMOVNTDQZ256mr }, | |||
6047 | { X86::VMOVNTPSZmr, X86::VMOVNTPDZmr, X86::VMOVNTDQZmr }, | |||
6048 | { X86::VMOVSDZmr, X86::VMOVSDZmr, X86::VMOVPQI2QIZmr }, | |||
6049 | { X86::VMOVSSZmr, X86::VMOVSSZmr, X86::VMOVPDI2DIZmr }, | |||
6050 | { X86::VMOVSDZrm, X86::VMOVSDZrm, X86::VMOVQI2PQIZrm }, | |||
6051 | { X86::VMOVSSZrm, X86::VMOVSSZrm, X86::VMOVDI2PDIZrm }, | |||
6052 | { X86::VBROADCASTSSZ128r, X86::VBROADCASTSSZ128r, X86::VPBROADCASTDZ128r }, | |||
6053 | { X86::VBROADCASTSSZ128m, X86::VBROADCASTSSZ128m, X86::VPBROADCASTDZ128m }, | |||
6054 | { X86::VBROADCASTSSZ256r, X86::VBROADCASTSSZ256r, X86::VPBROADCASTDZ256r }, | |||
6055 | { X86::VBROADCASTSSZ256m, X86::VBROADCASTSSZ256m, X86::VPBROADCASTDZ256m }, | |||
6056 | { X86::VBROADCASTSSZr, X86::VBROADCASTSSZr, X86::VPBROADCASTDZr }, | |||
6057 | { X86::VBROADCASTSSZm, X86::VBROADCASTSSZm, X86::VPBROADCASTDZm }, | |||
6058 | { X86::VBROADCASTSDZ256r, X86::VBROADCASTSDZ256r, X86::VPBROADCASTQZ256r }, | |||
6059 | { X86::VBROADCASTSDZ256m, X86::VBROADCASTSDZ256m, X86::VPBROADCASTQZ256m }, | |||
6060 | { X86::VBROADCASTSDZr, X86::VBROADCASTSDZr, X86::VPBROADCASTQZr }, | |||
6061 | { X86::VBROADCASTSDZm, X86::VBROADCASTSDZm, X86::VPBROADCASTQZm }, | |||
6062 | { X86::VINSERTF32x4Zrr, X86::VINSERTF32x4Zrr, X86::VINSERTI32x4Zrr }, | |||
6063 | { X86::VINSERTF32x4Zrm, X86::VINSERTF32x4Zrm, X86::VINSERTI32x4Zrm }, | |||
6064 | { X86::VINSERTF32x8Zrr, X86::VINSERTF32x8Zrr, X86::VINSERTI32x8Zrr }, | |||
6065 | { X86::VINSERTF32x8Zrm, X86::VINSERTF32x8Zrm, X86::VINSERTI32x8Zrm }, | |||
6066 | { X86::VINSERTF64x2Zrr, X86::VINSERTF64x2Zrr, X86::VINSERTI64x2Zrr }, | |||
6067 | { X86::VINSERTF64x2Zrm, X86::VINSERTF64x2Zrm, X86::VINSERTI64x2Zrm }, | |||
6068 | { X86::VINSERTF64x4Zrr, X86::VINSERTF64x4Zrr, X86::VINSERTI64x4Zrr }, | |||
6069 | { X86::VINSERTF64x4Zrm, X86::VINSERTF64x4Zrm, X86::VINSERTI64x4Zrm }, | |||
6070 | { X86::VINSERTF32x4Z256rr,X86::VINSERTF32x4Z256rr,X86::VINSERTI32x4Z256rr }, | |||
6071 | { X86::VINSERTF32x4Z256rm,X86::VINSERTF32x4Z256rm,X86::VINSERTI32x4Z256rm }, | |||
6072 | { X86::VINSERTF64x2Z256rr,X86::VINSERTF64x2Z256rr,X86::VINSERTI64x2Z256rr }, | |||
6073 | { X86::VINSERTF64x2Z256rm,X86::VINSERTF64x2Z256rm,X86::VINSERTI64x2Z256rm }, | |||
6074 | { X86::VEXTRACTF32x4Zrr, X86::VEXTRACTF32x4Zrr, X86::VEXTRACTI32x4Zrr }, | |||
6075 | { X86::VEXTRACTF32x4Zmr, X86::VEXTRACTF32x4Zmr, X86::VEXTRACTI32x4Zmr }, | |||
6076 | { X86::VEXTRACTF32x8Zrr, X86::VEXTRACTF32x8Zrr, X86::VEXTRACTI32x8Zrr }, | |||
6077 | { X86::VEXTRACTF32x8Zmr, X86::VEXTRACTF32x8Zmr, X86::VEXTRACTI32x8Zmr }, | |||
6078 | { X86::VEXTRACTF64x2Zrr, X86::VEXTRACTF64x2Zrr, X86::VEXTRACTI64x2Zrr }, | |||
6079 | { X86::VEXTRACTF64x2Zmr, X86::VEXTRACTF64x2Zmr, X86::VEXTRACTI64x2Zmr }, | |||
6080 | { X86::VEXTRACTF64x4Zrr, X86::VEXTRACTF64x4Zrr, X86::VEXTRACTI64x4Zrr }, | |||
6081 | { X86::VEXTRACTF64x4Zmr, X86::VEXTRACTF64x4Zmr, X86::VEXTRACTI64x4Zmr }, | |||
6082 | { X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTI32x4Z256rr }, | |||
6083 | { X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTI32x4Z256mr }, | |||
6084 | { X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTI64x2Z256rr }, | |||
6085 | { X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTI64x2Z256mr }, | |||
6086 | { X86::VPERMILPSmi, X86::VPERMILPSmi, X86::VPSHUFDmi }, | |||
6087 | { X86::VPERMILPSri, X86::VPERMILPSri, X86::VPSHUFDri }, | |||
6088 | { X86::VPERMILPSZ128mi, X86::VPERMILPSZ128mi, X86::VPSHUFDZ128mi }, | |||
6089 | { X86::VPERMILPSZ128ri, X86::VPERMILPSZ128ri, X86::VPSHUFDZ128ri }, | |||
6090 | { X86::VPERMILPSZ256mi, X86::VPERMILPSZ256mi, X86::VPSHUFDZ256mi }, | |||
6091 | { X86::VPERMILPSZ256ri, X86::VPERMILPSZ256ri, X86::VPSHUFDZ256ri }, | |||
6092 | { X86::VPERMILPSZmi, X86::VPERMILPSZmi, X86::VPSHUFDZmi }, | |||
6093 | { X86::VPERMILPSZri, X86::VPERMILPSZri, X86::VPSHUFDZri }, | |||
6094 | { X86::VPERMPSZ256rm, X86::VPERMPSZ256rm, X86::VPERMDZ256rm }, | |||
6095 | { X86::VPERMPSZ256rr, X86::VPERMPSZ256rr, X86::VPERMDZ256rr }, | |||
6096 | { X86::VPERMPDZ256mi, X86::VPERMPDZ256mi, X86::VPERMQZ256mi }, | |||
6097 | { X86::VPERMPDZ256ri, X86::VPERMPDZ256ri, X86::VPERMQZ256ri }, | |||
6098 | { X86::VPERMPDZ256rm, X86::VPERMPDZ256rm, X86::VPERMQZ256rm }, | |||
6099 | { X86::VPERMPDZ256rr, X86::VPERMPDZ256rr, X86::VPERMQZ256rr }, | |||
6100 | { X86::VPERMPSZrm, X86::VPERMPSZrm, X86::VPERMDZrm }, | |||
6101 | { X86::VPERMPSZrr, X86::VPERMPSZrr, X86::VPERMDZrr }, | |||
6102 | { X86::VPERMPDZmi, X86::VPERMPDZmi, X86::VPERMQZmi }, | |||
6103 | { X86::VPERMPDZri, X86::VPERMPDZri, X86::VPERMQZri }, | |||
6104 | { X86::VPERMPDZrm, X86::VPERMPDZrm, X86::VPERMQZrm }, | |||
6105 | { X86::VPERMPDZrr, X86::VPERMPDZrr, X86::VPERMQZrr }, | |||
6106 | { X86::VUNPCKLPDZ256rm, X86::VUNPCKLPDZ256rm, X86::VPUNPCKLQDQZ256rm }, | |||
6107 | { X86::VUNPCKLPDZ256rr, X86::VUNPCKLPDZ256rr, X86::VPUNPCKLQDQZ256rr }, | |||
6108 | { X86::VUNPCKHPDZ256rm, X86::VUNPCKHPDZ256rm, X86::VPUNPCKHQDQZ256rm }, | |||
6109 | { X86::VUNPCKHPDZ256rr, X86::VUNPCKHPDZ256rr, X86::VPUNPCKHQDQZ256rr }, | |||
6110 | { X86::VUNPCKLPSZ256rm, X86::VUNPCKLPSZ256rm, X86::VPUNPCKLDQZ256rm }, | |||
6111 | { X86::VUNPCKLPSZ256rr, X86::VUNPCKLPSZ256rr, X86::VPUNPCKLDQZ256rr }, | |||
6112 | { X86::VUNPCKHPSZ256rm, X86::VUNPCKHPSZ256rm, X86::VPUNPCKHDQZ256rm }, | |||
6113 | { X86::VUNPCKHPSZ256rr, X86::VUNPCKHPSZ256rr, X86::VPUNPCKHDQZ256rr }, | |||
6114 | { X86::VUNPCKLPDZ128rm, X86::VUNPCKLPDZ128rm, X86::VPUNPCKLQDQZ128rm }, | |||
6115 | { X86::VMOVLHPSZrr, X86::VUNPCKLPDZ128rr, X86::VPUNPCKLQDQZ128rr }, | |||
6116 | { X86::VUNPCKHPDZ128rm, X86::VUNPCKHPDZ128rm, X86::VPUNPCKHQDQZ128rm }, | |||
6117 | { X86::VUNPCKHPDZ128rr, X86::VUNPCKHPDZ128rr, X86::VPUNPCKHQDQZ128rr }, | |||
6118 | { X86::VUNPCKLPSZ128rm, X86::VUNPCKLPSZ128rm, X86::VPUNPCKLDQZ128rm }, | |||
6119 | { X86::VUNPCKLPSZ128rr, X86::VUNPCKLPSZ128rr, X86::VPUNPCKLDQZ128rr }, | |||
6120 | { X86::VUNPCKHPSZ128rm, X86::VUNPCKHPSZ128rm, X86::VPUNPCKHDQZ128rm }, | |||
6121 | { X86::VUNPCKHPSZ128rr, X86::VUNPCKHPSZ128rr, X86::VPUNPCKHDQZ128rr }, | |||
6122 | { X86::VUNPCKLPDZrm, X86::VUNPCKLPDZrm, X86::VPUNPCKLQDQZrm }, | |||
6123 | { X86::VUNPCKLPDZrr, X86::VUNPCKLPDZrr, X86::VPUNPCKLQDQZrr }, | |||
6124 | { X86::VUNPCKHPDZrm, X86::VUNPCKHPDZrm, X86::VPUNPCKHQDQZrm }, | |||
6125 | { X86::VUNPCKHPDZrr, X86::VUNPCKHPDZrr, X86::VPUNPCKHQDQZrr }, | |||
6126 | { X86::VUNPCKLPSZrm, X86::VUNPCKLPSZrm, X86::VPUNPCKLDQZrm }, | |||
6127 | { X86::VUNPCKLPSZrr, X86::VUNPCKLPSZrr, X86::VPUNPCKLDQZrr }, | |||
6128 | { X86::VUNPCKHPSZrm, X86::VUNPCKHPSZrm, X86::VPUNPCKHDQZrm }, | |||
6129 | { X86::VUNPCKHPSZrr, X86::VUNPCKHPSZrr, X86::VPUNPCKHDQZrr }, | |||
6130 | { X86::VEXTRACTPSZmr, X86::VEXTRACTPSZmr, X86::VPEXTRDZmr }, | |||
6131 | { X86::VEXTRACTPSZrr, X86::VEXTRACTPSZrr, X86::VPEXTRDZrr }, | |||
6132 | }; | |||
6133 | ||||
6134 | static const uint16_t ReplaceableInstrsAVX2[][3] = { | |||
6135 | //PackedSingle PackedDouble PackedInt | |||
6136 | { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNYrm }, | |||
6137 | { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNYrr }, | |||
6138 | { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDYrm }, | |||
6139 | { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDYrr }, | |||
6140 | { X86::VORPSYrm, X86::VORPDYrm, X86::VPORYrm }, | |||
6141 | { X86::VORPSYrr, X86::VORPDYrr, X86::VPORYrr }, | |||
6142 | { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORYrm }, | |||
6143 | { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORYrr }, | |||
6144 | { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, | |||
6145 | { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, | |||
6146 | { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, | |||
6147 | { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, | |||
6148 | { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, | |||
6149 | { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, | |||
6150 | { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, | |||
6151 | { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm}, | |||
6152 | { X86::VBROADCASTF128, X86::VBROADCASTF128, X86::VBROADCASTI128 }, | |||
6153 | { X86::VBLENDPSYrri, X86::VBLENDPSYrri, X86::VPBLENDDYrri }, | |||
6154 | { X86::VBLENDPSYrmi, X86::VBLENDPSYrmi, X86::VPBLENDDYrmi }, | |||
6155 | { X86::VPERMILPSYmi, X86::VPERMILPSYmi, X86::VPSHUFDYmi }, | |||
6156 | { X86::VPERMILPSYri, X86::VPERMILPSYri, X86::VPSHUFDYri }, | |||
6157 | { X86::VUNPCKLPDYrm, X86::VUNPCKLPDYrm, X86::VPUNPCKLQDQYrm }, | |||
6158 | { X86::VUNPCKLPDYrr, X86::VUNPCKLPDYrr, X86::VPUNPCKLQDQYrr }, | |||
6159 | { X86::VUNPCKHPDYrm, X86::VUNPCKHPDYrm, X86::VPUNPCKHQDQYrm }, | |||
6160 | { X86::VUNPCKHPDYrr, X86::VUNPCKHPDYrr, X86::VPUNPCKHQDQYrr }, | |||
6161 | { X86::VUNPCKLPSYrm, X86::VUNPCKLPSYrm, X86::VPUNPCKLDQYrm }, | |||
6162 | { X86::VUNPCKLPSYrr, X86::VUNPCKLPSYrr, X86::VPUNPCKLDQYrr }, | |||
6163 | { X86::VUNPCKHPSYrm, X86::VUNPCKHPSYrm, X86::VPUNPCKHDQYrm }, | |||
6164 | { X86::VUNPCKHPSYrr, X86::VUNPCKHPSYrr, X86::VPUNPCKHDQYrr }, | |||
6165 | }; | |||
6166 | ||||
6167 | static const uint16_t ReplaceableInstrsAVX2InsertExtract[][3] = { | |||
6168 | //PackedSingle PackedDouble PackedInt | |||
6169 | { X86::VEXTRACTF128mr, X86::VEXTRACTF128mr, X86::VEXTRACTI128mr }, | |||
6170 | { X86::VEXTRACTF128rr, X86::VEXTRACTF128rr, X86::VEXTRACTI128rr }, | |||
6171 | { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, | |||
6172 | { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, | |||
6173 | }; | |||
6174 | ||||
6175 | static const uint16_t ReplaceableInstrsAVX512[][4] = { | |||
6176 | // Two integer columns for 64-bit and 32-bit elements. | |||
6177 | //PackedSingle PackedDouble PackedInt PackedInt | |||
6178 | { X86::VMOVAPSZ128mr, X86::VMOVAPDZ128mr, X86::VMOVDQA64Z128mr, X86::VMOVDQA32Z128mr }, | |||
6179 | { X86::VMOVAPSZ128rm, X86::VMOVAPDZ128rm, X86::VMOVDQA64Z128rm, X86::VMOVDQA32Z128rm }, | |||
6180 | { X86::VMOVAPSZ128rr, X86::VMOVAPDZ128rr, X86::VMOVDQA64Z128rr, X86::VMOVDQA32Z128rr }, | |||
6181 | { X86::VMOVUPSZ128mr, X86::VMOVUPDZ128mr, X86::VMOVDQU64Z128mr, X86::VMOVDQU32Z128mr }, | |||
6182 | { X86::VMOVUPSZ128rm, X86::VMOVUPDZ128rm, X86::VMOVDQU64Z128rm, X86::VMOVDQU32Z128rm }, | |||
6183 | { X86::VMOVAPSZ256mr, X86::VMOVAPDZ256mr, X86::VMOVDQA64Z256mr, X86::VMOVDQA32Z256mr }, | |||
6184 | { X86::VMOVAPSZ256rm, X86::VMOVAPDZ256rm, X86::VMOVDQA64Z256rm, X86::VMOVDQA32Z256rm }, | |||
6185 | { X86::VMOVAPSZ256rr, X86::VMOVAPDZ256rr, X86::VMOVDQA64Z256rr, X86::VMOVDQA32Z256rr }, | |||
6186 | { X86::VMOVUPSZ256mr, X86::VMOVUPDZ256mr, X86::VMOVDQU64Z256mr, X86::VMOVDQU32Z256mr }, | |||
6187 | { X86::VMOVUPSZ256rm, X86::VMOVUPDZ256rm, X86::VMOVDQU64Z256rm, X86::VMOVDQU32Z256rm }, | |||
6188 | { X86::VMOVAPSZmr, X86::VMOVAPDZmr, X86::VMOVDQA64Zmr, X86::VMOVDQA32Zmr }, | |||
6189 | { X86::VMOVAPSZrm, X86::VMOVAPDZrm, X86::VMOVDQA64Zrm, X86::VMOVDQA32Zrm }, | |||
6190 | { X86::VMOVAPSZrr, X86::VMOVAPDZrr, X86::VMOVDQA64Zrr, X86::VMOVDQA32Zrr }, | |||
6191 | { X86::VMOVUPSZmr, X86::VMOVUPDZmr, X86::VMOVDQU64Zmr, X86::VMOVDQU32Zmr }, | |||
6192 | { X86::VMOVUPSZrm, X86::VMOVUPDZrm, X86::VMOVDQU64Zrm, X86::VMOVDQU32Zrm }, | |||
6193 | }; | |||
6194 | ||||
6195 | static const uint16_t ReplaceableInstrsAVX512DQ[][4] = { | |||
6196 | // Two integer columns for 64-bit and 32-bit elements. | |||
6197 | //PackedSingle PackedDouble PackedInt PackedInt | |||
6198 | { X86::VANDNPSZ128rm, X86::VANDNPDZ128rm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, | |||
6199 | { X86::VANDNPSZ128rr, X86::VANDNPDZ128rr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, | |||
6200 | { X86::VANDPSZ128rm, X86::VANDPDZ128rm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, | |||
6201 | { X86::VANDPSZ128rr, X86::VANDPDZ128rr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, | |||
6202 | { X86::VORPSZ128rm, X86::VORPDZ128rm, X86::VPORQZ128rm, X86::VPORDZ128rm }, | |||
6203 | { X86::VORPSZ128rr, X86::VORPDZ128rr, X86::VPORQZ128rr, X86::VPORDZ128rr }, | |||
6204 | { X86::VXORPSZ128rm, X86::VXORPDZ128rm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, | |||
6205 | { X86::VXORPSZ128rr, X86::VXORPDZ128rr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, | |||
6206 | { X86::VANDNPSZ256rm, X86::VANDNPDZ256rm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, | |||
6207 | { X86::VANDNPSZ256rr, X86::VANDNPDZ256rr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, | |||
6208 | { X86::VANDPSZ256rm, X86::VANDPDZ256rm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, | |||
6209 | { X86::VANDPSZ256rr, X86::VANDPDZ256rr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, | |||
6210 | { X86::VORPSZ256rm, X86::VORPDZ256rm, X86::VPORQZ256rm, X86::VPORDZ256rm }, | |||
6211 | { X86::VORPSZ256rr, X86::VORPDZ256rr, X86::VPORQZ256rr, X86::VPORDZ256rr }, | |||
6212 | { X86::VXORPSZ256rm, X86::VXORPDZ256rm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, | |||
6213 | { X86::VXORPSZ256rr, X86::VXORPDZ256rr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, | |||
6214 | { X86::VANDNPSZrm, X86::VANDNPDZrm, X86::VPANDNQZrm, X86::VPANDNDZrm }, | |||
6215 | { X86::VANDNPSZrr, X86::VANDNPDZrr, X86::VPANDNQZrr, X86::VPANDNDZrr }, | |||
6216 | { X86::VANDPSZrm, X86::VANDPDZrm, X86::VPANDQZrm, X86::VPANDDZrm }, | |||
6217 | { X86::VANDPSZrr, X86::VANDPDZrr, X86::VPANDQZrr, X86::VPANDDZrr }, | |||
6218 | { X86::VORPSZrm, X86::VORPDZrm, X86::VPORQZrm, X86::VPORDZrm }, | |||
6219 | { X86::VORPSZrr, X86::VORPDZrr, X86::VPORQZrr, X86::VPORDZrr }, | |||
6220 | { X86::VXORPSZrm, X86::VXORPDZrm, X86::VPXORQZrm, X86::VPXORDZrm }, | |||
6221 | { X86::VXORPSZrr, X86::VXORPDZrr, X86::VPXORQZrr, X86::VPXORDZrr }, | |||
6222 | }; | |||
6223 | ||||
6224 | static const uint16_t ReplaceableInstrsAVX512DQMasked[][4] = { | |||
6225 | // Two integer columns for 64-bit and 32-bit elements. | |||
6226 | //PackedSingle PackedDouble | |||
6227 | //PackedInt PackedInt | |||
6228 | { X86::VANDNPSZ128rmk, X86::VANDNPDZ128rmk, | |||
6229 | X86::VPANDNQZ128rmk, X86::VPANDNDZ128rmk }, | |||
6230 | { X86::VANDNPSZ128rmkz, X86::VANDNPDZ128rmkz, | |||
6231 | X86::VPANDNQZ128rmkz, X86::VPANDNDZ128rmkz }, | |||
6232 | { X86::VANDNPSZ128rrk, X86::VANDNPDZ128rrk, | |||
6233 | X86::VPANDNQZ128rrk, X86::VPANDNDZ128rrk }, | |||
6234 | { X86::VANDNPSZ128rrkz, X86::VANDNPDZ128rrkz, | |||
6235 | X86::VPANDNQZ128rrkz, X86::VPANDNDZ128rrkz }, | |||
6236 | { X86::VANDPSZ128rmk, X86::VANDPDZ128rmk, | |||
6237 | X86::VPANDQZ128rmk, X86::VPANDDZ128rmk }, | |||
6238 | { X86::VANDPSZ128rmkz, X86::VANDPDZ128rmkz, | |||
6239 | X86::VPANDQZ128rmkz, X86::VPANDDZ128rmkz }, | |||
6240 | { X86::VANDPSZ128rrk, X86::VANDPDZ128rrk, | |||
6241 | X86::VPANDQZ128rrk, X86::VPANDDZ128rrk }, | |||
6242 | { X86::VANDPSZ128rrkz, X86::VANDPDZ128rrkz, | |||
6243 | X86::VPANDQZ128rrkz, X86::VPANDDZ128rrkz }, | |||
6244 | { X86::VORPSZ128rmk, X86::VORPDZ128rmk, | |||
6245 | X86::VPORQZ128rmk, X86::VPORDZ128rmk }, | |||
6246 | { X86::VORPSZ128rmkz, X86::VORPDZ128rmkz, | |||
6247 | X86::VPORQZ128rmkz, X86::VPORDZ128rmkz }, | |||
6248 | { X86::VORPSZ128rrk, X86::VORPDZ128rrk, | |||
6249 | X86::VPORQZ128rrk, X86::VPORDZ128rrk }, | |||
6250 | { X86::VORPSZ128rrkz, X86::VORPDZ128rrkz, | |||
6251 | X86::VPORQZ128rrkz, X86::VPORDZ128rrkz }, | |||
6252 | { X86::VXORPSZ128rmk, X86::VXORPDZ128rmk, | |||
6253 | X86::VPXORQZ128rmk, X86::VPXORDZ128rmk }, | |||
6254 | { X86::VXORPSZ128rmkz, X86::VXORPDZ128rmkz, | |||
6255 | X86::VPXORQZ128rmkz, X86::VPXORDZ128rmkz }, | |||
6256 | { X86::VXORPSZ128rrk, X86::VXORPDZ128rrk, | |||
6257 | X86::VPXORQZ128rrk, X86::VPXORDZ128rrk }, | |||
6258 | { X86::VXORPSZ128rrkz, X86::VXORPDZ128rrkz, | |||
6259 | X86::VPXORQZ128rrkz, X86::VPXORDZ128rrkz }, | |||
6260 | { X86::VANDNPSZ256rmk, X86::VANDNPDZ256rmk, | |||
6261 | X86::VPANDNQZ256rmk, X86::VPANDNDZ256rmk }, | |||
6262 | { X86::VANDNPSZ256rmkz, X86::VANDNPDZ256rmkz, | |||
6263 | X86::VPANDNQZ256rmkz, X86::VPANDNDZ256rmkz }, | |||
6264 | { X86::VANDNPSZ256rrk, X86::VANDNPDZ256rrk, | |||
6265 | X86::VPANDNQZ256rrk, X86::VPANDNDZ256rrk }, | |||
6266 | { X86::VANDNPSZ256rrkz, X86::VANDNPDZ256rrkz, | |||
6267 | X86::VPANDNQZ256rrkz, X86::VPANDNDZ256rrkz }, | |||
6268 | { X86::VANDPSZ256rmk, X86::VANDPDZ256rmk, | |||
6269 | X86::VPANDQZ256rmk, X86::VPANDDZ256rmk }, | |||
6270 | { X86::VANDPSZ256rmkz, X86::VANDPDZ256rmkz, | |||
6271 | X86::VPANDQZ256rmkz, X86::VPANDDZ256rmkz }, | |||
6272 | { X86::VANDPSZ256rrk, X86::VANDPDZ256rrk, | |||
6273 | X86::VPANDQZ256rrk, X86::VPANDDZ256rrk }, | |||
6274 | { X86::VANDPSZ256rrkz, X86::VANDPDZ256rrkz, | |||
6275 | X86::VPANDQZ256rrkz, X86::VPANDDZ256rrkz }, | |||
6276 | { X86::VORPSZ256rmk, X86::VORPDZ256rmk, | |||
6277 | X86::VPORQZ256rmk, X86::VPORDZ256rmk }, | |||
6278 | { X86::VORPSZ256rmkz, X86::VORPDZ256rmkz, | |||
6279 | X86::VPORQZ256rmkz, X86::VPORDZ256rmkz }, | |||
6280 | { X86::VORPSZ256rrk, X86::VORPDZ256rrk, | |||
6281 | X86::VPORQZ256rrk, X86::VPORDZ256rrk }, | |||
6282 | { X86::VORPSZ256rrkz, X86::VORPDZ256rrkz, | |||
6283 | X86::VPORQZ256rrkz, X86::VPORDZ256rrkz }, | |||
6284 | { X86::VXORPSZ256rmk, X86::VXORPDZ256rmk, | |||
6285 | X86::VPXORQZ256rmk, X86::VPXORDZ256rmk }, | |||
6286 | { X86::VXORPSZ256rmkz, X86::VXORPDZ256rmkz, | |||
6287 | X86::VPXORQZ256rmkz, X86::VPXORDZ256rmkz }, | |||
6288 | { X86::VXORPSZ256rrk, X86::VXORPDZ256rrk, | |||
6289 | X86::VPXORQZ256rrk, X86::VPXORDZ256rrk }, | |||
6290 | { X86::VXORPSZ256rrkz, X86::VXORPDZ256rrkz, | |||
6291 | X86::VPXORQZ256rrkz, X86::VPXORDZ256rrkz }, | |||
6292 | { X86::VANDNPSZrmk, X86::VANDNPDZrmk, | |||
6293 | X86::VPANDNQZrmk, X86::VPANDNDZrmk }, | |||
6294 | { X86::VANDNPSZrmkz, X86::VANDNPDZrmkz, | |||
6295 | X86::VPANDNQZrmkz, X86::VPANDNDZrmkz }, | |||
6296 | { X86::VANDNPSZrrk, X86::VANDNPDZrrk, | |||
6297 | X86::VPANDNQZrrk, X86::VPANDNDZrrk }, | |||
6298 | { X86::VANDNPSZrrkz, X86::VANDNPDZrrkz, | |||
6299 | X86::VPANDNQZrrkz, X86::VPANDNDZrrkz }, | |||
6300 | { X86::VANDPSZrmk, X86::VANDPDZrmk, | |||
6301 | X86::VPANDQZrmk, X86::VPANDDZrmk }, | |||
6302 | { X86::VANDPSZrmkz, X86::VANDPDZrmkz, | |||
6303 | X86::VPANDQZrmkz, X86::VPANDDZrmkz }, | |||
6304 | { X86::VANDPSZrrk, X86::VANDPDZrrk, | |||
6305 | X86::VPANDQZrrk, X86::VPANDDZrrk }, | |||
6306 | { X86::VANDPSZrrkz, X86::VANDPDZrrkz, | |||
6307 | X86::VPANDQZrrkz, X86::VPANDDZrrkz }, | |||
6308 | { X86::VORPSZrmk, X86::VORPDZrmk, | |||
6309 | X86::VPORQZrmk, X86::VPORDZrmk }, | |||
6310 | { X86::VORPSZrmkz, X86::VORPDZrmkz, | |||
6311 | X86::VPORQZrmkz, X86::VPORDZrmkz }, | |||
6312 | { X86::VORPSZrrk, X86::VORPDZrrk, | |||
6313 | X86::VPORQZrrk, X86::VPORDZrrk }, | |||
6314 | { X86::VORPSZrrkz, X86::VORPDZrrkz, | |||
6315 | X86::VPORQZrrkz, X86::VPORDZrrkz }, | |||
6316 | { X86::VXORPSZrmk, X86::VXORPDZrmk, | |||
6317 | X86::VPXORQZrmk, X86::VPXORDZrmk }, | |||
6318 | { X86::VXORPSZrmkz, X86::VXORPDZrmkz, | |||
6319 | X86::VPXORQZrmkz, X86::VPXORDZrmkz }, | |||
6320 | { X86::VXORPSZrrk, X86::VXORPDZrrk, | |||
6321 | X86::VPXORQZrrk, X86::VPXORDZrrk }, | |||
6322 | { X86::VXORPSZrrkz, X86::VXORPDZrrkz, | |||
6323 | X86::VPXORQZrrkz, X86::VPXORDZrrkz }, | |||
6324 | // Broadcast loads can be handled the same as masked operations to avoid | |||
6325 | // changing element size. | |||
6326 | { X86::VANDNPSZ128rmb, X86::VANDNPDZ128rmb, | |||
6327 | X86::VPANDNQZ128rmb, X86::VPANDNDZ128rmb }, | |||
6328 | { X86::VANDPSZ128rmb, X86::VANDPDZ128rmb, | |||
6329 | X86::VPANDQZ128rmb, X86::VPANDDZ128rmb }, | |||
6330 | { X86::VORPSZ128rmb, X86::VORPDZ128rmb, | |||
6331 | X86::VPORQZ128rmb, X86::VPORDZ128rmb }, | |||
6332 | { X86::VXORPSZ128rmb, X86::VXORPDZ128rmb, | |||
6333 | X86::VPXORQZ128rmb, X86::VPXORDZ128rmb }, | |||
6334 | { X86::VANDNPSZ256rmb, X86::VANDNPDZ256rmb, | |||
6335 | X86::VPANDNQZ256rmb, X86::VPANDNDZ256rmb }, | |||
6336 | { X86::VANDPSZ256rmb, X86::VANDPDZ256rmb, | |||
6337 | X86::VPANDQZ256rmb, X86::VPANDDZ256rmb }, | |||
6338 | { X86::VORPSZ256rmb, X86::VORPDZ256rmb, | |||
6339 | X86::VPORQZ256rmb, X86::VPORDZ256rmb }, | |||
6340 | { X86::VXORPSZ256rmb, X86::VXORPDZ256rmb, | |||
6341 | X86::VPXORQZ256rmb, X86::VPXORDZ256rmb }, | |||
6342 | { X86::VANDNPSZrmb, X86::VANDNPDZrmb, | |||
6343 | X86::VPANDNQZrmb, X86::VPANDNDZrmb }, | |||
6344 | { X86::VANDPSZrmb, X86::VANDPDZrmb, | |||
6345 | X86::VPANDQZrmb, X86::VPANDDZrmb }, | |||
6346 | { X86::VANDPSZrmb, X86::VANDPDZrmb, | |||
6347 | X86::VPANDQZrmb, X86::VPANDDZrmb }, | |||
6348 | { X86::VORPSZrmb, X86::VORPDZrmb, | |||
6349 | X86::VPORQZrmb, X86::VPORDZrmb }, | |||
6350 | { X86::VXORPSZrmb, X86::VXORPDZrmb, | |||
6351 | X86::VPXORQZrmb, X86::VPXORDZrmb }, | |||
6352 | { X86::VANDNPSZ128rmbk, X86::VANDNPDZ128rmbk, | |||
6353 | X86::VPANDNQZ128rmbk, X86::VPANDNDZ128rmbk }, | |||
6354 | { X86::VANDPSZ128rmbk, X86::VANDPDZ128rmbk, | |||
6355 | X86::VPANDQZ128rmbk, X86::VPANDDZ128rmbk }, | |||
6356 | { X86::VORPSZ128rmbk, X86::VORPDZ128rmbk, | |||
6357 | X86::VPORQZ128rmbk, X86::VPORDZ128rmbk }, | |||
6358 | { X86::VXORPSZ128rmbk, X86::VXORPDZ128rmbk, | |||
6359 | X86::VPXORQZ128rmbk, X86::VPXORDZ128rmbk }, | |||
6360 | { X86::VANDNPSZ256rmbk, X86::VANDNPDZ256rmbk, | |||
6361 | X86::VPANDNQZ256rmbk, X86::VPANDNDZ256rmbk }, | |||
6362 | { X86::VANDPSZ256rmbk, X86::VANDPDZ256rmbk, | |||
6363 | X86::VPANDQZ256rmbk, X86::VPANDDZ256rmbk }, | |||
6364 | { X86::VORPSZ256rmbk, X86::VORPDZ256rmbk, | |||
6365 | X86::VPORQZ256rmbk, X86::VPORDZ256rmbk }, | |||
6366 | { X86::VXORPSZ256rmbk, X86::VXORPDZ256rmbk, | |||
6367 | X86::VPXORQZ256rmbk, X86::VPXORDZ256rmbk }, | |||
6368 | { X86::VANDNPSZrmbk, X86::VANDNPDZrmbk, | |||
6369 | X86::VPANDNQZrmbk, X86::VPANDNDZrmbk }, | |||
6370 | { X86::VANDPSZrmbk, X86::VANDPDZrmbk, | |||
6371 | X86::VPANDQZrmbk, X86::VPANDDZrmbk }, | |||
6372 | { X86::VANDPSZrmbk, X86::VANDPDZrmbk, | |||
6373 | X86::VPANDQZrmbk, X86::VPANDDZrmbk }, | |||
6374 | { X86::VORPSZrmbk, X86::VORPDZrmbk, | |||
6375 | X86::VPORQZrmbk, X86::VPORDZrmbk }, | |||
6376 | { X86::VXORPSZrmbk, X86::VXORPDZrmbk, | |||
6377 | X86::VPXORQZrmbk, X86::VPXORDZrmbk }, | |||
6378 | { X86::VANDNPSZ128rmbkz,X86::VANDNPDZ128rmbkz, | |||
6379 | X86::VPANDNQZ128rmbkz,X86::VPANDNDZ128rmbkz}, | |||
6380 | { X86::VANDPSZ128rmbkz, X86::VANDPDZ128rmbkz, | |||
6381 | X86::VPANDQZ128rmbkz, X86::VPANDDZ128rmbkz }, | |||
6382 | { X86::VORPSZ128rmbkz, X86::VORPDZ128rmbkz, | |||
6383 | X86::VPORQZ128rmbkz, X86::VPORDZ128rmbkz }, | |||
6384 | { X86::VXORPSZ128rmbkz, X86::VXORPDZ128rmbkz, | |||
6385 | X86::VPXORQZ128rmbkz, X86::VPXORDZ128rmbkz }, | |||
6386 | { X86::VANDNPSZ256rmbkz,X86::VANDNPDZ256rmbkz, | |||
6387 | X86::VPANDNQZ256rmbkz,X86::VPANDNDZ256rmbkz}, | |||
6388 | { X86::VANDPSZ256rmbkz, X86::VANDPDZ256rmbkz, | |||
6389 | X86::VPANDQZ256rmbkz, X86::VPANDDZ256rmbkz }, | |||
6390 | { X86::VORPSZ256rmbkz, X86::VORPDZ256rmbkz, | |||
6391 | X86::VPORQZ256rmbkz, X86::VPORDZ256rmbkz }, | |||
6392 | { X86::VXORPSZ256rmbkz, X86::VXORPDZ256rmbkz, | |||
6393 | X86::VPXORQZ256rmbkz, X86::VPXORDZ256rmbkz }, | |||
6394 | { X86::VANDNPSZrmbkz, X86::VANDNPDZrmbkz, | |||
6395 | X86::VPANDNQZrmbkz, X86::VPANDNDZrmbkz }, | |||
6396 | { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, | |||
6397 | X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, | |||
6398 | { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, | |||
6399 | X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, | |||
6400 | { X86::VORPSZrmbkz, X86::VORPDZrmbkz, | |||
6401 | X86::VPORQZrmbkz, X86::VPORDZrmbkz }, | |||
6402 | { X86::VXORPSZrmbkz, X86::VXORPDZrmbkz, | |||
6403 | X86::VPXORQZrmbkz, X86::VPXORDZrmbkz }, | |||
6404 | }; | |||
6405 | ||||
6406 | // NOTE: These should only be used by the custom domain methods. | |||
6407 | static const uint16_t ReplaceableCustomInstrs[][3] = { | |||
6408 | //PackedSingle PackedDouble PackedInt | |||
6409 | { X86::BLENDPSrmi, X86::BLENDPDrmi, X86::PBLENDWrmi }, | |||
6410 | { X86::BLENDPSrri, X86::BLENDPDrri, X86::PBLENDWrri }, | |||
6411 | { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDWrmi }, | |||
6412 | { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDWrri }, | |||
6413 | { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDWYrmi }, | |||
6414 | { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDWYrri }, | |||
6415 | }; | |||
6416 | static const uint16_t ReplaceableCustomAVX2Instrs[][3] = { | |||
6417 | //PackedSingle PackedDouble PackedInt | |||
6418 | { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDDrmi }, | |||
6419 | { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDDrri }, | |||
6420 | { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDDYrmi }, | |||
6421 | { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDDYrri }, | |||
6422 | }; | |||
6423 | ||||
6424 | // Special table for changing EVEX logic instructions to VEX. | |||
6425 | // TODO: Should we run EVEX->VEX earlier? | |||
6426 | static const uint16_t ReplaceableCustomAVX512LogicInstrs[][4] = { | |||
6427 | // Two integer columns for 64-bit and 32-bit elements. | |||
6428 | //PackedSingle PackedDouble PackedInt PackedInt | |||
6429 | { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, | |||
6430 | { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, | |||
6431 | { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, | |||
6432 | { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, | |||
6433 | { X86::VORPSrm, X86::VORPDrm, X86::VPORQZ128rm, X86::VPORDZ128rm }, | |||
6434 | { X86::VORPSrr, X86::VORPDrr, X86::VPORQZ128rr, X86::VPORDZ128rr }, | |||
6435 | { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, | |||
6436 | { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, | |||
6437 | { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, | |||
6438 | { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, | |||
6439 | { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, | |||
6440 | { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, | |||
6441 | { X86::VORPSYrm, X86::VORPDYrm, X86::VPORQZ256rm, X86::VPORDZ256rm }, | |||
6442 | { X86::VORPSYrr, X86::VORPDYrr, X86::VPORQZ256rr, X86::VPORDZ256rr }, | |||
6443 | { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, | |||
6444 | { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, | |||
6445 | }; | |||
6446 | ||||
6447 | // FIXME: Some shuffle and unpack instructions have equivalents in different | |||
6448 | // domains, but they require a bit more work than just switching opcodes. | |||
6449 | ||||
6450 | static const uint16_t *lookup(unsigned opcode, unsigned domain, | |||
6451 | ArrayRef<uint16_t[3]> Table) { | |||
6452 | for (const uint16_t (&Row)[3] : Table) | |||
6453 | if (Row[domain-1] == opcode) | |||
6454 | return Row; | |||
6455 | return nullptr; | |||
6456 | } | |||
6457 | ||||
6458 | static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain, | |||
6459 | ArrayRef<uint16_t[4]> Table) { | |||
6460 | // If this is the integer domain make sure to check both integer columns. | |||
6461 | for (const uint16_t (&Row)[4] : Table) | |||
6462 | if (Row[domain-1] == opcode || (domain == 3 && Row[3] == opcode)) | |||
6463 | return Row; | |||
6464 | return nullptr; | |||
6465 | } | |||
6466 | ||||
6467 | // Helper to attempt to widen/narrow blend masks. | |||
6468 | static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, | |||
6469 | unsigned NewWidth, unsigned *pNewMask = nullptr) { | |||
6470 | assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) &&((((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && "Illegal blend mask scale") ? static_cast<void> (0) : __assert_fail ("((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && \"Illegal blend mask scale\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6471, __PRETTY_FUNCTION__)) | |||
6471 | "Illegal blend mask scale")((((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && "Illegal blend mask scale") ? static_cast<void> (0) : __assert_fail ("((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && \"Illegal blend mask scale\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6471, __PRETTY_FUNCTION__)); | |||
6472 | unsigned NewMask = 0; | |||
6473 | ||||
6474 | if ((OldWidth % NewWidth) == 0) { | |||
6475 | unsigned Scale = OldWidth / NewWidth; | |||
6476 | unsigned SubMask = (1u << Scale) - 1; | |||
6477 | for (unsigned i = 0; i != NewWidth; ++i) { | |||
6478 | unsigned Sub = (OldMask >> (i * Scale)) & SubMask; | |||
6479 | if (Sub == SubMask) | |||
6480 | NewMask |= (1u << i); | |||
6481 | else if (Sub != 0x0) | |||
6482 | return false; | |||
6483 | } | |||
6484 | } else { | |||
6485 | unsigned Scale = NewWidth / OldWidth; | |||
6486 | unsigned SubMask = (1u << Scale) - 1; | |||
6487 | for (unsigned i = 0; i != OldWidth; ++i) { | |||
6488 | if (OldMask & (1 << i)) { | |||
6489 | NewMask |= (SubMask << (i * Scale)); | |||
6490 | } | |||
6491 | } | |||
6492 | } | |||
6493 | ||||
6494 | if (pNewMask) | |||
6495 | *pNewMask = NewMask; | |||
6496 | return true; | |||
6497 | } | |||
6498 | ||||
6499 | uint16_t X86InstrInfo::getExecutionDomainCustom(const MachineInstr &MI) const { | |||
6500 | unsigned Opcode = MI.getOpcode(); | |||
6501 | unsigned NumOperands = MI.getDesc().getNumOperands(); | |||
6502 | ||||
6503 | auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) { | |||
6504 | uint16_t validDomains = 0; | |||
6505 | if (MI.getOperand(NumOperands - 1).isImm()) { | |||
6506 | unsigned Imm = MI.getOperand(NumOperands - 1).getImm(); | |||
6507 | if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4)) | |||
6508 | validDomains |= 0x2; // PackedSingle | |||
6509 | if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2)) | |||
6510 | validDomains |= 0x4; // PackedDouble | |||
6511 | if (!Is256 || Subtarget.hasAVX2()) | |||
6512 | validDomains |= 0x8; // PackedInt | |||
6513 | } | |||
6514 | return validDomains; | |||
6515 | }; | |||
6516 | ||||
6517 | switch (Opcode) { | |||
6518 | case X86::BLENDPDrmi: | |||
6519 | case X86::BLENDPDrri: | |||
6520 | case X86::VBLENDPDrmi: | |||
6521 | case X86::VBLENDPDrri: | |||
6522 | return GetBlendDomains(2, false); | |||
6523 | case X86::VBLENDPDYrmi: | |||
6524 | case X86::VBLENDPDYrri: | |||
6525 | return GetBlendDomains(4, true); | |||
6526 | case X86::BLENDPSrmi: | |||
6527 | case X86::BLENDPSrri: | |||
6528 | case X86::VBLENDPSrmi: | |||
6529 | case X86::VBLENDPSrri: | |||
6530 | case X86::VPBLENDDrmi: | |||
6531 | case X86::VPBLENDDrri: | |||
6532 | return GetBlendDomains(4, false); | |||
6533 | case X86::VBLENDPSYrmi: | |||
6534 | case X86::VBLENDPSYrri: | |||
6535 | case X86::VPBLENDDYrmi: | |||
6536 | case X86::VPBLENDDYrri: | |||
6537 | return GetBlendDomains(8, true); | |||
6538 | case X86::PBLENDWrmi: | |||
6539 | case X86::PBLENDWrri: | |||
6540 | case X86::VPBLENDWrmi: | |||
6541 | case X86::VPBLENDWrri: | |||
6542 | // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks. | |||
6543 | case X86::VPBLENDWYrmi: | |||
6544 | case X86::VPBLENDWYrri: | |||
6545 | return GetBlendDomains(8, false); | |||
6546 | case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: | |||
6547 | case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: | |||
6548 | case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: | |||
6549 | case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: | |||
6550 | case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: | |||
6551 | case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: | |||
6552 | case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: | |||
6553 | case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: | |||
6554 | case X86::VPORDZ128rr: case X86::VPORDZ128rm: | |||
6555 | case X86::VPORDZ256rr: case X86::VPORDZ256rm: | |||
6556 | case X86::VPORQZ128rr: case X86::VPORQZ128rm: | |||
6557 | case X86::VPORQZ256rr: case X86::VPORQZ256rm: | |||
6558 | case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: | |||
6559 | case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: | |||
6560 | case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: | |||
6561 | case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: | |||
6562 | // If we don't have DQI see if we can still switch from an EVEX integer | |||
6563 | // instruction to a VEX floating point instruction. | |||
6564 | if (Subtarget.hasDQI()) | |||
6565 | return 0; | |||
6566 | ||||
6567 | if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16) | |||
6568 | return 0; | |||
6569 | if (RI.getEncodingValue(MI.getOperand(1).getReg()) >= 16) | |||
6570 | return 0; | |||
6571 | // Register forms will have 3 operands. Memory form will have more. | |||
6572 | if (NumOperands == 3 && | |||
6573 | RI.getEncodingValue(MI.getOperand(2).getReg()) >= 16) | |||
6574 | return 0; | |||
6575 | ||||
6576 | // All domains are valid. | |||
6577 | return 0xe; | |||
6578 | case X86::MOVHLPSrr: | |||
6579 | // We can swap domains when both inputs are the same register. | |||
6580 | // FIXME: This doesn't catch all the cases we would like. If the input | |||
6581 | // register isn't KILLed by the instruction, the two address instruction | |||
6582 | // pass puts a COPY on one input. The other input uses the original | |||
6583 | // register. This prevents the same physical register from being used by | |||
6584 | // both inputs. | |||
6585 | if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && | |||
6586 | MI.getOperand(0).getSubReg() == 0 && | |||
6587 | MI.getOperand(1).getSubReg() == 0 && | |||
6588 | MI.getOperand(2).getSubReg() == 0) | |||
6589 | return 0x6; | |||
6590 | return 0; | |||
6591 | } | |||
6592 | return 0; | |||
6593 | } | |||
6594 | ||||
6595 | bool X86InstrInfo::setExecutionDomainCustom(MachineInstr &MI, | |||
6596 | unsigned Domain) const { | |||
6597 | assert(Domain > 0 && Domain < 4 && "Invalid execution domain")((Domain > 0 && Domain < 4 && "Invalid execution domain" ) ? static_cast<void> (0) : __assert_fail ("Domain > 0 && Domain < 4 && \"Invalid execution domain\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6597, __PRETTY_FUNCTION__)); | |||
6598 | uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; | |||
6599 | assert(dom && "Not an SSE instruction")((dom && "Not an SSE instruction") ? static_cast<void > (0) : __assert_fail ("dom && \"Not an SSE instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6599, __PRETTY_FUNCTION__)); | |||
6600 | ||||
6601 | unsigned Opcode = MI.getOpcode(); | |||
6602 | unsigned NumOperands = MI.getDesc().getNumOperands(); | |||
6603 | ||||
6604 | auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) { | |||
6605 | if (MI.getOperand(NumOperands - 1).isImm()) { | |||
6606 | unsigned Imm = MI.getOperand(NumOperands - 1).getImm() & 255; | |||
6607 | Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm); | |||
6608 | unsigned NewImm = Imm; | |||
6609 | ||||
6610 | const uint16_t *table = lookup(Opcode, dom, ReplaceableCustomInstrs); | |||
6611 | if (!table) | |||
6612 | table = lookup(Opcode, dom, ReplaceableCustomAVX2Instrs); | |||
6613 | ||||
6614 | if (Domain == 1) { // PackedSingle | |||
6615 | AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); | |||
6616 | } else if (Domain == 2) { // PackedDouble | |||
6617 | AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2, &NewImm); | |||
6618 | } else if (Domain == 3) { // PackedInt | |||
6619 | if (Subtarget.hasAVX2()) { | |||
6620 | // If we are already VPBLENDW use that, else use VPBLENDD. | |||
6621 | if ((ImmWidth / (Is256 ? 2 : 1)) != 8) { | |||
6622 | table = lookup(Opcode, dom, ReplaceableCustomAVX2Instrs); | |||
6623 | AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); | |||
6624 | } | |||
6625 | } else { | |||
6626 | assert(!Is256 && "128-bit vector expected")((!Is256 && "128-bit vector expected") ? static_cast< void> (0) : __assert_fail ("!Is256 && \"128-bit vector expected\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6626, __PRETTY_FUNCTION__)); | |||
6627 | AdjustBlendMask(Imm, ImmWidth, 8, &NewImm); | |||
6628 | } | |||
6629 | } | |||
6630 | ||||
6631 | assert(table && table[Domain - 1] && "Unknown domain op")((table && table[Domain - 1] && "Unknown domain op" ) ? static_cast<void> (0) : __assert_fail ("table && table[Domain - 1] && \"Unknown domain op\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6631, __PRETTY_FUNCTION__)); | |||
6632 | MI.setDesc(get(table[Domain - 1])); | |||
6633 | MI.getOperand(NumOperands - 1).setImm(NewImm & 255); | |||
6634 | } | |||
6635 | return true; | |||
6636 | }; | |||
6637 | ||||
6638 | switch (Opcode) { | |||
6639 | case X86::BLENDPDrmi: | |||
6640 | case X86::BLENDPDrri: | |||
6641 | case X86::VBLENDPDrmi: | |||
6642 | case X86::VBLENDPDrri: | |||
6643 | return SetBlendDomain(2, false); | |||
6644 | case X86::VBLENDPDYrmi: | |||
6645 | case X86::VBLENDPDYrri: | |||
6646 | return SetBlendDomain(4, true); | |||
6647 | case X86::BLENDPSrmi: | |||
6648 | case X86::BLENDPSrri: | |||
6649 | case X86::VBLENDPSrmi: | |||
6650 | case X86::VBLENDPSrri: | |||
6651 | case X86::VPBLENDDrmi: | |||
6652 | case X86::VPBLENDDrri: | |||
6653 | return SetBlendDomain(4, false); | |||
6654 | case X86::VBLENDPSYrmi: | |||
6655 | case X86::VBLENDPSYrri: | |||
6656 | case X86::VPBLENDDYrmi: | |||
6657 | case X86::VPBLENDDYrri: | |||
6658 | return SetBlendDomain(8, true); | |||
6659 | case X86::PBLENDWrmi: | |||
6660 | case X86::PBLENDWrri: | |||
6661 | case X86::VPBLENDWrmi: | |||
6662 | case X86::VPBLENDWrri: | |||
6663 | return SetBlendDomain(8, false); | |||
6664 | case X86::VPBLENDWYrmi: | |||
6665 | case X86::VPBLENDWYrri: | |||
6666 | return SetBlendDomain(16, true); | |||
6667 | case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: | |||
6668 | case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: | |||
6669 | case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: | |||
6670 | case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: | |||
6671 | case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: | |||
6672 | case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: | |||
6673 | case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: | |||
6674 | case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: | |||
6675 | case X86::VPORDZ128rr: case X86::VPORDZ128rm: | |||
6676 | case X86::VPORDZ256rr: case X86::VPORDZ256rm: | |||
6677 | case X86::VPORQZ128rr: case X86::VPORQZ128rm: | |||
6678 | case X86::VPORQZ256rr: case X86::VPORQZ256rm: | |||
6679 | case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: | |||
6680 | case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: | |||
6681 | case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: | |||
6682 | case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: { | |||
6683 | // Without DQI, convert EVEX instructions to VEX instructions. | |||
6684 | if (Subtarget.hasDQI()) | |||
6685 | return false; | |||
6686 | ||||
6687 | const uint16_t *table = lookupAVX512(MI.getOpcode(), dom, | |||
6688 | ReplaceableCustomAVX512LogicInstrs); | |||
6689 | assert(table && "Instruction not found in table?")((table && "Instruction not found in table?") ? static_cast <void> (0) : __assert_fail ("table && \"Instruction not found in table?\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6689, __PRETTY_FUNCTION__)); | |||
6690 | // Don't change integer Q instructions to D instructions and | |||
6691 | // use D intructions if we started with a PS instruction. | |||
6692 | if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) | |||
6693 | Domain = 4; | |||
6694 | MI.setDesc(get(table[Domain - 1])); | |||
6695 | return true; | |||
6696 | } | |||
6697 | case X86::UNPCKHPDrr: | |||
6698 | case X86::MOVHLPSrr: | |||
6699 | // We just need to commute the instruction which will switch the domains. | |||
6700 | if (Domain != dom && Domain != 3 && | |||
6701 | MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && | |||
6702 | MI.getOperand(0).getSubReg() == 0 && | |||
6703 | MI.getOperand(1).getSubReg() == 0 && | |||
6704 | MI.getOperand(2).getSubReg() == 0) { | |||
6705 | commuteInstruction(MI, false); | |||
6706 | return true; | |||
6707 | } | |||
6708 | // We must always return true for MOVHLPSrr. | |||
6709 | if (Opcode == X86::MOVHLPSrr) | |||
6710 | return true; | |||
6711 | } | |||
6712 | return false; | |||
6713 | } | |||
6714 | ||||
6715 | std::pair<uint16_t, uint16_t> | |||
6716 | X86InstrInfo::getExecutionDomain(const MachineInstr &MI) const { | |||
6717 | uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; | |||
6718 | unsigned opcode = MI.getOpcode(); | |||
6719 | uint16_t validDomains = 0; | |||
6720 | if (domain) { | |||
6721 | // Attempt to match for custom instructions. | |||
6722 | validDomains = getExecutionDomainCustom(MI); | |||
6723 | if (validDomains) | |||
6724 | return std::make_pair(domain, validDomains); | |||
6725 | ||||
6726 | if (lookup(opcode, domain, ReplaceableInstrs)) { | |||
6727 | validDomains = 0xe; | |||
6728 | } else if (lookup(opcode, domain, ReplaceableInstrsAVX2)) { | |||
6729 | validDomains = Subtarget.hasAVX2() ? 0xe : 0x6; | |||
6730 | } else if (lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) { | |||
6731 | // Insert/extract instructions should only effect domain if AVX2 | |||
6732 | // is enabled. | |||
6733 | if (!Subtarget.hasAVX2()) | |||
6734 | return std::make_pair(0, 0); | |||
6735 | validDomains = 0xe; | |||
6736 | } else if (lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) { | |||
6737 | validDomains = 0xe; | |||
6738 | } else if (Subtarget.hasDQI() && lookupAVX512(opcode, domain, | |||
6739 | ReplaceableInstrsAVX512DQ)) { | |||
6740 | validDomains = 0xe; | |||
6741 | } else if (Subtarget.hasDQI()) { | |||
6742 | if (const uint16_t *table = lookupAVX512(opcode, domain, | |||
6743 | ReplaceableInstrsAVX512DQMasked)) { | |||
6744 | if (domain == 1 || (domain == 3 && table[3] == opcode)) | |||
6745 | validDomains = 0xa; | |||
6746 | else | |||
6747 | validDomains = 0xc; | |||
6748 | } | |||
6749 | } | |||
6750 | } | |||
6751 | return std::make_pair(domain, validDomains); | |||
6752 | } | |||
6753 | ||||
6754 | void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const { | |||
6755 | assert(Domain>0 && Domain<4 && "Invalid execution domain")((Domain>0 && Domain<4 && "Invalid execution domain" ) ? static_cast<void> (0) : __assert_fail ("Domain>0 && Domain<4 && \"Invalid execution domain\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6755, __PRETTY_FUNCTION__)); | |||
6756 | uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; | |||
6757 | assert(dom && "Not an SSE instruction")((dom && "Not an SSE instruction") ? static_cast<void > (0) : __assert_fail ("dom && \"Not an SSE instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6757, __PRETTY_FUNCTION__)); | |||
6758 | ||||
6759 | // Attempt to match for custom instructions. | |||
6760 | if (setExecutionDomainCustom(MI, Domain)) | |||
6761 | return; | |||
6762 | ||||
6763 | const uint16_t *table = lookup(MI.getOpcode(), dom, ReplaceableInstrs); | |||
6764 | if (!table) { // try the other table | |||
6765 | assert((Subtarget.hasAVX2() || Domain < 3) &&(((Subtarget.hasAVX2() || Domain < 3) && "256-bit vector operations only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasAVX2() || Domain < 3) && \"256-bit vector operations only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6766, __PRETTY_FUNCTION__)) | |||
6766 | "256-bit vector operations only available in AVX2")(((Subtarget.hasAVX2() || Domain < 3) && "256-bit vector operations only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasAVX2() || Domain < 3) && \"256-bit vector operations only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6766, __PRETTY_FUNCTION__)); | |||
6767 | table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2); | |||
6768 | } | |||
6769 | if (!table) { // try the other table | |||
6770 | assert(Subtarget.hasAVX2() &&((Subtarget.hasAVX2() && "256-bit insert/extract only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"256-bit insert/extract only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6771, __PRETTY_FUNCTION__)) | |||
6771 | "256-bit insert/extract only available in AVX2")((Subtarget.hasAVX2() && "256-bit insert/extract only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"256-bit insert/extract only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6771, __PRETTY_FUNCTION__)); | |||
6772 | table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract); | |||
6773 | } | |||
6774 | if (!table) { // try the AVX512 table | |||
6775 | assert(Subtarget.hasAVX512() && "Requires AVX-512")((Subtarget.hasAVX512() && "Requires AVX-512") ? static_cast <void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"Requires AVX-512\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6775, __PRETTY_FUNCTION__)); | |||
6776 | table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512); | |||
6777 | // Don't change integer Q instructions to D instructions. | |||
6778 | if (table && Domain == 3 && table[3] == MI.getOpcode()) | |||
6779 | Domain = 4; | |||
6780 | } | |||
6781 | if (!table) { // try the AVX512DQ table | |||
6782 | assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ")(((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasDQI() || Domain >= 3) && \"Requires AVX-512DQ\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6782, __PRETTY_FUNCTION__)); | |||
6783 | table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ); | |||
6784 | // Don't change integer Q instructions to D instructions and | |||
6785 | // use D intructions if we started with a PS instruction. | |||
6786 | if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) | |||
6787 | Domain = 4; | |||
6788 | } | |||
6789 | if (!table) { // try the AVX512DQMasked table | |||
6790 | assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ")(((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasDQI() || Domain >= 3) && \"Requires AVX-512DQ\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6790, __PRETTY_FUNCTION__)); | |||
6791 | table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked); | |||
6792 | if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) | |||
6793 | Domain = 4; | |||
6794 | } | |||
6795 | assert(table && "Cannot change domain")((table && "Cannot change domain") ? static_cast<void > (0) : __assert_fail ("table && \"Cannot change domain\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 6795, __PRETTY_FUNCTION__)); | |||
6796 | MI.setDesc(get(table[Domain - 1])); | |||
6797 | } | |||
6798 | ||||
6799 | /// Return the noop instruction to use for a noop. | |||
6800 | void X86InstrInfo::getNoop(MCInst &NopInst) const { | |||
6801 | NopInst.setOpcode(X86::NOOP); | |||
6802 | } | |||
6803 | ||||
6804 | bool X86InstrInfo::isHighLatencyDef(int opc) const { | |||
6805 | switch (opc) { | |||
6806 | default: return false; | |||
6807 | case X86::DIVPDrm: | |||
6808 | case X86::DIVPDrr: | |||
6809 | case X86::DIVPSrm: | |||
6810 | case X86::DIVPSrr: | |||
6811 | case X86::DIVSDrm: | |||
6812 | case X86::DIVSDrm_Int: | |||
6813 | case X86::DIVSDrr: | |||
6814 | case X86::DIVSDrr_Int: | |||
6815 | case X86::DIVSSrm: | |||
6816 | case X86::DIVSSrm_Int: | |||
6817 | case X86::DIVSSrr: | |||
6818 | case X86::DIVSSrr_Int: | |||
6819 | case X86::SQRTPDm: | |||
6820 | case X86::SQRTPDr: | |||
6821 | case X86::SQRTPSm: | |||
6822 | case X86::SQRTPSr: | |||
6823 | case X86::SQRTSDm: | |||
6824 | case X86::SQRTSDm_Int: | |||
6825 | case X86::SQRTSDr: | |||
6826 | case X86::SQRTSDr_Int: | |||
6827 | case X86::SQRTSSm: | |||
6828 | case X86::SQRTSSm_Int: | |||
6829 | case X86::SQRTSSr: | |||
6830 | case X86::SQRTSSr_Int: | |||
6831 | // AVX instructions with high latency | |||
6832 | case X86::VDIVPDrm: | |||
6833 | case X86::VDIVPDrr: | |||
6834 | case X86::VDIVPDYrm: | |||
6835 | case X86::VDIVPDYrr: | |||
6836 | case X86::VDIVPSrm: | |||
6837 | case X86::VDIVPSrr: | |||
6838 | case X86::VDIVPSYrm: | |||
6839 | case X86::VDIVPSYrr: | |||
6840 | case X86::VDIVSDrm: | |||
6841 | case X86::VDIVSDrm_Int: | |||
6842 | case X86::VDIVSDrr: | |||
6843 | case X86::VDIVSDrr_Int: | |||
6844 | case X86::VDIVSSrm: | |||
6845 | case X86::VDIVSSrm_Int: | |||
6846 | case X86::VDIVSSrr: | |||
6847 | case X86::VDIVSSrr_Int: | |||
6848 | case X86::VSQRTPDm: | |||
6849 | case X86::VSQRTPDr: | |||
6850 | case X86::VSQRTPDYm: | |||
6851 | case X86::VSQRTPDYr: | |||
6852 | case X86::VSQRTPSm: | |||
6853 | case X86::VSQRTPSr: | |||
6854 | case X86::VSQRTPSYm: | |||
6855 | case X86::VSQRTPSYr: | |||
6856 | case X86::VSQRTSDm: | |||
6857 | case X86::VSQRTSDm_Int: | |||
6858 | case X86::VSQRTSDr: | |||
6859 | case X86::VSQRTSDr_Int: | |||
6860 | case X86::VSQRTSSm: | |||
6861 | case X86::VSQRTSSm_Int: | |||
6862 | case X86::VSQRTSSr: | |||
6863 | case X86::VSQRTSSr_Int: | |||
6864 | // AVX512 instructions with high latency | |||
6865 | case X86::VDIVPDZ128rm: | |||
6866 | case X86::VDIVPDZ128rmb: | |||
6867 | case X86::VDIVPDZ128rmbk: | |||
6868 | case X86::VDIVPDZ128rmbkz: | |||
6869 | case X86::VDIVPDZ128rmk: | |||
6870 | case X86::VDIVPDZ128rmkz: | |||
6871 | case X86::VDIVPDZ128rr: | |||
6872 | case X86::VDIVPDZ128rrk: | |||
6873 | case X86::VDIVPDZ128rrkz: | |||
6874 | case X86::VDIVPDZ256rm: | |||
6875 | case X86::VDIVPDZ256rmb: | |||
6876 | case X86::VDIVPDZ256rmbk: | |||
6877 | case X86::VDIVPDZ256rmbkz: | |||
6878 | case X86::VDIVPDZ256rmk: | |||
6879 | case X86::VDIVPDZ256rmkz: | |||
6880 | case X86::VDIVPDZ256rr: | |||
6881 | case X86::VDIVPDZ256rrk: | |||
6882 | case X86::VDIVPDZ256rrkz: | |||
6883 | case X86::VDIVPDZrrb: | |||
6884 | case X86::VDIVPDZrrbk: | |||
6885 | case X86::VDIVPDZrrbkz: | |||
6886 | case X86::VDIVPDZrm: | |||
6887 | case X86::VDIVPDZrmb: | |||
6888 | case X86::VDIVPDZrmbk: | |||
6889 | case X86::VDIVPDZrmbkz: | |||
6890 | case X86::VDIVPDZrmk: | |||
6891 | case X86::VDIVPDZrmkz: | |||
6892 | case X86::VDIVPDZrr: | |||
6893 | case X86::VDIVPDZrrk: | |||
6894 | case X86::VDIVPDZrrkz: | |||
6895 | case X86::VDIVPSZ128rm: | |||
6896 | case X86::VDIVPSZ128rmb: | |||
6897 | case X86::VDIVPSZ128rmbk: | |||
6898 | case X86::VDIVPSZ128rmbkz: | |||
6899 | case X86::VDIVPSZ128rmk: | |||
6900 | case X86::VDIVPSZ128rmkz: | |||
6901 | case X86::VDIVPSZ128rr: | |||
6902 | case X86::VDIVPSZ128rrk: | |||
6903 | case X86::VDIVPSZ128rrkz: | |||
6904 | case X86::VDIVPSZ256rm: | |||
6905 | case X86::VDIVPSZ256rmb: | |||
6906 | case X86::VDIVPSZ256rmbk: | |||
6907 | case X86::VDIVPSZ256rmbkz: | |||
6908 | case X86::VDIVPSZ256rmk: | |||
6909 | case X86::VDIVPSZ256rmkz: | |||
6910 | case X86::VDIVPSZ256rr: | |||
6911 | case X86::VDIVPSZ256rrk: | |||
6912 | case X86::VDIVPSZ256rrkz: | |||
6913 | case X86::VDIVPSZrrb: | |||
6914 | case X86::VDIVPSZrrbk: | |||
6915 | case X86::VDIVPSZrrbkz: | |||
6916 | case X86::VDIVPSZrm: | |||
6917 | case X86::VDIVPSZrmb: | |||
6918 | case X86::VDIVPSZrmbk: | |||
6919 | case X86::VDIVPSZrmbkz: | |||
6920 | case X86::VDIVPSZrmk: | |||
6921 | case X86::VDIVPSZrmkz: | |||
6922 | case X86::VDIVPSZrr: | |||
6923 | case X86::VDIVPSZrrk: | |||
6924 | case X86::VDIVPSZrrkz: | |||
6925 | case X86::VDIVSDZrm: | |||
6926 | case X86::VDIVSDZrr: | |||
6927 | case X86::VDIVSDZrm_Int: | |||
6928 | case X86::VDIVSDZrm_Intk: | |||
6929 | case X86::VDIVSDZrm_Intkz: | |||
6930 | case X86::VDIVSDZrr_Int: | |||
6931 | case X86::VDIVSDZrr_Intk: | |||
6932 | case X86::VDIVSDZrr_Intkz: | |||
6933 | case X86::VDIVSDZrrb_Int: | |||
6934 | case X86::VDIVSDZrrb_Intk: | |||
6935 | case X86::VDIVSDZrrb_Intkz: | |||
6936 | case X86::VDIVSSZrm: | |||
6937 | case X86::VDIVSSZrr: | |||
6938 | case X86::VDIVSSZrm_Int: | |||
6939 | case X86::VDIVSSZrm_Intk: | |||
6940 | case X86::VDIVSSZrm_Intkz: | |||
6941 | case X86::VDIVSSZrr_Int: | |||
6942 | case X86::VDIVSSZrr_Intk: | |||
6943 | case X86::VDIVSSZrr_Intkz: | |||
6944 | case X86::VDIVSSZrrb_Int: | |||
6945 | case X86::VDIVSSZrrb_Intk: | |||
6946 | case X86::VDIVSSZrrb_Intkz: | |||
6947 | case X86::VSQRTPDZ128m: | |||
6948 | case X86::VSQRTPDZ128mb: | |||
6949 | case X86::VSQRTPDZ128mbk: | |||
6950 | case X86::VSQRTPDZ128mbkz: | |||
6951 | case X86::VSQRTPDZ128mk: | |||
6952 | case X86::VSQRTPDZ128mkz: | |||
6953 | case X86::VSQRTPDZ128r: | |||
6954 | case X86::VSQRTPDZ128rk: | |||
6955 | case X86::VSQRTPDZ128rkz: | |||
6956 | case X86::VSQRTPDZ256m: | |||
6957 | case X86::VSQRTPDZ256mb: | |||
6958 | case X86::VSQRTPDZ256mbk: | |||
6959 | case X86::VSQRTPDZ256mbkz: | |||
6960 | case X86::VSQRTPDZ256mk: | |||
6961 | case X86::VSQRTPDZ256mkz: | |||
6962 | case X86::VSQRTPDZ256r: | |||
6963 | case X86::VSQRTPDZ256rk: | |||
6964 | case X86::VSQRTPDZ256rkz: | |||
6965 | case X86::VSQRTPDZm: | |||
6966 | case X86::VSQRTPDZmb: | |||
6967 | case X86::VSQRTPDZmbk: | |||
6968 | case X86::VSQRTPDZmbkz: | |||
6969 | case X86::VSQRTPDZmk: | |||
6970 | case X86::VSQRTPDZmkz: | |||
6971 | case X86::VSQRTPDZr: | |||
6972 | case X86::VSQRTPDZrb: | |||
6973 | case X86::VSQRTPDZrbk: | |||
6974 | case X86::VSQRTPDZrbkz: | |||
6975 | case X86::VSQRTPDZrk: | |||
6976 | case X86::VSQRTPDZrkz: | |||
6977 | case X86::VSQRTPSZ128m: | |||
6978 | case X86::VSQRTPSZ128mb: | |||
6979 | case X86::VSQRTPSZ128mbk: | |||
6980 | case X86::VSQRTPSZ128mbkz: | |||
6981 | case X86::VSQRTPSZ128mk: | |||
6982 | case X86::VSQRTPSZ128mkz: | |||
6983 | case X86::VSQRTPSZ128r: | |||
6984 | case X86::VSQRTPSZ128rk: | |||
6985 | case X86::VSQRTPSZ128rkz: | |||
6986 | case X86::VSQRTPSZ256m: | |||
6987 | case X86::VSQRTPSZ256mb: | |||
6988 | case X86::VSQRTPSZ256mbk: | |||
6989 | case X86::VSQRTPSZ256mbkz: | |||
6990 | case X86::VSQRTPSZ256mk: | |||
6991 | case X86::VSQRTPSZ256mkz: | |||
6992 | case X86::VSQRTPSZ256r: | |||
6993 | case X86::VSQRTPSZ256rk: | |||
6994 | case X86::VSQRTPSZ256rkz: | |||
6995 | case X86::VSQRTPSZm: | |||
6996 | case X86::VSQRTPSZmb: | |||
6997 | case X86::VSQRTPSZmbk: | |||
6998 | case X86::VSQRTPSZmbkz: | |||
6999 | case X86::VSQRTPSZmk: | |||
7000 | case X86::VSQRTPSZmkz: | |||
7001 | case X86::VSQRTPSZr: | |||
7002 | case X86::VSQRTPSZrb: | |||
7003 | case X86::VSQRTPSZrbk: | |||
7004 | case X86::VSQRTPSZrbkz: | |||
7005 | case X86::VSQRTPSZrk: | |||
7006 | case X86::VSQRTPSZrkz: | |||
7007 | case X86::VSQRTSDZm: | |||
7008 | case X86::VSQRTSDZm_Int: | |||
7009 | case X86::VSQRTSDZm_Intk: | |||
7010 | case X86::VSQRTSDZm_Intkz: | |||
7011 | case X86::VSQRTSDZr: | |||
7012 | case X86::VSQRTSDZr_Int: | |||
7013 | case X86::VSQRTSDZr_Intk: | |||
7014 | case X86::VSQRTSDZr_Intkz: | |||
7015 | case X86::VSQRTSDZrb_Int: | |||
7016 | case X86::VSQRTSDZrb_Intk: | |||
7017 | case X86::VSQRTSDZrb_Intkz: | |||
7018 | case X86::VSQRTSSZm: | |||
7019 | case X86::VSQRTSSZm_Int: | |||
7020 | case X86::VSQRTSSZm_Intk: | |||
7021 | case X86::VSQRTSSZm_Intkz: | |||
7022 | case X86::VSQRTSSZr: | |||
7023 | case X86::VSQRTSSZr_Int: | |||
7024 | case X86::VSQRTSSZr_Intk: | |||
7025 | case X86::VSQRTSSZr_Intkz: | |||
7026 | case X86::VSQRTSSZrb_Int: | |||
7027 | case X86::VSQRTSSZrb_Intk: | |||
7028 | case X86::VSQRTSSZrb_Intkz: | |||
7029 | ||||
7030 | case X86::VGATHERDPDYrm: | |||
7031 | case X86::VGATHERDPDZ128rm: | |||
7032 | case X86::VGATHERDPDZ256rm: | |||
7033 | case X86::VGATHERDPDZrm: | |||
7034 | case X86::VGATHERDPDrm: | |||
7035 | case X86::VGATHERDPSYrm: | |||
7036 | case X86::VGATHERDPSZ128rm: | |||
7037 | case X86::VGATHERDPSZ256rm: | |||
7038 | case X86::VGATHERDPSZrm: | |||
7039 | case X86::VGATHERDPSrm: | |||
7040 | case X86::VGATHERPF0DPDm: | |||
7041 | case X86::VGATHERPF0DPSm: | |||
7042 | case X86::VGATHERPF0QPDm: | |||
7043 | case X86::VGATHERPF0QPSm: | |||
7044 | case X86::VGATHERPF1DPDm: | |||
7045 | case X86::VGATHERPF1DPSm: | |||
7046 | case X86::VGATHERPF1QPDm: | |||
7047 | case X86::VGATHERPF1QPSm: | |||
7048 | case X86::VGATHERQPDYrm: | |||
7049 | case X86::VGATHERQPDZ128rm: | |||
7050 | case X86::VGATHERQPDZ256rm: | |||
7051 | case X86::VGATHERQPDZrm: | |||
7052 | case X86::VGATHERQPDrm: | |||
7053 | case X86::VGATHERQPSYrm: | |||
7054 | case X86::VGATHERQPSZ128rm: | |||
7055 | case X86::VGATHERQPSZ256rm: | |||
7056 | case X86::VGATHERQPSZrm: | |||
7057 | case X86::VGATHERQPSrm: | |||
7058 | case X86::VPGATHERDDYrm: | |||
7059 | case X86::VPGATHERDDZ128rm: | |||
7060 | case X86::VPGATHERDDZ256rm: | |||
7061 | case X86::VPGATHERDDZrm: | |||
7062 | case X86::VPGATHERDDrm: | |||
7063 | case X86::VPGATHERDQYrm: | |||
7064 | case X86::VPGATHERDQZ128rm: | |||
7065 | case X86::VPGATHERDQZ256rm: | |||
7066 | case X86::VPGATHERDQZrm: | |||
7067 | case X86::VPGATHERDQrm: | |||
7068 | case X86::VPGATHERQDYrm: | |||
7069 | case X86::VPGATHERQDZ128rm: | |||
7070 | case X86::VPGATHERQDZ256rm: | |||
7071 | case X86::VPGATHERQDZrm: | |||
7072 | case X86::VPGATHERQDrm: | |||
7073 | case X86::VPGATHERQQYrm: | |||
7074 | case X86::VPGATHERQQZ128rm: | |||
7075 | case X86::VPGATHERQQZ256rm: | |||
7076 | case X86::VPGATHERQQZrm: | |||
7077 | case X86::VPGATHERQQrm: | |||
7078 | case X86::VSCATTERDPDZ128mr: | |||
7079 | case X86::VSCATTERDPDZ256mr: | |||
7080 | case X86::VSCATTERDPDZmr: | |||
7081 | case X86::VSCATTERDPSZ128mr: | |||
7082 | case X86::VSCATTERDPSZ256mr: | |||
7083 | case X86::VSCATTERDPSZmr: | |||
7084 | case X86::VSCATTERPF0DPDm: | |||
7085 | case X86::VSCATTERPF0DPSm: | |||
7086 | case X86::VSCATTERPF0QPDm: | |||
7087 | case X86::VSCATTERPF0QPSm: | |||
7088 | case X86::VSCATTERPF1DPDm: | |||
7089 | case X86::VSCATTERPF1DPSm: | |||
7090 | case X86::VSCATTERPF1QPDm: | |||
7091 | case X86::VSCATTERPF1QPSm: | |||
7092 | case X86::VSCATTERQPDZ128mr: | |||
7093 | case X86::VSCATTERQPDZ256mr: | |||
7094 | case X86::VSCATTERQPDZmr: | |||
7095 | case X86::VSCATTERQPSZ128mr: | |||
7096 | case X86::VSCATTERQPSZ256mr: | |||
7097 | case X86::VSCATTERQPSZmr: | |||
7098 | case X86::VPSCATTERDDZ128mr: | |||
7099 | case X86::VPSCATTERDDZ256mr: | |||
7100 | case X86::VPSCATTERDDZmr: | |||
7101 | case X86::VPSCATTERDQZ128mr: | |||
7102 | case X86::VPSCATTERDQZ256mr: | |||
7103 | case X86::VPSCATTERDQZmr: | |||
7104 | case X86::VPSCATTERQDZ128mr: | |||
7105 | case X86::VPSCATTERQDZ256mr: | |||
7106 | case X86::VPSCATTERQDZmr: | |||
7107 | case X86::VPSCATTERQQZ128mr: | |||
7108 | case X86::VPSCATTERQQZ256mr: | |||
7109 | case X86::VPSCATTERQQZmr: | |||
7110 | return true; | |||
7111 | } | |||
7112 | } | |||
7113 | ||||
7114 | bool X86InstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel, | |||
7115 | const MachineRegisterInfo *MRI, | |||
7116 | const MachineInstr &DefMI, | |||
7117 | unsigned DefIdx, | |||
7118 | const MachineInstr &UseMI, | |||
7119 | unsigned UseIdx) const { | |||
7120 | return isHighLatencyDef(DefMI.getOpcode()); | |||
7121 | } | |||
7122 | ||||
7123 | bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst, | |||
7124 | const MachineBasicBlock *MBB) const { | |||
7125 | assert((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) &&(((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && "Reassociation needs binary operators") ? static_cast<void > (0) : __assert_fail ("(Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && \"Reassociation needs binary operators\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7126, __PRETTY_FUNCTION__)) | |||
7126 | "Reassociation needs binary operators")(((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && "Reassociation needs binary operators") ? static_cast<void > (0) : __assert_fail ("(Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && \"Reassociation needs binary operators\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7126, __PRETTY_FUNCTION__)); | |||
7127 | ||||
7128 | // Integer binary math/logic instructions have a third source operand: | |||
7129 | // the EFLAGS register. That operand must be both defined here and never | |||
7130 | // used; ie, it must be dead. If the EFLAGS operand is live, then we can | |||
7131 | // not change anything because rearranging the operands could affect other | |||
7132 | // instructions that depend on the exact status flags (zero, sign, etc.) | |||
7133 | // that are set by using these particular operands with this operation. | |||
7134 | if (Inst.getNumOperands() == 4) { | |||
7135 | assert(Inst.getOperand(3).isReg() &&((Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg () == X86::EFLAGS && "Unexpected operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7137, __PRETTY_FUNCTION__)) | |||
7136 | Inst.getOperand(3).getReg() == X86::EFLAGS &&((Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg () == X86::EFLAGS && "Unexpected operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7137, __PRETTY_FUNCTION__)) | |||
7137 | "Unexpected operand in reassociable instruction")((Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg () == X86::EFLAGS && "Unexpected operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7137, __PRETTY_FUNCTION__)); | |||
7138 | if (!Inst.getOperand(3).isDead()) | |||
7139 | return false; | |||
7140 | } | |||
7141 | ||||
7142 | return TargetInstrInfo::hasReassociableOperands(Inst, MBB); | |||
7143 | } | |||
7144 | ||||
7145 | // TODO: There are many more machine instruction opcodes to match: | |||
7146 | // 1. Other data types (integer, vectors) | |||
7147 | // 2. Other math / logic operations (xor, or) | |||
7148 | // 3. Other forms of the same operation (intrinsics and other variants) | |||
7149 | bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const { | |||
7150 | switch (Inst.getOpcode()) { | |||
7151 | case X86::AND8rr: | |||
7152 | case X86::AND16rr: | |||
7153 | case X86::AND32rr: | |||
7154 | case X86::AND64rr: | |||
7155 | case X86::OR8rr: | |||
7156 | case X86::OR16rr: | |||
7157 | case X86::OR32rr: | |||
7158 | case X86::OR64rr: | |||
7159 | case X86::XOR8rr: | |||
7160 | case X86::XOR16rr: | |||
7161 | case X86::XOR32rr: | |||
7162 | case X86::XOR64rr: | |||
7163 | case X86::IMUL16rr: | |||
7164 | case X86::IMUL32rr: | |||
7165 | case X86::IMUL64rr: | |||
7166 | case X86::PANDrr: | |||
7167 | case X86::PORrr: | |||
7168 | case X86::PXORrr: | |||
7169 | case X86::ANDPDrr: | |||
7170 | case X86::ANDPSrr: | |||
7171 | case X86::ORPDrr: | |||
7172 | case X86::ORPSrr: | |||
7173 | case X86::XORPDrr: | |||
7174 | case X86::XORPSrr: | |||
7175 | case X86::PADDBrr: | |||
7176 | case X86::PADDWrr: | |||
7177 | case X86::PADDDrr: | |||
7178 | case X86::PADDQrr: | |||
7179 | case X86::VPANDrr: | |||
7180 | case X86::VPANDYrr: | |||
7181 | case X86::VPANDDZ128rr: | |||
7182 | case X86::VPANDDZ256rr: | |||
7183 | case X86::VPANDDZrr: | |||
7184 | case X86::VPANDQZ128rr: | |||
7185 | case X86::VPANDQZ256rr: | |||
7186 | case X86::VPANDQZrr: | |||
7187 | case X86::VPORrr: | |||
7188 | case X86::VPORYrr: | |||
7189 | case X86::VPORDZ128rr: | |||
7190 | case X86::VPORDZ256rr: | |||
7191 | case X86::VPORDZrr: | |||
7192 | case X86::VPORQZ128rr: | |||
7193 | case X86::VPORQZ256rr: | |||
7194 | case X86::VPORQZrr: | |||
7195 | case X86::VPXORrr: | |||
7196 | case X86::VPXORYrr: | |||
7197 | case X86::VPXORDZ128rr: | |||
7198 | case X86::VPXORDZ256rr: | |||
7199 | case X86::VPXORDZrr: | |||
7200 | case X86::VPXORQZ128rr: | |||
7201 | case X86::VPXORQZ256rr: | |||
7202 | case X86::VPXORQZrr: | |||
7203 | case X86::VANDPDrr: | |||
7204 | case X86::VANDPSrr: | |||
7205 | case X86::VANDPDYrr: | |||
7206 | case X86::VANDPSYrr: | |||
7207 | case X86::VANDPDZ128rr: | |||
7208 | case X86::VANDPSZ128rr: | |||
7209 | case X86::VANDPDZ256rr: | |||
7210 | case X86::VANDPSZ256rr: | |||
7211 | case X86::VANDPDZrr: | |||
7212 | case X86::VANDPSZrr: | |||
7213 | case X86::VORPDrr: | |||
7214 | case X86::VORPSrr: | |||
7215 | case X86::VORPDYrr: | |||
7216 | case X86::VORPSYrr: | |||
7217 | case X86::VORPDZ128rr: | |||
7218 | case X86::VORPSZ128rr: | |||
7219 | case X86::VORPDZ256rr: | |||
7220 | case X86::VORPSZ256rr: | |||
7221 | case X86::VORPDZrr: | |||
7222 | case X86::VORPSZrr: | |||
7223 | case X86::VXORPDrr: | |||
7224 | case X86::VXORPSrr: | |||
7225 | case X86::VXORPDYrr: | |||
7226 | case X86::VXORPSYrr: | |||
7227 | case X86::VXORPDZ128rr: | |||
7228 | case X86::VXORPSZ128rr: | |||
7229 | case X86::VXORPDZ256rr: | |||
7230 | case X86::VXORPSZ256rr: | |||
7231 | case X86::VXORPDZrr: | |||
7232 | case X86::VXORPSZrr: | |||
7233 | case X86::KADDBrr: | |||
7234 | case X86::KADDWrr: | |||
7235 | case X86::KADDDrr: | |||
7236 | case X86::KADDQrr: | |||
7237 | case X86::KANDBrr: | |||
7238 | case X86::KANDWrr: | |||
7239 | case X86::KANDDrr: | |||
7240 | case X86::KANDQrr: | |||
7241 | case X86::KORBrr: | |||
7242 | case X86::KORWrr: | |||
7243 | case X86::KORDrr: | |||
7244 | case X86::KORQrr: | |||
7245 | case X86::KXORBrr: | |||
7246 | case X86::KXORWrr: | |||
7247 | case X86::KXORDrr: | |||
7248 | case X86::KXORQrr: | |||
7249 | case X86::VPADDBrr: | |||
7250 | case X86::VPADDWrr: | |||
7251 | case X86::VPADDDrr: | |||
7252 | case X86::VPADDQrr: | |||
7253 | case X86::VPADDBYrr: | |||
7254 | case X86::VPADDWYrr: | |||
7255 | case X86::VPADDDYrr: | |||
7256 | case X86::VPADDQYrr: | |||
7257 | case X86::VPADDBZ128rr: | |||
7258 | case X86::VPADDWZ128rr: | |||
7259 | case X86::VPADDDZ128rr: | |||
7260 | case X86::VPADDQZ128rr: | |||
7261 | case X86::VPADDBZ256rr: | |||
7262 | case X86::VPADDWZ256rr: | |||
7263 | case X86::VPADDDZ256rr: | |||
7264 | case X86::VPADDQZ256rr: | |||
7265 | case X86::VPADDBZrr: | |||
7266 | case X86::VPADDWZrr: | |||
7267 | case X86::VPADDDZrr: | |||
7268 | case X86::VPADDQZrr: | |||
7269 | case X86::VPMULLWrr: | |||
7270 | case X86::VPMULLWYrr: | |||
7271 | case X86::VPMULLWZ128rr: | |||
7272 | case X86::VPMULLWZ256rr: | |||
7273 | case X86::VPMULLWZrr: | |||
7274 | case X86::VPMULLDrr: | |||
7275 | case X86::VPMULLDYrr: | |||
7276 | case X86::VPMULLDZ128rr: | |||
7277 | case X86::VPMULLDZ256rr: | |||
7278 | case X86::VPMULLDZrr: | |||
7279 | case X86::VPMULLQZ128rr: | |||
7280 | case X86::VPMULLQZ256rr: | |||
7281 | case X86::VPMULLQZrr: | |||
7282 | // Normal min/max instructions are not commutative because of NaN and signed | |||
7283 | // zero semantics, but these are. Thus, there's no need to check for global | |||
7284 | // relaxed math; the instructions themselves have the properties we need. | |||
7285 | case X86::MAXCPDrr: | |||
7286 | case X86::MAXCPSrr: | |||
7287 | case X86::MAXCSDrr: | |||
7288 | case X86::MAXCSSrr: | |||
7289 | case X86::MINCPDrr: | |||
7290 | case X86::MINCPSrr: | |||
7291 | case X86::MINCSDrr: | |||
7292 | case X86::MINCSSrr: | |||
7293 | case X86::VMAXCPDrr: | |||
7294 | case X86::VMAXCPSrr: | |||
7295 | case X86::VMAXCPDYrr: | |||
7296 | case X86::VMAXCPSYrr: | |||
7297 | case X86::VMAXCPDZ128rr: | |||
7298 | case X86::VMAXCPSZ128rr: | |||
7299 | case X86::VMAXCPDZ256rr: | |||
7300 | case X86::VMAXCPSZ256rr: | |||
7301 | case X86::VMAXCPDZrr: | |||
7302 | case X86::VMAXCPSZrr: | |||
7303 | case X86::VMAXCSDrr: | |||
7304 | case X86::VMAXCSSrr: | |||
7305 | case X86::VMAXCSDZrr: | |||
7306 | case X86::VMAXCSSZrr: | |||
7307 | case X86::VMINCPDrr: | |||
7308 | case X86::VMINCPSrr: | |||
7309 | case X86::VMINCPDYrr: | |||
7310 | case X86::VMINCPSYrr: | |||
7311 | case X86::VMINCPDZ128rr: | |||
7312 | case X86::VMINCPSZ128rr: | |||
7313 | case X86::VMINCPDZ256rr: | |||
7314 | case X86::VMINCPSZ256rr: | |||
7315 | case X86::VMINCPDZrr: | |||
7316 | case X86::VMINCPSZrr: | |||
7317 | case X86::VMINCSDrr: | |||
7318 | case X86::VMINCSSrr: | |||
7319 | case X86::VMINCSDZrr: | |||
7320 | case X86::VMINCSSZrr: | |||
7321 | return true; | |||
7322 | case X86::ADDPDrr: | |||
7323 | case X86::ADDPSrr: | |||
7324 | case X86::ADDSDrr: | |||
7325 | case X86::ADDSSrr: | |||
7326 | case X86::MULPDrr: | |||
7327 | case X86::MULPSrr: | |||
7328 | case X86::MULSDrr: | |||
7329 | case X86::MULSSrr: | |||
7330 | case X86::VADDPDrr: | |||
7331 | case X86::VADDPSrr: | |||
7332 | case X86::VADDPDYrr: | |||
7333 | case X86::VADDPSYrr: | |||
7334 | case X86::VADDPDZ128rr: | |||
7335 | case X86::VADDPSZ128rr: | |||
7336 | case X86::VADDPDZ256rr: | |||
7337 | case X86::VADDPSZ256rr: | |||
7338 | case X86::VADDPDZrr: | |||
7339 | case X86::VADDPSZrr: | |||
7340 | case X86::VADDSDrr: | |||
7341 | case X86::VADDSSrr: | |||
7342 | case X86::VADDSDZrr: | |||
7343 | case X86::VADDSSZrr: | |||
7344 | case X86::VMULPDrr: | |||
7345 | case X86::VMULPSrr: | |||
7346 | case X86::VMULPDYrr: | |||
7347 | case X86::VMULPSYrr: | |||
7348 | case X86::VMULPDZ128rr: | |||
7349 | case X86::VMULPSZ128rr: | |||
7350 | case X86::VMULPDZ256rr: | |||
7351 | case X86::VMULPSZ256rr: | |||
7352 | case X86::VMULPDZrr: | |||
7353 | case X86::VMULPSZrr: | |||
7354 | case X86::VMULSDrr: | |||
7355 | case X86::VMULSSrr: | |||
7356 | case X86::VMULSDZrr: | |||
7357 | case X86::VMULSSZrr: | |||
7358 | return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath; | |||
7359 | default: | |||
7360 | return false; | |||
7361 | } | |||
7362 | } | |||
7363 | ||||
7364 | /// This is an architecture-specific helper function of reassociateOps. | |||
7365 | /// Set special operand attributes for new instructions after reassociation. | |||
7366 | void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, | |||
7367 | MachineInstr &OldMI2, | |||
7368 | MachineInstr &NewMI1, | |||
7369 | MachineInstr &NewMI2) const { | |||
7370 | // Integer instructions define an implicit EFLAGS source register operand as | |||
7371 | // the third source (fourth total) operand. | |||
7372 | if (OldMI1.getNumOperands() != 4 || OldMI2.getNumOperands() != 4) | |||
7373 | return; | |||
7374 | ||||
7375 | assert(NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 &&((NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands () == 4 && "Unexpected instruction type for reassociation" ) ? static_cast<void> (0) : __assert_fail ("NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 && \"Unexpected instruction type for reassociation\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7376, __PRETTY_FUNCTION__)) | |||
7376 | "Unexpected instruction type for reassociation")((NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands () == 4 && "Unexpected instruction type for reassociation" ) ? static_cast<void> (0) : __assert_fail ("NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 && \"Unexpected instruction type for reassociation\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7376, __PRETTY_FUNCTION__)); | |||
7377 | ||||
7378 | MachineOperand &OldOp1 = OldMI1.getOperand(3); | |||
7379 | MachineOperand &OldOp2 = OldMI2.getOperand(3); | |||
7380 | MachineOperand &NewOp1 = NewMI1.getOperand(3); | |||
7381 | MachineOperand &NewOp2 = NewMI2.getOperand(3); | |||
7382 | ||||
7383 | assert(OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() &&((OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7384, __PRETTY_FUNCTION__)) | |||
7384 | "Must have dead EFLAGS operand in reassociable instruction")((OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7384, __PRETTY_FUNCTION__)); | |||
7385 | assert(OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() &&((OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7386, __PRETTY_FUNCTION__)) | |||
7386 | "Must have dead EFLAGS operand in reassociable instruction")((OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7386, __PRETTY_FUNCTION__)); | |||
7387 | ||||
7388 | (void)OldOp1; | |||
7389 | (void)OldOp2; | |||
7390 | ||||
7391 | assert(NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS &&((NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7392, __PRETTY_FUNCTION__)) | |||
7392 | "Unexpected operand in reassociable instruction")((NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7392, __PRETTY_FUNCTION__)); | |||
7393 | assert(NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS &&((NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7394, __PRETTY_FUNCTION__)) | |||
7394 | "Unexpected operand in reassociable instruction")((NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7394, __PRETTY_FUNCTION__)); | |||
7395 | ||||
7396 | // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations | |||
7397 | // of this pass or other passes. The EFLAGS operands must be dead in these new | |||
7398 | // instructions because the EFLAGS operands in the original instructions must | |||
7399 | // be dead in order for reassociation to occur. | |||
7400 | NewOp1.setIsDead(); | |||
7401 | NewOp2.setIsDead(); | |||
7402 | } | |||
7403 | ||||
7404 | std::pair<unsigned, unsigned> | |||
7405 | X86InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { | |||
7406 | return std::make_pair(TF, 0u); | |||
7407 | } | |||
7408 | ||||
7409 | ArrayRef<std::pair<unsigned, const char *>> | |||
7410 | X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const { | |||
7411 | using namespace X86II; | |||
7412 | static const std::pair<unsigned, const char *> TargetFlags[] = { | |||
7413 | {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"}, | |||
7414 | {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"}, | |||
7415 | {MO_GOT, "x86-got"}, | |||
7416 | {MO_GOTOFF, "x86-gotoff"}, | |||
7417 | {MO_GOTPCREL, "x86-gotpcrel"}, | |||
7418 | {MO_PLT, "x86-plt"}, | |||
7419 | {MO_TLSGD, "x86-tlsgd"}, | |||
7420 | {MO_TLSLD, "x86-tlsld"}, | |||
7421 | {MO_TLSLDM, "x86-tlsldm"}, | |||
7422 | {MO_GOTTPOFF, "x86-gottpoff"}, | |||
7423 | {MO_INDNTPOFF, "x86-indntpoff"}, | |||
7424 | {MO_TPOFF, "x86-tpoff"}, | |||
7425 | {MO_DTPOFF, "x86-dtpoff"}, | |||
7426 | {MO_NTPOFF, "x86-ntpoff"}, | |||
7427 | {MO_GOTNTPOFF, "x86-gotntpoff"}, | |||
7428 | {MO_DLLIMPORT, "x86-dllimport"}, | |||
7429 | {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"}, | |||
7430 | {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"}, | |||
7431 | {MO_TLVP, "x86-tlvp"}, | |||
7432 | {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"}, | |||
7433 | {MO_SECREL, "x86-secrel"}, | |||
7434 | {MO_COFFSTUB, "x86-coffstub"}}; | |||
7435 | return makeArrayRef(TargetFlags); | |||
7436 | } | |||
7437 | ||||
7438 | namespace { | |||
7439 | /// Create Global Base Reg pass. This initializes the PIC | |||
7440 | /// global base register for x86-32. | |||
7441 | struct CGBR : public MachineFunctionPass { | |||
7442 | static char ID; | |||
7443 | CGBR() : MachineFunctionPass(ID) {} | |||
7444 | ||||
7445 | bool runOnMachineFunction(MachineFunction &MF) override { | |||
7446 | const X86TargetMachine *TM = | |||
7447 | static_cast<const X86TargetMachine *>(&MF.getTarget()); | |||
7448 | const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); | |||
7449 | ||||
7450 | // Don't do anything in the 64-bit small and kernel code models. They use | |||
7451 | // RIP-relative addressing for everything. | |||
7452 | if (STI.is64Bit() && (TM->getCodeModel() == CodeModel::Small || | |||
7453 | TM->getCodeModel() == CodeModel::Kernel)) | |||
7454 | return false; | |||
7455 | ||||
7456 | // Only emit a global base reg in PIC mode. | |||
7457 | if (!TM->isPositionIndependent()) | |||
7458 | return false; | |||
7459 | ||||
7460 | X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); | |||
7461 | unsigned GlobalBaseReg = X86FI->getGlobalBaseReg(); | |||
7462 | ||||
7463 | // If we didn't need a GlobalBaseReg, don't insert code. | |||
7464 | if (GlobalBaseReg == 0) | |||
7465 | return false; | |||
7466 | ||||
7467 | // Insert the set of GlobalBaseReg into the first MBB of the function | |||
7468 | MachineBasicBlock &FirstMBB = MF.front(); | |||
7469 | MachineBasicBlock::iterator MBBI = FirstMBB.begin(); | |||
7470 | DebugLoc DL = FirstMBB.findDebugLoc(MBBI); | |||
7471 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | |||
7472 | const X86InstrInfo *TII = STI.getInstrInfo(); | |||
7473 | ||||
7474 | unsigned PC; | |||
7475 | if (STI.isPICStyleGOT()) | |||
7476 | PC = RegInfo.createVirtualRegister(&X86::GR32RegClass); | |||
7477 | else | |||
7478 | PC = GlobalBaseReg; | |||
7479 | ||||
7480 | if (STI.is64Bit()) { | |||
7481 | if (TM->getCodeModel() == CodeModel::Medium) { | |||
7482 | // In the medium code model, use a RIP-relative LEA to materialize the | |||
7483 | // GOT. | |||
7484 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) | |||
7485 | .addReg(X86::RIP) | |||
7486 | .addImm(0) | |||
7487 | .addReg(0) | |||
7488 | .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") | |||
7489 | .addReg(0); | |||
7490 | } else if (TM->getCodeModel() == CodeModel::Large) { | |||
7491 | // In the large code model, we are aiming for this code, though the | |||
7492 | // register allocation may vary: | |||
7493 | // leaq .LN$pb(%rip), %rax | |||
7494 | // movq $_GLOBAL_OFFSET_TABLE_ - .LN$pb, %rcx | |||
7495 | // addq %rcx, %rax | |||
7496 | // RAX now holds address of _GLOBAL_OFFSET_TABLE_. | |||
7497 | unsigned PBReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); | |||
7498 | unsigned GOTReg = | |||
7499 | RegInfo.createVirtualRegister(&X86::GR64RegClass); | |||
7500 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PBReg) | |||
7501 | .addReg(X86::RIP) | |||
7502 | .addImm(0) | |||
7503 | .addReg(0) | |||
7504 | .addSym(MF.getPICBaseSymbol()) | |||
7505 | .addReg(0); | |||
7506 | std::prev(MBBI)->setPreInstrSymbol(MF, MF.getPICBaseSymbol()); | |||
7507 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOV64ri), GOTReg) | |||
7508 | .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", | |||
7509 | X86II::MO_PIC_BASE_OFFSET); | |||
7510 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD64rr), PC) | |||
7511 | .addReg(PBReg, RegState::Kill) | |||
7512 | .addReg(GOTReg, RegState::Kill); | |||
7513 | } else { | |||
7514 | llvm_unreachable("unexpected code model")::llvm::llvm_unreachable_internal("unexpected code model", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/X86/X86InstrInfo.cpp" , 7514); | |||
7515 | } | |||
7516 | } else { | |||
7517 | // Operand of MovePCtoStack is completely ignored by asm printer. It's | |||
7518 | // only used in JIT code emission as displacement to pc. | |||
7519 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0); | |||
7520 | ||||
7521 | // If we're using vanilla 'GOT' PIC style, we should use relative | |||
7522 | // addressing not to pc, but to _GLOBAL_OFFSET_TABLE_ external. | |||
7523 | if (STI.isPICStyleGOT()) { | |||
7524 | // Generate addl $__GLOBAL_OFFSET_TABLE_ + [.-piclabel], | |||
7525 | // %some_register | |||
7526 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg) | |||
7527 | .addReg(PC) | |||
7528 | .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", | |||
7529 | X86II::MO_GOT_ABSOLUTE_ADDRESS); | |||
7530 | } | |||
7531 | } | |||
7532 | ||||
7533 | return true; | |||
7534 | } | |||
7535 | ||||
7536 | StringRef getPassName() const override { | |||
7537 | return "X86 PIC Global Base Reg Initialization"; | |||
7538 | } | |||
7539 | ||||
7540 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
7541 | AU.setPreservesCFG(); | |||
7542 | MachineFunctionPass::getAnalysisUsage(AU); | |||
7543 | } | |||
7544 | }; | |||
7545 | } | |||
7546 | ||||
7547 | char CGBR::ID = 0; | |||
7548 | FunctionPass* | |||
7549 | llvm::createX86GlobalBaseRegPass() { return new CGBR(); } | |||
7550 | ||||
7551 | namespace { | |||
7552 | struct LDTLSCleanup : public MachineFunctionPass { | |||
7553 | static char ID; | |||
7554 | LDTLSCleanup() : MachineFunctionPass(ID) {} | |||
7555 | ||||
7556 | bool runOnMachineFunction(MachineFunction &MF) override { | |||
7557 | if (skipFunction(MF.getFunction())) | |||
7558 | return false; | |||
7559 | ||||
7560 | X86MachineFunctionInfo *MFI = MF.getInfo<X86MachineFunctionInfo>(); | |||
7561 | if (MFI->getNumLocalDynamicTLSAccesses() < 2) { | |||
7562 | // No point folding accesses if there isn't at least two. | |||
7563 | return false; | |||
7564 | } | |||
7565 | ||||
7566 | MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>(); | |||
7567 | return VisitNode(DT->getRootNode(), 0); | |||
7568 | } | |||
7569 | ||||
7570 | // Visit the dominator subtree rooted at Node in pre-order. | |||
7571 | // If TLSBaseAddrReg is non-null, then use that to replace any | |||
7572 | // TLS_base_addr instructions. Otherwise, create the register | |||
7573 | // when the first such instruction is seen, and then use it | |||
7574 | // as we encounter more instructions. | |||
7575 | bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) { | |||
7576 | MachineBasicBlock *BB = Node->getBlock(); | |||
7577 | bool Changed = false; | |||
7578 | ||||
7579 | // Traverse the current block. | |||
7580 | for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; | |||
7581 | ++I) { | |||
7582 | switch (I->getOpcode()) { | |||
7583 | case X86::TLS_base_addr32: | |||
7584 | case X86::TLS_base_addr64: | |||
7585 | if (TLSBaseAddrReg) | |||
7586 | I = ReplaceTLSBaseAddrCall(*I, TLSBaseAddrReg); | |||
7587 | else | |||
7588 | I = SetRegister(*I, &TLSBaseAddrReg); | |||
7589 | Changed = true; | |||
7590 | break; | |||
7591 | default: | |||
7592 | break; | |||
7593 | } | |||
7594 | } | |||
7595 | ||||
7596 | // Visit the children of this block in the dominator tree. | |||
7597 | for (MachineDomTreeNode::iterator I = Node->begin(), E = Node->end(); | |||
7598 | I != E; ++I) { | |||
7599 | Changed |= VisitNode(*I, TLSBaseAddrReg); | |||
7600 | } | |||
7601 | ||||
7602 | return Changed; | |||
7603 | } | |||
7604 | ||||
7605 | // Replace the TLS_base_addr instruction I with a copy from | |||
7606 | // TLSBaseAddrReg, returning the new instruction. | |||
7607 | MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr &I, | |||
7608 | unsigned TLSBaseAddrReg) { | |||
7609 | MachineFunction *MF = I.getParent()->getParent(); | |||
7610 | const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); | |||
7611 | const bool is64Bit = STI.is64Bit(); | |||
7612 | const X86InstrInfo *TII = STI.getInstrInfo(); | |||
7613 | ||||
7614 | // Insert a Copy from TLSBaseAddrReg to RAX/EAX. | |||
7615 | MachineInstr *Copy = | |||
7616 | BuildMI(*I.getParent(), I, I.getDebugLoc(), | |||
7617 | TII->get(TargetOpcode::COPY), is64Bit ? X86::RAX : X86::EAX) | |||
7618 | .addReg(TLSBaseAddrReg); | |||
7619 | ||||
7620 | // Erase the TLS_base_addr instruction. | |||
7621 | I.eraseFromParent(); | |||
7622 | ||||
7623 | return Copy; | |||
7624 | } | |||
7625 | ||||
7626 | // Create a virtual register in *TLSBaseAddrReg, and populate it by | |||
7627 | // inserting a copy instruction after I. Returns the new instruction. | |||
7628 | MachineInstr *SetRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { | |||
7629 | MachineFunction *MF = I.getParent()->getParent(); | |||
7630 | const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); | |||
7631 | const bool is64Bit = STI.is64Bit(); | |||
7632 | const X86InstrInfo *TII = STI.getInstrInfo(); | |||
7633 | ||||
7634 | // Create a virtual register for the TLS base address. | |||
7635 | MachineRegisterInfo &RegInfo = MF->getRegInfo(); | |||
7636 | *TLSBaseAddrReg = RegInfo.createVirtualRegister(is64Bit | |||
7637 | ? &X86::GR64RegClass | |||
7638 | : &X86::GR32RegClass); | |||
7639 | ||||
7640 | // Insert a copy from RAX/EAX to TLSBaseAddrReg. | |||
7641 | MachineInstr *Next = I.getNextNode(); | |||
7642 | MachineInstr *Copy = | |||
7643 | BuildMI(*I.getParent(), Next, I.getDebugLoc(), | |||
7644 | TII->get(TargetOpcode::COPY), *TLSBaseAddrReg) | |||
7645 | .addReg(is64Bit ? X86::RAX : X86::EAX); | |||
7646 | ||||
7647 | return Copy; | |||
7648 | } | |||
7649 | ||||
7650 | StringRef getPassName() const override { | |||
7651 | return "Local Dynamic TLS Access Clean-up"; | |||
7652 | } | |||
7653 | ||||
7654 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
7655 | AU.setPreservesCFG(); | |||
7656 | AU.addRequired<MachineDominatorTree>(); | |||
7657 | MachineFunctionPass::getAnalysisUsage(AU); | |||
7658 | } | |||
7659 | }; | |||
7660 | } | |||
7661 | ||||
7662 | char LDTLSCleanup::ID = 0; | |||
7663 | FunctionPass* | |||
7664 | llvm::createCleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); } | |||
7665 | ||||
7666 | /// Constants defining how certain sequences should be outlined. | |||
7667 | /// | |||
7668 | /// \p MachineOutlinerDefault implies that the function is called with a call | |||
7669 | /// instruction, and a return must be emitted for the outlined function frame. | |||
7670 | /// | |||
7671 | /// That is, | |||
7672 | /// | |||
7673 | /// I1 OUTLINED_FUNCTION: | |||
7674 | /// I2 --> call OUTLINED_FUNCTION I1 | |||
7675 | /// I3 I2 | |||
7676 | /// I3 | |||
7677 | /// ret | |||
7678 | /// | |||
7679 | /// * Call construction overhead: 1 (call instruction) | |||
7680 | /// * Frame construction overhead: 1 (return instruction) | |||
7681 | /// | |||
7682 | /// \p MachineOutlinerTailCall implies that the function is being tail called. | |||
7683 | /// A jump is emitted instead of a call, and the return is already present in | |||
7684 | /// the outlined sequence. That is, | |||
7685 | /// | |||
7686 | /// I1 OUTLINED_FUNCTION: | |||
7687 | /// I2 --> jmp OUTLINED_FUNCTION I1 | |||
7688 | /// ret I2 | |||
7689 | /// ret | |||
7690 | /// | |||
7691 | /// * Call construction overhead: 1 (jump instruction) | |||
7692 | /// * Frame construction overhead: 0 (don't need to return) | |||
7693 | /// | |||
7694 | enum MachineOutlinerClass { | |||
7695 | MachineOutlinerDefault, | |||
7696 | MachineOutlinerTailCall | |||
7697 | }; | |||
7698 | ||||
7699 | outliner::OutlinedFunction X86InstrInfo::getOutliningCandidateInfo( | |||
7700 | std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { | |||
7701 | unsigned SequenceSize = | |||
7702 | std::accumulate(RepeatedSequenceLocs[0].front(), | |||
7703 | std::next(RepeatedSequenceLocs[0].back()), 0, | |||
7704 | [](unsigned Sum, const MachineInstr &MI) { | |||
7705 | // FIXME: x86 doesn't implement getInstSizeInBytes, so | |||
7706 | // we can't tell the cost. Just assume each instruction | |||
7707 | // is one byte. | |||
7708 | if (MI.isDebugInstr() || MI.isKill()) | |||
7709 | return Sum; | |||
7710 | return Sum + 1; | |||
7711 | }); | |||
7712 | ||||
7713 | // FIXME: Use real size in bytes for call and ret instructions. | |||
7714 | if (RepeatedSequenceLocs[0].back()->isTerminator()) { | |||
7715 | for (outliner::Candidate &C : RepeatedSequenceLocs) | |||
7716 | C.setCallInfo(MachineOutlinerTailCall, 1); | |||
7717 | ||||
7718 | return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, | |||
7719 | 0, // Number of bytes to emit frame. | |||
7720 | MachineOutlinerTailCall // Type of frame. | |||
7721 | ); | |||
7722 | } | |||
7723 | ||||
7724 | for (outliner::Candidate &C : RepeatedSequenceLocs) | |||
7725 | C.setCallInfo(MachineOutlinerDefault, 1); | |||
7726 | ||||
7727 | return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1, | |||
7728 | MachineOutlinerDefault); | |||
7729 | } | |||
7730 | ||||
7731 | bool X86InstrInfo::isFunctionSafeToOutlineFrom(MachineFunction &MF, | |||
7732 | bool OutlineFromLinkOnceODRs) const { | |||
7733 | const Function &F = MF.getFunction(); | |||
7734 | ||||
7735 | // Does the function use a red zone? If it does, then we can't risk messing | |||
7736 | // with the stack. | |||
7737 | if (!F.hasFnAttribute(Attribute::NoRedZone)) { | |||
7738 | // It could have a red zone. If it does, then we don't want to touch it. | |||
7739 | const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); | |||
7740 | if (!X86FI || X86FI->getUsesRedZone()) | |||
7741 | return false; | |||
7742 | } | |||
7743 | ||||
7744 | // If we *don't* want to outline from things that could potentially be deduped | |||
7745 | // then return false. | |||
7746 | if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) | |||
7747 | return false; | |||
7748 | ||||
7749 | // This function is viable for outlining, so return true. | |||
7750 | return true; | |||
7751 | } | |||
7752 | ||||
7753 | outliner::InstrType | |||
7754 | X86InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, unsigned Flags) const { | |||
7755 | MachineInstr &MI = *MIT; | |||
7756 | // Don't allow debug values to impact outlining type. | |||
7757 | if (MI.isDebugInstr() || MI.isIndirectDebugValue()) | |||
7758 | return outliner::InstrType::Invisible; | |||
7759 | ||||
7760 | // At this point, KILL instructions don't really tell us much so we can go | |||
7761 | // ahead and skip over them. | |||
7762 | if (MI.isKill()) | |||
7763 | return outliner::InstrType::Invisible; | |||
7764 | ||||
7765 | // Is this a tail call? If yes, we can outline as a tail call. | |||
7766 | if (isTailCall(MI)) | |||
7767 | return outliner::InstrType::Legal; | |||
7768 | ||||
7769 | // Is this the terminator of a basic block? | |||
7770 | if (MI.isTerminator() || MI.isReturn()) { | |||
7771 | ||||
7772 | // Does its parent have any successors in its MachineFunction? | |||
7773 | if (MI.getParent()->succ_empty()) | |||
7774 | return outliner::InstrType::Legal; | |||
7775 | ||||
7776 | // It does, so we can't tail call it. | |||
7777 | return outliner::InstrType::Illegal; | |||
7778 | } | |||
7779 | ||||
7780 | // Don't outline anything that modifies or reads from the stack pointer. | |||
7781 | // | |||
7782 | // FIXME: There are instructions which are being manually built without | |||
7783 | // explicit uses/defs so we also have to check the MCInstrDesc. We should be | |||
7784 | // able to remove the extra checks once those are fixed up. For example, | |||
7785 | // sometimes we might get something like %rax = POP64r 1. This won't be | |||
7786 | // caught by modifiesRegister or readsRegister even though the instruction | |||
7787 | // really ought to be formed so that modifiesRegister/readsRegister would | |||
7788 | // catch it. | |||
7789 | if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) || | |||
7790 | MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) || | |||
7791 | MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP)) | |||
7792 | return outliner::InstrType::Illegal; | |||
7793 | ||||
7794 | // Outlined calls change the instruction pointer, so don't read from it. | |||
7795 | if (MI.readsRegister(X86::RIP, &RI) || | |||
7796 | MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) || | |||
7797 | MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP)) | |||
7798 | return outliner::InstrType::Illegal; | |||
7799 | ||||
7800 | // Positions can't safely be outlined. | |||
7801 | if (MI.isPosition()) | |||
7802 | return outliner::InstrType::Illegal; | |||
7803 | ||||
7804 | // Make sure none of the operands of this instruction do anything tricky. | |||
7805 | for (const MachineOperand &MOP : MI.operands()) | |||
7806 | if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() || | |||
7807 | MOP.isTargetIndex()) | |||
7808 | return outliner::InstrType::Illegal; | |||
7809 | ||||
7810 | return outliner::InstrType::Legal; | |||
7811 | } | |||
7812 | ||||
7813 | void X86InstrInfo::buildOutlinedFrame(MachineBasicBlock &MBB, | |||
7814 | MachineFunction &MF, | |||
7815 | const outliner::OutlinedFunction &OF) | |||
7816 | const { | |||
7817 | // If we're a tail call, we already have a return, so don't do anything. | |||
7818 | if (OF.FrameConstructionID == MachineOutlinerTailCall) | |||
7819 | return; | |||
7820 | ||||
7821 | // We're a normal call, so our sequence doesn't have a return instruction. | |||
7822 | // Add it in. | |||
7823 | MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RETQ)); | |||
7824 | MBB.insert(MBB.end(), retq); | |||
7825 | } | |||
7826 | ||||
7827 | MachineBasicBlock::iterator | |||
7828 | X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB, | |||
7829 | MachineBasicBlock::iterator &It, | |||
7830 | MachineFunction &MF, | |||
7831 | const outliner::Candidate &C) const { | |||
7832 | // Is it a tail call? | |||
7833 | if (C.CallConstructionID == MachineOutlinerTailCall) { | |||
7834 | // Yes, just insert a JMP. | |||
7835 | It = MBB.insert(It, | |||
7836 | BuildMI(MF, DebugLoc(), get(X86::TAILJMPd64)) | |||
7837 | .addGlobalAddress(M.getNamedValue(MF.getName()))); | |||
7838 | } else { | |||
7839 | // No, insert a call. | |||
7840 | It = MBB.insert(It, | |||
7841 | BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32)) | |||
7842 | .addGlobalAddress(M.getNamedValue(MF.getName()))); | |||
7843 | } | |||
7844 | ||||
7845 | return It; | |||
7846 | } |