File: | lib/Target/X86/X86InstrInfo.cpp |
Warning: | line 3730, column 37 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===// | |||
2 | // | |||
3 | // The LLVM Compiler Infrastructure | |||
4 | // | |||
5 | // This file is distributed under the University of Illinois Open Source | |||
6 | // License. See LICENSE.TXT for details. | |||
7 | // | |||
8 | //===----------------------------------------------------------------------===// | |||
9 | // | |||
10 | // This file contains the X86 implementation of the TargetInstrInfo class. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "X86InstrInfo.h" | |||
15 | #include "X86.h" | |||
16 | #include "X86InstrBuilder.h" | |||
17 | #include "X86InstrFoldTables.h" | |||
18 | #include "X86MachineFunctionInfo.h" | |||
19 | #include "X86Subtarget.h" | |||
20 | #include "X86TargetMachine.h" | |||
21 | #include "llvm/ADT/STLExtras.h" | |||
22 | #include "llvm/ADT/Sequence.h" | |||
23 | #include "llvm/CodeGen/LivePhysRegs.h" | |||
24 | #include "llvm/CodeGen/LiveVariables.h" | |||
25 | #include "llvm/CodeGen/MachineConstantPool.h" | |||
26 | #include "llvm/CodeGen/MachineDominators.h" | |||
27 | #include "llvm/CodeGen/MachineFrameInfo.h" | |||
28 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
29 | #include "llvm/CodeGen/MachineModuleInfo.h" | |||
30 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
31 | #include "llvm/CodeGen/StackMaps.h" | |||
32 | #include "llvm/IR/DerivedTypes.h" | |||
33 | #include "llvm/IR/Function.h" | |||
34 | #include "llvm/IR/LLVMContext.h" | |||
35 | #include "llvm/MC/MCAsmInfo.h" | |||
36 | #include "llvm/MC/MCExpr.h" | |||
37 | #include "llvm/MC/MCInst.h" | |||
38 | #include "llvm/Support/CommandLine.h" | |||
39 | #include "llvm/Support/Debug.h" | |||
40 | #include "llvm/Support/ErrorHandling.h" | |||
41 | #include "llvm/Support/raw_ostream.h" | |||
42 | #include "llvm/Target/TargetOptions.h" | |||
43 | ||||
44 | using namespace llvm; | |||
45 | ||||
46 | #define DEBUG_TYPE"x86-instr-info" "x86-instr-info" | |||
47 | ||||
48 | #define GET_INSTRINFO_CTOR_DTOR | |||
49 | #include "X86GenInstrInfo.inc" | |||
50 | ||||
51 | static cl::opt<bool> | |||
52 | NoFusing("disable-spill-fusing", | |||
53 | cl::desc("Disable fusing of spill code into instructions"), | |||
54 | cl::Hidden); | |||
55 | static cl::opt<bool> | |||
56 | PrintFailedFusing("print-failed-fuse-candidates", | |||
57 | cl::desc("Print instructions that the allocator wants to" | |||
58 | " fuse, but the X86 backend currently can't"), | |||
59 | cl::Hidden); | |||
60 | static cl::opt<bool> | |||
61 | ReMatPICStubLoad("remat-pic-stub-load", | |||
62 | cl::desc("Re-materialize load from stub in PIC mode"), | |||
63 | cl::init(false), cl::Hidden); | |||
64 | static cl::opt<unsigned> | |||
65 | PartialRegUpdateClearance("partial-reg-update-clearance", | |||
66 | cl::desc("Clearance between two register writes " | |||
67 | "for inserting XOR to avoid partial " | |||
68 | "register update"), | |||
69 | cl::init(64), cl::Hidden); | |||
70 | static cl::opt<unsigned> | |||
71 | UndefRegClearance("undef-reg-clearance", | |||
72 | cl::desc("How many idle instructions we would like before " | |||
73 | "certain undef register reads"), | |||
74 | cl::init(128), cl::Hidden); | |||
75 | ||||
76 | ||||
77 | // Pin the vtable to this file. | |||
78 | void X86InstrInfo::anchor() {} | |||
79 | ||||
80 | X86InstrInfo::X86InstrInfo(X86Subtarget &STI) | |||
81 | : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64 | |||
82 | : X86::ADJCALLSTACKDOWN32), | |||
83 | (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64 | |||
84 | : X86::ADJCALLSTACKUP32), | |||
85 | X86::CATCHRET, | |||
86 | (STI.is64Bit() ? X86::RETQ : X86::RETL)), | |||
87 | Subtarget(STI), RI(STI.getTargetTriple()) { | |||
88 | } | |||
89 | ||||
90 | bool | |||
91 | X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, | |||
92 | unsigned &SrcReg, unsigned &DstReg, | |||
93 | unsigned &SubIdx) const { | |||
94 | switch (MI.getOpcode()) { | |||
95 | default: break; | |||
96 | case X86::MOVSX16rr8: | |||
97 | case X86::MOVZX16rr8: | |||
98 | case X86::MOVSX32rr8: | |||
99 | case X86::MOVZX32rr8: | |||
100 | case X86::MOVSX64rr8: | |||
101 | if (!Subtarget.is64Bit()) | |||
102 | // It's not always legal to reference the low 8-bit of the larger | |||
103 | // register in 32-bit mode. | |||
104 | return false; | |||
105 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
106 | case X86::MOVSX32rr16: | |||
107 | case X86::MOVZX32rr16: | |||
108 | case X86::MOVSX64rr16: | |||
109 | case X86::MOVSX64rr32: { | |||
110 | if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) | |||
111 | // Be conservative. | |||
112 | return false; | |||
113 | SrcReg = MI.getOperand(1).getReg(); | |||
114 | DstReg = MI.getOperand(0).getReg(); | |||
115 | switch (MI.getOpcode()) { | |||
116 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 116); | |||
117 | case X86::MOVSX16rr8: | |||
118 | case X86::MOVZX16rr8: | |||
119 | case X86::MOVSX32rr8: | |||
120 | case X86::MOVZX32rr8: | |||
121 | case X86::MOVSX64rr8: | |||
122 | SubIdx = X86::sub_8bit; | |||
123 | break; | |||
124 | case X86::MOVSX32rr16: | |||
125 | case X86::MOVZX32rr16: | |||
126 | case X86::MOVSX64rr16: | |||
127 | SubIdx = X86::sub_16bit; | |||
128 | break; | |||
129 | case X86::MOVSX64rr32: | |||
130 | SubIdx = X86::sub_32bit; | |||
131 | break; | |||
132 | } | |||
133 | return true; | |||
134 | } | |||
135 | } | |||
136 | return false; | |||
137 | } | |||
138 | ||||
139 | int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const { | |||
140 | const MachineFunction *MF = MI.getParent()->getParent(); | |||
141 | const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); | |||
142 | ||||
143 | if (isFrameInstr(MI)) { | |||
144 | unsigned StackAlign = TFI->getStackAlignment(); | |||
145 | int SPAdj = alignTo(getFrameSize(MI), StackAlign); | |||
146 | SPAdj -= getFrameAdjustment(MI); | |||
147 | if (!isFrameSetup(MI)) | |||
148 | SPAdj = -SPAdj; | |||
149 | return SPAdj; | |||
150 | } | |||
151 | ||||
152 | // To know whether a call adjusts the stack, we need information | |||
153 | // that is bound to the following ADJCALLSTACKUP pseudo. | |||
154 | // Look for the next ADJCALLSTACKUP that follows the call. | |||
155 | if (MI.isCall()) { | |||
156 | const MachineBasicBlock *MBB = MI.getParent(); | |||
157 | auto I = ++MachineBasicBlock::const_iterator(MI); | |||
158 | for (auto E = MBB->end(); I != E; ++I) { | |||
159 | if (I->getOpcode() == getCallFrameDestroyOpcode() || | |||
160 | I->isCall()) | |||
161 | break; | |||
162 | } | |||
163 | ||||
164 | // If we could not find a frame destroy opcode, then it has already | |||
165 | // been simplified, so we don't care. | |||
166 | if (I->getOpcode() != getCallFrameDestroyOpcode()) | |||
167 | return 0; | |||
168 | ||||
169 | return -(I->getOperand(1).getImm()); | |||
170 | } | |||
171 | ||||
172 | // Currently handle only PUSHes we can reasonably expect to see | |||
173 | // in call sequences | |||
174 | switch (MI.getOpcode()) { | |||
175 | default: | |||
176 | return 0; | |||
177 | case X86::PUSH32i8: | |||
178 | case X86::PUSH32r: | |||
179 | case X86::PUSH32rmm: | |||
180 | case X86::PUSH32rmr: | |||
181 | case X86::PUSHi32: | |||
182 | return 4; | |||
183 | case X86::PUSH64i8: | |||
184 | case X86::PUSH64r: | |||
185 | case X86::PUSH64rmm: | |||
186 | case X86::PUSH64rmr: | |||
187 | case X86::PUSH64i32: | |||
188 | return 8; | |||
189 | } | |||
190 | } | |||
191 | ||||
192 | /// Return true and the FrameIndex if the specified | |||
193 | /// operand and follow operands form a reference to the stack frame. | |||
194 | bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op, | |||
195 | int &FrameIndex) const { | |||
196 | if (MI.getOperand(Op + X86::AddrBaseReg).isFI() && | |||
197 | MI.getOperand(Op + X86::AddrScaleAmt).isImm() && | |||
198 | MI.getOperand(Op + X86::AddrIndexReg).isReg() && | |||
199 | MI.getOperand(Op + X86::AddrDisp).isImm() && | |||
200 | MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 && | |||
201 | MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 && | |||
202 | MI.getOperand(Op + X86::AddrDisp).getImm() == 0) { | |||
203 | FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex(); | |||
204 | return true; | |||
205 | } | |||
206 | return false; | |||
207 | } | |||
208 | ||||
209 | static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) { | |||
210 | switch (Opcode) { | |||
211 | default: | |||
212 | return false; | |||
213 | case X86::MOV8rm: | |||
214 | case X86::KMOVBkm: | |||
215 | MemBytes = 1; | |||
216 | return true; | |||
217 | case X86::MOV16rm: | |||
218 | case X86::KMOVWkm: | |||
219 | MemBytes = 2; | |||
220 | return true; | |||
221 | case X86::MOV32rm: | |||
222 | case X86::MOVSSrm: | |||
223 | case X86::VMOVSSZrm: | |||
224 | case X86::VMOVSSrm: | |||
225 | case X86::KMOVDkm: | |||
226 | MemBytes = 4; | |||
227 | return true; | |||
228 | case X86::MOV64rm: | |||
229 | case X86::LD_Fp64m: | |||
230 | case X86::MOVSDrm: | |||
231 | case X86::VMOVSDrm: | |||
232 | case X86::VMOVSDZrm: | |||
233 | case X86::MMX_MOVD64rm: | |||
234 | case X86::MMX_MOVQ64rm: | |||
235 | case X86::KMOVQkm: | |||
236 | MemBytes = 8; | |||
237 | return true; | |||
238 | case X86::MOVAPSrm: | |||
239 | case X86::MOVUPSrm: | |||
240 | case X86::MOVAPDrm: | |||
241 | case X86::MOVUPDrm: | |||
242 | case X86::MOVDQArm: | |||
243 | case X86::MOVDQUrm: | |||
244 | case X86::VMOVAPSrm: | |||
245 | case X86::VMOVUPSrm: | |||
246 | case X86::VMOVAPDrm: | |||
247 | case X86::VMOVUPDrm: | |||
248 | case X86::VMOVDQArm: | |||
249 | case X86::VMOVDQUrm: | |||
250 | case X86::VMOVAPSZ128rm: | |||
251 | case X86::VMOVUPSZ128rm: | |||
252 | case X86::VMOVAPSZ128rm_NOVLX: | |||
253 | case X86::VMOVUPSZ128rm_NOVLX: | |||
254 | case X86::VMOVAPDZ128rm: | |||
255 | case X86::VMOVUPDZ128rm: | |||
256 | case X86::VMOVDQU8Z128rm: | |||
257 | case X86::VMOVDQU16Z128rm: | |||
258 | case X86::VMOVDQA32Z128rm: | |||
259 | case X86::VMOVDQU32Z128rm: | |||
260 | case X86::VMOVDQA64Z128rm: | |||
261 | case X86::VMOVDQU64Z128rm: | |||
262 | MemBytes = 16; | |||
263 | return true; | |||
264 | case X86::VMOVAPSYrm: | |||
265 | case X86::VMOVUPSYrm: | |||
266 | case X86::VMOVAPDYrm: | |||
267 | case X86::VMOVUPDYrm: | |||
268 | case X86::VMOVDQAYrm: | |||
269 | case X86::VMOVDQUYrm: | |||
270 | case X86::VMOVAPSZ256rm: | |||
271 | case X86::VMOVUPSZ256rm: | |||
272 | case X86::VMOVAPSZ256rm_NOVLX: | |||
273 | case X86::VMOVUPSZ256rm_NOVLX: | |||
274 | case X86::VMOVAPDZ256rm: | |||
275 | case X86::VMOVUPDZ256rm: | |||
276 | case X86::VMOVDQU8Z256rm: | |||
277 | case X86::VMOVDQU16Z256rm: | |||
278 | case X86::VMOVDQA32Z256rm: | |||
279 | case X86::VMOVDQU32Z256rm: | |||
280 | case X86::VMOVDQA64Z256rm: | |||
281 | case X86::VMOVDQU64Z256rm: | |||
282 | MemBytes = 32; | |||
283 | return true; | |||
284 | case X86::VMOVAPSZrm: | |||
285 | case X86::VMOVUPSZrm: | |||
286 | case X86::VMOVAPDZrm: | |||
287 | case X86::VMOVUPDZrm: | |||
288 | case X86::VMOVDQU8Zrm: | |||
289 | case X86::VMOVDQU16Zrm: | |||
290 | case X86::VMOVDQA32Zrm: | |||
291 | case X86::VMOVDQU32Zrm: | |||
292 | case X86::VMOVDQA64Zrm: | |||
293 | case X86::VMOVDQU64Zrm: | |||
294 | MemBytes = 64; | |||
295 | return true; | |||
296 | } | |||
297 | } | |||
298 | ||||
299 | static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) { | |||
300 | switch (Opcode) { | |||
301 | default: | |||
302 | return false; | |||
303 | case X86::MOV8mr: | |||
304 | case X86::KMOVBmk: | |||
305 | MemBytes = 1; | |||
306 | return true; | |||
307 | case X86::MOV16mr: | |||
308 | case X86::KMOVWmk: | |||
309 | MemBytes = 2; | |||
310 | return true; | |||
311 | case X86::MOV32mr: | |||
312 | case X86::MOVSSmr: | |||
313 | case X86::VMOVSSmr: | |||
314 | case X86::VMOVSSZmr: | |||
315 | case X86::KMOVDmk: | |||
316 | MemBytes = 4; | |||
317 | return true; | |||
318 | case X86::MOV64mr: | |||
319 | case X86::ST_FpP64m: | |||
320 | case X86::MOVSDmr: | |||
321 | case X86::VMOVSDmr: | |||
322 | case X86::VMOVSDZmr: | |||
323 | case X86::MMX_MOVD64mr: | |||
324 | case X86::MMX_MOVQ64mr: | |||
325 | case X86::MMX_MOVNTQmr: | |||
326 | case X86::KMOVQmk: | |||
327 | MemBytes = 8; | |||
328 | return true; | |||
329 | case X86::MOVAPSmr: | |||
330 | case X86::MOVUPSmr: | |||
331 | case X86::MOVAPDmr: | |||
332 | case X86::MOVUPDmr: | |||
333 | case X86::MOVDQAmr: | |||
334 | case X86::MOVDQUmr: | |||
335 | case X86::VMOVAPSmr: | |||
336 | case X86::VMOVUPSmr: | |||
337 | case X86::VMOVAPDmr: | |||
338 | case X86::VMOVUPDmr: | |||
339 | case X86::VMOVDQAmr: | |||
340 | case X86::VMOVDQUmr: | |||
341 | case X86::VMOVUPSZ128mr: | |||
342 | case X86::VMOVAPSZ128mr: | |||
343 | case X86::VMOVUPSZ128mr_NOVLX: | |||
344 | case X86::VMOVAPSZ128mr_NOVLX: | |||
345 | case X86::VMOVUPDZ128mr: | |||
346 | case X86::VMOVAPDZ128mr: | |||
347 | case X86::VMOVDQA32Z128mr: | |||
348 | case X86::VMOVDQU32Z128mr: | |||
349 | case X86::VMOVDQA64Z128mr: | |||
350 | case X86::VMOVDQU64Z128mr: | |||
351 | case X86::VMOVDQU8Z128mr: | |||
352 | case X86::VMOVDQU16Z128mr: | |||
353 | MemBytes = 16; | |||
354 | return true; | |||
355 | case X86::VMOVUPSYmr: | |||
356 | case X86::VMOVAPSYmr: | |||
357 | case X86::VMOVUPDYmr: | |||
358 | case X86::VMOVAPDYmr: | |||
359 | case X86::VMOVDQUYmr: | |||
360 | case X86::VMOVDQAYmr: | |||
361 | case X86::VMOVUPSZ256mr: | |||
362 | case X86::VMOVAPSZ256mr: | |||
363 | case X86::VMOVUPSZ256mr_NOVLX: | |||
364 | case X86::VMOVAPSZ256mr_NOVLX: | |||
365 | case X86::VMOVUPDZ256mr: | |||
366 | case X86::VMOVAPDZ256mr: | |||
367 | case X86::VMOVDQU8Z256mr: | |||
368 | case X86::VMOVDQU16Z256mr: | |||
369 | case X86::VMOVDQA32Z256mr: | |||
370 | case X86::VMOVDQU32Z256mr: | |||
371 | case X86::VMOVDQA64Z256mr: | |||
372 | case X86::VMOVDQU64Z256mr: | |||
373 | MemBytes = 32; | |||
374 | return true; | |||
375 | case X86::VMOVUPSZmr: | |||
376 | case X86::VMOVAPSZmr: | |||
377 | case X86::VMOVUPDZmr: | |||
378 | case X86::VMOVAPDZmr: | |||
379 | case X86::VMOVDQU8Zmr: | |||
380 | case X86::VMOVDQU16Zmr: | |||
381 | case X86::VMOVDQA32Zmr: | |||
382 | case X86::VMOVDQU32Zmr: | |||
383 | case X86::VMOVDQA64Zmr: | |||
384 | case X86::VMOVDQU64Zmr: | |||
385 | MemBytes = 64; | |||
386 | return true; | |||
387 | } | |||
388 | return false; | |||
389 | } | |||
390 | ||||
391 | unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, | |||
392 | int &FrameIndex) const { | |||
393 | unsigned Dummy; | |||
394 | return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy); | |||
395 | } | |||
396 | ||||
397 | unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, | |||
398 | int &FrameIndex, | |||
399 | unsigned &MemBytes) const { | |||
400 | if (isFrameLoadOpcode(MI.getOpcode(), MemBytes)) | |||
401 | if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex)) | |||
402 | return MI.getOperand(0).getReg(); | |||
403 | return 0; | |||
404 | } | |||
405 | ||||
406 | unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI, | |||
407 | int &FrameIndex) const { | |||
408 | unsigned Dummy; | |||
409 | if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) { | |||
410 | unsigned Reg; | |||
411 | if ((Reg = isLoadFromStackSlot(MI, FrameIndex))) | |||
412 | return Reg; | |||
413 | // Check for post-frame index elimination operations | |||
414 | SmallVector<const MachineMemOperand *, 1> Accesses; | |||
415 | if (hasLoadFromStackSlot(MI, Accesses)) { | |||
416 | FrameIndex = | |||
417 | cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) | |||
418 | ->getFrameIndex(); | |||
419 | return 1; | |||
420 | } | |||
421 | } | |||
422 | return 0; | |||
423 | } | |||
424 | ||||
425 | unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, | |||
426 | int &FrameIndex) const { | |||
427 | unsigned Dummy; | |||
428 | return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy); | |||
429 | } | |||
430 | ||||
431 | unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, | |||
432 | int &FrameIndex, | |||
433 | unsigned &MemBytes) const { | |||
434 | if (isFrameStoreOpcode(MI.getOpcode(), MemBytes)) | |||
435 | if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 && | |||
436 | isFrameOperand(MI, 0, FrameIndex)) | |||
437 | return MI.getOperand(X86::AddrNumOperands).getReg(); | |||
438 | return 0; | |||
439 | } | |||
440 | ||||
441 | unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI, | |||
442 | int &FrameIndex) const { | |||
443 | unsigned Dummy; | |||
444 | if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) { | |||
445 | unsigned Reg; | |||
446 | if ((Reg = isStoreToStackSlot(MI, FrameIndex))) | |||
447 | return Reg; | |||
448 | // Check for post-frame index elimination operations | |||
449 | SmallVector<const MachineMemOperand *, 1> Accesses; | |||
450 | if (hasStoreToStackSlot(MI, Accesses)) { | |||
451 | FrameIndex = | |||
452 | cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) | |||
453 | ->getFrameIndex(); | |||
454 | return 1; | |||
455 | } | |||
456 | } | |||
457 | return 0; | |||
458 | } | |||
459 | ||||
460 | /// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r. | |||
461 | static bool regIsPICBase(unsigned BaseReg, const MachineRegisterInfo &MRI) { | |||
462 | // Don't waste compile time scanning use-def chains of physregs. | |||
463 | if (!TargetRegisterInfo::isVirtualRegister(BaseReg)) | |||
464 | return false; | |||
465 | bool isPICBase = false; | |||
466 | for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg), | |||
467 | E = MRI.def_instr_end(); I != E; ++I) { | |||
468 | MachineInstr *DefMI = &*I; | |||
469 | if (DefMI->getOpcode() != X86::MOVPC32r) | |||
470 | return false; | |||
471 | assert(!isPICBase && "More than one PIC base?")((!isPICBase && "More than one PIC base?") ? static_cast <void> (0) : __assert_fail ("!isPICBase && \"More than one PIC base?\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 471, __PRETTY_FUNCTION__)); | |||
472 | isPICBase = true; | |||
473 | } | |||
474 | return isPICBase; | |||
475 | } | |||
476 | ||||
477 | bool X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, | |||
478 | AliasAnalysis *AA) const { | |||
479 | switch (MI.getOpcode()) { | |||
480 | default: break; | |||
481 | case X86::MOV8rm: | |||
482 | case X86::MOV8rm_NOREX: | |||
483 | case X86::MOV16rm: | |||
484 | case X86::MOV32rm: | |||
485 | case X86::MOV64rm: | |||
486 | case X86::LD_Fp64m: | |||
487 | case X86::MOVSSrm: | |||
488 | case X86::MOVSDrm: | |||
489 | case X86::MOVAPSrm: | |||
490 | case X86::MOVUPSrm: | |||
491 | case X86::MOVAPDrm: | |||
492 | case X86::MOVUPDrm: | |||
493 | case X86::MOVDQArm: | |||
494 | case X86::MOVDQUrm: | |||
495 | case X86::VMOVSSrm: | |||
496 | case X86::VMOVSDrm: | |||
497 | case X86::VMOVAPSrm: | |||
498 | case X86::VMOVUPSrm: | |||
499 | case X86::VMOVAPDrm: | |||
500 | case X86::VMOVUPDrm: | |||
501 | case X86::VMOVDQArm: | |||
502 | case X86::VMOVDQUrm: | |||
503 | case X86::VMOVAPSYrm: | |||
504 | case X86::VMOVUPSYrm: | |||
505 | case X86::VMOVAPDYrm: | |||
506 | case X86::VMOVUPDYrm: | |||
507 | case X86::VMOVDQAYrm: | |||
508 | case X86::VMOVDQUYrm: | |||
509 | case X86::MMX_MOVD64rm: | |||
510 | case X86::MMX_MOVQ64rm: | |||
511 | // AVX-512 | |||
512 | case X86::VMOVSSZrm: | |||
513 | case X86::VMOVSDZrm: | |||
514 | case X86::VMOVAPDZ128rm: | |||
515 | case X86::VMOVAPDZ256rm: | |||
516 | case X86::VMOVAPDZrm: | |||
517 | case X86::VMOVAPSZ128rm: | |||
518 | case X86::VMOVAPSZ256rm: | |||
519 | case X86::VMOVAPSZ128rm_NOVLX: | |||
520 | case X86::VMOVAPSZ256rm_NOVLX: | |||
521 | case X86::VMOVAPSZrm: | |||
522 | case X86::VMOVDQA32Z128rm: | |||
523 | case X86::VMOVDQA32Z256rm: | |||
524 | case X86::VMOVDQA32Zrm: | |||
525 | case X86::VMOVDQA64Z128rm: | |||
526 | case X86::VMOVDQA64Z256rm: | |||
527 | case X86::VMOVDQA64Zrm: | |||
528 | case X86::VMOVDQU16Z128rm: | |||
529 | case X86::VMOVDQU16Z256rm: | |||
530 | case X86::VMOVDQU16Zrm: | |||
531 | case X86::VMOVDQU32Z128rm: | |||
532 | case X86::VMOVDQU32Z256rm: | |||
533 | case X86::VMOVDQU32Zrm: | |||
534 | case X86::VMOVDQU64Z128rm: | |||
535 | case X86::VMOVDQU64Z256rm: | |||
536 | case X86::VMOVDQU64Zrm: | |||
537 | case X86::VMOVDQU8Z128rm: | |||
538 | case X86::VMOVDQU8Z256rm: | |||
539 | case X86::VMOVDQU8Zrm: | |||
540 | case X86::VMOVUPDZ128rm: | |||
541 | case X86::VMOVUPDZ256rm: | |||
542 | case X86::VMOVUPDZrm: | |||
543 | case X86::VMOVUPSZ128rm: | |||
544 | case X86::VMOVUPSZ256rm: | |||
545 | case X86::VMOVUPSZ128rm_NOVLX: | |||
546 | case X86::VMOVUPSZ256rm_NOVLX: | |||
547 | case X86::VMOVUPSZrm: { | |||
548 | // Loads from constant pools are trivially rematerializable. | |||
549 | if (MI.getOperand(1 + X86::AddrBaseReg).isReg() && | |||
550 | MI.getOperand(1 + X86::AddrScaleAmt).isImm() && | |||
551 | MI.getOperand(1 + X86::AddrIndexReg).isReg() && | |||
552 | MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && | |||
553 | MI.isDereferenceableInvariantLoad(AA)) { | |||
554 | unsigned BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); | |||
555 | if (BaseReg == 0 || BaseReg == X86::RIP) | |||
556 | return true; | |||
557 | // Allow re-materialization of PIC load. | |||
558 | if (!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal()) | |||
559 | return false; | |||
560 | const MachineFunction &MF = *MI.getParent()->getParent(); | |||
561 | const MachineRegisterInfo &MRI = MF.getRegInfo(); | |||
562 | return regIsPICBase(BaseReg, MRI); | |||
563 | } | |||
564 | return false; | |||
565 | } | |||
566 | ||||
567 | case X86::LEA32r: | |||
568 | case X86::LEA64r: { | |||
569 | if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() && | |||
570 | MI.getOperand(1 + X86::AddrIndexReg).isReg() && | |||
571 | MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && | |||
572 | !MI.getOperand(1 + X86::AddrDisp).isReg()) { | |||
573 | // lea fi#, lea GV, etc. are all rematerializable. | |||
574 | if (!MI.getOperand(1 + X86::AddrBaseReg).isReg()) | |||
575 | return true; | |||
576 | unsigned BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); | |||
577 | if (BaseReg == 0) | |||
578 | return true; | |||
579 | // Allow re-materialization of lea PICBase + x. | |||
580 | const MachineFunction &MF = *MI.getParent()->getParent(); | |||
581 | const MachineRegisterInfo &MRI = MF.getRegInfo(); | |||
582 | return regIsPICBase(BaseReg, MRI); | |||
583 | } | |||
584 | return false; | |||
585 | } | |||
586 | } | |||
587 | ||||
588 | // All other instructions marked M_REMATERIALIZABLE are always trivially | |||
589 | // rematerializable. | |||
590 | return true; | |||
591 | } | |||
592 | ||||
593 | bool X86InstrInfo::isSafeToClobberEFLAGS(MachineBasicBlock &MBB, | |||
594 | MachineBasicBlock::iterator I) const { | |||
595 | MachineBasicBlock::iterator E = MBB.end(); | |||
596 | ||||
597 | // For compile time consideration, if we are not able to determine the | |||
598 | // safety after visiting 4 instructions in each direction, we will assume | |||
599 | // it's not safe. | |||
600 | MachineBasicBlock::iterator Iter = I; | |||
601 | for (unsigned i = 0; Iter != E && i < 4; ++i) { | |||
602 | bool SeenDef = false; | |||
603 | for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) { | |||
604 | MachineOperand &MO = Iter->getOperand(j); | |||
605 | if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS)) | |||
606 | SeenDef = true; | |||
607 | if (!MO.isReg()) | |||
608 | continue; | |||
609 | if (MO.getReg() == X86::EFLAGS) { | |||
610 | if (MO.isUse()) | |||
611 | return false; | |||
612 | SeenDef = true; | |||
613 | } | |||
614 | } | |||
615 | ||||
616 | if (SeenDef) | |||
617 | // This instruction defines EFLAGS, no need to look any further. | |||
618 | return true; | |||
619 | ++Iter; | |||
620 | // Skip over debug instructions. | |||
621 | while (Iter != E && Iter->isDebugInstr()) | |||
622 | ++Iter; | |||
623 | } | |||
624 | ||||
625 | // It is safe to clobber EFLAGS at the end of a block of no successor has it | |||
626 | // live in. | |||
627 | if (Iter == E) { | |||
628 | for (MachineBasicBlock *S : MBB.successors()) | |||
629 | if (S->isLiveIn(X86::EFLAGS)) | |||
630 | return false; | |||
631 | return true; | |||
632 | } | |||
633 | ||||
634 | MachineBasicBlock::iterator B = MBB.begin(); | |||
635 | Iter = I; | |||
636 | for (unsigned i = 0; i < 4; ++i) { | |||
637 | // If we make it to the beginning of the block, it's safe to clobber | |||
638 | // EFLAGS iff EFLAGS is not live-in. | |||
639 | if (Iter == B) | |||
640 | return !MBB.isLiveIn(X86::EFLAGS); | |||
641 | ||||
642 | --Iter; | |||
643 | // Skip over debug instructions. | |||
644 | while (Iter != B && Iter->isDebugInstr()) | |||
645 | --Iter; | |||
646 | ||||
647 | bool SawKill = false; | |||
648 | for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) { | |||
649 | MachineOperand &MO = Iter->getOperand(j); | |||
650 | // A register mask may clobber EFLAGS, but we should still look for a | |||
651 | // live EFLAGS def. | |||
652 | if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS)) | |||
653 | SawKill = true; | |||
654 | if (MO.isReg() && MO.getReg() == X86::EFLAGS) { | |||
655 | if (MO.isDef()) return MO.isDead(); | |||
656 | if (MO.isKill()) SawKill = true; | |||
657 | } | |||
658 | } | |||
659 | ||||
660 | if (SawKill) | |||
661 | // This instruction kills EFLAGS and doesn't redefine it, so | |||
662 | // there's no need to look further. | |||
663 | return true; | |||
664 | } | |||
665 | ||||
666 | // Conservative answer. | |||
667 | return false; | |||
668 | } | |||
669 | ||||
670 | void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB, | |||
671 | MachineBasicBlock::iterator I, | |||
672 | unsigned DestReg, unsigned SubIdx, | |||
673 | const MachineInstr &Orig, | |||
674 | const TargetRegisterInfo &TRI) const { | |||
675 | bool ClobbersEFLAGS = false; | |||
676 | for (const MachineOperand &MO : Orig.operands()) { | |||
677 | if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) { | |||
678 | ClobbersEFLAGS = true; | |||
679 | break; | |||
680 | } | |||
681 | } | |||
682 | ||||
683 | if (ClobbersEFLAGS && !isSafeToClobberEFLAGS(MBB, I)) { | |||
684 | // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side | |||
685 | // effects. | |||
686 | int Value; | |||
687 | switch (Orig.getOpcode()) { | |||
688 | case X86::MOV32r0: Value = 0; break; | |||
689 | case X86::MOV32r1: Value = 1; break; | |||
690 | case X86::MOV32r_1: Value = -1; break; | |||
691 | default: | |||
692 | llvm_unreachable("Unexpected instruction!")::llvm::llvm_unreachable_internal("Unexpected instruction!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 692); | |||
693 | } | |||
694 | ||||
695 | const DebugLoc &DL = Orig.getDebugLoc(); | |||
696 | BuildMI(MBB, I, DL, get(X86::MOV32ri)) | |||
697 | .add(Orig.getOperand(0)) | |||
698 | .addImm(Value); | |||
699 | } else { | |||
700 | MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); | |||
701 | MBB.insert(I, MI); | |||
702 | } | |||
703 | ||||
704 | MachineInstr &NewMI = *std::prev(I); | |||
705 | NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI); | |||
706 | } | |||
707 | ||||
708 | /// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead. | |||
709 | bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const { | |||
710 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
711 | MachineOperand &MO = MI.getOperand(i); | |||
712 | if (MO.isReg() && MO.isDef() && | |||
713 | MO.getReg() == X86::EFLAGS && !MO.isDead()) { | |||
714 | return true; | |||
715 | } | |||
716 | } | |||
717 | return false; | |||
718 | } | |||
719 | ||||
720 | /// Check whether the shift count for a machine operand is non-zero. | |||
721 | inline static unsigned getTruncatedShiftCount(MachineInstr &MI, | |||
722 | unsigned ShiftAmtOperandIdx) { | |||
723 | // The shift count is six bits with the REX.W prefix and five bits without. | |||
724 | unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31; | |||
725 | unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm(); | |||
726 | return Imm & ShiftCountMask; | |||
727 | } | |||
728 | ||||
729 | /// Check whether the given shift count is appropriate | |||
730 | /// can be represented by a LEA instruction. | |||
731 | inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) { | |||
732 | // Left shift instructions can be transformed into load-effective-address | |||
733 | // instructions if we can encode them appropriately. | |||
734 | // A LEA instruction utilizes a SIB byte to encode its scale factor. | |||
735 | // The SIB.scale field is two bits wide which means that we can encode any | |||
736 | // shift amount less than 4. | |||
737 | return ShAmt < 4 && ShAmt > 0; | |||
738 | } | |||
739 | ||||
740 | bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, | |||
741 | unsigned Opc, bool AllowSP, unsigned &NewSrc, | |||
742 | bool &isKill, MachineOperand &ImplicitOp, | |||
743 | LiveVariables *LV) const { | |||
744 | MachineFunction &MF = *MI.getParent()->getParent(); | |||
745 | const TargetRegisterClass *RC; | |||
746 | if (AllowSP) { | |||
747 | RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass; | |||
748 | } else { | |||
749 | RC = Opc != X86::LEA32r ? | |||
750 | &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass; | |||
751 | } | |||
752 | unsigned SrcReg = Src.getReg(); | |||
753 | ||||
754 | // For both LEA64 and LEA32 the register already has essentially the right | |||
755 | // type (32-bit or 64-bit) we may just need to forbid SP. | |||
756 | if (Opc != X86::LEA64_32r) { | |||
757 | NewSrc = SrcReg; | |||
758 | isKill = Src.isKill(); | |||
759 | assert(!Src.isUndef() && "Undef op doesn't need optimization")((!Src.isUndef() && "Undef op doesn't need optimization" ) ? static_cast<void> (0) : __assert_fail ("!Src.isUndef() && \"Undef op doesn't need optimization\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 759, __PRETTY_FUNCTION__)); | |||
760 | ||||
761 | if (TargetRegisterInfo::isVirtualRegister(NewSrc) && | |||
762 | !MF.getRegInfo().constrainRegClass(NewSrc, RC)) | |||
763 | return false; | |||
764 | ||||
765 | return true; | |||
766 | } | |||
767 | ||||
768 | // This is for an LEA64_32r and incoming registers are 32-bit. One way or | |||
769 | // another we need to add 64-bit registers to the final MI. | |||
770 | if (TargetRegisterInfo::isPhysicalRegister(SrcReg)) { | |||
771 | ImplicitOp = Src; | |||
772 | ImplicitOp.setImplicit(); | |||
773 | ||||
774 | NewSrc = getX86SubSuperRegister(Src.getReg(), 64); | |||
775 | isKill = Src.isKill(); | |||
776 | assert(!Src.isUndef() && "Undef op doesn't need optimization")((!Src.isUndef() && "Undef op doesn't need optimization" ) ? static_cast<void> (0) : __assert_fail ("!Src.isUndef() && \"Undef op doesn't need optimization\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 776, __PRETTY_FUNCTION__)); | |||
777 | } else { | |||
778 | // Virtual register of the wrong class, we have to create a temporary 64-bit | |||
779 | // vreg to feed into the LEA. | |||
780 | NewSrc = MF.getRegInfo().createVirtualRegister(RC); | |||
781 | MachineInstr *Copy = | |||
782 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
783 | .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit) | |||
784 | .add(Src); | |||
785 | ||||
786 | // Which is obviously going to be dead after we're done with it. | |||
787 | isKill = true; | |||
788 | ||||
789 | if (LV) | |||
790 | LV->replaceKillInstruction(SrcReg, MI, *Copy); | |||
791 | } | |||
792 | ||||
793 | // We've set all the parameters without issue. | |||
794 | return true; | |||
795 | } | |||
796 | ||||
797 | MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA( | |||
798 | unsigned MIOpc, MachineFunction::iterator &MFI, MachineInstr &MI, | |||
799 | LiveVariables *LV) const { | |||
800 | // We handle 8-bit adds and various 16-bit opcodes in the switch below. | |||
801 | bool Is16BitOp = !(MIOpc == X86::ADD8rr || MIOpc == X86::ADD8ri); | |||
802 | MachineRegisterInfo &RegInfo = MFI->getParent()->getRegInfo(); | |||
803 | assert((!Is16BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits((((!Is16BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits ( *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && "Unexpected type for LEA transform") ? static_cast<void> (0) : __assert_fail ("(!Is16BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits( *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && \"Unexpected type for LEA transform\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 805, __PRETTY_FUNCTION__)) | |||
804 | *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) &&(((!Is16BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits ( *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && "Unexpected type for LEA transform") ? static_cast<void> (0) : __assert_fail ("(!Is16BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits( *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && \"Unexpected type for LEA transform\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 805, __PRETTY_FUNCTION__)) | |||
805 | "Unexpected type for LEA transform")(((!Is16BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits ( *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && "Unexpected type for LEA transform") ? static_cast<void> (0) : __assert_fail ("(!Is16BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits( *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) && \"Unexpected type for LEA transform\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 805, __PRETTY_FUNCTION__)); | |||
806 | ||||
807 | // TODO: For a 32-bit target, we need to adjust the LEA variables with | |||
808 | // something like this: | |||
809 | // Opcode = X86::LEA32r; | |||
810 | // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); | |||
811 | // OutRegLEA = | |||
812 | // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass) | |||
813 | // : RegInfo.createVirtualRegister(&X86::GR32RegClass); | |||
814 | if (!Subtarget.is64Bit()) | |||
815 | return nullptr; | |||
816 | ||||
817 | unsigned Opcode = X86::LEA64_32r; | |||
818 | unsigned InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); | |||
819 | unsigned OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass); | |||
820 | ||||
821 | // Build and insert into an implicit UNDEF value. This is OK because | |||
822 | // we will be shifting and then extracting the lower 8/16-bits. | |||
823 | // This has the potential to cause partial register stall. e.g. | |||
824 | // movw (%rbp,%rcx,2), %dx | |||
825 | // leal -65(%rdx), %esi | |||
826 | // But testing has shown this *does* help performance in 64-bit mode (at | |||
827 | // least on modern x86 machines). | |||
828 | MachineBasicBlock::iterator MBBI = MI.getIterator(); | |||
829 | unsigned Dest = MI.getOperand(0).getReg(); | |||
830 | unsigned Src = MI.getOperand(1).getReg(); | |||
831 | bool IsDead = MI.getOperand(0).isDead(); | |||
832 | bool IsKill = MI.getOperand(1).isKill(); | |||
833 | unsigned SubReg = Is16BitOp ? X86::sub_16bit : X86::sub_8bit; | |||
834 | assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization")((!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization" ) ? static_cast<void> (0) : __assert_fail ("!MI.getOperand(1).isUndef() && \"Undef op doesn't need optimization\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 834, __PRETTY_FUNCTION__)); | |||
835 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA); | |||
836 | MachineInstr *InsMI = | |||
837 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
838 | .addReg(InRegLEA, RegState::Define, SubReg) | |||
839 | .addReg(Src, getKillRegState(IsKill)); | |||
840 | ||||
841 | MachineInstrBuilder MIB = | |||
842 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA); | |||
843 | switch (MIOpc) { | |||
844 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 844); | |||
845 | case X86::SHL16ri: { | |||
846 | unsigned ShAmt = MI.getOperand(2).getImm(); | |||
847 | MIB.addReg(0).addImm(1ULL << ShAmt) | |||
848 | .addReg(InRegLEA, RegState::Kill).addImm(0).addReg(0); | |||
849 | break; | |||
850 | } | |||
851 | case X86::INC16r: | |||
852 | addRegOffset(MIB, InRegLEA, true, 1); | |||
853 | break; | |||
854 | case X86::DEC16r: | |||
855 | addRegOffset(MIB, InRegLEA, true, -1); | |||
856 | break; | |||
857 | case X86::ADD8ri: | |||
858 | case X86::ADD16ri: | |||
859 | case X86::ADD16ri8: | |||
860 | case X86::ADD16ri_DB: | |||
861 | case X86::ADD16ri8_DB: | |||
862 | addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm()); | |||
863 | break; | |||
864 | case X86::ADD8rr: | |||
865 | case X86::ADD16rr: | |||
866 | case X86::ADD16rr_DB: { | |||
867 | unsigned Src2 = MI.getOperand(2).getReg(); | |||
868 | bool IsKill2 = MI.getOperand(2).isKill(); | |||
869 | assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization")((!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization" ) ? static_cast<void> (0) : __assert_fail ("!MI.getOperand(2).isUndef() && \"Undef op doesn't need optimization\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 869, __PRETTY_FUNCTION__)); | |||
870 | unsigned InRegLEA2 = 0; | |||
871 | MachineInstr *InsMI2 = nullptr; | |||
872 | if (Src == Src2) { | |||
873 | // ADD8rr/ADD16rr killed %reg1028, %reg1028 | |||
874 | // just a single insert_subreg. | |||
875 | addRegReg(MIB, InRegLEA, true, InRegLEA, false); | |||
876 | } else { | |||
877 | if (Subtarget.is64Bit()) | |||
878 | InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); | |||
879 | else | |||
880 | InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); | |||
881 | // Build and insert into an implicit UNDEF value. This is OK because | |||
882 | // we will be shifting and then extracting the lower 8/16-bits. | |||
883 | BuildMI(*MFI, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA2); | |||
884 | InsMI2 = BuildMI(*MFI, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
885 | .addReg(InRegLEA2, RegState::Define, SubReg) | |||
886 | .addReg(Src2, getKillRegState(IsKill2)); | |||
887 | addRegReg(MIB, InRegLEA, true, InRegLEA2, true); | |||
888 | } | |||
889 | if (LV && IsKill2 && InsMI2) | |||
890 | LV->replaceKillInstruction(Src2, MI, *InsMI2); | |||
891 | break; | |||
892 | } | |||
893 | } | |||
894 | ||||
895 | MachineInstr *NewMI = MIB; | |||
896 | MachineInstr *ExtMI = | |||
897 | BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) | |||
898 | .addReg(Dest, RegState::Define | getDeadRegState(IsDead)) | |||
899 | .addReg(OutRegLEA, RegState::Kill, SubReg); | |||
900 | ||||
901 | if (LV) { | |||
902 | // Update live variables. | |||
903 | LV->getVarInfo(InRegLEA).Kills.push_back(NewMI); | |||
904 | LV->getVarInfo(OutRegLEA).Kills.push_back(ExtMI); | |||
905 | if (IsKill) | |||
906 | LV->replaceKillInstruction(Src, MI, *InsMI); | |||
907 | if (IsDead) | |||
908 | LV->replaceKillInstruction(Dest, MI, *ExtMI); | |||
909 | } | |||
910 | ||||
911 | return ExtMI; | |||
912 | } | |||
913 | ||||
914 | /// This method must be implemented by targets that | |||
915 | /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target | |||
916 | /// may be able to convert a two-address instruction into a true | |||
917 | /// three-address instruction on demand. This allows the X86 target (for | |||
918 | /// example) to convert ADD and SHL instructions into LEA instructions if they | |||
919 | /// would require register copies due to two-addressness. | |||
920 | /// | |||
921 | /// This method returns a null pointer if the transformation cannot be | |||
922 | /// performed, otherwise it returns the new instruction. | |||
923 | /// | |||
924 | MachineInstr * | |||
925 | X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI, | |||
926 | MachineInstr &MI, LiveVariables *LV) const { | |||
927 | // The following opcodes also sets the condition code register(s). Only | |||
928 | // convert them to equivalent lea if the condition code register def's | |||
929 | // are dead! | |||
930 | if (hasLiveCondCodeDef(MI)) | |||
931 | return nullptr; | |||
932 | ||||
933 | MachineFunction &MF = *MI.getParent()->getParent(); | |||
934 | // All instructions input are two-addr instructions. Get the known operands. | |||
935 | const MachineOperand &Dest = MI.getOperand(0); | |||
936 | const MachineOperand &Src = MI.getOperand(1); | |||
937 | ||||
938 | // Ideally, operations with undef should be folded before we get here, but we | |||
939 | // can't guarantee it. Bail out because optimizing undefs is a waste of time. | |||
940 | // Without this, we have to forward undef state to new register operands to | |||
941 | // avoid machine verifier errors. | |||
942 | if (Src.isUndef()) | |||
943 | return nullptr; | |||
944 | if (MI.getNumOperands() > 2) | |||
945 | if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef()) | |||
946 | return nullptr; | |||
947 | ||||
948 | MachineInstr *NewMI = nullptr; | |||
949 | bool Is64Bit = Subtarget.is64Bit(); | |||
950 | ||||
951 | unsigned MIOpc = MI.getOpcode(); | |||
952 | switch (MIOpc) { | |||
953 | default: return nullptr; | |||
954 | case X86::SHL64ri: { | |||
955 | assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!")((MI.getNumOperands() >= 3 && "Unknown shift instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown shift instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 955, __PRETTY_FUNCTION__)); | |||
956 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
957 | if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; | |||
958 | ||||
959 | // LEA can't handle RSP. | |||
960 | if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) && | |||
961 | !MF.getRegInfo().constrainRegClass(Src.getReg(), | |||
962 | &X86::GR64_NOSPRegClass)) | |||
963 | return nullptr; | |||
964 | ||||
965 | NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)) | |||
966 | .add(Dest) | |||
967 | .addReg(0) | |||
968 | .addImm(1ULL << ShAmt) | |||
969 | .add(Src) | |||
970 | .addImm(0) | |||
971 | .addReg(0); | |||
972 | break; | |||
973 | } | |||
974 | case X86::SHL32ri: { | |||
975 | assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!")((MI.getNumOperands() >= 3 && "Unknown shift instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown shift instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 975, __PRETTY_FUNCTION__)); | |||
976 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
977 | if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; | |||
978 | ||||
979 | unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; | |||
980 | ||||
981 | // LEA can't handle ESP. | |||
982 | bool isKill; | |||
983 | unsigned SrcReg; | |||
984 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
985 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, | |||
986 | SrcReg, isKill, ImplicitOp, LV)) | |||
987 | return nullptr; | |||
988 | ||||
989 | MachineInstrBuilder MIB = | |||
990 | BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
991 | .add(Dest) | |||
992 | .addReg(0) | |||
993 | .addImm(1ULL << ShAmt) | |||
994 | .addReg(SrcReg, getKillRegState(isKill)) | |||
995 | .addImm(0) | |||
996 | .addReg(0); | |||
997 | if (ImplicitOp.getReg() != 0) | |||
998 | MIB.add(ImplicitOp); | |||
999 | NewMI = MIB; | |||
1000 | ||||
1001 | break; | |||
1002 | } | |||
1003 | case X86::SHL16ri: { | |||
1004 | assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!")((MI.getNumOperands() >= 3 && "Unknown shift instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown shift instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1004, __PRETTY_FUNCTION__)); | |||
1005 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
1006 | if (!isTruncatedShiftCountForLEA(ShAmt)) | |||
1007 | return nullptr; | |||
1008 | return convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV); | |||
1009 | } | |||
1010 | case X86::INC64r: | |||
1011 | case X86::INC32r: { | |||
1012 | assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!")((MI.getNumOperands() >= 2 && "Unknown inc instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 2 && \"Unknown inc instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1012, __PRETTY_FUNCTION__)); | |||
1013 | unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r : | |||
1014 | (Is64Bit ? X86::LEA64_32r : X86::LEA32r); | |||
1015 | bool isKill; | |||
1016 | unsigned SrcReg; | |||
1017 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1018 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, SrcReg, isKill, | |||
1019 | ImplicitOp, LV)) | |||
1020 | return nullptr; | |||
1021 | ||||
1022 | MachineInstrBuilder MIB = | |||
1023 | BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1024 | .add(Dest) | |||
1025 | .addReg(SrcReg, getKillRegState(isKill)); | |||
1026 | if (ImplicitOp.getReg() != 0) | |||
1027 | MIB.add(ImplicitOp); | |||
1028 | ||||
1029 | NewMI = addOffset(MIB, 1); | |||
1030 | break; | |||
1031 | } | |||
1032 | case X86::INC16r: | |||
1033 | return convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV); | |||
1034 | case X86::DEC64r: | |||
1035 | case X86::DEC32r: { | |||
1036 | assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!")((MI.getNumOperands() >= 2 && "Unknown dec instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 2 && \"Unknown dec instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1036, __PRETTY_FUNCTION__)); | |||
1037 | unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r | |||
1038 | : (Is64Bit ? X86::LEA64_32r : X86::LEA32r); | |||
1039 | ||||
1040 | bool isKill; | |||
1041 | unsigned SrcReg; | |||
1042 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1043 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, SrcReg, isKill, | |||
1044 | ImplicitOp, LV)) | |||
1045 | return nullptr; | |||
1046 | ||||
1047 | MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1048 | .add(Dest) | |||
1049 | .addReg(SrcReg, getKillRegState(isKill)); | |||
1050 | if (ImplicitOp.getReg() != 0) | |||
1051 | MIB.add(ImplicitOp); | |||
1052 | ||||
1053 | NewMI = addOffset(MIB, -1); | |||
1054 | ||||
1055 | break; | |||
1056 | } | |||
1057 | case X86::DEC16r: | |||
1058 | return convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV); | |||
1059 | case X86::ADD64rr: | |||
1060 | case X86::ADD64rr_DB: | |||
1061 | case X86::ADD32rr: | |||
1062 | case X86::ADD32rr_DB: { | |||
1063 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1063, __PRETTY_FUNCTION__)); | |||
1064 | unsigned Opc; | |||
1065 | if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB) | |||
1066 | Opc = X86::LEA64r; | |||
1067 | else | |||
1068 | Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; | |||
1069 | ||||
1070 | bool isKill; | |||
1071 | unsigned SrcReg; | |||
1072 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1073 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true, | |||
1074 | SrcReg, isKill, ImplicitOp, LV)) | |||
1075 | return nullptr; | |||
1076 | ||||
1077 | const MachineOperand &Src2 = MI.getOperand(2); | |||
1078 | bool isKill2; | |||
1079 | unsigned SrcReg2; | |||
1080 | MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false); | |||
1081 | if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/ false, | |||
1082 | SrcReg2, isKill2, ImplicitOp2, LV)) | |||
1083 | return nullptr; | |||
1084 | ||||
1085 | MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest); | |||
1086 | if (ImplicitOp.getReg() != 0) | |||
1087 | MIB.add(ImplicitOp); | |||
1088 | if (ImplicitOp2.getReg() != 0) | |||
1089 | MIB.add(ImplicitOp2); | |||
1090 | ||||
1091 | NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2); | |||
1092 | if (LV && Src2.isKill()) | |||
1093 | LV->replaceKillInstruction(SrcReg2, MI, *NewMI); | |||
1094 | break; | |||
1095 | } | |||
1096 | case X86::ADD8rr: | |||
1097 | case X86::ADD16rr: | |||
1098 | case X86::ADD16rr_DB: | |||
1099 | return convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV); | |||
1100 | case X86::ADD64ri32: | |||
1101 | case X86::ADD64ri8: | |||
1102 | case X86::ADD64ri32_DB: | |||
1103 | case X86::ADD64ri8_DB: | |||
1104 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1104, __PRETTY_FUNCTION__)); | |||
1105 | NewMI = addOffset( | |||
1106 | BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src), | |||
1107 | MI.getOperand(2)); | |||
1108 | break; | |||
1109 | case X86::ADD32ri: | |||
1110 | case X86::ADD32ri8: | |||
1111 | case X86::ADD32ri_DB: | |||
1112 | case X86::ADD32ri8_DB: { | |||
1113 | assert(MI.getNumOperands() >= 3 && "Unknown add instruction!")((MI.getNumOperands() >= 3 && "Unknown add instruction!" ) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() >= 3 && \"Unknown add instruction!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1113, __PRETTY_FUNCTION__)); | |||
1114 | unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r; | |||
1115 | ||||
1116 | bool isKill; | |||
1117 | unsigned SrcReg; | |||
1118 | MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); | |||
1119 | if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true, | |||
1120 | SrcReg, isKill, ImplicitOp, LV)) | |||
1121 | return nullptr; | |||
1122 | ||||
1123 | MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1124 | .add(Dest) | |||
1125 | .addReg(SrcReg, getKillRegState(isKill)); | |||
1126 | if (ImplicitOp.getReg() != 0) | |||
1127 | MIB.add(ImplicitOp); | |||
1128 | ||||
1129 | NewMI = addOffset(MIB, MI.getOperand(2)); | |||
1130 | break; | |||
1131 | } | |||
1132 | case X86::ADD8ri: | |||
1133 | case X86::ADD16ri: | |||
1134 | case X86::ADD16ri8: | |||
1135 | case X86::ADD16ri_DB: | |||
1136 | case X86::ADD16ri8_DB: | |||
1137 | return convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV); | |||
1138 | case X86::VMOVDQU8Z128rmk: | |||
1139 | case X86::VMOVDQU8Z256rmk: | |||
1140 | case X86::VMOVDQU8Zrmk: | |||
1141 | case X86::VMOVDQU16Z128rmk: | |||
1142 | case X86::VMOVDQU16Z256rmk: | |||
1143 | case X86::VMOVDQU16Zrmk: | |||
1144 | case X86::VMOVDQU32Z128rmk: case X86::VMOVDQA32Z128rmk: | |||
1145 | case X86::VMOVDQU32Z256rmk: case X86::VMOVDQA32Z256rmk: | |||
1146 | case X86::VMOVDQU32Zrmk: case X86::VMOVDQA32Zrmk: | |||
1147 | case X86::VMOVDQU64Z128rmk: case X86::VMOVDQA64Z128rmk: | |||
1148 | case X86::VMOVDQU64Z256rmk: case X86::VMOVDQA64Z256rmk: | |||
1149 | case X86::VMOVDQU64Zrmk: case X86::VMOVDQA64Zrmk: | |||
1150 | case X86::VMOVUPDZ128rmk: case X86::VMOVAPDZ128rmk: | |||
1151 | case X86::VMOVUPDZ256rmk: case X86::VMOVAPDZ256rmk: | |||
1152 | case X86::VMOVUPDZrmk: case X86::VMOVAPDZrmk: | |||
1153 | case X86::VMOVUPSZ128rmk: case X86::VMOVAPSZ128rmk: | |||
1154 | case X86::VMOVUPSZ256rmk: case X86::VMOVAPSZ256rmk: | |||
1155 | case X86::VMOVUPSZrmk: case X86::VMOVAPSZrmk: { | |||
1156 | unsigned Opc; | |||
1157 | switch (MIOpc) { | |||
1158 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1158); | |||
1159 | case X86::VMOVDQU8Z128rmk: Opc = X86::VPBLENDMBZ128rmk; break; | |||
1160 | case X86::VMOVDQU8Z256rmk: Opc = X86::VPBLENDMBZ256rmk; break; | |||
1161 | case X86::VMOVDQU8Zrmk: Opc = X86::VPBLENDMBZrmk; break; | |||
1162 | case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk; break; | |||
1163 | case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk; break; | |||
1164 | case X86::VMOVDQU16Zrmk: Opc = X86::VPBLENDMWZrmk; break; | |||
1165 | case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; | |||
1166 | case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; | |||
1167 | case X86::VMOVDQU32Zrmk: Opc = X86::VPBLENDMDZrmk; break; | |||
1168 | case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; | |||
1169 | case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; | |||
1170 | case X86::VMOVDQU64Zrmk: Opc = X86::VPBLENDMQZrmk; break; | |||
1171 | case X86::VMOVUPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; | |||
1172 | case X86::VMOVUPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; | |||
1173 | case X86::VMOVUPDZrmk: Opc = X86::VBLENDMPDZrmk; break; | |||
1174 | case X86::VMOVUPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; | |||
1175 | case X86::VMOVUPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; | |||
1176 | case X86::VMOVUPSZrmk: Opc = X86::VBLENDMPSZrmk; break; | |||
1177 | case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; | |||
1178 | case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; | |||
1179 | case X86::VMOVDQA32Zrmk: Opc = X86::VPBLENDMDZrmk; break; | |||
1180 | case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; | |||
1181 | case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; | |||
1182 | case X86::VMOVDQA64Zrmk: Opc = X86::VPBLENDMQZrmk; break; | |||
1183 | case X86::VMOVAPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; | |||
1184 | case X86::VMOVAPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; | |||
1185 | case X86::VMOVAPDZrmk: Opc = X86::VBLENDMPDZrmk; break; | |||
1186 | case X86::VMOVAPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; | |||
1187 | case X86::VMOVAPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; | |||
1188 | case X86::VMOVAPSZrmk: Opc = X86::VBLENDMPSZrmk; break; | |||
1189 | } | |||
1190 | ||||
1191 | NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1192 | .add(Dest) | |||
1193 | .add(MI.getOperand(2)) | |||
1194 | .add(Src) | |||
1195 | .add(MI.getOperand(3)) | |||
1196 | .add(MI.getOperand(4)) | |||
1197 | .add(MI.getOperand(5)) | |||
1198 | .add(MI.getOperand(6)) | |||
1199 | .add(MI.getOperand(7)); | |||
1200 | break; | |||
1201 | } | |||
1202 | case X86::VMOVDQU8Z128rrk: | |||
1203 | case X86::VMOVDQU8Z256rrk: | |||
1204 | case X86::VMOVDQU8Zrrk: | |||
1205 | case X86::VMOVDQU16Z128rrk: | |||
1206 | case X86::VMOVDQU16Z256rrk: | |||
1207 | case X86::VMOVDQU16Zrrk: | |||
1208 | case X86::VMOVDQU32Z128rrk: case X86::VMOVDQA32Z128rrk: | |||
1209 | case X86::VMOVDQU32Z256rrk: case X86::VMOVDQA32Z256rrk: | |||
1210 | case X86::VMOVDQU32Zrrk: case X86::VMOVDQA32Zrrk: | |||
1211 | case X86::VMOVDQU64Z128rrk: case X86::VMOVDQA64Z128rrk: | |||
1212 | case X86::VMOVDQU64Z256rrk: case X86::VMOVDQA64Z256rrk: | |||
1213 | case X86::VMOVDQU64Zrrk: case X86::VMOVDQA64Zrrk: | |||
1214 | case X86::VMOVUPDZ128rrk: case X86::VMOVAPDZ128rrk: | |||
1215 | case X86::VMOVUPDZ256rrk: case X86::VMOVAPDZ256rrk: | |||
1216 | case X86::VMOVUPDZrrk: case X86::VMOVAPDZrrk: | |||
1217 | case X86::VMOVUPSZ128rrk: case X86::VMOVAPSZ128rrk: | |||
1218 | case X86::VMOVUPSZ256rrk: case X86::VMOVAPSZ256rrk: | |||
1219 | case X86::VMOVUPSZrrk: case X86::VMOVAPSZrrk: { | |||
1220 | unsigned Opc; | |||
1221 | switch (MIOpc) { | |||
1222 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1222); | |||
1223 | case X86::VMOVDQU8Z128rrk: Opc = X86::VPBLENDMBZ128rrk; break; | |||
1224 | case X86::VMOVDQU8Z256rrk: Opc = X86::VPBLENDMBZ256rrk; break; | |||
1225 | case X86::VMOVDQU8Zrrk: Opc = X86::VPBLENDMBZrrk; break; | |||
1226 | case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk; break; | |||
1227 | case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk; break; | |||
1228 | case X86::VMOVDQU16Zrrk: Opc = X86::VPBLENDMWZrrk; break; | |||
1229 | case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; | |||
1230 | case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; | |||
1231 | case X86::VMOVDQU32Zrrk: Opc = X86::VPBLENDMDZrrk; break; | |||
1232 | case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; | |||
1233 | case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; | |||
1234 | case X86::VMOVDQU64Zrrk: Opc = X86::VPBLENDMQZrrk; break; | |||
1235 | case X86::VMOVUPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; | |||
1236 | case X86::VMOVUPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; | |||
1237 | case X86::VMOVUPDZrrk: Opc = X86::VBLENDMPDZrrk; break; | |||
1238 | case X86::VMOVUPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; | |||
1239 | case X86::VMOVUPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; | |||
1240 | case X86::VMOVUPSZrrk: Opc = X86::VBLENDMPSZrrk; break; | |||
1241 | case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; | |||
1242 | case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; | |||
1243 | case X86::VMOVDQA32Zrrk: Opc = X86::VPBLENDMDZrrk; break; | |||
1244 | case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; | |||
1245 | case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; | |||
1246 | case X86::VMOVDQA64Zrrk: Opc = X86::VPBLENDMQZrrk; break; | |||
1247 | case X86::VMOVAPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; | |||
1248 | case X86::VMOVAPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; | |||
1249 | case X86::VMOVAPDZrrk: Opc = X86::VBLENDMPDZrrk; break; | |||
1250 | case X86::VMOVAPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; | |||
1251 | case X86::VMOVAPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; | |||
1252 | case X86::VMOVAPSZrrk: Opc = X86::VBLENDMPSZrrk; break; | |||
1253 | } | |||
1254 | ||||
1255 | NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) | |||
1256 | .add(Dest) | |||
1257 | .add(MI.getOperand(2)) | |||
1258 | .add(Src) | |||
1259 | .add(MI.getOperand(3)); | |||
1260 | break; | |||
1261 | } | |||
1262 | } | |||
1263 | ||||
1264 | if (!NewMI) return nullptr; | |||
1265 | ||||
1266 | if (LV) { // Update live variables | |||
1267 | if (Src.isKill()) | |||
1268 | LV->replaceKillInstruction(Src.getReg(), MI, *NewMI); | |||
1269 | if (Dest.isDead()) | |||
1270 | LV->replaceKillInstruction(Dest.getReg(), MI, *NewMI); | |||
1271 | } | |||
1272 | ||||
1273 | MFI->insert(MI.getIterator(), NewMI); // Insert the new inst | |||
1274 | return NewMI; | |||
1275 | } | |||
1276 | ||||
1277 | /// This determines which of three possible cases of a three source commute | |||
1278 | /// the source indexes correspond to taking into account any mask operands. | |||
1279 | /// All prevents commuting a passthru operand. Returns -1 if the commute isn't | |||
1280 | /// possible. | |||
1281 | /// Case 0 - Possible to commute the first and second operands. | |||
1282 | /// Case 1 - Possible to commute the first and third operands. | |||
1283 | /// Case 2 - Possible to commute the second and third operands. | |||
1284 | static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, | |||
1285 | unsigned SrcOpIdx2) { | |||
1286 | // Put the lowest index to SrcOpIdx1 to simplify the checks below. | |||
1287 | if (SrcOpIdx1 > SrcOpIdx2) | |||
1288 | std::swap(SrcOpIdx1, SrcOpIdx2); | |||
1289 | ||||
1290 | unsigned Op1 = 1, Op2 = 2, Op3 = 3; | |||
1291 | if (X86II::isKMasked(TSFlags)) { | |||
1292 | Op2++; | |||
1293 | Op3++; | |||
1294 | } | |||
1295 | ||||
1296 | if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2) | |||
1297 | return 0; | |||
1298 | if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3) | |||
1299 | return 1; | |||
1300 | if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3) | |||
1301 | return 2; | |||
1302 | llvm_unreachable("Unknown three src commute case.")::llvm::llvm_unreachable_internal("Unknown three src commute case." , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1302); | |||
1303 | } | |||
1304 | ||||
1305 | unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands( | |||
1306 | const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, | |||
1307 | const X86InstrFMA3Group &FMA3Group) const { | |||
1308 | ||||
1309 | unsigned Opc = MI.getOpcode(); | |||
1310 | ||||
1311 | // TODO: Commuting the 1st operand of FMA*_Int requires some additional | |||
1312 | // analysis. The commute optimization is legal only if all users of FMA*_Int | |||
1313 | // use only the lowest element of the FMA*_Int instruction. Such analysis are | |||
1314 | // not implemented yet. So, just return 0 in that case. | |||
1315 | // When such analysis are available this place will be the right place for | |||
1316 | // calling it. | |||
1317 | assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) &&((!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && "Intrinsic instructions can't commute operand 1" ) ? static_cast<void> (0) : __assert_fail ("!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && \"Intrinsic instructions can't commute operand 1\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1318, __PRETTY_FUNCTION__)) | |||
1318 | "Intrinsic instructions can't commute operand 1")((!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && "Intrinsic instructions can't commute operand 1" ) ? static_cast<void> (0) : __assert_fail ("!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && \"Intrinsic instructions can't commute operand 1\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1318, __PRETTY_FUNCTION__)); | |||
1319 | ||||
1320 | // Determine which case this commute is or if it can't be done. | |||
1321 | unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, | |||
1322 | SrcOpIdx2); | |||
1323 | assert(Case < 3 && "Unexpected case number!")((Case < 3 && "Unexpected case number!") ? static_cast <void> (0) : __assert_fail ("Case < 3 && \"Unexpected case number!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1323, __PRETTY_FUNCTION__)); | |||
1324 | ||||
1325 | // Define the FMA forms mapping array that helps to map input FMA form | |||
1326 | // to output FMA form to preserve the operation semantics after | |||
1327 | // commuting the operands. | |||
1328 | const unsigned Form132Index = 0; | |||
1329 | const unsigned Form213Index = 1; | |||
1330 | const unsigned Form231Index = 2; | |||
1331 | static const unsigned FormMapping[][3] = { | |||
1332 | // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2; | |||
1333 | // FMA132 A, C, b; ==> FMA231 C, A, b; | |||
1334 | // FMA213 B, A, c; ==> FMA213 A, B, c; | |||
1335 | // FMA231 C, A, b; ==> FMA132 A, C, b; | |||
1336 | { Form231Index, Form213Index, Form132Index }, | |||
1337 | // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3; | |||
1338 | // FMA132 A, c, B; ==> FMA132 B, c, A; | |||
1339 | // FMA213 B, a, C; ==> FMA231 C, a, B; | |||
1340 | // FMA231 C, a, B; ==> FMA213 B, a, C; | |||
1341 | { Form132Index, Form231Index, Form213Index }, | |||
1342 | // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3; | |||
1343 | // FMA132 a, C, B; ==> FMA213 a, B, C; | |||
1344 | // FMA213 b, A, C; ==> FMA132 b, C, A; | |||
1345 | // FMA231 c, A, B; ==> FMA231 c, B, A; | |||
1346 | { Form213Index, Form132Index, Form231Index } | |||
1347 | }; | |||
1348 | ||||
1349 | unsigned FMAForms[3]; | |||
1350 | FMAForms[0] = FMA3Group.get132Opcode(); | |||
1351 | FMAForms[1] = FMA3Group.get213Opcode(); | |||
1352 | FMAForms[2] = FMA3Group.get231Opcode(); | |||
1353 | unsigned FormIndex; | |||
1354 | for (FormIndex = 0; FormIndex < 3; FormIndex++) | |||
1355 | if (Opc == FMAForms[FormIndex]) | |||
1356 | break; | |||
1357 | ||||
1358 | // Everything is ready, just adjust the FMA opcode and return it. | |||
1359 | FormIndex = FormMapping[Case][FormIndex]; | |||
1360 | return FMAForms[FormIndex]; | |||
1361 | } | |||
1362 | ||||
1363 | static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, | |||
1364 | unsigned SrcOpIdx2) { | |||
1365 | // Determine which case this commute is or if it can't be done. | |||
1366 | unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, | |||
1367 | SrcOpIdx2); | |||
1368 | assert(Case < 3 && "Unexpected case value!")((Case < 3 && "Unexpected case value!") ? static_cast <void> (0) : __assert_fail ("Case < 3 && \"Unexpected case value!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1368, __PRETTY_FUNCTION__)); | |||
1369 | ||||
1370 | // For each case we need to swap two pairs of bits in the final immediate. | |||
1371 | static const uint8_t SwapMasks[3][4] = { | |||
1372 | { 0x04, 0x10, 0x08, 0x20 }, // Swap bits 2/4 and 3/5. | |||
1373 | { 0x02, 0x10, 0x08, 0x40 }, // Swap bits 1/4 and 3/6. | |||
1374 | { 0x02, 0x04, 0x20, 0x40 }, // Swap bits 1/2 and 5/6. | |||
1375 | }; | |||
1376 | ||||
1377 | uint8_t Imm = MI.getOperand(MI.getNumOperands()-1).getImm(); | |||
1378 | // Clear out the bits we are swapping. | |||
1379 | uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] | | |||
1380 | SwapMasks[Case][2] | SwapMasks[Case][3]); | |||
1381 | // If the immediate had a bit of the pair set, then set the opposite bit. | |||
1382 | if (Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1]; | |||
1383 | if (Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0]; | |||
1384 | if (Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3]; | |||
1385 | if (Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2]; | |||
1386 | MI.getOperand(MI.getNumOperands()-1).setImm(NewImm); | |||
1387 | } | |||
1388 | ||||
1389 | // Returns true if this is a VPERMI2 or VPERMT2 instruction that can be | |||
1390 | // commuted. | |||
1391 | static bool isCommutableVPERMV3Instruction(unsigned Opcode) { | |||
1392 | #define VPERM_CASES(Suffix) \ | |||
1393 | case X86::VPERMI2##Suffix##128rr: case X86::VPERMT2##Suffix##128rr: \ | |||
1394 | case X86::VPERMI2##Suffix##256rr: case X86::VPERMT2##Suffix##256rr: \ | |||
1395 | case X86::VPERMI2##Suffix##rr: case X86::VPERMT2##Suffix##rr: \ | |||
1396 | case X86::VPERMI2##Suffix##128rm: case X86::VPERMT2##Suffix##128rm: \ | |||
1397 | case X86::VPERMI2##Suffix##256rm: case X86::VPERMT2##Suffix##256rm: \ | |||
1398 | case X86::VPERMI2##Suffix##rm: case X86::VPERMT2##Suffix##rm: \ | |||
1399 | case X86::VPERMI2##Suffix##128rrkz: case X86::VPERMT2##Suffix##128rrkz: \ | |||
1400 | case X86::VPERMI2##Suffix##256rrkz: case X86::VPERMT2##Suffix##256rrkz: \ | |||
1401 | case X86::VPERMI2##Suffix##rrkz: case X86::VPERMT2##Suffix##rrkz: \ | |||
1402 | case X86::VPERMI2##Suffix##128rmkz: case X86::VPERMT2##Suffix##128rmkz: \ | |||
1403 | case X86::VPERMI2##Suffix##256rmkz: case X86::VPERMT2##Suffix##256rmkz: \ | |||
1404 | case X86::VPERMI2##Suffix##rmkz: case X86::VPERMT2##Suffix##rmkz: | |||
1405 | ||||
1406 | #define VPERM_CASES_BROADCAST(Suffix) \ | |||
1407 | VPERM_CASES(Suffix) \ | |||
1408 | case X86::VPERMI2##Suffix##128rmb: case X86::VPERMT2##Suffix##128rmb: \ | |||
1409 | case X86::VPERMI2##Suffix##256rmb: case X86::VPERMT2##Suffix##256rmb: \ | |||
1410 | case X86::VPERMI2##Suffix##rmb: case X86::VPERMT2##Suffix##rmb: \ | |||
1411 | case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \ | |||
1412 | case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \ | |||
1413 | case X86::VPERMI2##Suffix##rmbkz: case X86::VPERMT2##Suffix##rmbkz: | |||
1414 | ||||
1415 | switch (Opcode) { | |||
1416 | default: return false; | |||
1417 | VPERM_CASES(B) | |||
1418 | VPERM_CASES_BROADCAST(D) | |||
1419 | VPERM_CASES_BROADCAST(PD) | |||
1420 | VPERM_CASES_BROADCAST(PS) | |||
1421 | VPERM_CASES_BROADCAST(Q) | |||
1422 | VPERM_CASES(W) | |||
1423 | return true; | |||
1424 | } | |||
1425 | #undef VPERM_CASES_BROADCAST | |||
1426 | #undef VPERM_CASES | |||
1427 | } | |||
1428 | ||||
1429 | // Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching | |||
1430 | // from the I opcode to the T opcode and vice versa. | |||
1431 | static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) { | |||
1432 | #define VPERM_CASES(Orig, New) \ | |||
1433 | case X86::Orig##128rr: return X86::New##128rr; \ | |||
1434 | case X86::Orig##128rrkz: return X86::New##128rrkz; \ | |||
1435 | case X86::Orig##128rm: return X86::New##128rm; \ | |||
1436 | case X86::Orig##128rmkz: return X86::New##128rmkz; \ | |||
1437 | case X86::Orig##256rr: return X86::New##256rr; \ | |||
1438 | case X86::Orig##256rrkz: return X86::New##256rrkz; \ | |||
1439 | case X86::Orig##256rm: return X86::New##256rm; \ | |||
1440 | case X86::Orig##256rmkz: return X86::New##256rmkz; \ | |||
1441 | case X86::Orig##rr: return X86::New##rr; \ | |||
1442 | case X86::Orig##rrkz: return X86::New##rrkz; \ | |||
1443 | case X86::Orig##rm: return X86::New##rm; \ | |||
1444 | case X86::Orig##rmkz: return X86::New##rmkz; | |||
1445 | ||||
1446 | #define VPERM_CASES_BROADCAST(Orig, New) \ | |||
1447 | VPERM_CASES(Orig, New) \ | |||
1448 | case X86::Orig##128rmb: return X86::New##128rmb; \ | |||
1449 | case X86::Orig##128rmbkz: return X86::New##128rmbkz; \ | |||
1450 | case X86::Orig##256rmb: return X86::New##256rmb; \ | |||
1451 | case X86::Orig##256rmbkz: return X86::New##256rmbkz; \ | |||
1452 | case X86::Orig##rmb: return X86::New##rmb; \ | |||
1453 | case X86::Orig##rmbkz: return X86::New##rmbkz; | |||
1454 | ||||
1455 | switch (Opcode) { | |||
1456 | VPERM_CASES(VPERMI2B, VPERMT2B) | |||
1457 | VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D) | |||
1458 | VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD) | |||
1459 | VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS) | |||
1460 | VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q) | |||
1461 | VPERM_CASES(VPERMI2W, VPERMT2W) | |||
1462 | VPERM_CASES(VPERMT2B, VPERMI2B) | |||
1463 | VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D) | |||
1464 | VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD) | |||
1465 | VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS) | |||
1466 | VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q) | |||
1467 | VPERM_CASES(VPERMT2W, VPERMI2W) | |||
1468 | } | |||
1469 | ||||
1470 | llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1470); | |||
1471 | #undef VPERM_CASES_BROADCAST | |||
1472 | #undef VPERM_CASES | |||
1473 | } | |||
1474 | ||||
1475 | MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, | |||
1476 | unsigned OpIdx1, | |||
1477 | unsigned OpIdx2) const { | |||
1478 | auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { | |||
1479 | if (NewMI) | |||
1480 | return *MI.getParent()->getParent()->CloneMachineInstr(&MI); | |||
1481 | return MI; | |||
1482 | }; | |||
1483 | ||||
1484 | switch (MI.getOpcode()) { | |||
1485 | case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I) | |||
1486 | case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I) | |||
1487 | case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I) | |||
1488 | case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I) | |||
1489 | case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I) | |||
1490 | case X86::SHLD64rri8:{// A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, (64-I) | |||
1491 | unsigned Opc; | |||
1492 | unsigned Size; | |||
1493 | switch (MI.getOpcode()) { | |||
1494 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1494); | |||
1495 | case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8; break; | |||
1496 | case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8; break; | |||
1497 | case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8; break; | |||
1498 | case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8; break; | |||
1499 | case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8; break; | |||
1500 | case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8; break; | |||
1501 | } | |||
1502 | unsigned Amt = MI.getOperand(3).getImm(); | |||
1503 | auto &WorkingMI = cloneIfNew(MI); | |||
1504 | WorkingMI.setDesc(get(Opc)); | |||
1505 | WorkingMI.getOperand(3).setImm(Size - Amt); | |||
1506 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1507 | OpIdx1, OpIdx2); | |||
1508 | } | |||
1509 | case X86::PFSUBrr: | |||
1510 | case X86::PFSUBRrr: { | |||
1511 | // PFSUB x, y: x = x - y | |||
1512 | // PFSUBR x, y: x = y - x | |||
1513 | unsigned Opc = | |||
1514 | (X86::PFSUBRrr == MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr); | |||
1515 | auto &WorkingMI = cloneIfNew(MI); | |||
1516 | WorkingMI.setDesc(get(Opc)); | |||
1517 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1518 | OpIdx1, OpIdx2); | |||
1519 | } | |||
1520 | case X86::BLENDPDrri: | |||
1521 | case X86::BLENDPSrri: | |||
1522 | case X86::VBLENDPDrri: | |||
1523 | case X86::VBLENDPSrri: | |||
1524 | // If we're optimizing for size, try to use MOVSD/MOVSS. | |||
1525 | if (MI.getParent()->getParent()->getFunction().optForSize()) { | |||
1526 | unsigned Mask, Opc; | |||
1527 | switch (MI.getOpcode()) { | |||
1528 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1528); | |||
1529 | case X86::BLENDPDrri: Opc = X86::MOVSDrr; Mask = 0x03; break; | |||
1530 | case X86::BLENDPSrri: Opc = X86::MOVSSrr; Mask = 0x0F; break; | |||
1531 | case X86::VBLENDPDrri: Opc = X86::VMOVSDrr; Mask = 0x03; break; | |||
1532 | case X86::VBLENDPSrri: Opc = X86::VMOVSSrr; Mask = 0x0F; break; | |||
1533 | } | |||
1534 | if ((MI.getOperand(3).getImm() ^ Mask) == 1) { | |||
1535 | auto &WorkingMI = cloneIfNew(MI); | |||
1536 | WorkingMI.setDesc(get(Opc)); | |||
1537 | WorkingMI.RemoveOperand(3); | |||
1538 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, | |||
1539 | /*NewMI=*/false, | |||
1540 | OpIdx1, OpIdx2); | |||
1541 | } | |||
1542 | } | |||
1543 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
1544 | case X86::PBLENDWrri: | |||
1545 | case X86::VBLENDPDYrri: | |||
1546 | case X86::VBLENDPSYrri: | |||
1547 | case X86::VPBLENDDrri: | |||
1548 | case X86::VPBLENDWrri: | |||
1549 | case X86::VPBLENDDYrri: | |||
1550 | case X86::VPBLENDWYrri:{ | |||
1551 | unsigned Mask; | |||
1552 | switch (MI.getOpcode()) { | |||
1553 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1553); | |||
1554 | case X86::BLENDPDrri: Mask = 0x03; break; | |||
1555 | case X86::BLENDPSrri: Mask = 0x0F; break; | |||
1556 | case X86::PBLENDWrri: Mask = 0xFF; break; | |||
1557 | case X86::VBLENDPDrri: Mask = 0x03; break; | |||
1558 | case X86::VBLENDPSrri: Mask = 0x0F; break; | |||
1559 | case X86::VBLENDPDYrri: Mask = 0x0F; break; | |||
1560 | case X86::VBLENDPSYrri: Mask = 0xFF; break; | |||
1561 | case X86::VPBLENDDrri: Mask = 0x0F; break; | |||
1562 | case X86::VPBLENDWrri: Mask = 0xFF; break; | |||
1563 | case X86::VPBLENDDYrri: Mask = 0xFF; break; | |||
1564 | case X86::VPBLENDWYrri: Mask = 0xFF; break; | |||
1565 | } | |||
1566 | // Only the least significant bits of Imm are used. | |||
1567 | unsigned Imm = MI.getOperand(3).getImm() & Mask; | |||
1568 | auto &WorkingMI = cloneIfNew(MI); | |||
1569 | WorkingMI.getOperand(3).setImm(Mask ^ Imm); | |||
1570 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1571 | OpIdx1, OpIdx2); | |||
1572 | } | |||
1573 | case X86::MOVSDrr: | |||
1574 | case X86::MOVSSrr: | |||
1575 | case X86::VMOVSDrr: | |||
1576 | case X86::VMOVSSrr:{ | |||
1577 | // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD. | |||
1578 | assert(Subtarget.hasSSE41() && "Commuting MOVSD/MOVSS requires SSE41!")((Subtarget.hasSSE41() && "Commuting MOVSD/MOVSS requires SSE41!" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE41() && \"Commuting MOVSD/MOVSS requires SSE41!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1578, __PRETTY_FUNCTION__)); | |||
1579 | ||||
1580 | unsigned Mask, Opc; | |||
1581 | switch (MI.getOpcode()) { | |||
1582 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1582); | |||
1583 | case X86::MOVSDrr: Opc = X86::BLENDPDrri; Mask = 0x02; break; | |||
1584 | case X86::MOVSSrr: Opc = X86::BLENDPSrri; Mask = 0x0E; break; | |||
1585 | case X86::VMOVSDrr: Opc = X86::VBLENDPDrri; Mask = 0x02; break; | |||
1586 | case X86::VMOVSSrr: Opc = X86::VBLENDPSrri; Mask = 0x0E; break; | |||
1587 | } | |||
1588 | ||||
1589 | auto &WorkingMI = cloneIfNew(MI); | |||
1590 | WorkingMI.setDesc(get(Opc)); | |||
1591 | WorkingMI.addOperand(MachineOperand::CreateImm(Mask)); | |||
1592 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1593 | OpIdx1, OpIdx2); | |||
1594 | } | |||
1595 | case X86::PCLMULQDQrr: | |||
1596 | case X86::VPCLMULQDQrr: | |||
1597 | case X86::VPCLMULQDQYrr: | |||
1598 | case X86::VPCLMULQDQZrr: | |||
1599 | case X86::VPCLMULQDQZ128rr: | |||
1600 | case X86::VPCLMULQDQZ256rr: { | |||
1601 | // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0] | |||
1602 | // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0] | |||
1603 | unsigned Imm = MI.getOperand(3).getImm(); | |||
1604 | unsigned Src1Hi = Imm & 0x01; | |||
1605 | unsigned Src2Hi = Imm & 0x10; | |||
1606 | auto &WorkingMI = cloneIfNew(MI); | |||
1607 | WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4)); | |||
1608 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1609 | OpIdx1, OpIdx2); | |||
1610 | } | |||
1611 | case X86::VPCMPBZ128rri: case X86::VPCMPUBZ128rri: | |||
1612 | case X86::VPCMPBZ256rri: case X86::VPCMPUBZ256rri: | |||
1613 | case X86::VPCMPBZrri: case X86::VPCMPUBZrri: | |||
1614 | case X86::VPCMPDZ128rri: case X86::VPCMPUDZ128rri: | |||
1615 | case X86::VPCMPDZ256rri: case X86::VPCMPUDZ256rri: | |||
1616 | case X86::VPCMPDZrri: case X86::VPCMPUDZrri: | |||
1617 | case X86::VPCMPQZ128rri: case X86::VPCMPUQZ128rri: | |||
1618 | case X86::VPCMPQZ256rri: case X86::VPCMPUQZ256rri: | |||
1619 | case X86::VPCMPQZrri: case X86::VPCMPUQZrri: | |||
1620 | case X86::VPCMPWZ128rri: case X86::VPCMPUWZ128rri: | |||
1621 | case X86::VPCMPWZ256rri: case X86::VPCMPUWZ256rri: | |||
1622 | case X86::VPCMPWZrri: case X86::VPCMPUWZrri: | |||
1623 | case X86::VPCMPBZ128rrik: case X86::VPCMPUBZ128rrik: | |||
1624 | case X86::VPCMPBZ256rrik: case X86::VPCMPUBZ256rrik: | |||
1625 | case X86::VPCMPBZrrik: case X86::VPCMPUBZrrik: | |||
1626 | case X86::VPCMPDZ128rrik: case X86::VPCMPUDZ128rrik: | |||
1627 | case X86::VPCMPDZ256rrik: case X86::VPCMPUDZ256rrik: | |||
1628 | case X86::VPCMPDZrrik: case X86::VPCMPUDZrrik: | |||
1629 | case X86::VPCMPQZ128rrik: case X86::VPCMPUQZ128rrik: | |||
1630 | case X86::VPCMPQZ256rrik: case X86::VPCMPUQZ256rrik: | |||
1631 | case X86::VPCMPQZrrik: case X86::VPCMPUQZrrik: | |||
1632 | case X86::VPCMPWZ128rrik: case X86::VPCMPUWZ128rrik: | |||
1633 | case X86::VPCMPWZ256rrik: case X86::VPCMPUWZ256rrik: | |||
1634 | case X86::VPCMPWZrrik: case X86::VPCMPUWZrrik: { | |||
1635 | // Flip comparison mode immediate (if necessary). | |||
1636 | unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7; | |||
1637 | Imm = X86::getSwappedVPCMPImm(Imm); | |||
1638 | auto &WorkingMI = cloneIfNew(MI); | |||
1639 | WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm); | |||
1640 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1641 | OpIdx1, OpIdx2); | |||
1642 | } | |||
1643 | case X86::VPCOMBri: case X86::VPCOMUBri: | |||
1644 | case X86::VPCOMDri: case X86::VPCOMUDri: | |||
1645 | case X86::VPCOMQri: case X86::VPCOMUQri: | |||
1646 | case X86::VPCOMWri: case X86::VPCOMUWri: { | |||
1647 | // Flip comparison mode immediate (if necessary). | |||
1648 | unsigned Imm = MI.getOperand(3).getImm() & 0x7; | |||
1649 | Imm = X86::getSwappedVPCOMImm(Imm); | |||
1650 | auto &WorkingMI = cloneIfNew(MI); | |||
1651 | WorkingMI.getOperand(3).setImm(Imm); | |||
1652 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1653 | OpIdx1, OpIdx2); | |||
1654 | } | |||
1655 | case X86::VPERM2F128rr: | |||
1656 | case X86::VPERM2I128rr: { | |||
1657 | // Flip permute source immediate. | |||
1658 | // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi. | |||
1659 | // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi. | |||
1660 | unsigned Imm = MI.getOperand(3).getImm() & 0xFF; | |||
1661 | auto &WorkingMI = cloneIfNew(MI); | |||
1662 | WorkingMI.getOperand(3).setImm(Imm ^ 0x22); | |||
1663 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1664 | OpIdx1, OpIdx2); | |||
1665 | } | |||
1666 | case X86::MOVHLPSrr: | |||
1667 | case X86::UNPCKHPDrr: | |||
1668 | case X86::VMOVHLPSrr: | |||
1669 | case X86::VUNPCKHPDrr: | |||
1670 | case X86::VMOVHLPSZrr: | |||
1671 | case X86::VUNPCKHPDZ128rr: { | |||
1672 | assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!")((Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Commuting MOVHLP/UNPCKHPD requires SSE2!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1672, __PRETTY_FUNCTION__)); | |||
1673 | ||||
1674 | unsigned Opc = MI.getOpcode(); | |||
1675 | switch (Opc) { | |||
1676 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1676); | |||
1677 | case X86::MOVHLPSrr: Opc = X86::UNPCKHPDrr; break; | |||
1678 | case X86::UNPCKHPDrr: Opc = X86::MOVHLPSrr; break; | |||
1679 | case X86::VMOVHLPSrr: Opc = X86::VUNPCKHPDrr; break; | |||
1680 | case X86::VUNPCKHPDrr: Opc = X86::VMOVHLPSrr; break; | |||
1681 | case X86::VMOVHLPSZrr: Opc = X86::VUNPCKHPDZ128rr; break; | |||
1682 | case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr; break; | |||
1683 | } | |||
1684 | auto &WorkingMI = cloneIfNew(MI); | |||
1685 | WorkingMI.setDesc(get(Opc)); | |||
1686 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1687 | OpIdx1, OpIdx2); | |||
1688 | } | |||
1689 | case X86::CMOVB16rr: case X86::CMOVB32rr: case X86::CMOVB64rr: | |||
1690 | case X86::CMOVAE16rr: case X86::CMOVAE32rr: case X86::CMOVAE64rr: | |||
1691 | case X86::CMOVE16rr: case X86::CMOVE32rr: case X86::CMOVE64rr: | |||
1692 | case X86::CMOVNE16rr: case X86::CMOVNE32rr: case X86::CMOVNE64rr: | |||
1693 | case X86::CMOVBE16rr: case X86::CMOVBE32rr: case X86::CMOVBE64rr: | |||
1694 | case X86::CMOVA16rr: case X86::CMOVA32rr: case X86::CMOVA64rr: | |||
1695 | case X86::CMOVL16rr: case X86::CMOVL32rr: case X86::CMOVL64rr: | |||
1696 | case X86::CMOVGE16rr: case X86::CMOVGE32rr: case X86::CMOVGE64rr: | |||
1697 | case X86::CMOVLE16rr: case X86::CMOVLE32rr: case X86::CMOVLE64rr: | |||
1698 | case X86::CMOVG16rr: case X86::CMOVG32rr: case X86::CMOVG64rr: | |||
1699 | case X86::CMOVS16rr: case X86::CMOVS32rr: case X86::CMOVS64rr: | |||
1700 | case X86::CMOVNS16rr: case X86::CMOVNS32rr: case X86::CMOVNS64rr: | |||
1701 | case X86::CMOVP16rr: case X86::CMOVP32rr: case X86::CMOVP64rr: | |||
1702 | case X86::CMOVNP16rr: case X86::CMOVNP32rr: case X86::CMOVNP64rr: | |||
1703 | case X86::CMOVO16rr: case X86::CMOVO32rr: case X86::CMOVO64rr: | |||
1704 | case X86::CMOVNO16rr: case X86::CMOVNO32rr: case X86::CMOVNO64rr: { | |||
1705 | unsigned Opc; | |||
1706 | switch (MI.getOpcode()) { | |||
1707 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 1707); | |||
1708 | case X86::CMOVB16rr: Opc = X86::CMOVAE16rr; break; | |||
1709 | case X86::CMOVB32rr: Opc = X86::CMOVAE32rr; break; | |||
1710 | case X86::CMOVB64rr: Opc = X86::CMOVAE64rr; break; | |||
1711 | case X86::CMOVAE16rr: Opc = X86::CMOVB16rr; break; | |||
1712 | case X86::CMOVAE32rr: Opc = X86::CMOVB32rr; break; | |||
1713 | case X86::CMOVAE64rr: Opc = X86::CMOVB64rr; break; | |||
1714 | case X86::CMOVE16rr: Opc = X86::CMOVNE16rr; break; | |||
1715 | case X86::CMOVE32rr: Opc = X86::CMOVNE32rr; break; | |||
1716 | case X86::CMOVE64rr: Opc = X86::CMOVNE64rr; break; | |||
1717 | case X86::CMOVNE16rr: Opc = X86::CMOVE16rr; break; | |||
1718 | case X86::CMOVNE32rr: Opc = X86::CMOVE32rr; break; | |||
1719 | case X86::CMOVNE64rr: Opc = X86::CMOVE64rr; break; | |||
1720 | case X86::CMOVBE16rr: Opc = X86::CMOVA16rr; break; | |||
1721 | case X86::CMOVBE32rr: Opc = X86::CMOVA32rr; break; | |||
1722 | case X86::CMOVBE64rr: Opc = X86::CMOVA64rr; break; | |||
1723 | case X86::CMOVA16rr: Opc = X86::CMOVBE16rr; break; | |||
1724 | case X86::CMOVA32rr: Opc = X86::CMOVBE32rr; break; | |||
1725 | case X86::CMOVA64rr: Opc = X86::CMOVBE64rr; break; | |||
1726 | case X86::CMOVL16rr: Opc = X86::CMOVGE16rr; break; | |||
1727 | case X86::CMOVL32rr: Opc = X86::CMOVGE32rr; break; | |||
1728 | case X86::CMOVL64rr: Opc = X86::CMOVGE64rr; break; | |||
1729 | case X86::CMOVGE16rr: Opc = X86::CMOVL16rr; break; | |||
1730 | case X86::CMOVGE32rr: Opc = X86::CMOVL32rr; break; | |||
1731 | case X86::CMOVGE64rr: Opc = X86::CMOVL64rr; break; | |||
1732 | case X86::CMOVLE16rr: Opc = X86::CMOVG16rr; break; | |||
1733 | case X86::CMOVLE32rr: Opc = X86::CMOVG32rr; break; | |||
1734 | case X86::CMOVLE64rr: Opc = X86::CMOVG64rr; break; | |||
1735 | case X86::CMOVG16rr: Opc = X86::CMOVLE16rr; break; | |||
1736 | case X86::CMOVG32rr: Opc = X86::CMOVLE32rr; break; | |||
1737 | case X86::CMOVG64rr: Opc = X86::CMOVLE64rr; break; | |||
1738 | case X86::CMOVS16rr: Opc = X86::CMOVNS16rr; break; | |||
1739 | case X86::CMOVS32rr: Opc = X86::CMOVNS32rr; break; | |||
1740 | case X86::CMOVS64rr: Opc = X86::CMOVNS64rr; break; | |||
1741 | case X86::CMOVNS16rr: Opc = X86::CMOVS16rr; break; | |||
1742 | case X86::CMOVNS32rr: Opc = X86::CMOVS32rr; break; | |||
1743 | case X86::CMOVNS64rr: Opc = X86::CMOVS64rr; break; | |||
1744 | case X86::CMOVP16rr: Opc = X86::CMOVNP16rr; break; | |||
1745 | case X86::CMOVP32rr: Opc = X86::CMOVNP32rr; break; | |||
1746 | case X86::CMOVP64rr: Opc = X86::CMOVNP64rr; break; | |||
1747 | case X86::CMOVNP16rr: Opc = X86::CMOVP16rr; break; | |||
1748 | case X86::CMOVNP32rr: Opc = X86::CMOVP32rr; break; | |||
1749 | case X86::CMOVNP64rr: Opc = X86::CMOVP64rr; break; | |||
1750 | case X86::CMOVO16rr: Opc = X86::CMOVNO16rr; break; | |||
1751 | case X86::CMOVO32rr: Opc = X86::CMOVNO32rr; break; | |||
1752 | case X86::CMOVO64rr: Opc = X86::CMOVNO64rr; break; | |||
1753 | case X86::CMOVNO16rr: Opc = X86::CMOVO16rr; break; | |||
1754 | case X86::CMOVNO32rr: Opc = X86::CMOVO32rr; break; | |||
1755 | case X86::CMOVNO64rr: Opc = X86::CMOVO64rr; break; | |||
1756 | } | |||
1757 | auto &WorkingMI = cloneIfNew(MI); | |||
1758 | WorkingMI.setDesc(get(Opc)); | |||
1759 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1760 | OpIdx1, OpIdx2); | |||
1761 | } | |||
1762 | case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: | |||
1763 | case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: | |||
1764 | case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: | |||
1765 | case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: | |||
1766 | case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: | |||
1767 | case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: | |||
1768 | case X86::VPTERNLOGDZrrik: | |||
1769 | case X86::VPTERNLOGDZ128rrik: | |||
1770 | case X86::VPTERNLOGDZ256rrik: | |||
1771 | case X86::VPTERNLOGQZrrik: | |||
1772 | case X86::VPTERNLOGQZ128rrik: | |||
1773 | case X86::VPTERNLOGQZ256rrik: | |||
1774 | case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: | |||
1775 | case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: | |||
1776 | case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: | |||
1777 | case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: | |||
1778 | case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: | |||
1779 | case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: | |||
1780 | case X86::VPTERNLOGDZ128rmbi: | |||
1781 | case X86::VPTERNLOGDZ256rmbi: | |||
1782 | case X86::VPTERNLOGDZrmbi: | |||
1783 | case X86::VPTERNLOGQZ128rmbi: | |||
1784 | case X86::VPTERNLOGQZ256rmbi: | |||
1785 | case X86::VPTERNLOGQZrmbi: | |||
1786 | case X86::VPTERNLOGDZ128rmbikz: | |||
1787 | case X86::VPTERNLOGDZ256rmbikz: | |||
1788 | case X86::VPTERNLOGDZrmbikz: | |||
1789 | case X86::VPTERNLOGQZ128rmbikz: | |||
1790 | case X86::VPTERNLOGQZ256rmbikz: | |||
1791 | case X86::VPTERNLOGQZrmbikz: { | |||
1792 | auto &WorkingMI = cloneIfNew(MI); | |||
1793 | commuteVPTERNLOG(WorkingMI, OpIdx1, OpIdx2); | |||
1794 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1795 | OpIdx1, OpIdx2); | |||
1796 | } | |||
1797 | default: { | |||
1798 | if (isCommutableVPERMV3Instruction(MI.getOpcode())) { | |||
1799 | unsigned Opc = getCommutedVPERMV3Opcode(MI.getOpcode()); | |||
1800 | auto &WorkingMI = cloneIfNew(MI); | |||
1801 | WorkingMI.setDesc(get(Opc)); | |||
1802 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1803 | OpIdx1, OpIdx2); | |||
1804 | } | |||
1805 | ||||
1806 | const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), | |||
1807 | MI.getDesc().TSFlags); | |||
1808 | if (FMA3Group) { | |||
1809 | unsigned Opc = | |||
1810 | getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group); | |||
1811 | auto &WorkingMI = cloneIfNew(MI); | |||
1812 | WorkingMI.setDesc(get(Opc)); | |||
1813 | return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, | |||
1814 | OpIdx1, OpIdx2); | |||
1815 | } | |||
1816 | ||||
1817 | return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); | |||
1818 | } | |||
1819 | } | |||
1820 | } | |||
1821 | ||||
1822 | bool | |||
1823 | X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI, | |||
1824 | unsigned &SrcOpIdx1, | |||
1825 | unsigned &SrcOpIdx2, | |||
1826 | bool IsIntrinsic) const { | |||
1827 | uint64_t TSFlags = MI.getDesc().TSFlags; | |||
1828 | ||||
1829 | unsigned FirstCommutableVecOp = 1; | |||
1830 | unsigned LastCommutableVecOp = 3; | |||
1831 | unsigned KMaskOp = -1U; | |||
1832 | if (X86II::isKMasked(TSFlags)) { | |||
1833 | // For k-zero-masked operations it is Ok to commute the first vector | |||
1834 | // operand. | |||
1835 | // For regular k-masked operations a conservative choice is done as the | |||
1836 | // elements of the first vector operand, for which the corresponding bit | |||
1837 | // in the k-mask operand is set to 0, are copied to the result of the | |||
1838 | // instruction. | |||
1839 | // TODO/FIXME: The commute still may be legal if it is known that the | |||
1840 | // k-mask operand is set to either all ones or all zeroes. | |||
1841 | // It is also Ok to commute the 1st operand if all users of MI use only | |||
1842 | // the elements enabled by the k-mask operand. For example, | |||
1843 | // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i] | |||
1844 | // : v1[i]; | |||
1845 | // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 -> | |||
1846 | // // Ok, to commute v1 in FMADD213PSZrk. | |||
1847 | ||||
1848 | // The k-mask operand has index = 2 for masked and zero-masked operations. | |||
1849 | KMaskOp = 2; | |||
1850 | ||||
1851 | // The operand with index = 1 is used as a source for those elements for | |||
1852 | // which the corresponding bit in the k-mask is set to 0. | |||
1853 | if (X86II::isKMergeMasked(TSFlags)) | |||
1854 | FirstCommutableVecOp = 3; | |||
1855 | ||||
1856 | LastCommutableVecOp++; | |||
1857 | } else if (IsIntrinsic) { | |||
1858 | // Commuting the first operand of an intrinsic instruction isn't possible | |||
1859 | // unless we can prove that only the lowest element of the result is used. | |||
1860 | FirstCommutableVecOp = 2; | |||
1861 | } | |||
1862 | ||||
1863 | if (isMem(MI, LastCommutableVecOp)) | |||
1864 | LastCommutableVecOp--; | |||
1865 | ||||
1866 | // Only the first RegOpsNum operands are commutable. | |||
1867 | // Also, the value 'CommuteAnyOperandIndex' is valid here as it means | |||
1868 | // that the operand is not specified/fixed. | |||
1869 | if (SrcOpIdx1 != CommuteAnyOperandIndex && | |||
1870 | (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp || | |||
1871 | SrcOpIdx1 == KMaskOp)) | |||
1872 | return false; | |||
1873 | if (SrcOpIdx2 != CommuteAnyOperandIndex && | |||
1874 | (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp || | |||
1875 | SrcOpIdx2 == KMaskOp)) | |||
1876 | return false; | |||
1877 | ||||
1878 | // Look for two different register operands assumed to be commutable | |||
1879 | // regardless of the FMA opcode. The FMA opcode is adjusted later. | |||
1880 | if (SrcOpIdx1 == CommuteAnyOperandIndex || | |||
1881 | SrcOpIdx2 == CommuteAnyOperandIndex) { | |||
1882 | unsigned CommutableOpIdx1 = SrcOpIdx1; | |||
1883 | unsigned CommutableOpIdx2 = SrcOpIdx2; | |||
1884 | ||||
1885 | // At least one of operands to be commuted is not specified and | |||
1886 | // this method is free to choose appropriate commutable operands. | |||
1887 | if (SrcOpIdx1 == SrcOpIdx2) | |||
1888 | // Both of operands are not fixed. By default set one of commutable | |||
1889 | // operands to the last register operand of the instruction. | |||
1890 | CommutableOpIdx2 = LastCommutableVecOp; | |||
1891 | else if (SrcOpIdx2 == CommuteAnyOperandIndex) | |||
1892 | // Only one of operands is not fixed. | |||
1893 | CommutableOpIdx2 = SrcOpIdx1; | |||
1894 | ||||
1895 | // CommutableOpIdx2 is well defined now. Let's choose another commutable | |||
1896 | // operand and assign its index to CommutableOpIdx1. | |||
1897 | unsigned Op2Reg = MI.getOperand(CommutableOpIdx2).getReg(); | |||
1898 | for (CommutableOpIdx1 = LastCommutableVecOp; | |||
1899 | CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) { | |||
1900 | // Just ignore and skip the k-mask operand. | |||
1901 | if (CommutableOpIdx1 == KMaskOp) | |||
1902 | continue; | |||
1903 | ||||
1904 | // The commuted operands must have different registers. | |||
1905 | // Otherwise, the commute transformation does not change anything and | |||
1906 | // is useless then. | |||
1907 | if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg()) | |||
1908 | break; | |||
1909 | } | |||
1910 | ||||
1911 | // No appropriate commutable operands were found. | |||
1912 | if (CommutableOpIdx1 < FirstCommutableVecOp) | |||
1913 | return false; | |||
1914 | ||||
1915 | // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2 | |||
1916 | // to return those values. | |||
1917 | if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, | |||
1918 | CommutableOpIdx1, CommutableOpIdx2)) | |||
1919 | return false; | |||
1920 | } | |||
1921 | ||||
1922 | return true; | |||
1923 | } | |||
1924 | ||||
1925 | bool X86InstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1, | |||
1926 | unsigned &SrcOpIdx2) const { | |||
1927 | const MCInstrDesc &Desc = MI.getDesc(); | |||
1928 | if (!Desc.isCommutable()) | |||
1929 | return false; | |||
1930 | ||||
1931 | switch (MI.getOpcode()) { | |||
1932 | case X86::CMPSDrr: | |||
1933 | case X86::CMPSSrr: | |||
1934 | case X86::CMPPDrri: | |||
1935 | case X86::CMPPSrri: | |||
1936 | case X86::VCMPSDrr: | |||
1937 | case X86::VCMPSSrr: | |||
1938 | case X86::VCMPPDrri: | |||
1939 | case X86::VCMPPSrri: | |||
1940 | case X86::VCMPPDYrri: | |||
1941 | case X86::VCMPPSYrri: | |||
1942 | case X86::VCMPSDZrr: | |||
1943 | case X86::VCMPSSZrr: | |||
1944 | case X86::VCMPPDZrri: | |||
1945 | case X86::VCMPPSZrri: | |||
1946 | case X86::VCMPPDZ128rri: | |||
1947 | case X86::VCMPPSZ128rri: | |||
1948 | case X86::VCMPPDZ256rri: | |||
1949 | case X86::VCMPPSZ256rri: { | |||
1950 | // Float comparison can be safely commuted for | |||
1951 | // Ordered/Unordered/Equal/NotEqual tests | |||
1952 | unsigned Imm = MI.getOperand(3).getImm() & 0x7; | |||
1953 | switch (Imm) { | |||
1954 | case 0x00: // EQUAL | |||
1955 | case 0x03: // UNORDERED | |||
1956 | case 0x04: // NOT EQUAL | |||
1957 | case 0x07: // ORDERED | |||
1958 | // The indices of the commutable operands are 1 and 2. | |||
1959 | // Assign them to the returned operand indices here. | |||
1960 | return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2); | |||
1961 | } | |||
1962 | return false; | |||
1963 | } | |||
1964 | case X86::MOVSDrr: | |||
1965 | case X86::MOVSSrr: | |||
1966 | case X86::VMOVSDrr: | |||
1967 | case X86::VMOVSSrr: | |||
1968 | if (Subtarget.hasSSE41()) | |||
1969 | return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
1970 | return false; | |||
1971 | case X86::MOVHLPSrr: | |||
1972 | case X86::UNPCKHPDrr: | |||
1973 | case X86::VMOVHLPSrr: | |||
1974 | case X86::VUNPCKHPDrr: | |||
1975 | case X86::VMOVHLPSZrr: | |||
1976 | case X86::VUNPCKHPDZ128rr: | |||
1977 | if (Subtarget.hasSSE2()) | |||
1978 | return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
1979 | return false; | |||
1980 | case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: | |||
1981 | case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: | |||
1982 | case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: | |||
1983 | case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: | |||
1984 | case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: | |||
1985 | case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: | |||
1986 | case X86::VPTERNLOGDZrrik: | |||
1987 | case X86::VPTERNLOGDZ128rrik: | |||
1988 | case X86::VPTERNLOGDZ256rrik: | |||
1989 | case X86::VPTERNLOGQZrrik: | |||
1990 | case X86::VPTERNLOGQZ128rrik: | |||
1991 | case X86::VPTERNLOGQZ256rrik: | |||
1992 | case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: | |||
1993 | case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: | |||
1994 | case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: | |||
1995 | case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: | |||
1996 | case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: | |||
1997 | case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: | |||
1998 | case X86::VPTERNLOGDZ128rmbi: | |||
1999 | case X86::VPTERNLOGDZ256rmbi: | |||
2000 | case X86::VPTERNLOGDZrmbi: | |||
2001 | case X86::VPTERNLOGQZ128rmbi: | |||
2002 | case X86::VPTERNLOGQZ256rmbi: | |||
2003 | case X86::VPTERNLOGQZrmbi: | |||
2004 | case X86::VPTERNLOGDZ128rmbikz: | |||
2005 | case X86::VPTERNLOGDZ256rmbikz: | |||
2006 | case X86::VPTERNLOGDZrmbikz: | |||
2007 | case X86::VPTERNLOGQZ128rmbikz: | |||
2008 | case X86::VPTERNLOGQZ256rmbikz: | |||
2009 | case X86::VPTERNLOGQZrmbikz: | |||
2010 | return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
2011 | case X86::VPMADD52HUQZ128r: | |||
2012 | case X86::VPMADD52HUQZ128rk: | |||
2013 | case X86::VPMADD52HUQZ128rkz: | |||
2014 | case X86::VPMADD52HUQZ256r: | |||
2015 | case X86::VPMADD52HUQZ256rk: | |||
2016 | case X86::VPMADD52HUQZ256rkz: | |||
2017 | case X86::VPMADD52HUQZr: | |||
2018 | case X86::VPMADD52HUQZrk: | |||
2019 | case X86::VPMADD52HUQZrkz: | |||
2020 | case X86::VPMADD52LUQZ128r: | |||
2021 | case X86::VPMADD52LUQZ128rk: | |||
2022 | case X86::VPMADD52LUQZ128rkz: | |||
2023 | case X86::VPMADD52LUQZ256r: | |||
2024 | case X86::VPMADD52LUQZ256rk: | |||
2025 | case X86::VPMADD52LUQZ256rkz: | |||
2026 | case X86::VPMADD52LUQZr: | |||
2027 | case X86::VPMADD52LUQZrk: | |||
2028 | case X86::VPMADD52LUQZrkz: { | |||
2029 | unsigned CommutableOpIdx1 = 2; | |||
2030 | unsigned CommutableOpIdx2 = 3; | |||
2031 | if (X86II::isKMasked(Desc.TSFlags)) { | |||
2032 | // Skip the mask register. | |||
2033 | ++CommutableOpIdx1; | |||
2034 | ++CommutableOpIdx2; | |||
2035 | } | |||
2036 | if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, | |||
2037 | CommutableOpIdx1, CommutableOpIdx2)) | |||
2038 | return false; | |||
2039 | if (!MI.getOperand(SrcOpIdx1).isReg() || | |||
2040 | !MI.getOperand(SrcOpIdx2).isReg()) | |||
2041 | // No idea. | |||
2042 | return false; | |||
2043 | return true; | |||
2044 | } | |||
2045 | ||||
2046 | default: | |||
2047 | const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), | |||
2048 | MI.getDesc().TSFlags); | |||
2049 | if (FMA3Group) | |||
2050 | return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2, | |||
2051 | FMA3Group->isIntrinsic()); | |||
2052 | ||||
2053 | // Handled masked instructions since we need to skip over the mask input | |||
2054 | // and the preserved input. | |||
2055 | if (X86II::isKMasked(Desc.TSFlags)) { | |||
2056 | // First assume that the first input is the mask operand and skip past it. | |||
2057 | unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1; | |||
2058 | unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2; | |||
2059 | // Check if the first input is tied. If there isn't one then we only | |||
2060 | // need to skip the mask operand which we did above. | |||
2061 | if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(), | |||
2062 | MCOI::TIED_TO) != -1)) { | |||
2063 | // If this is zero masking instruction with a tied operand, we need to | |||
2064 | // move the first index back to the first input since this must | |||
2065 | // be a 3 input instruction and we want the first two non-mask inputs. | |||
2066 | // Otherwise this is a 2 input instruction with a preserved input and | |||
2067 | // mask, so we need to move the indices to skip one more input. | |||
2068 | if (X86II::isKMergeMasked(Desc.TSFlags)) { | |||
2069 | ++CommutableOpIdx1; | |||
2070 | ++CommutableOpIdx2; | |||
2071 | } else { | |||
2072 | --CommutableOpIdx1; | |||
2073 | } | |||
2074 | } | |||
2075 | ||||
2076 | if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, | |||
2077 | CommutableOpIdx1, CommutableOpIdx2)) | |||
2078 | return false; | |||
2079 | ||||
2080 | if (!MI.getOperand(SrcOpIdx1).isReg() || | |||
2081 | !MI.getOperand(SrcOpIdx2).isReg()) | |||
2082 | // No idea. | |||
2083 | return false; | |||
2084 | return true; | |||
2085 | } | |||
2086 | ||||
2087 | return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); | |||
2088 | } | |||
2089 | return false; | |||
2090 | } | |||
2091 | ||||
2092 | X86::CondCode X86::getCondFromBranchOpc(unsigned BrOpc) { | |||
2093 | switch (BrOpc) { | |||
2094 | default: return X86::COND_INVALID; | |||
2095 | case X86::JE_1: return X86::COND_E; | |||
2096 | case X86::JNE_1: return X86::COND_NE; | |||
2097 | case X86::JL_1: return X86::COND_L; | |||
2098 | case X86::JLE_1: return X86::COND_LE; | |||
2099 | case X86::JG_1: return X86::COND_G; | |||
2100 | case X86::JGE_1: return X86::COND_GE; | |||
2101 | case X86::JB_1: return X86::COND_B; | |||
2102 | case X86::JBE_1: return X86::COND_BE; | |||
2103 | case X86::JA_1: return X86::COND_A; | |||
2104 | case X86::JAE_1: return X86::COND_AE; | |||
2105 | case X86::JS_1: return X86::COND_S; | |||
2106 | case X86::JNS_1: return X86::COND_NS; | |||
2107 | case X86::JP_1: return X86::COND_P; | |||
2108 | case X86::JNP_1: return X86::COND_NP; | |||
2109 | case X86::JO_1: return X86::COND_O; | |||
2110 | case X86::JNO_1: return X86::COND_NO; | |||
2111 | } | |||
2112 | } | |||
2113 | ||||
2114 | /// Return condition code of a SET opcode. | |||
2115 | X86::CondCode X86::getCondFromSETOpc(unsigned Opc) { | |||
2116 | switch (Opc) { | |||
2117 | default: return X86::COND_INVALID; | |||
2118 | case X86::SETAr: case X86::SETAm: return X86::COND_A; | |||
2119 | case X86::SETAEr: case X86::SETAEm: return X86::COND_AE; | |||
2120 | case X86::SETBr: case X86::SETBm: return X86::COND_B; | |||
2121 | case X86::SETBEr: case X86::SETBEm: return X86::COND_BE; | |||
2122 | case X86::SETEr: case X86::SETEm: return X86::COND_E; | |||
2123 | case X86::SETGr: case X86::SETGm: return X86::COND_G; | |||
2124 | case X86::SETGEr: case X86::SETGEm: return X86::COND_GE; | |||
2125 | case X86::SETLr: case X86::SETLm: return X86::COND_L; | |||
2126 | case X86::SETLEr: case X86::SETLEm: return X86::COND_LE; | |||
2127 | case X86::SETNEr: case X86::SETNEm: return X86::COND_NE; | |||
2128 | case X86::SETNOr: case X86::SETNOm: return X86::COND_NO; | |||
2129 | case X86::SETNPr: case X86::SETNPm: return X86::COND_NP; | |||
2130 | case X86::SETNSr: case X86::SETNSm: return X86::COND_NS; | |||
2131 | case X86::SETOr: case X86::SETOm: return X86::COND_O; | |||
2132 | case X86::SETPr: case X86::SETPm: return X86::COND_P; | |||
2133 | case X86::SETSr: case X86::SETSm: return X86::COND_S; | |||
2134 | } | |||
2135 | } | |||
2136 | ||||
2137 | /// Return condition code of a CMov opcode. | |||
2138 | X86::CondCode X86::getCondFromCMovOpc(unsigned Opc) { | |||
2139 | switch (Opc) { | |||
2140 | default: return X86::COND_INVALID; | |||
2141 | case X86::CMOVA16rm: case X86::CMOVA16rr: case X86::CMOVA32rm: | |||
2142 | case X86::CMOVA32rr: case X86::CMOVA64rm: case X86::CMOVA64rr: | |||
2143 | return X86::COND_A; | |||
2144 | case X86::CMOVAE16rm: case X86::CMOVAE16rr: case X86::CMOVAE32rm: | |||
2145 | case X86::CMOVAE32rr: case X86::CMOVAE64rm: case X86::CMOVAE64rr: | |||
2146 | return X86::COND_AE; | |||
2147 | case X86::CMOVB16rm: case X86::CMOVB16rr: case X86::CMOVB32rm: | |||
2148 | case X86::CMOVB32rr: case X86::CMOVB64rm: case X86::CMOVB64rr: | |||
2149 | return X86::COND_B; | |||
2150 | case X86::CMOVBE16rm: case X86::CMOVBE16rr: case X86::CMOVBE32rm: | |||
2151 | case X86::CMOVBE32rr: case X86::CMOVBE64rm: case X86::CMOVBE64rr: | |||
2152 | return X86::COND_BE; | |||
2153 | case X86::CMOVE16rm: case X86::CMOVE16rr: case X86::CMOVE32rm: | |||
2154 | case X86::CMOVE32rr: case X86::CMOVE64rm: case X86::CMOVE64rr: | |||
2155 | return X86::COND_E; | |||
2156 | case X86::CMOVG16rm: case X86::CMOVG16rr: case X86::CMOVG32rm: | |||
2157 | case X86::CMOVG32rr: case X86::CMOVG64rm: case X86::CMOVG64rr: | |||
2158 | return X86::COND_G; | |||
2159 | case X86::CMOVGE16rm: case X86::CMOVGE16rr: case X86::CMOVGE32rm: | |||
2160 | case X86::CMOVGE32rr: case X86::CMOVGE64rm: case X86::CMOVGE64rr: | |||
2161 | return X86::COND_GE; | |||
2162 | case X86::CMOVL16rm: case X86::CMOVL16rr: case X86::CMOVL32rm: | |||
2163 | case X86::CMOVL32rr: case X86::CMOVL64rm: case X86::CMOVL64rr: | |||
2164 | return X86::COND_L; | |||
2165 | case X86::CMOVLE16rm: case X86::CMOVLE16rr: case X86::CMOVLE32rm: | |||
2166 | case X86::CMOVLE32rr: case X86::CMOVLE64rm: case X86::CMOVLE64rr: | |||
2167 | return X86::COND_LE; | |||
2168 | case X86::CMOVNE16rm: case X86::CMOVNE16rr: case X86::CMOVNE32rm: | |||
2169 | case X86::CMOVNE32rr: case X86::CMOVNE64rm: case X86::CMOVNE64rr: | |||
2170 | return X86::COND_NE; | |||
2171 | case X86::CMOVNO16rm: case X86::CMOVNO16rr: case X86::CMOVNO32rm: | |||
2172 | case X86::CMOVNO32rr: case X86::CMOVNO64rm: case X86::CMOVNO64rr: | |||
2173 | return X86::COND_NO; | |||
2174 | case X86::CMOVNP16rm: case X86::CMOVNP16rr: case X86::CMOVNP32rm: | |||
2175 | case X86::CMOVNP32rr: case X86::CMOVNP64rm: case X86::CMOVNP64rr: | |||
2176 | return X86::COND_NP; | |||
2177 | case X86::CMOVNS16rm: case X86::CMOVNS16rr: case X86::CMOVNS32rm: | |||
2178 | case X86::CMOVNS32rr: case X86::CMOVNS64rm: case X86::CMOVNS64rr: | |||
2179 | return X86::COND_NS; | |||
2180 | case X86::CMOVO16rm: case X86::CMOVO16rr: case X86::CMOVO32rm: | |||
2181 | case X86::CMOVO32rr: case X86::CMOVO64rm: case X86::CMOVO64rr: | |||
2182 | return X86::COND_O; | |||
2183 | case X86::CMOVP16rm: case X86::CMOVP16rr: case X86::CMOVP32rm: | |||
2184 | case X86::CMOVP32rr: case X86::CMOVP64rm: case X86::CMOVP64rr: | |||
2185 | return X86::COND_P; | |||
2186 | case X86::CMOVS16rm: case X86::CMOVS16rr: case X86::CMOVS32rm: | |||
2187 | case X86::CMOVS32rr: case X86::CMOVS64rm: case X86::CMOVS64rr: | |||
2188 | return X86::COND_S; | |||
2189 | } | |||
2190 | } | |||
2191 | ||||
2192 | unsigned X86::GetCondBranchFromCond(X86::CondCode CC) { | |||
2193 | switch (CC) { | |||
2194 | default: llvm_unreachable("Illegal condition code!")::llvm::llvm_unreachable_internal("Illegal condition code!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2194); | |||
2195 | case X86::COND_E: return X86::JE_1; | |||
2196 | case X86::COND_NE: return X86::JNE_1; | |||
2197 | case X86::COND_L: return X86::JL_1; | |||
2198 | case X86::COND_LE: return X86::JLE_1; | |||
2199 | case X86::COND_G: return X86::JG_1; | |||
2200 | case X86::COND_GE: return X86::JGE_1; | |||
2201 | case X86::COND_B: return X86::JB_1; | |||
2202 | case X86::COND_BE: return X86::JBE_1; | |||
2203 | case X86::COND_A: return X86::JA_1; | |||
2204 | case X86::COND_AE: return X86::JAE_1; | |||
2205 | case X86::COND_S: return X86::JS_1; | |||
2206 | case X86::COND_NS: return X86::JNS_1; | |||
2207 | case X86::COND_P: return X86::JP_1; | |||
2208 | case X86::COND_NP: return X86::JNP_1; | |||
2209 | case X86::COND_O: return X86::JO_1; | |||
2210 | case X86::COND_NO: return X86::JNO_1; | |||
2211 | } | |||
2212 | } | |||
2213 | ||||
2214 | /// Return the inverse of the specified condition, | |||
2215 | /// e.g. turning COND_E to COND_NE. | |||
2216 | X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) { | |||
2217 | switch (CC) { | |||
2218 | default: llvm_unreachable("Illegal condition code!")::llvm::llvm_unreachable_internal("Illegal condition code!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2218); | |||
2219 | case X86::COND_E: return X86::COND_NE; | |||
2220 | case X86::COND_NE: return X86::COND_E; | |||
2221 | case X86::COND_L: return X86::COND_GE; | |||
2222 | case X86::COND_LE: return X86::COND_G; | |||
2223 | case X86::COND_G: return X86::COND_LE; | |||
2224 | case X86::COND_GE: return X86::COND_L; | |||
2225 | case X86::COND_B: return X86::COND_AE; | |||
2226 | case X86::COND_BE: return X86::COND_A; | |||
2227 | case X86::COND_A: return X86::COND_BE; | |||
2228 | case X86::COND_AE: return X86::COND_B; | |||
2229 | case X86::COND_S: return X86::COND_NS; | |||
2230 | case X86::COND_NS: return X86::COND_S; | |||
2231 | case X86::COND_P: return X86::COND_NP; | |||
2232 | case X86::COND_NP: return X86::COND_P; | |||
2233 | case X86::COND_O: return X86::COND_NO; | |||
2234 | case X86::COND_NO: return X86::COND_O; | |||
2235 | case X86::COND_NE_OR_P: return X86::COND_E_AND_NP; | |||
2236 | case X86::COND_E_AND_NP: return X86::COND_NE_OR_P; | |||
2237 | } | |||
2238 | } | |||
2239 | ||||
2240 | /// Assuming the flags are set by MI(a,b), return the condition code if we | |||
2241 | /// modify the instructions such that flags are set by MI(b,a). | |||
2242 | static X86::CondCode getSwappedCondition(X86::CondCode CC) { | |||
2243 | switch (CC) { | |||
2244 | default: return X86::COND_INVALID; | |||
2245 | case X86::COND_E: return X86::COND_E; | |||
2246 | case X86::COND_NE: return X86::COND_NE; | |||
2247 | case X86::COND_L: return X86::COND_G; | |||
2248 | case X86::COND_LE: return X86::COND_GE; | |||
2249 | case X86::COND_G: return X86::COND_L; | |||
2250 | case X86::COND_GE: return X86::COND_LE; | |||
2251 | case X86::COND_B: return X86::COND_A; | |||
2252 | case X86::COND_BE: return X86::COND_AE; | |||
2253 | case X86::COND_A: return X86::COND_B; | |||
2254 | case X86::COND_AE: return X86::COND_BE; | |||
2255 | } | |||
2256 | } | |||
2257 | ||||
2258 | std::pair<X86::CondCode, bool> | |||
2259 | X86::getX86ConditionCode(CmpInst::Predicate Predicate) { | |||
2260 | X86::CondCode CC = X86::COND_INVALID; | |||
2261 | bool NeedSwap = false; | |||
2262 | switch (Predicate) { | |||
2263 | default: break; | |||
2264 | // Floating-point Predicates | |||
2265 | case CmpInst::FCMP_UEQ: CC = X86::COND_E; break; | |||
2266 | case CmpInst::FCMP_OLT: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2267 | case CmpInst::FCMP_OGT: CC = X86::COND_A; break; | |||
2268 | case CmpInst::FCMP_OLE: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2269 | case CmpInst::FCMP_OGE: CC = X86::COND_AE; break; | |||
2270 | case CmpInst::FCMP_UGT: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2271 | case CmpInst::FCMP_ULT: CC = X86::COND_B; break; | |||
2272 | case CmpInst::FCMP_UGE: NeedSwap = true; LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2273 | case CmpInst::FCMP_ULE: CC = X86::COND_BE; break; | |||
2274 | case CmpInst::FCMP_ONE: CC = X86::COND_NE; break; | |||
2275 | case CmpInst::FCMP_UNO: CC = X86::COND_P; break; | |||
2276 | case CmpInst::FCMP_ORD: CC = X86::COND_NP; break; | |||
2277 | case CmpInst::FCMP_OEQ: LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2278 | case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break; | |||
2279 | ||||
2280 | // Integer Predicates | |||
2281 | case CmpInst::ICMP_EQ: CC = X86::COND_E; break; | |||
2282 | case CmpInst::ICMP_NE: CC = X86::COND_NE; break; | |||
2283 | case CmpInst::ICMP_UGT: CC = X86::COND_A; break; | |||
2284 | case CmpInst::ICMP_UGE: CC = X86::COND_AE; break; | |||
2285 | case CmpInst::ICMP_ULT: CC = X86::COND_B; break; | |||
2286 | case CmpInst::ICMP_ULE: CC = X86::COND_BE; break; | |||
2287 | case CmpInst::ICMP_SGT: CC = X86::COND_G; break; | |||
2288 | case CmpInst::ICMP_SGE: CC = X86::COND_GE; break; | |||
2289 | case CmpInst::ICMP_SLT: CC = X86::COND_L; break; | |||
2290 | case CmpInst::ICMP_SLE: CC = X86::COND_LE; break; | |||
2291 | } | |||
2292 | ||||
2293 | return std::make_pair(CC, NeedSwap); | |||
2294 | } | |||
2295 | ||||
2296 | /// Return a set opcode for the given condition and | |||
2297 | /// whether it has memory operand. | |||
2298 | unsigned X86::getSETFromCond(CondCode CC, bool HasMemoryOperand) { | |||
2299 | static const uint16_t Opc[16][2] = { | |||
2300 | { X86::SETAr, X86::SETAm }, | |||
2301 | { X86::SETAEr, X86::SETAEm }, | |||
2302 | { X86::SETBr, X86::SETBm }, | |||
2303 | { X86::SETBEr, X86::SETBEm }, | |||
2304 | { X86::SETEr, X86::SETEm }, | |||
2305 | { X86::SETGr, X86::SETGm }, | |||
2306 | { X86::SETGEr, X86::SETGEm }, | |||
2307 | { X86::SETLr, X86::SETLm }, | |||
2308 | { X86::SETLEr, X86::SETLEm }, | |||
2309 | { X86::SETNEr, X86::SETNEm }, | |||
2310 | { X86::SETNOr, X86::SETNOm }, | |||
2311 | { X86::SETNPr, X86::SETNPm }, | |||
2312 | { X86::SETNSr, X86::SETNSm }, | |||
2313 | { X86::SETOr, X86::SETOm }, | |||
2314 | { X86::SETPr, X86::SETPm }, | |||
2315 | { X86::SETSr, X86::SETSm } | |||
2316 | }; | |||
2317 | ||||
2318 | assert(CC <= LAST_VALID_COND && "Can only handle standard cond codes")((CC <= LAST_VALID_COND && "Can only handle standard cond codes" ) ? static_cast<void> (0) : __assert_fail ("CC <= LAST_VALID_COND && \"Can only handle standard cond codes\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2318, __PRETTY_FUNCTION__)); | |||
2319 | return Opc[CC][HasMemoryOperand ? 1 : 0]; | |||
2320 | } | |||
2321 | ||||
2322 | /// Return a cmov opcode for the given condition, | |||
2323 | /// register size in bytes, and operand type. | |||
2324 | unsigned X86::getCMovFromCond(CondCode CC, unsigned RegBytes, | |||
2325 | bool HasMemoryOperand) { | |||
2326 | static const uint16_t Opc[32][3] = { | |||
2327 | { X86::CMOVA16rr, X86::CMOVA32rr, X86::CMOVA64rr }, | |||
2328 | { X86::CMOVAE16rr, X86::CMOVAE32rr, X86::CMOVAE64rr }, | |||
2329 | { X86::CMOVB16rr, X86::CMOVB32rr, X86::CMOVB64rr }, | |||
2330 | { X86::CMOVBE16rr, X86::CMOVBE32rr, X86::CMOVBE64rr }, | |||
2331 | { X86::CMOVE16rr, X86::CMOVE32rr, X86::CMOVE64rr }, | |||
2332 | { X86::CMOVG16rr, X86::CMOVG32rr, X86::CMOVG64rr }, | |||
2333 | { X86::CMOVGE16rr, X86::CMOVGE32rr, X86::CMOVGE64rr }, | |||
2334 | { X86::CMOVL16rr, X86::CMOVL32rr, X86::CMOVL64rr }, | |||
2335 | { X86::CMOVLE16rr, X86::CMOVLE32rr, X86::CMOVLE64rr }, | |||
2336 | { X86::CMOVNE16rr, X86::CMOVNE32rr, X86::CMOVNE64rr }, | |||
2337 | { X86::CMOVNO16rr, X86::CMOVNO32rr, X86::CMOVNO64rr }, | |||
2338 | { X86::CMOVNP16rr, X86::CMOVNP32rr, X86::CMOVNP64rr }, | |||
2339 | { X86::CMOVNS16rr, X86::CMOVNS32rr, X86::CMOVNS64rr }, | |||
2340 | { X86::CMOVO16rr, X86::CMOVO32rr, X86::CMOVO64rr }, | |||
2341 | { X86::CMOVP16rr, X86::CMOVP32rr, X86::CMOVP64rr }, | |||
2342 | { X86::CMOVS16rr, X86::CMOVS32rr, X86::CMOVS64rr }, | |||
2343 | { X86::CMOVA16rm, X86::CMOVA32rm, X86::CMOVA64rm }, | |||
2344 | { X86::CMOVAE16rm, X86::CMOVAE32rm, X86::CMOVAE64rm }, | |||
2345 | { X86::CMOVB16rm, X86::CMOVB32rm, X86::CMOVB64rm }, | |||
2346 | { X86::CMOVBE16rm, X86::CMOVBE32rm, X86::CMOVBE64rm }, | |||
2347 | { X86::CMOVE16rm, X86::CMOVE32rm, X86::CMOVE64rm }, | |||
2348 | { X86::CMOVG16rm, X86::CMOVG32rm, X86::CMOVG64rm }, | |||
2349 | { X86::CMOVGE16rm, X86::CMOVGE32rm, X86::CMOVGE64rm }, | |||
2350 | { X86::CMOVL16rm, X86::CMOVL32rm, X86::CMOVL64rm }, | |||
2351 | { X86::CMOVLE16rm, X86::CMOVLE32rm, X86::CMOVLE64rm }, | |||
2352 | { X86::CMOVNE16rm, X86::CMOVNE32rm, X86::CMOVNE64rm }, | |||
2353 | { X86::CMOVNO16rm, X86::CMOVNO32rm, X86::CMOVNO64rm }, | |||
2354 | { X86::CMOVNP16rm, X86::CMOVNP32rm, X86::CMOVNP64rm }, | |||
2355 | { X86::CMOVNS16rm, X86::CMOVNS32rm, X86::CMOVNS64rm }, | |||
2356 | { X86::CMOVO16rm, X86::CMOVO32rm, X86::CMOVO64rm }, | |||
2357 | { X86::CMOVP16rm, X86::CMOVP32rm, X86::CMOVP64rm }, | |||
2358 | { X86::CMOVS16rm, X86::CMOVS32rm, X86::CMOVS64rm } | |||
2359 | }; | |||
2360 | ||||
2361 | assert(CC < 16 && "Can only handle standard cond codes")((CC < 16 && "Can only handle standard cond codes" ) ? static_cast<void> (0) : __assert_fail ("CC < 16 && \"Can only handle standard cond codes\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2361, __PRETTY_FUNCTION__)); | |||
2362 | unsigned Idx = HasMemoryOperand ? 16+CC : CC; | |||
2363 | switch(RegBytes) { | |||
2364 | default: llvm_unreachable("Illegal register size!")::llvm::llvm_unreachable_internal("Illegal register size!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2364); | |||
2365 | case 2: return Opc[Idx][0]; | |||
2366 | case 4: return Opc[Idx][1]; | |||
2367 | case 8: return Opc[Idx][2]; | |||
2368 | } | |||
2369 | } | |||
2370 | ||||
2371 | /// Get the VPCMP immediate for the given condition. | |||
2372 | unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) { | |||
2373 | switch (CC) { | |||
2374 | default: llvm_unreachable("Unexpected SETCC condition")::llvm::llvm_unreachable_internal("Unexpected SETCC condition" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2374); | |||
2375 | case ISD::SETNE: return 4; | |||
2376 | case ISD::SETEQ: return 0; | |||
2377 | case ISD::SETULT: | |||
2378 | case ISD::SETLT: return 1; | |||
2379 | case ISD::SETUGT: | |||
2380 | case ISD::SETGT: return 6; | |||
2381 | case ISD::SETUGE: | |||
2382 | case ISD::SETGE: return 5; | |||
2383 | case ISD::SETULE: | |||
2384 | case ISD::SETLE: return 2; | |||
2385 | } | |||
2386 | } | |||
2387 | ||||
2388 | /// Get the VPCMP immediate if the opcodes are swapped. | |||
2389 | unsigned X86::getSwappedVPCMPImm(unsigned Imm) { | |||
2390 | switch (Imm) { | |||
2391 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2391); | |||
2392 | case 0x01: Imm = 0x06; break; // LT -> NLE | |||
2393 | case 0x02: Imm = 0x05; break; // LE -> NLT | |||
2394 | case 0x05: Imm = 0x02; break; // NLT -> LE | |||
2395 | case 0x06: Imm = 0x01; break; // NLE -> LT | |||
2396 | case 0x00: // EQ | |||
2397 | case 0x03: // FALSE | |||
2398 | case 0x04: // NE | |||
2399 | case 0x07: // TRUE | |||
2400 | break; | |||
2401 | } | |||
2402 | ||||
2403 | return Imm; | |||
2404 | } | |||
2405 | ||||
2406 | /// Get the VPCOM immediate if the opcodes are swapped. | |||
2407 | unsigned X86::getSwappedVPCOMImm(unsigned Imm) { | |||
2408 | switch (Imm) { | |||
2409 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2409); | |||
2410 | case 0x00: Imm = 0x02; break; // LT -> GT | |||
2411 | case 0x01: Imm = 0x03; break; // LE -> GE | |||
2412 | case 0x02: Imm = 0x00; break; // GT -> LT | |||
2413 | case 0x03: Imm = 0x01; break; // GE -> LE | |||
2414 | case 0x04: // EQ | |||
2415 | case 0x05: // NE | |||
2416 | case 0x06: // FALSE | |||
2417 | case 0x07: // TRUE | |||
2418 | break; | |||
2419 | } | |||
2420 | ||||
2421 | return Imm; | |||
2422 | } | |||
2423 | ||||
2424 | bool X86InstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const { | |||
2425 | if (!MI.isTerminator()) return false; | |||
2426 | ||||
2427 | // Conditional branch is a special case. | |||
2428 | if (MI.isBranch() && !MI.isBarrier()) | |||
2429 | return true; | |||
2430 | if (!MI.isPredicable()) | |||
2431 | return true; | |||
2432 | return !isPredicated(MI); | |||
2433 | } | |||
2434 | ||||
2435 | bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const { | |||
2436 | switch (MI.getOpcode()) { | |||
2437 | case X86::TCRETURNdi: | |||
2438 | case X86::TCRETURNri: | |||
2439 | case X86::TCRETURNmi: | |||
2440 | case X86::TCRETURNdi64: | |||
2441 | case X86::TCRETURNri64: | |||
2442 | case X86::TCRETURNmi64: | |||
2443 | return true; | |||
2444 | default: | |||
2445 | return false; | |||
2446 | } | |||
2447 | } | |||
2448 | ||||
2449 | bool X86InstrInfo::canMakeTailCallConditional( | |||
2450 | SmallVectorImpl<MachineOperand> &BranchCond, | |||
2451 | const MachineInstr &TailCall) const { | |||
2452 | if (TailCall.getOpcode() != X86::TCRETURNdi && | |||
2453 | TailCall.getOpcode() != X86::TCRETURNdi64) { | |||
2454 | // Only direct calls can be done with a conditional branch. | |||
2455 | return false; | |||
2456 | } | |||
2457 | ||||
2458 | const MachineFunction *MF = TailCall.getParent()->getParent(); | |||
2459 | if (Subtarget.isTargetWin64() && MF->hasWinCFI()) { | |||
2460 | // Conditional tail calls confuse the Win64 unwinder. | |||
2461 | return false; | |||
2462 | } | |||
2463 | ||||
2464 | assert(BranchCond.size() == 1)((BranchCond.size() == 1) ? static_cast<void> (0) : __assert_fail ("BranchCond.size() == 1", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2464, __PRETTY_FUNCTION__)); | |||
2465 | if (BranchCond[0].getImm() > X86::LAST_VALID_COND) { | |||
2466 | // Can't make a conditional tail call with this condition. | |||
2467 | return false; | |||
2468 | } | |||
2469 | ||||
2470 | const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); | |||
2471 | if (X86FI->getTCReturnAddrDelta() != 0 || | |||
2472 | TailCall.getOperand(1).getImm() != 0) { | |||
2473 | // A conditional tail call cannot do any stack adjustment. | |||
2474 | return false; | |||
2475 | } | |||
2476 | ||||
2477 | return true; | |||
2478 | } | |||
2479 | ||||
2480 | void X86InstrInfo::replaceBranchWithTailCall( | |||
2481 | MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond, | |||
2482 | const MachineInstr &TailCall) const { | |||
2483 | assert(canMakeTailCallConditional(BranchCond, TailCall))((canMakeTailCallConditional(BranchCond, TailCall)) ? static_cast <void> (0) : __assert_fail ("canMakeTailCallConditional(BranchCond, TailCall)" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2483, __PRETTY_FUNCTION__)); | |||
2484 | ||||
2485 | MachineBasicBlock::iterator I = MBB.end(); | |||
2486 | while (I != MBB.begin()) { | |||
2487 | --I; | |||
2488 | if (I->isDebugInstr()) | |||
2489 | continue; | |||
2490 | if (!I->isBranch()) | |||
2491 | assert(0 && "Can't find the branch to replace!")((0 && "Can't find the branch to replace!") ? static_cast <void> (0) : __assert_fail ("0 && \"Can't find the branch to replace!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2491, __PRETTY_FUNCTION__)); | |||
2492 | ||||
2493 | X86::CondCode CC = X86::getCondFromBranchOpc(I->getOpcode()); | |||
2494 | assert(BranchCond.size() == 1)((BranchCond.size() == 1) ? static_cast<void> (0) : __assert_fail ("BranchCond.size() == 1", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2494, __PRETTY_FUNCTION__)); | |||
2495 | if (CC != BranchCond[0].getImm()) | |||
2496 | continue; | |||
2497 | ||||
2498 | break; | |||
2499 | } | |||
2500 | ||||
2501 | unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc | |||
2502 | : X86::TCRETURNdi64cc; | |||
2503 | ||||
2504 | auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc)); | |||
2505 | MIB->addOperand(TailCall.getOperand(0)); // Destination. | |||
2506 | MIB.addImm(0); // Stack offset (not used). | |||
2507 | MIB->addOperand(BranchCond[0]); // Condition. | |||
2508 | MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters. | |||
2509 | ||||
2510 | // Add implicit uses and defs of all live regs potentially clobbered by the | |||
2511 | // call. This way they still appear live across the call. | |||
2512 | LivePhysRegs LiveRegs(getRegisterInfo()); | |||
2513 | LiveRegs.addLiveOuts(MBB); | |||
2514 | SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers; | |||
2515 | LiveRegs.stepForward(*MIB, Clobbers); | |||
2516 | for (const auto &C : Clobbers) { | |||
2517 | MIB.addReg(C.first, RegState::Implicit); | |||
2518 | MIB.addReg(C.first, RegState::Implicit | RegState::Define); | |||
2519 | } | |||
2520 | ||||
2521 | I->eraseFromParent(); | |||
2522 | } | |||
2523 | ||||
2524 | // Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may | |||
2525 | // not be a fallthrough MBB now due to layout changes). Return nullptr if the | |||
2526 | // fallthrough MBB cannot be identified. | |||
2527 | static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB, | |||
2528 | MachineBasicBlock *TBB) { | |||
2529 | // Look for non-EHPad successors other than TBB. If we find exactly one, it | |||
2530 | // is the fallthrough MBB. If we find zero, then TBB is both the target MBB | |||
2531 | // and fallthrough MBB. If we find more than one, we cannot identify the | |||
2532 | // fallthrough MBB and should return nullptr. | |||
2533 | MachineBasicBlock *FallthroughBB = nullptr; | |||
2534 | for (auto SI = MBB->succ_begin(), SE = MBB->succ_end(); SI != SE; ++SI) { | |||
2535 | if ((*SI)->isEHPad() || (*SI == TBB && FallthroughBB)) | |||
2536 | continue; | |||
2537 | // Return a nullptr if we found more than one fallthrough successor. | |||
2538 | if (FallthroughBB && FallthroughBB != TBB) | |||
2539 | return nullptr; | |||
2540 | FallthroughBB = *SI; | |||
2541 | } | |||
2542 | return FallthroughBB; | |||
2543 | } | |||
2544 | ||||
2545 | bool X86InstrInfo::AnalyzeBranchImpl( | |||
2546 | MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, | |||
2547 | SmallVectorImpl<MachineOperand> &Cond, | |||
2548 | SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const { | |||
2549 | ||||
2550 | // Start from the bottom of the block and work up, examining the | |||
2551 | // terminator instructions. | |||
2552 | MachineBasicBlock::iterator I = MBB.end(); | |||
2553 | MachineBasicBlock::iterator UnCondBrIter = MBB.end(); | |||
2554 | while (I != MBB.begin()) { | |||
2555 | --I; | |||
2556 | if (I->isDebugInstr()) | |||
2557 | continue; | |||
2558 | ||||
2559 | // Working from the bottom, when we see a non-terminator instruction, we're | |||
2560 | // done. | |||
2561 | if (!isUnpredicatedTerminator(*I)) | |||
2562 | break; | |||
2563 | ||||
2564 | // A terminator that isn't a branch can't easily be handled by this | |||
2565 | // analysis. | |||
2566 | if (!I->isBranch()) | |||
2567 | return true; | |||
2568 | ||||
2569 | // Handle unconditional branches. | |||
2570 | if (I->getOpcode() == X86::JMP_1) { | |||
2571 | UnCondBrIter = I; | |||
2572 | ||||
2573 | if (!AllowModify) { | |||
2574 | TBB = I->getOperand(0).getMBB(); | |||
2575 | continue; | |||
2576 | } | |||
2577 | ||||
2578 | // If the block has any instructions after a JMP, delete them. | |||
2579 | while (std::next(I) != MBB.end()) | |||
2580 | std::next(I)->eraseFromParent(); | |||
2581 | ||||
2582 | Cond.clear(); | |||
2583 | FBB = nullptr; | |||
2584 | ||||
2585 | // Delete the JMP if it's equivalent to a fall-through. | |||
2586 | if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { | |||
2587 | TBB = nullptr; | |||
2588 | I->eraseFromParent(); | |||
2589 | I = MBB.end(); | |||
2590 | UnCondBrIter = MBB.end(); | |||
2591 | continue; | |||
2592 | } | |||
2593 | ||||
2594 | // TBB is used to indicate the unconditional destination. | |||
2595 | TBB = I->getOperand(0).getMBB(); | |||
2596 | continue; | |||
2597 | } | |||
2598 | ||||
2599 | // Handle conditional branches. | |||
2600 | X86::CondCode BranchCode = X86::getCondFromBranchOpc(I->getOpcode()); | |||
2601 | if (BranchCode == X86::COND_INVALID) | |||
2602 | return true; // Can't handle indirect branch. | |||
2603 | ||||
2604 | // In practice we should never have an undef eflags operand, if we do | |||
2605 | // abort here as we are not prepared to preserve the flag. | |||
2606 | if (I->getOperand(1).isUndef()) | |||
2607 | return true; | |||
2608 | ||||
2609 | // Working from the bottom, handle the first conditional branch. | |||
2610 | if (Cond.empty()) { | |||
2611 | MachineBasicBlock *TargetBB = I->getOperand(0).getMBB(); | |||
2612 | if (AllowModify && UnCondBrIter != MBB.end() && | |||
2613 | MBB.isLayoutSuccessor(TargetBB)) { | |||
2614 | // If we can modify the code and it ends in something like: | |||
2615 | // | |||
2616 | // jCC L1 | |||
2617 | // jmp L2 | |||
2618 | // L1: | |||
2619 | // ... | |||
2620 | // L2: | |||
2621 | // | |||
2622 | // Then we can change this to: | |||
2623 | // | |||
2624 | // jnCC L2 | |||
2625 | // L1: | |||
2626 | // ... | |||
2627 | // L2: | |||
2628 | // | |||
2629 | // Which is a bit more efficient. | |||
2630 | // We conditionally jump to the fall-through block. | |||
2631 | BranchCode = GetOppositeBranchCondition(BranchCode); | |||
2632 | unsigned JNCC = GetCondBranchFromCond(BranchCode); | |||
2633 | MachineBasicBlock::iterator OldInst = I; | |||
2634 | ||||
2635 | BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(JNCC)) | |||
2636 | .addMBB(UnCondBrIter->getOperand(0).getMBB()); | |||
2637 | BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(X86::JMP_1)) | |||
2638 | .addMBB(TargetBB); | |||
2639 | ||||
2640 | OldInst->eraseFromParent(); | |||
2641 | UnCondBrIter->eraseFromParent(); | |||
2642 | ||||
2643 | // Restart the analysis. | |||
2644 | UnCondBrIter = MBB.end(); | |||
2645 | I = MBB.end(); | |||
2646 | continue; | |||
2647 | } | |||
2648 | ||||
2649 | FBB = TBB; | |||
2650 | TBB = I->getOperand(0).getMBB(); | |||
2651 | Cond.push_back(MachineOperand::CreateImm(BranchCode)); | |||
2652 | CondBranches.push_back(&*I); | |||
2653 | continue; | |||
2654 | } | |||
2655 | ||||
2656 | // Handle subsequent conditional branches. Only handle the case where all | |||
2657 | // conditional branches branch to the same destination and their condition | |||
2658 | // opcodes fit one of the special multi-branch idioms. | |||
2659 | assert(Cond.size() == 1)((Cond.size() == 1) ? static_cast<void> (0) : __assert_fail ("Cond.size() == 1", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2659, __PRETTY_FUNCTION__)); | |||
2660 | assert(TBB)((TBB) ? static_cast<void> (0) : __assert_fail ("TBB", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2660, __PRETTY_FUNCTION__)); | |||
2661 | ||||
2662 | // If the conditions are the same, we can leave them alone. | |||
2663 | X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm(); | |||
2664 | auto NewTBB = I->getOperand(0).getMBB(); | |||
2665 | if (OldBranchCode == BranchCode && TBB == NewTBB) | |||
2666 | continue; | |||
2667 | ||||
2668 | // If they differ, see if they fit one of the known patterns. Theoretically, | |||
2669 | // we could handle more patterns here, but we shouldn't expect to see them | |||
2670 | // if instruction selection has done a reasonable job. | |||
2671 | if (TBB == NewTBB && | |||
2672 | ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) || | |||
2673 | (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) { | |||
2674 | BranchCode = X86::COND_NE_OR_P; | |||
2675 | } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) || | |||
2676 | (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) { | |||
2677 | if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB))) | |||
2678 | return true; | |||
2679 | ||||
2680 | // X86::COND_E_AND_NP usually has two different branch destinations. | |||
2681 | // | |||
2682 | // JP B1 | |||
2683 | // JE B2 | |||
2684 | // JMP B1 | |||
2685 | // B1: | |||
2686 | // B2: | |||
2687 | // | |||
2688 | // Here this condition branches to B2 only if NP && E. It has another | |||
2689 | // equivalent form: | |||
2690 | // | |||
2691 | // JNE B1 | |||
2692 | // JNP B2 | |||
2693 | // JMP B1 | |||
2694 | // B1: | |||
2695 | // B2: | |||
2696 | // | |||
2697 | // Similarly it branches to B2 only if E && NP. That is why this condition | |||
2698 | // is named with COND_E_AND_NP. | |||
2699 | BranchCode = X86::COND_E_AND_NP; | |||
2700 | } else | |||
2701 | return true; | |||
2702 | ||||
2703 | // Update the MachineOperand. | |||
2704 | Cond[0].setImm(BranchCode); | |||
2705 | CondBranches.push_back(&*I); | |||
2706 | } | |||
2707 | ||||
2708 | return false; | |||
2709 | } | |||
2710 | ||||
2711 | bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB, | |||
2712 | MachineBasicBlock *&TBB, | |||
2713 | MachineBasicBlock *&FBB, | |||
2714 | SmallVectorImpl<MachineOperand> &Cond, | |||
2715 | bool AllowModify) const { | |||
2716 | SmallVector<MachineInstr *, 4> CondBranches; | |||
2717 | return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify); | |||
2718 | } | |||
2719 | ||||
2720 | bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB, | |||
2721 | MachineBranchPredicate &MBP, | |||
2722 | bool AllowModify) const { | |||
2723 | using namespace std::placeholders; | |||
2724 | ||||
2725 | SmallVector<MachineOperand, 4> Cond; | |||
2726 | SmallVector<MachineInstr *, 4> CondBranches; | |||
2727 | if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches, | |||
2728 | AllowModify)) | |||
2729 | return true; | |||
2730 | ||||
2731 | if (Cond.size() != 1) | |||
2732 | return true; | |||
2733 | ||||
2734 | assert(MBP.TrueDest && "expected!")((MBP.TrueDest && "expected!") ? static_cast<void> (0) : __assert_fail ("MBP.TrueDest && \"expected!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2734, __PRETTY_FUNCTION__)); | |||
2735 | ||||
2736 | if (!MBP.FalseDest) | |||
2737 | MBP.FalseDest = MBB.getNextNode(); | |||
2738 | ||||
2739 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
2740 | ||||
2741 | MachineInstr *ConditionDef = nullptr; | |||
2742 | bool SingleUseCondition = true; | |||
2743 | ||||
2744 | for (auto I = std::next(MBB.rbegin()), E = MBB.rend(); I != E; ++I) { | |||
2745 | if (I->modifiesRegister(X86::EFLAGS, TRI)) { | |||
2746 | ConditionDef = &*I; | |||
2747 | break; | |||
2748 | } | |||
2749 | ||||
2750 | if (I->readsRegister(X86::EFLAGS, TRI)) | |||
2751 | SingleUseCondition = false; | |||
2752 | } | |||
2753 | ||||
2754 | if (!ConditionDef) | |||
2755 | return true; | |||
2756 | ||||
2757 | if (SingleUseCondition) { | |||
2758 | for (auto *Succ : MBB.successors()) | |||
2759 | if (Succ->isLiveIn(X86::EFLAGS)) | |||
2760 | SingleUseCondition = false; | |||
2761 | } | |||
2762 | ||||
2763 | MBP.ConditionDef = ConditionDef; | |||
2764 | MBP.SingleUseCondition = SingleUseCondition; | |||
2765 | ||||
2766 | // Currently we only recognize the simple pattern: | |||
2767 | // | |||
2768 | // test %reg, %reg | |||
2769 | // je %label | |||
2770 | // | |||
2771 | const unsigned TestOpcode = | |||
2772 | Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr; | |||
2773 | ||||
2774 | if (ConditionDef->getOpcode() == TestOpcode && | |||
2775 | ConditionDef->getNumOperands() == 3 && | |||
2776 | ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) && | |||
2777 | (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) { | |||
2778 | MBP.LHS = ConditionDef->getOperand(0); | |||
2779 | MBP.RHS = MachineOperand::CreateImm(0); | |||
2780 | MBP.Predicate = Cond[0].getImm() == X86::COND_NE | |||
2781 | ? MachineBranchPredicate::PRED_NE | |||
2782 | : MachineBranchPredicate::PRED_EQ; | |||
2783 | return false; | |||
2784 | } | |||
2785 | ||||
2786 | return true; | |||
2787 | } | |||
2788 | ||||
2789 | unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB, | |||
2790 | int *BytesRemoved) const { | |||
2791 | assert(!BytesRemoved && "code size not handled")((!BytesRemoved && "code size not handled") ? static_cast <void> (0) : __assert_fail ("!BytesRemoved && \"code size not handled\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2791, __PRETTY_FUNCTION__)); | |||
2792 | ||||
2793 | MachineBasicBlock::iterator I = MBB.end(); | |||
2794 | unsigned Count = 0; | |||
2795 | ||||
2796 | while (I != MBB.begin()) { | |||
2797 | --I; | |||
2798 | if (I->isDebugInstr()) | |||
2799 | continue; | |||
2800 | if (I->getOpcode() != X86::JMP_1 && | |||
2801 | X86::getCondFromBranchOpc(I->getOpcode()) == X86::COND_INVALID) | |||
2802 | break; | |||
2803 | // Remove the branch. | |||
2804 | I->eraseFromParent(); | |||
2805 | I = MBB.end(); | |||
2806 | ++Count; | |||
2807 | } | |||
2808 | ||||
2809 | return Count; | |||
2810 | } | |||
2811 | ||||
2812 | unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB, | |||
2813 | MachineBasicBlock *TBB, | |||
2814 | MachineBasicBlock *FBB, | |||
2815 | ArrayRef<MachineOperand> Cond, | |||
2816 | const DebugLoc &DL, | |||
2817 | int *BytesAdded) const { | |||
2818 | // Shouldn't be a fall through. | |||
2819 | assert(TBB && "insertBranch must not be told to insert a fallthrough")((TBB && "insertBranch must not be told to insert a fallthrough" ) ? static_cast<void> (0) : __assert_fail ("TBB && \"insertBranch must not be told to insert a fallthrough\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2819, __PRETTY_FUNCTION__)); | |||
2820 | assert((Cond.size() == 1 || Cond.size() == 0) &&(((Cond.size() == 1 || Cond.size() == 0) && "X86 branch conditions have one component!" ) ? static_cast<void> (0) : __assert_fail ("(Cond.size() == 1 || Cond.size() == 0) && \"X86 branch conditions have one component!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2821, __PRETTY_FUNCTION__)) | |||
2821 | "X86 branch conditions have one component!")(((Cond.size() == 1 || Cond.size() == 0) && "X86 branch conditions have one component!" ) ? static_cast<void> (0) : __assert_fail ("(Cond.size() == 1 || Cond.size() == 0) && \"X86 branch conditions have one component!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2821, __PRETTY_FUNCTION__)); | |||
2822 | assert(!BytesAdded && "code size not handled")((!BytesAdded && "code size not handled") ? static_cast <void> (0) : __assert_fail ("!BytesAdded && \"code size not handled\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2822, __PRETTY_FUNCTION__)); | |||
2823 | ||||
2824 | if (Cond.empty()) { | |||
2825 | // Unconditional branch? | |||
2826 | assert(!FBB && "Unconditional branch with multiple successors!")((!FBB && "Unconditional branch with multiple successors!" ) ? static_cast<void> (0) : __assert_fail ("!FBB && \"Unconditional branch with multiple successors!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2826, __PRETTY_FUNCTION__)); | |||
2827 | BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB); | |||
2828 | return 1; | |||
2829 | } | |||
2830 | ||||
2831 | // If FBB is null, it is implied to be a fall-through block. | |||
2832 | bool FallThru = FBB == nullptr; | |||
2833 | ||||
2834 | // Conditional branch. | |||
2835 | unsigned Count = 0; | |||
2836 | X86::CondCode CC = (X86::CondCode)Cond[0].getImm(); | |||
2837 | switch (CC) { | |||
2838 | case X86::COND_NE_OR_P: | |||
2839 | // Synthesize NE_OR_P with two branches. | |||
2840 | BuildMI(&MBB, DL, get(X86::JNE_1)).addMBB(TBB); | |||
2841 | ++Count; | |||
2842 | BuildMI(&MBB, DL, get(X86::JP_1)).addMBB(TBB); | |||
2843 | ++Count; | |||
2844 | break; | |||
2845 | case X86::COND_E_AND_NP: | |||
2846 | // Use the next block of MBB as FBB if it is null. | |||
2847 | if (FBB == nullptr) { | |||
2848 | FBB = getFallThroughMBB(&MBB, TBB); | |||
2849 | assert(FBB && "MBB cannot be the last block in function when the false "((FBB && "MBB cannot be the last block in function when the false " "body is a fall-through.") ? static_cast<void> (0) : __assert_fail ("FBB && \"MBB cannot be the last block in function when the false \" \"body is a fall-through.\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2850, __PRETTY_FUNCTION__)) | |||
2850 | "body is a fall-through.")((FBB && "MBB cannot be the last block in function when the false " "body is a fall-through.") ? static_cast<void> (0) : __assert_fail ("FBB && \"MBB cannot be the last block in function when the false \" \"body is a fall-through.\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2850, __PRETTY_FUNCTION__)); | |||
2851 | } | |||
2852 | // Synthesize COND_E_AND_NP with two branches. | |||
2853 | BuildMI(&MBB, DL, get(X86::JNE_1)).addMBB(FBB); | |||
2854 | ++Count; | |||
2855 | BuildMI(&MBB, DL, get(X86::JNP_1)).addMBB(TBB); | |||
2856 | ++Count; | |||
2857 | break; | |||
2858 | default: { | |||
2859 | unsigned Opc = GetCondBranchFromCond(CC); | |||
2860 | BuildMI(&MBB, DL, get(Opc)).addMBB(TBB); | |||
2861 | ++Count; | |||
2862 | } | |||
2863 | } | |||
2864 | if (!FallThru) { | |||
2865 | // Two-way Conditional branch. Insert the second branch. | |||
2866 | BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB); | |||
2867 | ++Count; | |||
2868 | } | |||
2869 | return Count; | |||
2870 | } | |||
2871 | ||||
2872 | bool X86InstrInfo:: | |||
2873 | canInsertSelect(const MachineBasicBlock &MBB, | |||
2874 | ArrayRef<MachineOperand> Cond, | |||
2875 | unsigned TrueReg, unsigned FalseReg, | |||
2876 | int &CondCycles, int &TrueCycles, int &FalseCycles) const { | |||
2877 | // Not all subtargets have cmov instructions. | |||
2878 | if (!Subtarget.hasCMov()) | |||
2879 | return false; | |||
2880 | if (Cond.size() != 1) | |||
2881 | return false; | |||
2882 | // We cannot do the composite conditions, at least not in SSA form. | |||
2883 | if ((X86::CondCode)Cond[0].getImm() > X86::COND_S) | |||
2884 | return false; | |||
2885 | ||||
2886 | // Check register classes. | |||
2887 | const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); | |||
2888 | const TargetRegisterClass *RC = | |||
2889 | RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); | |||
2890 | if (!RC) | |||
2891 | return false; | |||
2892 | ||||
2893 | // We have cmov instructions for 16, 32, and 64 bit general purpose registers. | |||
2894 | if (X86::GR16RegClass.hasSubClassEq(RC) || | |||
2895 | X86::GR32RegClass.hasSubClassEq(RC) || | |||
2896 | X86::GR64RegClass.hasSubClassEq(RC)) { | |||
2897 | // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy | |||
2898 | // Bridge. Probably Ivy Bridge as well. | |||
2899 | CondCycles = 2; | |||
2900 | TrueCycles = 2; | |||
2901 | FalseCycles = 2; | |||
2902 | return true; | |||
2903 | } | |||
2904 | ||||
2905 | // Can't do vectors. | |||
2906 | return false; | |||
2907 | } | |||
2908 | ||||
2909 | void X86InstrInfo::insertSelect(MachineBasicBlock &MBB, | |||
2910 | MachineBasicBlock::iterator I, | |||
2911 | const DebugLoc &DL, unsigned DstReg, | |||
2912 | ArrayRef<MachineOperand> Cond, unsigned TrueReg, | |||
2913 | unsigned FalseReg) const { | |||
2914 | MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); | |||
2915 | const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); | |||
2916 | const TargetRegisterClass &RC = *MRI.getRegClass(DstReg); | |||
2917 | assert(Cond.size() == 1 && "Invalid Cond array")((Cond.size() == 1 && "Invalid Cond array") ? static_cast <void> (0) : __assert_fail ("Cond.size() == 1 && \"Invalid Cond array\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2917, __PRETTY_FUNCTION__)); | |||
2918 | unsigned Opc = getCMovFromCond((X86::CondCode)Cond[0].getImm(), | |||
2919 | TRI.getRegSizeInBits(RC) / 8, | |||
2920 | false /*HasMemoryOperand*/); | |||
2921 | BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(FalseReg).addReg(TrueReg); | |||
2922 | } | |||
2923 | ||||
2924 | /// Test if the given register is a physical h register. | |||
2925 | static bool isHReg(unsigned Reg) { | |||
2926 | return X86::GR8_ABCD_HRegClass.contains(Reg); | |||
2927 | } | |||
2928 | ||||
2929 | // Try and copy between VR128/VR64 and GR64 registers. | |||
2930 | static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, | |||
2931 | const X86Subtarget &Subtarget) { | |||
2932 | bool HasAVX = Subtarget.hasAVX(); | |||
2933 | bool HasAVX512 = Subtarget.hasAVX512(); | |||
2934 | ||||
2935 | // SrcReg(MaskReg) -> DestReg(GR64) | |||
2936 | // SrcReg(MaskReg) -> DestReg(GR32) | |||
2937 | ||||
2938 | // All KMASK RegClasses hold the same k registers, can be tested against anyone. | |||
2939 | if (X86::VK16RegClass.contains(SrcReg)) { | |||
2940 | if (X86::GR64RegClass.contains(DestReg)) { | |||
2941 | assert(Subtarget.hasBWI())((Subtarget.hasBWI()) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI()", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2941, __PRETTY_FUNCTION__)); | |||
2942 | return X86::KMOVQrk; | |||
2943 | } | |||
2944 | if (X86::GR32RegClass.contains(DestReg)) | |||
2945 | return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk; | |||
2946 | } | |||
2947 | ||||
2948 | // SrcReg(GR64) -> DestReg(MaskReg) | |||
2949 | // SrcReg(GR32) -> DestReg(MaskReg) | |||
2950 | ||||
2951 | // All KMASK RegClasses hold the same k registers, can be tested against anyone. | |||
2952 | if (X86::VK16RegClass.contains(DestReg)) { | |||
2953 | if (X86::GR64RegClass.contains(SrcReg)) { | |||
2954 | assert(Subtarget.hasBWI())((Subtarget.hasBWI()) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI()", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 2954, __PRETTY_FUNCTION__)); | |||
2955 | return X86::KMOVQkr; | |||
2956 | } | |||
2957 | if (X86::GR32RegClass.contains(SrcReg)) | |||
2958 | return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr; | |||
2959 | } | |||
2960 | ||||
2961 | ||||
2962 | // SrcReg(VR128) -> DestReg(GR64) | |||
2963 | // SrcReg(VR64) -> DestReg(GR64) | |||
2964 | // SrcReg(GR64) -> DestReg(VR128) | |||
2965 | // SrcReg(GR64) -> DestReg(VR64) | |||
2966 | ||||
2967 | if (X86::GR64RegClass.contains(DestReg)) { | |||
2968 | if (X86::VR128XRegClass.contains(SrcReg)) | |||
2969 | // Copy from a VR128 register to a GR64 register. | |||
2970 | return HasAVX512 ? X86::VMOVPQIto64Zrr : | |||
2971 | HasAVX ? X86::VMOVPQIto64rr : | |||
2972 | X86::MOVPQIto64rr; | |||
2973 | if (X86::VR64RegClass.contains(SrcReg)) | |||
2974 | // Copy from a VR64 register to a GR64 register. | |||
2975 | return X86::MMX_MOVD64from64rr; | |||
2976 | } else if (X86::GR64RegClass.contains(SrcReg)) { | |||
2977 | // Copy from a GR64 register to a VR128 register. | |||
2978 | if (X86::VR128XRegClass.contains(DestReg)) | |||
2979 | return HasAVX512 ? X86::VMOV64toPQIZrr : | |||
2980 | HasAVX ? X86::VMOV64toPQIrr : | |||
2981 | X86::MOV64toPQIrr; | |||
2982 | // Copy from a GR64 register to a VR64 register. | |||
2983 | if (X86::VR64RegClass.contains(DestReg)) | |||
2984 | return X86::MMX_MOVD64to64rr; | |||
2985 | } | |||
2986 | ||||
2987 | // SrcReg(FR32) -> DestReg(GR32) | |||
2988 | // SrcReg(GR32) -> DestReg(FR32) | |||
2989 | ||||
2990 | if (X86::GR32RegClass.contains(DestReg) && | |||
2991 | X86::FR32XRegClass.contains(SrcReg)) | |||
2992 | // Copy from a FR32 register to a GR32 register. | |||
2993 | return HasAVX512 ? X86::VMOVSS2DIZrr : | |||
2994 | HasAVX ? X86::VMOVSS2DIrr : | |||
2995 | X86::MOVSS2DIrr; | |||
2996 | ||||
2997 | if (X86::FR32XRegClass.contains(DestReg) && | |||
2998 | X86::GR32RegClass.contains(SrcReg)) | |||
2999 | // Copy from a GR32 register to a FR32 register. | |||
3000 | return HasAVX512 ? X86::VMOVDI2SSZrr : | |||
3001 | HasAVX ? X86::VMOVDI2SSrr : | |||
3002 | X86::MOVDI2SSrr; | |||
3003 | return 0; | |||
3004 | } | |||
3005 | ||||
3006 | void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB, | |||
3007 | MachineBasicBlock::iterator MI, | |||
3008 | const DebugLoc &DL, unsigned DestReg, | |||
3009 | unsigned SrcReg, bool KillSrc) const { | |||
3010 | // First deal with the normal symmetric copies. | |||
3011 | bool HasAVX = Subtarget.hasAVX(); | |||
3012 | bool HasVLX = Subtarget.hasVLX(); | |||
3013 | unsigned Opc = 0; | |||
3014 | if (X86::GR64RegClass.contains(DestReg, SrcReg)) | |||
3015 | Opc = X86::MOV64rr; | |||
3016 | else if (X86::GR32RegClass.contains(DestReg, SrcReg)) | |||
3017 | Opc = X86::MOV32rr; | |||
3018 | else if (X86::GR16RegClass.contains(DestReg, SrcReg)) | |||
3019 | Opc = X86::MOV16rr; | |||
3020 | else if (X86::GR8RegClass.contains(DestReg, SrcReg)) { | |||
3021 | // Copying to or from a physical H register on x86-64 requires a NOREX | |||
3022 | // move. Otherwise use a normal move. | |||
3023 | if ((isHReg(DestReg) || isHReg(SrcReg)) && | |||
3024 | Subtarget.is64Bit()) { | |||
3025 | Opc = X86::MOV8rr_NOREX; | |||
3026 | // Both operands must be encodable without an REX prefix. | |||
3027 | assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&((X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && "8-bit H register can not be copied outside GR8_NOREX") ? static_cast <void> (0) : __assert_fail ("X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && \"8-bit H register can not be copied outside GR8_NOREX\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3028, __PRETTY_FUNCTION__)) | |||
3028 | "8-bit H register can not be copied outside GR8_NOREX")((X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && "8-bit H register can not be copied outside GR8_NOREX") ? static_cast <void> (0) : __assert_fail ("X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && \"8-bit H register can not be copied outside GR8_NOREX\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3028, __PRETTY_FUNCTION__)); | |||
3029 | } else | |||
3030 | Opc = X86::MOV8rr; | |||
3031 | } | |||
3032 | else if (X86::VR64RegClass.contains(DestReg, SrcReg)) | |||
3033 | Opc = X86::MMX_MOVQ64rr; | |||
3034 | else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) { | |||
3035 | if (HasVLX) | |||
3036 | Opc = X86::VMOVAPSZ128rr; | |||
3037 | else if (X86::VR128RegClass.contains(DestReg, SrcReg)) | |||
3038 | Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr; | |||
3039 | else { | |||
3040 | // If this an extended register and we don't have VLX we need to use a | |||
3041 | // 512-bit move. | |||
3042 | Opc = X86::VMOVAPSZrr; | |||
3043 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
3044 | DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, | |||
3045 | &X86::VR512RegClass); | |||
3046 | SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, | |||
3047 | &X86::VR512RegClass); | |||
3048 | } | |||
3049 | } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) { | |||
3050 | if (HasVLX) | |||
3051 | Opc = X86::VMOVAPSZ256rr; | |||
3052 | else if (X86::VR256RegClass.contains(DestReg, SrcReg)) | |||
3053 | Opc = X86::VMOVAPSYrr; | |||
3054 | else { | |||
3055 | // If this an extended register and we don't have VLX we need to use a | |||
3056 | // 512-bit move. | |||
3057 | Opc = X86::VMOVAPSZrr; | |||
3058 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
3059 | DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, | |||
3060 | &X86::VR512RegClass); | |||
3061 | SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, | |||
3062 | &X86::VR512RegClass); | |||
3063 | } | |||
3064 | } else if (X86::VR512RegClass.contains(DestReg, SrcReg)) | |||
3065 | Opc = X86::VMOVAPSZrr; | |||
3066 | // All KMASK RegClasses hold the same k registers, can be tested against anyone. | |||
3067 | else if (X86::VK16RegClass.contains(DestReg, SrcReg)) | |||
3068 | Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk; | |||
3069 | if (!Opc) | |||
3070 | Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget); | |||
3071 | ||||
3072 | if (Opc) { | |||
3073 | BuildMI(MBB, MI, DL, get(Opc), DestReg) | |||
3074 | .addReg(SrcReg, getKillRegState(KillSrc)); | |||
3075 | return; | |||
3076 | } | |||
3077 | ||||
3078 | if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) { | |||
3079 | // FIXME: We use a fatal error here because historically LLVM has tried | |||
3080 | // lower some of these physreg copies and we want to ensure we get | |||
3081 | // reasonable bug reports if someone encounters a case no other testing | |||
3082 | // found. This path should be removed after the LLVM 7 release. | |||
3083 | report_fatal_error("Unable to copy EFLAGS physical register!"); | |||
3084 | } | |||
3085 | ||||
3086 | LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " << RI.getName(DestReg ) << '\n'; } } while (false) | |||
3087 | << RI.getName(DestReg) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " << RI.getName(DestReg ) << '\n'; } } while (false); | |||
3088 | report_fatal_error("Cannot emit physreg copy instruction"); | |||
3089 | } | |||
3090 | ||||
3091 | bool X86InstrInfo::isCopyInstrImpl(const MachineInstr &MI, | |||
3092 | const MachineOperand *&Src, | |||
3093 | const MachineOperand *&Dest) const { | |||
3094 | if (MI.isMoveReg()) { | |||
3095 | Dest = &MI.getOperand(0); | |||
3096 | Src = &MI.getOperand(1); | |||
3097 | return true; | |||
3098 | } | |||
3099 | return false; | |||
3100 | } | |||
3101 | ||||
3102 | static unsigned getLoadStoreRegOpcode(unsigned Reg, | |||
3103 | const TargetRegisterClass *RC, | |||
3104 | bool isStackAligned, | |||
3105 | const X86Subtarget &STI, | |||
3106 | bool load) { | |||
3107 | bool HasAVX = STI.hasAVX(); | |||
3108 | bool HasAVX512 = STI.hasAVX512(); | |||
3109 | bool HasVLX = STI.hasVLX(); | |||
3110 | ||||
3111 | switch (STI.getRegisterInfo()->getSpillSize(*RC)) { | |||
3112 | default: | |||
3113 | llvm_unreachable("Unknown spill size")::llvm::llvm_unreachable_internal("Unknown spill size", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3113); | |||
3114 | case 1: | |||
3115 | assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass")((X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::GR8RegClass.hasSubClassEq(RC) && \"Unknown 1-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3115, __PRETTY_FUNCTION__)); | |||
3116 | if (STI.is64Bit()) | |||
3117 | // Copying to or from a physical H register on x86-64 requires a NOREX | |||
3118 | // move. Otherwise use a normal move. | |||
3119 | if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC)) | |||
3120 | return load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX; | |||
3121 | return load ? X86::MOV8rm : X86::MOV8mr; | |||
3122 | case 2: | |||
3123 | if (X86::VK16RegClass.hasSubClassEq(RC)) | |||
3124 | return load ? X86::KMOVWkm : X86::KMOVWmk; | |||
3125 | assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass")((X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::GR16RegClass.hasSubClassEq(RC) && \"Unknown 2-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3125, __PRETTY_FUNCTION__)); | |||
3126 | return load ? X86::MOV16rm : X86::MOV16mr; | |||
3127 | case 4: | |||
3128 | if (X86::GR32RegClass.hasSubClassEq(RC)) | |||
3129 | return load ? X86::MOV32rm : X86::MOV32mr; | |||
3130 | if (X86::FR32XRegClass.hasSubClassEq(RC)) | |||
3131 | return load ? | |||
3132 | (HasAVX512 ? X86::VMOVSSZrm : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) : | |||
3133 | (HasAVX512 ? X86::VMOVSSZmr : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr); | |||
3134 | if (X86::RFP32RegClass.hasSubClassEq(RC)) | |||
3135 | return load ? X86::LD_Fp32m : X86::ST_Fp32m; | |||
3136 | if (X86::VK32RegClass.hasSubClassEq(RC)) { | |||
3137 | assert(STI.hasBWI() && "KMOVD requires BWI")((STI.hasBWI() && "KMOVD requires BWI") ? static_cast <void> (0) : __assert_fail ("STI.hasBWI() && \"KMOVD requires BWI\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3137, __PRETTY_FUNCTION__)); | |||
3138 | return load ? X86::KMOVDkm : X86::KMOVDmk; | |||
3139 | } | |||
3140 | llvm_unreachable("Unknown 4-byte regclass")::llvm::llvm_unreachable_internal("Unknown 4-byte regclass", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3140); | |||
3141 | case 8: | |||
3142 | if (X86::GR64RegClass.hasSubClassEq(RC)) | |||
3143 | return load ? X86::MOV64rm : X86::MOV64mr; | |||
3144 | if (X86::FR64XRegClass.hasSubClassEq(RC)) | |||
3145 | return load ? | |||
3146 | (HasAVX512 ? X86::VMOVSDZrm : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) : | |||
3147 | (HasAVX512 ? X86::VMOVSDZmr : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr); | |||
3148 | if (X86::VR64RegClass.hasSubClassEq(RC)) | |||
3149 | return load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr; | |||
3150 | if (X86::RFP64RegClass.hasSubClassEq(RC)) | |||
3151 | return load ? X86::LD_Fp64m : X86::ST_Fp64m; | |||
3152 | if (X86::VK64RegClass.hasSubClassEq(RC)) { | |||
3153 | assert(STI.hasBWI() && "KMOVQ requires BWI")((STI.hasBWI() && "KMOVQ requires BWI") ? static_cast <void> (0) : __assert_fail ("STI.hasBWI() && \"KMOVQ requires BWI\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3153, __PRETTY_FUNCTION__)); | |||
3154 | return load ? X86::KMOVQkm : X86::KMOVQmk; | |||
3155 | } | |||
3156 | llvm_unreachable("Unknown 8-byte regclass")::llvm::llvm_unreachable_internal("Unknown 8-byte regclass", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3156); | |||
3157 | case 10: | |||
3158 | assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass")((X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::RFP80RegClass.hasSubClassEq(RC) && \"Unknown 10-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3158, __PRETTY_FUNCTION__)); | |||
3159 | return load ? X86::LD_Fp80m : X86::ST_FpP80m; | |||
3160 | case 16: { | |||
3161 | if (X86::VR128XRegClass.hasSubClassEq(RC)) { | |||
3162 | // If stack is realigned we can use aligned stores. | |||
3163 | if (isStackAligned) | |||
3164 | return load ? | |||
3165 | (HasVLX ? X86::VMOVAPSZ128rm : | |||
3166 | HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX : | |||
3167 | HasAVX ? X86::VMOVAPSrm : | |||
3168 | X86::MOVAPSrm): | |||
3169 | (HasVLX ? X86::VMOVAPSZ128mr : | |||
3170 | HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX : | |||
3171 | HasAVX ? X86::VMOVAPSmr : | |||
3172 | X86::MOVAPSmr); | |||
3173 | else | |||
3174 | return load ? | |||
3175 | (HasVLX ? X86::VMOVUPSZ128rm : | |||
3176 | HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX : | |||
3177 | HasAVX ? X86::VMOVUPSrm : | |||
3178 | X86::MOVUPSrm): | |||
3179 | (HasVLX ? X86::VMOVUPSZ128mr : | |||
3180 | HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX : | |||
3181 | HasAVX ? X86::VMOVUPSmr : | |||
3182 | X86::MOVUPSmr); | |||
3183 | } | |||
3184 | if (X86::BNDRRegClass.hasSubClassEq(RC)) { | |||
3185 | if (STI.is64Bit()) | |||
3186 | return load ? X86::BNDMOV64rm : X86::BNDMOV64mr; | |||
3187 | else | |||
3188 | return load ? X86::BNDMOV32rm : X86::BNDMOV32mr; | |||
3189 | } | |||
3190 | llvm_unreachable("Unknown 16-byte regclass")::llvm::llvm_unreachable_internal("Unknown 16-byte regclass", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3190); | |||
3191 | } | |||
3192 | case 32: | |||
3193 | assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass")((X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::VR256XRegClass.hasSubClassEq(RC) && \"Unknown 32-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3193, __PRETTY_FUNCTION__)); | |||
3194 | // If stack is realigned we can use aligned stores. | |||
3195 | if (isStackAligned) | |||
3196 | return load ? | |||
3197 | (HasVLX ? X86::VMOVAPSZ256rm : | |||
3198 | HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX : | |||
3199 | X86::VMOVAPSYrm) : | |||
3200 | (HasVLX ? X86::VMOVAPSZ256mr : | |||
3201 | HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX : | |||
3202 | X86::VMOVAPSYmr); | |||
3203 | else | |||
3204 | return load ? | |||
3205 | (HasVLX ? X86::VMOVUPSZ256rm : | |||
3206 | HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX : | |||
3207 | X86::VMOVUPSYrm) : | |||
3208 | (HasVLX ? X86::VMOVUPSZ256mr : | |||
3209 | HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX : | |||
3210 | X86::VMOVUPSYmr); | |||
3211 | case 64: | |||
3212 | assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass")((X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass" ) ? static_cast<void> (0) : __assert_fail ("X86::VR512RegClass.hasSubClassEq(RC) && \"Unknown 64-byte regclass\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3212, __PRETTY_FUNCTION__)); | |||
3213 | assert(STI.hasAVX512() && "Using 512-bit register requires AVX512")((STI.hasAVX512() && "Using 512-bit register requires AVX512" ) ? static_cast<void> (0) : __assert_fail ("STI.hasAVX512() && \"Using 512-bit register requires AVX512\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3213, __PRETTY_FUNCTION__)); | |||
3214 | if (isStackAligned) | |||
3215 | return load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; | |||
3216 | else | |||
3217 | return load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; | |||
3218 | } | |||
3219 | } | |||
3220 | ||||
3221 | bool X86InstrInfo::getMemOperandWithOffset( | |||
3222 | MachineInstr &MemOp, MachineOperand *&BaseOp, int64_t &Offset, | |||
3223 | const TargetRegisterInfo *TRI) const { | |||
3224 | const MCInstrDesc &Desc = MemOp.getDesc(); | |||
3225 | int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); | |||
3226 | if (MemRefBegin < 0) | |||
3227 | return false; | |||
3228 | ||||
3229 | MemRefBegin += X86II::getOperandBias(Desc); | |||
3230 | ||||
3231 | BaseOp = &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg); | |||
3232 | if (!BaseOp->isReg()) // Can be an MO_FrameIndex | |||
3233 | return false; | |||
3234 | ||||
3235 | if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1) | |||
3236 | return false; | |||
3237 | ||||
3238 | if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() != | |||
3239 | X86::NoRegister) | |||
3240 | return false; | |||
3241 | ||||
3242 | const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp); | |||
3243 | ||||
3244 | // Displacement can be symbolic | |||
3245 | if (!DispMO.isImm()) | |||
3246 | return false; | |||
3247 | ||||
3248 | Offset = DispMO.getImm(); | |||
3249 | ||||
3250 | assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "((BaseOp->isReg() && "getMemOperandWithOffset only supports base " "operands of type register.") ? static_cast<void> (0) : __assert_fail ("BaseOp->isReg() && \"getMemOperandWithOffset only supports base \" \"operands of type register.\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3251, __PRETTY_FUNCTION__)) | |||
3251 | "operands of type register.")((BaseOp->isReg() && "getMemOperandWithOffset only supports base " "operands of type register.") ? static_cast<void> (0) : __assert_fail ("BaseOp->isReg() && \"getMemOperandWithOffset only supports base \" \"operands of type register.\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3251, __PRETTY_FUNCTION__)); | |||
3252 | return true; | |||
3253 | } | |||
3254 | ||||
3255 | static unsigned getStoreRegOpcode(unsigned SrcReg, | |||
3256 | const TargetRegisterClass *RC, | |||
3257 | bool isStackAligned, | |||
3258 | const X86Subtarget &STI) { | |||
3259 | return getLoadStoreRegOpcode(SrcReg, RC, isStackAligned, STI, false); | |||
3260 | } | |||
3261 | ||||
3262 | ||||
3263 | static unsigned getLoadRegOpcode(unsigned DestReg, | |||
3264 | const TargetRegisterClass *RC, | |||
3265 | bool isStackAligned, | |||
3266 | const X86Subtarget &STI) { | |||
3267 | return getLoadStoreRegOpcode(DestReg, RC, isStackAligned, STI, true); | |||
3268 | } | |||
3269 | ||||
3270 | void X86InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, | |||
3271 | MachineBasicBlock::iterator MI, | |||
3272 | unsigned SrcReg, bool isKill, int FrameIdx, | |||
3273 | const TargetRegisterClass *RC, | |||
3274 | const TargetRegisterInfo *TRI) const { | |||
3275 | const MachineFunction &MF = *MBB.getParent(); | |||
3276 | assert(MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&((MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize (*RC) && "Stack slot too small for store") ? static_cast <void> (0) : __assert_fail ("MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && \"Stack slot too small for store\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3277, __PRETTY_FUNCTION__)) | |||
3277 | "Stack slot too small for store")((MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize (*RC) && "Stack slot too small for store") ? static_cast <void> (0) : __assert_fail ("MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && \"Stack slot too small for store\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3277, __PRETTY_FUNCTION__)); | |||
3278 | unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); | |||
3279 | bool isAligned = | |||
3280 | (Subtarget.getFrameLowering()->getStackAlignment() >= Alignment) || | |||
3281 | RI.canRealignStack(MF); | |||
3282 | unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); | |||
3283 | addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx) | |||
3284 | .addReg(SrcReg, getKillRegState(isKill)); | |||
3285 | } | |||
3286 | ||||
3287 | void X86InstrInfo::storeRegToAddr( | |||
3288 | MachineFunction &MF, unsigned SrcReg, bool isKill, | |||
3289 | SmallVectorImpl<MachineOperand> &Addr, const TargetRegisterClass *RC, | |||
3290 | ArrayRef<MachineMemOperand *> MMOs, | |||
3291 | SmallVectorImpl<MachineInstr *> &NewMIs) const { | |||
3292 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
3293 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
3294 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
3295 | unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); | |||
3296 | DebugLoc DL; | |||
3297 | MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc)); | |||
3298 | for (unsigned i = 0, e = Addr.size(); i != e; ++i) | |||
3299 | MIB.add(Addr[i]); | |||
3300 | MIB.addReg(SrcReg, getKillRegState(isKill)); | |||
3301 | MIB.setMemRefs(MMOs); | |||
3302 | NewMIs.push_back(MIB); | |||
3303 | } | |||
3304 | ||||
3305 | ||||
3306 | void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, | |||
3307 | MachineBasicBlock::iterator MI, | |||
3308 | unsigned DestReg, int FrameIdx, | |||
3309 | const TargetRegisterClass *RC, | |||
3310 | const TargetRegisterInfo *TRI) const { | |||
3311 | const MachineFunction &MF = *MBB.getParent(); | |||
3312 | unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); | |||
3313 | bool isAligned = | |||
3314 | (Subtarget.getFrameLowering()->getStackAlignment() >= Alignment) || | |||
3315 | RI.canRealignStack(MF); | |||
3316 | unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); | |||
3317 | addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), FrameIdx); | |||
3318 | } | |||
3319 | ||||
3320 | void X86InstrInfo::loadRegFromAddr( | |||
3321 | MachineFunction &MF, unsigned DestReg, | |||
3322 | SmallVectorImpl<MachineOperand> &Addr, const TargetRegisterClass *RC, | |||
3323 | ArrayRef<MachineMemOperand *> MMOs, | |||
3324 | SmallVectorImpl<MachineInstr *> &NewMIs) const { | |||
3325 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
3326 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
3327 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
3328 | unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); | |||
3329 | DebugLoc DL; | |||
3330 | MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), DestReg); | |||
3331 | for (unsigned i = 0, e = Addr.size(); i != e; ++i) | |||
3332 | MIB.add(Addr[i]); | |||
3333 | MIB.setMemRefs(MMOs); | |||
3334 | NewMIs.push_back(MIB); | |||
3335 | } | |||
3336 | ||||
3337 | bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg, | |||
3338 | unsigned &SrcReg2, int &CmpMask, | |||
3339 | int &CmpValue) const { | |||
3340 | switch (MI.getOpcode()) { | |||
3341 | default: break; | |||
3342 | case X86::CMP64ri32: | |||
3343 | case X86::CMP64ri8: | |||
3344 | case X86::CMP32ri: | |||
3345 | case X86::CMP32ri8: | |||
3346 | case X86::CMP16ri: | |||
3347 | case X86::CMP16ri8: | |||
3348 | case X86::CMP8ri: | |||
3349 | SrcReg = MI.getOperand(0).getReg(); | |||
3350 | SrcReg2 = 0; | |||
3351 | if (MI.getOperand(1).isImm()) { | |||
3352 | CmpMask = ~0; | |||
3353 | CmpValue = MI.getOperand(1).getImm(); | |||
3354 | } else { | |||
3355 | CmpMask = CmpValue = 0; | |||
3356 | } | |||
3357 | return true; | |||
3358 | // A SUB can be used to perform comparison. | |||
3359 | case X86::SUB64rm: | |||
3360 | case X86::SUB32rm: | |||
3361 | case X86::SUB16rm: | |||
3362 | case X86::SUB8rm: | |||
3363 | SrcReg = MI.getOperand(1).getReg(); | |||
3364 | SrcReg2 = 0; | |||
3365 | CmpMask = 0; | |||
3366 | CmpValue = 0; | |||
3367 | return true; | |||
3368 | case X86::SUB64rr: | |||
3369 | case X86::SUB32rr: | |||
3370 | case X86::SUB16rr: | |||
3371 | case X86::SUB8rr: | |||
3372 | SrcReg = MI.getOperand(1).getReg(); | |||
3373 | SrcReg2 = MI.getOperand(2).getReg(); | |||
3374 | CmpMask = 0; | |||
3375 | CmpValue = 0; | |||
3376 | return true; | |||
3377 | case X86::SUB64ri32: | |||
3378 | case X86::SUB64ri8: | |||
3379 | case X86::SUB32ri: | |||
3380 | case X86::SUB32ri8: | |||
3381 | case X86::SUB16ri: | |||
3382 | case X86::SUB16ri8: | |||
3383 | case X86::SUB8ri: | |||
3384 | SrcReg = MI.getOperand(1).getReg(); | |||
3385 | SrcReg2 = 0; | |||
3386 | if (MI.getOperand(2).isImm()) { | |||
3387 | CmpMask = ~0; | |||
3388 | CmpValue = MI.getOperand(2).getImm(); | |||
3389 | } else { | |||
3390 | CmpMask = CmpValue = 0; | |||
3391 | } | |||
3392 | return true; | |||
3393 | case X86::CMP64rr: | |||
3394 | case X86::CMP32rr: | |||
3395 | case X86::CMP16rr: | |||
3396 | case X86::CMP8rr: | |||
3397 | SrcReg = MI.getOperand(0).getReg(); | |||
3398 | SrcReg2 = MI.getOperand(1).getReg(); | |||
3399 | CmpMask = 0; | |||
3400 | CmpValue = 0; | |||
3401 | return true; | |||
3402 | case X86::TEST8rr: | |||
3403 | case X86::TEST16rr: | |||
3404 | case X86::TEST32rr: | |||
3405 | case X86::TEST64rr: | |||
3406 | SrcReg = MI.getOperand(0).getReg(); | |||
3407 | if (MI.getOperand(1).getReg() != SrcReg) | |||
3408 | return false; | |||
3409 | // Compare against zero. | |||
3410 | SrcReg2 = 0; | |||
3411 | CmpMask = ~0; | |||
3412 | CmpValue = 0; | |||
3413 | return true; | |||
3414 | } | |||
3415 | return false; | |||
3416 | } | |||
3417 | ||||
3418 | /// Check whether the first instruction, whose only | |||
3419 | /// purpose is to update flags, can be made redundant. | |||
3420 | /// CMPrr can be made redundant by SUBrr if the operands are the same. | |||
3421 | /// This function can be extended later on. | |||
3422 | /// SrcReg, SrcRegs: register operands for FlagI. | |||
3423 | /// ImmValue: immediate for FlagI if it takes an immediate. | |||
3424 | inline static bool isRedundantFlagInstr(MachineInstr &FlagI, unsigned SrcReg, | |||
3425 | unsigned SrcReg2, int ImmMask, | |||
3426 | int ImmValue, MachineInstr &OI) { | |||
3427 | if (((FlagI.getOpcode() == X86::CMP64rr && OI.getOpcode() == X86::SUB64rr) || | |||
3428 | (FlagI.getOpcode() == X86::CMP32rr && OI.getOpcode() == X86::SUB32rr) || | |||
3429 | (FlagI.getOpcode() == X86::CMP16rr && OI.getOpcode() == X86::SUB16rr) || | |||
3430 | (FlagI.getOpcode() == X86::CMP8rr && OI.getOpcode() == X86::SUB8rr)) && | |||
3431 | ((OI.getOperand(1).getReg() == SrcReg && | |||
3432 | OI.getOperand(2).getReg() == SrcReg2) || | |||
3433 | (OI.getOperand(1).getReg() == SrcReg2 && | |||
3434 | OI.getOperand(2).getReg() == SrcReg))) | |||
3435 | return true; | |||
3436 | ||||
3437 | if (ImmMask != 0 && | |||
3438 | ((FlagI.getOpcode() == X86::CMP64ri32 && | |||
3439 | OI.getOpcode() == X86::SUB64ri32) || | |||
3440 | (FlagI.getOpcode() == X86::CMP64ri8 && | |||
3441 | OI.getOpcode() == X86::SUB64ri8) || | |||
3442 | (FlagI.getOpcode() == X86::CMP32ri && OI.getOpcode() == X86::SUB32ri) || | |||
3443 | (FlagI.getOpcode() == X86::CMP32ri8 && | |||
3444 | OI.getOpcode() == X86::SUB32ri8) || | |||
3445 | (FlagI.getOpcode() == X86::CMP16ri && OI.getOpcode() == X86::SUB16ri) || | |||
3446 | (FlagI.getOpcode() == X86::CMP16ri8 && | |||
3447 | OI.getOpcode() == X86::SUB16ri8) || | |||
3448 | (FlagI.getOpcode() == X86::CMP8ri && OI.getOpcode() == X86::SUB8ri)) && | |||
3449 | OI.getOperand(1).getReg() == SrcReg && | |||
3450 | OI.getOperand(2).getImm() == ImmValue) | |||
3451 | return true; | |||
3452 | return false; | |||
3453 | } | |||
3454 | ||||
3455 | /// Check whether the definition can be converted | |||
3456 | /// to remove a comparison against zero. | |||
3457 | inline static bool isDefConvertible(MachineInstr &MI) { | |||
3458 | switch (MI.getOpcode()) { | |||
3459 | default: return false; | |||
3460 | ||||
3461 | // The shift instructions only modify ZF if their shift count is non-zero. | |||
3462 | // N.B.: The processor truncates the shift count depending on the encoding. | |||
3463 | case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri:case X86::SAR64ri: | |||
3464 | case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri:case X86::SHR64ri: | |||
3465 | return getTruncatedShiftCount(MI, 2) != 0; | |||
3466 | ||||
3467 | // Some left shift instructions can be turned into LEA instructions but only | |||
3468 | // if their flags aren't used. Avoid transforming such instructions. | |||
3469 | case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri:case X86::SHL64ri:{ | |||
3470 | unsigned ShAmt = getTruncatedShiftCount(MI, 2); | |||
3471 | if (isTruncatedShiftCountForLEA(ShAmt)) return false; | |||
3472 | return ShAmt != 0; | |||
3473 | } | |||
3474 | ||||
3475 | case X86::SHRD16rri8:case X86::SHRD32rri8:case X86::SHRD64rri8: | |||
3476 | case X86::SHLD16rri8:case X86::SHLD32rri8:case X86::SHLD64rri8: | |||
3477 | return getTruncatedShiftCount(MI, 3) != 0; | |||
3478 | ||||
3479 | case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB32ri: | |||
3480 | case X86::SUB32ri8: case X86::SUB16ri: case X86::SUB16ri8: | |||
3481 | case X86::SUB8ri: case X86::SUB64rr: case X86::SUB32rr: | |||
3482 | case X86::SUB16rr: case X86::SUB8rr: case X86::SUB64rm: | |||
3483 | case X86::SUB32rm: case X86::SUB16rm: case X86::SUB8rm: | |||
3484 | case X86::DEC64r: case X86::DEC32r: case X86::DEC16r: case X86::DEC8r: | |||
3485 | case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri: | |||
3486 | case X86::ADD32ri8: case X86::ADD16ri: case X86::ADD16ri8: | |||
3487 | case X86::ADD8ri: case X86::ADD64rr: case X86::ADD32rr: | |||
3488 | case X86::ADD16rr: case X86::ADD8rr: case X86::ADD64rm: | |||
3489 | case X86::ADD32rm: case X86::ADD16rm: case X86::ADD8rm: | |||
3490 | case X86::INC64r: case X86::INC32r: case X86::INC16r: case X86::INC8r: | |||
3491 | case X86::AND64ri32: case X86::AND64ri8: case X86::AND32ri: | |||
3492 | case X86::AND32ri8: case X86::AND16ri: case X86::AND16ri8: | |||
3493 | case X86::AND8ri: case X86::AND64rr: case X86::AND32rr: | |||
3494 | case X86::AND16rr: case X86::AND8rr: case X86::AND64rm: | |||
3495 | case X86::AND32rm: case X86::AND16rm: case X86::AND8rm: | |||
3496 | case X86::XOR64ri32: case X86::XOR64ri8: case X86::XOR32ri: | |||
3497 | case X86::XOR32ri8: case X86::XOR16ri: case X86::XOR16ri8: | |||
3498 | case X86::XOR8ri: case X86::XOR64rr: case X86::XOR32rr: | |||
3499 | case X86::XOR16rr: case X86::XOR8rr: case X86::XOR64rm: | |||
3500 | case X86::XOR32rm: case X86::XOR16rm: case X86::XOR8rm: | |||
3501 | case X86::OR64ri32: case X86::OR64ri8: case X86::OR32ri: | |||
3502 | case X86::OR32ri8: case X86::OR16ri: case X86::OR16ri8: | |||
3503 | case X86::OR8ri: case X86::OR64rr: case X86::OR32rr: | |||
3504 | case X86::OR16rr: case X86::OR8rr: case X86::OR64rm: | |||
3505 | case X86::OR32rm: case X86::OR16rm: case X86::OR8rm: | |||
3506 | case X86::ADC64ri32: case X86::ADC64ri8: case X86::ADC32ri: | |||
3507 | case X86::ADC32ri8: case X86::ADC16ri: case X86::ADC16ri8: | |||
3508 | case X86::ADC8ri: case X86::ADC64rr: case X86::ADC32rr: | |||
3509 | case X86::ADC16rr: case X86::ADC8rr: case X86::ADC64rm: | |||
3510 | case X86::ADC32rm: case X86::ADC16rm: case X86::ADC8rm: | |||
3511 | case X86::SBB64ri32: case X86::SBB64ri8: case X86::SBB32ri: | |||
3512 | case X86::SBB32ri8: case X86::SBB16ri: case X86::SBB16ri8: | |||
3513 | case X86::SBB8ri: case X86::SBB64rr: case X86::SBB32rr: | |||
3514 | case X86::SBB16rr: case X86::SBB8rr: case X86::SBB64rm: | |||
3515 | case X86::SBB32rm: case X86::SBB16rm: case X86::SBB8rm: | |||
3516 | case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r: | |||
3517 | case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1:case X86::SAR64r1: | |||
3518 | case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1:case X86::SHR64r1: | |||
3519 | case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1:case X86::SHL64r1: | |||
3520 | case X86::ANDN32rr: case X86::ANDN32rm: | |||
3521 | case X86::ANDN64rr: case X86::ANDN64rm: | |||
3522 | case X86::BEXTR32rr: case X86::BEXTR64rr: | |||
3523 | case X86::BEXTR32rm: case X86::BEXTR64rm: | |||
3524 | case X86::BLSI32rr: case X86::BLSI32rm: | |||
3525 | case X86::BLSI64rr: case X86::BLSI64rm: | |||
3526 | case X86::BLSMSK32rr:case X86::BLSMSK32rm: | |||
3527 | case X86::BLSMSK64rr:case X86::BLSMSK64rm: | |||
3528 | case X86::BLSR32rr: case X86::BLSR32rm: | |||
3529 | case X86::BLSR64rr: case X86::BLSR64rm: | |||
3530 | case X86::BZHI32rr: case X86::BZHI32rm: | |||
3531 | case X86::BZHI64rr: case X86::BZHI64rm: | |||
3532 | case X86::LZCNT16rr: case X86::LZCNT16rm: | |||
3533 | case X86::LZCNT32rr: case X86::LZCNT32rm: | |||
3534 | case X86::LZCNT64rr: case X86::LZCNT64rm: | |||
3535 | case X86::POPCNT16rr:case X86::POPCNT16rm: | |||
3536 | case X86::POPCNT32rr:case X86::POPCNT32rm: | |||
3537 | case X86::POPCNT64rr:case X86::POPCNT64rm: | |||
3538 | case X86::TZCNT16rr: case X86::TZCNT16rm: | |||
3539 | case X86::TZCNT32rr: case X86::TZCNT32rm: | |||
3540 | case X86::TZCNT64rr: case X86::TZCNT64rm: | |||
3541 | case X86::BEXTRI32ri: case X86::BEXTRI32mi: | |||
3542 | case X86::BEXTRI64ri: case X86::BEXTRI64mi: | |||
3543 | case X86::BLCFILL32rr: case X86::BLCFILL32rm: | |||
3544 | case X86::BLCFILL64rr: case X86::BLCFILL64rm: | |||
3545 | case X86::BLCI32rr: case X86::BLCI32rm: | |||
3546 | case X86::BLCI64rr: case X86::BLCI64rm: | |||
3547 | case X86::BLCIC32rr: case X86::BLCIC32rm: | |||
3548 | case X86::BLCIC64rr: case X86::BLCIC64rm: | |||
3549 | case X86::BLCMSK32rr: case X86::BLCMSK32rm: | |||
3550 | case X86::BLCMSK64rr: case X86::BLCMSK64rm: | |||
3551 | case X86::BLCS32rr: case X86::BLCS32rm: | |||
3552 | case X86::BLCS64rr: case X86::BLCS64rm: | |||
3553 | case X86::BLSFILL32rr: case X86::BLSFILL32rm: | |||
3554 | case X86::BLSFILL64rr: case X86::BLSFILL64rm: | |||
3555 | case X86::BLSIC32rr: case X86::BLSIC32rm: | |||
3556 | case X86::BLSIC64rr: case X86::BLSIC64rm: | |||
3557 | return true; | |||
3558 | } | |||
3559 | } | |||
3560 | ||||
3561 | /// Check whether the use can be converted to remove a comparison against zero. | |||
3562 | static X86::CondCode isUseDefConvertible(MachineInstr &MI) { | |||
3563 | switch (MI.getOpcode()) { | |||
3564 | default: return X86::COND_INVALID; | |||
3565 | case X86::LZCNT16rr: case X86::LZCNT16rm: | |||
3566 | case X86::LZCNT32rr: case X86::LZCNT32rm: | |||
3567 | case X86::LZCNT64rr: case X86::LZCNT64rm: | |||
3568 | return X86::COND_B; | |||
3569 | case X86::POPCNT16rr:case X86::POPCNT16rm: | |||
3570 | case X86::POPCNT32rr:case X86::POPCNT32rm: | |||
3571 | case X86::POPCNT64rr:case X86::POPCNT64rm: | |||
3572 | return X86::COND_E; | |||
3573 | case X86::TZCNT16rr: case X86::TZCNT16rm: | |||
3574 | case X86::TZCNT32rr: case X86::TZCNT32rm: | |||
3575 | case X86::TZCNT64rr: case X86::TZCNT64rm: | |||
3576 | return X86::COND_B; | |||
3577 | case X86::BSF16rr: | |||
3578 | case X86::BSF16rm: | |||
3579 | case X86::BSF32rr: | |||
3580 | case X86::BSF32rm: | |||
3581 | case X86::BSF64rr: | |||
3582 | case X86::BSF64rm: | |||
3583 | return X86::COND_E; | |||
3584 | } | |||
3585 | } | |||
3586 | ||||
3587 | /// Check if there exists an earlier instruction that | |||
3588 | /// operates on the same source operands and sets flags in the same way as | |||
3589 | /// Compare; remove Compare if possible. | |||
3590 | bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg, | |||
3591 | unsigned SrcReg2, int CmpMask, | |||
3592 | int CmpValue, | |||
3593 | const MachineRegisterInfo *MRI) const { | |||
3594 | // Check whether we can replace SUB with CMP. | |||
3595 | unsigned NewOpcode = 0; | |||
3596 | switch (CmpInstr.getOpcode()) { | |||
| ||||
3597 | default: break; | |||
3598 | case X86::SUB64ri32: | |||
3599 | case X86::SUB64ri8: | |||
3600 | case X86::SUB32ri: | |||
3601 | case X86::SUB32ri8: | |||
3602 | case X86::SUB16ri: | |||
3603 | case X86::SUB16ri8: | |||
3604 | case X86::SUB8ri: | |||
3605 | case X86::SUB64rm: | |||
3606 | case X86::SUB32rm: | |||
3607 | case X86::SUB16rm: | |||
3608 | case X86::SUB8rm: | |||
3609 | case X86::SUB64rr: | |||
3610 | case X86::SUB32rr: | |||
3611 | case X86::SUB16rr: | |||
3612 | case X86::SUB8rr: { | |||
3613 | if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) | |||
3614 | return false; | |||
3615 | // There is no use of the destination register, we can replace SUB with CMP. | |||
3616 | switch (CmpInstr.getOpcode()) { | |||
3617 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3617); | |||
3618 | case X86::SUB64rm: NewOpcode = X86::CMP64rm; break; | |||
3619 | case X86::SUB32rm: NewOpcode = X86::CMP32rm; break; | |||
3620 | case X86::SUB16rm: NewOpcode = X86::CMP16rm; break; | |||
3621 | case X86::SUB8rm: NewOpcode = X86::CMP8rm; break; | |||
3622 | case X86::SUB64rr: NewOpcode = X86::CMP64rr; break; | |||
3623 | case X86::SUB32rr: NewOpcode = X86::CMP32rr; break; | |||
3624 | case X86::SUB16rr: NewOpcode = X86::CMP16rr; break; | |||
3625 | case X86::SUB8rr: NewOpcode = X86::CMP8rr; break; | |||
3626 | case X86::SUB64ri32: NewOpcode = X86::CMP64ri32; break; | |||
3627 | case X86::SUB64ri8: NewOpcode = X86::CMP64ri8; break; | |||
3628 | case X86::SUB32ri: NewOpcode = X86::CMP32ri; break; | |||
3629 | case X86::SUB32ri8: NewOpcode = X86::CMP32ri8; break; | |||
3630 | case X86::SUB16ri: NewOpcode = X86::CMP16ri; break; | |||
3631 | case X86::SUB16ri8: NewOpcode = X86::CMP16ri8; break; | |||
3632 | case X86::SUB8ri: NewOpcode = X86::CMP8ri; break; | |||
3633 | } | |||
3634 | CmpInstr.setDesc(get(NewOpcode)); | |||
3635 | CmpInstr.RemoveOperand(0); | |||
3636 | // Fall through to optimize Cmp if Cmp is CMPrr or CMPri. | |||
3637 | if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm || | |||
3638 | NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm) | |||
3639 | return false; | |||
3640 | } | |||
3641 | } | |||
3642 | ||||
3643 | // Get the unique definition of SrcReg. | |||
3644 | MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); | |||
3645 | if (!MI) return false; | |||
3646 | ||||
3647 | // CmpInstr is the first instruction of the BB. | |||
3648 | MachineBasicBlock::iterator I = CmpInstr, Def = MI; | |||
3649 | ||||
3650 | // If we are comparing against zero, check whether we can use MI to update | |||
3651 | // EFLAGS. If MI is not in the same BB as CmpInstr, do not optimize. | |||
3652 | bool IsCmpZero = (CmpMask != 0 && CmpValue == 0); | |||
3653 | if (IsCmpZero && MI->getParent() != CmpInstr.getParent()) | |||
3654 | return false; | |||
3655 | ||||
3656 | // If we have a use of the source register between the def and our compare | |||
3657 | // instruction we can eliminate the compare iff the use sets EFLAGS in the | |||
3658 | // right way. | |||
3659 | bool ShouldUpdateCC = false; | |||
3660 | X86::CondCode NewCC = X86::COND_INVALID; | |||
3661 | if (IsCmpZero && !isDefConvertible(*MI)) { | |||
3662 | // Scan forward from the use until we hit the use we're looking for or the | |||
3663 | // compare instruction. | |||
3664 | for (MachineBasicBlock::iterator J = MI;; ++J) { | |||
3665 | // Do we have a convertible instruction? | |||
3666 | NewCC = isUseDefConvertible(*J); | |||
3667 | if (NewCC != X86::COND_INVALID && J->getOperand(1).isReg() && | |||
3668 | J->getOperand(1).getReg() == SrcReg) { | |||
3669 | assert(J->definesRegister(X86::EFLAGS) && "Must be an EFLAGS def!")((J->definesRegister(X86::EFLAGS) && "Must be an EFLAGS def!" ) ? static_cast<void> (0) : __assert_fail ("J->definesRegister(X86::EFLAGS) && \"Must be an EFLAGS def!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3669, __PRETTY_FUNCTION__)); | |||
3670 | ShouldUpdateCC = true; // Update CC later on. | |||
3671 | // This is not a def of SrcReg, but still a def of EFLAGS. Keep going | |||
3672 | // with the new def. | |||
3673 | Def = J; | |||
3674 | MI = &*Def; | |||
3675 | break; | |||
3676 | } | |||
3677 | ||||
3678 | if (J == I) | |||
3679 | return false; | |||
3680 | } | |||
3681 | } | |||
3682 | ||||
3683 | // We are searching for an earlier instruction that can make CmpInstr | |||
3684 | // redundant and that instruction will be saved in Sub. | |||
3685 | MachineInstr *Sub = nullptr; | |||
3686 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
3687 | ||||
3688 | // We iterate backward, starting from the instruction before CmpInstr and | |||
3689 | // stop when reaching the definition of a source register or done with the BB. | |||
3690 | // RI points to the instruction before CmpInstr. | |||
3691 | // If the definition is in this basic block, RE points to the definition; | |||
3692 | // otherwise, RE is the rend of the basic block. | |||
3693 | MachineBasicBlock::reverse_iterator | |||
3694 | RI = ++I.getReverse(), | |||
3695 | RE = CmpInstr.getParent() == MI->getParent() | |||
3696 | ? Def.getReverse() /* points to MI */ | |||
3697 | : CmpInstr.getParent()->rend(); | |||
3698 | MachineInstr *Movr0Inst = nullptr; | |||
3699 | for (; RI != RE; ++RI) { | |||
3700 | MachineInstr &Instr = *RI; | |||
3701 | // Check whether CmpInstr can be made redundant by the current instruction. | |||
3702 | if (!IsCmpZero && isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, | |||
3703 | CmpValue, Instr)) { | |||
3704 | Sub = &Instr; | |||
3705 | break; | |||
3706 | } | |||
3707 | ||||
3708 | if (Instr.modifiesRegister(X86::EFLAGS, TRI) || | |||
3709 | Instr.readsRegister(X86::EFLAGS, TRI)) { | |||
3710 | // This instruction modifies or uses EFLAGS. | |||
3711 | ||||
3712 | // MOV32r0 etc. are implemented with xor which clobbers condition code. | |||
3713 | // They are safe to move up, if the definition to EFLAGS is dead and | |||
3714 | // earlier instructions do not read or write EFLAGS. | |||
3715 | if (!Movr0Inst && Instr.getOpcode() == X86::MOV32r0 && | |||
3716 | Instr.registerDefIsDead(X86::EFLAGS, TRI)) { | |||
3717 | Movr0Inst = &Instr; | |||
3718 | continue; | |||
3719 | } | |||
3720 | ||||
3721 | // We can't remove CmpInstr. | |||
3722 | return false; | |||
3723 | } | |||
3724 | } | |||
3725 | ||||
3726 | // Return false if no candidates exist. | |||
3727 | if (!IsCmpZero && !Sub) | |||
3728 | return false; | |||
3729 | ||||
3730 | bool IsSwapped = (SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 && | |||
| ||||
3731 | Sub->getOperand(2).getReg() == SrcReg); | |||
3732 | ||||
3733 | // Scan forward from the instruction after CmpInstr for uses of EFLAGS. | |||
3734 | // It is safe to remove CmpInstr if EFLAGS is redefined or killed. | |||
3735 | // If we are done with the basic block, we need to check whether EFLAGS is | |||
3736 | // live-out. | |||
3737 | bool IsSafe = false; | |||
3738 | SmallVector<std::pair<MachineInstr*, unsigned /*NewOpc*/>, 4> OpsToUpdate; | |||
3739 | MachineBasicBlock::iterator E = CmpInstr.getParent()->end(); | |||
3740 | for (++I; I != E; ++I) { | |||
3741 | const MachineInstr &Instr = *I; | |||
3742 | bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI); | |||
3743 | bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI); | |||
3744 | // We should check the usage if this instruction uses and updates EFLAGS. | |||
3745 | if (!UseEFLAGS && ModifyEFLAGS) { | |||
3746 | // It is safe to remove CmpInstr if EFLAGS is updated again. | |||
3747 | IsSafe = true; | |||
3748 | break; | |||
3749 | } | |||
3750 | if (!UseEFLAGS && !ModifyEFLAGS) | |||
3751 | continue; | |||
3752 | ||||
3753 | // EFLAGS is used by this instruction. | |||
3754 | X86::CondCode OldCC = X86::COND_INVALID; | |||
3755 | bool OpcIsSET = false; | |||
3756 | if (IsCmpZero || IsSwapped) { | |||
3757 | // We decode the condition code from opcode. | |||
3758 | if (Instr.isBranch()) | |||
3759 | OldCC = X86::getCondFromBranchOpc(Instr.getOpcode()); | |||
3760 | else { | |||
3761 | OldCC = X86::getCondFromSETOpc(Instr.getOpcode()); | |||
3762 | if (OldCC != X86::COND_INVALID) | |||
3763 | OpcIsSET = true; | |||
3764 | else | |||
3765 | OldCC = X86::getCondFromCMovOpc(Instr.getOpcode()); | |||
3766 | } | |||
3767 | if (OldCC == X86::COND_INVALID) return false; | |||
3768 | } | |||
3769 | X86::CondCode ReplacementCC = X86::COND_INVALID; | |||
3770 | if (IsCmpZero) { | |||
3771 | switch (OldCC) { | |||
3772 | default: break; | |||
3773 | case X86::COND_A: case X86::COND_AE: | |||
3774 | case X86::COND_B: case X86::COND_BE: | |||
3775 | case X86::COND_G: case X86::COND_GE: | |||
3776 | case X86::COND_L: case X86::COND_LE: | |||
3777 | case X86::COND_O: case X86::COND_NO: | |||
3778 | // CF and OF are used, we can't perform this optimization. | |||
3779 | return false; | |||
3780 | } | |||
3781 | ||||
3782 | // If we're updating the condition code check if we have to reverse the | |||
3783 | // condition. | |||
3784 | if (ShouldUpdateCC) | |||
3785 | switch (OldCC) { | |||
3786 | default: | |||
3787 | return false; | |||
3788 | case X86::COND_E: | |||
3789 | ReplacementCC = NewCC; | |||
3790 | break; | |||
3791 | case X86::COND_NE: | |||
3792 | ReplacementCC = GetOppositeBranchCondition(NewCC); | |||
3793 | break; | |||
3794 | } | |||
3795 | } else if (IsSwapped) { | |||
3796 | // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs | |||
3797 | // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. | |||
3798 | // We swap the condition code and synthesize the new opcode. | |||
3799 | ReplacementCC = getSwappedCondition(OldCC); | |||
3800 | if (ReplacementCC == X86::COND_INVALID) return false; | |||
3801 | } | |||
3802 | ||||
3803 | if ((ShouldUpdateCC || IsSwapped) && ReplacementCC != OldCC) { | |||
3804 | // Synthesize the new opcode. | |||
3805 | bool HasMemoryOperand = Instr.hasOneMemOperand(); | |||
3806 | unsigned NewOpc; | |||
3807 | if (Instr.isBranch()) | |||
3808 | NewOpc = GetCondBranchFromCond(ReplacementCC); | |||
3809 | else if(OpcIsSET) | |||
3810 | NewOpc = getSETFromCond(ReplacementCC, HasMemoryOperand); | |||
3811 | else { | |||
3812 | unsigned DstReg = Instr.getOperand(0).getReg(); | |||
3813 | const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); | |||
3814 | NewOpc = getCMovFromCond(ReplacementCC, TRI->getRegSizeInBits(*DstRC)/8, | |||
3815 | HasMemoryOperand); | |||
3816 | } | |||
3817 | ||||
3818 | // Push the MachineInstr to OpsToUpdate. | |||
3819 | // If it is safe to remove CmpInstr, the condition code of these | |||
3820 | // instructions will be modified. | |||
3821 | OpsToUpdate.push_back(std::make_pair(&*I, NewOpc)); | |||
3822 | } | |||
3823 | if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) { | |||
3824 | // It is safe to remove CmpInstr if EFLAGS is updated again or killed. | |||
3825 | IsSafe = true; | |||
3826 | break; | |||
3827 | } | |||
3828 | } | |||
3829 | ||||
3830 | // If EFLAGS is not killed nor re-defined, we should check whether it is | |||
3831 | // live-out. If it is live-out, do not optimize. | |||
3832 | if ((IsCmpZero || IsSwapped) && !IsSafe) { | |||
3833 | MachineBasicBlock *MBB = CmpInstr.getParent(); | |||
3834 | for (MachineBasicBlock *Successor : MBB->successors()) | |||
3835 | if (Successor->isLiveIn(X86::EFLAGS)) | |||
3836 | return false; | |||
3837 | } | |||
3838 | ||||
3839 | // The instruction to be updated is either Sub or MI. | |||
3840 | Sub = IsCmpZero ? MI : Sub; | |||
3841 | // Move Movr0Inst to the appropriate place before Sub. | |||
3842 | if (Movr0Inst) { | |||
3843 | // Look backwards until we find a def that doesn't use the current EFLAGS. | |||
3844 | Def = Sub; | |||
3845 | MachineBasicBlock::reverse_iterator InsertI = Def.getReverse(), | |||
3846 | InsertE = Sub->getParent()->rend(); | |||
3847 | for (; InsertI != InsertE; ++InsertI) { | |||
3848 | MachineInstr *Instr = &*InsertI; | |||
3849 | if (!Instr->readsRegister(X86::EFLAGS, TRI) && | |||
3850 | Instr->modifiesRegister(X86::EFLAGS, TRI)) { | |||
3851 | Sub->getParent()->remove(Movr0Inst); | |||
3852 | Instr->getParent()->insert(MachineBasicBlock::iterator(Instr), | |||
3853 | Movr0Inst); | |||
3854 | break; | |||
3855 | } | |||
3856 | } | |||
3857 | if (InsertI == InsertE) | |||
3858 | return false; | |||
3859 | } | |||
3860 | ||||
3861 | // Make sure Sub instruction defines EFLAGS and mark the def live. | |||
3862 | unsigned i = 0, e = Sub->getNumOperands(); | |||
3863 | for (; i != e; ++i) { | |||
3864 | MachineOperand &MO = Sub->getOperand(i); | |||
3865 | if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) { | |||
3866 | MO.setIsDead(false); | |||
3867 | break; | |||
3868 | } | |||
3869 | } | |||
3870 | assert(i != e && "Unable to locate a def EFLAGS operand")((i != e && "Unable to locate a def EFLAGS operand") ? static_cast<void> (0) : __assert_fail ("i != e && \"Unable to locate a def EFLAGS operand\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3870, __PRETTY_FUNCTION__)); | |||
3871 | ||||
3872 | CmpInstr.eraseFromParent(); | |||
3873 | ||||
3874 | // Modify the condition code of instructions in OpsToUpdate. | |||
3875 | for (auto &Op : OpsToUpdate) | |||
3876 | Op.first->setDesc(get(Op.second)); | |||
3877 | return true; | |||
3878 | } | |||
3879 | ||||
3880 | /// Try to remove the load by folding it to a register | |||
3881 | /// operand at the use. We fold the load instructions if load defines a virtual | |||
3882 | /// register, the virtual register is used once in the same BB, and the | |||
3883 | /// instructions in-between do not load or store, and have no side effects. | |||
3884 | MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI, | |||
3885 | const MachineRegisterInfo *MRI, | |||
3886 | unsigned &FoldAsLoadDefReg, | |||
3887 | MachineInstr *&DefMI) const { | |||
3888 | // Check whether we can move DefMI here. | |||
3889 | DefMI = MRI->getVRegDef(FoldAsLoadDefReg); | |||
3890 | assert(DefMI)((DefMI) ? static_cast<void> (0) : __assert_fail ("DefMI" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3890, __PRETTY_FUNCTION__)); | |||
3891 | bool SawStore = false; | |||
3892 | if (!DefMI->isSafeToMove(nullptr, SawStore)) | |||
3893 | return nullptr; | |||
3894 | ||||
3895 | // Collect information about virtual register operands of MI. | |||
3896 | SmallVector<unsigned, 1> SrcOperandIds; | |||
3897 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
3898 | MachineOperand &MO = MI.getOperand(i); | |||
3899 | if (!MO.isReg()) | |||
3900 | continue; | |||
3901 | unsigned Reg = MO.getReg(); | |||
3902 | if (Reg != FoldAsLoadDefReg) | |||
3903 | continue; | |||
3904 | // Do not fold if we have a subreg use or a def. | |||
3905 | if (MO.getSubReg() || MO.isDef()) | |||
3906 | return nullptr; | |||
3907 | SrcOperandIds.push_back(i); | |||
3908 | } | |||
3909 | if (SrcOperandIds.empty()) | |||
3910 | return nullptr; | |||
3911 | ||||
3912 | // Check whether we can fold the def into SrcOperandId. | |||
3913 | if (MachineInstr *FoldMI = foldMemoryOperand(MI, SrcOperandIds, *DefMI)) { | |||
3914 | FoldAsLoadDefReg = 0; | |||
3915 | return FoldMI; | |||
3916 | } | |||
3917 | ||||
3918 | return nullptr; | |||
3919 | } | |||
3920 | ||||
3921 | /// Expand a single-def pseudo instruction to a two-addr | |||
3922 | /// instruction with two undef reads of the register being defined. | |||
3923 | /// This is used for mapping: | |||
3924 | /// %xmm4 = V_SET0 | |||
3925 | /// to: | |||
3926 | /// %xmm4 = PXORrr undef %xmm4, undef %xmm4 | |||
3927 | /// | |||
3928 | static bool Expand2AddrUndef(MachineInstrBuilder &MIB, | |||
3929 | const MCInstrDesc &Desc) { | |||
3930 | assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.")((Desc.getNumOperands() == 3 && "Expected two-addr instruction." ) ? static_cast<void> (0) : __assert_fail ("Desc.getNumOperands() == 3 && \"Expected two-addr instruction.\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3930, __PRETTY_FUNCTION__)); | |||
3931 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
3932 | MIB->setDesc(Desc); | |||
3933 | ||||
3934 | // MachineInstr::addOperand() will insert explicit operands before any | |||
3935 | // implicit operands. | |||
3936 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); | |||
3937 | // But we don't trust that. | |||
3938 | assert(MIB->getOperand(1).getReg() == Reg &&((MIB->getOperand(1).getReg() == Reg && MIB->getOperand (2).getReg() == Reg && "Misplaced operand") ? static_cast <void> (0) : __assert_fail ("MIB->getOperand(1).getReg() == Reg && MIB->getOperand(2).getReg() == Reg && \"Misplaced operand\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3939, __PRETTY_FUNCTION__)) | |||
3939 | MIB->getOperand(2).getReg() == Reg && "Misplaced operand")((MIB->getOperand(1).getReg() == Reg && MIB->getOperand (2).getReg() == Reg && "Misplaced operand") ? static_cast <void> (0) : __assert_fail ("MIB->getOperand(1).getReg() == Reg && MIB->getOperand(2).getReg() == Reg && \"Misplaced operand\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3939, __PRETTY_FUNCTION__)); | |||
3940 | return true; | |||
3941 | } | |||
3942 | ||||
3943 | /// Expand a single-def pseudo instruction to a two-addr | |||
3944 | /// instruction with two %k0 reads. | |||
3945 | /// This is used for mapping: | |||
3946 | /// %k4 = K_SET1 | |||
3947 | /// to: | |||
3948 | /// %k4 = KXNORrr %k0, %k0 | |||
3949 | static bool Expand2AddrKreg(MachineInstrBuilder &MIB, | |||
3950 | const MCInstrDesc &Desc, unsigned Reg) { | |||
3951 | assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.")((Desc.getNumOperands() == 3 && "Expected two-addr instruction." ) ? static_cast<void> (0) : __assert_fail ("Desc.getNumOperands() == 3 && \"Expected two-addr instruction.\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3951, __PRETTY_FUNCTION__)); | |||
3952 | MIB->setDesc(Desc); | |||
3953 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); | |||
3954 | return true; | |||
3955 | } | |||
3956 | ||||
3957 | static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, | |||
3958 | bool MinusOne) { | |||
3959 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
3960 | DebugLoc DL = MIB->getDebugLoc(); | |||
3961 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
3962 | ||||
3963 | // Insert the XOR. | |||
3964 | BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg) | |||
3965 | .addReg(Reg, RegState::Undef) | |||
3966 | .addReg(Reg, RegState::Undef); | |||
3967 | ||||
3968 | // Turn the pseudo into an INC or DEC. | |||
3969 | MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r)); | |||
3970 | MIB.addReg(Reg); | |||
3971 | ||||
3972 | return true; | |||
3973 | } | |||
3974 | ||||
3975 | static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, | |||
3976 | const TargetInstrInfo &TII, | |||
3977 | const X86Subtarget &Subtarget) { | |||
3978 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
3979 | DebugLoc DL = MIB->getDebugLoc(); | |||
3980 | int64_t Imm = MIB->getOperand(1).getImm(); | |||
3981 | assert(Imm != 0 && "Using push/pop for 0 is not efficient.")((Imm != 0 && "Using push/pop for 0 is not efficient." ) ? static_cast<void> (0) : __assert_fail ("Imm != 0 && \"Using push/pop for 0 is not efficient.\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3981, __PRETTY_FUNCTION__)); | |||
3982 | MachineBasicBlock::iterator I = MIB.getInstr(); | |||
3983 | ||||
3984 | int StackAdjustment; | |||
3985 | ||||
3986 | if (Subtarget.is64Bit()) { | |||
3987 | assert(MIB->getOpcode() == X86::MOV64ImmSExti8 ||((MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode () == X86::MOV32ImmSExti8) ? static_cast<void> (0) : __assert_fail ("MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode() == X86::MOV32ImmSExti8" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3988, __PRETTY_FUNCTION__)) | |||
3988 | MIB->getOpcode() == X86::MOV32ImmSExti8)((MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode () == X86::MOV32ImmSExti8) ? static_cast<void> (0) : __assert_fail ("MIB->getOpcode() == X86::MOV64ImmSExti8 || MIB->getOpcode() == X86::MOV32ImmSExti8" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 3988, __PRETTY_FUNCTION__)); | |||
3989 | ||||
3990 | // Can't use push/pop lowering if the function might write to the red zone. | |||
3991 | X86MachineFunctionInfo *X86FI = | |||
3992 | MBB.getParent()->getInfo<X86MachineFunctionInfo>(); | |||
3993 | if (X86FI->getUsesRedZone()) { | |||
3994 | MIB->setDesc(TII.get(MIB->getOpcode() == | |||
3995 | X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri)); | |||
3996 | return true; | |||
3997 | } | |||
3998 | ||||
3999 | // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and | |||
4000 | // widen the register if necessary. | |||
4001 | StackAdjustment = 8; | |||
4002 | BuildMI(MBB, I, DL, TII.get(X86::PUSH64i8)).addImm(Imm); | |||
4003 | MIB->setDesc(TII.get(X86::POP64r)); | |||
4004 | MIB->getOperand(0) | |||
4005 | .setReg(getX86SubSuperRegister(MIB->getOperand(0).getReg(), 64)); | |||
4006 | } else { | |||
4007 | assert(MIB->getOpcode() == X86::MOV32ImmSExti8)((MIB->getOpcode() == X86::MOV32ImmSExti8) ? static_cast< void> (0) : __assert_fail ("MIB->getOpcode() == X86::MOV32ImmSExti8" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 4007, __PRETTY_FUNCTION__)); | |||
4008 | StackAdjustment = 4; | |||
4009 | BuildMI(MBB, I, DL, TII.get(X86::PUSH32i8)).addImm(Imm); | |||
4010 | MIB->setDesc(TII.get(X86::POP32r)); | |||
4011 | } | |||
4012 | ||||
4013 | // Build CFI if necessary. | |||
4014 | MachineFunction &MF = *MBB.getParent(); | |||
4015 | const X86FrameLowering *TFL = Subtarget.getFrameLowering(); | |||
4016 | bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); | |||
4017 | bool NeedsDwarfCFI = | |||
4018 | !IsWin64Prologue && | |||
4019 | (MF.getMMI().hasDebugInfo() || MF.getFunction().needsUnwindTableEntry()); | |||
4020 | bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI; | |||
4021 | if (EmitCFI) { | |||
4022 | TFL->BuildCFI(MBB, I, DL, | |||
4023 | MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment)); | |||
4024 | TFL->BuildCFI(MBB, std::next(I), DL, | |||
4025 | MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment)); | |||
4026 | } | |||
4027 | ||||
4028 | return true; | |||
4029 | } | |||
4030 | ||||
4031 | // LoadStackGuard has so far only been implemented for 64-bit MachO. Different | |||
4032 | // code sequence is needed for other targets. | |||
4033 | static void expandLoadStackGuard(MachineInstrBuilder &MIB, | |||
4034 | const TargetInstrInfo &TII) { | |||
4035 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
4036 | DebugLoc DL = MIB->getDebugLoc(); | |||
4037 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4038 | const GlobalValue *GV = | |||
4039 | cast<GlobalValue>((*MIB->memoperands_begin())->getValue()); | |||
4040 | auto Flags = MachineMemOperand::MOLoad | | |||
4041 | MachineMemOperand::MODereferenceable | | |||
4042 | MachineMemOperand::MOInvariant; | |||
4043 | MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( | |||
4044 | MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, 8); | |||
4045 | MachineBasicBlock::iterator I = MIB.getInstr(); | |||
4046 | ||||
4047 | BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg).addReg(X86::RIP).addImm(1) | |||
4048 | .addReg(0).addGlobalAddress(GV, 0, X86II::MO_GOTPCREL).addReg(0) | |||
4049 | .addMemOperand(MMO); | |||
4050 | MIB->setDebugLoc(DL); | |||
4051 | MIB->setDesc(TII.get(X86::MOV64rm)); | |||
4052 | MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0); | |||
4053 | } | |||
4054 | ||||
4055 | static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) { | |||
4056 | MachineBasicBlock &MBB = *MIB->getParent(); | |||
4057 | MachineFunction &MF = *MBB.getParent(); | |||
4058 | const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>(); | |||
4059 | const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); | |||
4060 | unsigned XorOp = | |||
4061 | MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr; | |||
4062 | MIB->setDesc(TII.get(XorOp)); | |||
4063 | MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef); | |||
4064 | return true; | |||
4065 | } | |||
4066 | ||||
4067 | // This is used to handle spills for 128/256-bit registers when we have AVX512, | |||
4068 | // but not VLX. If it uses an extended register we need to use an instruction | |||
4069 | // that loads the lower 128/256-bit, but is available with only AVX512F. | |||
4070 | static bool expandNOVLXLoad(MachineInstrBuilder &MIB, | |||
4071 | const TargetRegisterInfo *TRI, | |||
4072 | const MCInstrDesc &LoadDesc, | |||
4073 | const MCInstrDesc &BroadcastDesc, | |||
4074 | unsigned SubIdx) { | |||
4075 | unsigned DestReg = MIB->getOperand(0).getReg(); | |||
4076 | // Check if DestReg is XMM16-31 or YMM16-31. | |||
4077 | if (TRI->getEncodingValue(DestReg) < 16) { | |||
4078 | // We can use a normal VEX encoded load. | |||
4079 | MIB->setDesc(LoadDesc); | |||
4080 | } else { | |||
4081 | // Use a 128/256-bit VBROADCAST instruction. | |||
4082 | MIB->setDesc(BroadcastDesc); | |||
4083 | // Change the destination to a 512-bit register. | |||
4084 | DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass); | |||
4085 | MIB->getOperand(0).setReg(DestReg); | |||
4086 | } | |||
4087 | return true; | |||
4088 | } | |||
4089 | ||||
4090 | // This is used to handle spills for 128/256-bit registers when we have AVX512, | |||
4091 | // but not VLX. If it uses an extended register we need to use an instruction | |||
4092 | // that stores the lower 128/256-bit, but is available with only AVX512F. | |||
4093 | static bool expandNOVLXStore(MachineInstrBuilder &MIB, | |||
4094 | const TargetRegisterInfo *TRI, | |||
4095 | const MCInstrDesc &StoreDesc, | |||
4096 | const MCInstrDesc &ExtractDesc, | |||
4097 | unsigned SubIdx) { | |||
4098 | unsigned SrcReg = MIB->getOperand(X86::AddrNumOperands).getReg(); | |||
4099 | // Check if DestReg is XMM16-31 or YMM16-31. | |||
4100 | if (TRI->getEncodingValue(SrcReg) < 16) { | |||
4101 | // We can use a normal VEX encoded store. | |||
4102 | MIB->setDesc(StoreDesc); | |||
4103 | } else { | |||
4104 | // Use a VEXTRACTF instruction. | |||
4105 | MIB->setDesc(ExtractDesc); | |||
4106 | // Change the destination to a 512-bit register. | |||
4107 | SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass); | |||
4108 | MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg); | |||
4109 | MIB.addImm(0x0); // Append immediate to extract from the lower bits. | |||
4110 | } | |||
4111 | ||||
4112 | return true; | |||
4113 | } | |||
4114 | bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { | |||
4115 | bool HasAVX = Subtarget.hasAVX(); | |||
4116 | MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); | |||
4117 | switch (MI.getOpcode()) { | |||
4118 | case X86::MOV32r0: | |||
4119 | return Expand2AddrUndef(MIB, get(X86::XOR32rr)); | |||
4120 | case X86::MOV32r1: | |||
4121 | return expandMOV32r1(MIB, *this, /*MinusOne=*/ false); | |||
4122 | case X86::MOV32r_1: | |||
4123 | return expandMOV32r1(MIB, *this, /*MinusOne=*/ true); | |||
4124 | case X86::MOV32ImmSExti8: | |||
4125 | case X86::MOV64ImmSExti8: | |||
4126 | return ExpandMOVImmSExti8(MIB, *this, Subtarget); | |||
4127 | case X86::SETB_C8r: | |||
4128 | return Expand2AddrUndef(MIB, get(X86::SBB8rr)); | |||
4129 | case X86::SETB_C16r: | |||
4130 | return Expand2AddrUndef(MIB, get(X86::SBB16rr)); | |||
4131 | case X86::SETB_C32r: | |||
4132 | return Expand2AddrUndef(MIB, get(X86::SBB32rr)); | |||
4133 | case X86::SETB_C64r: | |||
4134 | return Expand2AddrUndef(MIB, get(X86::SBB64rr)); | |||
4135 | case X86::MMX_SET0: | |||
4136 | return Expand2AddrUndef(MIB, get(X86::MMX_PXORirr)); | |||
4137 | case X86::V_SET0: | |||
4138 | case X86::FsFLD0SS: | |||
4139 | case X86::FsFLD0SD: | |||
4140 | return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr)); | |||
4141 | case X86::AVX_SET0: { | |||
4142 | assert(HasAVX && "AVX not supported")((HasAVX && "AVX not supported") ? static_cast<void > (0) : __assert_fail ("HasAVX && \"AVX not supported\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 4142, __PRETTY_FUNCTION__)); | |||
4143 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
4144 | unsigned SrcReg = MIB->getOperand(0).getReg(); | |||
4145 | unsigned XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); | |||
4146 | MIB->getOperand(0).setReg(XReg); | |||
4147 | Expand2AddrUndef(MIB, get(X86::VXORPSrr)); | |||
4148 | MIB.addReg(SrcReg, RegState::ImplicitDefine); | |||
4149 | return true; | |||
4150 | } | |||
4151 | case X86::AVX512_128_SET0: | |||
4152 | case X86::AVX512_FsFLD0SS: | |||
4153 | case X86::AVX512_FsFLD0SD: { | |||
4154 | bool HasVLX = Subtarget.hasVLX(); | |||
4155 | unsigned SrcReg = MIB->getOperand(0).getReg(); | |||
4156 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
4157 | if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) | |||
4158 | return Expand2AddrUndef(MIB, | |||
4159 | get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); | |||
4160 | // Extended register without VLX. Use a larger XOR. | |||
4161 | SrcReg = | |||
4162 | TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass); | |||
4163 | MIB->getOperand(0).setReg(SrcReg); | |||
4164 | return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); | |||
4165 | } | |||
4166 | case X86::AVX512_256_SET0: | |||
4167 | case X86::AVX512_512_SET0: { | |||
4168 | bool HasVLX = Subtarget.hasVLX(); | |||
4169 | unsigned SrcReg = MIB->getOperand(0).getReg(); | |||
4170 | const TargetRegisterInfo *TRI = &getRegisterInfo(); | |||
4171 | if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) { | |||
4172 | unsigned XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); | |||
4173 | MIB->getOperand(0).setReg(XReg); | |||
4174 | Expand2AddrUndef(MIB, | |||
4175 | get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); | |||
4176 | MIB.addReg(SrcReg, RegState::ImplicitDefine); | |||
4177 | return true; | |||
4178 | } | |||
4179 | return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); | |||
4180 | } | |||
4181 | case X86::V_SETALLONES: | |||
4182 | return Expand2AddrUndef(MIB, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr)); | |||
4183 | case X86::AVX2_SETALLONES: | |||
4184 | return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr)); | |||
4185 | case X86::AVX1_SETALLONES: { | |||
4186 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4187 | // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS. | |||
4188 | MIB->setDesc(get(X86::VCMPPSYrri)); | |||
4189 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf); | |||
4190 | return true; | |||
4191 | } | |||
4192 | case X86::AVX512_512_SETALLONES: { | |||
4193 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4194 | MIB->setDesc(get(X86::VPTERNLOGDZrri)); | |||
4195 | // VPTERNLOGD needs 3 register inputs and an immediate. | |||
4196 | // 0xff will return 1s for any input. | |||
4197 | MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef) | |||
4198 | .addReg(Reg, RegState::Undef).addImm(0xff); | |||
4199 | return true; | |||
4200 | } | |||
4201 | case X86::AVX512_512_SEXT_MASK_32: | |||
4202 | case X86::AVX512_512_SEXT_MASK_64: { | |||
4203 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4204 | unsigned MaskReg = MIB->getOperand(1).getReg(); | |||
4205 | unsigned MaskState = getRegState(MIB->getOperand(1)); | |||
4206 | unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64) ? | |||
4207 | X86::VPTERNLOGQZrrikz : X86::VPTERNLOGDZrrikz; | |||
4208 | MI.RemoveOperand(1); | |||
4209 | MIB->setDesc(get(Opc)); | |||
4210 | // VPTERNLOG needs 3 register inputs and an immediate. | |||
4211 | // 0xff will return 1s for any input. | |||
4212 | MIB.addReg(Reg, RegState::Undef).addReg(MaskReg, MaskState) | |||
4213 | .addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xff); | |||
4214 | return true; | |||
4215 | } | |||
4216 | case X86::VMOVAPSZ128rm_NOVLX: | |||
4217 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm), | |||
4218 | get(X86::VBROADCASTF32X4rm), X86::sub_xmm); | |||
4219 | case X86::VMOVUPSZ128rm_NOVLX: | |||
4220 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm), | |||
4221 | get(X86::VBROADCASTF32X4rm), X86::sub_xmm); | |||
4222 | case X86::VMOVAPSZ256rm_NOVLX: | |||
4223 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm), | |||
4224 | get(X86::VBROADCASTF64X4rm), X86::sub_ymm); | |||
4225 | case X86::VMOVUPSZ256rm_NOVLX: | |||
4226 | return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm), | |||
4227 | get(X86::VBROADCASTF64X4rm), X86::sub_ymm); | |||
4228 | case X86::VMOVAPSZ128mr_NOVLX: | |||
4229 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr), | |||
4230 | get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); | |||
4231 | case X86::VMOVUPSZ128mr_NOVLX: | |||
4232 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr), | |||
4233 | get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); | |||
4234 | case X86::VMOVAPSZ256mr_NOVLX: | |||
4235 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr), | |||
4236 | get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); | |||
4237 | case X86::VMOVUPSZ256mr_NOVLX: | |||
4238 | return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr), | |||
4239 | get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); | |||
4240 | case X86::MOV32ri64: { | |||
4241 | unsigned Reg = MIB->getOperand(0).getReg(); | |||
4242 | unsigned Reg32 = RI.getSubReg(Reg, X86::sub_32bit); | |||
4243 | MI.setDesc(get(X86::MOV32ri)); | |||
4244 | MIB->getOperand(0).setReg(Reg32); | |||
4245 | MIB.addReg(Reg, RegState::ImplicitDefine); | |||
4246 | return true; | |||
4247 | } | |||
4248 | ||||
4249 | // KNL does not recognize dependency-breaking idioms for mask registers, | |||
4250 | // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1. | |||
4251 | // Using %k0 as the undef input register is a performance heuristic based | |||
4252 | // on the assumption that %k0 is used less frequently than the other mask | |||
4253 | // registers, since it is not usable as a write mask. | |||
4254 | // FIXME: A more advanced approach would be to choose the best input mask | |||
4255 | // register based on context. | |||
4256 | case X86::KSET0W: return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0); | |||
4257 | case X86::KSET0D: return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0); | |||
4258 | case X86::KSET0Q: return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0); | |||
4259 | case X86::KSET1W: return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0); | |||
4260 | case X86::KSET1D: return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0); | |||
4261 | case X86::KSET1Q: return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0); | |||
4262 | case TargetOpcode::LOAD_STACK_GUARD: | |||
4263 | expandLoadStackGuard(MIB, *this); | |||
4264 | return true; | |||
4265 | case X86::XOR64_FP: | |||
4266 | case X86::XOR32_FP: | |||
4267 | return expandXorFP(MIB, *this); | |||
4268 | } | |||
4269 | return false; | |||
4270 | } | |||
4271 | ||||
4272 | /// Return true for all instructions that only update | |||
4273 | /// the first 32 or 64-bits of the destination register and leave the rest | |||
4274 | /// unmodified. This can be used to avoid folding loads if the instructions | |||
4275 | /// only update part of the destination register, and the non-updated part is | |||
4276 | /// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these | |||
4277 | /// instructions breaks the partial register dependency and it can improve | |||
4278 | /// performance. e.g.: | |||
4279 | /// | |||
4280 | /// movss (%rdi), %xmm0 | |||
4281 | /// cvtss2sd %xmm0, %xmm0 | |||
4282 | /// | |||
4283 | /// Instead of | |||
4284 | /// cvtss2sd (%rdi), %xmm0 | |||
4285 | /// | |||
4286 | /// FIXME: This should be turned into a TSFlags. | |||
4287 | /// | |||
4288 | static bool hasPartialRegUpdate(unsigned Opcode, | |||
4289 | const X86Subtarget &Subtarget) { | |||
4290 | switch (Opcode) { | |||
4291 | case X86::CVTSI2SSrr: | |||
4292 | case X86::CVTSI2SSrm: | |||
4293 | case X86::CVTSI642SSrr: | |||
4294 | case X86::CVTSI642SSrm: | |||
4295 | case X86::CVTSI2SDrr: | |||
4296 | case X86::CVTSI2SDrm: | |||
4297 | case X86::CVTSI642SDrr: | |||
4298 | case X86::CVTSI642SDrm: | |||
4299 | case X86::CVTSD2SSrr: | |||
4300 | case X86::CVTSD2SSrm: | |||
4301 | case X86::CVTSS2SDrr: | |||
4302 | case X86::CVTSS2SDrm: | |||
4303 | case X86::MOVHPDrm: | |||
4304 | case X86::MOVHPSrm: | |||
4305 | case X86::MOVLPDrm: | |||
4306 | case X86::MOVLPSrm: | |||
4307 | case X86::RCPSSr: | |||
4308 | case X86::RCPSSm: | |||
4309 | case X86::RCPSSr_Int: | |||
4310 | case X86::RCPSSm_Int: | |||
4311 | case X86::ROUNDSDr: | |||
4312 | case X86::ROUNDSDm: | |||
4313 | case X86::ROUNDSSr: | |||
4314 | case X86::ROUNDSSm: | |||
4315 | case X86::RSQRTSSr: | |||
4316 | case X86::RSQRTSSm: | |||
4317 | case X86::RSQRTSSr_Int: | |||
4318 | case X86::RSQRTSSm_Int: | |||
4319 | case X86::SQRTSSr: | |||
4320 | case X86::SQRTSSm: | |||
4321 | case X86::SQRTSSr_Int: | |||
4322 | case X86::SQRTSSm_Int: | |||
4323 | case X86::SQRTSDr: | |||
4324 | case X86::SQRTSDm: | |||
4325 | case X86::SQRTSDr_Int: | |||
4326 | case X86::SQRTSDm_Int: | |||
4327 | return true; | |||
4328 | // GPR | |||
4329 | case X86::POPCNT32rm: | |||
4330 | case X86::POPCNT32rr: | |||
4331 | case X86::POPCNT64rm: | |||
4332 | case X86::POPCNT64rr: | |||
4333 | return Subtarget.hasPOPCNTFalseDeps(); | |||
4334 | case X86::LZCNT32rm: | |||
4335 | case X86::LZCNT32rr: | |||
4336 | case X86::LZCNT64rm: | |||
4337 | case X86::LZCNT64rr: | |||
4338 | case X86::TZCNT32rm: | |||
4339 | case X86::TZCNT32rr: | |||
4340 | case X86::TZCNT64rm: | |||
4341 | case X86::TZCNT64rr: | |||
4342 | return Subtarget.hasLZCNTFalseDeps(); | |||
4343 | } | |||
4344 | ||||
4345 | return false; | |||
4346 | } | |||
4347 | ||||
4348 | /// Inform the BreakFalseDeps pass how many idle | |||
4349 | /// instructions we would like before a partial register update. | |||
4350 | unsigned X86InstrInfo::getPartialRegUpdateClearance( | |||
4351 | const MachineInstr &MI, unsigned OpNum, | |||
4352 | const TargetRegisterInfo *TRI) const { | |||
4353 | if (OpNum != 0 || !hasPartialRegUpdate(MI.getOpcode(), Subtarget)) | |||
4354 | return 0; | |||
4355 | ||||
4356 | // If MI is marked as reading Reg, the partial register update is wanted. | |||
4357 | const MachineOperand &MO = MI.getOperand(0); | |||
4358 | unsigned Reg = MO.getReg(); | |||
4359 | if (TargetRegisterInfo::isVirtualRegister(Reg)) { | |||
4360 | if (MO.readsReg() || MI.readsVirtualRegister(Reg)) | |||
4361 | return 0; | |||
4362 | } else { | |||
4363 | if (MI.readsRegister(Reg, TRI)) | |||
4364 | return 0; | |||
4365 | } | |||
4366 | ||||
4367 | // If any instructions in the clearance range are reading Reg, insert a | |||
4368 | // dependency breaking instruction, which is inexpensive and is likely to | |||
4369 | // be hidden in other instruction's cycles. | |||
4370 | return PartialRegUpdateClearance; | |||
4371 | } | |||
4372 | ||||
4373 | // Return true for any instruction the copies the high bits of the first source | |||
4374 | // operand into the unused high bits of the destination operand. | |||
4375 | static bool hasUndefRegUpdate(unsigned Opcode) { | |||
4376 | switch (Opcode) { | |||
4377 | case X86::VCVTSI2SSrr: | |||
4378 | case X86::VCVTSI2SSrm: | |||
4379 | case X86::VCVTSI2SSrr_Int: | |||
4380 | case X86::VCVTSI2SSrm_Int: | |||
4381 | case X86::VCVTSI642SSrr: | |||
4382 | case X86::VCVTSI642SSrm: | |||
4383 | case X86::VCVTSI642SSrr_Int: | |||
4384 | case X86::VCVTSI642SSrm_Int: | |||
4385 | case X86::VCVTSI2SDrr: | |||
4386 | case X86::VCVTSI2SDrm: | |||
4387 | case X86::VCVTSI2SDrr_Int: | |||
4388 | case X86::VCVTSI2SDrm_Int: | |||
4389 | case X86::VCVTSI642SDrr: | |||
4390 | case X86::VCVTSI642SDrm: | |||
4391 | case X86::VCVTSI642SDrr_Int: | |||
4392 | case X86::VCVTSI642SDrm_Int: | |||
4393 | case X86::VCVTSD2SSrr: | |||
4394 | case X86::VCVTSD2SSrm: | |||
4395 | case X86::VCVTSD2SSrr_Int: | |||
4396 | case X86::VCVTSD2SSrm_Int: | |||
4397 | case X86::VCVTSS2SDrr: | |||
4398 | case X86::VCVTSS2SDrm: | |||
4399 | case X86::VCVTSS2SDrr_Int: | |||
4400 | case X86::VCVTSS2SDrm_Int: | |||
4401 | case X86::VRCPSSr: | |||
4402 | case X86::VRCPSSr_Int: | |||
4403 | case X86::VRCPSSm: | |||
4404 | case X86::VRCPSSm_Int: | |||
4405 | case X86::VROUNDSDr: | |||
4406 | case X86::VROUNDSDm: | |||
4407 | case X86::VROUNDSDr_Int: | |||
4408 | case X86::VROUNDSDm_Int: | |||
4409 | case X86::VROUNDSSr: | |||
4410 | case X86::VROUNDSSm: | |||
4411 | case X86::VROUNDSSr_Int: | |||
4412 | case X86::VROUNDSSm_Int: | |||
4413 | case X86::VRSQRTSSr: | |||
4414 | case X86::VRSQRTSSr_Int: | |||
4415 | case X86::VRSQRTSSm: | |||
4416 | case X86::VRSQRTSSm_Int: | |||
4417 | case X86::VSQRTSSr: | |||
4418 | case X86::VSQRTSSr_Int: | |||
4419 | case X86::VSQRTSSm: | |||
4420 | case X86::VSQRTSSm_Int: | |||
4421 | case X86::VSQRTSDr: | |||
4422 | case X86::VSQRTSDr_Int: | |||
4423 | case X86::VSQRTSDm: | |||
4424 | case X86::VSQRTSDm_Int: | |||
4425 | // AVX-512 | |||
4426 | case X86::VCVTSI2SSZrr: | |||
4427 | case X86::VCVTSI2SSZrm: | |||
4428 | case X86::VCVTSI2SSZrr_Int: | |||
4429 | case X86::VCVTSI2SSZrrb_Int: | |||
4430 | case X86::VCVTSI2SSZrm_Int: | |||
4431 | case X86::VCVTSI642SSZrr: | |||
4432 | case X86::VCVTSI642SSZrm: | |||
4433 | case X86::VCVTSI642SSZrr_Int: | |||
4434 | case X86::VCVTSI642SSZrrb_Int: | |||
4435 | case X86::VCVTSI642SSZrm_Int: | |||
4436 | case X86::VCVTSI2SDZrr: | |||
4437 | case X86::VCVTSI2SDZrm: | |||
4438 | case X86::VCVTSI2SDZrr_Int: | |||
4439 | case X86::VCVTSI2SDZrrb_Int: | |||
4440 | case X86::VCVTSI2SDZrm_Int: | |||
4441 | case X86::VCVTSI642SDZrr: | |||
4442 | case X86::VCVTSI642SDZrm: | |||
4443 | case X86::VCVTSI642SDZrr_Int: | |||
4444 | case X86::VCVTSI642SDZrrb_Int: | |||
4445 | case X86::VCVTSI642SDZrm_Int: | |||
4446 | case X86::VCVTUSI2SSZrr: | |||
4447 | case X86::VCVTUSI2SSZrm: | |||
4448 | case X86::VCVTUSI2SSZrr_Int: | |||
4449 | case X86::VCVTUSI2SSZrrb_Int: | |||
4450 | case X86::VCVTUSI2SSZrm_Int: | |||
4451 | case X86::VCVTUSI642SSZrr: | |||
4452 | case X86::VCVTUSI642SSZrm: | |||
4453 | case X86::VCVTUSI642SSZrr_Int: | |||
4454 | case X86::VCVTUSI642SSZrrb_Int: | |||
4455 | case X86::VCVTUSI642SSZrm_Int: | |||
4456 | case X86::VCVTUSI2SDZrr: | |||
4457 | case X86::VCVTUSI2SDZrm: | |||
4458 | case X86::VCVTUSI2SDZrr_Int: | |||
4459 | case X86::VCVTUSI2SDZrm_Int: | |||
4460 | case X86::VCVTUSI642SDZrr: | |||
4461 | case X86::VCVTUSI642SDZrm: | |||
4462 | case X86::VCVTUSI642SDZrr_Int: | |||
4463 | case X86::VCVTUSI642SDZrrb_Int: | |||
4464 | case X86::VCVTUSI642SDZrm_Int: | |||
4465 | case X86::VCVTSD2SSZrr: | |||
4466 | case X86::VCVTSD2SSZrr_Int: | |||
4467 | case X86::VCVTSD2SSZrrb_Int: | |||
4468 | case X86::VCVTSD2SSZrm: | |||
4469 | case X86::VCVTSD2SSZrm_Int: | |||
4470 | case X86::VCVTSS2SDZrr: | |||
4471 | case X86::VCVTSS2SDZrr_Int: | |||
4472 | case X86::VCVTSS2SDZrrb_Int: | |||
4473 | case X86::VCVTSS2SDZrm: | |||
4474 | case X86::VCVTSS2SDZrm_Int: | |||
4475 | case X86::VGETEXPSDZr: | |||
4476 | case X86::VGETEXPSDZrb: | |||
4477 | case X86::VGETEXPSDZm: | |||
4478 | case X86::VGETEXPSSZr: | |||
4479 | case X86::VGETEXPSSZrb: | |||
4480 | case X86::VGETEXPSSZm: | |||
4481 | case X86::VGETMANTSDZrri: | |||
4482 | case X86::VGETMANTSDZrrib: | |||
4483 | case X86::VGETMANTSDZrmi: | |||
4484 | case X86::VGETMANTSSZrri: | |||
4485 | case X86::VGETMANTSSZrrib: | |||
4486 | case X86::VGETMANTSSZrmi: | |||
4487 | case X86::VRNDSCALESDZr: | |||
4488 | case X86::VRNDSCALESDZr_Int: | |||
4489 | case X86::VRNDSCALESDZrb_Int: | |||
4490 | case X86::VRNDSCALESDZm: | |||
4491 | case X86::VRNDSCALESDZm_Int: | |||
4492 | case X86::VRNDSCALESSZr: | |||
4493 | case X86::VRNDSCALESSZr_Int: | |||
4494 | case X86::VRNDSCALESSZrb_Int: | |||
4495 | case X86::VRNDSCALESSZm: | |||
4496 | case X86::VRNDSCALESSZm_Int: | |||
4497 | case X86::VRCP14SDZrr: | |||
4498 | case X86::VRCP14SDZrm: | |||
4499 | case X86::VRCP14SSZrr: | |||
4500 | case X86::VRCP14SSZrm: | |||
4501 | case X86::VRCP28SDZr: | |||
4502 | case X86::VRCP28SDZrb: | |||
4503 | case X86::VRCP28SDZm: | |||
4504 | case X86::VRCP28SSZr: | |||
4505 | case X86::VRCP28SSZrb: | |||
4506 | case X86::VRCP28SSZm: | |||
4507 | case X86::VREDUCESSZrmi: | |||
4508 | case X86::VREDUCESSZrri: | |||
4509 | case X86::VREDUCESSZrrib: | |||
4510 | case X86::VRSQRT14SDZrr: | |||
4511 | case X86::VRSQRT14SDZrm: | |||
4512 | case X86::VRSQRT14SSZrr: | |||
4513 | case X86::VRSQRT14SSZrm: | |||
4514 | case X86::VRSQRT28SDZr: | |||
4515 | case X86::VRSQRT28SDZrb: | |||
4516 | case X86::VRSQRT28SDZm: | |||
4517 | case X86::VRSQRT28SSZr: | |||
4518 | case X86::VRSQRT28SSZrb: | |||
4519 | case X86::VRSQRT28SSZm: | |||
4520 | case X86::VSQRTSSZr: | |||
4521 | case X86::VSQRTSSZr_Int: | |||
4522 | case X86::VSQRTSSZrb_Int: | |||
4523 | case X86::VSQRTSSZm: | |||
4524 | case X86::VSQRTSSZm_Int: | |||
4525 | case X86::VSQRTSDZr: | |||
4526 | case X86::VSQRTSDZr_Int: | |||
4527 | case X86::VSQRTSDZrb_Int: | |||
4528 | case X86::VSQRTSDZm: | |||
4529 | case X86::VSQRTSDZm_Int: | |||
4530 | return true; | |||
4531 | } | |||
4532 | ||||
4533 | return false; | |||
4534 | } | |||
4535 | ||||
4536 | /// Inform the BreakFalseDeps pass how many idle instructions we would like | |||
4537 | /// before certain undef register reads. | |||
4538 | /// | |||
4539 | /// This catches the VCVTSI2SD family of instructions: | |||
4540 | /// | |||
4541 | /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 | |||
4542 | /// | |||
4543 | /// We should to be careful *not* to catch VXOR idioms which are presumably | |||
4544 | /// handled specially in the pipeline: | |||
4545 | /// | |||
4546 | /// vxorps undef %xmm1, undef %xmm1, %xmm1 | |||
4547 | /// | |||
4548 | /// Like getPartialRegUpdateClearance, this makes a strong assumption that the | |||
4549 | /// high bits that are passed-through are not live. | |||
4550 | unsigned | |||
4551 | X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned &OpNum, | |||
4552 | const TargetRegisterInfo *TRI) const { | |||
4553 | if (!hasUndefRegUpdate(MI.getOpcode())) | |||
4554 | return 0; | |||
4555 | ||||
4556 | // Set the OpNum parameter to the first source operand. | |||
4557 | OpNum = 1; | |||
4558 | ||||
4559 | const MachineOperand &MO = MI.getOperand(OpNum); | |||
4560 | if (MO.isUndef() && TargetRegisterInfo::isPhysicalRegister(MO.getReg())) { | |||
4561 | return UndefRegClearance; | |||
4562 | } | |||
4563 | return 0; | |||
4564 | } | |||
4565 | ||||
4566 | void X86InstrInfo::breakPartialRegDependency( | |||
4567 | MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const { | |||
4568 | unsigned Reg = MI.getOperand(OpNum).getReg(); | |||
4569 | // If MI kills this register, the false dependence is already broken. | |||
4570 | if (MI.killsRegister(Reg, TRI)) | |||
4571 | return; | |||
4572 | ||||
4573 | if (X86::VR128RegClass.contains(Reg)) { | |||
4574 | // These instructions are all floating point domain, so xorps is the best | |||
4575 | // choice. | |||
4576 | unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr; | |||
4577 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg) | |||
4578 | .addReg(Reg, RegState::Undef) | |||
4579 | .addReg(Reg, RegState::Undef); | |||
4580 | MI.addRegisterKilled(Reg, TRI, true); | |||
4581 | } else if (X86::VR256RegClass.contains(Reg)) { | |||
4582 | // Use vxorps to clear the full ymm register. | |||
4583 | // It wants to read and write the xmm sub-register. | |||
4584 | unsigned XReg = TRI->getSubReg(Reg, X86::sub_xmm); | |||
4585 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg) | |||
4586 | .addReg(XReg, RegState::Undef) | |||
4587 | .addReg(XReg, RegState::Undef) | |||
4588 | .addReg(Reg, RegState::ImplicitDefine); | |||
4589 | MI.addRegisterKilled(Reg, TRI, true); | |||
4590 | } else if (X86::GR64RegClass.contains(Reg)) { | |||
4591 | // Using XOR32rr because it has shorter encoding and zeros up the upper bits | |||
4592 | // as well. | |||
4593 | unsigned XReg = TRI->getSubReg(Reg, X86::sub_32bit); | |||
4594 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg) | |||
4595 | .addReg(XReg, RegState::Undef) | |||
4596 | .addReg(XReg, RegState::Undef) | |||
4597 | .addReg(Reg, RegState::ImplicitDefine); | |||
4598 | MI.addRegisterKilled(Reg, TRI, true); | |||
4599 | } else if (X86::GR32RegClass.contains(Reg)) { | |||
4600 | BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg) | |||
4601 | .addReg(Reg, RegState::Undef) | |||
4602 | .addReg(Reg, RegState::Undef); | |||
4603 | MI.addRegisterKilled(Reg, TRI, true); | |||
4604 | } | |||
4605 | } | |||
4606 | ||||
4607 | static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs, | |||
4608 | int PtrOffset = 0) { | |||
4609 | unsigned NumAddrOps = MOs.size(); | |||
4610 | ||||
4611 | if (NumAddrOps < 4) { | |||
4612 | // FrameIndex only - add an immediate offset (whether its zero or not). | |||
4613 | for (unsigned i = 0; i != NumAddrOps; ++i) | |||
4614 | MIB.add(MOs[i]); | |||
4615 | addOffset(MIB, PtrOffset); | |||
4616 | } else { | |||
4617 | // General Memory Addressing - we need to add any offset to an existing | |||
4618 | // offset. | |||
4619 | assert(MOs.size() == 5 && "Unexpected memory operand list length")((MOs.size() == 5 && "Unexpected memory operand list length" ) ? static_cast<void> (0) : __assert_fail ("MOs.size() == 5 && \"Unexpected memory operand list length\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 4619, __PRETTY_FUNCTION__)); | |||
4620 | for (unsigned i = 0; i != NumAddrOps; ++i) { | |||
4621 | const MachineOperand &MO = MOs[i]; | |||
4622 | if (i == 3 && PtrOffset != 0) { | |||
4623 | MIB.addDisp(MO, PtrOffset); | |||
4624 | } else { | |||
4625 | MIB.add(MO); | |||
4626 | } | |||
4627 | } | |||
4628 | } | |||
4629 | } | |||
4630 | ||||
4631 | static void updateOperandRegConstraints(MachineFunction &MF, | |||
4632 | MachineInstr &NewMI, | |||
4633 | const TargetInstrInfo &TII) { | |||
4634 | MachineRegisterInfo &MRI = MF.getRegInfo(); | |||
4635 | const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); | |||
4636 | ||||
4637 | for (int Idx : llvm::seq<int>(0, NewMI.getNumOperands())) { | |||
4638 | MachineOperand &MO = NewMI.getOperand(Idx); | |||
4639 | // We only need to update constraints on virtual register operands. | |||
4640 | if (!MO.isReg()) | |||
4641 | continue; | |||
4642 | unsigned Reg = MO.getReg(); | |||
4643 | if (!TRI.isVirtualRegister(Reg)) | |||
4644 | continue; | |||
4645 | ||||
4646 | auto *NewRC = MRI.constrainRegClass( | |||
4647 | Reg, TII.getRegClass(NewMI.getDesc(), Idx, &TRI, MF)); | |||
4648 | if (!NewRC) { | |||
4649 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false) | |||
4650 | dbgs() << "WARNING: Unable to update register constraint for operand "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false) | |||
4651 | << Idx << " of instruction:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false) | |||
4652 | NewMI.dump(); dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("x86-instr-info")) { dbgs() << "WARNING: Unable to update register constraint for operand " << Idx << " of instruction:\n"; NewMI.dump(); dbgs () << "\n"; } } while (false); | |||
4653 | } | |||
4654 | } | |||
4655 | } | |||
4656 | ||||
4657 | static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, | |||
4658 | ArrayRef<MachineOperand> MOs, | |||
4659 | MachineBasicBlock::iterator InsertPt, | |||
4660 | MachineInstr &MI, | |||
4661 | const TargetInstrInfo &TII) { | |||
4662 | // Create the base instruction with the memory operand as the first part. | |||
4663 | // Omit the implicit operands, something BuildMI can't do. | |||
4664 | MachineInstr *NewMI = | |||
4665 | MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); | |||
4666 | MachineInstrBuilder MIB(MF, NewMI); | |||
4667 | addOperands(MIB, MOs); | |||
4668 | ||||
4669 | // Loop over the rest of the ri operands, converting them over. | |||
4670 | unsigned NumOps = MI.getDesc().getNumOperands() - 2; | |||
4671 | for (unsigned i = 0; i != NumOps; ++i) { | |||
4672 | MachineOperand &MO = MI.getOperand(i + 2); | |||
4673 | MIB.add(MO); | |||
4674 | } | |||
4675 | for (unsigned i = NumOps + 2, e = MI.getNumOperands(); i != e; ++i) { | |||
4676 | MachineOperand &MO = MI.getOperand(i); | |||
4677 | MIB.add(MO); | |||
4678 | } | |||
4679 | ||||
4680 | updateOperandRegConstraints(MF, *NewMI, TII); | |||
4681 | ||||
4682 | MachineBasicBlock *MBB = InsertPt->getParent(); | |||
4683 | MBB->insert(InsertPt, NewMI); | |||
4684 | ||||
4685 | return MIB; | |||
4686 | } | |||
4687 | ||||
4688 | static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode, | |||
4689 | unsigned OpNo, ArrayRef<MachineOperand> MOs, | |||
4690 | MachineBasicBlock::iterator InsertPt, | |||
4691 | MachineInstr &MI, const TargetInstrInfo &TII, | |||
4692 | int PtrOffset = 0) { | |||
4693 | // Omit the implicit operands, something BuildMI can't do. | |||
4694 | MachineInstr *NewMI = | |||
4695 | MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); | |||
4696 | MachineInstrBuilder MIB(MF, NewMI); | |||
4697 | ||||
4698 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
4699 | MachineOperand &MO = MI.getOperand(i); | |||
4700 | if (i == OpNo) { | |||
4701 | assert(MO.isReg() && "Expected to fold into reg operand!")((MO.isReg() && "Expected to fold into reg operand!") ? static_cast<void> (0) : __assert_fail ("MO.isReg() && \"Expected to fold into reg operand!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 4701, __PRETTY_FUNCTION__)); | |||
4702 | addOperands(MIB, MOs, PtrOffset); | |||
4703 | } else { | |||
4704 | MIB.add(MO); | |||
4705 | } | |||
4706 | } | |||
4707 | ||||
4708 | updateOperandRegConstraints(MF, *NewMI, TII); | |||
4709 | ||||
4710 | MachineBasicBlock *MBB = InsertPt->getParent(); | |||
4711 | MBB->insert(InsertPt, NewMI); | |||
4712 | ||||
4713 | return MIB; | |||
4714 | } | |||
4715 | ||||
4716 | static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, | |||
4717 | ArrayRef<MachineOperand> MOs, | |||
4718 | MachineBasicBlock::iterator InsertPt, | |||
4719 | MachineInstr &MI) { | |||
4720 | MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, | |||
4721 | MI.getDebugLoc(), TII.get(Opcode)); | |||
4722 | addOperands(MIB, MOs); | |||
4723 | return MIB.addImm(0); | |||
4724 | } | |||
4725 | ||||
4726 | MachineInstr *X86InstrInfo::foldMemoryOperandCustom( | |||
4727 | MachineFunction &MF, MachineInstr &MI, unsigned OpNum, | |||
4728 | ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, | |||
4729 | unsigned Size, unsigned Align) const { | |||
4730 | switch (MI.getOpcode()) { | |||
4731 | case X86::INSERTPSrr: | |||
4732 | case X86::VINSERTPSrr: | |||
4733 | case X86::VINSERTPSZrr: | |||
4734 | // Attempt to convert the load of inserted vector into a fold load | |||
4735 | // of a single float. | |||
4736 | if (OpNum == 2) { | |||
4737 | unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); | |||
4738 | unsigned ZMask = Imm & 15; | |||
4739 | unsigned DstIdx = (Imm >> 4) & 3; | |||
4740 | unsigned SrcIdx = (Imm >> 6) & 3; | |||
4741 | ||||
4742 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
4743 | const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); | |||
4744 | unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; | |||
4745 | if (Size <= RCSize && 4 <= Align) { | |||
4746 | int PtrOffset = SrcIdx * 4; | |||
4747 | unsigned NewImm = (DstIdx << 4) | ZMask; | |||
4748 | unsigned NewOpCode = | |||
4749 | (MI.getOpcode() == X86::VINSERTPSZrr) ? X86::VINSERTPSZrm : | |||
4750 | (MI.getOpcode() == X86::VINSERTPSrr) ? X86::VINSERTPSrm : | |||
4751 | X86::INSERTPSrm; | |||
4752 | MachineInstr *NewMI = | |||
4753 | FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, PtrOffset); | |||
4754 | NewMI->getOperand(NewMI->getNumOperands() - 1).setImm(NewImm); | |||
4755 | return NewMI; | |||
4756 | } | |||
4757 | } | |||
4758 | break; | |||
4759 | case X86::MOVHLPSrr: | |||
4760 | case X86::VMOVHLPSrr: | |||
4761 | case X86::VMOVHLPSZrr: | |||
4762 | // Move the upper 64-bits of the second operand to the lower 64-bits. | |||
4763 | // To fold the load, adjust the pointer to the upper and use (V)MOVLPS. | |||
4764 | // TODO: In most cases AVX doesn't have a 8-byte alignment requirement. | |||
4765 | if (OpNum == 2) { | |||
4766 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
4767 | const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); | |||
4768 | unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; | |||
4769 | if (Size <= RCSize && 8 <= Align) { | |||
4770 | unsigned NewOpCode = | |||
4771 | (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm : | |||
4772 | (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm : | |||
4773 | X86::MOVLPSrm; | |||
4774 | MachineInstr *NewMI = | |||
4775 | FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, 8); | |||
4776 | return NewMI; | |||
4777 | } | |||
4778 | } | |||
4779 | break; | |||
4780 | }; | |||
4781 | ||||
4782 | return nullptr; | |||
4783 | } | |||
4784 | ||||
4785 | static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, MachineInstr &MI) { | |||
4786 | if (MF.getFunction().optForSize() || !hasUndefRegUpdate(MI.getOpcode()) || | |||
4787 | !MI.getOperand(1).isReg()) | |||
4788 | return false; | |||
4789 | ||||
4790 | // The are two cases we need to handle depending on where in the pipeline | |||
4791 | // the folding attempt is being made. | |||
4792 | // -Register has the undef flag set. | |||
4793 | // -Register is produced by the IMPLICIT_DEF instruction. | |||
4794 | ||||
4795 | if (MI.getOperand(1).isUndef()) | |||
4796 | return true; | |||
4797 | ||||
4798 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | |||
4799 | MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(MI.getOperand(1).getReg()); | |||
4800 | return VRegDef && VRegDef->isImplicitDef(); | |||
4801 | } | |||
4802 | ||||
4803 | ||||
4804 | MachineInstr *X86InstrInfo::foldMemoryOperandImpl( | |||
4805 | MachineFunction &MF, MachineInstr &MI, unsigned OpNum, | |||
4806 | ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, | |||
4807 | unsigned Size, unsigned Align, bool AllowCommute) const { | |||
4808 | bool isSlowTwoMemOps = Subtarget.slowTwoMemOps(); | |||
4809 | bool isTwoAddrFold = false; | |||
4810 | ||||
4811 | // For CPUs that favor the register form of a call or push, | |||
4812 | // do not fold loads into calls or pushes, unless optimizing for size | |||
4813 | // aggressively. | |||
4814 | if (isSlowTwoMemOps && !MF.getFunction().optForMinSize() && | |||
4815 | (MI.getOpcode() == X86::CALL32r || MI.getOpcode() == X86::CALL64r || | |||
4816 | MI.getOpcode() == X86::PUSH16r || MI.getOpcode() == X86::PUSH32r || | |||
4817 | MI.getOpcode() == X86::PUSH64r)) | |||
4818 | return nullptr; | |||
4819 | ||||
4820 | // Avoid partial and undef register update stalls unless optimizing for size. | |||
4821 | if (!MF.getFunction().optForSize() && | |||
4822 | (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || | |||
4823 | shouldPreventUndefRegUpdateMemFold(MF, MI))) | |||
4824 | return nullptr; | |||
4825 | ||||
4826 | unsigned NumOps = MI.getDesc().getNumOperands(); | |||
4827 | bool isTwoAddr = | |||
4828 | NumOps > 1 && MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1; | |||
4829 | ||||
4830 | // FIXME: AsmPrinter doesn't know how to handle | |||
4831 | // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding. | |||
4832 | if (MI.getOpcode() == X86::ADD32ri && | |||
4833 | MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS) | |||
4834 | return nullptr; | |||
4835 | ||||
4836 | // GOTTPOFF relocation loads can only be folded into add instructions. | |||
4837 | // FIXME: Need to exclude other relocations that only support specific | |||
4838 | // instructions. | |||
4839 | if (MOs.size() == X86::AddrNumOperands && | |||
4840 | MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF && | |||
4841 | MI.getOpcode() != X86::ADD64rr) | |||
4842 | return nullptr; | |||
4843 | ||||
4844 | MachineInstr *NewMI = nullptr; | |||
4845 | ||||
4846 | // Attempt to fold any custom cases we have. | |||
4847 | if (MachineInstr *CustomMI = | |||
4848 | foldMemoryOperandCustom(MF, MI, OpNum, MOs, InsertPt, Size, Align)) | |||
4849 | return CustomMI; | |||
4850 | ||||
4851 | const X86MemoryFoldTableEntry *I = nullptr; | |||
4852 | ||||
4853 | // Folding a memory location into the two-address part of a two-address | |||
4854 | // instruction is different than folding it other places. It requires | |||
4855 | // replacing the *two* registers with the memory location. | |||
4856 | if (isTwoAddr && NumOps >= 2 && OpNum < 2 && MI.getOperand(0).isReg() && | |||
4857 | MI.getOperand(1).isReg() && | |||
4858 | MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) { | |||
4859 | I = lookupTwoAddrFoldTable(MI.getOpcode()); | |||
4860 | isTwoAddrFold = true; | |||
4861 | } else { | |||
4862 | if (OpNum == 0) { | |||
4863 | if (MI.getOpcode() == X86::MOV32r0) { | |||
4864 | NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, InsertPt, MI); | |||
4865 | if (NewMI) | |||
4866 | return NewMI; | |||
4867 | } | |||
4868 | } | |||
4869 | ||||
4870 | I = lookupFoldTable(MI.getOpcode(), OpNum); | |||
4871 | } | |||
4872 | ||||
4873 | if (I != nullptr) { | |||
4874 | unsigned Opcode = I->DstOp; | |||
4875 | unsigned MinAlign = (I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT; | |||
4876 | if (Align < MinAlign) | |||
4877 | return nullptr; | |||
4878 | bool NarrowToMOV32rm = false; | |||
4879 | if (Size) { | |||
4880 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
4881 | const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, | |||
4882 | &RI, MF); | |||
4883 | unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; | |||
4884 | if (Size < RCSize) { | |||
4885 | // Check if it's safe to fold the load. If the size of the object is | |||
4886 | // narrower than the load width, then it's not. | |||
4887 | if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4) | |||
4888 | return nullptr; | |||
4889 | // If this is a 64-bit load, but the spill slot is 32, then we can do | |||
4890 | // a 32-bit load which is implicitly zero-extended. This likely is | |||
4891 | // due to live interval analysis remat'ing a load from stack slot. | |||
4892 | if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) | |||
4893 | return nullptr; | |||
4894 | Opcode = X86::MOV32rm; | |||
4895 | NarrowToMOV32rm = true; | |||
4896 | } | |||
4897 | } | |||
4898 | ||||
4899 | if (isTwoAddrFold) | |||
4900 | NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this); | |||
4901 | else | |||
4902 | NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this); | |||
4903 | ||||
4904 | if (NarrowToMOV32rm) { | |||
4905 | // If this is the special case where we use a MOV32rm to load a 32-bit | |||
4906 | // value and zero-extend the top bits. Change the destination register | |||
4907 | // to a 32-bit one. | |||
4908 | unsigned DstReg = NewMI->getOperand(0).getReg(); | |||
4909 | if (TargetRegisterInfo::isPhysicalRegister(DstReg)) | |||
4910 | NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit)); | |||
4911 | else | |||
4912 | NewMI->getOperand(0).setSubReg(X86::sub_32bit); | |||
4913 | } | |||
4914 | return NewMI; | |||
4915 | } | |||
4916 | ||||
4917 | // If the instruction and target operand are commutable, commute the | |||
4918 | // instruction and try again. | |||
4919 | if (AllowCommute) { | |||
4920 | unsigned CommuteOpIdx1 = OpNum, CommuteOpIdx2 = CommuteAnyOperandIndex; | |||
4921 | if (findCommutedOpIndices(MI, CommuteOpIdx1, CommuteOpIdx2)) { | |||
4922 | bool HasDef = MI.getDesc().getNumDefs(); | |||
4923 | unsigned Reg0 = HasDef ? MI.getOperand(0).getReg() : 0; | |||
4924 | unsigned Reg1 = MI.getOperand(CommuteOpIdx1).getReg(); | |||
4925 | unsigned Reg2 = MI.getOperand(CommuteOpIdx2).getReg(); | |||
4926 | bool Tied1 = | |||
4927 | 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx1, MCOI::TIED_TO); | |||
4928 | bool Tied2 = | |||
4929 | 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx2, MCOI::TIED_TO); | |||
4930 | ||||
4931 | // If either of the commutable operands are tied to the destination | |||
4932 | // then we can not commute + fold. | |||
4933 | if ((HasDef && Reg0 == Reg1 && Tied1) || | |||
4934 | (HasDef && Reg0 == Reg2 && Tied2)) | |||
4935 | return nullptr; | |||
4936 | ||||
4937 | MachineInstr *CommutedMI = | |||
4938 | commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); | |||
4939 | if (!CommutedMI) { | |||
4940 | // Unable to commute. | |||
4941 | return nullptr; | |||
4942 | } | |||
4943 | if (CommutedMI != &MI) { | |||
4944 | // New instruction. We can't fold from this. | |||
4945 | CommutedMI->eraseFromParent(); | |||
4946 | return nullptr; | |||
4947 | } | |||
4948 | ||||
4949 | // Attempt to fold with the commuted version of the instruction. | |||
4950 | NewMI = foldMemoryOperandImpl(MF, MI, CommuteOpIdx2, MOs, InsertPt, | |||
4951 | Size, Align, /*AllowCommute=*/false); | |||
4952 | if (NewMI) | |||
4953 | return NewMI; | |||
4954 | ||||
4955 | // Folding failed again - undo the commute before returning. | |||
4956 | MachineInstr *UncommutedMI = | |||
4957 | commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); | |||
4958 | if (!UncommutedMI) { | |||
4959 | // Unable to commute. | |||
4960 | return nullptr; | |||
4961 | } | |||
4962 | if (UncommutedMI != &MI) { | |||
4963 | // New instruction. It doesn't need to be kept. | |||
4964 | UncommutedMI->eraseFromParent(); | |||
4965 | return nullptr; | |||
4966 | } | |||
4967 | ||||
4968 | // Return here to prevent duplicate fuse failure report. | |||
4969 | return nullptr; | |||
4970 | } | |||
4971 | } | |||
4972 | ||||
4973 | // No fusion | |||
4974 | if (PrintFailedFusing && !MI.isCopy()) | |||
4975 | dbgs() << "We failed to fuse operand " << OpNum << " in " << MI; | |||
4976 | return nullptr; | |||
4977 | } | |||
4978 | ||||
4979 | MachineInstr * | |||
4980 | X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, | |||
4981 | ArrayRef<unsigned> Ops, | |||
4982 | MachineBasicBlock::iterator InsertPt, | |||
4983 | int FrameIndex, LiveIntervals *LIS) const { | |||
4984 | // Check switch flag | |||
4985 | if (NoFusing) | |||
4986 | return nullptr; | |||
4987 | ||||
4988 | // Avoid partial and undef register update stalls unless optimizing for size. | |||
4989 | if (!MF.getFunction().optForSize() && | |||
4990 | (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || | |||
4991 | shouldPreventUndefRegUpdateMemFold(MF, MI))) | |||
4992 | return nullptr; | |||
4993 | ||||
4994 | // Don't fold subreg spills, or reloads that use a high subreg. | |||
4995 | for (auto Op : Ops) { | |||
4996 | MachineOperand &MO = MI.getOperand(Op); | |||
4997 | auto SubReg = MO.getSubReg(); | |||
4998 | if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi)) | |||
4999 | return nullptr; | |||
5000 | } | |||
5001 | ||||
5002 | const MachineFrameInfo &MFI = MF.getFrameInfo(); | |||
5003 | unsigned Size = MFI.getObjectSize(FrameIndex); | |||
5004 | unsigned Alignment = MFI.getObjectAlignment(FrameIndex); | |||
5005 | // If the function stack isn't realigned we don't want to fold instructions | |||
5006 | // that need increased alignment. | |||
5007 | if (!RI.needsStackRealignment(MF)) | |||
5008 | Alignment = | |||
5009 | std::min(Alignment, Subtarget.getFrameLowering()->getStackAlignment()); | |||
5010 | if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { | |||
5011 | unsigned NewOpc = 0; | |||
5012 | unsigned RCSize = 0; | |||
5013 | switch (MI.getOpcode()) { | |||
5014 | default: return nullptr; | |||
5015 | case X86::TEST8rr: NewOpc = X86::CMP8ri; RCSize = 1; break; | |||
5016 | case X86::TEST16rr: NewOpc = X86::CMP16ri8; RCSize = 2; break; | |||
5017 | case X86::TEST32rr: NewOpc = X86::CMP32ri8; RCSize = 4; break; | |||
5018 | case X86::TEST64rr: NewOpc = X86::CMP64ri8; RCSize = 8; break; | |||
5019 | } | |||
5020 | // Check if it's safe to fold the load. If the size of the object is | |||
5021 | // narrower than the load width, then it's not. | |||
5022 | if (Size < RCSize) | |||
5023 | return nullptr; | |||
5024 | // Change to CMPXXri r, 0 first. | |||
5025 | MI.setDesc(get(NewOpc)); | |||
5026 | MI.getOperand(1).ChangeToImmediate(0); | |||
5027 | } else if (Ops.size() != 1) | |||
5028 | return nullptr; | |||
5029 | ||||
5030 | return foldMemoryOperandImpl(MF, MI, Ops[0], | |||
5031 | MachineOperand::CreateFI(FrameIndex), InsertPt, | |||
5032 | Size, Alignment, /*AllowCommute=*/true); | |||
5033 | } | |||
5034 | ||||
5035 | /// Check if \p LoadMI is a partial register load that we can't fold into \p MI | |||
5036 | /// because the latter uses contents that wouldn't be defined in the folded | |||
5037 | /// version. For instance, this transformation isn't legal: | |||
5038 | /// movss (%rdi), %xmm0 | |||
5039 | /// addps %xmm0, %xmm0 | |||
5040 | /// -> | |||
5041 | /// addps (%rdi), %xmm0 | |||
5042 | /// | |||
5043 | /// But this one is: | |||
5044 | /// movss (%rdi), %xmm0 | |||
5045 | /// addss %xmm0, %xmm0 | |||
5046 | /// -> | |||
5047 | /// addss (%rdi), %xmm0 | |||
5048 | /// | |||
5049 | static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, | |||
5050 | const MachineInstr &UserMI, | |||
5051 | const MachineFunction &MF) { | |||
5052 | unsigned Opc = LoadMI.getOpcode(); | |||
5053 | unsigned UserOpc = UserMI.getOpcode(); | |||
5054 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
5055 | const TargetRegisterClass *RC = | |||
5056 | MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg()); | |||
5057 | unsigned RegSize = TRI.getRegSizeInBits(*RC); | |||
5058 | ||||
5059 | if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm) && | |||
5060 | RegSize > 32) { | |||
5061 | // These instructions only load 32 bits, we can't fold them if the | |||
5062 | // destination register is wider than 32 bits (4 bytes), and its user | |||
5063 | // instruction isn't scalar (SS). | |||
5064 | switch (UserOpc) { | |||
5065 | case X86::ADDSSrr_Int: case X86::VADDSSrr_Int: case X86::VADDSSZrr_Int: | |||
5066 | case X86::CMPSSrr_Int: case X86::VCMPSSrr_Int: case X86::VCMPSSZrr_Int: | |||
5067 | case X86::DIVSSrr_Int: case X86::VDIVSSrr_Int: case X86::VDIVSSZrr_Int: | |||
5068 | case X86::MAXSSrr_Int: case X86::VMAXSSrr_Int: case X86::VMAXSSZrr_Int: | |||
5069 | case X86::MINSSrr_Int: case X86::VMINSSrr_Int: case X86::VMINSSZrr_Int: | |||
5070 | case X86::MULSSrr_Int: case X86::VMULSSrr_Int: case X86::VMULSSZrr_Int: | |||
5071 | case X86::SUBSSrr_Int: case X86::VSUBSSrr_Int: case X86::VSUBSSZrr_Int: | |||
5072 | case X86::VADDSSZrr_Intk: case X86::VADDSSZrr_Intkz: | |||
5073 | case X86::VDIVSSZrr_Intk: case X86::VDIVSSZrr_Intkz: | |||
5074 | case X86::VMAXSSZrr_Intk: case X86::VMAXSSZrr_Intkz: | |||
5075 | case X86::VMINSSZrr_Intk: case X86::VMINSSZrr_Intkz: | |||
5076 | case X86::VMULSSZrr_Intk: case X86::VMULSSZrr_Intkz: | |||
5077 | case X86::VSUBSSZrr_Intk: case X86::VSUBSSZrr_Intkz: | |||
5078 | case X86::VFMADDSS4rr_Int: case X86::VFNMADDSS4rr_Int: | |||
5079 | case X86::VFMSUBSS4rr_Int: case X86::VFNMSUBSS4rr_Int: | |||
5080 | case X86::VFMADD132SSr_Int: case X86::VFNMADD132SSr_Int: | |||
5081 | case X86::VFMADD213SSr_Int: case X86::VFNMADD213SSr_Int: | |||
5082 | case X86::VFMADD231SSr_Int: case X86::VFNMADD231SSr_Int: | |||
5083 | case X86::VFMSUB132SSr_Int: case X86::VFNMSUB132SSr_Int: | |||
5084 | case X86::VFMSUB213SSr_Int: case X86::VFNMSUB213SSr_Int: | |||
5085 | case X86::VFMSUB231SSr_Int: case X86::VFNMSUB231SSr_Int: | |||
5086 | case X86::VFMADD132SSZr_Int: case X86::VFNMADD132SSZr_Int: | |||
5087 | case X86::VFMADD213SSZr_Int: case X86::VFNMADD213SSZr_Int: | |||
5088 | case X86::VFMADD231SSZr_Int: case X86::VFNMADD231SSZr_Int: | |||
5089 | case X86::VFMSUB132SSZr_Int: case X86::VFNMSUB132SSZr_Int: | |||
5090 | case X86::VFMSUB213SSZr_Int: case X86::VFNMSUB213SSZr_Int: | |||
5091 | case X86::VFMSUB231SSZr_Int: case X86::VFNMSUB231SSZr_Int: | |||
5092 | case X86::VFMADD132SSZr_Intk: case X86::VFNMADD132SSZr_Intk: | |||
5093 | case X86::VFMADD213SSZr_Intk: case X86::VFNMADD213SSZr_Intk: | |||
5094 | case X86::VFMADD231SSZr_Intk: case X86::VFNMADD231SSZr_Intk: | |||
5095 | case X86::VFMSUB132SSZr_Intk: case X86::VFNMSUB132SSZr_Intk: | |||
5096 | case X86::VFMSUB213SSZr_Intk: case X86::VFNMSUB213SSZr_Intk: | |||
5097 | case X86::VFMSUB231SSZr_Intk: case X86::VFNMSUB231SSZr_Intk: | |||
5098 | case X86::VFMADD132SSZr_Intkz: case X86::VFNMADD132SSZr_Intkz: | |||
5099 | case X86::VFMADD213SSZr_Intkz: case X86::VFNMADD213SSZr_Intkz: | |||
5100 | case X86::VFMADD231SSZr_Intkz: case X86::VFNMADD231SSZr_Intkz: | |||
5101 | case X86::VFMSUB132SSZr_Intkz: case X86::VFNMSUB132SSZr_Intkz: | |||
5102 | case X86::VFMSUB213SSZr_Intkz: case X86::VFNMSUB213SSZr_Intkz: | |||
5103 | case X86::VFMSUB231SSZr_Intkz: case X86::VFNMSUB231SSZr_Intkz: | |||
5104 | return false; | |||
5105 | default: | |||
5106 | return true; | |||
5107 | } | |||
5108 | } | |||
5109 | ||||
5110 | if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm) && | |||
5111 | RegSize > 64) { | |||
5112 | // These instructions only load 64 bits, we can't fold them if the | |||
5113 | // destination register is wider than 64 bits (8 bytes), and its user | |||
5114 | // instruction isn't scalar (SD). | |||
5115 | switch (UserOpc) { | |||
5116 | case X86::ADDSDrr_Int: case X86::VADDSDrr_Int: case X86::VADDSDZrr_Int: | |||
5117 | case X86::CMPSDrr_Int: case X86::VCMPSDrr_Int: case X86::VCMPSDZrr_Int: | |||
5118 | case X86::DIVSDrr_Int: case X86::VDIVSDrr_Int: case X86::VDIVSDZrr_Int: | |||
5119 | case X86::MAXSDrr_Int: case X86::VMAXSDrr_Int: case X86::VMAXSDZrr_Int: | |||
5120 | case X86::MINSDrr_Int: case X86::VMINSDrr_Int: case X86::VMINSDZrr_Int: | |||
5121 | case X86::MULSDrr_Int: case X86::VMULSDrr_Int: case X86::VMULSDZrr_Int: | |||
5122 | case X86::SUBSDrr_Int: case X86::VSUBSDrr_Int: case X86::VSUBSDZrr_Int: | |||
5123 | case X86::VADDSDZrr_Intk: case X86::VADDSDZrr_Intkz: | |||
5124 | case X86::VDIVSDZrr_Intk: case X86::VDIVSDZrr_Intkz: | |||
5125 | case X86::VMAXSDZrr_Intk: case X86::VMAXSDZrr_Intkz: | |||
5126 | case X86::VMINSDZrr_Intk: case X86::VMINSDZrr_Intkz: | |||
5127 | case X86::VMULSDZrr_Intk: case X86::VMULSDZrr_Intkz: | |||
5128 | case X86::VSUBSDZrr_Intk: case X86::VSUBSDZrr_Intkz: | |||
5129 | case X86::VFMADDSD4rr_Int: case X86::VFNMADDSD4rr_Int: | |||
5130 | case X86::VFMSUBSD4rr_Int: case X86::VFNMSUBSD4rr_Int: | |||
5131 | case X86::VFMADD132SDr_Int: case X86::VFNMADD132SDr_Int: | |||
5132 | case X86::VFMADD213SDr_Int: case X86::VFNMADD213SDr_Int: | |||
5133 | case X86::VFMADD231SDr_Int: case X86::VFNMADD231SDr_Int: | |||
5134 | case X86::VFMSUB132SDr_Int: case X86::VFNMSUB132SDr_Int: | |||
5135 | case X86::VFMSUB213SDr_Int: case X86::VFNMSUB213SDr_Int: | |||
5136 | case X86::VFMSUB231SDr_Int: case X86::VFNMSUB231SDr_Int: | |||
5137 | case X86::VFMADD132SDZr_Int: case X86::VFNMADD132SDZr_Int: | |||
5138 | case X86::VFMADD213SDZr_Int: case X86::VFNMADD213SDZr_Int: | |||
5139 | case X86::VFMADD231SDZr_Int: case X86::VFNMADD231SDZr_Int: | |||
5140 | case X86::VFMSUB132SDZr_Int: case X86::VFNMSUB132SDZr_Int: | |||
5141 | case X86::VFMSUB213SDZr_Int: case X86::VFNMSUB213SDZr_Int: | |||
5142 | case X86::VFMSUB231SDZr_Int: case X86::VFNMSUB231SDZr_Int: | |||
5143 | case X86::VFMADD132SDZr_Intk: case X86::VFNMADD132SDZr_Intk: | |||
5144 | case X86::VFMADD213SDZr_Intk: case X86::VFNMADD213SDZr_Intk: | |||
5145 | case X86::VFMADD231SDZr_Intk: case X86::VFNMADD231SDZr_Intk: | |||
5146 | case X86::VFMSUB132SDZr_Intk: case X86::VFNMSUB132SDZr_Intk: | |||
5147 | case X86::VFMSUB213SDZr_Intk: case X86::VFNMSUB213SDZr_Intk: | |||
5148 | case X86::VFMSUB231SDZr_Intk: case X86::VFNMSUB231SDZr_Intk: | |||
5149 | case X86::VFMADD132SDZr_Intkz: case X86::VFNMADD132SDZr_Intkz: | |||
5150 | case X86::VFMADD213SDZr_Intkz: case X86::VFNMADD213SDZr_Intkz: | |||
5151 | case X86::VFMADD231SDZr_Intkz: case X86::VFNMADD231SDZr_Intkz: | |||
5152 | case X86::VFMSUB132SDZr_Intkz: case X86::VFNMSUB132SDZr_Intkz: | |||
5153 | case X86::VFMSUB213SDZr_Intkz: case X86::VFNMSUB213SDZr_Intkz: | |||
5154 | case X86::VFMSUB231SDZr_Intkz: case X86::VFNMSUB231SDZr_Intkz: | |||
5155 | return false; | |||
5156 | default: | |||
5157 | return true; | |||
5158 | } | |||
5159 | } | |||
5160 | ||||
5161 | return false; | |||
5162 | } | |||
5163 | ||||
5164 | MachineInstr *X86InstrInfo::foldMemoryOperandImpl( | |||
5165 | MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, | |||
5166 | MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, | |||
5167 | LiveIntervals *LIS) const { | |||
5168 | ||||
5169 | // TODO: Support the case where LoadMI loads a wide register, but MI | |||
5170 | // only uses a subreg. | |||
5171 | for (auto Op : Ops) { | |||
5172 | if (MI.getOperand(Op).getSubReg()) | |||
5173 | return nullptr; | |||
5174 | } | |||
5175 | ||||
5176 | // If loading from a FrameIndex, fold directly from the FrameIndex. | |||
5177 | unsigned NumOps = LoadMI.getDesc().getNumOperands(); | |||
5178 | int FrameIndex; | |||
5179 | if (isLoadFromStackSlot(LoadMI, FrameIndex)) { | |||
5180 | if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) | |||
5181 | return nullptr; | |||
5182 | return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, LIS); | |||
5183 | } | |||
5184 | ||||
5185 | // Check switch flag | |||
5186 | if (NoFusing) return nullptr; | |||
5187 | ||||
5188 | // Avoid partial and undef register update stalls unless optimizing for size. | |||
5189 | if (!MF.getFunction().optForSize() && | |||
5190 | (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || | |||
5191 | shouldPreventUndefRegUpdateMemFold(MF, MI))) | |||
5192 | return nullptr; | |||
5193 | ||||
5194 | // Determine the alignment of the load. | |||
5195 | unsigned Alignment = 0; | |||
5196 | if (LoadMI.hasOneMemOperand()) | |||
5197 | Alignment = (*LoadMI.memoperands_begin())->getAlignment(); | |||
5198 | else | |||
5199 | switch (LoadMI.getOpcode()) { | |||
5200 | case X86::AVX512_512_SET0: | |||
5201 | case X86::AVX512_512_SETALLONES: | |||
5202 | Alignment = 64; | |||
5203 | break; | |||
5204 | case X86::AVX2_SETALLONES: | |||
5205 | case X86::AVX1_SETALLONES: | |||
5206 | case X86::AVX_SET0: | |||
5207 | case X86::AVX512_256_SET0: | |||
5208 | Alignment = 32; | |||
5209 | break; | |||
5210 | case X86::V_SET0: | |||
5211 | case X86::V_SETALLONES: | |||
5212 | case X86::AVX512_128_SET0: | |||
5213 | Alignment = 16; | |||
5214 | break; | |||
5215 | case X86::MMX_SET0: | |||
5216 | case X86::FsFLD0SD: | |||
5217 | case X86::AVX512_FsFLD0SD: | |||
5218 | Alignment = 8; | |||
5219 | break; | |||
5220 | case X86::FsFLD0SS: | |||
5221 | case X86::AVX512_FsFLD0SS: | |||
5222 | Alignment = 4; | |||
5223 | break; | |||
5224 | default: | |||
5225 | return nullptr; | |||
5226 | } | |||
5227 | if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { | |||
5228 | unsigned NewOpc = 0; | |||
5229 | switch (MI.getOpcode()) { | |||
5230 | default: return nullptr; | |||
5231 | case X86::TEST8rr: NewOpc = X86::CMP8ri; break; | |||
5232 | case X86::TEST16rr: NewOpc = X86::CMP16ri8; break; | |||
5233 | case X86::TEST32rr: NewOpc = X86::CMP32ri8; break; | |||
5234 | case X86::TEST64rr: NewOpc = X86::CMP64ri8; break; | |||
5235 | } | |||
5236 | // Change to CMPXXri r, 0 first. | |||
5237 | MI.setDesc(get(NewOpc)); | |||
5238 | MI.getOperand(1).ChangeToImmediate(0); | |||
5239 | } else if (Ops.size() != 1) | |||
5240 | return nullptr; | |||
5241 | ||||
5242 | // Make sure the subregisters match. | |||
5243 | // Otherwise we risk changing the size of the load. | |||
5244 | if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg()) | |||
5245 | return nullptr; | |||
5246 | ||||
5247 | SmallVector<MachineOperand,X86::AddrNumOperands> MOs; | |||
5248 | switch (LoadMI.getOpcode()) { | |||
5249 | case X86::MMX_SET0: | |||
5250 | case X86::V_SET0: | |||
5251 | case X86::V_SETALLONES: | |||
5252 | case X86::AVX2_SETALLONES: | |||
5253 | case X86::AVX1_SETALLONES: | |||
5254 | case X86::AVX_SET0: | |||
5255 | case X86::AVX512_128_SET0: | |||
5256 | case X86::AVX512_256_SET0: | |||
5257 | case X86::AVX512_512_SET0: | |||
5258 | case X86::AVX512_512_SETALLONES: | |||
5259 | case X86::FsFLD0SD: | |||
5260 | case X86::AVX512_FsFLD0SD: | |||
5261 | case X86::FsFLD0SS: | |||
5262 | case X86::AVX512_FsFLD0SS: { | |||
5263 | // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure. | |||
5264 | // Create a constant-pool entry and operands to load from it. | |||
5265 | ||||
5266 | // Medium and large mode can't fold loads this way. | |||
5267 | if (MF.getTarget().getCodeModel() != CodeModel::Small && | |||
5268 | MF.getTarget().getCodeModel() != CodeModel::Kernel) | |||
5269 | return nullptr; | |||
5270 | ||||
5271 | // x86-32 PIC requires a PIC base register for constant pools. | |||
5272 | unsigned PICBase = 0; | |||
5273 | if (MF.getTarget().isPositionIndependent()) { | |||
5274 | if (Subtarget.is64Bit()) | |||
5275 | PICBase = X86::RIP; | |||
5276 | else | |||
5277 | // FIXME: PICBase = getGlobalBaseReg(&MF); | |||
5278 | // This doesn't work for several reasons. | |||
5279 | // 1. GlobalBaseReg may have been spilled. | |||
5280 | // 2. It may not be live at MI. | |||
5281 | return nullptr; | |||
5282 | } | |||
5283 | ||||
5284 | // Create a constant-pool entry. | |||
5285 | MachineConstantPool &MCP = *MF.getConstantPool(); | |||
5286 | Type *Ty; | |||
5287 | unsigned Opc = LoadMI.getOpcode(); | |||
5288 | if (Opc == X86::FsFLD0SS || Opc == X86::AVX512_FsFLD0SS) | |||
5289 | Ty = Type::getFloatTy(MF.getFunction().getContext()); | |||
5290 | else if (Opc == X86::FsFLD0SD || Opc == X86::AVX512_FsFLD0SD) | |||
5291 | Ty = Type::getDoubleTy(MF.getFunction().getContext()); | |||
5292 | else if (Opc == X86::AVX512_512_SET0 || Opc == X86::AVX512_512_SETALLONES) | |||
5293 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()),16); | |||
5294 | else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0 || | |||
5295 | Opc == X86::AVX512_256_SET0 || Opc == X86::AVX1_SETALLONES) | |||
5296 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 8); | |||
5297 | else if (Opc == X86::MMX_SET0) | |||
5298 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 2); | |||
5299 | else | |||
5300 | Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 4); | |||
5301 | ||||
5302 | bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES || | |||
5303 | Opc == X86::AVX512_512_SETALLONES || | |||
5304 | Opc == X86::AVX1_SETALLONES); | |||
5305 | const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) : | |||
5306 | Constant::getNullValue(Ty); | |||
5307 | unsigned CPI = MCP.getConstantPoolIndex(C, Alignment); | |||
5308 | ||||
5309 | // Create operands to load from the constant pool entry. | |||
5310 | MOs.push_back(MachineOperand::CreateReg(PICBase, false)); | |||
5311 | MOs.push_back(MachineOperand::CreateImm(1)); | |||
5312 | MOs.push_back(MachineOperand::CreateReg(0, false)); | |||
5313 | MOs.push_back(MachineOperand::CreateCPI(CPI, 0)); | |||
5314 | MOs.push_back(MachineOperand::CreateReg(0, false)); | |||
5315 | break; | |||
5316 | } | |||
5317 | default: { | |||
5318 | if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) | |||
5319 | return nullptr; | |||
5320 | ||||
5321 | // Folding a normal load. Just copy the load's address operands. | |||
5322 | MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, | |||
5323 | LoadMI.operands_begin() + NumOps); | |||
5324 | break; | |||
5325 | } | |||
5326 | } | |||
5327 | return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt, | |||
5328 | /*Size=*/0, Alignment, /*AllowCommute=*/true); | |||
5329 | } | |||
5330 | ||||
5331 | static SmallVector<MachineMemOperand *, 2> | |||
5332 | extractLoadMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { | |||
5333 | SmallVector<MachineMemOperand *, 2> LoadMMOs; | |||
5334 | ||||
5335 | for (MachineMemOperand *MMO : MMOs) { | |||
5336 | if (!MMO->isLoad()) | |||
5337 | continue; | |||
5338 | ||||
5339 | if (!MMO->isStore()) { | |||
5340 | // Reuse the MMO. | |||
5341 | LoadMMOs.push_back(MMO); | |||
5342 | } else { | |||
5343 | // Clone the MMO and unset the store flag. | |||
5344 | LoadMMOs.push_back(MF.getMachineMemOperand( | |||
5345 | MMO->getPointerInfo(), MMO->getFlags() & ~MachineMemOperand::MOStore, | |||
5346 | MMO->getSize(), MMO->getBaseAlignment(), MMO->getAAInfo(), nullptr, | |||
5347 | MMO->getSyncScopeID(), MMO->getOrdering(), | |||
5348 | MMO->getFailureOrdering())); | |||
5349 | } | |||
5350 | } | |||
5351 | ||||
5352 | return LoadMMOs; | |||
5353 | } | |||
5354 | ||||
5355 | static SmallVector<MachineMemOperand *, 2> | |||
5356 | extractStoreMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { | |||
5357 | SmallVector<MachineMemOperand *, 2> StoreMMOs; | |||
5358 | ||||
5359 | for (MachineMemOperand *MMO : MMOs) { | |||
5360 | if (!MMO->isStore()) | |||
5361 | continue; | |||
5362 | ||||
5363 | if (!MMO->isLoad()) { | |||
5364 | // Reuse the MMO. | |||
5365 | StoreMMOs.push_back(MMO); | |||
5366 | } else { | |||
5367 | // Clone the MMO and unset the load flag. | |||
5368 | StoreMMOs.push_back(MF.getMachineMemOperand( | |||
5369 | MMO->getPointerInfo(), MMO->getFlags() & ~MachineMemOperand::MOLoad, | |||
5370 | MMO->getSize(), MMO->getBaseAlignment(), MMO->getAAInfo(), nullptr, | |||
5371 | MMO->getSyncScopeID(), MMO->getOrdering(), | |||
5372 | MMO->getFailureOrdering())); | |||
5373 | } | |||
5374 | } | |||
5375 | ||||
5376 | return StoreMMOs; | |||
5377 | } | |||
5378 | ||||
5379 | bool X86InstrInfo::unfoldMemoryOperand( | |||
5380 | MachineFunction &MF, MachineInstr &MI, unsigned Reg, bool UnfoldLoad, | |||
5381 | bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const { | |||
5382 | const X86MemoryFoldTableEntry *I = lookupUnfoldTable(MI.getOpcode()); | |||
5383 | if (I == nullptr) | |||
5384 | return false; | |||
5385 | unsigned Opc = I->DstOp; | |||
5386 | unsigned Index = I->Flags & TB_INDEX_MASK; | |||
5387 | bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; | |||
5388 | bool FoldedStore = I->Flags & TB_FOLDED_STORE; | |||
5389 | if (UnfoldLoad && !FoldedLoad) | |||
5390 | return false; | |||
5391 | UnfoldLoad &= FoldedLoad; | |||
5392 | if (UnfoldStore && !FoldedStore) | |||
5393 | return false; | |||
5394 | UnfoldStore &= FoldedStore; | |||
5395 | ||||
5396 | const MCInstrDesc &MCID = get(Opc); | |||
5397 | const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); | |||
5398 | // TODO: Check if 32-byte or greater accesses are slow too? | |||
5399 | if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass && | |||
5400 | Subtarget.isUnalignedMem16Slow()) | |||
5401 | // Without memoperands, loadRegFromAddr and storeRegToStackSlot will | |||
5402 | // conservatively assume the address is unaligned. That's bad for | |||
5403 | // performance. | |||
5404 | return false; | |||
5405 | SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps; | |||
5406 | SmallVector<MachineOperand,2> BeforeOps; | |||
5407 | SmallVector<MachineOperand,2> AfterOps; | |||
5408 | SmallVector<MachineOperand,4> ImpOps; | |||
5409 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
5410 | MachineOperand &Op = MI.getOperand(i); | |||
5411 | if (i >= Index && i < Index + X86::AddrNumOperands) | |||
5412 | AddrOps.push_back(Op); | |||
5413 | else if (Op.isReg() && Op.isImplicit()) | |||
5414 | ImpOps.push_back(Op); | |||
5415 | else if (i < Index) | |||
5416 | BeforeOps.push_back(Op); | |||
5417 | else if (i > Index) | |||
5418 | AfterOps.push_back(Op); | |||
5419 | } | |||
5420 | ||||
5421 | // Emit the load instruction. | |||
5422 | if (UnfoldLoad) { | |||
5423 | auto MMOs = extractLoadMMOs(MI.memoperands(), MF); | |||
5424 | loadRegFromAddr(MF, Reg, AddrOps, RC, MMOs, NewMIs); | |||
5425 | if (UnfoldStore) { | |||
5426 | // Address operands cannot be marked isKill. | |||
5427 | for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) { | |||
5428 | MachineOperand &MO = NewMIs[0]->getOperand(i); | |||
5429 | if (MO.isReg()) | |||
5430 | MO.setIsKill(false); | |||
5431 | } | |||
5432 | } | |||
5433 | } | |||
5434 | ||||
5435 | // Emit the data processing instruction. | |||
5436 | MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI.getDebugLoc(), true); | |||
5437 | MachineInstrBuilder MIB(MF, DataMI); | |||
5438 | ||||
5439 | if (FoldedStore) | |||
5440 | MIB.addReg(Reg, RegState::Define); | |||
5441 | for (MachineOperand &BeforeOp : BeforeOps) | |||
5442 | MIB.add(BeforeOp); | |||
5443 | if (FoldedLoad) | |||
5444 | MIB.addReg(Reg); | |||
5445 | for (MachineOperand &AfterOp : AfterOps) | |||
5446 | MIB.add(AfterOp); | |||
5447 | for (MachineOperand &ImpOp : ImpOps) { | |||
5448 | MIB.addReg(ImpOp.getReg(), | |||
5449 | getDefRegState(ImpOp.isDef()) | | |||
5450 | RegState::Implicit | | |||
5451 | getKillRegState(ImpOp.isKill()) | | |||
5452 | getDeadRegState(ImpOp.isDead()) | | |||
5453 | getUndefRegState(ImpOp.isUndef())); | |||
5454 | } | |||
5455 | // Change CMP32ri r, 0 back to TEST32rr r, r, etc. | |||
5456 | switch (DataMI->getOpcode()) { | |||
5457 | default: break; | |||
5458 | case X86::CMP64ri32: | |||
5459 | case X86::CMP64ri8: | |||
5460 | case X86::CMP32ri: | |||
5461 | case X86::CMP32ri8: | |||
5462 | case X86::CMP16ri: | |||
5463 | case X86::CMP16ri8: | |||
5464 | case X86::CMP8ri: { | |||
5465 | MachineOperand &MO0 = DataMI->getOperand(0); | |||
5466 | MachineOperand &MO1 = DataMI->getOperand(1); | |||
5467 | if (MO1.getImm() == 0) { | |||
5468 | unsigned NewOpc; | |||
5469 | switch (DataMI->getOpcode()) { | |||
5470 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5470); | |||
5471 | case X86::CMP64ri8: | |||
5472 | case X86::CMP64ri32: NewOpc = X86::TEST64rr; break; | |||
5473 | case X86::CMP32ri8: | |||
5474 | case X86::CMP32ri: NewOpc = X86::TEST32rr; break; | |||
5475 | case X86::CMP16ri8: | |||
5476 | case X86::CMP16ri: NewOpc = X86::TEST16rr; break; | |||
5477 | case X86::CMP8ri: NewOpc = X86::TEST8rr; break; | |||
5478 | } | |||
5479 | DataMI->setDesc(get(NewOpc)); | |||
5480 | MO1.ChangeToRegister(MO0.getReg(), false); | |||
5481 | } | |||
5482 | } | |||
5483 | } | |||
5484 | NewMIs.push_back(DataMI); | |||
5485 | ||||
5486 | // Emit the store instruction. | |||
5487 | if (UnfoldStore) { | |||
5488 | const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF); | |||
5489 | auto MMOs = extractStoreMMOs(MI.memoperands(), MF); | |||
5490 | storeRegToAddr(MF, Reg, true, AddrOps, DstRC, MMOs, NewMIs); | |||
5491 | } | |||
5492 | ||||
5493 | return true; | |||
5494 | } | |||
5495 | ||||
5496 | bool | |||
5497 | X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, | |||
5498 | SmallVectorImpl<SDNode*> &NewNodes) const { | |||
5499 | if (!N->isMachineOpcode()) | |||
5500 | return false; | |||
5501 | ||||
5502 | const X86MemoryFoldTableEntry *I = lookupUnfoldTable(N->getMachineOpcode()); | |||
5503 | if (I == nullptr) | |||
5504 | return false; | |||
5505 | unsigned Opc = I->DstOp; | |||
5506 | unsigned Index = I->Flags & TB_INDEX_MASK; | |||
5507 | bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; | |||
5508 | bool FoldedStore = I->Flags & TB_FOLDED_STORE; | |||
5509 | const MCInstrDesc &MCID = get(Opc); | |||
5510 | MachineFunction &MF = DAG.getMachineFunction(); | |||
5511 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | |||
5512 | const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); | |||
5513 | unsigned NumDefs = MCID.NumDefs; | |||
5514 | std::vector<SDValue> AddrOps; | |||
5515 | std::vector<SDValue> BeforeOps; | |||
5516 | std::vector<SDValue> AfterOps; | |||
5517 | SDLoc dl(N); | |||
5518 | unsigned NumOps = N->getNumOperands(); | |||
5519 | for (unsigned i = 0; i != NumOps-1; ++i) { | |||
5520 | SDValue Op = N->getOperand(i); | |||
5521 | if (i >= Index-NumDefs && i < Index-NumDefs + X86::AddrNumOperands) | |||
5522 | AddrOps.push_back(Op); | |||
5523 | else if (i < Index-NumDefs) | |||
5524 | BeforeOps.push_back(Op); | |||
5525 | else if (i > Index-NumDefs) | |||
5526 | AfterOps.push_back(Op); | |||
5527 | } | |||
5528 | SDValue Chain = N->getOperand(NumOps-1); | |||
5529 | AddrOps.push_back(Chain); | |||
5530 | ||||
5531 | // Emit the load instruction. | |||
5532 | SDNode *Load = nullptr; | |||
5533 | if (FoldedLoad) { | |||
5534 | EVT VT = *TRI.legalclasstypes_begin(*RC); | |||
5535 | auto MMOs = extractLoadMMOs(cast<MachineSDNode>(N)->memoperands(), MF); | |||
5536 | if (MMOs.empty() && RC == &X86::VR128RegClass && | |||
5537 | Subtarget.isUnalignedMem16Slow()) | |||
5538 | // Do not introduce a slow unaligned load. | |||
5539 | return false; | |||
5540 | // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte | |||
5541 | // memory access is slow above. | |||
5542 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
5543 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
5544 | Load = DAG.getMachineNode(getLoadRegOpcode(0, RC, isAligned, Subtarget), dl, | |||
5545 | VT, MVT::Other, AddrOps); | |||
5546 | NewNodes.push_back(Load); | |||
5547 | ||||
5548 | // Preserve memory reference information. | |||
5549 | DAG.setNodeMemRefs(cast<MachineSDNode>(Load), MMOs); | |||
5550 | } | |||
5551 | ||||
5552 | // Emit the data processing instruction. | |||
5553 | std::vector<EVT> VTs; | |||
5554 | const TargetRegisterClass *DstRC = nullptr; | |||
5555 | if (MCID.getNumDefs() > 0) { | |||
5556 | DstRC = getRegClass(MCID, 0, &RI, MF); | |||
5557 | VTs.push_back(*TRI.legalclasstypes_begin(*DstRC)); | |||
5558 | } | |||
5559 | for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { | |||
5560 | EVT VT = N->getValueType(i); | |||
5561 | if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs()) | |||
5562 | VTs.push_back(VT); | |||
5563 | } | |||
5564 | if (Load) | |||
5565 | BeforeOps.push_back(SDValue(Load, 0)); | |||
5566 | BeforeOps.insert(BeforeOps.end(), AfterOps.begin(), AfterOps.end()); | |||
5567 | // Change CMP32ri r, 0 back to TEST32rr r, r, etc. | |||
5568 | switch (Opc) { | |||
5569 | default: break; | |||
5570 | case X86::CMP64ri32: | |||
5571 | case X86::CMP64ri8: | |||
5572 | case X86::CMP32ri: | |||
5573 | case X86::CMP32ri8: | |||
5574 | case X86::CMP16ri: | |||
5575 | case X86::CMP16ri8: | |||
5576 | case X86::CMP8ri: | |||
5577 | if (isNullConstant(BeforeOps[1])) { | |||
5578 | switch (Opc) { | |||
5579 | default: llvm_unreachable("Unreachable!")::llvm::llvm_unreachable_internal("Unreachable!", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5579); | |||
5580 | case X86::CMP64ri8: | |||
5581 | case X86::CMP64ri32: Opc = X86::TEST64rr; break; | |||
5582 | case X86::CMP32ri8: | |||
5583 | case X86::CMP32ri: Opc = X86::TEST32rr; break; | |||
5584 | case X86::CMP16ri8: | |||
5585 | case X86::CMP16ri: Opc = X86::TEST16rr; break; | |||
5586 | case X86::CMP8ri: Opc = X86::TEST8rr; break; | |||
5587 | } | |||
5588 | BeforeOps[1] = BeforeOps[0]; | |||
5589 | } | |||
5590 | } | |||
5591 | SDNode *NewNode= DAG.getMachineNode(Opc, dl, VTs, BeforeOps); | |||
5592 | NewNodes.push_back(NewNode); | |||
5593 | ||||
5594 | // Emit the store instruction. | |||
5595 | if (FoldedStore) { | |||
5596 | AddrOps.pop_back(); | |||
5597 | AddrOps.push_back(SDValue(NewNode, 0)); | |||
5598 | AddrOps.push_back(Chain); | |||
5599 | auto MMOs = extractStoreMMOs(cast<MachineSDNode>(N)->memoperands(), MF); | |||
5600 | if (MMOs.empty() && RC == &X86::VR128RegClass && | |||
5601 | Subtarget.isUnalignedMem16Slow()) | |||
5602 | // Do not introduce a slow unaligned store. | |||
5603 | return false; | |||
5604 | // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte | |||
5605 | // memory access is slow above. | |||
5606 | unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); | |||
5607 | bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; | |||
5608 | SDNode *Store = | |||
5609 | DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget), | |||
5610 | dl, MVT::Other, AddrOps); | |||
5611 | NewNodes.push_back(Store); | |||
5612 | ||||
5613 | // Preserve memory reference information. | |||
5614 | DAG.setNodeMemRefs(cast<MachineSDNode>(Store), MMOs); | |||
5615 | } | |||
5616 | ||||
5617 | return true; | |||
5618 | } | |||
5619 | ||||
5620 | unsigned X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc, | |||
5621 | bool UnfoldLoad, bool UnfoldStore, | |||
5622 | unsigned *LoadRegIndex) const { | |||
5623 | const X86MemoryFoldTableEntry *I = lookupUnfoldTable(Opc); | |||
5624 | if (I == nullptr) | |||
5625 | return 0; | |||
5626 | bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; | |||
5627 | bool FoldedStore = I->Flags & TB_FOLDED_STORE; | |||
5628 | if (UnfoldLoad && !FoldedLoad) | |||
5629 | return 0; | |||
5630 | if (UnfoldStore && !FoldedStore) | |||
5631 | return 0; | |||
5632 | if (LoadRegIndex) | |||
5633 | *LoadRegIndex = I->Flags & TB_INDEX_MASK; | |||
5634 | return I->DstOp; | |||
5635 | } | |||
5636 | ||||
5637 | bool | |||
5638 | X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, | |||
5639 | int64_t &Offset1, int64_t &Offset2) const { | |||
5640 | if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) | |||
5641 | return false; | |||
5642 | unsigned Opc1 = Load1->getMachineOpcode(); | |||
5643 | unsigned Opc2 = Load2->getMachineOpcode(); | |||
5644 | switch (Opc1) { | |||
5645 | default: return false; | |||
5646 | case X86::MOV8rm: | |||
5647 | case X86::MOV16rm: | |||
5648 | case X86::MOV32rm: | |||
5649 | case X86::MOV64rm: | |||
5650 | case X86::LD_Fp32m: | |||
5651 | case X86::LD_Fp64m: | |||
5652 | case X86::LD_Fp80m: | |||
5653 | case X86::MOVSSrm: | |||
5654 | case X86::MOVSDrm: | |||
5655 | case X86::MMX_MOVD64rm: | |||
5656 | case X86::MMX_MOVQ64rm: | |||
5657 | case X86::MOVAPSrm: | |||
5658 | case X86::MOVUPSrm: | |||
5659 | case X86::MOVAPDrm: | |||
5660 | case X86::MOVUPDrm: | |||
5661 | case X86::MOVDQArm: | |||
5662 | case X86::MOVDQUrm: | |||
5663 | // AVX load instructions | |||
5664 | case X86::VMOVSSrm: | |||
5665 | case X86::VMOVSDrm: | |||
5666 | case X86::VMOVAPSrm: | |||
5667 | case X86::VMOVUPSrm: | |||
5668 | case X86::VMOVAPDrm: | |||
5669 | case X86::VMOVUPDrm: | |||
5670 | case X86::VMOVDQArm: | |||
5671 | case X86::VMOVDQUrm: | |||
5672 | case X86::VMOVAPSYrm: | |||
5673 | case X86::VMOVUPSYrm: | |||
5674 | case X86::VMOVAPDYrm: | |||
5675 | case X86::VMOVUPDYrm: | |||
5676 | case X86::VMOVDQAYrm: | |||
5677 | case X86::VMOVDQUYrm: | |||
5678 | // AVX512 load instructions | |||
5679 | case X86::VMOVSSZrm: | |||
5680 | case X86::VMOVSDZrm: | |||
5681 | case X86::VMOVAPSZ128rm: | |||
5682 | case X86::VMOVUPSZ128rm: | |||
5683 | case X86::VMOVAPSZ128rm_NOVLX: | |||
5684 | case X86::VMOVUPSZ128rm_NOVLX: | |||
5685 | case X86::VMOVAPDZ128rm: | |||
5686 | case X86::VMOVUPDZ128rm: | |||
5687 | case X86::VMOVDQU8Z128rm: | |||
5688 | case X86::VMOVDQU16Z128rm: | |||
5689 | case X86::VMOVDQA32Z128rm: | |||
5690 | case X86::VMOVDQU32Z128rm: | |||
5691 | case X86::VMOVDQA64Z128rm: | |||
5692 | case X86::VMOVDQU64Z128rm: | |||
5693 | case X86::VMOVAPSZ256rm: | |||
5694 | case X86::VMOVUPSZ256rm: | |||
5695 | case X86::VMOVAPSZ256rm_NOVLX: | |||
5696 | case X86::VMOVUPSZ256rm_NOVLX: | |||
5697 | case X86::VMOVAPDZ256rm: | |||
5698 | case X86::VMOVUPDZ256rm: | |||
5699 | case X86::VMOVDQU8Z256rm: | |||
5700 | case X86::VMOVDQU16Z256rm: | |||
5701 | case X86::VMOVDQA32Z256rm: | |||
5702 | case X86::VMOVDQU32Z256rm: | |||
5703 | case X86::VMOVDQA64Z256rm: | |||
5704 | case X86::VMOVDQU64Z256rm: | |||
5705 | case X86::VMOVAPSZrm: | |||
5706 | case X86::VMOVUPSZrm: | |||
5707 | case X86::VMOVAPDZrm: | |||
5708 | case X86::VMOVUPDZrm: | |||
5709 | case X86::VMOVDQU8Zrm: | |||
5710 | case X86::VMOVDQU16Zrm: | |||
5711 | case X86::VMOVDQA32Zrm: | |||
5712 | case X86::VMOVDQU32Zrm: | |||
5713 | case X86::VMOVDQA64Zrm: | |||
5714 | case X86::VMOVDQU64Zrm: | |||
5715 | case X86::KMOVBkm: | |||
5716 | case X86::KMOVWkm: | |||
5717 | case X86::KMOVDkm: | |||
5718 | case X86::KMOVQkm: | |||
5719 | break; | |||
5720 | } | |||
5721 | switch (Opc2) { | |||
5722 | default: return false; | |||
5723 | case X86::MOV8rm: | |||
5724 | case X86::MOV16rm: | |||
5725 | case X86::MOV32rm: | |||
5726 | case X86::MOV64rm: | |||
5727 | case X86::LD_Fp32m: | |||
5728 | case X86::LD_Fp64m: | |||
5729 | case X86::LD_Fp80m: | |||
5730 | case X86::MOVSSrm: | |||
5731 | case X86::MOVSDrm: | |||
5732 | case X86::MMX_MOVD64rm: | |||
5733 | case X86::MMX_MOVQ64rm: | |||
5734 | case X86::MOVAPSrm: | |||
5735 | case X86::MOVUPSrm: | |||
5736 | case X86::MOVAPDrm: | |||
5737 | case X86::MOVUPDrm: | |||
5738 | case X86::MOVDQArm: | |||
5739 | case X86::MOVDQUrm: | |||
5740 | // AVX load instructions | |||
5741 | case X86::VMOVSSrm: | |||
5742 | case X86::VMOVSDrm: | |||
5743 | case X86::VMOVAPSrm: | |||
5744 | case X86::VMOVUPSrm: | |||
5745 | case X86::VMOVAPDrm: | |||
5746 | case X86::VMOVUPDrm: | |||
5747 | case X86::VMOVDQArm: | |||
5748 | case X86::VMOVDQUrm: | |||
5749 | case X86::VMOVAPSYrm: | |||
5750 | case X86::VMOVUPSYrm: | |||
5751 | case X86::VMOVAPDYrm: | |||
5752 | case X86::VMOVUPDYrm: | |||
5753 | case X86::VMOVDQAYrm: | |||
5754 | case X86::VMOVDQUYrm: | |||
5755 | // AVX512 load instructions | |||
5756 | case X86::VMOVSSZrm: | |||
5757 | case X86::VMOVSDZrm: | |||
5758 | case X86::VMOVAPSZ128rm: | |||
5759 | case X86::VMOVUPSZ128rm: | |||
5760 | case X86::VMOVAPSZ128rm_NOVLX: | |||
5761 | case X86::VMOVUPSZ128rm_NOVLX: | |||
5762 | case X86::VMOVAPDZ128rm: | |||
5763 | case X86::VMOVUPDZ128rm: | |||
5764 | case X86::VMOVDQU8Z128rm: | |||
5765 | case X86::VMOVDQU16Z128rm: | |||
5766 | case X86::VMOVDQA32Z128rm: | |||
5767 | case X86::VMOVDQU32Z128rm: | |||
5768 | case X86::VMOVDQA64Z128rm: | |||
5769 | case X86::VMOVDQU64Z128rm: | |||
5770 | case X86::VMOVAPSZ256rm: | |||
5771 | case X86::VMOVUPSZ256rm: | |||
5772 | case X86::VMOVAPSZ256rm_NOVLX: | |||
5773 | case X86::VMOVUPSZ256rm_NOVLX: | |||
5774 | case X86::VMOVAPDZ256rm: | |||
5775 | case X86::VMOVUPDZ256rm: | |||
5776 | case X86::VMOVDQU8Z256rm: | |||
5777 | case X86::VMOVDQU16Z256rm: | |||
5778 | case X86::VMOVDQA32Z256rm: | |||
5779 | case X86::VMOVDQU32Z256rm: | |||
5780 | case X86::VMOVDQA64Z256rm: | |||
5781 | case X86::VMOVDQU64Z256rm: | |||
5782 | case X86::VMOVAPSZrm: | |||
5783 | case X86::VMOVUPSZrm: | |||
5784 | case X86::VMOVAPDZrm: | |||
5785 | case X86::VMOVUPDZrm: | |||
5786 | case X86::VMOVDQU8Zrm: | |||
5787 | case X86::VMOVDQU16Zrm: | |||
5788 | case X86::VMOVDQA32Zrm: | |||
5789 | case X86::VMOVDQU32Zrm: | |||
5790 | case X86::VMOVDQA64Zrm: | |||
5791 | case X86::VMOVDQU64Zrm: | |||
5792 | case X86::KMOVBkm: | |||
5793 | case X86::KMOVWkm: | |||
5794 | case X86::KMOVDkm: | |||
5795 | case X86::KMOVQkm: | |||
5796 | break; | |||
5797 | } | |||
5798 | ||||
5799 | // Lambda to check if both the loads have the same value for an operand index. | |||
5800 | auto HasSameOp = [&](int I) { | |||
5801 | return Load1->getOperand(I) == Load2->getOperand(I); | |||
5802 | }; | |||
5803 | ||||
5804 | // All operands except the displacement should match. | |||
5805 | if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) || | |||
5806 | !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg)) | |||
5807 | return false; | |||
5808 | ||||
5809 | // Chain Operand must be the same. | |||
5810 | if (!HasSameOp(5)) | |||
5811 | return false; | |||
5812 | ||||
5813 | // Now let's examine if the displacements are constants. | |||
5814 | auto Disp1 = dyn_cast<ConstantSDNode>(Load1->getOperand(X86::AddrDisp)); | |||
5815 | auto Disp2 = dyn_cast<ConstantSDNode>(Load2->getOperand(X86::AddrDisp)); | |||
5816 | if (!Disp1 || !Disp2) | |||
5817 | return false; | |||
5818 | ||||
5819 | Offset1 = Disp1->getSExtValue(); | |||
5820 | Offset2 = Disp2->getSExtValue(); | |||
5821 | return true; | |||
5822 | } | |||
5823 | ||||
5824 | bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, | |||
5825 | int64_t Offset1, int64_t Offset2, | |||
5826 | unsigned NumLoads) const { | |||
5827 | assert(Offset2 > Offset1)((Offset2 > Offset1) ? static_cast<void> (0) : __assert_fail ("Offset2 > Offset1", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5827, __PRETTY_FUNCTION__)); | |||
5828 | if ((Offset2 - Offset1) / 8 > 64) | |||
5829 | return false; | |||
5830 | ||||
5831 | unsigned Opc1 = Load1->getMachineOpcode(); | |||
5832 | unsigned Opc2 = Load2->getMachineOpcode(); | |||
5833 | if (Opc1 != Opc2) | |||
5834 | return false; // FIXME: overly conservative? | |||
5835 | ||||
5836 | switch (Opc1) { | |||
5837 | default: break; | |||
5838 | case X86::LD_Fp32m: | |||
5839 | case X86::LD_Fp64m: | |||
5840 | case X86::LD_Fp80m: | |||
5841 | case X86::MMX_MOVD64rm: | |||
5842 | case X86::MMX_MOVQ64rm: | |||
5843 | return false; | |||
5844 | } | |||
5845 | ||||
5846 | EVT VT = Load1->getValueType(0); | |||
5847 | switch (VT.getSimpleVT().SimpleTy) { | |||
5848 | default: | |||
5849 | // XMM registers. In 64-bit mode we can be a bit more aggressive since we | |||
5850 | // have 16 of them to play with. | |||
5851 | if (Subtarget.is64Bit()) { | |||
5852 | if (NumLoads >= 3) | |||
5853 | return false; | |||
5854 | } else if (NumLoads) { | |||
5855 | return false; | |||
5856 | } | |||
5857 | break; | |||
5858 | case MVT::i8: | |||
5859 | case MVT::i16: | |||
5860 | case MVT::i32: | |||
5861 | case MVT::i64: | |||
5862 | case MVT::f32: | |||
5863 | case MVT::f64: | |||
5864 | if (NumLoads) | |||
5865 | return false; | |||
5866 | break; | |||
5867 | } | |||
5868 | ||||
5869 | return true; | |||
5870 | } | |||
5871 | ||||
5872 | bool X86InstrInfo:: | |||
5873 | reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { | |||
5874 | assert(Cond.size() == 1 && "Invalid X86 branch condition!")((Cond.size() == 1 && "Invalid X86 branch condition!" ) ? static_cast<void> (0) : __assert_fail ("Cond.size() == 1 && \"Invalid X86 branch condition!\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5874, __PRETTY_FUNCTION__)); | |||
5875 | X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm()); | |||
5876 | Cond[0].setImm(GetOppositeBranchCondition(CC)); | |||
5877 | return false; | |||
5878 | } | |||
5879 | ||||
5880 | bool X86InstrInfo:: | |||
5881 | isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const { | |||
5882 | // FIXME: Return false for x87 stack register classes for now. We can't | |||
5883 | // allow any loads of these registers before FpGet_ST0_80. | |||
5884 | return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass || | |||
5885 | RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass || | |||
5886 | RC == &X86::RFP80RegClass); | |||
5887 | } | |||
5888 | ||||
5889 | /// Return a virtual register initialized with the | |||
5890 | /// the global base register value. Output instructions required to | |||
5891 | /// initialize the register in the function entry block, if necessary. | |||
5892 | /// | |||
5893 | /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. | |||
5894 | /// | |||
5895 | unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { | |||
5896 | assert((!Subtarget.is64Bit() ||(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5899, __PRETTY_FUNCTION__)) | |||
5897 | MF->getTarget().getCodeModel() == CodeModel::Medium ||(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5899, __PRETTY_FUNCTION__)) | |||
5898 | MF->getTarget().getCodeModel() == CodeModel::Large) &&(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5899, __PRETTY_FUNCTION__)) | |||
5899 | "X86-64 PIC uses RIP relative addressing")(((!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel ::Large) && "X86-64 PIC uses RIP relative addressing" ) ? static_cast<void> (0) : __assert_fail ("(!Subtarget.is64Bit() || MF->getTarget().getCodeModel() == CodeModel::Medium || MF->getTarget().getCodeModel() == CodeModel::Large) && \"X86-64 PIC uses RIP relative addressing\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 5899, __PRETTY_FUNCTION__)); | |||
5900 | ||||
5901 | X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); | |||
5902 | unsigned GlobalBaseReg = X86FI->getGlobalBaseReg(); | |||
5903 | if (GlobalBaseReg != 0) | |||
5904 | return GlobalBaseReg; | |||
5905 | ||||
5906 | // Create the register. The code to initialize it is inserted | |||
5907 | // later, by the CGBR pass (below). | |||
5908 | MachineRegisterInfo &RegInfo = MF->getRegInfo(); | |||
5909 | GlobalBaseReg = RegInfo.createVirtualRegister( | |||
5910 | Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass); | |||
5911 | X86FI->setGlobalBaseReg(GlobalBaseReg); | |||
5912 | return GlobalBaseReg; | |||
5913 | } | |||
5914 | ||||
5915 | // These are the replaceable SSE instructions. Some of these have Int variants | |||
5916 | // that we don't include here. We don't want to replace instructions selected | |||
5917 | // by intrinsics. | |||
5918 | static const uint16_t ReplaceableInstrs[][3] = { | |||
5919 | //PackedSingle PackedDouble PackedInt | |||
5920 | { X86::MOVAPSmr, X86::MOVAPDmr, X86::MOVDQAmr }, | |||
5921 | { X86::MOVAPSrm, X86::MOVAPDrm, X86::MOVDQArm }, | |||
5922 | { X86::MOVAPSrr, X86::MOVAPDrr, X86::MOVDQArr }, | |||
5923 | { X86::MOVUPSmr, X86::MOVUPDmr, X86::MOVDQUmr }, | |||
5924 | { X86::MOVUPSrm, X86::MOVUPDrm, X86::MOVDQUrm }, | |||
5925 | { X86::MOVLPSmr, X86::MOVLPDmr, X86::MOVPQI2QImr }, | |||
5926 | { X86::MOVSDmr, X86::MOVSDmr, X86::MOVPQI2QImr }, | |||
5927 | { X86::MOVSSmr, X86::MOVSSmr, X86::MOVPDI2DImr }, | |||
5928 | { X86::MOVSDrm, X86::MOVSDrm, X86::MOVQI2PQIrm }, | |||
5929 | { X86::MOVSSrm, X86::MOVSSrm, X86::MOVDI2PDIrm }, | |||
5930 | { X86::MOVNTPSmr, X86::MOVNTPDmr, X86::MOVNTDQmr }, | |||
5931 | { X86::ANDNPSrm, X86::ANDNPDrm, X86::PANDNrm }, | |||
5932 | { X86::ANDNPSrr, X86::ANDNPDrr, X86::PANDNrr }, | |||
5933 | { X86::ANDPSrm, X86::ANDPDrm, X86::PANDrm }, | |||
5934 | { X86::ANDPSrr, X86::ANDPDrr, X86::PANDrr }, | |||
5935 | { X86::ORPSrm, X86::ORPDrm, X86::PORrm }, | |||
5936 | { X86::ORPSrr, X86::ORPDrr, X86::PORrr }, | |||
5937 | { X86::XORPSrm, X86::XORPDrm, X86::PXORrm }, | |||
5938 | { X86::XORPSrr, X86::XORPDrr, X86::PXORrr }, | |||
5939 | { X86::UNPCKLPDrm, X86::UNPCKLPDrm, X86::PUNPCKLQDQrm }, | |||
5940 | { X86::MOVLHPSrr, X86::UNPCKLPDrr, X86::PUNPCKLQDQrr }, | |||
5941 | { X86::UNPCKHPDrm, X86::UNPCKHPDrm, X86::PUNPCKHQDQrm }, | |||
5942 | { X86::UNPCKHPDrr, X86::UNPCKHPDrr, X86::PUNPCKHQDQrr }, | |||
5943 | { X86::UNPCKLPSrm, X86::UNPCKLPSrm, X86::PUNPCKLDQrm }, | |||
5944 | { X86::UNPCKLPSrr, X86::UNPCKLPSrr, X86::PUNPCKLDQrr }, | |||
5945 | { X86::UNPCKHPSrm, X86::UNPCKHPSrm, X86::PUNPCKHDQrm }, | |||
5946 | { X86::UNPCKHPSrr, X86::UNPCKHPSrr, X86::PUNPCKHDQrr }, | |||
5947 | { X86::EXTRACTPSmr, X86::EXTRACTPSmr, X86::PEXTRDmr }, | |||
5948 | { X86::EXTRACTPSrr, X86::EXTRACTPSrr, X86::PEXTRDrr }, | |||
5949 | // AVX 128-bit support | |||
5950 | { X86::VMOVAPSmr, X86::VMOVAPDmr, X86::VMOVDQAmr }, | |||
5951 | { X86::VMOVAPSrm, X86::VMOVAPDrm, X86::VMOVDQArm }, | |||
5952 | { X86::VMOVAPSrr, X86::VMOVAPDrr, X86::VMOVDQArr }, | |||
5953 | { X86::VMOVUPSmr, X86::VMOVUPDmr, X86::VMOVDQUmr }, | |||
5954 | { X86::VMOVUPSrm, X86::VMOVUPDrm, X86::VMOVDQUrm }, | |||
5955 | { X86::VMOVLPSmr, X86::VMOVLPDmr, X86::VMOVPQI2QImr }, | |||
5956 | { X86::VMOVSDmr, X86::VMOVSDmr, X86::VMOVPQI2QImr }, | |||
5957 | { X86::VMOVSSmr, X86::VMOVSSmr, X86::VMOVPDI2DImr }, | |||
5958 | { X86::VMOVSDrm, X86::VMOVSDrm, X86::VMOVQI2PQIrm }, | |||
5959 | { X86::VMOVSSrm, X86::VMOVSSrm, X86::VMOVDI2PDIrm }, | |||
5960 | { X86::VMOVNTPSmr, X86::VMOVNTPDmr, X86::VMOVNTDQmr }, | |||
5961 | { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNrm }, | |||
5962 | { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNrr }, | |||
5963 | { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDrm }, | |||
5964 | { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDrr }, | |||
5965 | { X86::VORPSrm, X86::VORPDrm, X86::VPORrm }, | |||
5966 | { X86::VORPSrr, X86::VORPDrr, X86::VPORrr }, | |||
5967 | { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORrm }, | |||
5968 | { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORrr }, | |||
5969 | { X86::VUNPCKLPDrm, X86::VUNPCKLPDrm, X86::VPUNPCKLQDQrm }, | |||
5970 | { X86::VMOVLHPSrr, X86::VUNPCKLPDrr, X86::VPUNPCKLQDQrr }, | |||
5971 | { X86::VUNPCKHPDrm, X86::VUNPCKHPDrm, X86::VPUNPCKHQDQrm }, | |||
5972 | { X86::VUNPCKHPDrr, X86::VUNPCKHPDrr, X86::VPUNPCKHQDQrr }, | |||
5973 | { X86::VUNPCKLPSrm, X86::VUNPCKLPSrm, X86::VPUNPCKLDQrm }, | |||
5974 | { X86::VUNPCKLPSrr, X86::VUNPCKLPSrr, X86::VPUNPCKLDQrr }, | |||
5975 | { X86::VUNPCKHPSrm, X86::VUNPCKHPSrm, X86::VPUNPCKHDQrm }, | |||
5976 | { X86::VUNPCKHPSrr, X86::VUNPCKHPSrr, X86::VPUNPCKHDQrr }, | |||
5977 | { X86::VEXTRACTPSmr, X86::VEXTRACTPSmr, X86::VPEXTRDmr }, | |||
5978 | { X86::VEXTRACTPSrr, X86::VEXTRACTPSrr, X86::VPEXTRDrr }, | |||
5979 | // AVX 256-bit support | |||
5980 | { X86::VMOVAPSYmr, X86::VMOVAPDYmr, X86::VMOVDQAYmr }, | |||
5981 | { X86::VMOVAPSYrm, X86::VMOVAPDYrm, X86::VMOVDQAYrm }, | |||
5982 | { X86::VMOVAPSYrr, X86::VMOVAPDYrr, X86::VMOVDQAYrr }, | |||
5983 | { X86::VMOVUPSYmr, X86::VMOVUPDYmr, X86::VMOVDQUYmr }, | |||
5984 | { X86::VMOVUPSYrm, X86::VMOVUPDYrm, X86::VMOVDQUYrm }, | |||
5985 | { X86::VMOVNTPSYmr, X86::VMOVNTPDYmr, X86::VMOVNTDQYmr }, | |||
5986 | { X86::VPERMPSYrm, X86::VPERMPSYrm, X86::VPERMDYrm }, | |||
5987 | { X86::VPERMPSYrr, X86::VPERMPSYrr, X86::VPERMDYrr }, | |||
5988 | { X86::VPERMPDYmi, X86::VPERMPDYmi, X86::VPERMQYmi }, | |||
5989 | { X86::VPERMPDYri, X86::VPERMPDYri, X86::VPERMQYri }, | |||
5990 | // AVX512 support | |||
5991 | { X86::VMOVLPSZ128mr, X86::VMOVLPDZ128mr, X86::VMOVPQI2QIZmr }, | |||
5992 | { X86::VMOVNTPSZ128mr, X86::VMOVNTPDZ128mr, X86::VMOVNTDQZ128mr }, | |||
5993 | { X86::VMOVNTPSZ256mr, X86::VMOVNTPDZ256mr, X86::VMOVNTDQZ256mr }, | |||
5994 | { X86::VMOVNTPSZmr, X86::VMOVNTPDZmr, X86::VMOVNTDQZmr }, | |||
5995 | { X86::VMOVSDZmr, X86::VMOVSDZmr, X86::VMOVPQI2QIZmr }, | |||
5996 | { X86::VMOVSSZmr, X86::VMOVSSZmr, X86::VMOVPDI2DIZmr }, | |||
5997 | { X86::VMOVSDZrm, X86::VMOVSDZrm, X86::VMOVQI2PQIZrm }, | |||
5998 | { X86::VMOVSSZrm, X86::VMOVSSZrm, X86::VMOVDI2PDIZrm }, | |||
5999 | { X86::VBROADCASTSSZ128r, X86::VBROADCASTSSZ128r, X86::VPBROADCASTDZ128r }, | |||
6000 | { X86::VBROADCASTSSZ128m, X86::VBROADCASTSSZ128m, X86::VPBROADCASTDZ128m }, | |||
6001 | { X86::VBROADCASTSSZ256r, X86::VBROADCASTSSZ256r, X86::VPBROADCASTDZ256r }, | |||
6002 | { X86::VBROADCASTSSZ256m, X86::VBROADCASTSSZ256m, X86::VPBROADCASTDZ256m }, | |||
6003 | { X86::VBROADCASTSSZr, X86::VBROADCASTSSZr, X86::VPBROADCASTDZr }, | |||
6004 | { X86::VBROADCASTSSZm, X86::VBROADCASTSSZm, X86::VPBROADCASTDZm }, | |||
6005 | { X86::VBROADCASTSDZ256r, X86::VBROADCASTSDZ256r, X86::VPBROADCASTQZ256r }, | |||
6006 | { X86::VBROADCASTSDZ256m, X86::VBROADCASTSDZ256m, X86::VPBROADCASTQZ256m }, | |||
6007 | { X86::VBROADCASTSDZr, X86::VBROADCASTSDZr, X86::VPBROADCASTQZr }, | |||
6008 | { X86::VBROADCASTSDZm, X86::VBROADCASTSDZm, X86::VPBROADCASTQZm }, | |||
6009 | { X86::VINSERTF32x4Zrr, X86::VINSERTF32x4Zrr, X86::VINSERTI32x4Zrr }, | |||
6010 | { X86::VINSERTF32x4Zrm, X86::VINSERTF32x4Zrm, X86::VINSERTI32x4Zrm }, | |||
6011 | { X86::VINSERTF32x8Zrr, X86::VINSERTF32x8Zrr, X86::VINSERTI32x8Zrr }, | |||
6012 | { X86::VINSERTF32x8Zrm, X86::VINSERTF32x8Zrm, X86::VINSERTI32x8Zrm }, | |||
6013 | { X86::VINSERTF64x2Zrr, X86::VINSERTF64x2Zrr, X86::VINSERTI64x2Zrr }, | |||
6014 | { X86::VINSERTF64x2Zrm, X86::VINSERTF64x2Zrm, X86::VINSERTI64x2Zrm }, | |||
6015 | { X86::VINSERTF64x4Zrr, X86::VINSERTF64x4Zrr, X86::VINSERTI64x4Zrr }, | |||
6016 | { X86::VINSERTF64x4Zrm, X86::VINSERTF64x4Zrm, X86::VINSERTI64x4Zrm }, | |||
6017 | { X86::VINSERTF32x4Z256rr,X86::VINSERTF32x4Z256rr,X86::VINSERTI32x4Z256rr }, | |||
6018 | { X86::VINSERTF32x4Z256rm,X86::VINSERTF32x4Z256rm,X86::VINSERTI32x4Z256rm }, | |||
6019 | { X86::VINSERTF64x2Z256rr,X86::VINSERTF64x2Z256rr,X86::VINSERTI64x2Z256rr }, | |||
6020 | { X86::VINSERTF64x2Z256rm,X86::VINSERTF64x2Z256rm,X86::VINSERTI64x2Z256rm }, | |||
6021 | { X86::VEXTRACTF32x4Zrr, X86::VEXTRACTF32x4Zrr, X86::VEXTRACTI32x4Zrr }, | |||
6022 | { X86::VEXTRACTF32x4Zmr, X86::VEXTRACTF32x4Zmr, X86::VEXTRACTI32x4Zmr }, | |||
6023 | { X86::VEXTRACTF32x8Zrr, X86::VEXTRACTF32x8Zrr, X86::VEXTRACTI32x8Zrr }, | |||
6024 | { X86::VEXTRACTF32x8Zmr, X86::VEXTRACTF32x8Zmr, X86::VEXTRACTI32x8Zmr }, | |||
6025 | { X86::VEXTRACTF64x2Zrr, X86::VEXTRACTF64x2Zrr, X86::VEXTRACTI64x2Zrr }, | |||
6026 | { X86::VEXTRACTF64x2Zmr, X86::VEXTRACTF64x2Zmr, X86::VEXTRACTI64x2Zmr }, | |||
6027 | { X86::VEXTRACTF64x4Zrr, X86::VEXTRACTF64x4Zrr, X86::VEXTRACTI64x4Zrr }, | |||
6028 | { X86::VEXTRACTF64x4Zmr, X86::VEXTRACTF64x4Zmr, X86::VEXTRACTI64x4Zmr }, | |||
6029 | { X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTI32x4Z256rr }, | |||
6030 | { X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTI32x4Z256mr }, | |||
6031 | { X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTI64x2Z256rr }, | |||
6032 | { X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTI64x2Z256mr }, | |||
6033 | { X86::VPERMILPSmi, X86::VPERMILPSmi, X86::VPSHUFDmi }, | |||
6034 | { X86::VPERMILPSri, X86::VPERMILPSri, X86::VPSHUFDri }, | |||
6035 | { X86::VPERMILPSZ128mi, X86::VPERMILPSZ128mi, X86::VPSHUFDZ128mi }, | |||
6036 | { X86::VPERMILPSZ128ri, X86::VPERMILPSZ128ri, X86::VPSHUFDZ128ri }, | |||
6037 | { X86::VPERMILPSZ256mi, X86::VPERMILPSZ256mi, X86::VPSHUFDZ256mi }, | |||
6038 | { X86::VPERMILPSZ256ri, X86::VPERMILPSZ256ri, X86::VPSHUFDZ256ri }, | |||
6039 | { X86::VPERMILPSZmi, X86::VPERMILPSZmi, X86::VPSHUFDZmi }, | |||
6040 | { X86::VPERMILPSZri, X86::VPERMILPSZri, X86::VPSHUFDZri }, | |||
6041 | { X86::VPERMPSZ256rm, X86::VPERMPSZ256rm, X86::VPERMDZ256rm }, | |||
6042 | { X86::VPERMPSZ256rr, X86::VPERMPSZ256rr, X86::VPERMDZ256rr }, | |||
6043 | { X86::VPERMPDZ256mi, X86::VPERMPDZ256mi, X86::VPERMQZ256mi }, | |||
6044 | { X86::VPERMPDZ256ri, X86::VPERMPDZ256ri, X86::VPERMQZ256ri }, | |||
6045 | { X86::VPERMPDZ256rm, X86::VPERMPDZ256rm, X86::VPERMQZ256rm }, | |||
6046 | { X86::VPERMPDZ256rr, X86::VPERMPDZ256rr, X86::VPERMQZ256rr }, | |||
6047 | { X86::VPERMPSZrm, X86::VPERMPSZrm, X86::VPERMDZrm }, | |||
6048 | { X86::VPERMPSZrr, X86::VPERMPSZrr, X86::VPERMDZrr }, | |||
6049 | { X86::VPERMPDZmi, X86::VPERMPDZmi, X86::VPERMQZmi }, | |||
6050 | { X86::VPERMPDZri, X86::VPERMPDZri, X86::VPERMQZri }, | |||
6051 | { X86::VPERMPDZrm, X86::VPERMPDZrm, X86::VPERMQZrm }, | |||
6052 | { X86::VPERMPDZrr, X86::VPERMPDZrr, X86::VPERMQZrr }, | |||
6053 | { X86::VUNPCKLPDZ256rm, X86::VUNPCKLPDZ256rm, X86::VPUNPCKLQDQZ256rm }, | |||
6054 | { X86::VUNPCKLPDZ256rr, X86::VUNPCKLPDZ256rr, X86::VPUNPCKLQDQZ256rr }, | |||
6055 | { X86::VUNPCKHPDZ256rm, X86::VUNPCKHPDZ256rm, X86::VPUNPCKHQDQZ256rm }, | |||
6056 | { X86::VUNPCKHPDZ256rr, X86::VUNPCKHPDZ256rr, X86::VPUNPCKHQDQZ256rr }, | |||
6057 | { X86::VUNPCKLPSZ256rm, X86::VUNPCKLPSZ256rm, X86::VPUNPCKLDQZ256rm }, | |||
6058 | { X86::VUNPCKLPSZ256rr, X86::VUNPCKLPSZ256rr, X86::VPUNPCKLDQZ256rr }, | |||
6059 | { X86::VUNPCKHPSZ256rm, X86::VUNPCKHPSZ256rm, X86::VPUNPCKHDQZ256rm }, | |||
6060 | { X86::VUNPCKHPSZ256rr, X86::VUNPCKHPSZ256rr, X86::VPUNPCKHDQZ256rr }, | |||
6061 | { X86::VUNPCKLPDZ128rm, X86::VUNPCKLPDZ128rm, X86::VPUNPCKLQDQZ128rm }, | |||
6062 | { X86::VMOVLHPSZrr, X86::VUNPCKLPDZ128rr, X86::VPUNPCKLQDQZ128rr }, | |||
6063 | { X86::VUNPCKHPDZ128rm, X86::VUNPCKHPDZ128rm, X86::VPUNPCKHQDQZ128rm }, | |||
6064 | { X86::VUNPCKHPDZ128rr, X86::VUNPCKHPDZ128rr, X86::VPUNPCKHQDQZ128rr }, | |||
6065 | { X86::VUNPCKLPSZ128rm, X86::VUNPCKLPSZ128rm, X86::VPUNPCKLDQZ128rm }, | |||
6066 | { X86::VUNPCKLPSZ128rr, X86::VUNPCKLPSZ128rr, X86::VPUNPCKLDQZ128rr }, | |||
6067 | { X86::VUNPCKHPSZ128rm, X86::VUNPCKHPSZ128rm, X86::VPUNPCKHDQZ128rm }, | |||
6068 | { X86::VUNPCKHPSZ128rr, X86::VUNPCKHPSZ128rr, X86::VPUNPCKHDQZ128rr }, | |||
6069 | { X86::VUNPCKLPDZrm, X86::VUNPCKLPDZrm, X86::VPUNPCKLQDQZrm }, | |||
6070 | { X86::VUNPCKLPDZrr, X86::VUNPCKLPDZrr, X86::VPUNPCKLQDQZrr }, | |||
6071 | { X86::VUNPCKHPDZrm, X86::VUNPCKHPDZrm, X86::VPUNPCKHQDQZrm }, | |||
6072 | { X86::VUNPCKHPDZrr, X86::VUNPCKHPDZrr, X86::VPUNPCKHQDQZrr }, | |||
6073 | { X86::VUNPCKLPSZrm, X86::VUNPCKLPSZrm, X86::VPUNPCKLDQZrm }, | |||
6074 | { X86::VUNPCKLPSZrr, X86::VUNPCKLPSZrr, X86::VPUNPCKLDQZrr }, | |||
6075 | { X86::VUNPCKHPSZrm, X86::VUNPCKHPSZrm, X86::VPUNPCKHDQZrm }, | |||
6076 | { X86::VUNPCKHPSZrr, X86::VUNPCKHPSZrr, X86::VPUNPCKHDQZrr }, | |||
6077 | { X86::VEXTRACTPSZmr, X86::VEXTRACTPSZmr, X86::VPEXTRDZmr }, | |||
6078 | { X86::VEXTRACTPSZrr, X86::VEXTRACTPSZrr, X86::VPEXTRDZrr }, | |||
6079 | }; | |||
6080 | ||||
6081 | static const uint16_t ReplaceableInstrsAVX2[][3] = { | |||
6082 | //PackedSingle PackedDouble PackedInt | |||
6083 | { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNYrm }, | |||
6084 | { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNYrr }, | |||
6085 | { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDYrm }, | |||
6086 | { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDYrr }, | |||
6087 | { X86::VORPSYrm, X86::VORPDYrm, X86::VPORYrm }, | |||
6088 | { X86::VORPSYrr, X86::VORPDYrr, X86::VPORYrr }, | |||
6089 | { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORYrm }, | |||
6090 | { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORYrr }, | |||
6091 | { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, | |||
6092 | { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, | |||
6093 | { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, | |||
6094 | { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, | |||
6095 | { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, | |||
6096 | { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, | |||
6097 | { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, | |||
6098 | { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm}, | |||
6099 | { X86::VBROADCASTF128, X86::VBROADCASTF128, X86::VBROADCASTI128 }, | |||
6100 | { X86::VBLENDPSYrri, X86::VBLENDPSYrri, X86::VPBLENDDYrri }, | |||
6101 | { X86::VBLENDPSYrmi, X86::VBLENDPSYrmi, X86::VPBLENDDYrmi }, | |||
6102 | { X86::VPERMILPSYmi, X86::VPERMILPSYmi, X86::VPSHUFDYmi }, | |||
6103 | { X86::VPERMILPSYri, X86::VPERMILPSYri, X86::VPSHUFDYri }, | |||
6104 | { X86::VUNPCKLPDYrm, X86::VUNPCKLPDYrm, X86::VPUNPCKLQDQYrm }, | |||
6105 | { X86::VUNPCKLPDYrr, X86::VUNPCKLPDYrr, X86::VPUNPCKLQDQYrr }, | |||
6106 | { X86::VUNPCKHPDYrm, X86::VUNPCKHPDYrm, X86::VPUNPCKHQDQYrm }, | |||
6107 | { X86::VUNPCKHPDYrr, X86::VUNPCKHPDYrr, X86::VPUNPCKHQDQYrr }, | |||
6108 | { X86::VUNPCKLPSYrm, X86::VUNPCKLPSYrm, X86::VPUNPCKLDQYrm }, | |||
6109 | { X86::VUNPCKLPSYrr, X86::VUNPCKLPSYrr, X86::VPUNPCKLDQYrr }, | |||
6110 | { X86::VUNPCKHPSYrm, X86::VUNPCKHPSYrm, X86::VPUNPCKHDQYrm }, | |||
6111 | { X86::VUNPCKHPSYrr, X86::VUNPCKHPSYrr, X86::VPUNPCKHDQYrr }, | |||
6112 | }; | |||
6113 | ||||
6114 | static const uint16_t ReplaceableInstrsAVX2InsertExtract[][3] = { | |||
6115 | //PackedSingle PackedDouble PackedInt | |||
6116 | { X86::VEXTRACTF128mr, X86::VEXTRACTF128mr, X86::VEXTRACTI128mr }, | |||
6117 | { X86::VEXTRACTF128rr, X86::VEXTRACTF128rr, X86::VEXTRACTI128rr }, | |||
6118 | { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, | |||
6119 | { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, | |||
6120 | }; | |||
6121 | ||||
6122 | static const uint16_t ReplaceableInstrsAVX512[][4] = { | |||
6123 | // Two integer columns for 64-bit and 32-bit elements. | |||
6124 | //PackedSingle PackedDouble PackedInt PackedInt | |||
6125 | { X86::VMOVAPSZ128mr, X86::VMOVAPDZ128mr, X86::VMOVDQA64Z128mr, X86::VMOVDQA32Z128mr }, | |||
6126 | { X86::VMOVAPSZ128rm, X86::VMOVAPDZ128rm, X86::VMOVDQA64Z128rm, X86::VMOVDQA32Z128rm }, | |||
6127 | { X86::VMOVAPSZ128rr, X86::VMOVAPDZ128rr, X86::VMOVDQA64Z128rr, X86::VMOVDQA32Z128rr }, | |||
6128 | { X86::VMOVUPSZ128mr, X86::VMOVUPDZ128mr, X86::VMOVDQU64Z128mr, X86::VMOVDQU32Z128mr }, | |||
6129 | { X86::VMOVUPSZ128rm, X86::VMOVUPDZ128rm, X86::VMOVDQU64Z128rm, X86::VMOVDQU32Z128rm }, | |||
6130 | { X86::VMOVAPSZ256mr, X86::VMOVAPDZ256mr, X86::VMOVDQA64Z256mr, X86::VMOVDQA32Z256mr }, | |||
6131 | { X86::VMOVAPSZ256rm, X86::VMOVAPDZ256rm, X86::VMOVDQA64Z256rm, X86::VMOVDQA32Z256rm }, | |||
6132 | { X86::VMOVAPSZ256rr, X86::VMOVAPDZ256rr, X86::VMOVDQA64Z256rr, X86::VMOVDQA32Z256rr }, | |||
6133 | { X86::VMOVUPSZ256mr, X86::VMOVUPDZ256mr, X86::VMOVDQU64Z256mr, X86::VMOVDQU32Z256mr }, | |||
6134 | { X86::VMOVUPSZ256rm, X86::VMOVUPDZ256rm, X86::VMOVDQU64Z256rm, X86::VMOVDQU32Z256rm }, | |||
6135 | { X86::VMOVAPSZmr, X86::VMOVAPDZmr, X86::VMOVDQA64Zmr, X86::VMOVDQA32Zmr }, | |||
6136 | { X86::VMOVAPSZrm, X86::VMOVAPDZrm, X86::VMOVDQA64Zrm, X86::VMOVDQA32Zrm }, | |||
6137 | { X86::VMOVAPSZrr, X86::VMOVAPDZrr, X86::VMOVDQA64Zrr, X86::VMOVDQA32Zrr }, | |||
6138 | { X86::VMOVUPSZmr, X86::VMOVUPDZmr, X86::VMOVDQU64Zmr, X86::VMOVDQU32Zmr }, | |||
6139 | { X86::VMOVUPSZrm, X86::VMOVUPDZrm, X86::VMOVDQU64Zrm, X86::VMOVDQU32Zrm }, | |||
6140 | }; | |||
6141 | ||||
6142 | static const uint16_t ReplaceableInstrsAVX512DQ[][4] = { | |||
6143 | // Two integer columns for 64-bit and 32-bit elements. | |||
6144 | //PackedSingle PackedDouble PackedInt PackedInt | |||
6145 | { X86::VANDNPSZ128rm, X86::VANDNPDZ128rm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, | |||
6146 | { X86::VANDNPSZ128rr, X86::VANDNPDZ128rr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, | |||
6147 | { X86::VANDPSZ128rm, X86::VANDPDZ128rm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, | |||
6148 | { X86::VANDPSZ128rr, X86::VANDPDZ128rr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, | |||
6149 | { X86::VORPSZ128rm, X86::VORPDZ128rm, X86::VPORQZ128rm, X86::VPORDZ128rm }, | |||
6150 | { X86::VORPSZ128rr, X86::VORPDZ128rr, X86::VPORQZ128rr, X86::VPORDZ128rr }, | |||
6151 | { X86::VXORPSZ128rm, X86::VXORPDZ128rm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, | |||
6152 | { X86::VXORPSZ128rr, X86::VXORPDZ128rr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, | |||
6153 | { X86::VANDNPSZ256rm, X86::VANDNPDZ256rm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, | |||
6154 | { X86::VANDNPSZ256rr, X86::VANDNPDZ256rr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, | |||
6155 | { X86::VANDPSZ256rm, X86::VANDPDZ256rm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, | |||
6156 | { X86::VANDPSZ256rr, X86::VANDPDZ256rr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, | |||
6157 | { X86::VORPSZ256rm, X86::VORPDZ256rm, X86::VPORQZ256rm, X86::VPORDZ256rm }, | |||
6158 | { X86::VORPSZ256rr, X86::VORPDZ256rr, X86::VPORQZ256rr, X86::VPORDZ256rr }, | |||
6159 | { X86::VXORPSZ256rm, X86::VXORPDZ256rm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, | |||
6160 | { X86::VXORPSZ256rr, X86::VXORPDZ256rr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, | |||
6161 | { X86::VANDNPSZrm, X86::VANDNPDZrm, X86::VPANDNQZrm, X86::VPANDNDZrm }, | |||
6162 | { X86::VANDNPSZrr, X86::VANDNPDZrr, X86::VPANDNQZrr, X86::VPANDNDZrr }, | |||
6163 | { X86::VANDPSZrm, X86::VANDPDZrm, X86::VPANDQZrm, X86::VPANDDZrm }, | |||
6164 | { X86::VANDPSZrr, X86::VANDPDZrr, X86::VPANDQZrr, X86::VPANDDZrr }, | |||
6165 | { X86::VORPSZrm, X86::VORPDZrm, X86::VPORQZrm, X86::VPORDZrm }, | |||
6166 | { X86::VORPSZrr, X86::VORPDZrr, X86::VPORQZrr, X86::VPORDZrr }, | |||
6167 | { X86::VXORPSZrm, X86::VXORPDZrm, X86::VPXORQZrm, X86::VPXORDZrm }, | |||
6168 | { X86::VXORPSZrr, X86::VXORPDZrr, X86::VPXORQZrr, X86::VPXORDZrr }, | |||
6169 | }; | |||
6170 | ||||
6171 | static const uint16_t ReplaceableInstrsAVX512DQMasked[][4] = { | |||
6172 | // Two integer columns for 64-bit and 32-bit elements. | |||
6173 | //PackedSingle PackedDouble | |||
6174 | //PackedInt PackedInt | |||
6175 | { X86::VANDNPSZ128rmk, X86::VANDNPDZ128rmk, | |||
6176 | X86::VPANDNQZ128rmk, X86::VPANDNDZ128rmk }, | |||
6177 | { X86::VANDNPSZ128rmkz, X86::VANDNPDZ128rmkz, | |||
6178 | X86::VPANDNQZ128rmkz, X86::VPANDNDZ128rmkz }, | |||
6179 | { X86::VANDNPSZ128rrk, X86::VANDNPDZ128rrk, | |||
6180 | X86::VPANDNQZ128rrk, X86::VPANDNDZ128rrk }, | |||
6181 | { X86::VANDNPSZ128rrkz, X86::VANDNPDZ128rrkz, | |||
6182 | X86::VPANDNQZ128rrkz, X86::VPANDNDZ128rrkz }, | |||
6183 | { X86::VANDPSZ128rmk, X86::VANDPDZ128rmk, | |||
6184 | X86::VPANDQZ128rmk, X86::VPANDDZ128rmk }, | |||
6185 | { X86::VANDPSZ128rmkz, X86::VANDPDZ128rmkz, | |||
6186 | X86::VPANDQZ128rmkz, X86::VPANDDZ128rmkz }, | |||
6187 | { X86::VANDPSZ128rrk, X86::VANDPDZ128rrk, | |||
6188 | X86::VPANDQZ128rrk, X86::VPANDDZ128rrk }, | |||
6189 | { X86::VANDPSZ128rrkz, X86::VANDPDZ128rrkz, | |||
6190 | X86::VPANDQZ128rrkz, X86::VPANDDZ128rrkz }, | |||
6191 | { X86::VORPSZ128rmk, X86::VORPDZ128rmk, | |||
6192 | X86::VPORQZ128rmk, X86::VPORDZ128rmk }, | |||
6193 | { X86::VORPSZ128rmkz, X86::VORPDZ128rmkz, | |||
6194 | X86::VPORQZ128rmkz, X86::VPORDZ128rmkz }, | |||
6195 | { X86::VORPSZ128rrk, X86::VORPDZ128rrk, | |||
6196 | X86::VPORQZ128rrk, X86::VPORDZ128rrk }, | |||
6197 | { X86::VORPSZ128rrkz, X86::VORPDZ128rrkz, | |||
6198 | X86::VPORQZ128rrkz, X86::VPORDZ128rrkz }, | |||
6199 | { X86::VXORPSZ128rmk, X86::VXORPDZ128rmk, | |||
6200 | X86::VPXORQZ128rmk, X86::VPXORDZ128rmk }, | |||
6201 | { X86::VXORPSZ128rmkz, X86::VXORPDZ128rmkz, | |||
6202 | X86::VPXORQZ128rmkz, X86::VPXORDZ128rmkz }, | |||
6203 | { X86::VXORPSZ128rrk, X86::VXORPDZ128rrk, | |||
6204 | X86::VPXORQZ128rrk, X86::VPXORDZ128rrk }, | |||
6205 | { X86::VXORPSZ128rrkz, X86::VXORPDZ128rrkz, | |||
6206 | X86::VPXORQZ128rrkz, X86::VPXORDZ128rrkz }, | |||
6207 | { X86::VANDNPSZ256rmk, X86::VANDNPDZ256rmk, | |||
6208 | X86::VPANDNQZ256rmk, X86::VPANDNDZ256rmk }, | |||
6209 | { X86::VANDNPSZ256rmkz, X86::VANDNPDZ256rmkz, | |||
6210 | X86::VPANDNQZ256rmkz, X86::VPANDNDZ256rmkz }, | |||
6211 | { X86::VANDNPSZ256rrk, X86::VANDNPDZ256rrk, | |||
6212 | X86::VPANDNQZ256rrk, X86::VPANDNDZ256rrk }, | |||
6213 | { X86::VANDNPSZ256rrkz, X86::VANDNPDZ256rrkz, | |||
6214 | X86::VPANDNQZ256rrkz, X86::VPANDNDZ256rrkz }, | |||
6215 | { X86::VANDPSZ256rmk, X86::VANDPDZ256rmk, | |||
6216 | X86::VPANDQZ256rmk, X86::VPANDDZ256rmk }, | |||
6217 | { X86::VANDPSZ256rmkz, X86::VANDPDZ256rmkz, | |||
6218 | X86::VPANDQZ256rmkz, X86::VPANDDZ256rmkz }, | |||
6219 | { X86::VANDPSZ256rrk, X86::VANDPDZ256rrk, | |||
6220 | X86::VPANDQZ256rrk, X86::VPANDDZ256rrk }, | |||
6221 | { X86::VANDPSZ256rrkz, X86::VANDPDZ256rrkz, | |||
6222 | X86::VPANDQZ256rrkz, X86::VPANDDZ256rrkz }, | |||
6223 | { X86::VORPSZ256rmk, X86::VORPDZ256rmk, | |||
6224 | X86::VPORQZ256rmk, X86::VPORDZ256rmk }, | |||
6225 | { X86::VORPSZ256rmkz, X86::VORPDZ256rmkz, | |||
6226 | X86::VPORQZ256rmkz, X86::VPORDZ256rmkz }, | |||
6227 | { X86::VORPSZ256rrk, X86::VORPDZ256rrk, | |||
6228 | X86::VPORQZ256rrk, X86::VPORDZ256rrk }, | |||
6229 | { X86::VORPSZ256rrkz, X86::VORPDZ256rrkz, | |||
6230 | X86::VPORQZ256rrkz, X86::VPORDZ256rrkz }, | |||
6231 | { X86::VXORPSZ256rmk, X86::VXORPDZ256rmk, | |||
6232 | X86::VPXORQZ256rmk, X86::VPXORDZ256rmk }, | |||
6233 | { X86::VXORPSZ256rmkz, X86::VXORPDZ256rmkz, | |||
6234 | X86::VPXORQZ256rmkz, X86::VPXORDZ256rmkz }, | |||
6235 | { X86::VXORPSZ256rrk, X86::VXORPDZ256rrk, | |||
6236 | X86::VPXORQZ256rrk, X86::VPXORDZ256rrk }, | |||
6237 | { X86::VXORPSZ256rrkz, X86::VXORPDZ256rrkz, | |||
6238 | X86::VPXORQZ256rrkz, X86::VPXORDZ256rrkz }, | |||
6239 | { X86::VANDNPSZrmk, X86::VANDNPDZrmk, | |||
6240 | X86::VPANDNQZrmk, X86::VPANDNDZrmk }, | |||
6241 | { X86::VANDNPSZrmkz, X86::VANDNPDZrmkz, | |||
6242 | X86::VPANDNQZrmkz, X86::VPANDNDZrmkz }, | |||
6243 | { X86::VANDNPSZrrk, X86::VANDNPDZrrk, | |||
6244 | X86::VPANDNQZrrk, X86::VPANDNDZrrk }, | |||
6245 | { X86::VANDNPSZrrkz, X86::VANDNPDZrrkz, | |||
6246 | X86::VPANDNQZrrkz, X86::VPANDNDZrrkz }, | |||
6247 | { X86::VANDPSZrmk, X86::VANDPDZrmk, | |||
6248 | X86::VPANDQZrmk, X86::VPANDDZrmk }, | |||
6249 | { X86::VANDPSZrmkz, X86::VANDPDZrmkz, | |||
6250 | X86::VPANDQZrmkz, X86::VPANDDZrmkz }, | |||
6251 | { X86::VANDPSZrrk, X86::VANDPDZrrk, | |||
6252 | X86::VPANDQZrrk, X86::VPANDDZrrk }, | |||
6253 | { X86::VANDPSZrrkz, X86::VANDPDZrrkz, | |||
6254 | X86::VPANDQZrrkz, X86::VPANDDZrrkz }, | |||
6255 | { X86::VORPSZrmk, X86::VORPDZrmk, | |||
6256 | X86::VPORQZrmk, X86::VPORDZrmk }, | |||
6257 | { X86::VORPSZrmkz, X86::VORPDZrmkz, | |||
6258 | X86::VPORQZrmkz, X86::VPORDZrmkz }, | |||
6259 | { X86::VORPSZrrk, X86::VORPDZrrk, | |||
6260 | X86::VPORQZrrk, X86::VPORDZrrk }, | |||
6261 | { X86::VORPSZrrkz, X86::VORPDZrrkz, | |||
6262 | X86::VPORQZrrkz, X86::VPORDZrrkz }, | |||
6263 | { X86::VXORPSZrmk, X86::VXORPDZrmk, | |||
6264 | X86::VPXORQZrmk, X86::VPXORDZrmk }, | |||
6265 | { X86::VXORPSZrmkz, X86::VXORPDZrmkz, | |||
6266 | X86::VPXORQZrmkz, X86::VPXORDZrmkz }, | |||
6267 | { X86::VXORPSZrrk, X86::VXORPDZrrk, | |||
6268 | X86::VPXORQZrrk, X86::VPXORDZrrk }, | |||
6269 | { X86::VXORPSZrrkz, X86::VXORPDZrrkz, | |||
6270 | X86::VPXORQZrrkz, X86::VPXORDZrrkz }, | |||
6271 | // Broadcast loads can be handled the same as masked operations to avoid | |||
6272 | // changing element size. | |||
6273 | { X86::VANDNPSZ128rmb, X86::VANDNPDZ128rmb, | |||
6274 | X86::VPANDNQZ128rmb, X86::VPANDNDZ128rmb }, | |||
6275 | { X86::VANDPSZ128rmb, X86::VANDPDZ128rmb, | |||
6276 | X86::VPANDQZ128rmb, X86::VPANDDZ128rmb }, | |||
6277 | { X86::VORPSZ128rmb, X86::VORPDZ128rmb, | |||
6278 | X86::VPORQZ128rmb, X86::VPORDZ128rmb }, | |||
6279 | { X86::VXORPSZ128rmb, X86::VXORPDZ128rmb, | |||
6280 | X86::VPXORQZ128rmb, X86::VPXORDZ128rmb }, | |||
6281 | { X86::VANDNPSZ256rmb, X86::VANDNPDZ256rmb, | |||
6282 | X86::VPANDNQZ256rmb, X86::VPANDNDZ256rmb }, | |||
6283 | { X86::VANDPSZ256rmb, X86::VANDPDZ256rmb, | |||
6284 | X86::VPANDQZ256rmb, X86::VPANDDZ256rmb }, | |||
6285 | { X86::VORPSZ256rmb, X86::VORPDZ256rmb, | |||
6286 | X86::VPORQZ256rmb, X86::VPORDZ256rmb }, | |||
6287 | { X86::VXORPSZ256rmb, X86::VXORPDZ256rmb, | |||
6288 | X86::VPXORQZ256rmb, X86::VPXORDZ256rmb }, | |||
6289 | { X86::VANDNPSZrmb, X86::VANDNPDZrmb, | |||
6290 | X86::VPANDNQZrmb, X86::VPANDNDZrmb }, | |||
6291 | { X86::VANDPSZrmb, X86::VANDPDZrmb, | |||
6292 | X86::VPANDQZrmb, X86::VPANDDZrmb }, | |||
6293 | { X86::VANDPSZrmb, X86::VANDPDZrmb, | |||
6294 | X86::VPANDQZrmb, X86::VPANDDZrmb }, | |||
6295 | { X86::VORPSZrmb, X86::VORPDZrmb, | |||
6296 | X86::VPORQZrmb, X86::VPORDZrmb }, | |||
6297 | { X86::VXORPSZrmb, X86::VXORPDZrmb, | |||
6298 | X86::VPXORQZrmb, X86::VPXORDZrmb }, | |||
6299 | { X86::VANDNPSZ128rmbk, X86::VANDNPDZ128rmbk, | |||
6300 | X86::VPANDNQZ128rmbk, X86::VPANDNDZ128rmbk }, | |||
6301 | { X86::VANDPSZ128rmbk, X86::VANDPDZ128rmbk, | |||
6302 | X86::VPANDQZ128rmbk, X86::VPANDDZ128rmbk }, | |||
6303 | { X86::VORPSZ128rmbk, X86::VORPDZ128rmbk, | |||
6304 | X86::VPORQZ128rmbk, X86::VPORDZ128rmbk }, | |||
6305 | { X86::VXORPSZ128rmbk, X86::VXORPDZ128rmbk, | |||
6306 | X86::VPXORQZ128rmbk, X86::VPXORDZ128rmbk }, | |||
6307 | { X86::VANDNPSZ256rmbk, X86::VANDNPDZ256rmbk, | |||
6308 | X86::VPANDNQZ256rmbk, X86::VPANDNDZ256rmbk }, | |||
6309 | { X86::VANDPSZ256rmbk, X86::VANDPDZ256rmbk, | |||
6310 | X86::VPANDQZ256rmbk, X86::VPANDDZ256rmbk }, | |||
6311 | { X86::VORPSZ256rmbk, X86::VORPDZ256rmbk, | |||
6312 | X86::VPORQZ256rmbk, X86::VPORDZ256rmbk }, | |||
6313 | { X86::VXORPSZ256rmbk, X86::VXORPDZ256rmbk, | |||
6314 | X86::VPXORQZ256rmbk, X86::VPXORDZ256rmbk }, | |||
6315 | { X86::VANDNPSZrmbk, X86::VANDNPDZrmbk, | |||
6316 | X86::VPANDNQZrmbk, X86::VPANDNDZrmbk }, | |||
6317 | { X86::VANDPSZrmbk, X86::VANDPDZrmbk, | |||
6318 | X86::VPANDQZrmbk, X86::VPANDDZrmbk }, | |||
6319 | { X86::VANDPSZrmbk, X86::VANDPDZrmbk, | |||
6320 | X86::VPANDQZrmbk, X86::VPANDDZrmbk }, | |||
6321 | { X86::VORPSZrmbk, X86::VORPDZrmbk, | |||
6322 | X86::VPORQZrmbk, X86::VPORDZrmbk }, | |||
6323 | { X86::VXORPSZrmbk, X86::VXORPDZrmbk, | |||
6324 | X86::VPXORQZrmbk, X86::VPXORDZrmbk }, | |||
6325 | { X86::VANDNPSZ128rmbkz,X86::VANDNPDZ128rmbkz, | |||
6326 | X86::VPANDNQZ128rmbkz,X86::VPANDNDZ128rmbkz}, | |||
6327 | { X86::VANDPSZ128rmbkz, X86::VANDPDZ128rmbkz, | |||
6328 | X86::VPANDQZ128rmbkz, X86::VPANDDZ128rmbkz }, | |||
6329 | { X86::VORPSZ128rmbkz, X86::VORPDZ128rmbkz, | |||
6330 | X86::VPORQZ128rmbkz, X86::VPORDZ128rmbkz }, | |||
6331 | { X86::VXORPSZ128rmbkz, X86::VXORPDZ128rmbkz, | |||
6332 | X86::VPXORQZ128rmbkz, X86::VPXORDZ128rmbkz }, | |||
6333 | { X86::VANDNPSZ256rmbkz,X86::VANDNPDZ256rmbkz, | |||
6334 | X86::VPANDNQZ256rmbkz,X86::VPANDNDZ256rmbkz}, | |||
6335 | { X86::VANDPSZ256rmbkz, X86::VANDPDZ256rmbkz, | |||
6336 | X86::VPANDQZ256rmbkz, X86::VPANDDZ256rmbkz }, | |||
6337 | { X86::VORPSZ256rmbkz, X86::VORPDZ256rmbkz, | |||
6338 | X86::VPORQZ256rmbkz, X86::VPORDZ256rmbkz }, | |||
6339 | { X86::VXORPSZ256rmbkz, X86::VXORPDZ256rmbkz, | |||
6340 | X86::VPXORQZ256rmbkz, X86::VPXORDZ256rmbkz }, | |||
6341 | { X86::VANDNPSZrmbkz, X86::VANDNPDZrmbkz, | |||
6342 | X86::VPANDNQZrmbkz, X86::VPANDNDZrmbkz }, | |||
6343 | { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, | |||
6344 | X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, | |||
6345 | { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, | |||
6346 | X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, | |||
6347 | { X86::VORPSZrmbkz, X86::VORPDZrmbkz, | |||
6348 | X86::VPORQZrmbkz, X86::VPORDZrmbkz }, | |||
6349 | { X86::VXORPSZrmbkz, X86::VXORPDZrmbkz, | |||
6350 | X86::VPXORQZrmbkz, X86::VPXORDZrmbkz }, | |||
6351 | }; | |||
6352 | ||||
6353 | // NOTE: These should only be used by the custom domain methods. | |||
6354 | static const uint16_t ReplaceableCustomInstrs[][3] = { | |||
6355 | //PackedSingle PackedDouble PackedInt | |||
6356 | { X86::BLENDPSrmi, X86::BLENDPDrmi, X86::PBLENDWrmi }, | |||
6357 | { X86::BLENDPSrri, X86::BLENDPDrri, X86::PBLENDWrri }, | |||
6358 | { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDWrmi }, | |||
6359 | { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDWrri }, | |||
6360 | { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDWYrmi }, | |||
6361 | { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDWYrri }, | |||
6362 | }; | |||
6363 | static const uint16_t ReplaceableCustomAVX2Instrs[][3] = { | |||
6364 | //PackedSingle PackedDouble PackedInt | |||
6365 | { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDDrmi }, | |||
6366 | { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDDrri }, | |||
6367 | { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDDYrmi }, | |||
6368 | { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDDYrri }, | |||
6369 | }; | |||
6370 | ||||
6371 | // Special table for changing EVEX logic instructions to VEX. | |||
6372 | // TODO: Should we run EVEX->VEX earlier? | |||
6373 | static const uint16_t ReplaceableCustomAVX512LogicInstrs[][4] = { | |||
6374 | // Two integer columns for 64-bit and 32-bit elements. | |||
6375 | //PackedSingle PackedDouble PackedInt PackedInt | |||
6376 | { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, | |||
6377 | { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, | |||
6378 | { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, | |||
6379 | { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, | |||
6380 | { X86::VORPSrm, X86::VORPDrm, X86::VPORQZ128rm, X86::VPORDZ128rm }, | |||
6381 | { X86::VORPSrr, X86::VORPDrr, X86::VPORQZ128rr, X86::VPORDZ128rr }, | |||
6382 | { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, | |||
6383 | { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, | |||
6384 | { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, | |||
6385 | { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, | |||
6386 | { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, | |||
6387 | { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, | |||
6388 | { X86::VORPSYrm, X86::VORPDYrm, X86::VPORQZ256rm, X86::VPORDZ256rm }, | |||
6389 | { X86::VORPSYrr, X86::VORPDYrr, X86::VPORQZ256rr, X86::VPORDZ256rr }, | |||
6390 | { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, | |||
6391 | { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, | |||
6392 | }; | |||
6393 | ||||
6394 | // FIXME: Some shuffle and unpack instructions have equivalents in different | |||
6395 | // domains, but they require a bit more work than just switching opcodes. | |||
6396 | ||||
6397 | static const uint16_t *lookup(unsigned opcode, unsigned domain, | |||
6398 | ArrayRef<uint16_t[3]> Table) { | |||
6399 | for (const uint16_t (&Row)[3] : Table) | |||
6400 | if (Row[domain-1] == opcode) | |||
6401 | return Row; | |||
6402 | return nullptr; | |||
6403 | } | |||
6404 | ||||
6405 | static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain, | |||
6406 | ArrayRef<uint16_t[4]> Table) { | |||
6407 | // If this is the integer domain make sure to check both integer columns. | |||
6408 | for (const uint16_t (&Row)[4] : Table) | |||
6409 | if (Row[domain-1] == opcode || (domain == 3 && Row[3] == opcode)) | |||
6410 | return Row; | |||
6411 | return nullptr; | |||
6412 | } | |||
6413 | ||||
6414 | // Helper to attempt to widen/narrow blend masks. | |||
6415 | static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, | |||
6416 | unsigned NewWidth, unsigned *pNewMask = nullptr) { | |||
6417 | assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) &&((((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && "Illegal blend mask scale") ? static_cast<void> (0) : __assert_fail ("((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && \"Illegal blend mask scale\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6418, __PRETTY_FUNCTION__)) | |||
6418 | "Illegal blend mask scale")((((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && "Illegal blend mask scale") ? static_cast<void> (0) : __assert_fail ("((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && \"Illegal blend mask scale\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6418, __PRETTY_FUNCTION__)); | |||
6419 | unsigned NewMask = 0; | |||
6420 | ||||
6421 | if ((OldWidth % NewWidth) == 0) { | |||
6422 | unsigned Scale = OldWidth / NewWidth; | |||
6423 | unsigned SubMask = (1u << Scale) - 1; | |||
6424 | for (unsigned i = 0; i != NewWidth; ++i) { | |||
6425 | unsigned Sub = (OldMask >> (i * Scale)) & SubMask; | |||
6426 | if (Sub == SubMask) | |||
6427 | NewMask |= (1u << i); | |||
6428 | else if (Sub != 0x0) | |||
6429 | return false; | |||
6430 | } | |||
6431 | } else { | |||
6432 | unsigned Scale = NewWidth / OldWidth; | |||
6433 | unsigned SubMask = (1u << Scale) - 1; | |||
6434 | for (unsigned i = 0; i != OldWidth; ++i) { | |||
6435 | if (OldMask & (1 << i)) { | |||
6436 | NewMask |= (SubMask << (i * Scale)); | |||
6437 | } | |||
6438 | } | |||
6439 | } | |||
6440 | ||||
6441 | if (pNewMask) | |||
6442 | *pNewMask = NewMask; | |||
6443 | return true; | |||
6444 | } | |||
6445 | ||||
6446 | uint16_t X86InstrInfo::getExecutionDomainCustom(const MachineInstr &MI) const { | |||
6447 | unsigned Opcode = MI.getOpcode(); | |||
6448 | unsigned NumOperands = MI.getDesc().getNumOperands(); | |||
6449 | ||||
6450 | auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) { | |||
6451 | uint16_t validDomains = 0; | |||
6452 | if (MI.getOperand(NumOperands - 1).isImm()) { | |||
6453 | unsigned Imm = MI.getOperand(NumOperands - 1).getImm(); | |||
6454 | if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4)) | |||
6455 | validDomains |= 0x2; // PackedSingle | |||
6456 | if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2)) | |||
6457 | validDomains |= 0x4; // PackedDouble | |||
6458 | if (!Is256 || Subtarget.hasAVX2()) | |||
6459 | validDomains |= 0x8; // PackedInt | |||
6460 | } | |||
6461 | return validDomains; | |||
6462 | }; | |||
6463 | ||||
6464 | switch (Opcode) { | |||
6465 | case X86::BLENDPDrmi: | |||
6466 | case X86::BLENDPDrri: | |||
6467 | case X86::VBLENDPDrmi: | |||
6468 | case X86::VBLENDPDrri: | |||
6469 | return GetBlendDomains(2, false); | |||
6470 | case X86::VBLENDPDYrmi: | |||
6471 | case X86::VBLENDPDYrri: | |||
6472 | return GetBlendDomains(4, true); | |||
6473 | case X86::BLENDPSrmi: | |||
6474 | case X86::BLENDPSrri: | |||
6475 | case X86::VBLENDPSrmi: | |||
6476 | case X86::VBLENDPSrri: | |||
6477 | case X86::VPBLENDDrmi: | |||
6478 | case X86::VPBLENDDrri: | |||
6479 | return GetBlendDomains(4, false); | |||
6480 | case X86::VBLENDPSYrmi: | |||
6481 | case X86::VBLENDPSYrri: | |||
6482 | case X86::VPBLENDDYrmi: | |||
6483 | case X86::VPBLENDDYrri: | |||
6484 | return GetBlendDomains(8, true); | |||
6485 | case X86::PBLENDWrmi: | |||
6486 | case X86::PBLENDWrri: | |||
6487 | case X86::VPBLENDWrmi: | |||
6488 | case X86::VPBLENDWrri: | |||
6489 | // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks. | |||
6490 | case X86::VPBLENDWYrmi: | |||
6491 | case X86::VPBLENDWYrri: | |||
6492 | return GetBlendDomains(8, false); | |||
6493 | case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: | |||
6494 | case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: | |||
6495 | case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: | |||
6496 | case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: | |||
6497 | case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: | |||
6498 | case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: | |||
6499 | case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: | |||
6500 | case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: | |||
6501 | case X86::VPORDZ128rr: case X86::VPORDZ128rm: | |||
6502 | case X86::VPORDZ256rr: case X86::VPORDZ256rm: | |||
6503 | case X86::VPORQZ128rr: case X86::VPORQZ128rm: | |||
6504 | case X86::VPORQZ256rr: case X86::VPORQZ256rm: | |||
6505 | case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: | |||
6506 | case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: | |||
6507 | case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: | |||
6508 | case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: | |||
6509 | // If we don't have DQI see if we can still switch from an EVEX integer | |||
6510 | // instruction to a VEX floating point instruction. | |||
6511 | if (Subtarget.hasDQI()) | |||
6512 | return 0; | |||
6513 | ||||
6514 | if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16) | |||
6515 | return 0; | |||
6516 | if (RI.getEncodingValue(MI.getOperand(1).getReg()) >= 16) | |||
6517 | return 0; | |||
6518 | // Register forms will have 3 operands. Memory form will have more. | |||
6519 | if (NumOperands == 3 && | |||
6520 | RI.getEncodingValue(MI.getOperand(2).getReg()) >= 16) | |||
6521 | return 0; | |||
6522 | ||||
6523 | // All domains are valid. | |||
6524 | return 0xe; | |||
6525 | case X86::MOVHLPSrr: | |||
6526 | // We can swap domains when both inputs are the same register. | |||
6527 | // FIXME: This doesn't catch all the cases we would like. If the input | |||
6528 | // register isn't KILLed by the instruction, the two address instruction | |||
6529 | // pass puts a COPY on one input. The other input uses the original | |||
6530 | // register. This prevents the same physical register from being used by | |||
6531 | // both inputs. | |||
6532 | if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && | |||
6533 | MI.getOperand(0).getSubReg() == 0 && | |||
6534 | MI.getOperand(1).getSubReg() == 0 && | |||
6535 | MI.getOperand(2).getSubReg() == 0) | |||
6536 | return 0x6; | |||
6537 | return 0; | |||
6538 | } | |||
6539 | return 0; | |||
6540 | } | |||
6541 | ||||
6542 | bool X86InstrInfo::setExecutionDomainCustom(MachineInstr &MI, | |||
6543 | unsigned Domain) const { | |||
6544 | assert(Domain > 0 && Domain < 4 && "Invalid execution domain")((Domain > 0 && Domain < 4 && "Invalid execution domain" ) ? static_cast<void> (0) : __assert_fail ("Domain > 0 && Domain < 4 && \"Invalid execution domain\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6544, __PRETTY_FUNCTION__)); | |||
6545 | uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; | |||
6546 | assert(dom && "Not an SSE instruction")((dom && "Not an SSE instruction") ? static_cast<void > (0) : __assert_fail ("dom && \"Not an SSE instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6546, __PRETTY_FUNCTION__)); | |||
6547 | ||||
6548 | unsigned Opcode = MI.getOpcode(); | |||
6549 | unsigned NumOperands = MI.getDesc().getNumOperands(); | |||
6550 | ||||
6551 | auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) { | |||
6552 | if (MI.getOperand(NumOperands - 1).isImm()) { | |||
6553 | unsigned Imm = MI.getOperand(NumOperands - 1).getImm() & 255; | |||
6554 | Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm); | |||
6555 | unsigned NewImm = Imm; | |||
6556 | ||||
6557 | const uint16_t *table = lookup(Opcode, dom, ReplaceableCustomInstrs); | |||
6558 | if (!table) | |||
6559 | table = lookup(Opcode, dom, ReplaceableCustomAVX2Instrs); | |||
6560 | ||||
6561 | if (Domain == 1) { // PackedSingle | |||
6562 | AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); | |||
6563 | } else if (Domain == 2) { // PackedDouble | |||
6564 | AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2, &NewImm); | |||
6565 | } else if (Domain == 3) { // PackedInt | |||
6566 | if (Subtarget.hasAVX2()) { | |||
6567 | // If we are already VPBLENDW use that, else use VPBLENDD. | |||
6568 | if ((ImmWidth / (Is256 ? 2 : 1)) != 8) { | |||
6569 | table = lookup(Opcode, dom, ReplaceableCustomAVX2Instrs); | |||
6570 | AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); | |||
6571 | } | |||
6572 | } else { | |||
6573 | assert(!Is256 && "128-bit vector expected")((!Is256 && "128-bit vector expected") ? static_cast< void> (0) : __assert_fail ("!Is256 && \"128-bit vector expected\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6573, __PRETTY_FUNCTION__)); | |||
6574 | AdjustBlendMask(Imm, ImmWidth, 8, &NewImm); | |||
6575 | } | |||
6576 | } | |||
6577 | ||||
6578 | assert(table && table[Domain - 1] && "Unknown domain op")((table && table[Domain - 1] && "Unknown domain op" ) ? static_cast<void> (0) : __assert_fail ("table && table[Domain - 1] && \"Unknown domain op\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6578, __PRETTY_FUNCTION__)); | |||
6579 | MI.setDesc(get(table[Domain - 1])); | |||
6580 | MI.getOperand(NumOperands - 1).setImm(NewImm & 255); | |||
6581 | } | |||
6582 | return true; | |||
6583 | }; | |||
6584 | ||||
6585 | switch (Opcode) { | |||
6586 | case X86::BLENDPDrmi: | |||
6587 | case X86::BLENDPDrri: | |||
6588 | case X86::VBLENDPDrmi: | |||
6589 | case X86::VBLENDPDrri: | |||
6590 | return SetBlendDomain(2, false); | |||
6591 | case X86::VBLENDPDYrmi: | |||
6592 | case X86::VBLENDPDYrri: | |||
6593 | return SetBlendDomain(4, true); | |||
6594 | case X86::BLENDPSrmi: | |||
6595 | case X86::BLENDPSrri: | |||
6596 | case X86::VBLENDPSrmi: | |||
6597 | case X86::VBLENDPSrri: | |||
6598 | case X86::VPBLENDDrmi: | |||
6599 | case X86::VPBLENDDrri: | |||
6600 | return SetBlendDomain(4, false); | |||
6601 | case X86::VBLENDPSYrmi: | |||
6602 | case X86::VBLENDPSYrri: | |||
6603 | case X86::VPBLENDDYrmi: | |||
6604 | case X86::VPBLENDDYrri: | |||
6605 | return SetBlendDomain(8, true); | |||
6606 | case X86::PBLENDWrmi: | |||
6607 | case X86::PBLENDWrri: | |||
6608 | case X86::VPBLENDWrmi: | |||
6609 | case X86::VPBLENDWrri: | |||
6610 | return SetBlendDomain(8, false); | |||
6611 | case X86::VPBLENDWYrmi: | |||
6612 | case X86::VPBLENDWYrri: | |||
6613 | return SetBlendDomain(16, true); | |||
6614 | case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: | |||
6615 | case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: | |||
6616 | case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: | |||
6617 | case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: | |||
6618 | case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: | |||
6619 | case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: | |||
6620 | case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: | |||
6621 | case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: | |||
6622 | case X86::VPORDZ128rr: case X86::VPORDZ128rm: | |||
6623 | case X86::VPORDZ256rr: case X86::VPORDZ256rm: | |||
6624 | case X86::VPORQZ128rr: case X86::VPORQZ128rm: | |||
6625 | case X86::VPORQZ256rr: case X86::VPORQZ256rm: | |||
6626 | case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: | |||
6627 | case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: | |||
6628 | case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: | |||
6629 | case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: { | |||
6630 | // Without DQI, convert EVEX instructions to VEX instructions. | |||
6631 | if (Subtarget.hasDQI()) | |||
6632 | return false; | |||
6633 | ||||
6634 | const uint16_t *table = lookupAVX512(MI.getOpcode(), dom, | |||
6635 | ReplaceableCustomAVX512LogicInstrs); | |||
6636 | assert(table && "Instruction not found in table?")((table && "Instruction not found in table?") ? static_cast <void> (0) : __assert_fail ("table && \"Instruction not found in table?\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6636, __PRETTY_FUNCTION__)); | |||
6637 | // Don't change integer Q instructions to D instructions and | |||
6638 | // use D intructions if we started with a PS instruction. | |||
6639 | if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) | |||
6640 | Domain = 4; | |||
6641 | MI.setDesc(get(table[Domain - 1])); | |||
6642 | return true; | |||
6643 | } | |||
6644 | case X86::UNPCKHPDrr: | |||
6645 | case X86::MOVHLPSrr: | |||
6646 | // We just need to commute the instruction which will switch the domains. | |||
6647 | if (Domain != dom && Domain != 3 && | |||
6648 | MI.getOperand(1).getReg() == MI.getOperand(2).getReg() && | |||
6649 | MI.getOperand(0).getSubReg() == 0 && | |||
6650 | MI.getOperand(1).getSubReg() == 0 && | |||
6651 | MI.getOperand(2).getSubReg() == 0) { | |||
6652 | commuteInstruction(MI, false); | |||
6653 | return true; | |||
6654 | } | |||
6655 | // We must always return true for MOVHLPSrr. | |||
6656 | if (Opcode == X86::MOVHLPSrr) | |||
6657 | return true; | |||
6658 | } | |||
6659 | return false; | |||
6660 | } | |||
6661 | ||||
6662 | std::pair<uint16_t, uint16_t> | |||
6663 | X86InstrInfo::getExecutionDomain(const MachineInstr &MI) const { | |||
6664 | uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; | |||
6665 | unsigned opcode = MI.getOpcode(); | |||
6666 | uint16_t validDomains = 0; | |||
6667 | if (domain) { | |||
6668 | // Attempt to match for custom instructions. | |||
6669 | validDomains = getExecutionDomainCustom(MI); | |||
6670 | if (validDomains) | |||
6671 | return std::make_pair(domain, validDomains); | |||
6672 | ||||
6673 | if (lookup(opcode, domain, ReplaceableInstrs)) { | |||
6674 | validDomains = 0xe; | |||
6675 | } else if (lookup(opcode, domain, ReplaceableInstrsAVX2)) { | |||
6676 | validDomains = Subtarget.hasAVX2() ? 0xe : 0x6; | |||
6677 | } else if (lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) { | |||
6678 | // Insert/extract instructions should only effect domain if AVX2 | |||
6679 | // is enabled. | |||
6680 | if (!Subtarget.hasAVX2()) | |||
6681 | return std::make_pair(0, 0); | |||
6682 | validDomains = 0xe; | |||
6683 | } else if (lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) { | |||
6684 | validDomains = 0xe; | |||
6685 | } else if (Subtarget.hasDQI() && lookupAVX512(opcode, domain, | |||
6686 | ReplaceableInstrsAVX512DQ)) { | |||
6687 | validDomains = 0xe; | |||
6688 | } else if (Subtarget.hasDQI()) { | |||
6689 | if (const uint16_t *table = lookupAVX512(opcode, domain, | |||
6690 | ReplaceableInstrsAVX512DQMasked)) { | |||
6691 | if (domain == 1 || (domain == 3 && table[3] == opcode)) | |||
6692 | validDomains = 0xa; | |||
6693 | else | |||
6694 | validDomains = 0xc; | |||
6695 | } | |||
6696 | } | |||
6697 | } | |||
6698 | return std::make_pair(domain, validDomains); | |||
6699 | } | |||
6700 | ||||
6701 | void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const { | |||
6702 | assert(Domain>0 && Domain<4 && "Invalid execution domain")((Domain>0 && Domain<4 && "Invalid execution domain" ) ? static_cast<void> (0) : __assert_fail ("Domain>0 && Domain<4 && \"Invalid execution domain\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6702, __PRETTY_FUNCTION__)); | |||
6703 | uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; | |||
6704 | assert(dom && "Not an SSE instruction")((dom && "Not an SSE instruction") ? static_cast<void > (0) : __assert_fail ("dom && \"Not an SSE instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6704, __PRETTY_FUNCTION__)); | |||
6705 | ||||
6706 | // Attempt to match for custom instructions. | |||
6707 | if (setExecutionDomainCustom(MI, Domain)) | |||
6708 | return; | |||
6709 | ||||
6710 | const uint16_t *table = lookup(MI.getOpcode(), dom, ReplaceableInstrs); | |||
6711 | if (!table) { // try the other table | |||
6712 | assert((Subtarget.hasAVX2() || Domain < 3) &&(((Subtarget.hasAVX2() || Domain < 3) && "256-bit vector operations only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasAVX2() || Domain < 3) && \"256-bit vector operations only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6713, __PRETTY_FUNCTION__)) | |||
6713 | "256-bit vector operations only available in AVX2")(((Subtarget.hasAVX2() || Domain < 3) && "256-bit vector operations only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasAVX2() || Domain < 3) && \"256-bit vector operations only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6713, __PRETTY_FUNCTION__)); | |||
6714 | table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2); | |||
6715 | } | |||
6716 | if (!table) { // try the other table | |||
6717 | assert(Subtarget.hasAVX2() &&((Subtarget.hasAVX2() && "256-bit insert/extract only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"256-bit insert/extract only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6718, __PRETTY_FUNCTION__)) | |||
6718 | "256-bit insert/extract only available in AVX2")((Subtarget.hasAVX2() && "256-bit insert/extract only available in AVX2" ) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"256-bit insert/extract only available in AVX2\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6718, __PRETTY_FUNCTION__)); | |||
6719 | table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract); | |||
6720 | } | |||
6721 | if (!table) { // try the AVX512 table | |||
6722 | assert(Subtarget.hasAVX512() && "Requires AVX-512")((Subtarget.hasAVX512() && "Requires AVX-512") ? static_cast <void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"Requires AVX-512\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6722, __PRETTY_FUNCTION__)); | |||
6723 | table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512); | |||
6724 | // Don't change integer Q instructions to D instructions. | |||
6725 | if (table && Domain == 3 && table[3] == MI.getOpcode()) | |||
6726 | Domain = 4; | |||
6727 | } | |||
6728 | if (!table) { // try the AVX512DQ table | |||
6729 | assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ")(((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasDQI() || Domain >= 3) && \"Requires AVX-512DQ\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6729, __PRETTY_FUNCTION__)); | |||
6730 | table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ); | |||
6731 | // Don't change integer Q instructions to D instructions and | |||
6732 | // use D intructions if we started with a PS instruction. | |||
6733 | if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) | |||
6734 | Domain = 4; | |||
6735 | } | |||
6736 | if (!table) { // try the AVX512DQMasked table | |||
6737 | assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ")(((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ" ) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasDQI() || Domain >= 3) && \"Requires AVX-512DQ\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6737, __PRETTY_FUNCTION__)); | |||
6738 | table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked); | |||
6739 | if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) | |||
6740 | Domain = 4; | |||
6741 | } | |||
6742 | assert(table && "Cannot change domain")((table && "Cannot change domain") ? static_cast<void > (0) : __assert_fail ("table && \"Cannot change domain\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 6742, __PRETTY_FUNCTION__)); | |||
6743 | MI.setDesc(get(table[Domain - 1])); | |||
6744 | } | |||
6745 | ||||
6746 | /// Return the noop instruction to use for a noop. | |||
6747 | void X86InstrInfo::getNoop(MCInst &NopInst) const { | |||
6748 | NopInst.setOpcode(X86::NOOP); | |||
6749 | } | |||
6750 | ||||
6751 | bool X86InstrInfo::isHighLatencyDef(int opc) const { | |||
6752 | switch (opc) { | |||
6753 | default: return false; | |||
6754 | case X86::DIVPDrm: | |||
6755 | case X86::DIVPDrr: | |||
6756 | case X86::DIVPSrm: | |||
6757 | case X86::DIVPSrr: | |||
6758 | case X86::DIVSDrm: | |||
6759 | case X86::DIVSDrm_Int: | |||
6760 | case X86::DIVSDrr: | |||
6761 | case X86::DIVSDrr_Int: | |||
6762 | case X86::DIVSSrm: | |||
6763 | case X86::DIVSSrm_Int: | |||
6764 | case X86::DIVSSrr: | |||
6765 | case X86::DIVSSrr_Int: | |||
6766 | case X86::SQRTPDm: | |||
6767 | case X86::SQRTPDr: | |||
6768 | case X86::SQRTPSm: | |||
6769 | case X86::SQRTPSr: | |||
6770 | case X86::SQRTSDm: | |||
6771 | case X86::SQRTSDm_Int: | |||
6772 | case X86::SQRTSDr: | |||
6773 | case X86::SQRTSDr_Int: | |||
6774 | case X86::SQRTSSm: | |||
6775 | case X86::SQRTSSm_Int: | |||
6776 | case X86::SQRTSSr: | |||
6777 | case X86::SQRTSSr_Int: | |||
6778 | // AVX instructions with high latency | |||
6779 | case X86::VDIVPDrm: | |||
6780 | case X86::VDIVPDrr: | |||
6781 | case X86::VDIVPDYrm: | |||
6782 | case X86::VDIVPDYrr: | |||
6783 | case X86::VDIVPSrm: | |||
6784 | case X86::VDIVPSrr: | |||
6785 | case X86::VDIVPSYrm: | |||
6786 | case X86::VDIVPSYrr: | |||
6787 | case X86::VDIVSDrm: | |||
6788 | case X86::VDIVSDrm_Int: | |||
6789 | case X86::VDIVSDrr: | |||
6790 | case X86::VDIVSDrr_Int: | |||
6791 | case X86::VDIVSSrm: | |||
6792 | case X86::VDIVSSrm_Int: | |||
6793 | case X86::VDIVSSrr: | |||
6794 | case X86::VDIVSSrr_Int: | |||
6795 | case X86::VSQRTPDm: | |||
6796 | case X86::VSQRTPDr: | |||
6797 | case X86::VSQRTPDYm: | |||
6798 | case X86::VSQRTPDYr: | |||
6799 | case X86::VSQRTPSm: | |||
6800 | case X86::VSQRTPSr: | |||
6801 | case X86::VSQRTPSYm: | |||
6802 | case X86::VSQRTPSYr: | |||
6803 | case X86::VSQRTSDm: | |||
6804 | case X86::VSQRTSDm_Int: | |||
6805 | case X86::VSQRTSDr: | |||
6806 | case X86::VSQRTSDr_Int: | |||
6807 | case X86::VSQRTSSm: | |||
6808 | case X86::VSQRTSSm_Int: | |||
6809 | case X86::VSQRTSSr: | |||
6810 | case X86::VSQRTSSr_Int: | |||
6811 | // AVX512 instructions with high latency | |||
6812 | case X86::VDIVPDZ128rm: | |||
6813 | case X86::VDIVPDZ128rmb: | |||
6814 | case X86::VDIVPDZ128rmbk: | |||
6815 | case X86::VDIVPDZ128rmbkz: | |||
6816 | case X86::VDIVPDZ128rmk: | |||
6817 | case X86::VDIVPDZ128rmkz: | |||
6818 | case X86::VDIVPDZ128rr: | |||
6819 | case X86::VDIVPDZ128rrk: | |||
6820 | case X86::VDIVPDZ128rrkz: | |||
6821 | case X86::VDIVPDZ256rm: | |||
6822 | case X86::VDIVPDZ256rmb: | |||
6823 | case X86::VDIVPDZ256rmbk: | |||
6824 | case X86::VDIVPDZ256rmbkz: | |||
6825 | case X86::VDIVPDZ256rmk: | |||
6826 | case X86::VDIVPDZ256rmkz: | |||
6827 | case X86::VDIVPDZ256rr: | |||
6828 | case X86::VDIVPDZ256rrk: | |||
6829 | case X86::VDIVPDZ256rrkz: | |||
6830 | case X86::VDIVPDZrrb: | |||
6831 | case X86::VDIVPDZrrbk: | |||
6832 | case X86::VDIVPDZrrbkz: | |||
6833 | case X86::VDIVPDZrm: | |||
6834 | case X86::VDIVPDZrmb: | |||
6835 | case X86::VDIVPDZrmbk: | |||
6836 | case X86::VDIVPDZrmbkz: | |||
6837 | case X86::VDIVPDZrmk: | |||
6838 | case X86::VDIVPDZrmkz: | |||
6839 | case X86::VDIVPDZrr: | |||
6840 | case X86::VDIVPDZrrk: | |||
6841 | case X86::VDIVPDZrrkz: | |||
6842 | case X86::VDIVPSZ128rm: | |||
6843 | case X86::VDIVPSZ128rmb: | |||
6844 | case X86::VDIVPSZ128rmbk: | |||
6845 | case X86::VDIVPSZ128rmbkz: | |||
6846 | case X86::VDIVPSZ128rmk: | |||
6847 | case X86::VDIVPSZ128rmkz: | |||
6848 | case X86::VDIVPSZ128rr: | |||
6849 | case X86::VDIVPSZ128rrk: | |||
6850 | case X86::VDIVPSZ128rrkz: | |||
6851 | case X86::VDIVPSZ256rm: | |||
6852 | case X86::VDIVPSZ256rmb: | |||
6853 | case X86::VDIVPSZ256rmbk: | |||
6854 | case X86::VDIVPSZ256rmbkz: | |||
6855 | case X86::VDIVPSZ256rmk: | |||
6856 | case X86::VDIVPSZ256rmkz: | |||
6857 | case X86::VDIVPSZ256rr: | |||
6858 | case X86::VDIVPSZ256rrk: | |||
6859 | case X86::VDIVPSZ256rrkz: | |||
6860 | case X86::VDIVPSZrrb: | |||
6861 | case X86::VDIVPSZrrbk: | |||
6862 | case X86::VDIVPSZrrbkz: | |||
6863 | case X86::VDIVPSZrm: | |||
6864 | case X86::VDIVPSZrmb: | |||
6865 | case X86::VDIVPSZrmbk: | |||
6866 | case X86::VDIVPSZrmbkz: | |||
6867 | case X86::VDIVPSZrmk: | |||
6868 | case X86::VDIVPSZrmkz: | |||
6869 | case X86::VDIVPSZrr: | |||
6870 | case X86::VDIVPSZrrk: | |||
6871 | case X86::VDIVPSZrrkz: | |||
6872 | case X86::VDIVSDZrm: | |||
6873 | case X86::VDIVSDZrr: | |||
6874 | case X86::VDIVSDZrm_Int: | |||
6875 | case X86::VDIVSDZrm_Intk: | |||
6876 | case X86::VDIVSDZrm_Intkz: | |||
6877 | case X86::VDIVSDZrr_Int: | |||
6878 | case X86::VDIVSDZrr_Intk: | |||
6879 | case X86::VDIVSDZrr_Intkz: | |||
6880 | case X86::VDIVSDZrrb_Int: | |||
6881 | case X86::VDIVSDZrrb_Intk: | |||
6882 | case X86::VDIVSDZrrb_Intkz: | |||
6883 | case X86::VDIVSSZrm: | |||
6884 | case X86::VDIVSSZrr: | |||
6885 | case X86::VDIVSSZrm_Int: | |||
6886 | case X86::VDIVSSZrm_Intk: | |||
6887 | case X86::VDIVSSZrm_Intkz: | |||
6888 | case X86::VDIVSSZrr_Int: | |||
6889 | case X86::VDIVSSZrr_Intk: | |||
6890 | case X86::VDIVSSZrr_Intkz: | |||
6891 | case X86::VDIVSSZrrb_Int: | |||
6892 | case X86::VDIVSSZrrb_Intk: | |||
6893 | case X86::VDIVSSZrrb_Intkz: | |||
6894 | case X86::VSQRTPDZ128m: | |||
6895 | case X86::VSQRTPDZ128mb: | |||
6896 | case X86::VSQRTPDZ128mbk: | |||
6897 | case X86::VSQRTPDZ128mbkz: | |||
6898 | case X86::VSQRTPDZ128mk: | |||
6899 | case X86::VSQRTPDZ128mkz: | |||
6900 | case X86::VSQRTPDZ128r: | |||
6901 | case X86::VSQRTPDZ128rk: | |||
6902 | case X86::VSQRTPDZ128rkz: | |||
6903 | case X86::VSQRTPDZ256m: | |||
6904 | case X86::VSQRTPDZ256mb: | |||
6905 | case X86::VSQRTPDZ256mbk: | |||
6906 | case X86::VSQRTPDZ256mbkz: | |||
6907 | case X86::VSQRTPDZ256mk: | |||
6908 | case X86::VSQRTPDZ256mkz: | |||
6909 | case X86::VSQRTPDZ256r: | |||
6910 | case X86::VSQRTPDZ256rk: | |||
6911 | case X86::VSQRTPDZ256rkz: | |||
6912 | case X86::VSQRTPDZm: | |||
6913 | case X86::VSQRTPDZmb: | |||
6914 | case X86::VSQRTPDZmbk: | |||
6915 | case X86::VSQRTPDZmbkz: | |||
6916 | case X86::VSQRTPDZmk: | |||
6917 | case X86::VSQRTPDZmkz: | |||
6918 | case X86::VSQRTPDZr: | |||
6919 | case X86::VSQRTPDZrb: | |||
6920 | case X86::VSQRTPDZrbk: | |||
6921 | case X86::VSQRTPDZrbkz: | |||
6922 | case X86::VSQRTPDZrk: | |||
6923 | case X86::VSQRTPDZrkz: | |||
6924 | case X86::VSQRTPSZ128m: | |||
6925 | case X86::VSQRTPSZ128mb: | |||
6926 | case X86::VSQRTPSZ128mbk: | |||
6927 | case X86::VSQRTPSZ128mbkz: | |||
6928 | case X86::VSQRTPSZ128mk: | |||
6929 | case X86::VSQRTPSZ128mkz: | |||
6930 | case X86::VSQRTPSZ128r: | |||
6931 | case X86::VSQRTPSZ128rk: | |||
6932 | case X86::VSQRTPSZ128rkz: | |||
6933 | case X86::VSQRTPSZ256m: | |||
6934 | case X86::VSQRTPSZ256mb: | |||
6935 | case X86::VSQRTPSZ256mbk: | |||
6936 | case X86::VSQRTPSZ256mbkz: | |||
6937 | case X86::VSQRTPSZ256mk: | |||
6938 | case X86::VSQRTPSZ256mkz: | |||
6939 | case X86::VSQRTPSZ256r: | |||
6940 | case X86::VSQRTPSZ256rk: | |||
6941 | case X86::VSQRTPSZ256rkz: | |||
6942 | case X86::VSQRTPSZm: | |||
6943 | case X86::VSQRTPSZmb: | |||
6944 | case X86::VSQRTPSZmbk: | |||
6945 | case X86::VSQRTPSZmbkz: | |||
6946 | case X86::VSQRTPSZmk: | |||
6947 | case X86::VSQRTPSZmkz: | |||
6948 | case X86::VSQRTPSZr: | |||
6949 | case X86::VSQRTPSZrb: | |||
6950 | case X86::VSQRTPSZrbk: | |||
6951 | case X86::VSQRTPSZrbkz: | |||
6952 | case X86::VSQRTPSZrk: | |||
6953 | case X86::VSQRTPSZrkz: | |||
6954 | case X86::VSQRTSDZm: | |||
6955 | case X86::VSQRTSDZm_Int: | |||
6956 | case X86::VSQRTSDZm_Intk: | |||
6957 | case X86::VSQRTSDZm_Intkz: | |||
6958 | case X86::VSQRTSDZr: | |||
6959 | case X86::VSQRTSDZr_Int: | |||
6960 | case X86::VSQRTSDZr_Intk: | |||
6961 | case X86::VSQRTSDZr_Intkz: | |||
6962 | case X86::VSQRTSDZrb_Int: | |||
6963 | case X86::VSQRTSDZrb_Intk: | |||
6964 | case X86::VSQRTSDZrb_Intkz: | |||
6965 | case X86::VSQRTSSZm: | |||
6966 | case X86::VSQRTSSZm_Int: | |||
6967 | case X86::VSQRTSSZm_Intk: | |||
6968 | case X86::VSQRTSSZm_Intkz: | |||
6969 | case X86::VSQRTSSZr: | |||
6970 | case X86::VSQRTSSZr_Int: | |||
6971 | case X86::VSQRTSSZr_Intk: | |||
6972 | case X86::VSQRTSSZr_Intkz: | |||
6973 | case X86::VSQRTSSZrb_Int: | |||
6974 | case X86::VSQRTSSZrb_Intk: | |||
6975 | case X86::VSQRTSSZrb_Intkz: | |||
6976 | ||||
6977 | case X86::VGATHERDPDYrm: | |||
6978 | case X86::VGATHERDPDZ128rm: | |||
6979 | case X86::VGATHERDPDZ256rm: | |||
6980 | case X86::VGATHERDPDZrm: | |||
6981 | case X86::VGATHERDPDrm: | |||
6982 | case X86::VGATHERDPSYrm: | |||
6983 | case X86::VGATHERDPSZ128rm: | |||
6984 | case X86::VGATHERDPSZ256rm: | |||
6985 | case X86::VGATHERDPSZrm: | |||
6986 | case X86::VGATHERDPSrm: | |||
6987 | case X86::VGATHERPF0DPDm: | |||
6988 | case X86::VGATHERPF0DPSm: | |||
6989 | case X86::VGATHERPF0QPDm: | |||
6990 | case X86::VGATHERPF0QPSm: | |||
6991 | case X86::VGATHERPF1DPDm: | |||
6992 | case X86::VGATHERPF1DPSm: | |||
6993 | case X86::VGATHERPF1QPDm: | |||
6994 | case X86::VGATHERPF1QPSm: | |||
6995 | case X86::VGATHERQPDYrm: | |||
6996 | case X86::VGATHERQPDZ128rm: | |||
6997 | case X86::VGATHERQPDZ256rm: | |||
6998 | case X86::VGATHERQPDZrm: | |||
6999 | case X86::VGATHERQPDrm: | |||
7000 | case X86::VGATHERQPSYrm: | |||
7001 | case X86::VGATHERQPSZ128rm: | |||
7002 | case X86::VGATHERQPSZ256rm: | |||
7003 | case X86::VGATHERQPSZrm: | |||
7004 | case X86::VGATHERQPSrm: | |||
7005 | case X86::VPGATHERDDYrm: | |||
7006 | case X86::VPGATHERDDZ128rm: | |||
7007 | case X86::VPGATHERDDZ256rm: | |||
7008 | case X86::VPGATHERDDZrm: | |||
7009 | case X86::VPGATHERDDrm: | |||
7010 | case X86::VPGATHERDQYrm: | |||
7011 | case X86::VPGATHERDQZ128rm: | |||
7012 | case X86::VPGATHERDQZ256rm: | |||
7013 | case X86::VPGATHERDQZrm: | |||
7014 | case X86::VPGATHERDQrm: | |||
7015 | case X86::VPGATHERQDYrm: | |||
7016 | case X86::VPGATHERQDZ128rm: | |||
7017 | case X86::VPGATHERQDZ256rm: | |||
7018 | case X86::VPGATHERQDZrm: | |||
7019 | case X86::VPGATHERQDrm: | |||
7020 | case X86::VPGATHERQQYrm: | |||
7021 | case X86::VPGATHERQQZ128rm: | |||
7022 | case X86::VPGATHERQQZ256rm: | |||
7023 | case X86::VPGATHERQQZrm: | |||
7024 | case X86::VPGATHERQQrm: | |||
7025 | case X86::VSCATTERDPDZ128mr: | |||
7026 | case X86::VSCATTERDPDZ256mr: | |||
7027 | case X86::VSCATTERDPDZmr: | |||
7028 | case X86::VSCATTERDPSZ128mr: | |||
7029 | case X86::VSCATTERDPSZ256mr: | |||
7030 | case X86::VSCATTERDPSZmr: | |||
7031 | case X86::VSCATTERPF0DPDm: | |||
7032 | case X86::VSCATTERPF0DPSm: | |||
7033 | case X86::VSCATTERPF0QPDm: | |||
7034 | case X86::VSCATTERPF0QPSm: | |||
7035 | case X86::VSCATTERPF1DPDm: | |||
7036 | case X86::VSCATTERPF1DPSm: | |||
7037 | case X86::VSCATTERPF1QPDm: | |||
7038 | case X86::VSCATTERPF1QPSm: | |||
7039 | case X86::VSCATTERQPDZ128mr: | |||
7040 | case X86::VSCATTERQPDZ256mr: | |||
7041 | case X86::VSCATTERQPDZmr: | |||
7042 | case X86::VSCATTERQPSZ128mr: | |||
7043 | case X86::VSCATTERQPSZ256mr: | |||
7044 | case X86::VSCATTERQPSZmr: | |||
7045 | case X86::VPSCATTERDDZ128mr: | |||
7046 | case X86::VPSCATTERDDZ256mr: | |||
7047 | case X86::VPSCATTERDDZmr: | |||
7048 | case X86::VPSCATTERDQZ128mr: | |||
7049 | case X86::VPSCATTERDQZ256mr: | |||
7050 | case X86::VPSCATTERDQZmr: | |||
7051 | case X86::VPSCATTERQDZ128mr: | |||
7052 | case X86::VPSCATTERQDZ256mr: | |||
7053 | case X86::VPSCATTERQDZmr: | |||
7054 | case X86::VPSCATTERQQZ128mr: | |||
7055 | case X86::VPSCATTERQQZ256mr: | |||
7056 | case X86::VPSCATTERQQZmr: | |||
7057 | return true; | |||
7058 | } | |||
7059 | } | |||
7060 | ||||
7061 | bool X86InstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel, | |||
7062 | const MachineRegisterInfo *MRI, | |||
7063 | const MachineInstr &DefMI, | |||
7064 | unsigned DefIdx, | |||
7065 | const MachineInstr &UseMI, | |||
7066 | unsigned UseIdx) const { | |||
7067 | return isHighLatencyDef(DefMI.getOpcode()); | |||
7068 | } | |||
7069 | ||||
7070 | bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst, | |||
7071 | const MachineBasicBlock *MBB) const { | |||
7072 | assert((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) &&(((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && "Reassociation needs binary operators") ? static_cast<void > (0) : __assert_fail ("(Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && \"Reassociation needs binary operators\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7073, __PRETTY_FUNCTION__)) | |||
7073 | "Reassociation needs binary operators")(((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && "Reassociation needs binary operators") ? static_cast<void > (0) : __assert_fail ("(Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && \"Reassociation needs binary operators\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7073, __PRETTY_FUNCTION__)); | |||
7074 | ||||
7075 | // Integer binary math/logic instructions have a third source operand: | |||
7076 | // the EFLAGS register. That operand must be both defined here and never | |||
7077 | // used; ie, it must be dead. If the EFLAGS operand is live, then we can | |||
7078 | // not change anything because rearranging the operands could affect other | |||
7079 | // instructions that depend on the exact status flags (zero, sign, etc.) | |||
7080 | // that are set by using these particular operands with this operation. | |||
7081 | if (Inst.getNumOperands() == 4) { | |||
7082 | assert(Inst.getOperand(3).isReg() &&((Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg () == X86::EFLAGS && "Unexpected operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7084, __PRETTY_FUNCTION__)) | |||
7083 | Inst.getOperand(3).getReg() == X86::EFLAGS &&((Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg () == X86::EFLAGS && "Unexpected operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7084, __PRETTY_FUNCTION__)) | |||
7084 | "Unexpected operand in reassociable instruction")((Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg () == X86::EFLAGS && "Unexpected operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(3).isReg() && Inst.getOperand(3).getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7084, __PRETTY_FUNCTION__)); | |||
7085 | if (!Inst.getOperand(3).isDead()) | |||
7086 | return false; | |||
7087 | } | |||
7088 | ||||
7089 | return TargetInstrInfo::hasReassociableOperands(Inst, MBB); | |||
7090 | } | |||
7091 | ||||
7092 | // TODO: There are many more machine instruction opcodes to match: | |||
7093 | // 1. Other data types (integer, vectors) | |||
7094 | // 2. Other math / logic operations (xor, or) | |||
7095 | // 3. Other forms of the same operation (intrinsics and other variants) | |||
7096 | bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const { | |||
7097 | switch (Inst.getOpcode()) { | |||
7098 | case X86::AND8rr: | |||
7099 | case X86::AND16rr: | |||
7100 | case X86::AND32rr: | |||
7101 | case X86::AND64rr: | |||
7102 | case X86::OR8rr: | |||
7103 | case X86::OR16rr: | |||
7104 | case X86::OR32rr: | |||
7105 | case X86::OR64rr: | |||
7106 | case X86::XOR8rr: | |||
7107 | case X86::XOR16rr: | |||
7108 | case X86::XOR32rr: | |||
7109 | case X86::XOR64rr: | |||
7110 | case X86::IMUL16rr: | |||
7111 | case X86::IMUL32rr: | |||
7112 | case X86::IMUL64rr: | |||
7113 | case X86::PANDrr: | |||
7114 | case X86::PORrr: | |||
7115 | case X86::PXORrr: | |||
7116 | case X86::ANDPDrr: | |||
7117 | case X86::ANDPSrr: | |||
7118 | case X86::ORPDrr: | |||
7119 | case X86::ORPSrr: | |||
7120 | case X86::XORPDrr: | |||
7121 | case X86::XORPSrr: | |||
7122 | case X86::PADDBrr: | |||
7123 | case X86::PADDWrr: | |||
7124 | case X86::PADDDrr: | |||
7125 | case X86::PADDQrr: | |||
7126 | case X86::VPANDrr: | |||
7127 | case X86::VPANDYrr: | |||
7128 | case X86::VPANDDZ128rr: | |||
7129 | case X86::VPANDDZ256rr: | |||
7130 | case X86::VPANDDZrr: | |||
7131 | case X86::VPANDQZ128rr: | |||
7132 | case X86::VPANDQZ256rr: | |||
7133 | case X86::VPANDQZrr: | |||
7134 | case X86::VPORrr: | |||
7135 | case X86::VPORYrr: | |||
7136 | case X86::VPORDZ128rr: | |||
7137 | case X86::VPORDZ256rr: | |||
7138 | case X86::VPORDZrr: | |||
7139 | case X86::VPORQZ128rr: | |||
7140 | case X86::VPORQZ256rr: | |||
7141 | case X86::VPORQZrr: | |||
7142 | case X86::VPXORrr: | |||
7143 | case X86::VPXORYrr: | |||
7144 | case X86::VPXORDZ128rr: | |||
7145 | case X86::VPXORDZ256rr: | |||
7146 | case X86::VPXORDZrr: | |||
7147 | case X86::VPXORQZ128rr: | |||
7148 | case X86::VPXORQZ256rr: | |||
7149 | case X86::VPXORQZrr: | |||
7150 | case X86::VANDPDrr: | |||
7151 | case X86::VANDPSrr: | |||
7152 | case X86::VANDPDYrr: | |||
7153 | case X86::VANDPSYrr: | |||
7154 | case X86::VANDPDZ128rr: | |||
7155 | case X86::VANDPSZ128rr: | |||
7156 | case X86::VANDPDZ256rr: | |||
7157 | case X86::VANDPSZ256rr: | |||
7158 | case X86::VANDPDZrr: | |||
7159 | case X86::VANDPSZrr: | |||
7160 | case X86::VORPDrr: | |||
7161 | case X86::VORPSrr: | |||
7162 | case X86::VORPDYrr: | |||
7163 | case X86::VORPSYrr: | |||
7164 | case X86::VORPDZ128rr: | |||
7165 | case X86::VORPSZ128rr: | |||
7166 | case X86::VORPDZ256rr: | |||
7167 | case X86::VORPSZ256rr: | |||
7168 | case X86::VORPDZrr: | |||
7169 | case X86::VORPSZrr: | |||
7170 | case X86::VXORPDrr: | |||
7171 | case X86::VXORPSrr: | |||
7172 | case X86::VXORPDYrr: | |||
7173 | case X86::VXORPSYrr: | |||
7174 | case X86::VXORPDZ128rr: | |||
7175 | case X86::VXORPSZ128rr: | |||
7176 | case X86::VXORPDZ256rr: | |||
7177 | case X86::VXORPSZ256rr: | |||
7178 | case X86::VXORPDZrr: | |||
7179 | case X86::VXORPSZrr: | |||
7180 | case X86::KADDBrr: | |||
7181 | case X86::KADDWrr: | |||
7182 | case X86::KADDDrr: | |||
7183 | case X86::KADDQrr: | |||
7184 | case X86::KANDBrr: | |||
7185 | case X86::KANDWrr: | |||
7186 | case X86::KANDDrr: | |||
7187 | case X86::KANDQrr: | |||
7188 | case X86::KORBrr: | |||
7189 | case X86::KORWrr: | |||
7190 | case X86::KORDrr: | |||
7191 | case X86::KORQrr: | |||
7192 | case X86::KXORBrr: | |||
7193 | case X86::KXORWrr: | |||
7194 | case X86::KXORDrr: | |||
7195 | case X86::KXORQrr: | |||
7196 | case X86::VPADDBrr: | |||
7197 | case X86::VPADDWrr: | |||
7198 | case X86::VPADDDrr: | |||
7199 | case X86::VPADDQrr: | |||
7200 | case X86::VPADDBYrr: | |||
7201 | case X86::VPADDWYrr: | |||
7202 | case X86::VPADDDYrr: | |||
7203 | case X86::VPADDQYrr: | |||
7204 | case X86::VPADDBZ128rr: | |||
7205 | case X86::VPADDWZ128rr: | |||
7206 | case X86::VPADDDZ128rr: | |||
7207 | case X86::VPADDQZ128rr: | |||
7208 | case X86::VPADDBZ256rr: | |||
7209 | case X86::VPADDWZ256rr: | |||
7210 | case X86::VPADDDZ256rr: | |||
7211 | case X86::VPADDQZ256rr: | |||
7212 | case X86::VPADDBZrr: | |||
7213 | case X86::VPADDWZrr: | |||
7214 | case X86::VPADDDZrr: | |||
7215 | case X86::VPADDQZrr: | |||
7216 | case X86::VPMULLWrr: | |||
7217 | case X86::VPMULLWYrr: | |||
7218 | case X86::VPMULLWZ128rr: | |||
7219 | case X86::VPMULLWZ256rr: | |||
7220 | case X86::VPMULLWZrr: | |||
7221 | case X86::VPMULLDrr: | |||
7222 | case X86::VPMULLDYrr: | |||
7223 | case X86::VPMULLDZ128rr: | |||
7224 | case X86::VPMULLDZ256rr: | |||
7225 | case X86::VPMULLDZrr: | |||
7226 | case X86::VPMULLQZ128rr: | |||
7227 | case X86::VPMULLQZ256rr: | |||
7228 | case X86::VPMULLQZrr: | |||
7229 | // Normal min/max instructions are not commutative because of NaN and signed | |||
7230 | // zero semantics, but these are. Thus, there's no need to check for global | |||
7231 | // relaxed math; the instructions themselves have the properties we need. | |||
7232 | case X86::MAXCPDrr: | |||
7233 | case X86::MAXCPSrr: | |||
7234 | case X86::MAXCSDrr: | |||
7235 | case X86::MAXCSSrr: | |||
7236 | case X86::MINCPDrr: | |||
7237 | case X86::MINCPSrr: | |||
7238 | case X86::MINCSDrr: | |||
7239 | case X86::MINCSSrr: | |||
7240 | case X86::VMAXCPDrr: | |||
7241 | case X86::VMAXCPSrr: | |||
7242 | case X86::VMAXCPDYrr: | |||
7243 | case X86::VMAXCPSYrr: | |||
7244 | case X86::VMAXCPDZ128rr: | |||
7245 | case X86::VMAXCPSZ128rr: | |||
7246 | case X86::VMAXCPDZ256rr: | |||
7247 | case X86::VMAXCPSZ256rr: | |||
7248 | case X86::VMAXCPDZrr: | |||
7249 | case X86::VMAXCPSZrr: | |||
7250 | case X86::VMAXCSDrr: | |||
7251 | case X86::VMAXCSSrr: | |||
7252 | case X86::VMAXCSDZrr: | |||
7253 | case X86::VMAXCSSZrr: | |||
7254 | case X86::VMINCPDrr: | |||
7255 | case X86::VMINCPSrr: | |||
7256 | case X86::VMINCPDYrr: | |||
7257 | case X86::VMINCPSYrr: | |||
7258 | case X86::VMINCPDZ128rr: | |||
7259 | case X86::VMINCPSZ128rr: | |||
7260 | case X86::VMINCPDZ256rr: | |||
7261 | case X86::VMINCPSZ256rr: | |||
7262 | case X86::VMINCPDZrr: | |||
7263 | case X86::VMINCPSZrr: | |||
7264 | case X86::VMINCSDrr: | |||
7265 | case X86::VMINCSSrr: | |||
7266 | case X86::VMINCSDZrr: | |||
7267 | case X86::VMINCSSZrr: | |||
7268 | return true; | |||
7269 | case X86::ADDPDrr: | |||
7270 | case X86::ADDPSrr: | |||
7271 | case X86::ADDSDrr: | |||
7272 | case X86::ADDSSrr: | |||
7273 | case X86::MULPDrr: | |||
7274 | case X86::MULPSrr: | |||
7275 | case X86::MULSDrr: | |||
7276 | case X86::MULSSrr: | |||
7277 | case X86::VADDPDrr: | |||
7278 | case X86::VADDPSrr: | |||
7279 | case X86::VADDPDYrr: | |||
7280 | case X86::VADDPSYrr: | |||
7281 | case X86::VADDPDZ128rr: | |||
7282 | case X86::VADDPSZ128rr: | |||
7283 | case X86::VADDPDZ256rr: | |||
7284 | case X86::VADDPSZ256rr: | |||
7285 | case X86::VADDPDZrr: | |||
7286 | case X86::VADDPSZrr: | |||
7287 | case X86::VADDSDrr: | |||
7288 | case X86::VADDSSrr: | |||
7289 | case X86::VADDSDZrr: | |||
7290 | case X86::VADDSSZrr: | |||
7291 | case X86::VMULPDrr: | |||
7292 | case X86::VMULPSrr: | |||
7293 | case X86::VMULPDYrr: | |||
7294 | case X86::VMULPSYrr: | |||
7295 | case X86::VMULPDZ128rr: | |||
7296 | case X86::VMULPSZ128rr: | |||
7297 | case X86::VMULPDZ256rr: | |||
7298 | case X86::VMULPSZ256rr: | |||
7299 | case X86::VMULPDZrr: | |||
7300 | case X86::VMULPSZrr: | |||
7301 | case X86::VMULSDrr: | |||
7302 | case X86::VMULSSrr: | |||
7303 | case X86::VMULSDZrr: | |||
7304 | case X86::VMULSSZrr: | |||
7305 | return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath; | |||
7306 | default: | |||
7307 | return false; | |||
7308 | } | |||
7309 | } | |||
7310 | ||||
7311 | /// This is an architecture-specific helper function of reassociateOps. | |||
7312 | /// Set special operand attributes for new instructions after reassociation. | |||
7313 | void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, | |||
7314 | MachineInstr &OldMI2, | |||
7315 | MachineInstr &NewMI1, | |||
7316 | MachineInstr &NewMI2) const { | |||
7317 | // Integer instructions define an implicit EFLAGS source register operand as | |||
7318 | // the third source (fourth total) operand. | |||
7319 | if (OldMI1.getNumOperands() != 4 || OldMI2.getNumOperands() != 4) | |||
7320 | return; | |||
7321 | ||||
7322 | assert(NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 &&((NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands () == 4 && "Unexpected instruction type for reassociation" ) ? static_cast<void> (0) : __assert_fail ("NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 && \"Unexpected instruction type for reassociation\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7323, __PRETTY_FUNCTION__)) | |||
7323 | "Unexpected instruction type for reassociation")((NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands () == 4 && "Unexpected instruction type for reassociation" ) ? static_cast<void> (0) : __assert_fail ("NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 && \"Unexpected instruction type for reassociation\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7323, __PRETTY_FUNCTION__)); | |||
7324 | ||||
7325 | MachineOperand &OldOp1 = OldMI1.getOperand(3); | |||
7326 | MachineOperand &OldOp2 = OldMI2.getOperand(3); | |||
7327 | MachineOperand &NewOp1 = NewMI1.getOperand(3); | |||
7328 | MachineOperand &NewOp2 = NewMI2.getOperand(3); | |||
7329 | ||||
7330 | assert(OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() &&((OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7331, __PRETTY_FUNCTION__)) | |||
7331 | "Must have dead EFLAGS operand in reassociable instruction")((OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7331, __PRETTY_FUNCTION__)); | |||
7332 | assert(OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() &&((OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7333, __PRETTY_FUNCTION__)) | |||
7333 | "Must have dead EFLAGS operand in reassociable instruction")((OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && "Must have dead EFLAGS operand in reassociable instruction" ) ? static_cast<void> (0) : __assert_fail ("OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && \"Must have dead EFLAGS operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7333, __PRETTY_FUNCTION__)); | |||
7334 | ||||
7335 | (void)OldOp1; | |||
7336 | (void)OldOp2; | |||
7337 | ||||
7338 | assert(NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS &&((NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7339, __PRETTY_FUNCTION__)) | |||
7339 | "Unexpected operand in reassociable instruction")((NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7339, __PRETTY_FUNCTION__)); | |||
7340 | assert(NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS &&((NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7341, __PRETTY_FUNCTION__)) | |||
7341 | "Unexpected operand in reassociable instruction")((NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && "Unexpected operand in reassociable instruction") ? static_cast <void> (0) : __assert_fail ("NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && \"Unexpected operand in reassociable instruction\"" , "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7341, __PRETTY_FUNCTION__)); | |||
7342 | ||||
7343 | // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations | |||
7344 | // of this pass or other passes. The EFLAGS operands must be dead in these new | |||
7345 | // instructions because the EFLAGS operands in the original instructions must | |||
7346 | // be dead in order for reassociation to occur. | |||
7347 | NewOp1.setIsDead(); | |||
7348 | NewOp2.setIsDead(); | |||
7349 | } | |||
7350 | ||||
7351 | std::pair<unsigned, unsigned> | |||
7352 | X86InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { | |||
7353 | return std::make_pair(TF, 0u); | |||
7354 | } | |||
7355 | ||||
7356 | ArrayRef<std::pair<unsigned, const char *>> | |||
7357 | X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const { | |||
7358 | using namespace X86II; | |||
7359 | static const std::pair<unsigned, const char *> TargetFlags[] = { | |||
7360 | {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"}, | |||
7361 | {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"}, | |||
7362 | {MO_GOT, "x86-got"}, | |||
7363 | {MO_GOTOFF, "x86-gotoff"}, | |||
7364 | {MO_GOTPCREL, "x86-gotpcrel"}, | |||
7365 | {MO_PLT, "x86-plt"}, | |||
7366 | {MO_TLSGD, "x86-tlsgd"}, | |||
7367 | {MO_TLSLD, "x86-tlsld"}, | |||
7368 | {MO_TLSLDM, "x86-tlsldm"}, | |||
7369 | {MO_GOTTPOFF, "x86-gottpoff"}, | |||
7370 | {MO_INDNTPOFF, "x86-indntpoff"}, | |||
7371 | {MO_TPOFF, "x86-tpoff"}, | |||
7372 | {MO_DTPOFF, "x86-dtpoff"}, | |||
7373 | {MO_NTPOFF, "x86-ntpoff"}, | |||
7374 | {MO_GOTNTPOFF, "x86-gotntpoff"}, | |||
7375 | {MO_DLLIMPORT, "x86-dllimport"}, | |||
7376 | {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"}, | |||
7377 | {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"}, | |||
7378 | {MO_TLVP, "x86-tlvp"}, | |||
7379 | {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"}, | |||
7380 | {MO_SECREL, "x86-secrel"}, | |||
7381 | {MO_COFFSTUB, "x86-coffstub"}}; | |||
7382 | return makeArrayRef(TargetFlags); | |||
7383 | } | |||
7384 | ||||
7385 | namespace { | |||
7386 | /// Create Global Base Reg pass. This initializes the PIC | |||
7387 | /// global base register for x86-32. | |||
7388 | struct CGBR : public MachineFunctionPass { | |||
7389 | static char ID; | |||
7390 | CGBR() : MachineFunctionPass(ID) {} | |||
7391 | ||||
7392 | bool runOnMachineFunction(MachineFunction &MF) override { | |||
7393 | const X86TargetMachine *TM = | |||
7394 | static_cast<const X86TargetMachine *>(&MF.getTarget()); | |||
7395 | const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); | |||
7396 | ||||
7397 | // Don't do anything in the 64-bit small and kernel code models. They use | |||
7398 | // RIP-relative addressing for everything. | |||
7399 | if (STI.is64Bit() && (TM->getCodeModel() == CodeModel::Small || | |||
7400 | TM->getCodeModel() == CodeModel::Kernel)) | |||
7401 | return false; | |||
7402 | ||||
7403 | // Only emit a global base reg in PIC mode. | |||
7404 | if (!TM->isPositionIndependent()) | |||
7405 | return false; | |||
7406 | ||||
7407 | X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); | |||
7408 | unsigned GlobalBaseReg = X86FI->getGlobalBaseReg(); | |||
7409 | ||||
7410 | // If we didn't need a GlobalBaseReg, don't insert code. | |||
7411 | if (GlobalBaseReg == 0) | |||
7412 | return false; | |||
7413 | ||||
7414 | // Insert the set of GlobalBaseReg into the first MBB of the function | |||
7415 | MachineBasicBlock &FirstMBB = MF.front(); | |||
7416 | MachineBasicBlock::iterator MBBI = FirstMBB.begin(); | |||
7417 | DebugLoc DL = FirstMBB.findDebugLoc(MBBI); | |||
7418 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | |||
7419 | const X86InstrInfo *TII = STI.getInstrInfo(); | |||
7420 | ||||
7421 | unsigned PC; | |||
7422 | if (STI.isPICStyleGOT()) | |||
7423 | PC = RegInfo.createVirtualRegister(&X86::GR32RegClass); | |||
7424 | else | |||
7425 | PC = GlobalBaseReg; | |||
7426 | ||||
7427 | if (STI.is64Bit()) { | |||
7428 | if (TM->getCodeModel() == CodeModel::Medium) { | |||
7429 | // In the medium code model, use a RIP-relative LEA to materialize the | |||
7430 | // GOT. | |||
7431 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) | |||
7432 | .addReg(X86::RIP) | |||
7433 | .addImm(0) | |||
7434 | .addReg(0) | |||
7435 | .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") | |||
7436 | .addReg(0); | |||
7437 | } else if (TM->getCodeModel() == CodeModel::Large) { | |||
7438 | // In the large code model, we are aiming for this code, though the | |||
7439 | // register allocation may vary: | |||
7440 | // leaq .LN$pb(%rip), %rax | |||
7441 | // movq $_GLOBAL_OFFSET_TABLE_ - .LN$pb, %rcx | |||
7442 | // addq %rcx, %rax | |||
7443 | // RAX now holds address of _GLOBAL_OFFSET_TABLE_. | |||
7444 | unsigned PBReg = RegInfo.createVirtualRegister(&X86::GR64RegClass); | |||
7445 | unsigned GOTReg = | |||
7446 | RegInfo.createVirtualRegister(&X86::GR64RegClass); | |||
7447 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PBReg) | |||
7448 | .addReg(X86::RIP) | |||
7449 | .addImm(0) | |||
7450 | .addReg(0) | |||
7451 | .addSym(MF.getPICBaseSymbol()) | |||
7452 | .addReg(0); | |||
7453 | std::prev(MBBI)->setPreInstrSymbol(MF, MF.getPICBaseSymbol()); | |||
7454 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOV64ri), GOTReg) | |||
7455 | .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", | |||
7456 | X86II::MO_PIC_BASE_OFFSET); | |||
7457 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD64rr), PC) | |||
7458 | .addReg(PBReg, RegState::Kill) | |||
7459 | .addReg(GOTReg, RegState::Kill); | |||
7460 | } else { | |||
7461 | llvm_unreachable("unexpected code model")::llvm::llvm_unreachable_internal("unexpected code model", "/build/llvm-toolchain-snapshot-8~svn349319/lib/Target/X86/X86InstrInfo.cpp" , 7461); | |||
7462 | } | |||
7463 | } else { | |||
7464 | // Operand of MovePCtoStack is completely ignored by asm printer. It's | |||
7465 | // only used in JIT code emission as displacement to pc. | |||
7466 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0); | |||
7467 | ||||
7468 | // If we're using vanilla 'GOT' PIC style, we should use relative | |||
7469 | // addressing not to pc, but to _GLOBAL_OFFSET_TABLE_ external. | |||
7470 | if (STI.isPICStyleGOT()) { | |||
7471 | // Generate addl $__GLOBAL_OFFSET_TABLE_ + [.-piclabel], | |||
7472 | // %some_register | |||
7473 | BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg) | |||
7474 | .addReg(PC) | |||
7475 | .addExternalSymbol("_GLOBAL_OFFSET_TABLE_", | |||
7476 | X86II::MO_GOT_ABSOLUTE_ADDRESS); | |||
7477 | } | |||
7478 | } | |||
7479 | ||||
7480 | return true; | |||
7481 | } | |||
7482 | ||||
7483 | StringRef getPassName() const override { | |||
7484 | return "X86 PIC Global Base Reg Initialization"; | |||
7485 | } | |||
7486 | ||||
7487 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
7488 | AU.setPreservesCFG(); | |||
7489 | MachineFunctionPass::getAnalysisUsage(AU); | |||
7490 | } | |||
7491 | }; | |||
7492 | } | |||
7493 | ||||
7494 | char CGBR::ID = 0; | |||
7495 | FunctionPass* | |||
7496 | llvm::createX86GlobalBaseRegPass() { return new CGBR(); } | |||
7497 | ||||
7498 | namespace { | |||
7499 | struct LDTLSCleanup : public MachineFunctionPass { | |||
7500 | static char ID; | |||
7501 | LDTLSCleanup() : MachineFunctionPass(ID) {} | |||
7502 | ||||
7503 | bool runOnMachineFunction(MachineFunction &MF) override { | |||
7504 | if (skipFunction(MF.getFunction())) | |||
7505 | return false; | |||
7506 | ||||
7507 | X86MachineFunctionInfo *MFI = MF.getInfo<X86MachineFunctionInfo>(); | |||
7508 | if (MFI->getNumLocalDynamicTLSAccesses() < 2) { | |||
7509 | // No point folding accesses if there isn't at least two. | |||
7510 | return false; | |||
7511 | } | |||
7512 | ||||
7513 | MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>(); | |||
7514 | return VisitNode(DT->getRootNode(), 0); | |||
7515 | } | |||
7516 | ||||
7517 | // Visit the dominator subtree rooted at Node in pre-order. | |||
7518 | // If TLSBaseAddrReg is non-null, then use that to replace any | |||
7519 | // TLS_base_addr instructions. Otherwise, create the register | |||
7520 | // when the first such instruction is seen, and then use it | |||
7521 | // as we encounter more instructions. | |||
7522 | bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) { | |||
7523 | MachineBasicBlock *BB = Node->getBlock(); | |||
7524 | bool Changed = false; | |||
7525 | ||||
7526 | // Traverse the current block. | |||
7527 | for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; | |||
7528 | ++I) { | |||
7529 | switch (I->getOpcode()) { | |||
7530 | case X86::TLS_base_addr32: | |||
7531 | case X86::TLS_base_addr64: | |||
7532 | if (TLSBaseAddrReg) | |||
7533 | I = ReplaceTLSBaseAddrCall(*I, TLSBaseAddrReg); | |||
7534 | else | |||
7535 | I = SetRegister(*I, &TLSBaseAddrReg); | |||
7536 | Changed = true; | |||
7537 | break; | |||
7538 | default: | |||
7539 | break; | |||
7540 | } | |||
7541 | } | |||
7542 | ||||
7543 | // Visit the children of this block in the dominator tree. | |||
7544 | for (MachineDomTreeNode::iterator I = Node->begin(), E = Node->end(); | |||
7545 | I != E; ++I) { | |||
7546 | Changed |= VisitNode(*I, TLSBaseAddrReg); | |||
7547 | } | |||
7548 | ||||
7549 | return Changed; | |||
7550 | } | |||
7551 | ||||
7552 | // Replace the TLS_base_addr instruction I with a copy from | |||
7553 | // TLSBaseAddrReg, returning the new instruction. | |||
7554 | MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr &I, | |||
7555 | unsigned TLSBaseAddrReg) { | |||
7556 | MachineFunction *MF = I.getParent()->getParent(); | |||
7557 | const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); | |||
7558 | const bool is64Bit = STI.is64Bit(); | |||
7559 | const X86InstrInfo *TII = STI.getInstrInfo(); | |||
7560 | ||||
7561 | // Insert a Copy from TLSBaseAddrReg to RAX/EAX. | |||
7562 | MachineInstr *Copy = | |||
7563 | BuildMI(*I.getParent(), I, I.getDebugLoc(), | |||
7564 | TII->get(TargetOpcode::COPY), is64Bit ? X86::RAX : X86::EAX) | |||
7565 | .addReg(TLSBaseAddrReg); | |||
7566 | ||||
7567 | // Erase the TLS_base_addr instruction. | |||
7568 | I.eraseFromParent(); | |||
7569 | ||||
7570 | return Copy; | |||
7571 | } | |||
7572 | ||||
7573 | // Create a virtual register in *TLSBaseAddrReg, and populate it by | |||
7574 | // inserting a copy instruction after I. Returns the new instruction. | |||
7575 | MachineInstr *SetRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { | |||
7576 | MachineFunction *MF = I.getParent()->getParent(); | |||
7577 | const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); | |||
7578 | const bool is64Bit = STI.is64Bit(); | |||
7579 | const X86InstrInfo *TII = STI.getInstrInfo(); | |||
7580 | ||||
7581 | // Create a virtual register for the TLS base address. | |||
7582 | MachineRegisterInfo &RegInfo = MF->getRegInfo(); | |||
7583 | *TLSBaseAddrReg = RegInfo.createVirtualRegister(is64Bit | |||
7584 | ? &X86::GR64RegClass | |||
7585 | : &X86::GR32RegClass); | |||
7586 | ||||
7587 | // Insert a copy from RAX/EAX to TLSBaseAddrReg. | |||
7588 | MachineInstr *Next = I.getNextNode(); | |||
7589 | MachineInstr *Copy = | |||
7590 | BuildMI(*I.getParent(), Next, I.getDebugLoc(), | |||
7591 | TII->get(TargetOpcode::COPY), *TLSBaseAddrReg) | |||
7592 | .addReg(is64Bit ? X86::RAX : X86::EAX); | |||
7593 | ||||
7594 | return Copy; | |||
7595 | } | |||
7596 | ||||
7597 | StringRef getPassName() const override { | |||
7598 | return "Local Dynamic TLS Access Clean-up"; | |||
7599 | } | |||
7600 | ||||
7601 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
7602 | AU.setPreservesCFG(); | |||
7603 | AU.addRequired<MachineDominatorTree>(); | |||
7604 | MachineFunctionPass::getAnalysisUsage(AU); | |||
7605 | } | |||
7606 | }; | |||
7607 | } | |||
7608 | ||||
7609 | char LDTLSCleanup::ID = 0; | |||
7610 | FunctionPass* | |||
7611 | llvm::createCleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); } | |||
7612 | ||||
7613 | /// Constants defining how certain sequences should be outlined. | |||
7614 | /// | |||
7615 | /// \p MachineOutlinerDefault implies that the function is called with a call | |||
7616 | /// instruction, and a return must be emitted for the outlined function frame. | |||
7617 | /// | |||
7618 | /// That is, | |||
7619 | /// | |||
7620 | /// I1 OUTLINED_FUNCTION: | |||
7621 | /// I2 --> call OUTLINED_FUNCTION I1 | |||
7622 | /// I3 I2 | |||
7623 | /// I3 | |||
7624 | /// ret | |||
7625 | /// | |||
7626 | /// * Call construction overhead: 1 (call instruction) | |||
7627 | /// * Frame construction overhead: 1 (return instruction) | |||
7628 | /// | |||
7629 | /// \p MachineOutlinerTailCall implies that the function is being tail called. | |||
7630 | /// A jump is emitted instead of a call, and the return is already present in | |||
7631 | /// the outlined sequence. That is, | |||
7632 | /// | |||
7633 | /// I1 OUTLINED_FUNCTION: | |||
7634 | /// I2 --> jmp OUTLINED_FUNCTION I1 | |||
7635 | /// ret I2 | |||
7636 | /// ret | |||
7637 | /// | |||
7638 | /// * Call construction overhead: 1 (jump instruction) | |||
7639 | /// * Frame construction overhead: 0 (don't need to return) | |||
7640 | /// | |||
7641 | enum MachineOutlinerClass { | |||
7642 | MachineOutlinerDefault, | |||
7643 | MachineOutlinerTailCall | |||
7644 | }; | |||
7645 | ||||
7646 | outliner::OutlinedFunction X86InstrInfo::getOutliningCandidateInfo( | |||
7647 | std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { | |||
7648 | unsigned SequenceSize = | |||
7649 | std::accumulate(RepeatedSequenceLocs[0].front(), | |||
7650 | std::next(RepeatedSequenceLocs[0].back()), 0, | |||
7651 | [](unsigned Sum, const MachineInstr &MI) { | |||
7652 | // FIXME: x86 doesn't implement getInstSizeInBytes, so | |||
7653 | // we can't tell the cost. Just assume each instruction | |||
7654 | // is one byte. | |||
7655 | if (MI.isDebugInstr() || MI.isKill()) | |||
7656 | return Sum; | |||
7657 | return Sum + 1; | |||
7658 | }); | |||
7659 | ||||
7660 | // FIXME: Use real size in bytes for call and ret instructions. | |||
7661 | if (RepeatedSequenceLocs[0].back()->isTerminator()) { | |||
7662 | for (outliner::Candidate &C : RepeatedSequenceLocs) | |||
7663 | C.setCallInfo(MachineOutlinerTailCall, 1); | |||
7664 | ||||
7665 | return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, | |||
7666 | 0, // Number of bytes to emit frame. | |||
7667 | MachineOutlinerTailCall // Type of frame. | |||
7668 | ); | |||
7669 | } | |||
7670 | ||||
7671 | for (outliner::Candidate &C : RepeatedSequenceLocs) | |||
7672 | C.setCallInfo(MachineOutlinerDefault, 1); | |||
7673 | ||||
7674 | return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1, | |||
7675 | MachineOutlinerDefault); | |||
7676 | } | |||
7677 | ||||
7678 | bool X86InstrInfo::isFunctionSafeToOutlineFrom(MachineFunction &MF, | |||
7679 | bool OutlineFromLinkOnceODRs) const { | |||
7680 | const Function &F = MF.getFunction(); | |||
7681 | ||||
7682 | // Does the function use a red zone? If it does, then we can't risk messing | |||
7683 | // with the stack. | |||
7684 | if (!F.hasFnAttribute(Attribute::NoRedZone)) { | |||
7685 | // It could have a red zone. If it does, then we don't want to touch it. | |||
7686 | const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); | |||
7687 | if (!X86FI || X86FI->getUsesRedZone()) | |||
7688 | return false; | |||
7689 | } | |||
7690 | ||||
7691 | // If we *don't* want to outline from things that could potentially be deduped | |||
7692 | // then return false. | |||
7693 | if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) | |||
7694 | return false; | |||
7695 | ||||
7696 | // This function is viable for outlining, so return true. | |||
7697 | return true; | |||
7698 | } | |||
7699 | ||||
7700 | outliner::InstrType | |||
7701 | X86InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, unsigned Flags) const { | |||
7702 | MachineInstr &MI = *MIT; | |||
7703 | // Don't allow debug values to impact outlining type. | |||
7704 | if (MI.isDebugInstr() || MI.isIndirectDebugValue()) | |||
7705 | return outliner::InstrType::Invisible; | |||
7706 | ||||
7707 | // At this point, KILL instructions don't really tell us much so we can go | |||
7708 | // ahead and skip over them. | |||
7709 | if (MI.isKill()) | |||
7710 | return outliner::InstrType::Invisible; | |||
7711 | ||||
7712 | // Is this a tail call? If yes, we can outline as a tail call. | |||
7713 | if (isTailCall(MI)) | |||
7714 | return outliner::InstrType::Legal; | |||
7715 | ||||
7716 | // Is this the terminator of a basic block? | |||
7717 | if (MI.isTerminator() || MI.isReturn()) { | |||
7718 | ||||
7719 | // Does its parent have any successors in its MachineFunction? | |||
7720 | if (MI.getParent()->succ_empty()) | |||
7721 | return outliner::InstrType::Legal; | |||
7722 | ||||
7723 | // It does, so we can't tail call it. | |||
7724 | return outliner::InstrType::Illegal; | |||
7725 | } | |||
7726 | ||||
7727 | // Don't outline anything that modifies or reads from the stack pointer. | |||
7728 | // | |||
7729 | // FIXME: There are instructions which are being manually built without | |||
7730 | // explicit uses/defs so we also have to check the MCInstrDesc. We should be | |||
7731 | // able to remove the extra checks once those are fixed up. For example, | |||
7732 | // sometimes we might get something like %rax = POP64r 1. This won't be | |||
7733 | // caught by modifiesRegister or readsRegister even though the instruction | |||
7734 | // really ought to be formed so that modifiesRegister/readsRegister would | |||
7735 | // catch it. | |||
7736 | if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) || | |||
7737 | MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) || | |||
7738 | MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP)) | |||
7739 | return outliner::InstrType::Illegal; | |||
7740 | ||||
7741 | // Outlined calls change the instruction pointer, so don't read from it. | |||
7742 | if (MI.readsRegister(X86::RIP, &RI) || | |||
7743 | MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) || | |||
7744 | MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP)) | |||
7745 | return outliner::InstrType::Illegal; | |||
7746 | ||||
7747 | // Positions can't safely be outlined. | |||
7748 | if (MI.isPosition()) | |||
7749 | return outliner::InstrType::Illegal; | |||
7750 | ||||
7751 | // Make sure none of the operands of this instruction do anything tricky. | |||
7752 | for (const MachineOperand &MOP : MI.operands()) | |||
7753 | if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() || | |||
7754 | MOP.isTargetIndex()) | |||
7755 | return outliner::InstrType::Illegal; | |||
7756 | ||||
7757 | return outliner::InstrType::Legal; | |||
7758 | } | |||
7759 | ||||
7760 | void X86InstrInfo::buildOutlinedFrame(MachineBasicBlock &MBB, | |||
7761 | MachineFunction &MF, | |||
7762 | const outliner::OutlinedFunction &OF) | |||
7763 | const { | |||
7764 | // If we're a tail call, we already have a return, so don't do anything. | |||
7765 | if (OF.FrameConstructionID == MachineOutlinerTailCall) | |||
7766 | return; | |||
7767 | ||||
7768 | // We're a normal call, so our sequence doesn't have a return instruction. | |||
7769 | // Add it in. | |||
7770 | MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RETQ)); | |||
7771 | MBB.insert(MBB.end(), retq); | |||
7772 | } | |||
7773 | ||||
7774 | MachineBasicBlock::iterator | |||
7775 | X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB, | |||
7776 | MachineBasicBlock::iterator &It, | |||
7777 | MachineFunction &MF, | |||
7778 | const outliner::Candidate &C) const { | |||
7779 | // Is it a tail call? | |||
7780 | if (C.CallConstructionID == MachineOutlinerTailCall) { | |||
7781 | // Yes, just insert a JMP. | |||
7782 | It = MBB.insert(It, | |||
7783 | BuildMI(MF, DebugLoc(), get(X86::TAILJMPd64)) | |||
7784 | .addGlobalAddress(M.getNamedValue(MF.getName()))); | |||
7785 | } else { | |||
7786 | // No, insert a call. | |||
7787 | It = MBB.insert(It, | |||
7788 | BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32)) | |||
7789 | .addGlobalAddress(M.getNamedValue(MF.getName()))); | |||
7790 | } | |||
7791 | ||||
7792 | return It; | |||
7793 | } | |||
7794 | ||||
7795 | #define GET_INSTRINFO_HELPERS | |||
7796 | #include "X86GenInstrInfo.inc" |