File: | llvm/lib/Target/X86/X86MCInstLower.cpp |
Warning: | line 634, column 7 1st function call argument is an uninitialized value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file contains code to lower X86 MachineInstrs to their corresponding | |||
10 | // MCInst records. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "MCTargetDesc/X86ATTInstPrinter.h" | |||
15 | #include "MCTargetDesc/X86BaseInfo.h" | |||
16 | #include "MCTargetDesc/X86InstComments.h" | |||
17 | #include "MCTargetDesc/X86TargetStreamer.h" | |||
18 | #include "Utils/X86ShuffleDecode.h" | |||
19 | #include "X86AsmPrinter.h" | |||
20 | #include "X86RegisterInfo.h" | |||
21 | #include "X86ShuffleDecodeConstantPool.h" | |||
22 | #include "llvm/ADT/Optional.h" | |||
23 | #include "llvm/ADT/SmallString.h" | |||
24 | #include "llvm/ADT/iterator_range.h" | |||
25 | #include "llvm/CodeGen/MachineConstantPool.h" | |||
26 | #include "llvm/CodeGen/MachineFunction.h" | |||
27 | #include "llvm/CodeGen/MachineModuleInfoImpls.h" | |||
28 | #include "llvm/CodeGen/MachineOperand.h" | |||
29 | #include "llvm/CodeGen/StackMaps.h" | |||
30 | #include "llvm/IR/DataLayout.h" | |||
31 | #include "llvm/IR/GlobalValue.h" | |||
32 | #include "llvm/IR/Mangler.h" | |||
33 | #include "llvm/MC/MCAsmInfo.h" | |||
34 | #include "llvm/MC/MCCodeEmitter.h" | |||
35 | #include "llvm/MC/MCContext.h" | |||
36 | #include "llvm/MC/MCExpr.h" | |||
37 | #include "llvm/MC/MCFixup.h" | |||
38 | #include "llvm/MC/MCInst.h" | |||
39 | #include "llvm/MC/MCInstBuilder.h" | |||
40 | #include "llvm/MC/MCSection.h" | |||
41 | #include "llvm/MC/MCSectionELF.h" | |||
42 | #include "llvm/MC/MCStreamer.h" | |||
43 | #include "llvm/MC/MCSymbol.h" | |||
44 | #include "llvm/MC/MCSymbolELF.h" | |||
45 | #include "llvm/Target/TargetLoweringObjectFile.h" | |||
46 | ||||
47 | using namespace llvm; | |||
48 | ||||
49 | namespace { | |||
50 | ||||
51 | /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst. | |||
52 | class X86MCInstLower { | |||
53 | MCContext &Ctx; | |||
54 | const MachineFunction &MF; | |||
55 | const TargetMachine &TM; | |||
56 | const MCAsmInfo &MAI; | |||
57 | X86AsmPrinter &AsmPrinter; | |||
58 | ||||
59 | public: | |||
60 | X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter); | |||
61 | ||||
62 | Optional<MCOperand> LowerMachineOperand(const MachineInstr *MI, | |||
63 | const MachineOperand &MO) const; | |||
64 | void Lower(const MachineInstr *MI, MCInst &OutMI) const; | |||
65 | ||||
66 | MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const; | |||
67 | MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const; | |||
68 | ||||
69 | private: | |||
70 | MachineModuleInfoMachO &getMachOMMI() const; | |||
71 | }; | |||
72 | ||||
73 | } // end anonymous namespace | |||
74 | ||||
75 | // Emit a minimal sequence of nops spanning NumBytes bytes. | |||
76 | static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, | |||
77 | const MCSubtargetInfo &STI); | |||
78 | ||||
79 | void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst, | |||
80 | const MCSubtargetInfo &STI, | |||
81 | MCCodeEmitter *CodeEmitter) { | |||
82 | if (InShadow) { | |||
83 | SmallString<256> Code; | |||
84 | SmallVector<MCFixup, 4> Fixups; | |||
85 | raw_svector_ostream VecOS(Code); | |||
86 | CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI); | |||
87 | CurrentShadowSize += Code.size(); | |||
88 | if (CurrentShadowSize >= RequiredShadowSize) | |||
89 | InShadow = false; // The shadow is big enough. Stop counting. | |||
90 | } | |||
91 | } | |||
92 | ||||
93 | void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding( | |||
94 | MCStreamer &OutStreamer, const MCSubtargetInfo &STI) { | |||
95 | if (InShadow && CurrentShadowSize < RequiredShadowSize) { | |||
96 | InShadow = false; | |||
97 | EmitNops(OutStreamer, RequiredShadowSize - CurrentShadowSize, | |||
98 | MF->getSubtarget<X86Subtarget>().is64Bit(), STI); | |||
99 | } | |||
100 | } | |||
101 | ||||
102 | void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) { | |||
103 | OutStreamer->EmitInstruction(Inst, getSubtargetInfo()); | |||
104 | SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get()); | |||
105 | } | |||
106 | ||||
107 | X86MCInstLower::X86MCInstLower(const MachineFunction &mf, | |||
108 | X86AsmPrinter &asmprinter) | |||
109 | : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()), | |||
110 | AsmPrinter(asmprinter) {} | |||
111 | ||||
112 | MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const { | |||
113 | return MF.getMMI().getObjFileInfo<MachineModuleInfoMachO>(); | |||
114 | } | |||
115 | ||||
116 | /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol | |||
117 | /// operand to an MCSymbol. | |||
118 | MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const { | |||
119 | const DataLayout &DL = MF.getDataLayout(); | |||
120 | assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&(((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && "Isn't a symbol reference" ) ? static_cast<void> (0) : __assert_fail ("(MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && \"Isn't a symbol reference\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 121, __PRETTY_FUNCTION__)) | |||
121 | "Isn't a symbol reference")(((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && "Isn't a symbol reference" ) ? static_cast<void> (0) : __assert_fail ("(MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && \"Isn't a symbol reference\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 121, __PRETTY_FUNCTION__)); | |||
122 | ||||
123 | MCSymbol *Sym = nullptr; | |||
124 | SmallString<128> Name; | |||
125 | StringRef Suffix; | |||
126 | ||||
127 | switch (MO.getTargetFlags()) { | |||
128 | case X86II::MO_DLLIMPORT: | |||
129 | // Handle dllimport linkage. | |||
130 | Name += "__imp_"; | |||
131 | break; | |||
132 | case X86II::MO_COFFSTUB: | |||
133 | Name += ".refptr."; | |||
134 | break; | |||
135 | case X86II::MO_DARWIN_NONLAZY: | |||
136 | case X86II::MO_DARWIN_NONLAZY_PIC_BASE: | |||
137 | Suffix = "$non_lazy_ptr"; | |||
138 | break; | |||
139 | } | |||
140 | ||||
141 | if (!Suffix.empty()) | |||
142 | Name += DL.getPrivateGlobalPrefix(); | |||
143 | ||||
144 | if (MO.isGlobal()) { | |||
145 | const GlobalValue *GV = MO.getGlobal(); | |||
146 | AsmPrinter.getNameWithPrefix(Name, GV); | |||
147 | } else if (MO.isSymbol()) { | |||
148 | Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL); | |||
149 | } else if (MO.isMBB()) { | |||
150 | assert(Suffix.empty())((Suffix.empty()) ? static_cast<void> (0) : __assert_fail ("Suffix.empty()", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 150, __PRETTY_FUNCTION__)); | |||
151 | Sym = MO.getMBB()->getSymbol(); | |||
152 | } | |||
153 | ||||
154 | Name += Suffix; | |||
155 | if (!Sym) | |||
156 | Sym = Ctx.getOrCreateSymbol(Name); | |||
157 | ||||
158 | // If the target flags on the operand changes the name of the symbol, do that | |||
159 | // before we return the symbol. | |||
160 | switch (MO.getTargetFlags()) { | |||
161 | default: | |||
162 | break; | |||
163 | case X86II::MO_COFFSTUB: { | |||
164 | MachineModuleInfoCOFF &MMICOFF = | |||
165 | MF.getMMI().getObjFileInfo<MachineModuleInfoCOFF>(); | |||
166 | MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym); | |||
167 | if (!StubSym.getPointer()) { | |||
168 | assert(MO.isGlobal() && "Extern symbol not handled yet")((MO.isGlobal() && "Extern symbol not handled yet") ? static_cast<void> (0) : __assert_fail ("MO.isGlobal() && \"Extern symbol not handled yet\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 168, __PRETTY_FUNCTION__)); | |||
169 | StubSym = MachineModuleInfoImpl::StubValueTy( | |||
170 | AsmPrinter.getSymbol(MO.getGlobal()), true); | |||
171 | } | |||
172 | break; | |||
173 | } | |||
174 | case X86II::MO_DARWIN_NONLAZY: | |||
175 | case X86II::MO_DARWIN_NONLAZY_PIC_BASE: { | |||
176 | MachineModuleInfoImpl::StubValueTy &StubSym = | |||
177 | getMachOMMI().getGVStubEntry(Sym); | |||
178 | if (!StubSym.getPointer()) { | |||
179 | assert(MO.isGlobal() && "Extern symbol not handled yet")((MO.isGlobal() && "Extern symbol not handled yet") ? static_cast<void> (0) : __assert_fail ("MO.isGlobal() && \"Extern symbol not handled yet\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 179, __PRETTY_FUNCTION__)); | |||
180 | StubSym = MachineModuleInfoImpl::StubValueTy( | |||
181 | AsmPrinter.getSymbol(MO.getGlobal()), | |||
182 | !MO.getGlobal()->hasInternalLinkage()); | |||
183 | } | |||
184 | break; | |||
185 | } | |||
186 | } | |||
187 | ||||
188 | return Sym; | |||
189 | } | |||
190 | ||||
191 | MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO, | |||
192 | MCSymbol *Sym) const { | |||
193 | // FIXME: We would like an efficient form for this, so we don't have to do a | |||
194 | // lot of extra uniquing. | |||
195 | const MCExpr *Expr = nullptr; | |||
196 | MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None; | |||
197 | ||||
198 | switch (MO.getTargetFlags()) { | |||
199 | default: | |||
200 | llvm_unreachable("Unknown target flag on GV operand")::llvm::llvm_unreachable_internal("Unknown target flag on GV operand" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 200); | |||
201 | case X86II::MO_NO_FLAG: // No flag. | |||
202 | // These affect the name of the symbol, not any suffix. | |||
203 | case X86II::MO_DARWIN_NONLAZY: | |||
204 | case X86II::MO_DLLIMPORT: | |||
205 | case X86II::MO_COFFSTUB: | |||
206 | break; | |||
207 | ||||
208 | case X86II::MO_TLVP: | |||
209 | RefKind = MCSymbolRefExpr::VK_TLVP; | |||
210 | break; | |||
211 | case X86II::MO_TLVP_PIC_BASE: | |||
212 | Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx); | |||
213 | // Subtract the pic base. | |||
214 | Expr = MCBinaryExpr::createSub( | |||
215 | Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx); | |||
216 | break; | |||
217 | case X86II::MO_SECREL: | |||
218 | RefKind = MCSymbolRefExpr::VK_SECREL; | |||
219 | break; | |||
220 | case X86II::MO_TLSGD: | |||
221 | RefKind = MCSymbolRefExpr::VK_TLSGD; | |||
222 | break; | |||
223 | case X86II::MO_TLSLD: | |||
224 | RefKind = MCSymbolRefExpr::VK_TLSLD; | |||
225 | break; | |||
226 | case X86II::MO_TLSLDM: | |||
227 | RefKind = MCSymbolRefExpr::VK_TLSLDM; | |||
228 | break; | |||
229 | case X86II::MO_GOTTPOFF: | |||
230 | RefKind = MCSymbolRefExpr::VK_GOTTPOFF; | |||
231 | break; | |||
232 | case X86II::MO_INDNTPOFF: | |||
233 | RefKind = MCSymbolRefExpr::VK_INDNTPOFF; | |||
234 | break; | |||
235 | case X86II::MO_TPOFF: | |||
236 | RefKind = MCSymbolRefExpr::VK_TPOFF; | |||
237 | break; | |||
238 | case X86II::MO_DTPOFF: | |||
239 | RefKind = MCSymbolRefExpr::VK_DTPOFF; | |||
240 | break; | |||
241 | case X86II::MO_NTPOFF: | |||
242 | RefKind = MCSymbolRefExpr::VK_NTPOFF; | |||
243 | break; | |||
244 | case X86II::MO_GOTNTPOFF: | |||
245 | RefKind = MCSymbolRefExpr::VK_GOTNTPOFF; | |||
246 | break; | |||
247 | case X86II::MO_GOTPCREL: | |||
248 | RefKind = MCSymbolRefExpr::VK_GOTPCREL; | |||
249 | break; | |||
250 | case X86II::MO_GOT: | |||
251 | RefKind = MCSymbolRefExpr::VK_GOT; | |||
252 | break; | |||
253 | case X86II::MO_GOTOFF: | |||
254 | RefKind = MCSymbolRefExpr::VK_GOTOFF; | |||
255 | break; | |||
256 | case X86II::MO_PLT: | |||
257 | RefKind = MCSymbolRefExpr::VK_PLT; | |||
258 | break; | |||
259 | case X86II::MO_ABS8: | |||
260 | RefKind = MCSymbolRefExpr::VK_X86_ABS8; | |||
261 | break; | |||
262 | case X86II::MO_PIC_BASE_OFFSET: | |||
263 | case X86II::MO_DARWIN_NONLAZY_PIC_BASE: | |||
264 | Expr = MCSymbolRefExpr::create(Sym, Ctx); | |||
265 | // Subtract the pic base. | |||
266 | Expr = MCBinaryExpr::createSub( | |||
267 | Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx); | |||
268 | if (MO.isJTI()) { | |||
269 | assert(MAI.doesSetDirectiveSuppressReloc())((MAI.doesSetDirectiveSuppressReloc()) ? static_cast<void> (0) : __assert_fail ("MAI.doesSetDirectiveSuppressReloc()", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 269, __PRETTY_FUNCTION__)); | |||
270 | // If .set directive is supported, use it to reduce the number of | |||
271 | // relocations the assembler will generate for differences between | |||
272 | // local labels. This is only safe when the symbols are in the same | |||
273 | // section so we are restricting it to jumptable references. | |||
274 | MCSymbol *Label = Ctx.createTempSymbol(); | |||
275 | AsmPrinter.OutStreamer->EmitAssignment(Label, Expr); | |||
276 | Expr = MCSymbolRefExpr::create(Label, Ctx); | |||
277 | } | |||
278 | break; | |||
279 | } | |||
280 | ||||
281 | if (!Expr) | |||
282 | Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx); | |||
283 | ||||
284 | if (!MO.isJTI() && !MO.isMBB() && MO.getOffset()) | |||
285 | Expr = MCBinaryExpr::createAdd( | |||
286 | Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx); | |||
287 | return MCOperand::createExpr(Expr); | |||
288 | } | |||
289 | ||||
290 | /// Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with | |||
291 | /// a short fixed-register form. | |||
292 | static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode) { | |||
293 | unsigned ImmOp = Inst.getNumOperands() - 1; | |||
294 | assert(Inst.getOperand(0).isReg() &&((Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp ).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst .getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 299, __PRETTY_FUNCTION__)) | |||
295 | (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) &&((Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp ).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst .getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 299, __PRETTY_FUNCTION__)) | |||
296 | ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() &&((Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp ).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst .getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 299, __PRETTY_FUNCTION__)) | |||
297 | Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) ||((Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp ).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst .getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 299, __PRETTY_FUNCTION__)) | |||
298 | Inst.getNumOperands() == 2) &&((Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp ).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst .getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 299, __PRETTY_FUNCTION__)) | |||
299 | "Unexpected instruction!")((Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp ).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst .getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getOperand(0).isReg() && (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) && ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() && Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) || Inst.getNumOperands() == 2) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 299, __PRETTY_FUNCTION__)); | |||
300 | ||||
301 | // Check whether the destination register can be fixed. | |||
302 | unsigned Reg = Inst.getOperand(0).getReg(); | |||
303 | if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX) | |||
304 | return; | |||
305 | ||||
306 | // If so, rewrite the instruction. | |||
307 | MCOperand Saved = Inst.getOperand(ImmOp); | |||
308 | Inst = MCInst(); | |||
309 | Inst.setOpcode(Opcode); | |||
310 | Inst.addOperand(Saved); | |||
311 | } | |||
312 | ||||
313 | /// If a movsx instruction has a shorter encoding for the used register | |||
314 | /// simplify the instruction to use it instead. | |||
315 | static void SimplifyMOVSX(MCInst &Inst) { | |||
316 | unsigned NewOpcode = 0; | |||
317 | unsigned Op0 = Inst.getOperand(0).getReg(), Op1 = Inst.getOperand(1).getReg(); | |||
318 | switch (Inst.getOpcode()) { | |||
319 | default: | |||
320 | llvm_unreachable("Unexpected instruction!")::llvm::llvm_unreachable_internal("Unexpected instruction!", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 320); | |||
321 | case X86::MOVSX16rr8: // movsbw %al, %ax --> cbtw | |||
322 | if (Op0 == X86::AX && Op1 == X86::AL) | |||
323 | NewOpcode = X86::CBW; | |||
324 | break; | |||
325 | case X86::MOVSX32rr16: // movswl %ax, %eax --> cwtl | |||
326 | if (Op0 == X86::EAX && Op1 == X86::AX) | |||
327 | NewOpcode = X86::CWDE; | |||
328 | break; | |||
329 | case X86::MOVSX64rr32: // movslq %eax, %rax --> cltq | |||
330 | if (Op0 == X86::RAX && Op1 == X86::EAX) | |||
331 | NewOpcode = X86::CDQE; | |||
332 | break; | |||
333 | } | |||
334 | ||||
335 | if (NewOpcode != 0) { | |||
336 | Inst = MCInst(); | |||
337 | Inst.setOpcode(NewOpcode); | |||
338 | } | |||
339 | } | |||
340 | ||||
341 | /// Simplify things like MOV32rm to MOV32o32a. | |||
342 | static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst, | |||
343 | unsigned Opcode) { | |||
344 | // Don't make these simplifications in 64-bit mode; other assemblers don't | |||
345 | // perform them because they make the code larger. | |||
346 | if (Printer.getSubtarget().is64Bit()) | |||
347 | return; | |||
348 | ||||
349 | bool IsStore = Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg(); | |||
350 | unsigned AddrBase = IsStore; | |||
351 | unsigned RegOp = IsStore ? 0 : 5; | |||
352 | unsigned AddrOp = AddrBase + 3; | |||
353 | assert(((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)) | |||
354 | Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() &&((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)) | |||
355 | Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() &&((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)) | |||
356 | Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() &&((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)) | |||
357 | Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() &&((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)) | |||
358 | Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() &&((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)) | |||
359 | (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) &&((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)) | |||
360 | "Unexpected instruction!")((Inst.getNumOperands() == 6 && Inst.getOperand(RegOp ).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt ).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg ).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg ).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst .getOperand(AddrOp).isImm()) && "Unexpected instruction!" ) ? static_cast<void> (0) : __assert_fail ("Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() && Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() && Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() && Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() && Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() && (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) && \"Unexpected instruction!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 360, __PRETTY_FUNCTION__)); | |||
361 | ||||
362 | // Check whether the destination register can be fixed. | |||
363 | unsigned Reg = Inst.getOperand(RegOp).getReg(); | |||
364 | if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX) | |||
365 | return; | |||
366 | ||||
367 | // Check whether this is an absolute address. | |||
368 | // FIXME: We know TLVP symbol refs aren't, but there should be a better way | |||
369 | // to do this here. | |||
370 | bool Absolute = true; | |||
371 | if (Inst.getOperand(AddrOp).isExpr()) { | |||
372 | const MCExpr *MCE = Inst.getOperand(AddrOp).getExpr(); | |||
373 | if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE)) | |||
374 | if (SRE->getKind() == MCSymbolRefExpr::VK_TLVP) | |||
375 | Absolute = false; | |||
376 | } | |||
377 | ||||
378 | if (Absolute && | |||
379 | (Inst.getOperand(AddrBase + X86::AddrBaseReg).getReg() != 0 || | |||
380 | Inst.getOperand(AddrBase + X86::AddrScaleAmt).getImm() != 1 || | |||
381 | Inst.getOperand(AddrBase + X86::AddrIndexReg).getReg() != 0)) | |||
382 | return; | |||
383 | ||||
384 | // If so, rewrite the instruction. | |||
385 | MCOperand Saved = Inst.getOperand(AddrOp); | |||
386 | MCOperand Seg = Inst.getOperand(AddrBase + X86::AddrSegmentReg); | |||
387 | Inst = MCInst(); | |||
388 | Inst.setOpcode(Opcode); | |||
389 | Inst.addOperand(Saved); | |||
390 | Inst.addOperand(Seg); | |||
391 | } | |||
392 | ||||
393 | static unsigned getRetOpcode(const X86Subtarget &Subtarget) { | |||
394 | return Subtarget.is64Bit() ? X86::RETQ : X86::RETL; | |||
395 | } | |||
396 | ||||
397 | Optional<MCOperand> | |||
398 | X86MCInstLower::LowerMachineOperand(const MachineInstr *MI, | |||
399 | const MachineOperand &MO) const { | |||
400 | switch (MO.getType()) { | |||
401 | default: | |||
402 | MI->print(errs()); | |||
403 | llvm_unreachable("unknown operand type")::llvm::llvm_unreachable_internal("unknown operand type", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 403); | |||
404 | case MachineOperand::MO_Register: | |||
405 | // Ignore all implicit register operands. | |||
406 | if (MO.isImplicit()) | |||
407 | return None; | |||
408 | return MCOperand::createReg(MO.getReg()); | |||
409 | case MachineOperand::MO_Immediate: | |||
410 | return MCOperand::createImm(MO.getImm()); | |||
411 | case MachineOperand::MO_MachineBasicBlock: | |||
412 | case MachineOperand::MO_GlobalAddress: | |||
413 | case MachineOperand::MO_ExternalSymbol: | |||
414 | return LowerSymbolOperand(MO, GetSymbolFromOperand(MO)); | |||
415 | case MachineOperand::MO_MCSymbol: | |||
416 | return LowerSymbolOperand(MO, MO.getMCSymbol()); | |||
417 | case MachineOperand::MO_JumpTableIndex: | |||
418 | return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex())); | |||
419 | case MachineOperand::MO_ConstantPoolIndex: | |||
420 | return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex())); | |||
421 | case MachineOperand::MO_BlockAddress: | |||
422 | return LowerSymbolOperand( | |||
423 | MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress())); | |||
424 | case MachineOperand::MO_RegisterMask: | |||
425 | // Ignore call clobbers. | |||
426 | return None; | |||
427 | } | |||
428 | } | |||
429 | ||||
430 | // Replace TAILJMP opcodes with their equivalent opcodes that have encoding | |||
431 | // information. | |||
432 | static unsigned convertTailJumpOpcode(unsigned Opcode) { | |||
433 | switch (Opcode) { | |||
434 | case X86::TAILJMPr: | |||
435 | Opcode = X86::JMP32r; | |||
436 | break; | |||
437 | case X86::TAILJMPm: | |||
438 | Opcode = X86::JMP32m; | |||
439 | break; | |||
440 | case X86::TAILJMPr64: | |||
441 | Opcode = X86::JMP64r; | |||
442 | break; | |||
443 | case X86::TAILJMPm64: | |||
444 | Opcode = X86::JMP64m; | |||
445 | break; | |||
446 | case X86::TAILJMPr64_REX: | |||
447 | Opcode = X86::JMP64r_REX; | |||
448 | break; | |||
449 | case X86::TAILJMPm64_REX: | |||
450 | Opcode = X86::JMP64m_REX; | |||
451 | break; | |||
452 | case X86::TAILJMPd: | |||
453 | case X86::TAILJMPd64: | |||
454 | Opcode = X86::JMP_1; | |||
455 | break; | |||
456 | case X86::TAILJMPd_CC: | |||
457 | case X86::TAILJMPd64_CC: | |||
458 | Opcode = X86::JCC_1; | |||
459 | break; | |||
460 | } | |||
461 | ||||
462 | return Opcode; | |||
463 | } | |||
464 | ||||
465 | void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { | |||
466 | OutMI.setOpcode(MI->getOpcode()); | |||
467 | ||||
468 | for (const MachineOperand &MO : MI->operands()) | |||
| ||||
469 | if (auto MaybeMCOp = LowerMachineOperand(MI, MO)) | |||
470 | OutMI.addOperand(MaybeMCOp.getValue()); | |||
471 | ||||
472 | // Handle a few special cases to eliminate operand modifiers. | |||
473 | switch (OutMI.getOpcode()) { | |||
474 | case X86::LEA64_32r: | |||
475 | case X86::LEA64r: | |||
476 | case X86::LEA16r: | |||
477 | case X86::LEA32r: | |||
478 | // LEA should have a segment register, but it must be empty. | |||
479 | assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&((OutMI.getNumOperands() == 1 + X86::AddrNumOperands && "Unexpected # of LEA operands") ? static_cast<void> (0 ) : __assert_fail ("OutMI.getNumOperands() == 1 + X86::AddrNumOperands && \"Unexpected # of LEA operands\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 480, __PRETTY_FUNCTION__)) | |||
480 | "Unexpected # of LEA operands")((OutMI.getNumOperands() == 1 + X86::AddrNumOperands && "Unexpected # of LEA operands") ? static_cast<void> (0 ) : __assert_fail ("OutMI.getNumOperands() == 1 + X86::AddrNumOperands && \"Unexpected # of LEA operands\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 480, __PRETTY_FUNCTION__)); | |||
481 | assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&((OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 && "LEA has segment specified!") ? static_cast<void> (0) : __assert_fail ("OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 && \"LEA has segment specified!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 482, __PRETTY_FUNCTION__)) | |||
482 | "LEA has segment specified!")((OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 && "LEA has segment specified!") ? static_cast<void> (0) : __assert_fail ("OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 && \"LEA has segment specified!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 482, __PRETTY_FUNCTION__)); | |||
483 | break; | |||
484 | ||||
485 | // Commute operands to get a smaller encoding by using VEX.R instead of VEX.B | |||
486 | // if one of the registers is extended, but other isn't. | |||
487 | case X86::VMOVZPQILo2PQIrr: | |||
488 | case X86::VMOVAPDrr: | |||
489 | case X86::VMOVAPDYrr: | |||
490 | case X86::VMOVAPSrr: | |||
491 | case X86::VMOVAPSYrr: | |||
492 | case X86::VMOVDQArr: | |||
493 | case X86::VMOVDQAYrr: | |||
494 | case X86::VMOVDQUrr: | |||
495 | case X86::VMOVDQUYrr: | |||
496 | case X86::VMOVUPDrr: | |||
497 | case X86::VMOVUPDYrr: | |||
498 | case X86::VMOVUPSrr: | |||
499 | case X86::VMOVUPSYrr: { | |||
500 | if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) && | |||
501 | X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg())) { | |||
502 | unsigned NewOpc; | |||
503 | switch (OutMI.getOpcode()) { | |||
504 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 504); | |||
505 | case X86::VMOVZPQILo2PQIrr: NewOpc = X86::VMOVPQI2QIrr; break; | |||
506 | case X86::VMOVAPDrr: NewOpc = X86::VMOVAPDrr_REV; break; | |||
507 | case X86::VMOVAPDYrr: NewOpc = X86::VMOVAPDYrr_REV; break; | |||
508 | case X86::VMOVAPSrr: NewOpc = X86::VMOVAPSrr_REV; break; | |||
509 | case X86::VMOVAPSYrr: NewOpc = X86::VMOVAPSYrr_REV; break; | |||
510 | case X86::VMOVDQArr: NewOpc = X86::VMOVDQArr_REV; break; | |||
511 | case X86::VMOVDQAYrr: NewOpc = X86::VMOVDQAYrr_REV; break; | |||
512 | case X86::VMOVDQUrr: NewOpc = X86::VMOVDQUrr_REV; break; | |||
513 | case X86::VMOVDQUYrr: NewOpc = X86::VMOVDQUYrr_REV; break; | |||
514 | case X86::VMOVUPDrr: NewOpc = X86::VMOVUPDrr_REV; break; | |||
515 | case X86::VMOVUPDYrr: NewOpc = X86::VMOVUPDYrr_REV; break; | |||
516 | case X86::VMOVUPSrr: NewOpc = X86::VMOVUPSrr_REV; break; | |||
517 | case X86::VMOVUPSYrr: NewOpc = X86::VMOVUPSYrr_REV; break; | |||
518 | } | |||
519 | OutMI.setOpcode(NewOpc); | |||
520 | } | |||
521 | break; | |||
522 | } | |||
523 | case X86::VMOVSDrr: | |||
524 | case X86::VMOVSSrr: { | |||
525 | if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(0).getReg()) && | |||
526 | X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) { | |||
527 | unsigned NewOpc; | |||
528 | switch (OutMI.getOpcode()) { | |||
529 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 529); | |||
530 | case X86::VMOVSDrr: NewOpc = X86::VMOVSDrr_REV; break; | |||
531 | case X86::VMOVSSrr: NewOpc = X86::VMOVSSrr_REV; break; | |||
532 | } | |||
533 | OutMI.setOpcode(NewOpc); | |||
534 | } | |||
535 | break; | |||
536 | } | |||
537 | ||||
538 | case X86::VPCMPBZ128rmi: case X86::VPCMPBZ128rmik: | |||
539 | case X86::VPCMPBZ128rri: case X86::VPCMPBZ128rrik: | |||
540 | case X86::VPCMPBZ256rmi: case X86::VPCMPBZ256rmik: | |||
541 | case X86::VPCMPBZ256rri: case X86::VPCMPBZ256rrik: | |||
542 | case X86::VPCMPBZrmi: case X86::VPCMPBZrmik: | |||
543 | case X86::VPCMPBZrri: case X86::VPCMPBZrrik: | |||
544 | case X86::VPCMPDZ128rmi: case X86::VPCMPDZ128rmik: | |||
545 | case X86::VPCMPDZ128rmib: case X86::VPCMPDZ128rmibk: | |||
546 | case X86::VPCMPDZ128rri: case X86::VPCMPDZ128rrik: | |||
547 | case X86::VPCMPDZ256rmi: case X86::VPCMPDZ256rmik: | |||
548 | case X86::VPCMPDZ256rmib: case X86::VPCMPDZ256rmibk: | |||
549 | case X86::VPCMPDZ256rri: case X86::VPCMPDZ256rrik: | |||
550 | case X86::VPCMPDZrmi: case X86::VPCMPDZrmik: | |||
551 | case X86::VPCMPDZrmib: case X86::VPCMPDZrmibk: | |||
552 | case X86::VPCMPDZrri: case X86::VPCMPDZrrik: | |||
553 | case X86::VPCMPQZ128rmi: case X86::VPCMPQZ128rmik: | |||
554 | case X86::VPCMPQZ128rmib: case X86::VPCMPQZ128rmibk: | |||
555 | case X86::VPCMPQZ128rri: case X86::VPCMPQZ128rrik: | |||
556 | case X86::VPCMPQZ256rmi: case X86::VPCMPQZ256rmik: | |||
557 | case X86::VPCMPQZ256rmib: case X86::VPCMPQZ256rmibk: | |||
558 | case X86::VPCMPQZ256rri: case X86::VPCMPQZ256rrik: | |||
559 | case X86::VPCMPQZrmi: case X86::VPCMPQZrmik: | |||
560 | case X86::VPCMPQZrmib: case X86::VPCMPQZrmibk: | |||
561 | case X86::VPCMPQZrri: case X86::VPCMPQZrrik: | |||
562 | case X86::VPCMPWZ128rmi: case X86::VPCMPWZ128rmik: | |||
563 | case X86::VPCMPWZ128rri: case X86::VPCMPWZ128rrik: | |||
564 | case X86::VPCMPWZ256rmi: case X86::VPCMPWZ256rmik: | |||
565 | case X86::VPCMPWZ256rri: case X86::VPCMPWZ256rrik: | |||
566 | case X86::VPCMPWZrmi: case X86::VPCMPWZrmik: | |||
567 | case X86::VPCMPWZrri: case X86::VPCMPWZrrik: { | |||
568 | // Turn immediate 0 into the VPCMPEQ instruction. | |||
569 | if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 0) { | |||
570 | unsigned NewOpc; | |||
571 | switch (OutMI.getOpcode()) { | |||
572 | case X86::VPCMPBZ128rmi: NewOpc = X86::VPCMPEQBZ128rm; break; | |||
573 | case X86::VPCMPBZ128rmik: NewOpc = X86::VPCMPEQBZ128rmk; break; | |||
574 | case X86::VPCMPBZ128rri: NewOpc = X86::VPCMPEQBZ128rr; break; | |||
575 | case X86::VPCMPBZ128rrik: NewOpc = X86::VPCMPEQBZ128rrk; break; | |||
576 | case X86::VPCMPBZ256rmi: NewOpc = X86::VPCMPEQBZ256rm; break; | |||
577 | case X86::VPCMPBZ256rmik: NewOpc = X86::VPCMPEQBZ256rmk; break; | |||
578 | case X86::VPCMPBZ256rri: NewOpc = X86::VPCMPEQBZ256rr; break; | |||
579 | case X86::VPCMPBZ256rrik: NewOpc = X86::VPCMPEQBZ256rrk; break; | |||
580 | case X86::VPCMPBZrmi: NewOpc = X86::VPCMPEQBZrm; break; | |||
581 | case X86::VPCMPBZrmik: NewOpc = X86::VPCMPEQBZrmk; break; | |||
582 | case X86::VPCMPBZrri: NewOpc = X86::VPCMPEQBZrr; break; | |||
583 | case X86::VPCMPBZrrik: NewOpc = X86::VPCMPEQBZrrk; break; | |||
584 | case X86::VPCMPDZ128rmi: NewOpc = X86::VPCMPEQDZ128rm; break; | |||
585 | case X86::VPCMPDZ128rmib: NewOpc = X86::VPCMPEQDZ128rmb; break; | |||
586 | case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPEQDZ128rmbk; break; | |||
587 | case X86::VPCMPDZ128rmik: NewOpc = X86::VPCMPEQDZ128rmk; break; | |||
588 | case X86::VPCMPDZ128rri: NewOpc = X86::VPCMPEQDZ128rr; break; | |||
589 | case X86::VPCMPDZ128rrik: NewOpc = X86::VPCMPEQDZ128rrk; break; | |||
590 | case X86::VPCMPDZ256rmi: NewOpc = X86::VPCMPEQDZ256rm; break; | |||
591 | case X86::VPCMPDZ256rmib: NewOpc = X86::VPCMPEQDZ256rmb; break; | |||
592 | case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPEQDZ256rmbk; break; | |||
593 | case X86::VPCMPDZ256rmik: NewOpc = X86::VPCMPEQDZ256rmk; break; | |||
594 | case X86::VPCMPDZ256rri: NewOpc = X86::VPCMPEQDZ256rr; break; | |||
595 | case X86::VPCMPDZ256rrik: NewOpc = X86::VPCMPEQDZ256rrk; break; | |||
596 | case X86::VPCMPDZrmi: NewOpc = X86::VPCMPEQDZrm; break; | |||
597 | case X86::VPCMPDZrmib: NewOpc = X86::VPCMPEQDZrmb; break; | |||
598 | case X86::VPCMPDZrmibk: NewOpc = X86::VPCMPEQDZrmbk; break; | |||
599 | case X86::VPCMPDZrmik: NewOpc = X86::VPCMPEQDZrmk; break; | |||
600 | case X86::VPCMPDZrri: NewOpc = X86::VPCMPEQDZrr; break; | |||
601 | case X86::VPCMPDZrrik: NewOpc = X86::VPCMPEQDZrrk; break; | |||
602 | case X86::VPCMPQZ128rmi: NewOpc = X86::VPCMPEQQZ128rm; break; | |||
603 | case X86::VPCMPQZ128rmib: NewOpc = X86::VPCMPEQQZ128rmb; break; | |||
604 | case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPEQQZ128rmbk; break; | |||
605 | case X86::VPCMPQZ128rmik: NewOpc = X86::VPCMPEQQZ128rmk; break; | |||
606 | case X86::VPCMPQZ128rri: NewOpc = X86::VPCMPEQQZ128rr; break; | |||
607 | case X86::VPCMPQZ128rrik: NewOpc = X86::VPCMPEQQZ128rrk; break; | |||
608 | case X86::VPCMPQZ256rmi: NewOpc = X86::VPCMPEQQZ256rm; break; | |||
609 | case X86::VPCMPQZ256rmib: NewOpc = X86::VPCMPEQQZ256rmb; break; | |||
610 | case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPEQQZ256rmbk; break; | |||
611 | case X86::VPCMPQZ256rmik: NewOpc = X86::VPCMPEQQZ256rmk; break; | |||
612 | case X86::VPCMPQZ256rri: NewOpc = X86::VPCMPEQQZ256rr; break; | |||
613 | case X86::VPCMPQZ256rrik: NewOpc = X86::VPCMPEQQZ256rrk; break; | |||
614 | case X86::VPCMPQZrmi: NewOpc = X86::VPCMPEQQZrm; break; | |||
615 | case X86::VPCMPQZrmib: NewOpc = X86::VPCMPEQQZrmb; break; | |||
616 | case X86::VPCMPQZrmibk: NewOpc = X86::VPCMPEQQZrmbk; break; | |||
617 | case X86::VPCMPQZrmik: NewOpc = X86::VPCMPEQQZrmk; break; | |||
618 | case X86::VPCMPQZrri: NewOpc = X86::VPCMPEQQZrr; break; | |||
619 | case X86::VPCMPQZrrik: NewOpc = X86::VPCMPEQQZrrk; break; | |||
620 | case X86::VPCMPWZ128rmi: NewOpc = X86::VPCMPEQWZ128rm; break; | |||
621 | case X86::VPCMPWZ128rmik: NewOpc = X86::VPCMPEQWZ128rmk; break; | |||
622 | case X86::VPCMPWZ128rri: NewOpc = X86::VPCMPEQWZ128rr; break; | |||
623 | case X86::VPCMPWZ128rrik: NewOpc = X86::VPCMPEQWZ128rrk; break; | |||
624 | case X86::VPCMPWZ256rmi: NewOpc = X86::VPCMPEQWZ256rm; break; | |||
625 | case X86::VPCMPWZ256rmik: NewOpc = X86::VPCMPEQWZ256rmk; break; | |||
626 | case X86::VPCMPWZ256rri: NewOpc = X86::VPCMPEQWZ256rr; break; | |||
627 | case X86::VPCMPWZ256rrik: NewOpc = X86::VPCMPEQWZ256rrk; break; | |||
628 | case X86::VPCMPWZrmi: NewOpc = X86::VPCMPEQWZrm; break; | |||
629 | case X86::VPCMPWZrmik: NewOpc = X86::VPCMPEQWZrmk; break; | |||
630 | case X86::VPCMPWZrri: NewOpc = X86::VPCMPEQWZrr; break; | |||
631 | case X86::VPCMPWZrrik: NewOpc = X86::VPCMPEQWZrrk; break; | |||
632 | } | |||
633 | ||||
634 | OutMI.setOpcode(NewOpc); | |||
| ||||
635 | OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1)); | |||
636 | break; | |||
637 | } | |||
638 | ||||
639 | // Turn immediate 6 into the VPCMPGT instruction. | |||
640 | if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 6) { | |||
641 | unsigned NewOpc; | |||
642 | switch (OutMI.getOpcode()) { | |||
643 | case X86::VPCMPBZ128rmi: NewOpc = X86::VPCMPGTBZ128rm; break; | |||
644 | case X86::VPCMPBZ128rmik: NewOpc = X86::VPCMPGTBZ128rmk; break; | |||
645 | case X86::VPCMPBZ128rri: NewOpc = X86::VPCMPGTBZ128rr; break; | |||
646 | case X86::VPCMPBZ128rrik: NewOpc = X86::VPCMPGTBZ128rrk; break; | |||
647 | case X86::VPCMPBZ256rmi: NewOpc = X86::VPCMPGTBZ256rm; break; | |||
648 | case X86::VPCMPBZ256rmik: NewOpc = X86::VPCMPGTBZ256rmk; break; | |||
649 | case X86::VPCMPBZ256rri: NewOpc = X86::VPCMPGTBZ256rr; break; | |||
650 | case X86::VPCMPBZ256rrik: NewOpc = X86::VPCMPGTBZ256rrk; break; | |||
651 | case X86::VPCMPBZrmi: NewOpc = X86::VPCMPGTBZrm; break; | |||
652 | case X86::VPCMPBZrmik: NewOpc = X86::VPCMPGTBZrmk; break; | |||
653 | case X86::VPCMPBZrri: NewOpc = X86::VPCMPGTBZrr; break; | |||
654 | case X86::VPCMPBZrrik: NewOpc = X86::VPCMPGTBZrrk; break; | |||
655 | case X86::VPCMPDZ128rmi: NewOpc = X86::VPCMPGTDZ128rm; break; | |||
656 | case X86::VPCMPDZ128rmib: NewOpc = X86::VPCMPGTDZ128rmb; break; | |||
657 | case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPGTDZ128rmbk; break; | |||
658 | case X86::VPCMPDZ128rmik: NewOpc = X86::VPCMPGTDZ128rmk; break; | |||
659 | case X86::VPCMPDZ128rri: NewOpc = X86::VPCMPGTDZ128rr; break; | |||
660 | case X86::VPCMPDZ128rrik: NewOpc = X86::VPCMPGTDZ128rrk; break; | |||
661 | case X86::VPCMPDZ256rmi: NewOpc = X86::VPCMPGTDZ256rm; break; | |||
662 | case X86::VPCMPDZ256rmib: NewOpc = X86::VPCMPGTDZ256rmb; break; | |||
663 | case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPGTDZ256rmbk; break; | |||
664 | case X86::VPCMPDZ256rmik: NewOpc = X86::VPCMPGTDZ256rmk; break; | |||
665 | case X86::VPCMPDZ256rri: NewOpc = X86::VPCMPGTDZ256rr; break; | |||
666 | case X86::VPCMPDZ256rrik: NewOpc = X86::VPCMPGTDZ256rrk; break; | |||
667 | case X86::VPCMPDZrmi: NewOpc = X86::VPCMPGTDZrm; break; | |||
668 | case X86::VPCMPDZrmib: NewOpc = X86::VPCMPGTDZrmb; break; | |||
669 | case X86::VPCMPDZrmibk: NewOpc = X86::VPCMPGTDZrmbk; break; | |||
670 | case X86::VPCMPDZrmik: NewOpc = X86::VPCMPGTDZrmk; break; | |||
671 | case X86::VPCMPDZrri: NewOpc = X86::VPCMPGTDZrr; break; | |||
672 | case X86::VPCMPDZrrik: NewOpc = X86::VPCMPGTDZrrk; break; | |||
673 | case X86::VPCMPQZ128rmi: NewOpc = X86::VPCMPGTQZ128rm; break; | |||
674 | case X86::VPCMPQZ128rmib: NewOpc = X86::VPCMPGTQZ128rmb; break; | |||
675 | case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPGTQZ128rmbk; break; | |||
676 | case X86::VPCMPQZ128rmik: NewOpc = X86::VPCMPGTQZ128rmk; break; | |||
677 | case X86::VPCMPQZ128rri: NewOpc = X86::VPCMPGTQZ128rr; break; | |||
678 | case X86::VPCMPQZ128rrik: NewOpc = X86::VPCMPGTQZ128rrk; break; | |||
679 | case X86::VPCMPQZ256rmi: NewOpc = X86::VPCMPGTQZ256rm; break; | |||
680 | case X86::VPCMPQZ256rmib: NewOpc = X86::VPCMPGTQZ256rmb; break; | |||
681 | case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPGTQZ256rmbk; break; | |||
682 | case X86::VPCMPQZ256rmik: NewOpc = X86::VPCMPGTQZ256rmk; break; | |||
683 | case X86::VPCMPQZ256rri: NewOpc = X86::VPCMPGTQZ256rr; break; | |||
684 | case X86::VPCMPQZ256rrik: NewOpc = X86::VPCMPGTQZ256rrk; break; | |||
685 | case X86::VPCMPQZrmi: NewOpc = X86::VPCMPGTQZrm; break; | |||
686 | case X86::VPCMPQZrmib: NewOpc = X86::VPCMPGTQZrmb; break; | |||
687 | case X86::VPCMPQZrmibk: NewOpc = X86::VPCMPGTQZrmbk; break; | |||
688 | case X86::VPCMPQZrmik: NewOpc = X86::VPCMPGTQZrmk; break; | |||
689 | case X86::VPCMPQZrri: NewOpc = X86::VPCMPGTQZrr; break; | |||
690 | case X86::VPCMPQZrrik: NewOpc = X86::VPCMPGTQZrrk; break; | |||
691 | case X86::VPCMPWZ128rmi: NewOpc = X86::VPCMPGTWZ128rm; break; | |||
692 | case X86::VPCMPWZ128rmik: NewOpc = X86::VPCMPGTWZ128rmk; break; | |||
693 | case X86::VPCMPWZ128rri: NewOpc = X86::VPCMPGTWZ128rr; break; | |||
694 | case X86::VPCMPWZ128rrik: NewOpc = X86::VPCMPGTWZ128rrk; break; | |||
695 | case X86::VPCMPWZ256rmi: NewOpc = X86::VPCMPGTWZ256rm; break; | |||
696 | case X86::VPCMPWZ256rmik: NewOpc = X86::VPCMPGTWZ256rmk; break; | |||
697 | case X86::VPCMPWZ256rri: NewOpc = X86::VPCMPGTWZ256rr; break; | |||
698 | case X86::VPCMPWZ256rrik: NewOpc = X86::VPCMPGTWZ256rrk; break; | |||
699 | case X86::VPCMPWZrmi: NewOpc = X86::VPCMPGTWZrm; break; | |||
700 | case X86::VPCMPWZrmik: NewOpc = X86::VPCMPGTWZrmk; break; | |||
701 | case X86::VPCMPWZrri: NewOpc = X86::VPCMPGTWZrr; break; | |||
702 | case X86::VPCMPWZrrik: NewOpc = X86::VPCMPGTWZrrk; break; | |||
703 | } | |||
704 | ||||
705 | OutMI.setOpcode(NewOpc); | |||
706 | OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1)); | |||
707 | break; | |||
708 | } | |||
709 | ||||
710 | break; | |||
711 | } | |||
712 | ||||
713 | // CALL64r, CALL64pcrel32 - These instructions used to have | |||
714 | // register inputs modeled as normal uses instead of implicit uses. As such, | |||
715 | // they we used to truncate off all but the first operand (the callee). This | |||
716 | // issue seems to have been fixed at some point. This assert verifies that. | |||
717 | case X86::CALL64r: | |||
718 | case X86::CALL64pcrel32: | |||
719 | assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!")((OutMI.getNumOperands() == 1 && "Unexpected number of operands!" ) ? static_cast<void> (0) : __assert_fail ("OutMI.getNumOperands() == 1 && \"Unexpected number of operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 719, __PRETTY_FUNCTION__)); | |||
720 | break; | |||
721 | ||||
722 | case X86::EH_RETURN: | |||
723 | case X86::EH_RETURN64: { | |||
724 | OutMI = MCInst(); | |||
725 | OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget())); | |||
726 | break; | |||
727 | } | |||
728 | ||||
729 | case X86::CLEANUPRET: { | |||
730 | // Replace CLEANUPRET with the appropriate RET. | |||
731 | OutMI = MCInst(); | |||
732 | OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget())); | |||
733 | break; | |||
734 | } | |||
735 | ||||
736 | case X86::CATCHRET: { | |||
737 | // Replace CATCHRET with the appropriate RET. | |||
738 | const X86Subtarget &Subtarget = AsmPrinter.getSubtarget(); | |||
739 | unsigned ReturnReg = Subtarget.is64Bit() ? X86::RAX : X86::EAX; | |||
740 | OutMI = MCInst(); | |||
741 | OutMI.setOpcode(getRetOpcode(Subtarget)); | |||
742 | OutMI.addOperand(MCOperand::createReg(ReturnReg)); | |||
743 | break; | |||
744 | } | |||
745 | ||||
746 | // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump | |||
747 | // instruction. | |||
748 | case X86::TAILJMPr: | |||
749 | case X86::TAILJMPr64: | |||
750 | case X86::TAILJMPr64_REX: | |||
751 | case X86::TAILJMPd: | |||
752 | case X86::TAILJMPd64: | |||
753 | assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!")((OutMI.getNumOperands() == 1 && "Unexpected number of operands!" ) ? static_cast<void> (0) : __assert_fail ("OutMI.getNumOperands() == 1 && \"Unexpected number of operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 753, __PRETTY_FUNCTION__)); | |||
754 | OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode())); | |||
755 | break; | |||
756 | ||||
757 | case X86::TAILJMPd_CC: | |||
758 | case X86::TAILJMPd64_CC: | |||
759 | assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!")((OutMI.getNumOperands() == 2 && "Unexpected number of operands!" ) ? static_cast<void> (0) : __assert_fail ("OutMI.getNumOperands() == 2 && \"Unexpected number of operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 759, __PRETTY_FUNCTION__)); | |||
760 | OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode())); | |||
761 | break; | |||
762 | ||||
763 | case X86::TAILJMPm: | |||
764 | case X86::TAILJMPm64: | |||
765 | case X86::TAILJMPm64_REX: | |||
766 | assert(OutMI.getNumOperands() == X86::AddrNumOperands &&((OutMI.getNumOperands() == X86::AddrNumOperands && "Unexpected number of operands!" ) ? static_cast<void> (0) : __assert_fail ("OutMI.getNumOperands() == X86::AddrNumOperands && \"Unexpected number of operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 767, __PRETTY_FUNCTION__)) | |||
767 | "Unexpected number of operands!")((OutMI.getNumOperands() == X86::AddrNumOperands && "Unexpected number of operands!" ) ? static_cast<void> (0) : __assert_fail ("OutMI.getNumOperands() == X86::AddrNumOperands && \"Unexpected number of operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 767, __PRETTY_FUNCTION__)); | |||
768 | OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode())); | |||
769 | break; | |||
770 | ||||
771 | case X86::DEC16r: | |||
772 | case X86::DEC32r: | |||
773 | case X86::INC16r: | |||
774 | case X86::INC32r: | |||
775 | // If we aren't in 64-bit mode we can use the 1-byte inc/dec instructions. | |||
776 | if (!AsmPrinter.getSubtarget().is64Bit()) { | |||
777 | unsigned Opcode; | |||
778 | switch (OutMI.getOpcode()) { | |||
779 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 779); | |||
780 | case X86::DEC16r: Opcode = X86::DEC16r_alt; break; | |||
781 | case X86::DEC32r: Opcode = X86::DEC32r_alt; break; | |||
782 | case X86::INC16r: Opcode = X86::INC16r_alt; break; | |||
783 | case X86::INC32r: Opcode = X86::INC32r_alt; break; | |||
784 | } | |||
785 | OutMI.setOpcode(Opcode); | |||
786 | } | |||
787 | break; | |||
788 | ||||
789 | // We don't currently select the correct instruction form for instructions | |||
790 | // which have a short %eax, etc. form. Handle this by custom lowering, for | |||
791 | // now. | |||
792 | // | |||
793 | // Note, we are currently not handling the following instructions: | |||
794 | // MOV64ao8, MOV64o8a | |||
795 | // XCHG16ar, XCHG32ar, XCHG64ar | |||
796 | case X86::MOV8mr_NOREX: | |||
797 | case X86::MOV8mr: | |||
798 | case X86::MOV8rm_NOREX: | |||
799 | case X86::MOV8rm: | |||
800 | case X86::MOV16mr: | |||
801 | case X86::MOV16rm: | |||
802 | case X86::MOV32mr: | |||
803 | case X86::MOV32rm: { | |||
804 | unsigned NewOpc; | |||
805 | switch (OutMI.getOpcode()) { | |||
806 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 806); | |||
807 | case X86::MOV8mr_NOREX: | |||
808 | case X86::MOV8mr: NewOpc = X86::MOV8o32a; break; | |||
809 | case X86::MOV8rm_NOREX: | |||
810 | case X86::MOV8rm: NewOpc = X86::MOV8ao32; break; | |||
811 | case X86::MOV16mr: NewOpc = X86::MOV16o32a; break; | |||
812 | case X86::MOV16rm: NewOpc = X86::MOV16ao32; break; | |||
813 | case X86::MOV32mr: NewOpc = X86::MOV32o32a; break; | |||
814 | case X86::MOV32rm: NewOpc = X86::MOV32ao32; break; | |||
815 | } | |||
816 | SimplifyShortMoveForm(AsmPrinter, OutMI, NewOpc); | |||
817 | break; | |||
818 | } | |||
819 | ||||
820 | case X86::ADC8ri: case X86::ADC16ri: case X86::ADC32ri: case X86::ADC64ri32: | |||
821 | case X86::ADD8ri: case X86::ADD16ri: case X86::ADD32ri: case X86::ADD64ri32: | |||
822 | case X86::AND8ri: case X86::AND16ri: case X86::AND32ri: case X86::AND64ri32: | |||
823 | case X86::CMP8ri: case X86::CMP16ri: case X86::CMP32ri: case X86::CMP64ri32: | |||
824 | case X86::OR8ri: case X86::OR16ri: case X86::OR32ri: case X86::OR64ri32: | |||
825 | case X86::SBB8ri: case X86::SBB16ri: case X86::SBB32ri: case X86::SBB64ri32: | |||
826 | case X86::SUB8ri: case X86::SUB16ri: case X86::SUB32ri: case X86::SUB64ri32: | |||
827 | case X86::TEST8ri:case X86::TEST16ri:case X86::TEST32ri:case X86::TEST64ri32: | |||
828 | case X86::XOR8ri: case X86::XOR16ri: case X86::XOR32ri: case X86::XOR64ri32: { | |||
829 | unsigned NewOpc; | |||
830 | switch (OutMI.getOpcode()) { | |||
831 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 831); | |||
832 | case X86::ADC8ri: NewOpc = X86::ADC8i8; break; | |||
833 | case X86::ADC16ri: NewOpc = X86::ADC16i16; break; | |||
834 | case X86::ADC32ri: NewOpc = X86::ADC32i32; break; | |||
835 | case X86::ADC64ri32: NewOpc = X86::ADC64i32; break; | |||
836 | case X86::ADD8ri: NewOpc = X86::ADD8i8; break; | |||
837 | case X86::ADD16ri: NewOpc = X86::ADD16i16; break; | |||
838 | case X86::ADD32ri: NewOpc = X86::ADD32i32; break; | |||
839 | case X86::ADD64ri32: NewOpc = X86::ADD64i32; break; | |||
840 | case X86::AND8ri: NewOpc = X86::AND8i8; break; | |||
841 | case X86::AND16ri: NewOpc = X86::AND16i16; break; | |||
842 | case X86::AND32ri: NewOpc = X86::AND32i32; break; | |||
843 | case X86::AND64ri32: NewOpc = X86::AND64i32; break; | |||
844 | case X86::CMP8ri: NewOpc = X86::CMP8i8; break; | |||
845 | case X86::CMP16ri: NewOpc = X86::CMP16i16; break; | |||
846 | case X86::CMP32ri: NewOpc = X86::CMP32i32; break; | |||
847 | case X86::CMP64ri32: NewOpc = X86::CMP64i32; break; | |||
848 | case X86::OR8ri: NewOpc = X86::OR8i8; break; | |||
849 | case X86::OR16ri: NewOpc = X86::OR16i16; break; | |||
850 | case X86::OR32ri: NewOpc = X86::OR32i32; break; | |||
851 | case X86::OR64ri32: NewOpc = X86::OR64i32; break; | |||
852 | case X86::SBB8ri: NewOpc = X86::SBB8i8; break; | |||
853 | case X86::SBB16ri: NewOpc = X86::SBB16i16; break; | |||
854 | case X86::SBB32ri: NewOpc = X86::SBB32i32; break; | |||
855 | case X86::SBB64ri32: NewOpc = X86::SBB64i32; break; | |||
856 | case X86::SUB8ri: NewOpc = X86::SUB8i8; break; | |||
857 | case X86::SUB16ri: NewOpc = X86::SUB16i16; break; | |||
858 | case X86::SUB32ri: NewOpc = X86::SUB32i32; break; | |||
859 | case X86::SUB64ri32: NewOpc = X86::SUB64i32; break; | |||
860 | case X86::TEST8ri: NewOpc = X86::TEST8i8; break; | |||
861 | case X86::TEST16ri: NewOpc = X86::TEST16i16; break; | |||
862 | case X86::TEST32ri: NewOpc = X86::TEST32i32; break; | |||
863 | case X86::TEST64ri32: NewOpc = X86::TEST64i32; break; | |||
864 | case X86::XOR8ri: NewOpc = X86::XOR8i8; break; | |||
865 | case X86::XOR16ri: NewOpc = X86::XOR16i16; break; | |||
866 | case X86::XOR32ri: NewOpc = X86::XOR32i32; break; | |||
867 | case X86::XOR64ri32: NewOpc = X86::XOR64i32; break; | |||
868 | } | |||
869 | SimplifyShortImmForm(OutMI, NewOpc); | |||
870 | break; | |||
871 | } | |||
872 | ||||
873 | // Try to shrink some forms of movsx. | |||
874 | case X86::MOVSX16rr8: | |||
875 | case X86::MOVSX32rr16: | |||
876 | case X86::MOVSX64rr32: | |||
877 | SimplifyMOVSX(OutMI); | |||
878 | break; | |||
879 | ||||
880 | case X86::VCMPPDrri: | |||
881 | case X86::VCMPPDYrri: | |||
882 | case X86::VCMPPSrri: | |||
883 | case X86::VCMPPSYrri: | |||
884 | case X86::VCMPSDrr: | |||
885 | case X86::VCMPSSrr: { | |||
886 | // Swap the operands if it will enable a 2 byte VEX encoding. | |||
887 | // FIXME: Change the immediate to improve opportunities? | |||
888 | if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) && | |||
889 | X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) { | |||
890 | unsigned Imm = MI->getOperand(3).getImm() & 0x7; | |||
891 | switch (Imm) { | |||
892 | default: break; | |||
893 | case 0x00: // EQUAL | |||
894 | case 0x03: // UNORDERED | |||
895 | case 0x04: // NOT EQUAL | |||
896 | case 0x07: // ORDERED | |||
897 | std::swap(OutMI.getOperand(1), OutMI.getOperand(2)); | |||
898 | break; | |||
899 | } | |||
900 | } | |||
901 | break; | |||
902 | } | |||
903 | ||||
904 | case X86::VMOVHLPSrr: | |||
905 | case X86::VUNPCKHPDrr: | |||
906 | // These are not truly commutable so hide them from the default case. | |||
907 | break; | |||
908 | ||||
909 | default: { | |||
910 | // If the instruction is a commutable arithmetic instruction we might be | |||
911 | // able to commute the operands to get a 2 byte VEX prefix. | |||
912 | uint64_t TSFlags = MI->getDesc().TSFlags; | |||
913 | if (MI->getDesc().isCommutable() && | |||
914 | (TSFlags & X86II::EncodingMask) == X86II::VEX && | |||
915 | (TSFlags & X86II::OpMapMask) == X86II::TB && | |||
916 | (TSFlags & X86II::FormMask) == X86II::MRMSrcReg && | |||
917 | !(TSFlags & X86II::VEX_W) && (TSFlags & X86II::VEX_4V) && | |||
918 | OutMI.getNumOperands() == 3) { | |||
919 | if (!X86II::isX86_64ExtendedReg(OutMI.getOperand(1).getReg()) && | |||
920 | X86II::isX86_64ExtendedReg(OutMI.getOperand(2).getReg())) | |||
921 | std::swap(OutMI.getOperand(1), OutMI.getOperand(2)); | |||
922 | } | |||
923 | break; | |||
924 | } | |||
925 | } | |||
926 | } | |||
927 | ||||
928 | void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, | |||
929 | const MachineInstr &MI) { | |||
930 | bool Is64Bits = MI.getOpcode() == X86::TLS_addr64 || | |||
931 | MI.getOpcode() == X86::TLS_base_addr64; | |||
932 | MCContext &Ctx = OutStreamer->getContext(); | |||
933 | ||||
934 | MCSymbolRefExpr::VariantKind SRVK; | |||
935 | switch (MI.getOpcode()) { | |||
936 | case X86::TLS_addr32: | |||
937 | case X86::TLS_addr64: | |||
938 | SRVK = MCSymbolRefExpr::VK_TLSGD; | |||
939 | break; | |||
940 | case X86::TLS_base_addr32: | |||
941 | SRVK = MCSymbolRefExpr::VK_TLSLDM; | |||
942 | break; | |||
943 | case X86::TLS_base_addr64: | |||
944 | SRVK = MCSymbolRefExpr::VK_TLSLD; | |||
945 | break; | |||
946 | default: | |||
947 | llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 947); | |||
948 | } | |||
949 | ||||
950 | const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create( | |||
951 | MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx); | |||
952 | ||||
953 | // As of binutils 2.32, ld has a bogus TLS relaxation error when the GD/LD | |||
954 | // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is | |||
955 | // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by | |||
956 | // only using GOT when GOTPCRELX is enabled. | |||
957 | // TODO Delete the workaround when GOTPCRELX becomes commonplace. | |||
958 | bool UseGot = MMI->getModule()->getRtLibUseGOT() && | |||
959 | Ctx.getAsmInfo()->canRelaxRelocations(); | |||
960 | ||||
961 | if (Is64Bits) { | |||
962 | bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD; | |||
963 | if (NeedsPadding) | |||
964 | EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); | |||
965 | EmitAndCountInstruction(MCInstBuilder(X86::LEA64r) | |||
966 | .addReg(X86::RDI) | |||
967 | .addReg(X86::RIP) | |||
968 | .addImm(1) | |||
969 | .addReg(0) | |||
970 | .addExpr(Sym) | |||
971 | .addReg(0)); | |||
972 | const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr"); | |||
973 | if (NeedsPadding) { | |||
974 | if (!UseGot) | |||
975 | EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); | |||
976 | EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); | |||
977 | EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX)); | |||
978 | } | |||
979 | if (UseGot) { | |||
980 | const MCExpr *Expr = MCSymbolRefExpr::create( | |||
981 | TlsGetAddr, MCSymbolRefExpr::VK_GOTPCREL, Ctx); | |||
982 | EmitAndCountInstruction(MCInstBuilder(X86::CALL64m) | |||
983 | .addReg(X86::RIP) | |||
984 | .addImm(1) | |||
985 | .addReg(0) | |||
986 | .addExpr(Expr) | |||
987 | .addReg(0)); | |||
988 | } else { | |||
989 | EmitAndCountInstruction( | |||
990 | MCInstBuilder(X86::CALL64pcrel32) | |||
991 | .addExpr(MCSymbolRefExpr::create(TlsGetAddr, | |||
992 | MCSymbolRefExpr::VK_PLT, Ctx))); | |||
993 | } | |||
994 | } else { | |||
995 | if (SRVK == MCSymbolRefExpr::VK_TLSGD && !UseGot) { | |||
996 | EmitAndCountInstruction(MCInstBuilder(X86::LEA32r) | |||
997 | .addReg(X86::EAX) | |||
998 | .addReg(0) | |||
999 | .addImm(1) | |||
1000 | .addReg(X86::EBX) | |||
1001 | .addExpr(Sym) | |||
1002 | .addReg(0)); | |||
1003 | } else { | |||
1004 | EmitAndCountInstruction(MCInstBuilder(X86::LEA32r) | |||
1005 | .addReg(X86::EAX) | |||
1006 | .addReg(X86::EBX) | |||
1007 | .addImm(1) | |||
1008 | .addReg(0) | |||
1009 | .addExpr(Sym) | |||
1010 | .addReg(0)); | |||
1011 | } | |||
1012 | ||||
1013 | const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr"); | |||
1014 | if (UseGot) { | |||
1015 | const MCExpr *Expr = | |||
1016 | MCSymbolRefExpr::create(TlsGetAddr, MCSymbolRefExpr::VK_GOT, Ctx); | |||
1017 | EmitAndCountInstruction(MCInstBuilder(X86::CALL32m) | |||
1018 | .addReg(X86::EBX) | |||
1019 | .addImm(1) | |||
1020 | .addReg(0) | |||
1021 | .addExpr(Expr) | |||
1022 | .addReg(0)); | |||
1023 | } else { | |||
1024 | EmitAndCountInstruction( | |||
1025 | MCInstBuilder(X86::CALLpcrel32) | |||
1026 | .addExpr(MCSymbolRefExpr::create(TlsGetAddr, | |||
1027 | MCSymbolRefExpr::VK_PLT, Ctx))); | |||
1028 | } | |||
1029 | } | |||
1030 | } | |||
1031 | ||||
1032 | /// Emit the largest nop instruction smaller than or equal to \p NumBytes | |||
1033 | /// bytes. Return the size of nop emitted. | |||
1034 | static unsigned EmitNop(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, | |||
1035 | const MCSubtargetInfo &STI) { | |||
1036 | // This works only for 64bit. For 32bit we have to do additional checking if | |||
1037 | // the CPU supports multi-byte nops. | |||
1038 | assert(Is64Bit && "EmitNops only supports X86-64")((Is64Bit && "EmitNops only supports X86-64") ? static_cast <void> (0) : __assert_fail ("Is64Bit && \"EmitNops only supports X86-64\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1038, __PRETTY_FUNCTION__)); | |||
1039 | ||||
1040 | unsigned NopSize; | |||
1041 | unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg; | |||
1042 | IndexReg = Displacement = SegmentReg = 0; | |||
1043 | BaseReg = X86::RAX; | |||
1044 | ScaleVal = 1; | |||
1045 | switch (NumBytes) { | |||
1046 | case 0: | |||
1047 | llvm_unreachable("Zero nops?")::llvm::llvm_unreachable_internal("Zero nops?", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1047); | |||
1048 | break; | |||
1049 | case 1: | |||
1050 | NopSize = 1; | |||
1051 | Opc = X86::NOOP; | |||
1052 | break; | |||
1053 | case 2: | |||
1054 | NopSize = 2; | |||
1055 | Opc = X86::XCHG16ar; | |||
1056 | break; | |||
1057 | case 3: | |||
1058 | NopSize = 3; | |||
1059 | Opc = X86::NOOPL; | |||
1060 | break; | |||
1061 | case 4: | |||
1062 | NopSize = 4; | |||
1063 | Opc = X86::NOOPL; | |||
1064 | Displacement = 8; | |||
1065 | break; | |||
1066 | case 5: | |||
1067 | NopSize = 5; | |||
1068 | Opc = X86::NOOPL; | |||
1069 | Displacement = 8; | |||
1070 | IndexReg = X86::RAX; | |||
1071 | break; | |||
1072 | case 6: | |||
1073 | NopSize = 6; | |||
1074 | Opc = X86::NOOPW; | |||
1075 | Displacement = 8; | |||
1076 | IndexReg = X86::RAX; | |||
1077 | break; | |||
1078 | case 7: | |||
1079 | NopSize = 7; | |||
1080 | Opc = X86::NOOPL; | |||
1081 | Displacement = 512; | |||
1082 | break; | |||
1083 | case 8: | |||
1084 | NopSize = 8; | |||
1085 | Opc = X86::NOOPL; | |||
1086 | Displacement = 512; | |||
1087 | IndexReg = X86::RAX; | |||
1088 | break; | |||
1089 | case 9: | |||
1090 | NopSize = 9; | |||
1091 | Opc = X86::NOOPW; | |||
1092 | Displacement = 512; | |||
1093 | IndexReg = X86::RAX; | |||
1094 | break; | |||
1095 | default: | |||
1096 | NopSize = 10; | |||
1097 | Opc = X86::NOOPW; | |||
1098 | Displacement = 512; | |||
1099 | IndexReg = X86::RAX; | |||
1100 | SegmentReg = X86::CS; | |||
1101 | break; | |||
1102 | } | |||
1103 | ||||
1104 | unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U); | |||
1105 | NopSize += NumPrefixes; | |||
1106 | for (unsigned i = 0; i != NumPrefixes; ++i) | |||
1107 | OS.EmitBytes("\x66"); | |||
1108 | ||||
1109 | switch (Opc) { | |||
1110 | default: llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1110); | |||
1111 | case X86::NOOP: | |||
1112 | OS.EmitInstruction(MCInstBuilder(Opc), STI); | |||
1113 | break; | |||
1114 | case X86::XCHG16ar: | |||
1115 | OS.EmitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX), STI); | |||
1116 | break; | |||
1117 | case X86::NOOPL: | |||
1118 | case X86::NOOPW: | |||
1119 | OS.EmitInstruction(MCInstBuilder(Opc) | |||
1120 | .addReg(BaseReg) | |||
1121 | .addImm(ScaleVal) | |||
1122 | .addReg(IndexReg) | |||
1123 | .addImm(Displacement) | |||
1124 | .addReg(SegmentReg), | |||
1125 | STI); | |||
1126 | break; | |||
1127 | } | |||
1128 | assert(NopSize <= NumBytes && "We overemitted?")((NopSize <= NumBytes && "We overemitted?") ? static_cast <void> (0) : __assert_fail ("NopSize <= NumBytes && \"We overemitted?\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1128, __PRETTY_FUNCTION__)); | |||
1129 | return NopSize; | |||
1130 | } | |||
1131 | ||||
1132 | /// Emit the optimal amount of multi-byte nops on X86. | |||
1133 | static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, | |||
1134 | const MCSubtargetInfo &STI) { | |||
1135 | unsigned NopsToEmit = NumBytes; | |||
1136 | (void)NopsToEmit; | |||
1137 | while (NumBytes) { | |||
1138 | NumBytes -= EmitNop(OS, NumBytes, Is64Bit, STI); | |||
1139 | assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!")((NopsToEmit >= NumBytes && "Emitted more than I asked for!" ) ? static_cast<void> (0) : __assert_fail ("NopsToEmit >= NumBytes && \"Emitted more than I asked for!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1139, __PRETTY_FUNCTION__)); | |||
1140 | } | |||
1141 | } | |||
1142 | ||||
1143 | void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI, | |||
1144 | X86MCInstLower &MCIL) { | |||
1145 | assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64")((Subtarget->is64Bit() && "Statepoint currently only supports X86-64" ) ? static_cast<void> (0) : __assert_fail ("Subtarget->is64Bit() && \"Statepoint currently only supports X86-64\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1145, __PRETTY_FUNCTION__)); | |||
1146 | ||||
1147 | StatepointOpers SOpers(&MI); | |||
1148 | if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { | |||
1149 | EmitNops(*OutStreamer, PatchBytes, Subtarget->is64Bit(), | |||
1150 | getSubtargetInfo()); | |||
1151 | } else { | |||
1152 | // Lower call target and choose correct opcode | |||
1153 | const MachineOperand &CallTarget = SOpers.getCallTarget(); | |||
1154 | MCOperand CallTargetMCOp; | |||
1155 | unsigned CallOpcode; | |||
1156 | switch (CallTarget.getType()) { | |||
1157 | case MachineOperand::MO_GlobalAddress: | |||
1158 | case MachineOperand::MO_ExternalSymbol: | |||
1159 | CallTargetMCOp = MCIL.LowerSymbolOperand( | |||
1160 | CallTarget, MCIL.GetSymbolFromOperand(CallTarget)); | |||
1161 | CallOpcode = X86::CALL64pcrel32; | |||
1162 | // Currently, we only support relative addressing with statepoints. | |||
1163 | // Otherwise, we'll need a scratch register to hold the target | |||
1164 | // address. You'll fail asserts during load & relocation if this | |||
1165 | // symbol is to far away. (TODO: support non-relative addressing) | |||
1166 | break; | |||
1167 | case MachineOperand::MO_Immediate: | |||
1168 | CallTargetMCOp = MCOperand::createImm(CallTarget.getImm()); | |||
1169 | CallOpcode = X86::CALL64pcrel32; | |||
1170 | // Currently, we only support relative addressing with statepoints. | |||
1171 | // Otherwise, we'll need a scratch register to hold the target | |||
1172 | // immediate. You'll fail asserts during load & relocation if this | |||
1173 | // address is to far away. (TODO: support non-relative addressing) | |||
1174 | break; | |||
1175 | case MachineOperand::MO_Register: | |||
1176 | // FIXME: Add retpoline support and remove this. | |||
1177 | if (Subtarget->useRetpolineIndirectCalls()) | |||
1178 | report_fatal_error("Lowering register statepoints with retpoline not " | |||
1179 | "yet implemented."); | |||
1180 | CallTargetMCOp = MCOperand::createReg(CallTarget.getReg()); | |||
1181 | CallOpcode = X86::CALL64r; | |||
1182 | break; | |||
1183 | default: | |||
1184 | llvm_unreachable("Unsupported operand type in statepoint call target")::llvm::llvm_unreachable_internal("Unsupported operand type in statepoint call target" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1184); | |||
1185 | break; | |||
1186 | } | |||
1187 | ||||
1188 | // Emit call | |||
1189 | MCInst CallInst; | |||
1190 | CallInst.setOpcode(CallOpcode); | |||
1191 | CallInst.addOperand(CallTargetMCOp); | |||
1192 | OutStreamer->EmitInstruction(CallInst, getSubtargetInfo()); | |||
1193 | } | |||
1194 | ||||
1195 | // Record our statepoint node in the same section used by STACKMAP | |||
1196 | // and PATCHPOINT | |||
1197 | auto &Ctx = OutStreamer->getContext(); | |||
1198 | MCSymbol *MILabel = Ctx.createTempSymbol(); | |||
1199 | OutStreamer->EmitLabel(MILabel); | |||
1200 | SM.recordStatepoint(*MILabel, MI); | |||
1201 | } | |||
1202 | ||||
1203 | void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI, | |||
1204 | X86MCInstLower &MCIL) { | |||
1205 | // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>, | |||
1206 | // <opcode>, <operands> | |||
1207 | ||||
1208 | Register DefRegister = FaultingMI.getOperand(0).getReg(); | |||
1209 | FaultMaps::FaultKind FK = | |||
1210 | static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm()); | |||
1211 | MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol(); | |||
1212 | unsigned Opcode = FaultingMI.getOperand(3).getImm(); | |||
1213 | unsigned OperandsBeginIdx = 4; | |||
1214 | ||||
1215 | auto &Ctx = OutStreamer->getContext(); | |||
1216 | MCSymbol *FaultingLabel = Ctx.createTempSymbol(); | |||
1217 | OutStreamer->EmitLabel(FaultingLabel); | |||
1218 | ||||
1219 | assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!")((FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!" ) ? static_cast<void> (0) : __assert_fail ("FK < FaultMaps::FaultKindMax && \"Invalid Faulting Kind!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1219, __PRETTY_FUNCTION__)); | |||
1220 | FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel); | |||
1221 | ||||
1222 | MCInst MI; | |||
1223 | MI.setOpcode(Opcode); | |||
1224 | ||||
1225 | if (DefRegister != X86::NoRegister) | |||
1226 | MI.addOperand(MCOperand::createReg(DefRegister)); | |||
1227 | ||||
1228 | for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx, | |||
1229 | E = FaultingMI.operands_end(); | |||
1230 | I != E; ++I) | |||
1231 | if (auto MaybeOperand = MCIL.LowerMachineOperand(&FaultingMI, *I)) | |||
1232 | MI.addOperand(MaybeOperand.getValue()); | |||
1233 | ||||
1234 | OutStreamer->AddComment("on-fault: " + HandlerLabel->getName()); | |||
1235 | OutStreamer->EmitInstruction(MI, getSubtargetInfo()); | |||
1236 | } | |||
1237 | ||||
1238 | void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI, | |||
1239 | X86MCInstLower &MCIL) { | |||
1240 | bool Is64Bits = Subtarget->is64Bit(); | |||
1241 | MCContext &Ctx = OutStreamer->getContext(); | |||
1242 | MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__"); | |||
1243 | const MCSymbolRefExpr *Op = | |||
1244 | MCSymbolRefExpr::create(fentry, MCSymbolRefExpr::VK_None, Ctx); | |||
1245 | ||||
1246 | EmitAndCountInstruction( | |||
1247 | MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32) | |||
1248 | .addExpr(Op)); | |||
1249 | } | |||
1250 | ||||
1251 | void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI, | |||
1252 | X86MCInstLower &MCIL) { | |||
1253 | // PATCHABLE_OP minsize, opcode, operands | |||
1254 | ||||
1255 | unsigned MinSize = MI.getOperand(0).getImm(); | |||
1256 | unsigned Opcode = MI.getOperand(1).getImm(); | |||
1257 | ||||
1258 | MCInst MCI; | |||
1259 | MCI.setOpcode(Opcode); | |||
1260 | for (auto &MO : make_range(MI.operands_begin() + 2, MI.operands_end())) | |||
1261 | if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO)) | |||
1262 | MCI.addOperand(MaybeOperand.getValue()); | |||
1263 | ||||
1264 | SmallString<256> Code; | |||
1265 | SmallVector<MCFixup, 4> Fixups; | |||
1266 | raw_svector_ostream VecOS(Code); | |||
1267 | CodeEmitter->encodeInstruction(MCI, VecOS, Fixups, getSubtargetInfo()); | |||
1268 | ||||
1269 | if (Code.size() < MinSize) { | |||
1270 | if (MinSize == 2 && Opcode == X86::PUSH64r) { | |||
1271 | // This is an optimization that lets us get away without emitting a nop in | |||
1272 | // many cases. | |||
1273 | // | |||
1274 | // NB! In some cases the encoding for PUSH64r (e.g. PUSH64r %r9) takes two | |||
1275 | // bytes too, so the check on MinSize is important. | |||
1276 | MCI.setOpcode(X86::PUSH64rmr); | |||
1277 | } else { | |||
1278 | unsigned NopSize = EmitNop(*OutStreamer, MinSize, Subtarget->is64Bit(), | |||
1279 | getSubtargetInfo()); | |||
1280 | assert(NopSize == MinSize && "Could not implement MinSize!")((NopSize == MinSize && "Could not implement MinSize!" ) ? static_cast<void> (0) : __assert_fail ("NopSize == MinSize && \"Could not implement MinSize!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1280, __PRETTY_FUNCTION__)); | |||
1281 | (void)NopSize; | |||
1282 | } | |||
1283 | } | |||
1284 | ||||
1285 | OutStreamer->EmitInstruction(MCI, getSubtargetInfo()); | |||
1286 | } | |||
1287 | ||||
1288 | // Lower a stackmap of the form: | |||
1289 | // <id>, <shadowBytes>, ... | |||
1290 | void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) { | |||
1291 | SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); | |||
1292 | ||||
1293 | auto &Ctx = OutStreamer->getContext(); | |||
1294 | MCSymbol *MILabel = Ctx.createTempSymbol(); | |||
1295 | OutStreamer->EmitLabel(MILabel); | |||
1296 | ||||
1297 | SM.recordStackMap(*MILabel, MI); | |||
1298 | unsigned NumShadowBytes = MI.getOperand(1).getImm(); | |||
1299 | SMShadowTracker.reset(NumShadowBytes); | |||
1300 | } | |||
1301 | ||||
1302 | // Lower a patchpoint of the form: | |||
1303 | // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ... | |||
1304 | void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, | |||
1305 | X86MCInstLower &MCIL) { | |||
1306 | assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64")((Subtarget->is64Bit() && "Patchpoint currently only supports X86-64" ) ? static_cast<void> (0) : __assert_fail ("Subtarget->is64Bit() && \"Patchpoint currently only supports X86-64\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1306, __PRETTY_FUNCTION__)); | |||
1307 | ||||
1308 | SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); | |||
1309 | ||||
1310 | auto &Ctx = OutStreamer->getContext(); | |||
1311 | MCSymbol *MILabel = Ctx.createTempSymbol(); | |||
1312 | OutStreamer->EmitLabel(MILabel); | |||
1313 | SM.recordPatchPoint(*MILabel, MI); | |||
1314 | ||||
1315 | PatchPointOpers opers(&MI); | |||
1316 | unsigned ScratchIdx = opers.getNextScratchIdx(); | |||
1317 | unsigned EncodedBytes = 0; | |||
1318 | const MachineOperand &CalleeMO = opers.getCallTarget(); | |||
1319 | ||||
1320 | // Check for null target. If target is non-null (i.e. is non-zero or is | |||
1321 | // symbolic) then emit a call. | |||
1322 | if (!(CalleeMO.isImm() && !CalleeMO.getImm())) { | |||
1323 | MCOperand CalleeMCOp; | |||
1324 | switch (CalleeMO.getType()) { | |||
1325 | default: | |||
1326 | /// FIXME: Add a verifier check for bad callee types. | |||
1327 | llvm_unreachable("Unrecognized callee operand type.")::llvm::llvm_unreachable_internal("Unrecognized callee operand type." , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1327); | |||
1328 | case MachineOperand::MO_Immediate: | |||
1329 | if (CalleeMO.getImm()) | |||
1330 | CalleeMCOp = MCOperand::createImm(CalleeMO.getImm()); | |||
1331 | break; | |||
1332 | case MachineOperand::MO_ExternalSymbol: | |||
1333 | case MachineOperand::MO_GlobalAddress: | |||
1334 | CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO, | |||
1335 | MCIL.GetSymbolFromOperand(CalleeMO)); | |||
1336 | break; | |||
1337 | } | |||
1338 | ||||
1339 | // Emit MOV to materialize the target address and the CALL to target. | |||
1340 | // This is encoded with 12-13 bytes, depending on which register is used. | |||
1341 | Register ScratchReg = MI.getOperand(ScratchIdx).getReg(); | |||
1342 | if (X86II::isX86_64ExtendedReg(ScratchReg)) | |||
1343 | EncodedBytes = 13; | |||
1344 | else | |||
1345 | EncodedBytes = 12; | |||
1346 | ||||
1347 | EmitAndCountInstruction( | |||
1348 | MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp)); | |||
1349 | // FIXME: Add retpoline support and remove this. | |||
1350 | if (Subtarget->useRetpolineIndirectCalls()) | |||
1351 | report_fatal_error( | |||
1352 | "Lowering patchpoint with retpoline not yet implemented."); | |||
1353 | EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg)); | |||
1354 | } | |||
1355 | ||||
1356 | // Emit padding. | |||
1357 | unsigned NumBytes = opers.getNumPatchBytes(); | |||
1358 | assert(NumBytes >= EncodedBytes &&((NumBytes >= EncodedBytes && "Patchpoint can't request size less than the length of a call." ) ? static_cast<void> (0) : __assert_fail ("NumBytes >= EncodedBytes && \"Patchpoint can't request size less than the length of a call.\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1359, __PRETTY_FUNCTION__)) | |||
1359 | "Patchpoint can't request size less than the length of a call.")((NumBytes >= EncodedBytes && "Patchpoint can't request size less than the length of a call." ) ? static_cast<void> (0) : __assert_fail ("NumBytes >= EncodedBytes && \"Patchpoint can't request size less than the length of a call.\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1359, __PRETTY_FUNCTION__)); | |||
1360 | ||||
1361 | EmitNops(*OutStreamer, NumBytes - EncodedBytes, Subtarget->is64Bit(), | |||
1362 | getSubtargetInfo()); | |||
1363 | } | |||
1364 | ||||
1365 | void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, | |||
1366 | X86MCInstLower &MCIL) { | |||
1367 | assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64")((Subtarget->is64Bit() && "XRay custom events only supports X86-64" ) ? static_cast<void> (0) : __assert_fail ("Subtarget->is64Bit() && \"XRay custom events only supports X86-64\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1367, __PRETTY_FUNCTION__)); | |||
1368 | ||||
1369 | // We want to emit the following pattern, which follows the x86 calling | |||
1370 | // convention to prepare for the trampoline call to be patched in. | |||
1371 | // | |||
1372 | // .p2align 1, ... | |||
1373 | // .Lxray_event_sled_N: | |||
1374 | // jmp +N // jump across the instrumentation sled | |||
1375 | // ... // set up arguments in register | |||
1376 | // callq __xray_CustomEvent@plt // force dependency to symbol | |||
1377 | // ... | |||
1378 | // <jump here> | |||
1379 | // | |||
1380 | // After patching, it would look something like: | |||
1381 | // | |||
1382 | // nopw (2-byte nop) | |||
1383 | // ... | |||
1384 | // callq __xrayCustomEvent // already lowered | |||
1385 | // ... | |||
1386 | // | |||
1387 | // --- | |||
1388 | // First we emit the label and the jump. | |||
1389 | auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true); | |||
1390 | OutStreamer->AddComment("# XRay Custom Event Log"); | |||
1391 | OutStreamer->EmitCodeAlignment(2); | |||
1392 | OutStreamer->EmitLabel(CurSled); | |||
1393 | ||||
1394 | // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as | |||
1395 | // an operand (computed as an offset from the jmp instruction). | |||
1396 | // FIXME: Find another less hacky way do force the relative jump. | |||
1397 | OutStreamer->EmitBinaryData("\xeb\x0f"); | |||
1398 | ||||
1399 | // The default C calling convention will place two arguments into %rcx and | |||
1400 | // %rdx -- so we only work with those. | |||
1401 | const Register DestRegs[] = {X86::RDI, X86::RSI}; | |||
1402 | bool UsedMask[] = {false, false}; | |||
1403 | // Filled out in loop. | |||
1404 | Register SrcRegs[] = {0, 0}; | |||
1405 | ||||
1406 | // Then we put the operands in the %rdi and %rsi registers. We spill the | |||
1407 | // values in the register before we clobber them, and mark them as used in | |||
1408 | // UsedMask. In case the arguments are already in the correct register, we use | |||
1409 | // emit nops appropriately sized to keep the sled the same size in every | |||
1410 | // situation. | |||
1411 | for (unsigned I = 0; I < MI.getNumOperands(); ++I) | |||
1412 | if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) { | |||
1413 | assert(Op->isReg() && "Only support arguments in registers")((Op->isReg() && "Only support arguments in registers" ) ? static_cast<void> (0) : __assert_fail ("Op->isReg() && \"Only support arguments in registers\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1413, __PRETTY_FUNCTION__)); | |||
1414 | SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64); | |||
1415 | if (SrcRegs[I] != DestRegs[I]) { | |||
1416 | UsedMask[I] = true; | |||
1417 | EmitAndCountInstruction( | |||
1418 | MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I])); | |||
1419 | } else { | |||
1420 | EmitNops(*OutStreamer, 4, Subtarget->is64Bit(), getSubtargetInfo()); | |||
1421 | } | |||
1422 | } | |||
1423 | ||||
1424 | // Now that the register values are stashed, mov arguments into place. | |||
1425 | // FIXME: This doesn't work if one of the later SrcRegs is equal to an | |||
1426 | // earlier DestReg. We will have already overwritten over the register before | |||
1427 | // we can copy from it. | |||
1428 | for (unsigned I = 0; I < MI.getNumOperands(); ++I) | |||
1429 | if (SrcRegs[I] != DestRegs[I]) | |||
1430 | EmitAndCountInstruction( | |||
1431 | MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I])); | |||
1432 | ||||
1433 | // We emit a hard dependency on the __xray_CustomEvent symbol, which is the | |||
1434 | // name of the trampoline to be implemented by the XRay runtime. | |||
1435 | auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent"); | |||
1436 | MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym); | |||
1437 | if (isPositionIndependent()) | |||
1438 | TOp.setTargetFlags(X86II::MO_PLT); | |||
1439 | ||||
1440 | // Emit the call instruction. | |||
1441 | EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32) | |||
1442 | .addOperand(MCIL.LowerSymbolOperand(TOp, TSym))); | |||
1443 | ||||
1444 | // Restore caller-saved and used registers. | |||
1445 | for (unsigned I = sizeof UsedMask; I-- > 0;) | |||
1446 | if (UsedMask[I]) | |||
1447 | EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I])); | |||
1448 | else | |||
1449 | EmitNops(*OutStreamer, 1, Subtarget->is64Bit(), getSubtargetInfo()); | |||
1450 | ||||
1451 | OutStreamer->AddComment("xray custom event end."); | |||
1452 | ||||
1453 | // Record the sled version. Older versions of this sled were spelled | |||
1454 | // differently, so we let the runtime handle the different offsets we're | |||
1455 | // using. | |||
1456 | recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 1); | |||
1457 | } | |||
1458 | ||||
1459 | void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI, | |||
1460 | X86MCInstLower &MCIL) { | |||
1461 | assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64")((Subtarget->is64Bit() && "XRay typed events only supports X86-64" ) ? static_cast<void> (0) : __assert_fail ("Subtarget->is64Bit() && \"XRay typed events only supports X86-64\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1461, __PRETTY_FUNCTION__)); | |||
1462 | ||||
1463 | // We want to emit the following pattern, which follows the x86 calling | |||
1464 | // convention to prepare for the trampoline call to be patched in. | |||
1465 | // | |||
1466 | // .p2align 1, ... | |||
1467 | // .Lxray_event_sled_N: | |||
1468 | // jmp +N // jump across the instrumentation sled | |||
1469 | // ... // set up arguments in register | |||
1470 | // callq __xray_TypedEvent@plt // force dependency to symbol | |||
1471 | // ... | |||
1472 | // <jump here> | |||
1473 | // | |||
1474 | // After patching, it would look something like: | |||
1475 | // | |||
1476 | // nopw (2-byte nop) | |||
1477 | // ... | |||
1478 | // callq __xrayTypedEvent // already lowered | |||
1479 | // ... | |||
1480 | // | |||
1481 | // --- | |||
1482 | // First we emit the label and the jump. | |||
1483 | auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true); | |||
1484 | OutStreamer->AddComment("# XRay Typed Event Log"); | |||
1485 | OutStreamer->EmitCodeAlignment(2); | |||
1486 | OutStreamer->EmitLabel(CurSled); | |||
1487 | ||||
1488 | // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as | |||
1489 | // an operand (computed as an offset from the jmp instruction). | |||
1490 | // FIXME: Find another less hacky way do force the relative jump. | |||
1491 | OutStreamer->EmitBinaryData("\xeb\x14"); | |||
1492 | ||||
1493 | // An x86-64 convention may place three arguments into %rcx, %rdx, and R8, | |||
1494 | // so we'll work with those. Or we may be called via SystemV, in which case | |||
1495 | // we don't have to do any translation. | |||
1496 | const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX}; | |||
1497 | bool UsedMask[] = {false, false, false}; | |||
1498 | ||||
1499 | // Will fill out src regs in the loop. | |||
1500 | Register SrcRegs[] = {0, 0, 0}; | |||
1501 | ||||
1502 | // Then we put the operands in the SystemV registers. We spill the values in | |||
1503 | // the registers before we clobber them, and mark them as used in UsedMask. | |||
1504 | // In case the arguments are already in the correct register, we emit nops | |||
1505 | // appropriately sized to keep the sled the same size in every situation. | |||
1506 | for (unsigned I = 0; I < MI.getNumOperands(); ++I) | |||
1507 | if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) { | |||
1508 | // TODO: Is register only support adequate? | |||
1509 | assert(Op->isReg() && "Only supports arguments in registers")((Op->isReg() && "Only supports arguments in registers" ) ? static_cast<void> (0) : __assert_fail ("Op->isReg() && \"Only supports arguments in registers\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1509, __PRETTY_FUNCTION__)); | |||
1510 | SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64); | |||
1511 | if (SrcRegs[I] != DestRegs[I]) { | |||
1512 | UsedMask[I] = true; | |||
1513 | EmitAndCountInstruction( | |||
1514 | MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I])); | |||
1515 | } else { | |||
1516 | EmitNops(*OutStreamer, 4, Subtarget->is64Bit(), getSubtargetInfo()); | |||
1517 | } | |||
1518 | } | |||
1519 | ||||
1520 | // In the above loop we only stash all of the destination registers or emit | |||
1521 | // nops if the arguments are already in the right place. Doing the actually | |||
1522 | // moving is postponed until after all the registers are stashed so nothing | |||
1523 | // is clobbers. We've already added nops to account for the size of mov and | |||
1524 | // push if the register is in the right place, so we only have to worry about | |||
1525 | // emitting movs. | |||
1526 | // FIXME: This doesn't work if one of the later SrcRegs is equal to an | |||
1527 | // earlier DestReg. We will have already overwritten over the register before | |||
1528 | // we can copy from it. | |||
1529 | for (unsigned I = 0; I < MI.getNumOperands(); ++I) | |||
1530 | if (UsedMask[I]) | |||
1531 | EmitAndCountInstruction( | |||
1532 | MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I])); | |||
1533 | ||||
1534 | // We emit a hard dependency on the __xray_TypedEvent symbol, which is the | |||
1535 | // name of the trampoline to be implemented by the XRay runtime. | |||
1536 | auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent"); | |||
1537 | MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym); | |||
1538 | if (isPositionIndependent()) | |||
1539 | TOp.setTargetFlags(X86II::MO_PLT); | |||
1540 | ||||
1541 | // Emit the call instruction. | |||
1542 | EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32) | |||
1543 | .addOperand(MCIL.LowerSymbolOperand(TOp, TSym))); | |||
1544 | ||||
1545 | // Restore caller-saved and used registers. | |||
1546 | for (unsigned I = sizeof UsedMask; I-- > 0;) | |||
1547 | if (UsedMask[I]) | |||
1548 | EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I])); | |||
1549 | else | |||
1550 | EmitNops(*OutStreamer, 1, Subtarget->is64Bit(), getSubtargetInfo()); | |||
1551 | ||||
1552 | OutStreamer->AddComment("xray typed event end."); | |||
1553 | ||||
1554 | // Record the sled version. | |||
1555 | recordSled(CurSled, MI, SledKind::TYPED_EVENT, 0); | |||
1556 | } | |||
1557 | ||||
1558 | void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI, | |||
1559 | X86MCInstLower &MCIL) { | |||
1560 | // We want to emit the following pattern: | |||
1561 | // | |||
1562 | // .p2align 1, ... | |||
1563 | // .Lxray_sled_N: | |||
1564 | // jmp .tmpN | |||
1565 | // # 9 bytes worth of noops | |||
1566 | // | |||
1567 | // We need the 9 bytes because at runtime, we'd be patching over the full 11 | |||
1568 | // bytes with the following pattern: | |||
1569 | // | |||
1570 | // mov %r10, <function id, 32-bit> // 6 bytes | |||
1571 | // call <relative offset, 32-bits> // 5 bytes | |||
1572 | // | |||
1573 | auto CurSled = OutContext.createTempSymbol("xray_sled_", true); | |||
1574 | OutStreamer->EmitCodeAlignment(2); | |||
1575 | OutStreamer->EmitLabel(CurSled); | |||
1576 | ||||
1577 | // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as | |||
1578 | // an operand (computed as an offset from the jmp instruction). | |||
1579 | // FIXME: Find another less hacky way do force the relative jump. | |||
1580 | OutStreamer->EmitBytes("\xeb\x09"); | |||
1581 | EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo()); | |||
1582 | recordSled(CurSled, MI, SledKind::FUNCTION_ENTER); | |||
1583 | } | |||
1584 | ||||
1585 | void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI, | |||
1586 | X86MCInstLower &MCIL) { | |||
1587 | // Since PATCHABLE_RET takes the opcode of the return statement as an | |||
1588 | // argument, we use that to emit the correct form of the RET that we want. | |||
1589 | // i.e. when we see this: | |||
1590 | // | |||
1591 | // PATCHABLE_RET X86::RET ... | |||
1592 | // | |||
1593 | // We should emit the RET followed by sleds. | |||
1594 | // | |||
1595 | // .p2align 1, ... | |||
1596 | // .Lxray_sled_N: | |||
1597 | // ret # or equivalent instruction | |||
1598 | // # 10 bytes worth of noops | |||
1599 | // | |||
1600 | // This just makes sure that the alignment for the next instruction is 2. | |||
1601 | auto CurSled = OutContext.createTempSymbol("xray_sled_", true); | |||
1602 | OutStreamer->EmitCodeAlignment(2); | |||
1603 | OutStreamer->EmitLabel(CurSled); | |||
1604 | unsigned OpCode = MI.getOperand(0).getImm(); | |||
1605 | MCInst Ret; | |||
1606 | Ret.setOpcode(OpCode); | |||
1607 | for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end())) | |||
1608 | if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO)) | |||
1609 | Ret.addOperand(MaybeOperand.getValue()); | |||
1610 | OutStreamer->EmitInstruction(Ret, getSubtargetInfo()); | |||
1611 | EmitNops(*OutStreamer, 10, Subtarget->is64Bit(), getSubtargetInfo()); | |||
1612 | recordSled(CurSled, MI, SledKind::FUNCTION_EXIT); | |||
1613 | } | |||
1614 | ||||
1615 | void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, | |||
1616 | X86MCInstLower &MCIL) { | |||
1617 | // Like PATCHABLE_RET, we have the actual instruction in the operands to this | |||
1618 | // instruction so we lower that particular instruction and its operands. | |||
1619 | // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how | |||
1620 | // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to | |||
1621 | // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual | |||
1622 | // tail call much like how we have it in PATCHABLE_RET. | |||
1623 | auto CurSled = OutContext.createTempSymbol("xray_sled_", true); | |||
1624 | OutStreamer->EmitCodeAlignment(2); | |||
1625 | OutStreamer->EmitLabel(CurSled); | |||
1626 | auto Target = OutContext.createTempSymbol(); | |||
1627 | ||||
1628 | // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as | |||
1629 | // an operand (computed as an offset from the jmp instruction). | |||
1630 | // FIXME: Find another less hacky way do force the relative jump. | |||
1631 | OutStreamer->EmitBytes("\xeb\x09"); | |||
1632 | EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo()); | |||
1633 | OutStreamer->EmitLabel(Target); | |||
1634 | recordSled(CurSled, MI, SledKind::TAIL_CALL); | |||
1635 | ||||
1636 | unsigned OpCode = MI.getOperand(0).getImm(); | |||
1637 | OpCode = convertTailJumpOpcode(OpCode); | |||
1638 | MCInst TC; | |||
1639 | TC.setOpcode(OpCode); | |||
1640 | ||||
1641 | // Before emitting the instruction, add a comment to indicate that this is | |||
1642 | // indeed a tail call. | |||
1643 | OutStreamer->AddComment("TAILCALL"); | |||
1644 | for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end())) | |||
1645 | if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO)) | |||
1646 | TC.addOperand(MaybeOperand.getValue()); | |||
1647 | OutStreamer->EmitInstruction(TC, getSubtargetInfo()); | |||
1648 | } | |||
1649 | ||||
1650 | // Returns instruction preceding MBBI in MachineFunction. | |||
1651 | // If MBBI is the first instruction of the first basic block, returns null. | |||
1652 | static MachineBasicBlock::const_iterator | |||
1653 | PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) { | |||
1654 | const MachineBasicBlock *MBB = MBBI->getParent(); | |||
1655 | while (MBBI == MBB->begin()) { | |||
1656 | if (MBB == &MBB->getParent()->front()) | |||
1657 | return MachineBasicBlock::const_iterator(); | |||
1658 | MBB = MBB->getPrevNode(); | |||
1659 | MBBI = MBB->end(); | |||
1660 | } | |||
1661 | --MBBI; | |||
1662 | return MBBI; | |||
1663 | } | |||
1664 | ||||
1665 | static const Constant *getConstantFromPool(const MachineInstr &MI, | |||
1666 | const MachineOperand &Op) { | |||
1667 | if (!Op.isCPI() || Op.getOffset() != 0) | |||
1668 | return nullptr; | |||
1669 | ||||
1670 | ArrayRef<MachineConstantPoolEntry> Constants = | |||
1671 | MI.getParent()->getParent()->getConstantPool()->getConstants(); | |||
1672 | const MachineConstantPoolEntry &ConstantEntry = Constants[Op.getIndex()]; | |||
1673 | ||||
1674 | // Bail if this is a machine constant pool entry, we won't be able to dig out | |||
1675 | // anything useful. | |||
1676 | if (ConstantEntry.isMachineConstantPoolEntry()) | |||
1677 | return nullptr; | |||
1678 | ||||
1679 | const Constant *C = ConstantEntry.Val.ConstVal; | |||
1680 | assert((!C || ConstantEntry.getType() == C->getType()) &&(((!C || ConstantEntry.getType() == C->getType()) && "Expected a constant of the same type!") ? static_cast<void > (0) : __assert_fail ("(!C || ConstantEntry.getType() == C->getType()) && \"Expected a constant of the same type!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1681, __PRETTY_FUNCTION__)) | |||
1681 | "Expected a constant of the same type!")(((!C || ConstantEntry.getType() == C->getType()) && "Expected a constant of the same type!") ? static_cast<void > (0) : __assert_fail ("(!C || ConstantEntry.getType() == C->getType()) && \"Expected a constant of the same type!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1681, __PRETTY_FUNCTION__)); | |||
1682 | return C; | |||
1683 | } | |||
1684 | ||||
1685 | static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx, | |||
1686 | unsigned SrcOp2Idx, ArrayRef<int> Mask) { | |||
1687 | std::string Comment; | |||
1688 | ||||
1689 | // Compute the name for a register. This is really goofy because we have | |||
1690 | // multiple instruction printers that could (in theory) use different | |||
1691 | // names. Fortunately most people use the ATT style (outside of Windows) | |||
1692 | // and they actually agree on register naming here. Ultimately, this is | |||
1693 | // a comment, and so its OK if it isn't perfect. | |||
1694 | auto GetRegisterName = [](unsigned RegNum) -> StringRef { | |||
1695 | return X86ATTInstPrinter::getRegisterName(RegNum); | |||
1696 | }; | |||
1697 | ||||
1698 | const MachineOperand &DstOp = MI->getOperand(0); | |||
1699 | const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx); | |||
1700 | const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx); | |||
1701 | ||||
1702 | StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem"; | |||
1703 | StringRef Src1Name = | |||
1704 | SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem"; | |||
1705 | StringRef Src2Name = | |||
1706 | SrcOp2.isReg() ? GetRegisterName(SrcOp2.getReg()) : "mem"; | |||
1707 | ||||
1708 | // One source operand, fix the mask to print all elements in one span. | |||
1709 | SmallVector<int, 8> ShuffleMask(Mask.begin(), Mask.end()); | |||
1710 | if (Src1Name == Src2Name) | |||
1711 | for (int i = 0, e = ShuffleMask.size(); i != e; ++i) | |||
1712 | if (ShuffleMask[i] >= e) | |||
1713 | ShuffleMask[i] -= e; | |||
1714 | ||||
1715 | raw_string_ostream CS(Comment); | |||
1716 | CS << DstName; | |||
1717 | ||||
1718 | // Handle AVX512 MASK/MASXZ write mask comments. | |||
1719 | // MASK: zmmX {%kY} | |||
1720 | // MASKZ: zmmX {%kY} {z} | |||
1721 | if (SrcOp1Idx > 1) { | |||
1722 | assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask")(((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask" ) ? static_cast<void> (0) : __assert_fail ("(SrcOp1Idx == 2 || SrcOp1Idx == 3) && \"Unexpected writemask\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1722, __PRETTY_FUNCTION__)); | |||
1723 | ||||
1724 | const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1); | |||
1725 | if (WriteMaskOp.isReg()) { | |||
1726 | CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}"; | |||
1727 | ||||
1728 | if (SrcOp1Idx == 2) { | |||
1729 | CS << " {z}"; | |||
1730 | } | |||
1731 | } | |||
1732 | } | |||
1733 | ||||
1734 | CS << " = "; | |||
1735 | ||||
1736 | for (int i = 0, e = ShuffleMask.size(); i != e; ++i) { | |||
1737 | if (i != 0) | |||
1738 | CS << ","; | |||
1739 | if (ShuffleMask[i] == SM_SentinelZero) { | |||
1740 | CS << "zero"; | |||
1741 | continue; | |||
1742 | } | |||
1743 | ||||
1744 | // Otherwise, it must come from src1 or src2. Print the span of elements | |||
1745 | // that comes from this src. | |||
1746 | bool isSrc1 = ShuffleMask[i] < (int)e; | |||
1747 | CS << (isSrc1 ? Src1Name : Src2Name) << '['; | |||
1748 | ||||
1749 | bool IsFirst = true; | |||
1750 | while (i != e && ShuffleMask[i] != SM_SentinelZero && | |||
1751 | (ShuffleMask[i] < (int)e) == isSrc1) { | |||
1752 | if (!IsFirst) | |||
1753 | CS << ','; | |||
1754 | else | |||
1755 | IsFirst = false; | |||
1756 | if (ShuffleMask[i] == SM_SentinelUndef) | |||
1757 | CS << "u"; | |||
1758 | else | |||
1759 | CS << ShuffleMask[i] % (int)e; | |||
1760 | ++i; | |||
1761 | } | |||
1762 | CS << ']'; | |||
1763 | --i; // For loop increments element #. | |||
1764 | } | |||
1765 | CS.flush(); | |||
1766 | ||||
1767 | return Comment; | |||
1768 | } | |||
1769 | ||||
1770 | static void printConstant(const APInt &Val, raw_ostream &CS) { | |||
1771 | if (Val.getBitWidth() <= 64) { | |||
1772 | CS << Val.getZExtValue(); | |||
1773 | } else { | |||
1774 | // print multi-word constant as (w0,w1) | |||
1775 | CS << "("; | |||
1776 | for (int i = 0, N = Val.getNumWords(); i < N; ++i) { | |||
1777 | if (i > 0) | |||
1778 | CS << ","; | |||
1779 | CS << Val.getRawData()[i]; | |||
1780 | } | |||
1781 | CS << ")"; | |||
1782 | } | |||
1783 | } | |||
1784 | ||||
1785 | static void printConstant(const APFloat &Flt, raw_ostream &CS) { | |||
1786 | SmallString<32> Str; | |||
1787 | // Force scientific notation to distinquish from integers. | |||
1788 | Flt.toString(Str, 0, 0); | |||
1789 | CS << Str; | |||
1790 | } | |||
1791 | ||||
1792 | static void printConstant(const Constant *COp, raw_ostream &CS) { | |||
1793 | if (isa<UndefValue>(COp)) { | |||
1794 | CS << "u"; | |||
1795 | } else if (auto *CI = dyn_cast<ConstantInt>(COp)) { | |||
1796 | printConstant(CI->getValue(), CS); | |||
1797 | } else if (auto *CF = dyn_cast<ConstantFP>(COp)) { | |||
1798 | printConstant(CF->getValueAPF(), CS); | |||
1799 | } else { | |||
1800 | CS << "?"; | |||
1801 | } | |||
1802 | } | |||
1803 | ||||
1804 | void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) { | |||
1805 | assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?")((MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?" ) ? static_cast<void> (0) : __assert_fail ("MF->hasWinCFI() && \"SEH_ instruction in function without WinCFI?\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1805, __PRETTY_FUNCTION__)); | |||
1806 | assert(getSubtarget().isOSWindows() && "SEH_ instruction Windows only")((getSubtarget().isOSWindows() && "SEH_ instruction Windows only" ) ? static_cast<void> (0) : __assert_fail ("getSubtarget().isOSWindows() && \"SEH_ instruction Windows only\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1806, __PRETTY_FUNCTION__)); | |||
1807 | ||||
1808 | // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86. | |||
1809 | if (EmitFPOData) { | |||
1810 | X86TargetStreamer *XTS = | |||
1811 | static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer()); | |||
1812 | switch (MI->getOpcode()) { | |||
1813 | case X86::SEH_PushReg: | |||
1814 | XTS->emitFPOPushReg(MI->getOperand(0).getImm()); | |||
1815 | break; | |||
1816 | case X86::SEH_StackAlloc: | |||
1817 | XTS->emitFPOStackAlloc(MI->getOperand(0).getImm()); | |||
1818 | break; | |||
1819 | case X86::SEH_StackAlign: | |||
1820 | XTS->emitFPOStackAlign(MI->getOperand(0).getImm()); | |||
1821 | break; | |||
1822 | case X86::SEH_SetFrame: | |||
1823 | assert(MI->getOperand(1).getImm() == 0 &&((MI->getOperand(1).getImm() == 0 && ".cv_fpo_setframe takes no offset" ) ? static_cast<void> (0) : __assert_fail ("MI->getOperand(1).getImm() == 0 && \".cv_fpo_setframe takes no offset\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1824, __PRETTY_FUNCTION__)) | |||
1824 | ".cv_fpo_setframe takes no offset")((MI->getOperand(1).getImm() == 0 && ".cv_fpo_setframe takes no offset" ) ? static_cast<void> (0) : __assert_fail ("MI->getOperand(1).getImm() == 0 && \".cv_fpo_setframe takes no offset\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1824, __PRETTY_FUNCTION__)); | |||
1825 | XTS->emitFPOSetFrame(MI->getOperand(0).getImm()); | |||
1826 | break; | |||
1827 | case X86::SEH_EndPrologue: | |||
1828 | XTS->emitFPOEndPrologue(); | |||
1829 | break; | |||
1830 | case X86::SEH_SaveReg: | |||
1831 | case X86::SEH_SaveXMM: | |||
1832 | case X86::SEH_PushFrame: | |||
1833 | llvm_unreachable("SEH_ directive incompatible with FPO")::llvm::llvm_unreachable_internal("SEH_ directive incompatible with FPO" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1833); | |||
1834 | break; | |||
1835 | default: | |||
1836 | llvm_unreachable("expected SEH_ instruction")::llvm::llvm_unreachable_internal("expected SEH_ instruction" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1836); | |||
1837 | } | |||
1838 | return; | |||
1839 | } | |||
1840 | ||||
1841 | // Otherwise, use the .seh_ directives for all other Windows platforms. | |||
1842 | switch (MI->getOpcode()) { | |||
1843 | case X86::SEH_PushReg: | |||
1844 | OutStreamer->EmitWinCFIPushReg(MI->getOperand(0).getImm()); | |||
1845 | break; | |||
1846 | ||||
1847 | case X86::SEH_SaveReg: | |||
1848 | OutStreamer->EmitWinCFISaveReg(MI->getOperand(0).getImm(), | |||
1849 | MI->getOperand(1).getImm()); | |||
1850 | break; | |||
1851 | ||||
1852 | case X86::SEH_SaveXMM: | |||
1853 | OutStreamer->EmitWinCFISaveXMM(MI->getOperand(0).getImm(), | |||
1854 | MI->getOperand(1).getImm()); | |||
1855 | break; | |||
1856 | ||||
1857 | case X86::SEH_StackAlloc: | |||
1858 | OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm()); | |||
1859 | break; | |||
1860 | ||||
1861 | case X86::SEH_SetFrame: | |||
1862 | OutStreamer->EmitWinCFISetFrame(MI->getOperand(0).getImm(), | |||
1863 | MI->getOperand(1).getImm()); | |||
1864 | break; | |||
1865 | ||||
1866 | case X86::SEH_PushFrame: | |||
1867 | OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm()); | |||
1868 | break; | |||
1869 | ||||
1870 | case X86::SEH_EndPrologue: | |||
1871 | OutStreamer->EmitWinCFIEndProlog(); | |||
1872 | break; | |||
1873 | ||||
1874 | default: | |||
1875 | llvm_unreachable("expected SEH_ instruction")::llvm::llvm_unreachable_internal("expected SEH_ instruction" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1875); | |||
1876 | } | |||
1877 | } | |||
1878 | ||||
1879 | static unsigned getRegisterWidth(const MCOperandInfo &Info) { | |||
1880 | if (Info.RegClass == X86::VR128RegClassID || | |||
1881 | Info.RegClass == X86::VR128XRegClassID) | |||
1882 | return 128; | |||
1883 | if (Info.RegClass == X86::VR256RegClassID || | |||
1884 | Info.RegClass == X86::VR256XRegClassID) | |||
1885 | return 256; | |||
1886 | if (Info.RegClass == X86::VR512RegClassID) | |||
1887 | return 512; | |||
1888 | llvm_unreachable("Unknown register class!")::llvm::llvm_unreachable_internal("Unknown register class!", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1888); | |||
1889 | } | |||
1890 | ||||
1891 | void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { | |||
1892 | X86MCInstLower MCInstLowering(*MF, *this); | |||
1893 | const X86RegisterInfo *RI = | |||
1894 | MF->getSubtarget<X86Subtarget>().getRegisterInfo(); | |||
1895 | ||||
1896 | // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that | |||
1897 | // are compressed from EVEX encoding to VEX encoding. | |||
1898 | if (TM.Options.MCOptions.ShowMCEncoding) { | |||
1899 | if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX) | |||
1900 | OutStreamer->AddComment("EVEX TO VEX Compression ", false); | |||
1901 | } | |||
1902 | ||||
1903 | switch (MI->getOpcode()) { | |||
1904 | case TargetOpcode::DBG_VALUE: | |||
1905 | llvm_unreachable("Should be handled target independently")::llvm::llvm_unreachable_internal("Should be handled target independently" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1905); | |||
1906 | ||||
1907 | // Emit nothing here but a comment if we can. | |||
1908 | case X86::Int_MemBarrier: | |||
1909 | OutStreamer->emitRawComment("MEMBARRIER"); | |||
1910 | return; | |||
1911 | ||||
1912 | case X86::EH_RETURN: | |||
1913 | case X86::EH_RETURN64: { | |||
1914 | // Lower these as normal, but add some comments. | |||
1915 | Register Reg = MI->getOperand(0).getReg(); | |||
1916 | OutStreamer->AddComment(StringRef("eh_return, addr: %") + | |||
1917 | X86ATTInstPrinter::getRegisterName(Reg)); | |||
1918 | break; | |||
1919 | } | |||
1920 | case X86::CLEANUPRET: { | |||
1921 | // Lower these as normal, but add some comments. | |||
1922 | OutStreamer->AddComment("CLEANUPRET"); | |||
1923 | break; | |||
1924 | } | |||
1925 | ||||
1926 | case X86::CATCHRET: { | |||
1927 | // Lower these as normal, but add some comments. | |||
1928 | OutStreamer->AddComment("CATCHRET"); | |||
1929 | break; | |||
1930 | } | |||
1931 | ||||
1932 | case X86::TAILJMPr: | |||
1933 | case X86::TAILJMPm: | |||
1934 | case X86::TAILJMPd: | |||
1935 | case X86::TAILJMPd_CC: | |||
1936 | case X86::TAILJMPr64: | |||
1937 | case X86::TAILJMPm64: | |||
1938 | case X86::TAILJMPd64: | |||
1939 | case X86::TAILJMPd64_CC: | |||
1940 | case X86::TAILJMPr64_REX: | |||
1941 | case X86::TAILJMPm64_REX: | |||
1942 | // Lower these as normal, but add some comments. | |||
1943 | OutStreamer->AddComment("TAILCALL"); | |||
1944 | break; | |||
1945 | ||||
1946 | case X86::TLS_addr32: | |||
1947 | case X86::TLS_addr64: | |||
1948 | case X86::TLS_base_addr32: | |||
1949 | case X86::TLS_base_addr64: | |||
1950 | return LowerTlsAddr(MCInstLowering, *MI); | |||
1951 | ||||
1952 | // Loading/storing mask pairs requires two kmov operations. The second one of these | |||
1953 | // needs a 2 byte displacement relative to the specified address (with 32 bit spill | |||
1954 | // size). The pairs of 1bit masks up to 16 bit masks all use the same spill size, | |||
1955 | // they all are stored using MASKPAIR16STORE, loaded using MASKPAIR16LOAD. | |||
1956 | // | |||
1957 | // The displacement value might wrap around in theory, thus the asserts in both | |||
1958 | // cases. | |||
1959 | case X86::MASKPAIR16LOAD: { | |||
1960 | int64_t Disp = MI->getOperand(1 + X86::AddrDisp).getImm(); | |||
1961 | assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement")((Disp >= 0 && Disp <= (2147483647) - 2 && "Unexpected displacement") ? static_cast<void> (0) : __assert_fail ("Disp >= 0 && Disp <= INT32_MAX - 2 && \"Unexpected displacement\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1961, __PRETTY_FUNCTION__)); | |||
1962 | Register Reg = MI->getOperand(0).getReg(); | |||
1963 | Register Reg0 = RI->getSubReg(Reg, X86::sub_mask_0); | |||
1964 | Register Reg1 = RI->getSubReg(Reg, X86::sub_mask_1); | |||
1965 | ||||
1966 | // Load the first mask register | |||
1967 | MCInstBuilder MIB = MCInstBuilder(X86::KMOVWkm); | |||
1968 | MIB.addReg(Reg0); | |||
1969 | for (int i = 0; i < X86::AddrNumOperands; ++i) { | |||
1970 | auto Op = MCInstLowering.LowerMachineOperand(MI, MI->getOperand(1 + i)); | |||
1971 | MIB.addOperand(Op.getValue()); | |||
1972 | } | |||
1973 | EmitAndCountInstruction(MIB); | |||
1974 | ||||
1975 | // Load the second mask register of the pair | |||
1976 | MIB = MCInstBuilder(X86::KMOVWkm); | |||
1977 | MIB.addReg(Reg1); | |||
1978 | for (int i = 0; i < X86::AddrNumOperands; ++i) { | |||
1979 | if (i == X86::AddrDisp) { | |||
1980 | MIB.addImm(Disp + 2); | |||
1981 | } else { | |||
1982 | auto Op = MCInstLowering.LowerMachineOperand(MI, MI->getOperand(1 + i)); | |||
1983 | MIB.addOperand(Op.getValue()); | |||
1984 | } | |||
1985 | } | |||
1986 | EmitAndCountInstruction(MIB); | |||
1987 | return; | |||
1988 | } | |||
1989 | ||||
1990 | case X86::MASKPAIR16STORE: { | |||
1991 | int64_t Disp = MI->getOperand(X86::AddrDisp).getImm(); | |||
1992 | assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement")((Disp >= 0 && Disp <= (2147483647) - 2 && "Unexpected displacement") ? static_cast<void> (0) : __assert_fail ("Disp >= 0 && Disp <= INT32_MAX - 2 && \"Unexpected displacement\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 1992, __PRETTY_FUNCTION__)); | |||
1993 | Register Reg = MI->getOperand(X86::AddrNumOperands).getReg(); | |||
1994 | Register Reg0 = RI->getSubReg(Reg, X86::sub_mask_0); | |||
1995 | Register Reg1 = RI->getSubReg(Reg, X86::sub_mask_1); | |||
1996 | ||||
1997 | // Store the first mask register | |||
1998 | MCInstBuilder MIB = MCInstBuilder(X86::KMOVWmk); | |||
1999 | for (int i = 0; i < X86::AddrNumOperands; ++i) | |||
2000 | MIB.addOperand(MCInstLowering.LowerMachineOperand(MI, MI->getOperand(i)).getValue()); | |||
2001 | MIB.addReg(Reg0); | |||
2002 | EmitAndCountInstruction(MIB); | |||
2003 | ||||
2004 | // Store the second mask register of the pair | |||
2005 | MIB = MCInstBuilder(X86::KMOVWmk); | |||
2006 | for (int i = 0; i < X86::AddrNumOperands; ++i) { | |||
2007 | if (i == X86::AddrDisp) { | |||
2008 | MIB.addImm(Disp + 2); | |||
2009 | } else { | |||
2010 | auto Op = MCInstLowering.LowerMachineOperand(MI, MI->getOperand(0 + i)); | |||
2011 | MIB.addOperand(Op.getValue()); | |||
2012 | } | |||
2013 | } | |||
2014 | MIB.addReg(Reg1); | |||
2015 | EmitAndCountInstruction(MIB); | |||
2016 | return; | |||
2017 | } | |||
2018 | ||||
2019 | case X86::MOVPC32r: { | |||
2020 | // This is a pseudo op for a two instruction sequence with a label, which | |||
2021 | // looks like: | |||
2022 | // call "L1$pb" | |||
2023 | // "L1$pb": | |||
2024 | // popl %esi | |||
2025 | ||||
2026 | // Emit the call. | |||
2027 | MCSymbol *PICBase = MF->getPICBaseSymbol(); | |||
2028 | // FIXME: We would like an efficient form for this, so we don't have to do a | |||
2029 | // lot of extra uniquing. | |||
2030 | EmitAndCountInstruction( | |||
2031 | MCInstBuilder(X86::CALLpcrel32) | |||
2032 | .addExpr(MCSymbolRefExpr::create(PICBase, OutContext))); | |||
2033 | ||||
2034 | const X86FrameLowering *FrameLowering = | |||
2035 | MF->getSubtarget<X86Subtarget>().getFrameLowering(); | |||
2036 | bool hasFP = FrameLowering->hasFP(*MF); | |||
2037 | ||||
2038 | // TODO: This is needed only if we require precise CFA. | |||
2039 | bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() && | |||
2040 | !OutStreamer->getDwarfFrameInfos().back().End; | |||
2041 | ||||
2042 | int stackGrowth = -RI->getSlotSize(); | |||
2043 | ||||
2044 | if (HasActiveDwarfFrame && !hasFP) { | |||
2045 | OutStreamer->EmitCFIAdjustCfaOffset(-stackGrowth); | |||
2046 | } | |||
2047 | ||||
2048 | // Emit the label. | |||
2049 | OutStreamer->EmitLabel(PICBase); | |||
2050 | ||||
2051 | // popl $reg | |||
2052 | EmitAndCountInstruction( | |||
2053 | MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg())); | |||
2054 | ||||
2055 | if (HasActiveDwarfFrame && !hasFP) { | |||
2056 | OutStreamer->EmitCFIAdjustCfaOffset(stackGrowth); | |||
2057 | } | |||
2058 | return; | |||
2059 | } | |||
2060 | ||||
2061 | case X86::ADD32ri: { | |||
2062 | // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri. | |||
2063 | if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS) | |||
2064 | break; | |||
2065 | ||||
2066 | // Okay, we have something like: | |||
2067 | // EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL) | |||
2068 | ||||
2069 | // For this, we want to print something like: | |||
2070 | // MYGLOBAL + (. - PICBASE) | |||
2071 | // However, we can't generate a ".", so just emit a new label here and refer | |||
2072 | // to it. | |||
2073 | MCSymbol *DotSym = OutContext.createTempSymbol(); | |||
2074 | OutStreamer->EmitLabel(DotSym); | |||
2075 | ||||
2076 | // Now that we have emitted the label, lower the complex operand expression. | |||
2077 | MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2)); | |||
2078 | ||||
2079 | const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext); | |||
2080 | const MCExpr *PICBase = | |||
2081 | MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext); | |||
2082 | DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext); | |||
2083 | ||||
2084 | DotExpr = MCBinaryExpr::createAdd( | |||
2085 | MCSymbolRefExpr::create(OpSym, OutContext), DotExpr, OutContext); | |||
2086 | ||||
2087 | EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri) | |||
2088 | .addReg(MI->getOperand(0).getReg()) | |||
2089 | .addReg(MI->getOperand(1).getReg()) | |||
2090 | .addExpr(DotExpr)); | |||
2091 | return; | |||
2092 | } | |||
2093 | case TargetOpcode::STATEPOINT: | |||
2094 | return LowerSTATEPOINT(*MI, MCInstLowering); | |||
2095 | ||||
2096 | case TargetOpcode::FAULTING_OP: | |||
2097 | return LowerFAULTING_OP(*MI, MCInstLowering); | |||
2098 | ||||
2099 | case TargetOpcode::FENTRY_CALL: | |||
2100 | return LowerFENTRY_CALL(*MI, MCInstLowering); | |||
2101 | ||||
2102 | case TargetOpcode::PATCHABLE_OP: | |||
2103 | return LowerPATCHABLE_OP(*MI, MCInstLowering); | |||
2104 | ||||
2105 | case TargetOpcode::STACKMAP: | |||
2106 | return LowerSTACKMAP(*MI); | |||
2107 | ||||
2108 | case TargetOpcode::PATCHPOINT: | |||
2109 | return LowerPATCHPOINT(*MI, MCInstLowering); | |||
2110 | ||||
2111 | case TargetOpcode::PATCHABLE_FUNCTION_ENTER: | |||
2112 | return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering); | |||
2113 | ||||
2114 | case TargetOpcode::PATCHABLE_RET: | |||
2115 | return LowerPATCHABLE_RET(*MI, MCInstLowering); | |||
2116 | ||||
2117 | case TargetOpcode::PATCHABLE_TAIL_CALL: | |||
2118 | return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering); | |||
2119 | ||||
2120 | case TargetOpcode::PATCHABLE_EVENT_CALL: | |||
2121 | return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering); | |||
2122 | ||||
2123 | case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL: | |||
2124 | return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering); | |||
2125 | ||||
2126 | case X86::MORESTACK_RET: | |||
2127 | EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); | |||
2128 | return; | |||
2129 | ||||
2130 | case X86::MORESTACK_RET_RESTORE_R10: | |||
2131 | // Return, then restore R10. | |||
2132 | EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); | |||
2133 | EmitAndCountInstruction( | |||
2134 | MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX)); | |||
2135 | return; | |||
2136 | ||||
2137 | case X86::SEH_PushReg: | |||
2138 | case X86::SEH_SaveReg: | |||
2139 | case X86::SEH_SaveXMM: | |||
2140 | case X86::SEH_StackAlloc: | |||
2141 | case X86::SEH_StackAlign: | |||
2142 | case X86::SEH_SetFrame: | |||
2143 | case X86::SEH_PushFrame: | |||
2144 | case X86::SEH_EndPrologue: | |||
2145 | EmitSEHInstruction(MI); | |||
2146 | return; | |||
2147 | ||||
2148 | case X86::SEH_Epilogue: { | |||
2149 | assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?")((MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?" ) ? static_cast<void> (0) : __assert_fail ("MF->hasWinCFI() && \"SEH_ instruction in function without WinCFI?\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2149, __PRETTY_FUNCTION__)); | |||
2150 | MachineBasicBlock::const_iterator MBBI(MI); | |||
2151 | // Check if preceded by a call and emit nop if so. | |||
2152 | for (MBBI = PrevCrossBBInst(MBBI); | |||
2153 | MBBI != MachineBasicBlock::const_iterator(); | |||
2154 | MBBI = PrevCrossBBInst(MBBI)) { | |||
2155 | // Conservatively assume that pseudo instructions don't emit code and keep | |||
2156 | // looking for a call. We may emit an unnecessary nop in some cases. | |||
2157 | if (!MBBI->isPseudo()) { | |||
2158 | if (MBBI->isCall()) | |||
2159 | EmitAndCountInstruction(MCInstBuilder(X86::NOOP)); | |||
2160 | break; | |||
2161 | } | |||
2162 | } | |||
2163 | return; | |||
2164 | } | |||
2165 | ||||
2166 | // Lower PSHUFB and VPERMILP normally but add a comment if we can find | |||
2167 | // a constant shuffle mask. We won't be able to do this at the MC layer | |||
2168 | // because the mask isn't an immediate. | |||
2169 | case X86::PSHUFBrm: | |||
2170 | case X86::VPSHUFBrm: | |||
2171 | case X86::VPSHUFBYrm: | |||
2172 | case X86::VPSHUFBZ128rm: | |||
2173 | case X86::VPSHUFBZ128rmk: | |||
2174 | case X86::VPSHUFBZ128rmkz: | |||
2175 | case X86::VPSHUFBZ256rm: | |||
2176 | case X86::VPSHUFBZ256rmk: | |||
2177 | case X86::VPSHUFBZ256rmkz: | |||
2178 | case X86::VPSHUFBZrm: | |||
2179 | case X86::VPSHUFBZrmk: | |||
2180 | case X86::VPSHUFBZrmkz: { | |||
2181 | if (!OutStreamer->isVerboseAsm()) | |||
2182 | break; | |||
2183 | unsigned SrcIdx, MaskIdx; | |||
2184 | switch (MI->getOpcode()) { | |||
2185 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2185); | |||
2186 | case X86::PSHUFBrm: | |||
2187 | case X86::VPSHUFBrm: | |||
2188 | case X86::VPSHUFBYrm: | |||
2189 | case X86::VPSHUFBZ128rm: | |||
2190 | case X86::VPSHUFBZ256rm: | |||
2191 | case X86::VPSHUFBZrm: | |||
2192 | SrcIdx = 1; MaskIdx = 5; break; | |||
2193 | case X86::VPSHUFBZ128rmkz: | |||
2194 | case X86::VPSHUFBZ256rmkz: | |||
2195 | case X86::VPSHUFBZrmkz: | |||
2196 | SrcIdx = 2; MaskIdx = 6; break; | |||
2197 | case X86::VPSHUFBZ128rmk: | |||
2198 | case X86::VPSHUFBZ256rmk: | |||
2199 | case X86::VPSHUFBZrmk: | |||
2200 | SrcIdx = 3; MaskIdx = 7; break; | |||
2201 | } | |||
2202 | ||||
2203 | assert(MI->getNumOperands() >= 6 &&((MI->getNumOperands() >= 6 && "We should always have at least 6 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 6 && \"We should always have at least 6 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2204, __PRETTY_FUNCTION__)) | |||
2204 | "We should always have at least 6 operands!")((MI->getNumOperands() >= 6 && "We should always have at least 6 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 6 && \"We should always have at least 6 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2204, __PRETTY_FUNCTION__)); | |||
2205 | ||||
2206 | const MachineOperand &MaskOp = MI->getOperand(MaskIdx); | |||
2207 | if (auto *C = getConstantFromPool(*MI, MaskOp)) { | |||
2208 | unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]); | |||
2209 | SmallVector<int, 64> Mask; | |||
2210 | DecodePSHUFBMask(C, Width, Mask); | |||
2211 | if (!Mask.empty()) | |||
2212 | OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); | |||
2213 | } | |||
2214 | break; | |||
2215 | } | |||
2216 | ||||
2217 | case X86::VPERMILPSrm: | |||
2218 | case X86::VPERMILPSYrm: | |||
2219 | case X86::VPERMILPSZ128rm: | |||
2220 | case X86::VPERMILPSZ128rmk: | |||
2221 | case X86::VPERMILPSZ128rmkz: | |||
2222 | case X86::VPERMILPSZ256rm: | |||
2223 | case X86::VPERMILPSZ256rmk: | |||
2224 | case X86::VPERMILPSZ256rmkz: | |||
2225 | case X86::VPERMILPSZrm: | |||
2226 | case X86::VPERMILPSZrmk: | |||
2227 | case X86::VPERMILPSZrmkz: | |||
2228 | case X86::VPERMILPDrm: | |||
2229 | case X86::VPERMILPDYrm: | |||
2230 | case X86::VPERMILPDZ128rm: | |||
2231 | case X86::VPERMILPDZ128rmk: | |||
2232 | case X86::VPERMILPDZ128rmkz: | |||
2233 | case X86::VPERMILPDZ256rm: | |||
2234 | case X86::VPERMILPDZ256rmk: | |||
2235 | case X86::VPERMILPDZ256rmkz: | |||
2236 | case X86::VPERMILPDZrm: | |||
2237 | case X86::VPERMILPDZrmk: | |||
2238 | case X86::VPERMILPDZrmkz: { | |||
2239 | if (!OutStreamer->isVerboseAsm()) | |||
2240 | break; | |||
2241 | unsigned SrcIdx, MaskIdx; | |||
2242 | unsigned ElSize; | |||
2243 | switch (MI->getOpcode()) { | |||
2244 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2244); | |||
2245 | case X86::VPERMILPSrm: | |||
2246 | case X86::VPERMILPSYrm: | |||
2247 | case X86::VPERMILPSZ128rm: | |||
2248 | case X86::VPERMILPSZ256rm: | |||
2249 | case X86::VPERMILPSZrm: | |||
2250 | SrcIdx = 1; MaskIdx = 5; ElSize = 32; break; | |||
2251 | case X86::VPERMILPSZ128rmkz: | |||
2252 | case X86::VPERMILPSZ256rmkz: | |||
2253 | case X86::VPERMILPSZrmkz: | |||
2254 | SrcIdx = 2; MaskIdx = 6; ElSize = 32; break; | |||
2255 | case X86::VPERMILPSZ128rmk: | |||
2256 | case X86::VPERMILPSZ256rmk: | |||
2257 | case X86::VPERMILPSZrmk: | |||
2258 | SrcIdx = 3; MaskIdx = 7; ElSize = 32; break; | |||
2259 | case X86::VPERMILPDrm: | |||
2260 | case X86::VPERMILPDYrm: | |||
2261 | case X86::VPERMILPDZ128rm: | |||
2262 | case X86::VPERMILPDZ256rm: | |||
2263 | case X86::VPERMILPDZrm: | |||
2264 | SrcIdx = 1; MaskIdx = 5; ElSize = 64; break; | |||
2265 | case X86::VPERMILPDZ128rmkz: | |||
2266 | case X86::VPERMILPDZ256rmkz: | |||
2267 | case X86::VPERMILPDZrmkz: | |||
2268 | SrcIdx = 2; MaskIdx = 6; ElSize = 64; break; | |||
2269 | case X86::VPERMILPDZ128rmk: | |||
2270 | case X86::VPERMILPDZ256rmk: | |||
2271 | case X86::VPERMILPDZrmk: | |||
2272 | SrcIdx = 3; MaskIdx = 7; ElSize = 64; break; | |||
2273 | } | |||
2274 | ||||
2275 | assert(MI->getNumOperands() >= 6 &&((MI->getNumOperands() >= 6 && "We should always have at least 6 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 6 && \"We should always have at least 6 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2276, __PRETTY_FUNCTION__)) | |||
2276 | "We should always have at least 6 operands!")((MI->getNumOperands() >= 6 && "We should always have at least 6 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 6 && \"We should always have at least 6 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2276, __PRETTY_FUNCTION__)); | |||
2277 | ||||
2278 | const MachineOperand &MaskOp = MI->getOperand(MaskIdx); | |||
2279 | if (auto *C = getConstantFromPool(*MI, MaskOp)) { | |||
2280 | unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]); | |||
2281 | SmallVector<int, 16> Mask; | |||
2282 | DecodeVPERMILPMask(C, ElSize, Width, Mask); | |||
2283 | if (!Mask.empty()) | |||
2284 | OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); | |||
2285 | } | |||
2286 | break; | |||
2287 | } | |||
2288 | ||||
2289 | case X86::VPERMIL2PDrm: | |||
2290 | case X86::VPERMIL2PSrm: | |||
2291 | case X86::VPERMIL2PDYrm: | |||
2292 | case X86::VPERMIL2PSYrm: { | |||
2293 | if (!OutStreamer->isVerboseAsm()) | |||
2294 | break; | |||
2295 | assert(MI->getNumOperands() >= 8 &&((MI->getNumOperands() >= 8 && "We should always have at least 8 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 8 && \"We should always have at least 8 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2296, __PRETTY_FUNCTION__)) | |||
2296 | "We should always have at least 8 operands!")((MI->getNumOperands() >= 8 && "We should always have at least 8 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 8 && \"We should always have at least 8 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2296, __PRETTY_FUNCTION__)); | |||
2297 | ||||
2298 | const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1); | |||
2299 | if (!CtrlOp.isImm()) | |||
2300 | break; | |||
2301 | ||||
2302 | unsigned ElSize; | |||
2303 | switch (MI->getOpcode()) { | |||
2304 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2304); | |||
2305 | case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break; | |||
2306 | case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break; | |||
2307 | } | |||
2308 | ||||
2309 | const MachineOperand &MaskOp = MI->getOperand(6); | |||
2310 | if (auto *C = getConstantFromPool(*MI, MaskOp)) { | |||
2311 | unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]); | |||
2312 | SmallVector<int, 16> Mask; | |||
2313 | DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask); | |||
2314 | if (!Mask.empty()) | |||
2315 | OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask)); | |||
2316 | } | |||
2317 | break; | |||
2318 | } | |||
2319 | ||||
2320 | case X86::VPPERMrrm: { | |||
2321 | if (!OutStreamer->isVerboseAsm()) | |||
2322 | break; | |||
2323 | assert(MI->getNumOperands() >= 7 &&((MI->getNumOperands() >= 7 && "We should always have at least 7 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 7 && \"We should always have at least 7 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2324, __PRETTY_FUNCTION__)) | |||
2324 | "We should always have at least 7 operands!")((MI->getNumOperands() >= 7 && "We should always have at least 7 operands!" ) ? static_cast<void> (0) : __assert_fail ("MI->getNumOperands() >= 7 && \"We should always have at least 7 operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2324, __PRETTY_FUNCTION__)); | |||
2325 | ||||
2326 | const MachineOperand &MaskOp = MI->getOperand(6); | |||
2327 | if (auto *C = getConstantFromPool(*MI, MaskOp)) { | |||
2328 | unsigned Width = getRegisterWidth(MI->getDesc().OpInfo[0]); | |||
2329 | SmallVector<int, 16> Mask; | |||
2330 | DecodeVPPERMMask(C, Width, Mask); | |||
2331 | if (!Mask.empty()) | |||
2332 | OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask)); | |||
2333 | } | |||
2334 | break; | |||
2335 | } | |||
2336 | ||||
2337 | case X86::MMX_MOVQ64rm: { | |||
2338 | if (!OutStreamer->isVerboseAsm()) | |||
2339 | break; | |||
2340 | if (MI->getNumOperands() <= 4) | |||
2341 | break; | |||
2342 | if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { | |||
2343 | std::string Comment; | |||
2344 | raw_string_ostream CS(Comment); | |||
2345 | const MachineOperand &DstOp = MI->getOperand(0); | |||
2346 | CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = "; | |||
2347 | if (auto *CF = dyn_cast<ConstantFP>(C)) { | |||
2348 | CS << "0x" << CF->getValueAPF().bitcastToAPInt().toString(16, false); | |||
2349 | OutStreamer->AddComment(CS.str()); | |||
2350 | } | |||
2351 | } | |||
2352 | break; | |||
2353 | } | |||
2354 | ||||
2355 | #define MOV_CASE(Prefix, Suffix)case X86::PrefixMOVAPDSuffixrm: case X86::PrefixMOVAPSSuffixrm : case X86::PrefixMOVUPDSuffixrm: case X86::PrefixMOVUPSSuffixrm : case X86::PrefixMOVDQASuffixrm: case X86::PrefixMOVDQUSuffixrm : \ | |||
2356 | case X86::Prefix##MOVAPD##Suffix##rm: \ | |||
2357 | case X86::Prefix##MOVAPS##Suffix##rm: \ | |||
2358 | case X86::Prefix##MOVUPD##Suffix##rm: \ | |||
2359 | case X86::Prefix##MOVUPS##Suffix##rm: \ | |||
2360 | case X86::Prefix##MOVDQA##Suffix##rm: \ | |||
2361 | case X86::Prefix##MOVDQU##Suffix##rm: | |||
2362 | ||||
2363 | #define MOV_AVX512_CASE(Suffix)case X86::VMOVDQA64Suffixrm: case X86::VMOVDQA32Suffixrm: case X86::VMOVDQU64Suffixrm: case X86::VMOVDQU32Suffixrm: case X86 ::VMOVDQU16Suffixrm: case X86::VMOVDQU8Suffixrm: case X86::VMOVAPSSuffixrm : case X86::VMOVAPDSuffixrm: case X86::VMOVUPSSuffixrm: case X86 ::VMOVUPDSuffixrm: \ | |||
2364 | case X86::VMOVDQA64##Suffix##rm: \ | |||
2365 | case X86::VMOVDQA32##Suffix##rm: \ | |||
2366 | case X86::VMOVDQU64##Suffix##rm: \ | |||
2367 | case X86::VMOVDQU32##Suffix##rm: \ | |||
2368 | case X86::VMOVDQU16##Suffix##rm: \ | |||
2369 | case X86::VMOVDQU8##Suffix##rm: \ | |||
2370 | case X86::VMOVAPS##Suffix##rm: \ | |||
2371 | case X86::VMOVAPD##Suffix##rm: \ | |||
2372 | case X86::VMOVUPS##Suffix##rm: \ | |||
2373 | case X86::VMOVUPD##Suffix##rm: | |||
2374 | ||||
2375 | #define CASE_ALL_MOV_RM()case X86::MOVAPDrm: case X86::MOVAPSrm: case X86::MOVUPDrm: case X86::MOVUPSrm: case X86::MOVDQArm: case X86::MOVDQUrm: case X86 ::VMOVAPDrm: case X86::VMOVAPSrm: case X86::VMOVUPDrm: case X86 ::VMOVUPSrm: case X86::VMOVDQArm: case X86::VMOVDQUrm: case X86 ::VMOVAPDYrm: case X86::VMOVAPSYrm: case X86::VMOVUPDYrm: case X86::VMOVUPSYrm: case X86::VMOVDQAYrm: case X86::VMOVDQUYrm: case X86::VMOVDQA64Zrm: case X86::VMOVDQA32Zrm: case X86::VMOVDQU64Zrm : case X86::VMOVDQU32Zrm: case X86::VMOVDQU16Zrm: case X86::VMOVDQU8Zrm : case X86::VMOVAPSZrm: case X86::VMOVAPDZrm: case X86::VMOVUPSZrm : case X86::VMOVUPDZrm: case X86::VMOVDQA64Z256rm: case X86:: VMOVDQA32Z256rm: case X86::VMOVDQU64Z256rm: case X86::VMOVDQU32Z256rm : case X86::VMOVDQU16Z256rm: case X86::VMOVDQU8Z256rm: case X86 ::VMOVAPSZ256rm: case X86::VMOVAPDZ256rm: case X86::VMOVUPSZ256rm : case X86::VMOVUPDZ256rm: case X86::VMOVDQA64Z128rm: case X86 ::VMOVDQA32Z128rm: case X86::VMOVDQU64Z128rm: case X86::VMOVDQU32Z128rm : case X86::VMOVDQU16Z128rm: case X86::VMOVDQU8Z128rm: case X86 ::VMOVAPSZ128rm: case X86::VMOVAPDZ128rm: case X86::VMOVUPSZ128rm : case X86::VMOVUPDZ128rm: \ | |||
2376 | MOV_CASE(, )case X86::MOVAPDrm: case X86::MOVAPSrm: case X86::MOVUPDrm: case X86::MOVUPSrm: case X86::MOVDQArm: case X86::MOVDQUrm: /* SSE */ \ | |||
2377 | MOV_CASE(V, )case X86::VMOVAPDrm: case X86::VMOVAPSrm: case X86::VMOVUPDrm : case X86::VMOVUPSrm: case X86::VMOVDQArm: case X86::VMOVDQUrm : /* AVX-128 */ \ | |||
2378 | MOV_CASE(V, Y)case X86::VMOVAPDYrm: case X86::VMOVAPSYrm: case X86::VMOVUPDYrm : case X86::VMOVUPSYrm: case X86::VMOVDQAYrm: case X86::VMOVDQUYrm : /* AVX-256 */ \ | |||
2379 | MOV_AVX512_CASE(Z)case X86::VMOVDQA64Zrm: case X86::VMOVDQA32Zrm: case X86::VMOVDQU64Zrm : case X86::VMOVDQU32Zrm: case X86::VMOVDQU16Zrm: case X86::VMOVDQU8Zrm : case X86::VMOVAPSZrm: case X86::VMOVAPDZrm: case X86::VMOVUPSZrm : case X86::VMOVUPDZrm: \ | |||
2380 | MOV_AVX512_CASE(Z256)case X86::VMOVDQA64Z256rm: case X86::VMOVDQA32Z256rm: case X86 ::VMOVDQU64Z256rm: case X86::VMOVDQU32Z256rm: case X86::VMOVDQU16Z256rm : case X86::VMOVDQU8Z256rm: case X86::VMOVAPSZ256rm: case X86 ::VMOVAPDZ256rm: case X86::VMOVUPSZ256rm: case X86::VMOVUPDZ256rm : \ | |||
2381 | MOV_AVX512_CASE(Z128)case X86::VMOVDQA64Z128rm: case X86::VMOVDQA32Z128rm: case X86 ::VMOVDQU64Z128rm: case X86::VMOVDQU32Z128rm: case X86::VMOVDQU16Z128rm : case X86::VMOVDQU8Z128rm: case X86::VMOVAPSZ128rm: case X86 ::VMOVAPDZ128rm: case X86::VMOVUPSZ128rm: case X86::VMOVUPDZ128rm : | |||
2382 | ||||
2383 | // For loads from a constant pool to a vector register, print the constant | |||
2384 | // loaded. | |||
2385 | CASE_ALL_MOV_RM()case X86::MOVAPDrm: case X86::MOVAPSrm: case X86::MOVUPDrm: case X86::MOVUPSrm: case X86::MOVDQArm: case X86::MOVDQUrm: case X86 ::VMOVAPDrm: case X86::VMOVAPSrm: case X86::VMOVUPDrm: case X86 ::VMOVUPSrm: case X86::VMOVDQArm: case X86::VMOVDQUrm: case X86 ::VMOVAPDYrm: case X86::VMOVAPSYrm: case X86::VMOVUPDYrm: case X86::VMOVUPSYrm: case X86::VMOVDQAYrm: case X86::VMOVDQUYrm: case X86::VMOVDQA64Zrm: case X86::VMOVDQA32Zrm: case X86::VMOVDQU64Zrm : case X86::VMOVDQU32Zrm: case X86::VMOVDQU16Zrm: case X86::VMOVDQU8Zrm : case X86::VMOVAPSZrm: case X86::VMOVAPDZrm: case X86::VMOVUPSZrm : case X86::VMOVUPDZrm: case X86::VMOVDQA64Z256rm: case X86:: VMOVDQA32Z256rm: case X86::VMOVDQU64Z256rm: case X86::VMOVDQU32Z256rm : case X86::VMOVDQU16Z256rm: case X86::VMOVDQU8Z256rm: case X86 ::VMOVAPSZ256rm: case X86::VMOVAPDZ256rm: case X86::VMOVUPSZ256rm : case X86::VMOVUPDZ256rm: case X86::VMOVDQA64Z128rm: case X86 ::VMOVDQA32Z128rm: case X86::VMOVDQU64Z128rm: case X86::VMOVDQU32Z128rm : case X86::VMOVDQU16Z128rm: case X86::VMOVDQU8Z128rm: case X86 ::VMOVAPSZ128rm: case X86::VMOVAPDZ128rm: case X86::VMOVUPSZ128rm : case X86::VMOVUPDZ128rm: | |||
2386 | case X86::VBROADCASTF128: | |||
2387 | case X86::VBROADCASTI128: | |||
2388 | case X86::VBROADCASTF32X4Z256rm: | |||
2389 | case X86::VBROADCASTF32X4rm: | |||
2390 | case X86::VBROADCASTF32X8rm: | |||
2391 | case X86::VBROADCASTF64X2Z128rm: | |||
2392 | case X86::VBROADCASTF64X2rm: | |||
2393 | case X86::VBROADCASTF64X4rm: | |||
2394 | case X86::VBROADCASTI32X4Z256rm: | |||
2395 | case X86::VBROADCASTI32X4rm: | |||
2396 | case X86::VBROADCASTI32X8rm: | |||
2397 | case X86::VBROADCASTI64X2Z128rm: | |||
2398 | case X86::VBROADCASTI64X2rm: | |||
2399 | case X86::VBROADCASTI64X4rm: | |||
2400 | if (!OutStreamer->isVerboseAsm()) | |||
2401 | break; | |||
2402 | if (MI->getNumOperands() <= 4) | |||
2403 | break; | |||
2404 | if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { | |||
2405 | int NumLanes = 1; | |||
2406 | // Override NumLanes for the broadcast instructions. | |||
2407 | switch (MI->getOpcode()) { | |||
2408 | case X86::VBROADCASTF128: NumLanes = 2; break; | |||
2409 | case X86::VBROADCASTI128: NumLanes = 2; break; | |||
2410 | case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; break; | |||
2411 | case X86::VBROADCASTF32X4rm: NumLanes = 4; break; | |||
2412 | case X86::VBROADCASTF32X8rm: NumLanes = 2; break; | |||
2413 | case X86::VBROADCASTF64X2Z128rm: NumLanes = 2; break; | |||
2414 | case X86::VBROADCASTF64X2rm: NumLanes = 4; break; | |||
2415 | case X86::VBROADCASTF64X4rm: NumLanes = 2; break; | |||
2416 | case X86::VBROADCASTI32X4Z256rm: NumLanes = 2; break; | |||
2417 | case X86::VBROADCASTI32X4rm: NumLanes = 4; break; | |||
2418 | case X86::VBROADCASTI32X8rm: NumLanes = 2; break; | |||
2419 | case X86::VBROADCASTI64X2Z128rm: NumLanes = 2; break; | |||
2420 | case X86::VBROADCASTI64X2rm: NumLanes = 4; break; | |||
2421 | case X86::VBROADCASTI64X4rm: NumLanes = 2; break; | |||
2422 | } | |||
2423 | ||||
2424 | std::string Comment; | |||
2425 | raw_string_ostream CS(Comment); | |||
2426 | const MachineOperand &DstOp = MI->getOperand(0); | |||
2427 | CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = "; | |||
2428 | if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) { | |||
2429 | CS << "["; | |||
2430 | for (int l = 0; l != NumLanes; ++l) { | |||
2431 | for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements; | |||
2432 | ++i) { | |||
2433 | if (i != 0 || l != 0) | |||
2434 | CS << ","; | |||
2435 | if (CDS->getElementType()->isIntegerTy()) | |||
2436 | printConstant(CDS->getElementAsAPInt(i), CS); | |||
2437 | else if (CDS->getElementType()->isHalfTy() || | |||
2438 | CDS->getElementType()->isFloatTy() || | |||
2439 | CDS->getElementType()->isDoubleTy()) | |||
2440 | printConstant(CDS->getElementAsAPFloat(i), CS); | |||
2441 | else | |||
2442 | CS << "?"; | |||
2443 | } | |||
2444 | } | |||
2445 | CS << "]"; | |||
2446 | OutStreamer->AddComment(CS.str()); | |||
2447 | } else if (auto *CV = dyn_cast<ConstantVector>(C)) { | |||
2448 | CS << "<"; | |||
2449 | for (int l = 0; l != NumLanes; ++l) { | |||
2450 | for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands; | |||
2451 | ++i) { | |||
2452 | if (i != 0 || l != 0) | |||
2453 | CS << ","; | |||
2454 | printConstant(CV->getOperand(i), CS); | |||
2455 | } | |||
2456 | } | |||
2457 | CS << ">"; | |||
2458 | OutStreamer->AddComment(CS.str()); | |||
2459 | } | |||
2460 | } | |||
2461 | break; | |||
2462 | case X86::MOVDDUPrm: | |||
2463 | case X86::VMOVDDUPrm: | |||
2464 | case X86::VMOVDDUPZ128rm: | |||
2465 | case X86::VBROADCASTSSrm: | |||
2466 | case X86::VBROADCASTSSYrm: | |||
2467 | case X86::VBROADCASTSSZ128m: | |||
2468 | case X86::VBROADCASTSSZ256m: | |||
2469 | case X86::VBROADCASTSSZm: | |||
2470 | case X86::VBROADCASTSDYrm: | |||
2471 | case X86::VBROADCASTSDZ256m: | |||
2472 | case X86::VBROADCASTSDZm: | |||
2473 | case X86::VPBROADCASTBrm: | |||
2474 | case X86::VPBROADCASTBYrm: | |||
2475 | case X86::VPBROADCASTBZ128m: | |||
2476 | case X86::VPBROADCASTBZ256m: | |||
2477 | case X86::VPBROADCASTBZm: | |||
2478 | case X86::VPBROADCASTDrm: | |||
2479 | case X86::VPBROADCASTDYrm: | |||
2480 | case X86::VPBROADCASTDZ128m: | |||
2481 | case X86::VPBROADCASTDZ256m: | |||
2482 | case X86::VPBROADCASTDZm: | |||
2483 | case X86::VPBROADCASTQrm: | |||
2484 | case X86::VPBROADCASTQYrm: | |||
2485 | case X86::VPBROADCASTQZ128m: | |||
2486 | case X86::VPBROADCASTQZ256m: | |||
2487 | case X86::VPBROADCASTQZm: | |||
2488 | case X86::VPBROADCASTWrm: | |||
2489 | case X86::VPBROADCASTWYrm: | |||
2490 | case X86::VPBROADCASTWZ128m: | |||
2491 | case X86::VPBROADCASTWZ256m: | |||
2492 | case X86::VPBROADCASTWZm: | |||
2493 | if (!OutStreamer->isVerboseAsm()) | |||
2494 | break; | |||
2495 | if (MI->getNumOperands() <= 4) | |||
2496 | break; | |||
2497 | if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { | |||
2498 | int NumElts; | |||
2499 | switch (MI->getOpcode()) { | |||
2500 | default: llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "/build/llvm-toolchain-snapshot-10~++20200106111110+0efc9e5a8cc/llvm/lib/Target/X86/X86MCInstLower.cpp" , 2500); | |||
2501 | case X86::MOVDDUPrm: NumElts = 2; break; | |||
2502 | case X86::VMOVDDUPrm: NumElts = 2; break; | |||
2503 | case X86::VMOVDDUPZ128rm: NumElts = 2; break; | |||
2504 | case X86::VBROADCASTSSrm: NumElts = 4; break; | |||
2505 | case X86::VBROADCASTSSYrm: NumElts = 8; break; | |||
2506 | case X86::VBROADCASTSSZ128m: NumElts = 4; break; | |||
2507 | case X86::VBROADCASTSSZ256m: NumElts = 8; break; | |||
2508 | case X86::VBROADCASTSSZm: NumElts = 16; break; | |||
2509 | case X86::VBROADCASTSDYrm: NumElts = 4; break; | |||
2510 | case X86::VBROADCASTSDZ256m: NumElts = 4; break; | |||
2511 | case X86::VBROADCASTSDZm: NumElts = 8; break; | |||
2512 | case X86::VPBROADCASTBrm: NumElts = 16; break; | |||
2513 | case X86::VPBROADCASTBYrm: NumElts = 32; break; | |||
2514 | case X86::VPBROADCASTBZ128m: NumElts = 16; break; | |||
2515 | case X86::VPBROADCASTBZ256m: NumElts = 32; break; | |||
2516 | case X86::VPBROADCASTBZm: NumElts = 64; break; | |||
2517 | case X86::VPBROADCASTDrm: NumElts = 4; break; | |||
2518 | case X86::VPBROADCASTDYrm: NumElts = 8; break; | |||
2519 | case X86::VPBROADCASTDZ128m: NumElts = 4; break; | |||
2520 | case X86::VPBROADCASTDZ256m: NumElts = 8; break; | |||
2521 | case X86::VPBROADCASTDZm: NumElts = 16; break; | |||
2522 | case X86::VPBROADCASTQrm: NumElts = 2; break; | |||
2523 | case X86::VPBROADCASTQYrm: NumElts = 4; break; | |||
2524 | case X86::VPBROADCASTQZ128m: NumElts = 2; break; | |||
2525 | case X86::VPBROADCASTQZ256m: NumElts = 4; break; | |||
2526 | case X86::VPBROADCASTQZm: NumElts = 8; break; | |||
2527 | case X86::VPBROADCASTWrm: NumElts = 8; break; | |||
2528 | case X86::VPBROADCASTWYrm: NumElts = 16; break; | |||
2529 | case X86::VPBROADCASTWZ128m: NumElts = 8; break; | |||
2530 | case X86::VPBROADCASTWZ256m: NumElts = 16; break; | |||
2531 | case X86::VPBROADCASTWZm: NumElts = 32; break; | |||
2532 | } | |||
2533 | ||||
2534 | std::string Comment; | |||
2535 | raw_string_ostream CS(Comment); | |||
2536 | const MachineOperand &DstOp = MI->getOperand(0); | |||
2537 | CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = "; | |||
2538 | CS << "["; | |||
2539 | for (int i = 0; i != NumElts; ++i) { | |||
2540 | if (i != 0) | |||
2541 | CS << ","; | |||
2542 | printConstant(C, CS); | |||
2543 | } | |||
2544 | CS << "]"; | |||
2545 | OutStreamer->AddComment(CS.str()); | |||
2546 | } | |||
2547 | } | |||
2548 | ||||
2549 | MCInst TmpInst; | |||
2550 | MCInstLowering.Lower(MI, TmpInst); | |||
2551 | ||||
2552 | // Stackmap shadows cannot include branch targets, so we can count the bytes | |||
2553 | // in a call towards the shadow, but must ensure that the no thread returns | |||
2554 | // in to the stackmap shadow. The only way to achieve this is if the call | |||
2555 | // is at the end of the shadow. | |||
2556 | if (MI->isCall()) { | |||
2557 | // Count then size of the call towards the shadow | |||
2558 | SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get()); | |||
2559 | // Then flush the shadow so that we fill with nops before the call, not | |||
2560 | // after it. | |||
2561 | SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); | |||
2562 | // Then emit the call | |||
2563 | OutStreamer->EmitInstruction(TmpInst, getSubtargetInfo()); | |||
2564 | return; | |||
2565 | } | |||
2566 | ||||
2567 | EmitAndCountInstruction(TmpInst); | |||
2568 | } |