File: | llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp |
Warning: | line 893, column 7 6th function call argument is an uninitialized value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==// | ||||||
2 | // | ||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
6 | // | ||||||
7 | //===----------------------------------------------------------------------===// | ||||||
8 | /// \file | ||||||
9 | /// This file implements the targeting of the InstructionSelector class for | ||||||
10 | /// AArch64. | ||||||
11 | /// \todo This should be generated by TableGen. | ||||||
12 | //===----------------------------------------------------------------------===// | ||||||
13 | |||||||
14 | #include "AArch64InstrInfo.h" | ||||||
15 | #include "AArch64MachineFunctionInfo.h" | ||||||
16 | #include "AArch64RegisterBankInfo.h" | ||||||
17 | #include "AArch64RegisterInfo.h" | ||||||
18 | #include "AArch64Subtarget.h" | ||||||
19 | #include "AArch64TargetMachine.h" | ||||||
20 | #include "MCTargetDesc/AArch64AddressingModes.h" | ||||||
21 | #include "MCTargetDesc/AArch64MCTargetDesc.h" | ||||||
22 | #include "llvm/ADT/Optional.h" | ||||||
23 | #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" | ||||||
24 | #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" | ||||||
25 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" | ||||||
26 | #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" | ||||||
27 | #include "llvm/CodeGen/GlobalISel/Utils.h" | ||||||
28 | #include "llvm/CodeGen/MachineBasicBlock.h" | ||||||
29 | #include "llvm/CodeGen/MachineConstantPool.h" | ||||||
30 | #include "llvm/CodeGen/MachineFunction.h" | ||||||
31 | #include "llvm/CodeGen/MachineInstr.h" | ||||||
32 | #include "llvm/CodeGen/MachineInstrBuilder.h" | ||||||
33 | #include "llvm/CodeGen/MachineOperand.h" | ||||||
34 | #include "llvm/CodeGen/MachineRegisterInfo.h" | ||||||
35 | #include "llvm/CodeGen/TargetOpcodes.h" | ||||||
36 | #include "llvm/IR/Constants.h" | ||||||
37 | #include "llvm/IR/Instructions.h" | ||||||
38 | #include "llvm/IR/PatternMatch.h" | ||||||
39 | #include "llvm/IR/Type.h" | ||||||
40 | #include "llvm/IR/IntrinsicsAArch64.h" | ||||||
41 | #include "llvm/Pass.h" | ||||||
42 | #include "llvm/Support/Debug.h" | ||||||
43 | #include "llvm/Support/raw_ostream.h" | ||||||
44 | |||||||
45 | #define DEBUG_TYPE"aarch64-isel" "aarch64-isel" | ||||||
46 | |||||||
47 | using namespace llvm; | ||||||
48 | using namespace MIPatternMatch; | ||||||
49 | |||||||
50 | namespace { | ||||||
51 | |||||||
52 | #define GET_GLOBALISEL_PREDICATE_BITSET | ||||||
53 | #include "AArch64GenGlobalISel.inc" | ||||||
54 | #undef GET_GLOBALISEL_PREDICATE_BITSET | ||||||
55 | |||||||
56 | class AArch64InstructionSelector : public InstructionSelector { | ||||||
57 | public: | ||||||
58 | AArch64InstructionSelector(const AArch64TargetMachine &TM, | ||||||
59 | const AArch64Subtarget &STI, | ||||||
60 | const AArch64RegisterBankInfo &RBI); | ||||||
61 | |||||||
62 | bool select(MachineInstr &I) override; | ||||||
63 | static const char *getName() { return DEBUG_TYPE"aarch64-isel"; } | ||||||
64 | |||||||
65 | void setupMF(MachineFunction &MF, GISelKnownBits &KB, | ||||||
66 | CodeGenCoverage &CoverageInfo) override { | ||||||
67 | InstructionSelector::setupMF(MF, KB, CoverageInfo); | ||||||
68 | |||||||
69 | // hasFnAttribute() is expensive to call on every BRCOND selection, so | ||||||
70 | // cache it here for each run of the selector. | ||||||
71 | ProduceNonFlagSettingCondBr = | ||||||
72 | !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening); | ||||||
73 | MFReturnAddr = Register(); | ||||||
74 | |||||||
75 | processPHIs(MF); | ||||||
76 | } | ||||||
77 | |||||||
78 | private: | ||||||
79 | /// tblgen-erated 'select' implementation, used as the initial selector for | ||||||
80 | /// the patterns that don't require complex C++. | ||||||
81 | bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; | ||||||
82 | |||||||
83 | // A lowering phase that runs before any selection attempts. | ||||||
84 | // Returns true if the instruction was modified. | ||||||
85 | bool preISelLower(MachineInstr &I); | ||||||
86 | |||||||
87 | // An early selection function that runs before the selectImpl() call. | ||||||
88 | bool earlySelect(MachineInstr &I) const; | ||||||
89 | |||||||
90 | // Do some preprocessing of G_PHIs before we begin selection. | ||||||
91 | void processPHIs(MachineFunction &MF); | ||||||
92 | |||||||
93 | bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
94 | |||||||
95 | /// Eliminate same-sized cross-bank copies into stores before selectImpl(). | ||||||
96 | bool contractCrossBankCopyIntoStore(MachineInstr &I, | ||||||
97 | MachineRegisterInfo &MRI); | ||||||
98 | |||||||
99 | bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI); | ||||||
100 | |||||||
101 | bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF, | ||||||
102 | MachineRegisterInfo &MRI) const; | ||||||
103 | bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF, | ||||||
104 | MachineRegisterInfo &MRI) const; | ||||||
105 | |||||||
106 | ///@{ | ||||||
107 | /// Helper functions for selectCompareBranch. | ||||||
108 | bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp, | ||||||
109 | MachineIRBuilder &MIB) const; | ||||||
110 | bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp, | ||||||
111 | MachineIRBuilder &MIB) const; | ||||||
112 | bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp, | ||||||
113 | MachineIRBuilder &MIB) const; | ||||||
114 | bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert, | ||||||
115 | MachineBasicBlock *DstMBB, | ||||||
116 | MachineIRBuilder &MIB) const; | ||||||
117 | ///@} | ||||||
118 | |||||||
119 | bool selectCompareBranch(MachineInstr &I, MachineFunction &MF, | ||||||
120 | MachineRegisterInfo &MRI) const; | ||||||
121 | |||||||
122 | bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
123 | bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
124 | |||||||
125 | // Helper to generate an equivalent of scalar_to_vector into a new register, | ||||||
126 | // returned via 'Dst'. | ||||||
127 | MachineInstr *emitScalarToVector(unsigned EltSize, | ||||||
128 | const TargetRegisterClass *DstRC, | ||||||
129 | Register Scalar, | ||||||
130 | MachineIRBuilder &MIRBuilder) const; | ||||||
131 | |||||||
132 | /// Emit a lane insert into \p DstReg, or a new vector register if None is | ||||||
133 | /// provided. | ||||||
134 | /// | ||||||
135 | /// The lane inserted into is defined by \p LaneIdx. The vector source | ||||||
136 | /// register is given by \p SrcReg. The register containing the element is | ||||||
137 | /// given by \p EltReg. | ||||||
138 | MachineInstr *emitLaneInsert(Optional<Register> DstReg, Register SrcReg, | ||||||
139 | Register EltReg, unsigned LaneIdx, | ||||||
140 | const RegisterBank &RB, | ||||||
141 | MachineIRBuilder &MIRBuilder) const; | ||||||
142 | bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
143 | bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy, | ||||||
144 | MachineRegisterInfo &MRI) const; | ||||||
145 | bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
146 | bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
147 | bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
148 | |||||||
149 | bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
150 | bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
151 | bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
152 | bool selectSplitVectorUnmerge(MachineInstr &I, | ||||||
153 | MachineRegisterInfo &MRI) const; | ||||||
154 | bool selectIntrinsicWithSideEffects(MachineInstr &I, | ||||||
155 | MachineRegisterInfo &MRI) const; | ||||||
156 | bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI); | ||||||
157 | bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
158 | bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
159 | bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
160 | bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
161 | bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
162 | bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
163 | bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI) const; | ||||||
164 | |||||||
165 | unsigned emitConstantPoolEntry(const Constant *CPVal, | ||||||
166 | MachineFunction &MF) const; | ||||||
167 | MachineInstr *emitLoadFromConstantPool(const Constant *CPVal, | ||||||
168 | MachineIRBuilder &MIRBuilder) const; | ||||||
169 | |||||||
170 | // Emit a vector concat operation. | ||||||
171 | MachineInstr *emitVectorConcat(Optional<Register> Dst, Register Op1, | ||||||
172 | Register Op2, | ||||||
173 | MachineIRBuilder &MIRBuilder) const; | ||||||
174 | |||||||
175 | // Emit an integer compare between LHS and RHS, which checks for Predicate. | ||||||
176 | MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS, | ||||||
177 | MachineOperand &Predicate, | ||||||
178 | MachineIRBuilder &MIRBuilder) const; | ||||||
179 | |||||||
180 | /// Emit a floating point comparison between \p LHS and \p RHS. | ||||||
181 | /// \p Pred if given is the intended predicate to use. | ||||||
182 | MachineInstr *emitFPCompare(Register LHS, Register RHS, | ||||||
183 | MachineIRBuilder &MIRBuilder, | ||||||
184 | Optional<CmpInst::Predicate> = None) const; | ||||||
185 | |||||||
186 | MachineInstr *emitInstr(unsigned Opcode, | ||||||
187 | std::initializer_list<llvm::DstOp> DstOps, | ||||||
188 | std::initializer_list<llvm::SrcOp> SrcOps, | ||||||
189 | MachineIRBuilder &MIRBuilder, | ||||||
190 | const ComplexRendererFns &RenderFns = None) const; | ||||||
191 | /// Helper function to emit an add or sub instruction. | ||||||
192 | /// | ||||||
193 | /// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above | ||||||
194 | /// in a specific order. | ||||||
195 | /// | ||||||
196 | /// Below is an example of the expected input to \p AddrModeAndSizeToOpcode. | ||||||
197 | /// | ||||||
198 | /// \code | ||||||
199 | /// const std::array<std::array<unsigned, 2>, 4> Table { | ||||||
200 | /// {{AArch64::ADDXri, AArch64::ADDWri}, | ||||||
201 | /// {AArch64::ADDXrs, AArch64::ADDWrs}, | ||||||
202 | /// {AArch64::ADDXrr, AArch64::ADDWrr}, | ||||||
203 | /// {AArch64::SUBXri, AArch64::SUBWri}, | ||||||
204 | /// {AArch64::ADDXrx, AArch64::ADDWrx}}}; | ||||||
205 | /// \endcode | ||||||
206 | /// | ||||||
207 | /// Each row in the table corresponds to a different addressing mode. Each | ||||||
208 | /// column corresponds to a different register size. | ||||||
209 | /// | ||||||
210 | /// \attention Rows must be structured as follows: | ||||||
211 | /// - Row 0: The ri opcode variants | ||||||
212 | /// - Row 1: The rs opcode variants | ||||||
213 | /// - Row 2: The rr opcode variants | ||||||
214 | /// - Row 3: The ri opcode variants for negative immediates | ||||||
215 | /// - Row 4: The rx opcode variants | ||||||
216 | /// | ||||||
217 | /// \attention Columns must be structured as follows: | ||||||
218 | /// - Column 0: The 64-bit opcode variants | ||||||
219 | /// - Column 1: The 32-bit opcode variants | ||||||
220 | /// | ||||||
221 | /// \p Dst is the destination register of the binop to emit. | ||||||
222 | /// \p LHS is the left-hand operand of the binop to emit. | ||||||
223 | /// \p RHS is the right-hand operand of the binop to emit. | ||||||
224 | MachineInstr *emitAddSub( | ||||||
225 | const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode, | ||||||
226 | Register Dst, MachineOperand &LHS, MachineOperand &RHS, | ||||||
227 | MachineIRBuilder &MIRBuilder) const; | ||||||
228 | MachineInstr *emitADD(Register DefReg, MachineOperand &LHS, | ||||||
229 | MachineOperand &RHS, | ||||||
230 | MachineIRBuilder &MIRBuilder) const; | ||||||
231 | MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS, | ||||||
232 | MachineIRBuilder &MIRBuilder) const; | ||||||
233 | MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS, | ||||||
234 | MachineIRBuilder &MIRBuilder) const; | ||||||
235 | MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS, | ||||||
236 | MachineIRBuilder &MIRBuilder) const; | ||||||
237 | MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS, | ||||||
238 | MachineIRBuilder &MIRBuilder) const; | ||||||
239 | MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS, | ||||||
240 | AArch64CC::CondCode CC, | ||||||
241 | MachineIRBuilder &MIRBuilder) const; | ||||||
242 | MachineInstr *emitExtractVectorElt(Optional<Register> DstReg, | ||||||
243 | const RegisterBank &DstRB, LLT ScalarTy, | ||||||
244 | Register VecReg, unsigned LaneIdx, | ||||||
245 | MachineIRBuilder &MIRBuilder) const; | ||||||
246 | |||||||
247 | /// Helper function for selecting G_FCONSTANT. If the G_FCONSTANT can be | ||||||
248 | /// materialized using a FMOV instruction, then update MI and return it. | ||||||
249 | /// Otherwise, do nothing and return a nullptr. | ||||||
250 | MachineInstr *emitFMovForFConstant(MachineInstr &MI, | ||||||
251 | MachineRegisterInfo &MRI) const; | ||||||
252 | |||||||
253 | /// Emit a CSet for an integer compare. | ||||||
254 | /// | ||||||
255 | /// \p DefReg is expected to be a 32-bit scalar register. | ||||||
256 | MachineInstr *emitCSetForICMP(Register DefReg, unsigned Pred, | ||||||
257 | MachineIRBuilder &MIRBuilder) const; | ||||||
258 | /// Emit a CSet for a FP compare. | ||||||
259 | /// | ||||||
260 | /// \p Dst is expected to be a 32-bit scalar register. | ||||||
261 | MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred, | ||||||
262 | MachineIRBuilder &MIRBuilder) const; | ||||||
263 | |||||||
264 | /// Emit the overflow op for \p Opcode. | ||||||
265 | /// | ||||||
266 | /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO, | ||||||
267 | /// G_USUBO, etc. | ||||||
268 | std::pair<MachineInstr *, AArch64CC::CondCode> | ||||||
269 | emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS, | ||||||
270 | MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const; | ||||||
271 | |||||||
272 | /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg. | ||||||
273 | /// \p IsNegative is true if the test should be "not zero". | ||||||
274 | /// This will also optimize the test bit instruction when possible. | ||||||
275 | MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative, | ||||||
276 | MachineBasicBlock *DstMBB, | ||||||
277 | MachineIRBuilder &MIB) const; | ||||||
278 | |||||||
279 | /// Emit a CB(N)Z instruction which branches to \p DestMBB. | ||||||
280 | MachineInstr *emitCBZ(Register CompareReg, bool IsNegative, | ||||||
281 | MachineBasicBlock *DestMBB, | ||||||
282 | MachineIRBuilder &MIB) const; | ||||||
283 | |||||||
284 | // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td. | ||||||
285 | // We use these manually instead of using the importer since it doesn't | ||||||
286 | // support SDNodeXForm. | ||||||
287 | ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const; | ||||||
288 | ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const; | ||||||
289 | ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const; | ||||||
290 | ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const; | ||||||
291 | |||||||
292 | ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const; | ||||||
293 | ComplexRendererFns selectArithImmed(MachineOperand &Root) const; | ||||||
294 | ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const; | ||||||
295 | |||||||
296 | ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root, | ||||||
297 | unsigned Size) const; | ||||||
298 | |||||||
299 | ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const { | ||||||
300 | return selectAddrModeUnscaled(Root, 1); | ||||||
301 | } | ||||||
302 | ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const { | ||||||
303 | return selectAddrModeUnscaled(Root, 2); | ||||||
304 | } | ||||||
305 | ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const { | ||||||
306 | return selectAddrModeUnscaled(Root, 4); | ||||||
307 | } | ||||||
308 | ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const { | ||||||
309 | return selectAddrModeUnscaled(Root, 8); | ||||||
310 | } | ||||||
311 | ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const { | ||||||
312 | return selectAddrModeUnscaled(Root, 16); | ||||||
313 | } | ||||||
314 | |||||||
315 | /// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used | ||||||
316 | /// from complex pattern matchers like selectAddrModeIndexed(). | ||||||
317 | ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size, | ||||||
318 | MachineRegisterInfo &MRI) const; | ||||||
319 | |||||||
320 | ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root, | ||||||
321 | unsigned Size) const; | ||||||
322 | template <int Width> | ||||||
323 | ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const { | ||||||
324 | return selectAddrModeIndexed(Root, Width / 8); | ||||||
325 | } | ||||||
326 | |||||||
327 | bool isWorthFoldingIntoExtendedReg(MachineInstr &MI, | ||||||
328 | const MachineRegisterInfo &MRI) const; | ||||||
329 | ComplexRendererFns | ||||||
330 | selectAddrModeShiftedExtendXReg(MachineOperand &Root, | ||||||
331 | unsigned SizeInBytes) const; | ||||||
332 | |||||||
333 | /// Returns a \p ComplexRendererFns which contains a base, offset, and whether | ||||||
334 | /// or not a shift + extend should be folded into an addressing mode. Returns | ||||||
335 | /// None when this is not profitable or possible. | ||||||
336 | ComplexRendererFns | ||||||
337 | selectExtendedSHL(MachineOperand &Root, MachineOperand &Base, | ||||||
338 | MachineOperand &Offset, unsigned SizeInBytes, | ||||||
339 | bool WantsExt) const; | ||||||
340 | ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const; | ||||||
341 | ComplexRendererFns selectAddrModeXRO(MachineOperand &Root, | ||||||
342 | unsigned SizeInBytes) const; | ||||||
343 | template <int Width> | ||||||
344 | ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const { | ||||||
345 | return selectAddrModeXRO(Root, Width / 8); | ||||||
346 | } | ||||||
347 | |||||||
348 | ComplexRendererFns selectAddrModeWRO(MachineOperand &Root, | ||||||
349 | unsigned SizeInBytes) const; | ||||||
350 | template <int Width> | ||||||
351 | ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const { | ||||||
352 | return selectAddrModeWRO(Root, Width / 8); | ||||||
353 | } | ||||||
354 | |||||||
355 | ComplexRendererFns selectShiftedRegister(MachineOperand &Root) const; | ||||||
356 | |||||||
357 | ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const { | ||||||
358 | return selectShiftedRegister(Root); | ||||||
359 | } | ||||||
360 | |||||||
361 | ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const { | ||||||
362 | // TODO: selectShiftedRegister should allow for rotates on logical shifts. | ||||||
363 | // For now, make them the same. The only difference between the two is that | ||||||
364 | // logical shifts are allowed to fold in rotates. Otherwise, these are | ||||||
365 | // functionally the same. | ||||||
366 | return selectShiftedRegister(Root); | ||||||
367 | } | ||||||
368 | |||||||
369 | /// Given an extend instruction, determine the correct shift-extend type for | ||||||
370 | /// that instruction. | ||||||
371 | /// | ||||||
372 | /// If the instruction is going to be used in a load or store, pass | ||||||
373 | /// \p IsLoadStore = true. | ||||||
374 | AArch64_AM::ShiftExtendType | ||||||
375 | getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI, | ||||||
376 | bool IsLoadStore = false) const; | ||||||
377 | |||||||
378 | /// Move \p Reg to \p RC if \p Reg is not already on \p RC. | ||||||
379 | /// | ||||||
380 | /// \returns Either \p Reg if no change was necessary, or the new register | ||||||
381 | /// created by moving \p Reg. | ||||||
382 | /// | ||||||
383 | /// Note: This uses emitCopy right now. | ||||||
384 | Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC, | ||||||
385 | MachineIRBuilder &MIB) const; | ||||||
386 | |||||||
387 | ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const; | ||||||
388 | |||||||
389 | void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI, | ||||||
390 | int OpIdx = -1) const; | ||||||
391 | void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I, | ||||||
392 | int OpIdx = -1) const; | ||||||
393 | void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I, | ||||||
394 | int OpIdx = -1) const; | ||||||
395 | |||||||
396 | // Materialize a GlobalValue or BlockAddress using a movz+movk sequence. | ||||||
397 | void materializeLargeCMVal(MachineInstr &I, const Value *V, | ||||||
398 | unsigned OpFlags) const; | ||||||
399 | |||||||
400 | // Optimization methods. | ||||||
401 | bool tryOptSelect(MachineInstr &MI) const; | ||||||
402 | MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS, | ||||||
403 | MachineOperand &Predicate, | ||||||
404 | MachineIRBuilder &MIRBuilder) const; | ||||||
405 | |||||||
406 | /// Return true if \p MI is a load or store of \p NumBytes bytes. | ||||||
407 | bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const; | ||||||
408 | |||||||
409 | /// Returns true if \p MI is guaranteed to have the high-half of a 64-bit | ||||||
410 | /// register zeroed out. In other words, the result of MI has been explicitly | ||||||
411 | /// zero extended. | ||||||
412 | bool isDef32(const MachineInstr &MI) const; | ||||||
413 | |||||||
414 | const AArch64TargetMachine &TM; | ||||||
415 | const AArch64Subtarget &STI; | ||||||
416 | const AArch64InstrInfo &TII; | ||||||
417 | const AArch64RegisterInfo &TRI; | ||||||
418 | const AArch64RegisterBankInfo &RBI; | ||||||
419 | |||||||
420 | bool ProduceNonFlagSettingCondBr = false; | ||||||
421 | |||||||
422 | // Some cached values used during selection. | ||||||
423 | // We use LR as a live-in register, and we keep track of it here as it can be | ||||||
424 | // clobbered by calls. | ||||||
425 | Register MFReturnAddr; | ||||||
426 | |||||||
427 | #define GET_GLOBALISEL_PREDICATES_DECL | ||||||
428 | #include "AArch64GenGlobalISel.inc" | ||||||
429 | #undef GET_GLOBALISEL_PREDICATES_DECL | ||||||
430 | |||||||
431 | // We declare the temporaries used by selectImpl() in the class to minimize the | ||||||
432 | // cost of constructing placeholder values. | ||||||
433 | #define GET_GLOBALISEL_TEMPORARIES_DECL | ||||||
434 | #include "AArch64GenGlobalISel.inc" | ||||||
435 | #undef GET_GLOBALISEL_TEMPORARIES_DECL | ||||||
436 | }; | ||||||
437 | |||||||
438 | } // end anonymous namespace | ||||||
439 | |||||||
440 | #define GET_GLOBALISEL_IMPL | ||||||
441 | #include "AArch64GenGlobalISel.inc" | ||||||
442 | #undef GET_GLOBALISEL_IMPL | ||||||
443 | |||||||
444 | AArch64InstructionSelector::AArch64InstructionSelector( | ||||||
445 | const AArch64TargetMachine &TM, const AArch64Subtarget &STI, | ||||||
446 | const AArch64RegisterBankInfo &RBI) | ||||||
447 | : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), | ||||||
448 | TRI(*STI.getRegisterInfo()), RBI(RBI), | ||||||
449 | #define GET_GLOBALISEL_PREDICATES_INIT | ||||||
450 | #include "AArch64GenGlobalISel.inc" | ||||||
451 | #undef GET_GLOBALISEL_PREDICATES_INIT | ||||||
452 | #define GET_GLOBALISEL_TEMPORARIES_INIT | ||||||
453 | #include "AArch64GenGlobalISel.inc" | ||||||
454 | #undef GET_GLOBALISEL_TEMPORARIES_INIT | ||||||
455 | { | ||||||
456 | } | ||||||
457 | |||||||
458 | // FIXME: This should be target-independent, inferred from the types declared | ||||||
459 | // for each class in the bank. | ||||||
460 | static const TargetRegisterClass * | ||||||
461 | getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB, | ||||||
462 | const RegisterBankInfo &RBI, | ||||||
463 | bool GetAllRegSet = false) { | ||||||
464 | if (RB.getID() == AArch64::GPRRegBankID) { | ||||||
465 | if (Ty.getSizeInBits() <= 32) | ||||||
466 | return GetAllRegSet ? &AArch64::GPR32allRegClass | ||||||
467 | : &AArch64::GPR32RegClass; | ||||||
468 | if (Ty.getSizeInBits() == 64) | ||||||
469 | return GetAllRegSet ? &AArch64::GPR64allRegClass | ||||||
470 | : &AArch64::GPR64RegClass; | ||||||
471 | return nullptr; | ||||||
472 | } | ||||||
473 | |||||||
474 | if (RB.getID() == AArch64::FPRRegBankID) { | ||||||
475 | if (Ty.getSizeInBits() <= 16) | ||||||
476 | return &AArch64::FPR16RegClass; | ||||||
477 | if (Ty.getSizeInBits() == 32) | ||||||
478 | return &AArch64::FPR32RegClass; | ||||||
479 | if (Ty.getSizeInBits() == 64) | ||||||
480 | return &AArch64::FPR64RegClass; | ||||||
481 | if (Ty.getSizeInBits() == 128) | ||||||
482 | return &AArch64::FPR128RegClass; | ||||||
483 | return nullptr; | ||||||
484 | } | ||||||
485 | |||||||
486 | return nullptr; | ||||||
487 | } | ||||||
488 | |||||||
489 | /// Given a register bank, and size in bits, return the smallest register class | ||||||
490 | /// that can represent that combination. | ||||||
491 | static const TargetRegisterClass * | ||||||
492 | getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits, | ||||||
493 | bool GetAllRegSet = false) { | ||||||
494 | unsigned RegBankID = RB.getID(); | ||||||
495 | |||||||
496 | if (RegBankID == AArch64::GPRRegBankID) { | ||||||
497 | if (SizeInBits <= 32) | ||||||
498 | return GetAllRegSet ? &AArch64::GPR32allRegClass | ||||||
499 | : &AArch64::GPR32RegClass; | ||||||
500 | if (SizeInBits == 64) | ||||||
501 | return GetAllRegSet ? &AArch64::GPR64allRegClass | ||||||
502 | : &AArch64::GPR64RegClass; | ||||||
503 | } | ||||||
504 | |||||||
505 | if (RegBankID == AArch64::FPRRegBankID) { | ||||||
506 | switch (SizeInBits) { | ||||||
507 | default: | ||||||
508 | return nullptr; | ||||||
509 | case 8: | ||||||
510 | return &AArch64::FPR8RegClass; | ||||||
511 | case 16: | ||||||
512 | return &AArch64::FPR16RegClass; | ||||||
513 | case 32: | ||||||
514 | return &AArch64::FPR32RegClass; | ||||||
515 | case 64: | ||||||
516 | return &AArch64::FPR64RegClass; | ||||||
517 | case 128: | ||||||
518 | return &AArch64::FPR128RegClass; | ||||||
519 | } | ||||||
520 | } | ||||||
521 | |||||||
522 | return nullptr; | ||||||
523 | } | ||||||
524 | |||||||
525 | /// Returns the correct subregister to use for a given register class. | ||||||
526 | static bool getSubRegForClass(const TargetRegisterClass *RC, | ||||||
527 | const TargetRegisterInfo &TRI, unsigned &SubReg) { | ||||||
528 | switch (TRI.getRegSizeInBits(*RC)) { | ||||||
529 | case 8: | ||||||
530 | SubReg = AArch64::bsub; | ||||||
531 | break; | ||||||
532 | case 16: | ||||||
533 | SubReg = AArch64::hsub; | ||||||
534 | break; | ||||||
535 | case 32: | ||||||
536 | if (RC != &AArch64::FPR32RegClass) | ||||||
537 | SubReg = AArch64::sub_32; | ||||||
538 | else | ||||||
539 | SubReg = AArch64::ssub; | ||||||
540 | break; | ||||||
541 | case 64: | ||||||
542 | SubReg = AArch64::dsub; | ||||||
543 | break; | ||||||
544 | default: | ||||||
545 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class." ; } } while (false) | ||||||
546 | dbgs() << "Couldn't find appropriate subregister for register class.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class." ; } } while (false); | ||||||
547 | return false; | ||||||
548 | } | ||||||
549 | |||||||
550 | return true; | ||||||
551 | } | ||||||
552 | |||||||
553 | /// Returns the minimum size the given register bank can hold. | ||||||
554 | static unsigned getMinSizeForRegBank(const RegisterBank &RB) { | ||||||
555 | switch (RB.getID()) { | ||||||
556 | case AArch64::GPRRegBankID: | ||||||
557 | return 32; | ||||||
558 | case AArch64::FPRRegBankID: | ||||||
559 | return 8; | ||||||
560 | default: | ||||||
561 | llvm_unreachable("Tried to get minimum size for unknown register bank.")::llvm::llvm_unreachable_internal("Tried to get minimum size for unknown register bank." , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 561); | ||||||
562 | } | ||||||
563 | } | ||||||
564 | |||||||
565 | static Optional<uint64_t> getImmedFromMO(const MachineOperand &Root) { | ||||||
566 | auto &MI = *Root.getParent(); | ||||||
567 | auto &MBB = *MI.getParent(); | ||||||
568 | auto &MF = *MBB.getParent(); | ||||||
569 | auto &MRI = MF.getRegInfo(); | ||||||
570 | uint64_t Immed; | ||||||
571 | if (Root.isImm()) | ||||||
572 | Immed = Root.getImm(); | ||||||
573 | else if (Root.isCImm()) | ||||||
574 | Immed = Root.getCImm()->getZExtValue(); | ||||||
575 | else if (Root.isReg()) { | ||||||
576 | auto ValAndVReg = | ||||||
577 | getConstantVRegValWithLookThrough(Root.getReg(), MRI, true); | ||||||
578 | if (!ValAndVReg) | ||||||
579 | return None; | ||||||
580 | Immed = ValAndVReg->Value.getSExtValue(); | ||||||
581 | } else | ||||||
582 | return None; | ||||||
583 | return Immed; | ||||||
584 | } | ||||||
585 | |||||||
586 | /// Check whether \p I is a currently unsupported binary operation: | ||||||
587 | /// - it has an unsized type | ||||||
588 | /// - an operand is not a vreg | ||||||
589 | /// - all operands are not in the same bank | ||||||
590 | /// These are checks that should someday live in the verifier, but right now, | ||||||
591 | /// these are mostly limitations of the aarch64 selector. | ||||||
592 | static bool unsupportedBinOp(const MachineInstr &I, | ||||||
593 | const AArch64RegisterBankInfo &RBI, | ||||||
594 | const MachineRegisterInfo &MRI, | ||||||
595 | const AArch64RegisterInfo &TRI) { | ||||||
596 | LLT Ty = MRI.getType(I.getOperand(0).getReg()); | ||||||
597 | if (!Ty.isValid()) { | ||||||
598 | LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Generic binop register should be typed\n" ; } } while (false); | ||||||
599 | return true; | ||||||
600 | } | ||||||
601 | |||||||
602 | const RegisterBank *PrevOpBank = nullptr; | ||||||
603 | for (auto &MO : I.operands()) { | ||||||
604 | // FIXME: Support non-register operands. | ||||||
605 | if (!MO.isReg()) { | ||||||
606 | LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Generic inst non-reg operands are unsupported\n" ; } } while (false); | ||||||
607 | return true; | ||||||
608 | } | ||||||
609 | |||||||
610 | // FIXME: Can generic operations have physical registers operands? If | ||||||
611 | // so, this will need to be taught about that, and we'll need to get the | ||||||
612 | // bank out of the minimal class for the register. | ||||||
613 | // Either way, this needs to be documented (and possibly verified). | ||||||
614 | if (!Register::isVirtualRegister(MO.getReg())) { | ||||||
615 | LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Generic inst has physical register operand\n" ; } } while (false); | ||||||
616 | return true; | ||||||
617 | } | ||||||
618 | |||||||
619 | const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI); | ||||||
620 | if (!OpBank) { | ||||||
621 | LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Generic register has no bank or class\n" ; } } while (false); | ||||||
622 | return true; | ||||||
623 | } | ||||||
624 | |||||||
625 | if (PrevOpBank && OpBank != PrevOpBank) { | ||||||
626 | LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Generic inst operands have different banks\n" ; } } while (false); | ||||||
627 | return true; | ||||||
628 | } | ||||||
629 | PrevOpBank = OpBank; | ||||||
630 | } | ||||||
631 | return false; | ||||||
632 | } | ||||||
633 | |||||||
634 | /// Select the AArch64 opcode for the basic binary operation \p GenericOpc | ||||||
635 | /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID | ||||||
636 | /// and of size \p OpSize. | ||||||
637 | /// \returns \p GenericOpc if the combination is unsupported. | ||||||
638 | static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, | ||||||
639 | unsigned OpSize) { | ||||||
640 | switch (RegBankID) { | ||||||
641 | case AArch64::GPRRegBankID: | ||||||
642 | if (OpSize == 32) { | ||||||
643 | switch (GenericOpc) { | ||||||
644 | case TargetOpcode::G_SHL: | ||||||
645 | return AArch64::LSLVWr; | ||||||
646 | case TargetOpcode::G_LSHR: | ||||||
647 | return AArch64::LSRVWr; | ||||||
648 | case TargetOpcode::G_ASHR: | ||||||
649 | return AArch64::ASRVWr; | ||||||
650 | default: | ||||||
651 | return GenericOpc; | ||||||
652 | } | ||||||
653 | } else if (OpSize == 64) { | ||||||
654 | switch (GenericOpc) { | ||||||
655 | case TargetOpcode::G_PTR_ADD: | ||||||
656 | return AArch64::ADDXrr; | ||||||
657 | case TargetOpcode::G_SHL: | ||||||
658 | return AArch64::LSLVXr; | ||||||
659 | case TargetOpcode::G_LSHR: | ||||||
660 | return AArch64::LSRVXr; | ||||||
661 | case TargetOpcode::G_ASHR: | ||||||
662 | return AArch64::ASRVXr; | ||||||
663 | default: | ||||||
664 | return GenericOpc; | ||||||
665 | } | ||||||
666 | } | ||||||
667 | break; | ||||||
668 | case AArch64::FPRRegBankID: | ||||||
669 | switch (OpSize) { | ||||||
670 | case 32: | ||||||
671 | switch (GenericOpc) { | ||||||
672 | case TargetOpcode::G_FADD: | ||||||
673 | return AArch64::FADDSrr; | ||||||
674 | case TargetOpcode::G_FSUB: | ||||||
675 | return AArch64::FSUBSrr; | ||||||
676 | case TargetOpcode::G_FMUL: | ||||||
677 | return AArch64::FMULSrr; | ||||||
678 | case TargetOpcode::G_FDIV: | ||||||
679 | return AArch64::FDIVSrr; | ||||||
680 | default: | ||||||
681 | return GenericOpc; | ||||||
682 | } | ||||||
683 | case 64: | ||||||
684 | switch (GenericOpc) { | ||||||
685 | case TargetOpcode::G_FADD: | ||||||
686 | return AArch64::FADDDrr; | ||||||
687 | case TargetOpcode::G_FSUB: | ||||||
688 | return AArch64::FSUBDrr; | ||||||
689 | case TargetOpcode::G_FMUL: | ||||||
690 | return AArch64::FMULDrr; | ||||||
691 | case TargetOpcode::G_FDIV: | ||||||
692 | return AArch64::FDIVDrr; | ||||||
693 | case TargetOpcode::G_OR: | ||||||
694 | return AArch64::ORRv8i8; | ||||||
695 | default: | ||||||
696 | return GenericOpc; | ||||||
697 | } | ||||||
698 | } | ||||||
699 | break; | ||||||
700 | } | ||||||
701 | return GenericOpc; | ||||||
702 | } | ||||||
703 | |||||||
704 | /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc, | ||||||
705 | /// appropriate for the (value) register bank \p RegBankID and of memory access | ||||||
706 | /// size \p OpSize. This returns the variant with the base+unsigned-immediate | ||||||
707 | /// addressing mode (e.g., LDRXui). | ||||||
708 | /// \returns \p GenericOpc if the combination is unsupported. | ||||||
709 | static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, | ||||||
710 | unsigned OpSize) { | ||||||
711 | const bool isStore = GenericOpc == TargetOpcode::G_STORE; | ||||||
712 | switch (RegBankID) { | ||||||
713 | case AArch64::GPRRegBankID: | ||||||
714 | switch (OpSize) { | ||||||
715 | case 8: | ||||||
716 | return isStore ? AArch64::STRBBui : AArch64::LDRBBui; | ||||||
717 | case 16: | ||||||
718 | return isStore ? AArch64::STRHHui : AArch64::LDRHHui; | ||||||
719 | case 32: | ||||||
720 | return isStore ? AArch64::STRWui : AArch64::LDRWui; | ||||||
721 | case 64: | ||||||
722 | return isStore ? AArch64::STRXui : AArch64::LDRXui; | ||||||
723 | } | ||||||
724 | break; | ||||||
725 | case AArch64::FPRRegBankID: | ||||||
726 | switch (OpSize) { | ||||||
727 | case 8: | ||||||
728 | return isStore ? AArch64::STRBui : AArch64::LDRBui; | ||||||
729 | case 16: | ||||||
730 | return isStore ? AArch64::STRHui : AArch64::LDRHui; | ||||||
731 | case 32: | ||||||
732 | return isStore ? AArch64::STRSui : AArch64::LDRSui; | ||||||
733 | case 64: | ||||||
734 | return isStore ? AArch64::STRDui : AArch64::LDRDui; | ||||||
735 | } | ||||||
736 | break; | ||||||
737 | } | ||||||
738 | return GenericOpc; | ||||||
739 | } | ||||||
740 | |||||||
741 | #ifndef NDEBUG | ||||||
742 | /// Helper function that verifies that we have a valid copy at the end of | ||||||
743 | /// selectCopy. Verifies that the source and dest have the expected sizes and | ||||||
744 | /// then returns true. | ||||||
745 | static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank, | ||||||
746 | const MachineRegisterInfo &MRI, | ||||||
747 | const TargetRegisterInfo &TRI, | ||||||
748 | const RegisterBankInfo &RBI) { | ||||||
749 | const Register DstReg = I.getOperand(0).getReg(); | ||||||
750 | const Register SrcReg = I.getOperand(1).getReg(); | ||||||
751 | const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); | ||||||
752 | const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); | ||||||
753 | |||||||
754 | // Make sure the size of the source and dest line up. | ||||||
755 | assert((((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
756 | (DstSize == SrcSize ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
757 | // Copies are a mean to setup initial types, the number of(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
758 | // bits may not exactly match.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
759 | (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
760 | // Copies are a mean to copy bits around, as long as we are(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
761 | // on the same register class, that's fine. Otherwise, that(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
762 | // means we need some SUBREG_TO_REG or AND & co.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
763 | (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)) | ||||||
764 | "Copy with different width?!")(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg ) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && "Copy with different width?!") ? static_cast<void> (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 764, __PRETTY_FUNCTION__)); | ||||||
765 | |||||||
766 | // Check the size of the destination. | ||||||
767 | assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) &&(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID ) && "GPRs cannot get more than 64-bit width values") ? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 768, __PRETTY_FUNCTION__)) | ||||||
768 | "GPRs cannot get more than 64-bit width values")(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID ) && "GPRs cannot get more than 64-bit width values") ? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 768, __PRETTY_FUNCTION__)); | ||||||
769 | |||||||
770 | return true; | ||||||
771 | } | ||||||
772 | #endif | ||||||
773 | |||||||
774 | /// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg | ||||||
775 | /// to \p *To. | ||||||
776 | /// | ||||||
777 | /// E.g "To = COPY SrcReg:SubReg" | ||||||
778 | static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI, | ||||||
779 | const RegisterBankInfo &RBI, Register SrcReg, | ||||||
780 | const TargetRegisterClass *To, unsigned SubReg) { | ||||||
781 | assert(SrcReg.isValid() && "Expected a valid source register?")((SrcReg.isValid() && "Expected a valid source register?" ) ? static_cast<void> (0) : __assert_fail ("SrcReg.isValid() && \"Expected a valid source register?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 781, __PRETTY_FUNCTION__)); | ||||||
782 | assert(To && "Destination register class cannot be null")((To && "Destination register class cannot be null") ? static_cast<void> (0) : __assert_fail ("To && \"Destination register class cannot be null\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 782, __PRETTY_FUNCTION__)); | ||||||
783 | assert(SubReg && "Expected a valid subregister")((SubReg && "Expected a valid subregister") ? static_cast <void> (0) : __assert_fail ("SubReg && \"Expected a valid subregister\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 783, __PRETTY_FUNCTION__)); | ||||||
784 | |||||||
785 | MachineIRBuilder MIB(I); | ||||||
786 | auto SubRegCopy = | ||||||
787 | MIB.buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, 0, SubReg); | ||||||
788 | MachineOperand &RegOp = I.getOperand(1); | ||||||
789 | RegOp.setReg(SubRegCopy.getReg(0)); | ||||||
790 | |||||||
791 | // It's possible that the destination register won't be constrained. Make | ||||||
792 | // sure that happens. | ||||||
793 | if (!Register::isPhysicalRegister(I.getOperand(0).getReg())) | ||||||
794 | RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI); | ||||||
795 | |||||||
796 | return true; | ||||||
797 | } | ||||||
798 | |||||||
799 | /// Helper function to get the source and destination register classes for a | ||||||
800 | /// copy. Returns a std::pair containing the source register class for the | ||||||
801 | /// copy, and the destination register class for the copy. If a register class | ||||||
802 | /// cannot be determined, then it will be nullptr. | ||||||
803 | static std::pair<const TargetRegisterClass *, const TargetRegisterClass *> | ||||||
804 | getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII, | ||||||
805 | MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, | ||||||
806 | const RegisterBankInfo &RBI) { | ||||||
807 | Register DstReg = I.getOperand(0).getReg(); | ||||||
808 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
809 | const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); | ||||||
810 | const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); | ||||||
811 | unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); | ||||||
812 | unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); | ||||||
813 | |||||||
814 | // Special casing for cross-bank copies of s1s. We can technically represent | ||||||
815 | // a 1-bit value with any size of register. The minimum size for a GPR is 32 | ||||||
816 | // bits. So, we need to put the FPR on 32 bits as well. | ||||||
817 | // | ||||||
818 | // FIXME: I'm not sure if this case holds true outside of copies. If it does, | ||||||
819 | // then we can pull it into the helpers that get the appropriate class for a | ||||||
820 | // register bank. Or make a new helper that carries along some constraint | ||||||
821 | // information. | ||||||
822 | if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1)) | ||||||
823 | SrcSize = DstSize = 32; | ||||||
824 | |||||||
825 | return {getMinClassForRegBank(SrcRegBank, SrcSize, true), | ||||||
826 | getMinClassForRegBank(DstRegBank, DstSize, true)}; | ||||||
827 | } | ||||||
828 | |||||||
829 | static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, | ||||||
830 | MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, | ||||||
831 | const RegisterBankInfo &RBI) { | ||||||
832 | Register DstReg = I.getOperand(0).getReg(); | ||||||
833 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
834 | const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); | ||||||
835 | const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); | ||||||
836 | |||||||
837 | // Find the correct register classes for the source and destination registers. | ||||||
838 | const TargetRegisterClass *SrcRC; | ||||||
839 | const TargetRegisterClass *DstRC; | ||||||
840 | std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI); | ||||||
841 | |||||||
842 | if (!DstRC) { | ||||||
843 | LLVM_DEBUG(dbgs() << "Unexpected dest size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unexpected dest size " << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while (false) | ||||||
844 | << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unexpected dest size " << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while (false); | ||||||
845 | return false; | ||||||
846 | } | ||||||
847 | |||||||
848 | // A couple helpers below, for making sure that the copy we produce is valid. | ||||||
849 | |||||||
850 | // Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want | ||||||
851 | // to verify that the src and dst are the same size, since that's handled by | ||||||
852 | // the SUBREG_TO_REG. | ||||||
853 | bool KnownValid = false; | ||||||
854 | |||||||
855 | // Returns true, or asserts if something we don't expect happens. Instead of | ||||||
856 | // returning true, we return isValidCopy() to ensure that we verify the | ||||||
857 | // result. | ||||||
858 | auto CheckCopy = [&]() { | ||||||
859 | // If we have a bitcast or something, we can't have physical registers. | ||||||
860 | assert((I.isCopy() ||(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand( 0).getReg()) && !Register::isPhysicalRegister(I.getOperand (1).getReg()))) && "No phys reg on generic operator!" ) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 863, __PRETTY_FUNCTION__)) | ||||||
861 | (!Register::isPhysicalRegister(I.getOperand(0).getReg()) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand( 0).getReg()) && !Register::isPhysicalRegister(I.getOperand (1).getReg()))) && "No phys reg on generic operator!" ) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 863, __PRETTY_FUNCTION__)) | ||||||
862 | !Register::isPhysicalRegister(I.getOperand(1).getReg()))) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand( 0).getReg()) && !Register::isPhysicalRegister(I.getOperand (1).getReg()))) && "No phys reg on generic operator!" ) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 863, __PRETTY_FUNCTION__)) | ||||||
863 | "No phys reg on generic operator!")(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand( 0).getReg()) && !Register::isPhysicalRegister(I.getOperand (1).getReg()))) && "No phys reg on generic operator!" ) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 863, __PRETTY_FUNCTION__)); | ||||||
864 | bool ValidCopy = true; | ||||||
865 | #ifndef NDEBUG | ||||||
866 | ValidCopy = KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI); | ||||||
867 | assert(ValidCopy && "Invalid copy.")((ValidCopy && "Invalid copy.") ? static_cast<void > (0) : __assert_fail ("ValidCopy && \"Invalid copy.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 867, __PRETTY_FUNCTION__)); | ||||||
868 | #endif | ||||||
869 | return ValidCopy; | ||||||
870 | }; | ||||||
871 | |||||||
872 | // Is this a copy? If so, then we may need to insert a subregister copy. | ||||||
873 | if (I.isCopy()) { | ||||||
874 | // Yes. Check if there's anything to fix up. | ||||||
875 | if (!SrcRC) { | ||||||
876 | LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Couldn't determine source register class\n" ; } } while (false); | ||||||
877 | return false; | ||||||
878 | } | ||||||
879 | |||||||
880 | unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC); | ||||||
881 | unsigned DstSize = TRI.getRegSizeInBits(*DstRC); | ||||||
882 | unsigned SubReg; | ||||||
883 | |||||||
884 | // If the source bank doesn't support a subregister copy small enough, | ||||||
885 | // then we first need to copy to the destination bank. | ||||||
886 | if (getMinSizeForRegBank(SrcRegBank) > DstSize) { | ||||||
887 | const TargetRegisterClass *DstTempRC = | ||||||
888 | getMinClassForRegBank(DstRegBank, SrcSize, /* GetAllRegSet */ true); | ||||||
889 | getSubRegForClass(DstRC, TRI, SubReg); | ||||||
890 | |||||||
891 | MachineIRBuilder MIB(I); | ||||||
892 | auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg}); | ||||||
893 | copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg); | ||||||
| |||||||
894 | } else if (SrcSize > DstSize) { | ||||||
895 | // If the source register is bigger than the destination we need to | ||||||
896 | // perform a subregister copy. | ||||||
897 | const TargetRegisterClass *SubRegRC = | ||||||
898 | getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true); | ||||||
899 | getSubRegForClass(SubRegRC, TRI, SubReg); | ||||||
900 | copySubReg(I, MRI, RBI, SrcReg, DstRC, SubReg); | ||||||
901 | } else if (DstSize > SrcSize) { | ||||||
902 | // If the destination register is bigger than the source we need to do | ||||||
903 | // a promotion using SUBREG_TO_REG. | ||||||
904 | const TargetRegisterClass *PromotionRC = | ||||||
905 | getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true); | ||||||
906 | getSubRegForClass(SrcRC, TRI, SubReg); | ||||||
907 | |||||||
908 | Register PromoteReg = MRI.createVirtualRegister(PromotionRC); | ||||||
909 | BuildMI(*I.getParent(), I, I.getDebugLoc(), | ||||||
910 | TII.get(AArch64::SUBREG_TO_REG), PromoteReg) | ||||||
911 | .addImm(0) | ||||||
912 | .addUse(SrcReg) | ||||||
913 | .addImm(SubReg); | ||||||
914 | MachineOperand &RegOp = I.getOperand(1); | ||||||
915 | RegOp.setReg(PromoteReg); | ||||||
916 | |||||||
917 | // Promise that the copy is implicitly validated by the SUBREG_TO_REG. | ||||||
918 | KnownValid = true; | ||||||
919 | } | ||||||
920 | |||||||
921 | // If the destination is a physical register, then there's nothing to | ||||||
922 | // change, so we're done. | ||||||
923 | if (Register::isPhysicalRegister(DstReg)) | ||||||
924 | return CheckCopy(); | ||||||
925 | } | ||||||
926 | |||||||
927 | // No need to constrain SrcReg. It will get constrained when we hit another | ||||||
928 | // of its use or its defs. Copies do not have constraints. | ||||||
929 | if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | ||||||
930 | LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) << " operand\n"; } } while ( false) | ||||||
931 | << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) << " operand\n"; } } while ( false); | ||||||
932 | return false; | ||||||
933 | } | ||||||
934 | I.setDesc(TII.get(AArch64::COPY)); | ||||||
935 | return CheckCopy(); | ||||||
936 | } | ||||||
937 | |||||||
938 | static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) { | ||||||
939 | if (!DstTy.isScalar() || !SrcTy.isScalar()) | ||||||
940 | return GenericOpc; | ||||||
941 | |||||||
942 | const unsigned DstSize = DstTy.getSizeInBits(); | ||||||
943 | const unsigned SrcSize = SrcTy.getSizeInBits(); | ||||||
944 | |||||||
945 | switch (DstSize) { | ||||||
946 | case 32: | ||||||
947 | switch (SrcSize) { | ||||||
948 | case 32: | ||||||
949 | switch (GenericOpc) { | ||||||
950 | case TargetOpcode::G_SITOFP: | ||||||
951 | return AArch64::SCVTFUWSri; | ||||||
952 | case TargetOpcode::G_UITOFP: | ||||||
953 | return AArch64::UCVTFUWSri; | ||||||
954 | case TargetOpcode::G_FPTOSI: | ||||||
955 | return AArch64::FCVTZSUWSr; | ||||||
956 | case TargetOpcode::G_FPTOUI: | ||||||
957 | return AArch64::FCVTZUUWSr; | ||||||
958 | default: | ||||||
959 | return GenericOpc; | ||||||
960 | } | ||||||
961 | case 64: | ||||||
962 | switch (GenericOpc) { | ||||||
963 | case TargetOpcode::G_SITOFP: | ||||||
964 | return AArch64::SCVTFUXSri; | ||||||
965 | case TargetOpcode::G_UITOFP: | ||||||
966 | return AArch64::UCVTFUXSri; | ||||||
967 | case TargetOpcode::G_FPTOSI: | ||||||
968 | return AArch64::FCVTZSUWDr; | ||||||
969 | case TargetOpcode::G_FPTOUI: | ||||||
970 | return AArch64::FCVTZUUWDr; | ||||||
971 | default: | ||||||
972 | return GenericOpc; | ||||||
973 | } | ||||||
974 | default: | ||||||
975 | return GenericOpc; | ||||||
976 | } | ||||||
977 | case 64: | ||||||
978 | switch (SrcSize) { | ||||||
979 | case 32: | ||||||
980 | switch (GenericOpc) { | ||||||
981 | case TargetOpcode::G_SITOFP: | ||||||
982 | return AArch64::SCVTFUWDri; | ||||||
983 | case TargetOpcode::G_UITOFP: | ||||||
984 | return AArch64::UCVTFUWDri; | ||||||
985 | case TargetOpcode::G_FPTOSI: | ||||||
986 | return AArch64::FCVTZSUXSr; | ||||||
987 | case TargetOpcode::G_FPTOUI: | ||||||
988 | return AArch64::FCVTZUUXSr; | ||||||
989 | default: | ||||||
990 | return GenericOpc; | ||||||
991 | } | ||||||
992 | case 64: | ||||||
993 | switch (GenericOpc) { | ||||||
994 | case TargetOpcode::G_SITOFP: | ||||||
995 | return AArch64::SCVTFUXDri; | ||||||
996 | case TargetOpcode::G_UITOFP: | ||||||
997 | return AArch64::UCVTFUXDri; | ||||||
998 | case TargetOpcode::G_FPTOSI: | ||||||
999 | return AArch64::FCVTZSUXDr; | ||||||
1000 | case TargetOpcode::G_FPTOUI: | ||||||
1001 | return AArch64::FCVTZUUXDr; | ||||||
1002 | default: | ||||||
1003 | return GenericOpc; | ||||||
1004 | } | ||||||
1005 | default: | ||||||
1006 | return GenericOpc; | ||||||
1007 | } | ||||||
1008 | default: | ||||||
1009 | return GenericOpc; | ||||||
1010 | }; | ||||||
1011 | return GenericOpc; | ||||||
1012 | } | ||||||
1013 | |||||||
1014 | MachineInstr * | ||||||
1015 | AArch64InstructionSelector::emitSelect(Register Dst, Register True, | ||||||
1016 | Register False, AArch64CC::CondCode CC, | ||||||
1017 | MachineIRBuilder &MIB) const { | ||||||
1018 | MachineRegisterInfo &MRI = *MIB.getMRI(); | ||||||
1019 | assert(RBI.getRegBank(False, MRI, TRI)->getID() ==((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank (True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?" ) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1021, __PRETTY_FUNCTION__)) | ||||||
1020 | RBI.getRegBank(True, MRI, TRI)->getID() &&((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank (True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?" ) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1021, __PRETTY_FUNCTION__)) | ||||||
1021 | "Expected both select operands to have the same regbank?")((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank (True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?" ) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1021, __PRETTY_FUNCTION__)); | ||||||
1022 | LLT Ty = MRI.getType(True); | ||||||
1023 | if (Ty.isVector()) | ||||||
1024 | return nullptr; | ||||||
1025 | const unsigned Size = Ty.getSizeInBits(); | ||||||
1026 | assert((Size == 32 || Size == 64) &&(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit select only?" ) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1027, __PRETTY_FUNCTION__)) | ||||||
1027 | "Expected 32 bit or 64 bit select only?")(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit select only?" ) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1027, __PRETTY_FUNCTION__)); | ||||||
1028 | const bool Is32Bit = Size == 32; | ||||||
1029 | if (RBI.getRegBank(True, MRI, TRI)->getID() != AArch64::GPRRegBankID) { | ||||||
1030 | unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr; | ||||||
1031 | auto FCSel = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC); | ||||||
1032 | constrainSelectedInstRegOperands(*FCSel, TII, TRI, RBI); | ||||||
1033 | return &*FCSel; | ||||||
1034 | } | ||||||
1035 | |||||||
1036 | // By default, we'll try and emit a CSEL. | ||||||
1037 | unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr; | ||||||
1038 | bool Optimized = false; | ||||||
1039 | auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI, | ||||||
1040 | &Optimized](Register &Reg, Register &OtherReg, | ||||||
1041 | bool Invert) { | ||||||
1042 | if (Optimized) | ||||||
1043 | return false; | ||||||
1044 | |||||||
1045 | // Attempt to fold: | ||||||
1046 | // | ||||||
1047 | // %sub = G_SUB 0, %x | ||||||
1048 | // %select = G_SELECT cc, %reg, %sub | ||||||
1049 | // | ||||||
1050 | // Into: | ||||||
1051 | // %select = CSNEG %reg, %x, cc | ||||||
1052 | Register MatchReg; | ||||||
1053 | if (mi_match(Reg, MRI, m_Neg(m_Reg(MatchReg)))) { | ||||||
1054 | Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr; | ||||||
1055 | Reg = MatchReg; | ||||||
1056 | if (Invert) { | ||||||
1057 | CC = AArch64CC::getInvertedCondCode(CC); | ||||||
1058 | std::swap(Reg, OtherReg); | ||||||
1059 | } | ||||||
1060 | return true; | ||||||
1061 | } | ||||||
1062 | |||||||
1063 | // Attempt to fold: | ||||||
1064 | // | ||||||
1065 | // %xor = G_XOR %x, -1 | ||||||
1066 | // %select = G_SELECT cc, %reg, %xor | ||||||
1067 | // | ||||||
1068 | // Into: | ||||||
1069 | // %select = CSINV %reg, %x, cc | ||||||
1070 | if (mi_match(Reg, MRI, m_Not(m_Reg(MatchReg)))) { | ||||||
1071 | Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr; | ||||||
1072 | Reg = MatchReg; | ||||||
1073 | if (Invert) { | ||||||
1074 | CC = AArch64CC::getInvertedCondCode(CC); | ||||||
1075 | std::swap(Reg, OtherReg); | ||||||
1076 | } | ||||||
1077 | return true; | ||||||
1078 | } | ||||||
1079 | |||||||
1080 | // Attempt to fold: | ||||||
1081 | // | ||||||
1082 | // %add = G_ADD %x, 1 | ||||||
1083 | // %select = G_SELECT cc, %reg, %add | ||||||
1084 | // | ||||||
1085 | // Into: | ||||||
1086 | // %select = CSINC %reg, %x, cc | ||||||
1087 | if (mi_match(Reg, MRI, m_GAdd(m_Reg(MatchReg), m_SpecificICst(1)))) { | ||||||
1088 | Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr; | ||||||
1089 | Reg = MatchReg; | ||||||
1090 | if (Invert) { | ||||||
1091 | CC = AArch64CC::getInvertedCondCode(CC); | ||||||
1092 | std::swap(Reg, OtherReg); | ||||||
1093 | } | ||||||
1094 | return true; | ||||||
1095 | } | ||||||
1096 | |||||||
1097 | return false; | ||||||
1098 | }; | ||||||
1099 | |||||||
1100 | // Helper lambda which tries to use CSINC/CSINV for the instruction when its | ||||||
1101 | // true/false values are constants. | ||||||
1102 | // FIXME: All of these patterns already exist in tablegen. We should be | ||||||
1103 | // able to import these. | ||||||
1104 | auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI, | ||||||
1105 | &Optimized]() { | ||||||
1106 | if (Optimized) | ||||||
1107 | return false; | ||||||
1108 | auto TrueCst = getConstantVRegValWithLookThrough(True, MRI); | ||||||
1109 | auto FalseCst = getConstantVRegValWithLookThrough(False, MRI); | ||||||
1110 | if (!TrueCst && !FalseCst) | ||||||
1111 | return false; | ||||||
1112 | |||||||
1113 | Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR; | ||||||
1114 | if (TrueCst && FalseCst) { | ||||||
1115 | int64_t T = TrueCst->Value.getSExtValue(); | ||||||
1116 | int64_t F = FalseCst->Value.getSExtValue(); | ||||||
1117 | |||||||
1118 | if (T == 0 && F == 1) { | ||||||
1119 | // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc | ||||||
1120 | Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr; | ||||||
1121 | True = ZReg; | ||||||
1122 | False = ZReg; | ||||||
1123 | return true; | ||||||
1124 | } | ||||||
1125 | |||||||
1126 | if (T == 0 && F == -1) { | ||||||
1127 | // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc | ||||||
1128 | Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr; | ||||||
1129 | True = ZReg; | ||||||
1130 | False = ZReg; | ||||||
1131 | return true; | ||||||
1132 | } | ||||||
1133 | } | ||||||
1134 | |||||||
1135 | if (TrueCst) { | ||||||
1136 | int64_t T = TrueCst->Value.getSExtValue(); | ||||||
1137 | if (T == 1) { | ||||||
1138 | // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc | ||||||
1139 | Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr; | ||||||
1140 | True = False; | ||||||
1141 | False = ZReg; | ||||||
1142 | CC = AArch64CC::getInvertedCondCode(CC); | ||||||
1143 | return true; | ||||||
1144 | } | ||||||
1145 | |||||||
1146 | if (T == -1) { | ||||||
1147 | // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc | ||||||
1148 | Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr; | ||||||
1149 | True = False; | ||||||
1150 | False = ZReg; | ||||||
1151 | CC = AArch64CC::getInvertedCondCode(CC); | ||||||
1152 | return true; | ||||||
1153 | } | ||||||
1154 | } | ||||||
1155 | |||||||
1156 | if (FalseCst) { | ||||||
1157 | int64_t F = FalseCst->Value.getSExtValue(); | ||||||
1158 | if (F == 1) { | ||||||
1159 | // G_SELECT cc, t, 1 -> CSINC t, zreg, cc | ||||||
1160 | Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr; | ||||||
1161 | False = ZReg; | ||||||
1162 | return true; | ||||||
1163 | } | ||||||
1164 | |||||||
1165 | if (F == -1) { | ||||||
1166 | // G_SELECT cc, t, -1 -> CSINC t, zreg, cc | ||||||
1167 | Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr; | ||||||
1168 | False = ZReg; | ||||||
1169 | return true; | ||||||
1170 | } | ||||||
1171 | } | ||||||
1172 | return false; | ||||||
1173 | }; | ||||||
1174 | |||||||
1175 | Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false); | ||||||
1176 | Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true); | ||||||
1177 | Optimized |= TryOptSelectCst(); | ||||||
1178 | auto SelectInst = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC); | ||||||
1179 | constrainSelectedInstRegOperands(*SelectInst, TII, TRI, RBI); | ||||||
1180 | return &*SelectInst; | ||||||
1181 | } | ||||||
1182 | |||||||
1183 | static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) { | ||||||
1184 | switch (P) { | ||||||
1185 | default: | ||||||
1186 | llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1186); | ||||||
1187 | case CmpInst::ICMP_NE: | ||||||
1188 | return AArch64CC::NE; | ||||||
1189 | case CmpInst::ICMP_EQ: | ||||||
1190 | return AArch64CC::EQ; | ||||||
1191 | case CmpInst::ICMP_SGT: | ||||||
1192 | return AArch64CC::GT; | ||||||
1193 | case CmpInst::ICMP_SGE: | ||||||
1194 | return AArch64CC::GE; | ||||||
1195 | case CmpInst::ICMP_SLT: | ||||||
1196 | return AArch64CC::LT; | ||||||
1197 | case CmpInst::ICMP_SLE: | ||||||
1198 | return AArch64CC::LE; | ||||||
1199 | case CmpInst::ICMP_UGT: | ||||||
1200 | return AArch64CC::HI; | ||||||
1201 | case CmpInst::ICMP_UGE: | ||||||
1202 | return AArch64CC::HS; | ||||||
1203 | case CmpInst::ICMP_ULT: | ||||||
1204 | return AArch64CC::LO; | ||||||
1205 | case CmpInst::ICMP_ULE: | ||||||
1206 | return AArch64CC::LS; | ||||||
1207 | } | ||||||
1208 | } | ||||||
1209 | |||||||
1210 | static void changeFCMPPredToAArch64CC(CmpInst::Predicate P, | ||||||
1211 | AArch64CC::CondCode &CondCode, | ||||||
1212 | AArch64CC::CondCode &CondCode2) { | ||||||
1213 | CondCode2 = AArch64CC::AL; | ||||||
1214 | switch (P) { | ||||||
1215 | default: | ||||||
1216 | llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1216); | ||||||
1217 | case CmpInst::FCMP_OEQ: | ||||||
1218 | CondCode = AArch64CC::EQ; | ||||||
1219 | break; | ||||||
1220 | case CmpInst::FCMP_OGT: | ||||||
1221 | CondCode = AArch64CC::GT; | ||||||
1222 | break; | ||||||
1223 | case CmpInst::FCMP_OGE: | ||||||
1224 | CondCode = AArch64CC::GE; | ||||||
1225 | break; | ||||||
1226 | case CmpInst::FCMP_OLT: | ||||||
1227 | CondCode = AArch64CC::MI; | ||||||
1228 | break; | ||||||
1229 | case CmpInst::FCMP_OLE: | ||||||
1230 | CondCode = AArch64CC::LS; | ||||||
1231 | break; | ||||||
1232 | case CmpInst::FCMP_ONE: | ||||||
1233 | CondCode = AArch64CC::MI; | ||||||
1234 | CondCode2 = AArch64CC::GT; | ||||||
1235 | break; | ||||||
1236 | case CmpInst::FCMP_ORD: | ||||||
1237 | CondCode = AArch64CC::VC; | ||||||
1238 | break; | ||||||
1239 | case CmpInst::FCMP_UNO: | ||||||
1240 | CondCode = AArch64CC::VS; | ||||||
1241 | break; | ||||||
1242 | case CmpInst::FCMP_UEQ: | ||||||
1243 | CondCode = AArch64CC::EQ; | ||||||
1244 | CondCode2 = AArch64CC::VS; | ||||||
1245 | break; | ||||||
1246 | case CmpInst::FCMP_UGT: | ||||||
1247 | CondCode = AArch64CC::HI; | ||||||
1248 | break; | ||||||
1249 | case CmpInst::FCMP_UGE: | ||||||
1250 | CondCode = AArch64CC::PL; | ||||||
1251 | break; | ||||||
1252 | case CmpInst::FCMP_ULT: | ||||||
1253 | CondCode = AArch64CC::LT; | ||||||
1254 | break; | ||||||
1255 | case CmpInst::FCMP_ULE: | ||||||
1256 | CondCode = AArch64CC::LE; | ||||||
1257 | break; | ||||||
1258 | case CmpInst::FCMP_UNE: | ||||||
1259 | CondCode = AArch64CC::NE; | ||||||
1260 | break; | ||||||
1261 | } | ||||||
1262 | } | ||||||
1263 | |||||||
1264 | /// Return a register which can be used as a bit to test in a TB(N)Z. | ||||||
1265 | static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert, | ||||||
1266 | MachineRegisterInfo &MRI) { | ||||||
1267 | assert(Reg.isValid() && "Expected valid register!")((Reg.isValid() && "Expected valid register!") ? static_cast <void> (0) : __assert_fail ("Reg.isValid() && \"Expected valid register!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1267, __PRETTY_FUNCTION__)); | ||||||
1268 | while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) { | ||||||
1269 | unsigned Opc = MI->getOpcode(); | ||||||
1270 | |||||||
1271 | if (!MI->getOperand(0).isReg() || | ||||||
1272 | !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg())) | ||||||
1273 | break; | ||||||
1274 | |||||||
1275 | // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits. | ||||||
1276 | // | ||||||
1277 | // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number | ||||||
1278 | // on the truncated x is the same as the bit number on x. | ||||||
1279 | if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT || | ||||||
1280 | Opc == TargetOpcode::G_TRUNC) { | ||||||
1281 | Register NextReg = MI->getOperand(1).getReg(); | ||||||
1282 | // Did we find something worth folding? | ||||||
1283 | if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(NextReg)) | ||||||
1284 | break; | ||||||
1285 | |||||||
1286 | // NextReg is worth folding. Keep looking. | ||||||
1287 | Reg = NextReg; | ||||||
1288 | continue; | ||||||
1289 | } | ||||||
1290 | |||||||
1291 | // Attempt to find a suitable operation with a constant on one side. | ||||||
1292 | Optional<uint64_t> C; | ||||||
1293 | Register TestReg; | ||||||
1294 | switch (Opc) { | ||||||
1295 | default: | ||||||
1296 | break; | ||||||
1297 | case TargetOpcode::G_AND: | ||||||
1298 | case TargetOpcode::G_XOR: { | ||||||
1299 | TestReg = MI->getOperand(1).getReg(); | ||||||
1300 | Register ConstantReg = MI->getOperand(2).getReg(); | ||||||
1301 | auto VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI); | ||||||
1302 | if (!VRegAndVal) { | ||||||
1303 | // AND commutes, check the other side for a constant. | ||||||
1304 | // FIXME: Can we canonicalize the constant so that it's always on the | ||||||
1305 | // same side at some point earlier? | ||||||
1306 | std::swap(ConstantReg, TestReg); | ||||||
1307 | VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI); | ||||||
1308 | } | ||||||
1309 | if (VRegAndVal) | ||||||
1310 | C = VRegAndVal->Value.getSExtValue(); | ||||||
1311 | break; | ||||||
1312 | } | ||||||
1313 | case TargetOpcode::G_ASHR: | ||||||
1314 | case TargetOpcode::G_LSHR: | ||||||
1315 | case TargetOpcode::G_SHL: { | ||||||
1316 | TestReg = MI->getOperand(1).getReg(); | ||||||
1317 | auto VRegAndVal = | ||||||
1318 | getConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI); | ||||||
1319 | if (VRegAndVal) | ||||||
1320 | C = VRegAndVal->Value.getSExtValue(); | ||||||
1321 | break; | ||||||
1322 | } | ||||||
1323 | } | ||||||
1324 | |||||||
1325 | // Didn't find a constant or viable register. Bail out of the loop. | ||||||
1326 | if (!C || !TestReg.isValid()) | ||||||
1327 | break; | ||||||
1328 | |||||||
1329 | // We found a suitable instruction with a constant. Check to see if we can | ||||||
1330 | // walk through the instruction. | ||||||
1331 | Register NextReg; | ||||||
1332 | unsigned TestRegSize = MRI.getType(TestReg).getSizeInBits(); | ||||||
1333 | switch (Opc) { | ||||||
1334 | default: | ||||||
1335 | break; | ||||||
1336 | case TargetOpcode::G_AND: | ||||||
1337 | // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set. | ||||||
1338 | if ((*C >> Bit) & 1) | ||||||
1339 | NextReg = TestReg; | ||||||
1340 | break; | ||||||
1341 | case TargetOpcode::G_SHL: | ||||||
1342 | // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in | ||||||
1343 | // the type of the register. | ||||||
1344 | if (*C <= Bit && (Bit - *C) < TestRegSize) { | ||||||
1345 | NextReg = TestReg; | ||||||
1346 | Bit = Bit - *C; | ||||||
1347 | } | ||||||
1348 | break; | ||||||
1349 | case TargetOpcode::G_ASHR: | ||||||
1350 | // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits | ||||||
1351 | // in x | ||||||
1352 | NextReg = TestReg; | ||||||
1353 | Bit = Bit + *C; | ||||||
1354 | if (Bit >= TestRegSize) | ||||||
1355 | Bit = TestRegSize - 1; | ||||||
1356 | break; | ||||||
1357 | case TargetOpcode::G_LSHR: | ||||||
1358 | // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x | ||||||
1359 | if ((Bit + *C) < TestRegSize) { | ||||||
1360 | NextReg = TestReg; | ||||||
1361 | Bit = Bit + *C; | ||||||
1362 | } | ||||||
1363 | break; | ||||||
1364 | case TargetOpcode::G_XOR: | ||||||
1365 | // We can walk through a G_XOR by inverting whether we use tbz/tbnz when | ||||||
1366 | // appropriate. | ||||||
1367 | // | ||||||
1368 | // e.g. If x' = xor x, c, and the b-th bit is set in c then | ||||||
1369 | // | ||||||
1370 | // tbz x', b -> tbnz x, b | ||||||
1371 | // | ||||||
1372 | // Because x' only has the b-th bit set if x does not. | ||||||
1373 | if ((*C >> Bit) & 1) | ||||||
1374 | Invert = !Invert; | ||||||
1375 | NextReg = TestReg; | ||||||
1376 | break; | ||||||
1377 | } | ||||||
1378 | |||||||
1379 | // Check if we found anything worth folding. | ||||||
1380 | if (!NextReg.isValid()) | ||||||
1381 | return Reg; | ||||||
1382 | Reg = NextReg; | ||||||
1383 | } | ||||||
1384 | |||||||
1385 | return Reg; | ||||||
1386 | } | ||||||
1387 | |||||||
1388 | MachineInstr *AArch64InstructionSelector::emitTestBit( | ||||||
1389 | Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB, | ||||||
1390 | MachineIRBuilder &MIB) const { | ||||||
1391 | assert(TestReg.isValid())((TestReg.isValid()) ? static_cast<void> (0) : __assert_fail ("TestReg.isValid()", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1391, __PRETTY_FUNCTION__)); | ||||||
1392 | assert(ProduceNonFlagSettingCondBr &&((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!" ) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1393, __PRETTY_FUNCTION__)) | ||||||
1393 | "Cannot emit TB(N)Z with speculation tracking!")((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!" ) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1393, __PRETTY_FUNCTION__)); | ||||||
1394 | MachineRegisterInfo &MRI = *MIB.getMRI(); | ||||||
1395 | |||||||
1396 | // Attempt to optimize the test bit by walking over instructions. | ||||||
1397 | TestReg = getTestBitReg(TestReg, Bit, IsNegative, MRI); | ||||||
1398 | LLT Ty = MRI.getType(TestReg); | ||||||
1399 | unsigned Size = Ty.getSizeInBits(); | ||||||
1400 | assert(!Ty.isVector() && "Expected a scalar!")((!Ty.isVector() && "Expected a scalar!") ? static_cast <void> (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1400, __PRETTY_FUNCTION__)); | ||||||
1401 | assert(Bit < 64 && "Bit is too large!")((Bit < 64 && "Bit is too large!") ? static_cast< void> (0) : __assert_fail ("Bit < 64 && \"Bit is too large!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1401, __PRETTY_FUNCTION__)); | ||||||
1402 | |||||||
1403 | // When the test register is a 64-bit register, we have to narrow to make | ||||||
1404 | // TBNZW work. | ||||||
1405 | bool UseWReg = Bit < 32; | ||||||
1406 | unsigned NecessarySize = UseWReg ? 32 : 64; | ||||||
1407 | if (Size != NecessarySize) | ||||||
1408 | TestReg = moveScalarRegClass( | ||||||
1409 | TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass, | ||||||
1410 | MIB); | ||||||
1411 | |||||||
1412 | static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX}, | ||||||
1413 | {AArch64::TBZW, AArch64::TBNZW}}; | ||||||
1414 | unsigned Opc = OpcTable[UseWReg][IsNegative]; | ||||||
1415 | auto TestBitMI = | ||||||
1416 | MIB.buildInstr(Opc).addReg(TestReg).addImm(Bit).addMBB(DstMBB); | ||||||
1417 | constrainSelectedInstRegOperands(*TestBitMI, TII, TRI, RBI); | ||||||
1418 | return &*TestBitMI; | ||||||
1419 | } | ||||||
1420 | |||||||
1421 | bool AArch64InstructionSelector::tryOptAndIntoCompareBranch( | ||||||
1422 | MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB, | ||||||
1423 | MachineIRBuilder &MIB) const { | ||||||
1424 | assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?")((AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?" ) ? static_cast<void> (0) : __assert_fail ("AndInst.getOpcode() == TargetOpcode::G_AND && \"Expected G_AND only?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1424, __PRETTY_FUNCTION__)); | ||||||
1425 | // Given something like this: | ||||||
1426 | // | ||||||
1427 | // %x = ...Something... | ||||||
1428 | // %one = G_CONSTANT i64 1 | ||||||
1429 | // %zero = G_CONSTANT i64 0 | ||||||
1430 | // %and = G_AND %x, %one | ||||||
1431 | // %cmp = G_ICMP intpred(ne), %and, %zero | ||||||
1432 | // %cmp_trunc = G_TRUNC %cmp | ||||||
1433 | // G_BRCOND %cmp_trunc, %bb.3 | ||||||
1434 | // | ||||||
1435 | // We want to try and fold the AND into the G_BRCOND and produce either a | ||||||
1436 | // TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)). | ||||||
1437 | // | ||||||
1438 | // In this case, we'd get | ||||||
1439 | // | ||||||
1440 | // TBNZ %x %bb.3 | ||||||
1441 | // | ||||||
1442 | |||||||
1443 | // Check if the AND has a constant on its RHS which we can use as a mask. | ||||||
1444 | // If it's a power of 2, then it's the same as checking a specific bit. | ||||||
1445 | // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set) | ||||||
1446 | auto MaybeBit = getConstantVRegValWithLookThrough( | ||||||
1447 | AndInst.getOperand(2).getReg(), *MIB.getMRI()); | ||||||
1448 | if (!MaybeBit) | ||||||
1449 | return false; | ||||||
1450 | |||||||
1451 | int32_t Bit = MaybeBit->Value.exactLogBase2(); | ||||||
1452 | if (Bit < 0) | ||||||
1453 | return false; | ||||||
1454 | |||||||
1455 | Register TestReg = AndInst.getOperand(1).getReg(); | ||||||
1456 | |||||||
1457 | // Emit a TB(N)Z. | ||||||
1458 | emitTestBit(TestReg, Bit, Invert, DstMBB, MIB); | ||||||
1459 | return true; | ||||||
1460 | } | ||||||
1461 | |||||||
1462 | MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg, | ||||||
1463 | bool IsNegative, | ||||||
1464 | MachineBasicBlock *DestMBB, | ||||||
1465 | MachineIRBuilder &MIB) const { | ||||||
1466 | assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!")((ProduceNonFlagSettingCondBr && "CBZ does not set flags!" ) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"CBZ does not set flags!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1466, __PRETTY_FUNCTION__)); | ||||||
1467 | MachineRegisterInfo &MRI = *MIB.getMRI(); | ||||||
1468 | assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64 ::GPRRegBankID && "Expected GPRs only?") ? static_cast <void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1470, __PRETTY_FUNCTION__)) | ||||||
1469 | AArch64::GPRRegBankID &&((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64 ::GPRRegBankID && "Expected GPRs only?") ? static_cast <void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1470, __PRETTY_FUNCTION__)) | ||||||
1470 | "Expected GPRs only?")((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64 ::GPRRegBankID && "Expected GPRs only?") ? static_cast <void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1470, __PRETTY_FUNCTION__)); | ||||||
1471 | auto Ty = MRI.getType(CompareReg); | ||||||
1472 | unsigned Width = Ty.getSizeInBits(); | ||||||
1473 | assert(!Ty.isVector() && "Expected scalar only?")((!Ty.isVector() && "Expected scalar only?") ? static_cast <void> (0) : __assert_fail ("!Ty.isVector() && \"Expected scalar only?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1473, __PRETTY_FUNCTION__)); | ||||||
1474 | assert(Width <= 64 && "Expected width to be at most 64?")((Width <= 64 && "Expected width to be at most 64?" ) ? static_cast<void> (0) : __assert_fail ("Width <= 64 && \"Expected width to be at most 64?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1474, __PRETTY_FUNCTION__)); | ||||||
1475 | static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX}, | ||||||
1476 | {AArch64::CBNZW, AArch64::CBNZX}}; | ||||||
1477 | unsigned Opc = OpcTable[IsNegative][Width == 64]; | ||||||
1478 | auto BranchMI = MIB.buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB); | ||||||
1479 | constrainSelectedInstRegOperands(*BranchMI, TII, TRI, RBI); | ||||||
1480 | return &*BranchMI; | ||||||
1481 | } | ||||||
1482 | |||||||
1483 | bool AArch64InstructionSelector::selectCompareBranchFedByFCmp( | ||||||
1484 | MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const { | ||||||
1485 | assert(FCmp.getOpcode() == TargetOpcode::G_FCMP)((FCmp.getOpcode() == TargetOpcode::G_FCMP) ? static_cast< void> (0) : __assert_fail ("FCmp.getOpcode() == TargetOpcode::G_FCMP" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1485, __PRETTY_FUNCTION__)); | ||||||
1486 | assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void > (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1486, __PRETTY_FUNCTION__)); | ||||||
1487 | // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't | ||||||
1488 | // totally clean. Some of them require two branches to implement. | ||||||
1489 | auto Pred = (CmpInst::Predicate)FCmp.getOperand(1).getPredicate(); | ||||||
1490 | emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB, | ||||||
1491 | Pred); | ||||||
1492 | AArch64CC::CondCode CC1, CC2; | ||||||
1493 | changeFCMPPredToAArch64CC(static_cast<CmpInst::Predicate>(Pred), CC1, CC2); | ||||||
1494 | MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); | ||||||
1495 | MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB); | ||||||
1496 | if (CC2 != AArch64CC::AL) | ||||||
1497 | MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB); | ||||||
1498 | I.eraseFromParent(); | ||||||
1499 | return true; | ||||||
1500 | } | ||||||
1501 | |||||||
1502 | bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp( | ||||||
1503 | MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const { | ||||||
1504 | assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)((ICmp.getOpcode() == TargetOpcode::G_ICMP) ? static_cast< void> (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1504, __PRETTY_FUNCTION__)); | ||||||
1505 | assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void > (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1505, __PRETTY_FUNCTION__)); | ||||||
1506 | // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z. | ||||||
1507 | // | ||||||
1508 | // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z | ||||||
1509 | // instructions will not be produced, as they are conditional branch | ||||||
1510 | // instructions that do not set flags. | ||||||
1511 | if (!ProduceNonFlagSettingCondBr) | ||||||
1512 | return false; | ||||||
1513 | |||||||
1514 | MachineRegisterInfo &MRI = *MIB.getMRI(); | ||||||
1515 | MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); | ||||||
1516 | auto Pred = | ||||||
1517 | static_cast<CmpInst::Predicate>(ICmp.getOperand(1).getPredicate()); | ||||||
1518 | Register LHS = ICmp.getOperand(2).getReg(); | ||||||
1519 | Register RHS = ICmp.getOperand(3).getReg(); | ||||||
1520 | |||||||
1521 | // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that. | ||||||
1522 | auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI); | ||||||
1523 | MachineInstr *AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI); | ||||||
1524 | |||||||
1525 | // When we can emit a TB(N)Z, prefer that. | ||||||
1526 | // | ||||||
1527 | // Handle non-commutative condition codes first. | ||||||
1528 | // Note that we don't want to do this when we have a G_AND because it can | ||||||
1529 | // become a tst. The tst will make the test bit in the TB(N)Z redundant. | ||||||
1530 | if (VRegAndVal && !AndInst) { | ||||||
1531 | int64_t C = VRegAndVal->Value.getSExtValue(); | ||||||
1532 | |||||||
1533 | // When we have a greater-than comparison, we can just test if the msb is | ||||||
1534 | // zero. | ||||||
1535 | if (C == -1 && Pred == CmpInst::ICMP_SGT) { | ||||||
1536 | uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1; | ||||||
1537 | emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB); | ||||||
1538 | I.eraseFromParent(); | ||||||
1539 | return true; | ||||||
1540 | } | ||||||
1541 | |||||||
1542 | // When we have a less than comparison, we can just test if the msb is not | ||||||
1543 | // zero. | ||||||
1544 | if (C == 0 && Pred == CmpInst::ICMP_SLT) { | ||||||
1545 | uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1; | ||||||
1546 | emitTestBit(LHS, Bit, /*IsNegative = */ true, DestMBB, MIB); | ||||||
1547 | I.eraseFromParent(); | ||||||
1548 | return true; | ||||||
1549 | } | ||||||
1550 | } | ||||||
1551 | |||||||
1552 | // Attempt to handle commutative condition codes. Right now, that's only | ||||||
1553 | // eq/ne. | ||||||
1554 | if (ICmpInst::isEquality(Pred)) { | ||||||
1555 | if (!VRegAndVal) { | ||||||
1556 | std::swap(RHS, LHS); | ||||||
1557 | VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI); | ||||||
1558 | AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI); | ||||||
1559 | } | ||||||
1560 | |||||||
1561 | if (VRegAndVal && VRegAndVal->Value == 0) { | ||||||
1562 | // If there's a G_AND feeding into this branch, try to fold it away by | ||||||
1563 | // emitting a TB(N)Z instead. | ||||||
1564 | // | ||||||
1565 | // Note: If we have LT, then it *is* possible to fold, but it wouldn't be | ||||||
1566 | // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding | ||||||
1567 | // would be redundant. | ||||||
1568 | if (AndInst && | ||||||
1569 | tryOptAndIntoCompareBranch( | ||||||
1570 | *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB)) { | ||||||
1571 | I.eraseFromParent(); | ||||||
1572 | return true; | ||||||
1573 | } | ||||||
1574 | |||||||
1575 | // Otherwise, try to emit a CB(N)Z instead. | ||||||
1576 | auto LHSTy = MRI.getType(LHS); | ||||||
1577 | if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) { | ||||||
1578 | emitCBZ(LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB); | ||||||
1579 | I.eraseFromParent(); | ||||||
1580 | return true; | ||||||
1581 | } | ||||||
1582 | } | ||||||
1583 | } | ||||||
1584 | |||||||
1585 | return false; | ||||||
1586 | } | ||||||
1587 | |||||||
1588 | bool AArch64InstructionSelector::selectCompareBranchFedByICmp( | ||||||
1589 | MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const { | ||||||
1590 | assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)((ICmp.getOpcode() == TargetOpcode::G_ICMP) ? static_cast< void> (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1590, __PRETTY_FUNCTION__)); | ||||||
1591 | assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void > (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1591, __PRETTY_FUNCTION__)); | ||||||
1592 | if (tryOptCompareBranchFedByICmp(I, ICmp, MIB)) | ||||||
1593 | return true; | ||||||
1594 | |||||||
1595 | // Couldn't optimize. Emit a compare + a Bcc. | ||||||
1596 | MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); | ||||||
1597 | auto PredOp = ICmp.getOperand(1); | ||||||
1598 | emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB); | ||||||
1599 | const AArch64CC::CondCode CC = changeICMPPredToAArch64CC( | ||||||
1600 | static_cast<CmpInst::Predicate>(PredOp.getPredicate())); | ||||||
1601 | MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB); | ||||||
1602 | I.eraseFromParent(); | ||||||
1603 | return true; | ||||||
1604 | } | ||||||
1605 | |||||||
1606 | bool AArch64InstructionSelector::selectCompareBranch( | ||||||
1607 | MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { | ||||||
1608 | Register CondReg = I.getOperand(0).getReg(); | ||||||
1609 | MachineInstr *CCMI = MRI.getVRegDef(CondReg); | ||||||
1610 | if (CCMI->getOpcode() == TargetOpcode::G_TRUNC) { | ||||||
1611 | CondReg = CCMI->getOperand(1).getReg(); | ||||||
1612 | CCMI = MRI.getVRegDef(CondReg); | ||||||
1613 | } | ||||||
1614 | |||||||
1615 | // Try to select the G_BRCOND using whatever is feeding the condition if | ||||||
1616 | // possible. | ||||||
1617 | MachineIRBuilder MIB(I); | ||||||
1618 | unsigned CCMIOpc = CCMI->getOpcode(); | ||||||
1619 | if (CCMIOpc == TargetOpcode::G_FCMP) | ||||||
1620 | return selectCompareBranchFedByFCmp(I, *CCMI, MIB); | ||||||
1621 | if (CCMIOpc == TargetOpcode::G_ICMP) | ||||||
1622 | return selectCompareBranchFedByICmp(I, *CCMI, MIB); | ||||||
1623 | |||||||
1624 | // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z | ||||||
1625 | // instructions will not be produced, as they are conditional branch | ||||||
1626 | // instructions that do not set flags. | ||||||
1627 | if (ProduceNonFlagSettingCondBr) { | ||||||
1628 | emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true, | ||||||
1629 | I.getOperand(1).getMBB(), MIB); | ||||||
1630 | I.eraseFromParent(); | ||||||
1631 | return true; | ||||||
1632 | } | ||||||
1633 | |||||||
1634 | // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead. | ||||||
1635 | auto TstMI = | ||||||
1636 | MIB.buildInstr(AArch64::ANDSWri, {LLT::scalar(32)}, {CondReg}).addImm(1); | ||||||
1637 | constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI); | ||||||
1638 | auto Bcc = MIB.buildInstr(AArch64::Bcc) | ||||||
1639 | .addImm(AArch64CC::EQ) | ||||||
1640 | .addMBB(I.getOperand(1).getMBB()); | ||||||
1641 | I.eraseFromParent(); | ||||||
1642 | return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI); | ||||||
1643 | } | ||||||
1644 | |||||||
1645 | /// Returns the element immediate value of a vector shift operand if found. | ||||||
1646 | /// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR. | ||||||
1647 | static Optional<int64_t> getVectorShiftImm(Register Reg, | ||||||
1648 | MachineRegisterInfo &MRI) { | ||||||
1649 | assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand")((MRI.getType(Reg).isVector() && "Expected a *vector* shift operand" ) ? static_cast<void> (0) : __assert_fail ("MRI.getType(Reg).isVector() && \"Expected a *vector* shift operand\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1649, __PRETTY_FUNCTION__)); | ||||||
1650 | MachineInstr *OpMI = MRI.getVRegDef(Reg); | ||||||
1651 | assert(OpMI && "Expected to find a vreg def for vector shift operand")((OpMI && "Expected to find a vreg def for vector shift operand" ) ? static_cast<void> (0) : __assert_fail ("OpMI && \"Expected to find a vreg def for vector shift operand\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1651, __PRETTY_FUNCTION__)); | ||||||
1652 | if (OpMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR) | ||||||
1653 | return None; | ||||||
1654 | |||||||
1655 | // Check all operands are identical immediates. | ||||||
1656 | int64_t ImmVal = 0; | ||||||
1657 | for (unsigned Idx = 1; Idx < OpMI->getNumOperands(); ++Idx) { | ||||||
1658 | auto VRegAndVal = getConstantVRegValWithLookThrough(OpMI->getOperand(Idx).getReg(), MRI); | ||||||
1659 | if (!VRegAndVal) | ||||||
1660 | return None; | ||||||
1661 | |||||||
1662 | if (Idx == 1) | ||||||
1663 | ImmVal = VRegAndVal->Value.getSExtValue(); | ||||||
1664 | if (ImmVal != VRegAndVal->Value.getSExtValue()) | ||||||
1665 | return None; | ||||||
1666 | } | ||||||
1667 | |||||||
1668 | return ImmVal; | ||||||
1669 | } | ||||||
1670 | |||||||
1671 | /// Matches and returns the shift immediate value for a SHL instruction given | ||||||
1672 | /// a shift operand. | ||||||
1673 | static Optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI) { | ||||||
1674 | Optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI); | ||||||
1675 | if (!ShiftImm) | ||||||
1676 | return None; | ||||||
1677 | // Check the immediate is in range for a SHL. | ||||||
1678 | int64_t Imm = *ShiftImm; | ||||||
1679 | if (Imm < 0) | ||||||
1680 | return None; | ||||||
1681 | switch (SrcTy.getElementType().getSizeInBits()) { | ||||||
1682 | default: | ||||||
1683 | LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unhandled element type for vector shift" ; } } while (false); | ||||||
1684 | return None; | ||||||
1685 | case 8: | ||||||
1686 | if (Imm > 7) | ||||||
1687 | return None; | ||||||
1688 | break; | ||||||
1689 | case 16: | ||||||
1690 | if (Imm > 15) | ||||||
1691 | return None; | ||||||
1692 | break; | ||||||
1693 | case 32: | ||||||
1694 | if (Imm > 31) | ||||||
1695 | return None; | ||||||
1696 | break; | ||||||
1697 | case 64: | ||||||
1698 | if (Imm > 63) | ||||||
1699 | return None; | ||||||
1700 | break; | ||||||
1701 | } | ||||||
1702 | return Imm; | ||||||
1703 | } | ||||||
1704 | |||||||
1705 | bool AArch64InstructionSelector::selectVectorSHL( | ||||||
1706 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
1707 | assert(I.getOpcode() == TargetOpcode::G_SHL)((I.getOpcode() == TargetOpcode::G_SHL) ? static_cast<void > (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1707, __PRETTY_FUNCTION__)); | ||||||
1708 | Register DstReg = I.getOperand(0).getReg(); | ||||||
1709 | const LLT Ty = MRI.getType(DstReg); | ||||||
1710 | Register Src1Reg = I.getOperand(1).getReg(); | ||||||
1711 | Register Src2Reg = I.getOperand(2).getReg(); | ||||||
1712 | |||||||
1713 | if (!Ty.isVector()) | ||||||
1714 | return false; | ||||||
1715 | |||||||
1716 | // Check if we have a vector of constants on RHS that we can select as the | ||||||
1717 | // immediate form. | ||||||
1718 | Optional<int64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI); | ||||||
1719 | |||||||
1720 | unsigned Opc = 0; | ||||||
1721 | if (Ty == LLT::vector(2, 64)) { | ||||||
1722 | Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64; | ||||||
1723 | } else if (Ty == LLT::vector(4, 32)) { | ||||||
1724 | Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32; | ||||||
1725 | } else if (Ty == LLT::vector(2, 32)) { | ||||||
1726 | Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32; | ||||||
1727 | } else if (Ty == LLT::vector(4, 16)) { | ||||||
1728 | Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16; | ||||||
1729 | } else if (Ty == LLT::vector(8, 16)) { | ||||||
1730 | Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16; | ||||||
1731 | } else if (Ty == LLT::vector(16, 8)) { | ||||||
1732 | Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8; | ||||||
1733 | } else if (Ty == LLT::vector(8, 8)) { | ||||||
1734 | Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8; | ||||||
1735 | } else { | ||||||
1736 | LLVM_DEBUG(dbgs() << "Unhandled G_SHL type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unhandled G_SHL type"; } } while (false); | ||||||
1737 | return false; | ||||||
1738 | } | ||||||
1739 | |||||||
1740 | MachineIRBuilder MIB(I); | ||||||
1741 | auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg}); | ||||||
1742 | if (ImmVal) | ||||||
1743 | Shl.addImm(*ImmVal); | ||||||
1744 | else | ||||||
1745 | Shl.addUse(Src2Reg); | ||||||
1746 | constrainSelectedInstRegOperands(*Shl, TII, TRI, RBI); | ||||||
1747 | I.eraseFromParent(); | ||||||
1748 | return true; | ||||||
1749 | } | ||||||
1750 | |||||||
1751 | bool AArch64InstructionSelector::selectVectorAshrLshr( | ||||||
1752 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
1753 | assert(I.getOpcode() == TargetOpcode::G_ASHR ||((I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode ::G_LSHR) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1754, __PRETTY_FUNCTION__)) | ||||||
1754 | I.getOpcode() == TargetOpcode::G_LSHR)((I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode ::G_LSHR) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1754, __PRETTY_FUNCTION__)); | ||||||
1755 | Register DstReg = I.getOperand(0).getReg(); | ||||||
1756 | const LLT Ty = MRI.getType(DstReg); | ||||||
1757 | Register Src1Reg = I.getOperand(1).getReg(); | ||||||
1758 | Register Src2Reg = I.getOperand(2).getReg(); | ||||||
1759 | |||||||
1760 | if (!Ty.isVector()) | ||||||
1761 | return false; | ||||||
1762 | |||||||
1763 | bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR; | ||||||
1764 | |||||||
1765 | // We expect the immediate case to be lowered in the PostLegalCombiner to | ||||||
1766 | // AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents. | ||||||
1767 | |||||||
1768 | // There is not a shift right register instruction, but the shift left | ||||||
1769 | // register instruction takes a signed value, where negative numbers specify a | ||||||
1770 | // right shift. | ||||||
1771 | |||||||
1772 | unsigned Opc = 0; | ||||||
1773 | unsigned NegOpc = 0; | ||||||
1774 | const TargetRegisterClass *RC = | ||||||
1775 | getRegClassForTypeOnBank(Ty, RBI.getRegBank(AArch64::FPRRegBankID), RBI); | ||||||
1776 | if (Ty == LLT::vector(2, 64)) { | ||||||
1777 | Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64; | ||||||
1778 | NegOpc = AArch64::NEGv2i64; | ||||||
1779 | } else if (Ty == LLT::vector(4, 32)) { | ||||||
1780 | Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32; | ||||||
1781 | NegOpc = AArch64::NEGv4i32; | ||||||
1782 | } else if (Ty == LLT::vector(2, 32)) { | ||||||
1783 | Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32; | ||||||
1784 | NegOpc = AArch64::NEGv2i32; | ||||||
1785 | } else if (Ty == LLT::vector(4, 16)) { | ||||||
1786 | Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16; | ||||||
1787 | NegOpc = AArch64::NEGv4i16; | ||||||
1788 | } else if (Ty == LLT::vector(8, 16)) { | ||||||
1789 | Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16; | ||||||
1790 | NegOpc = AArch64::NEGv8i16; | ||||||
1791 | } else if (Ty == LLT::vector(16, 8)) { | ||||||
1792 | Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8; | ||||||
1793 | NegOpc = AArch64::NEGv8i16; | ||||||
1794 | } else if (Ty == LLT::vector(8, 8)) { | ||||||
1795 | Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8; | ||||||
1796 | NegOpc = AArch64::NEGv8i8; | ||||||
1797 | } else { | ||||||
1798 | LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unhandled G_ASHR type"; } } while (false); | ||||||
1799 | return false; | ||||||
1800 | } | ||||||
1801 | |||||||
1802 | MachineIRBuilder MIB(I); | ||||||
1803 | auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg}); | ||||||
1804 | constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI); | ||||||
1805 | auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg}); | ||||||
1806 | constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI); | ||||||
1807 | I.eraseFromParent(); | ||||||
1808 | return true; | ||||||
1809 | } | ||||||
1810 | |||||||
1811 | bool AArch64InstructionSelector::selectVaStartAAPCS( | ||||||
1812 | MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { | ||||||
1813 | return false; | ||||||
1814 | } | ||||||
1815 | |||||||
1816 | bool AArch64InstructionSelector::selectVaStartDarwin( | ||||||
1817 | MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const { | ||||||
1818 | AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>(); | ||||||
1819 | Register ListReg = I.getOperand(0).getReg(); | ||||||
1820 | |||||||
1821 | Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); | ||||||
1822 | |||||||
1823 | auto MIB = | ||||||
1824 | BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri)) | ||||||
1825 | .addDef(ArgsAddrReg) | ||||||
1826 | .addFrameIndex(FuncInfo->getVarArgsStackIndex()) | ||||||
1827 | .addImm(0) | ||||||
1828 | .addImm(0); | ||||||
1829 | |||||||
1830 | constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); | ||||||
1831 | |||||||
1832 | MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui)) | ||||||
1833 | .addUse(ArgsAddrReg) | ||||||
1834 | .addUse(ListReg) | ||||||
1835 | .addImm(0) | ||||||
1836 | .addMemOperand(*I.memoperands_begin()); | ||||||
1837 | |||||||
1838 | constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); | ||||||
1839 | I.eraseFromParent(); | ||||||
1840 | return true; | ||||||
1841 | } | ||||||
1842 | |||||||
1843 | void AArch64InstructionSelector::materializeLargeCMVal( | ||||||
1844 | MachineInstr &I, const Value *V, unsigned OpFlags) const { | ||||||
1845 | MachineBasicBlock &MBB = *I.getParent(); | ||||||
1846 | MachineFunction &MF = *MBB.getParent(); | ||||||
1847 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||
1848 | MachineIRBuilder MIB(I); | ||||||
1849 | |||||||
1850 | auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {}); | ||||||
1851 | MovZ->addOperand(MF, I.getOperand(1)); | ||||||
1852 | MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 | | ||||||
1853 | AArch64II::MO_NC); | ||||||
1854 | MovZ->addOperand(MF, MachineOperand::CreateImm(0)); | ||||||
1855 | constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI); | ||||||
1856 | |||||||
1857 | auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset, | ||||||
1858 | Register ForceDstReg) { | ||||||
1859 | Register DstReg = ForceDstReg | ||||||
1860 | ? ForceDstReg | ||||||
1861 | : MRI.createVirtualRegister(&AArch64::GPR64RegClass); | ||||||
1862 | auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg); | ||||||
1863 | if (auto *GV = dyn_cast<GlobalValue>(V)) { | ||||||
1864 | MovI->addOperand(MF, MachineOperand::CreateGA( | ||||||
1865 | GV, MovZ->getOperand(1).getOffset(), Flags)); | ||||||
1866 | } else { | ||||||
1867 | MovI->addOperand( | ||||||
1868 | MF, MachineOperand::CreateBA(cast<BlockAddress>(V), | ||||||
1869 | MovZ->getOperand(1).getOffset(), Flags)); | ||||||
1870 | } | ||||||
1871 | MovI->addOperand(MF, MachineOperand::CreateImm(Offset)); | ||||||
1872 | constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI); | ||||||
1873 | return DstReg; | ||||||
1874 | }; | ||||||
1875 | Register DstReg = BuildMovK(MovZ.getReg(0), | ||||||
1876 | AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0); | ||||||
1877 | DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0); | ||||||
1878 | BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg()); | ||||||
1879 | } | ||||||
1880 | |||||||
1881 | bool AArch64InstructionSelector::preISelLower(MachineInstr &I) { | ||||||
1882 | MachineBasicBlock &MBB = *I.getParent(); | ||||||
1883 | MachineFunction &MF = *MBB.getParent(); | ||||||
1884 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||
1885 | |||||||
1886 | switch (I.getOpcode()) { | ||||||
1887 | case TargetOpcode::G_SHL: | ||||||
1888 | case TargetOpcode::G_ASHR: | ||||||
1889 | case TargetOpcode::G_LSHR: { | ||||||
1890 | // These shifts are legalized to have 64 bit shift amounts because we want | ||||||
1891 | // to take advantage of the existing imported selection patterns that assume | ||||||
1892 | // the immediates are s64s. However, if the shifted type is 32 bits and for | ||||||
1893 | // some reason we receive input GMIR that has an s64 shift amount that's not | ||||||
1894 | // a G_CONSTANT, insert a truncate so that we can still select the s32 | ||||||
1895 | // register-register variant. | ||||||
1896 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
1897 | Register ShiftReg = I.getOperand(2).getReg(); | ||||||
1898 | const LLT ShiftTy = MRI.getType(ShiftReg); | ||||||
1899 | const LLT SrcTy = MRI.getType(SrcReg); | ||||||
1900 | if (SrcTy.isVector()) | ||||||
1901 | return false; | ||||||
1902 | assert(!ShiftTy.isVector() && "unexpected vector shift ty")((!ShiftTy.isVector() && "unexpected vector shift ty" ) ? static_cast<void> (0) : __assert_fail ("!ShiftTy.isVector() && \"unexpected vector shift ty\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1902, __PRETTY_FUNCTION__)); | ||||||
1903 | if (SrcTy.getSizeInBits() != 32 || ShiftTy.getSizeInBits() != 64) | ||||||
1904 | return false; | ||||||
1905 | auto *AmtMI = MRI.getVRegDef(ShiftReg); | ||||||
1906 | assert(AmtMI && "could not find a vreg definition for shift amount")((AmtMI && "could not find a vreg definition for shift amount" ) ? static_cast<void> (0) : __assert_fail ("AmtMI && \"could not find a vreg definition for shift amount\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1906, __PRETTY_FUNCTION__)); | ||||||
1907 | if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) { | ||||||
1908 | // Insert a subregister copy to implement a 64->32 trunc | ||||||
1909 | MachineIRBuilder MIB(I); | ||||||
1910 | auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {}) | ||||||
1911 | .addReg(ShiftReg, 0, AArch64::sub_32); | ||||||
1912 | MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID)); | ||||||
1913 | I.getOperand(2).setReg(Trunc.getReg(0)); | ||||||
1914 | } | ||||||
1915 | return true; | ||||||
1916 | } | ||||||
1917 | case TargetOpcode::G_STORE: | ||||||
1918 | return contractCrossBankCopyIntoStore(I, MRI); | ||||||
1919 | case TargetOpcode::G_PTR_ADD: | ||||||
1920 | return convertPtrAddToAdd(I, MRI); | ||||||
1921 | case TargetOpcode::G_LOAD: { | ||||||
1922 | // For scalar loads of pointers, we try to convert the dest type from p0 | ||||||
1923 | // to s64 so that our imported patterns can match. Like with the G_PTR_ADD | ||||||
1924 | // conversion, this should be ok because all users should have been | ||||||
1925 | // selected already, so the type doesn't matter for them. | ||||||
1926 | Register DstReg = I.getOperand(0).getReg(); | ||||||
1927 | const LLT DstTy = MRI.getType(DstReg); | ||||||
1928 | if (!DstTy.isPointer()) | ||||||
1929 | return false; | ||||||
1930 | MRI.setType(DstReg, LLT::scalar(64)); | ||||||
1931 | return true; | ||||||
1932 | } | ||||||
1933 | case AArch64::G_DUP: { | ||||||
1934 | // Convert the type from p0 to s64 to help selection. | ||||||
1935 | LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
1936 | if (!DstTy.getElementType().isPointer()) | ||||||
1937 | return false; | ||||||
1938 | MachineIRBuilder MIB(I); | ||||||
1939 | auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(1).getReg()); | ||||||
1940 | MRI.setType(I.getOperand(0).getReg(), | ||||||
1941 | DstTy.changeElementType(LLT::scalar(64))); | ||||||
1942 | MRI.setRegBank(NewSrc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID)); | ||||||
1943 | I.getOperand(1).setReg(NewSrc.getReg(0)); | ||||||
1944 | return true; | ||||||
1945 | } | ||||||
1946 | case TargetOpcode::G_UITOFP: | ||||||
1947 | case TargetOpcode::G_SITOFP: { | ||||||
1948 | // If both source and destination regbanks are FPR, then convert the opcode | ||||||
1949 | // to G_SITOF so that the importer can select it to an fpr variant. | ||||||
1950 | // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank | ||||||
1951 | // copy. | ||||||
1952 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
1953 | LLT SrcTy = MRI.getType(SrcReg); | ||||||
1954 | LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
1955 | if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits()) | ||||||
1956 | return false; | ||||||
1957 | |||||||
1958 | if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) { | ||||||
1959 | if (I.getOpcode() == TargetOpcode::G_SITOFP) | ||||||
1960 | I.setDesc(TII.get(AArch64::G_SITOF)); | ||||||
1961 | else | ||||||
1962 | I.setDesc(TII.get(AArch64::G_UITOF)); | ||||||
1963 | return true; | ||||||
1964 | } | ||||||
1965 | return false; | ||||||
1966 | } | ||||||
1967 | default: | ||||||
1968 | return false; | ||||||
1969 | } | ||||||
1970 | } | ||||||
1971 | |||||||
1972 | /// This lowering tries to look for G_PTR_ADD instructions and then converts | ||||||
1973 | /// them to a standard G_ADD with a COPY on the source. | ||||||
1974 | /// | ||||||
1975 | /// The motivation behind this is to expose the add semantics to the imported | ||||||
1976 | /// tablegen patterns. We shouldn't need to check for uses being loads/stores, | ||||||
1977 | /// because the selector works bottom up, uses before defs. By the time we | ||||||
1978 | /// end up trying to select a G_PTR_ADD, we should have already attempted to | ||||||
1979 | /// fold this into addressing modes and were therefore unsuccessful. | ||||||
1980 | bool AArch64InstructionSelector::convertPtrAddToAdd( | ||||||
1981 | MachineInstr &I, MachineRegisterInfo &MRI) { | ||||||
1982 | assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD")((I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_PTR_ADD && \"Expected G_PTR_ADD\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 1982, __PRETTY_FUNCTION__)); | ||||||
1983 | Register DstReg = I.getOperand(0).getReg(); | ||||||
1984 | Register AddOp1Reg = I.getOperand(1).getReg(); | ||||||
1985 | const LLT PtrTy = MRI.getType(DstReg); | ||||||
1986 | if (PtrTy.getAddressSpace() != 0) | ||||||
1987 | return false; | ||||||
1988 | |||||||
1989 | MachineIRBuilder MIB(I); | ||||||
1990 | const LLT CastPtrTy = PtrTy.isVector() ? LLT::vector(2, 64) : LLT::scalar(64); | ||||||
1991 | auto PtrToInt = MIB.buildPtrToInt(CastPtrTy, AddOp1Reg); | ||||||
1992 | // Set regbanks on the registers. | ||||||
1993 | if (PtrTy.isVector()) | ||||||
1994 | MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID)); | ||||||
1995 | else | ||||||
1996 | MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID)); | ||||||
1997 | |||||||
1998 | // Now turn the %dst(p0) = G_PTR_ADD %base, off into: | ||||||
1999 | // %dst(intty) = G_ADD %intbase, off | ||||||
2000 | I.setDesc(TII.get(TargetOpcode::G_ADD)); | ||||||
2001 | MRI.setType(DstReg, CastPtrTy); | ||||||
2002 | I.getOperand(1).setReg(PtrToInt.getReg(0)); | ||||||
2003 | if (!select(*PtrToInt)) { | ||||||
2004 | LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd" ; } } while (false); | ||||||
2005 | return false; | ||||||
2006 | } | ||||||
2007 | |||||||
2008 | // Also take the opportunity here to try to do some optimization. | ||||||
2009 | // Try to convert this into a G_SUB if the offset is a 0-x negate idiom. | ||||||
2010 | Register NegatedReg; | ||||||
2011 | if (!mi_match(I.getOperand(2).getReg(), MRI, m_Neg(m_Reg(NegatedReg)))) | ||||||
2012 | return true; | ||||||
2013 | I.getOperand(2).setReg(NegatedReg); | ||||||
2014 | I.setDesc(TII.get(TargetOpcode::G_SUB)); | ||||||
2015 | return true; | ||||||
2016 | } | ||||||
2017 | |||||||
2018 | bool AArch64InstructionSelector::earlySelectSHL( | ||||||
2019 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
2020 | // We try to match the immediate variant of LSL, which is actually an alias | ||||||
2021 | // for a special case of UBFM. Otherwise, we fall back to the imported | ||||||
2022 | // selector which will match the register variant. | ||||||
2023 | assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op")((I.getOpcode() == TargetOpcode::G_SHL && "unexpected op" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL && \"unexpected op\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2023, __PRETTY_FUNCTION__)); | ||||||
2024 | const auto &MO = I.getOperand(2); | ||||||
2025 | auto VRegAndVal = getConstantVRegVal(MO.getReg(), MRI); | ||||||
2026 | if (!VRegAndVal) | ||||||
2027 | return false; | ||||||
2028 | |||||||
2029 | const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
2030 | if (DstTy.isVector()) | ||||||
2031 | return false; | ||||||
2032 | bool Is64Bit = DstTy.getSizeInBits() == 64; | ||||||
2033 | auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO); | ||||||
2034 | auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO); | ||||||
2035 | MachineIRBuilder MIB(I); | ||||||
2036 | |||||||
2037 | if (!Imm1Fn || !Imm2Fn) | ||||||
2038 | return false; | ||||||
2039 | |||||||
2040 | auto NewI = | ||||||
2041 | MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri, | ||||||
2042 | {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()}); | ||||||
2043 | |||||||
2044 | for (auto &RenderFn : *Imm1Fn) | ||||||
2045 | RenderFn(NewI); | ||||||
2046 | for (auto &RenderFn : *Imm2Fn) | ||||||
2047 | RenderFn(NewI); | ||||||
2048 | |||||||
2049 | I.eraseFromParent(); | ||||||
2050 | return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI); | ||||||
2051 | } | ||||||
2052 | |||||||
2053 | bool AArch64InstructionSelector::contractCrossBankCopyIntoStore( | ||||||
2054 | MachineInstr &I, MachineRegisterInfo &MRI) { | ||||||
2055 | assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE")((I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_STORE && \"Expected G_STORE\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2055, __PRETTY_FUNCTION__)); | ||||||
2056 | // If we're storing a scalar, it doesn't matter what register bank that | ||||||
2057 | // scalar is on. All that matters is the size. | ||||||
2058 | // | ||||||
2059 | // So, if we see something like this (with a 32-bit scalar as an example): | ||||||
2060 | // | ||||||
2061 | // %x:gpr(s32) = ... something ... | ||||||
2062 | // %y:fpr(s32) = COPY %x:gpr(s32) | ||||||
2063 | // G_STORE %y:fpr(s32) | ||||||
2064 | // | ||||||
2065 | // We can fix this up into something like this: | ||||||
2066 | // | ||||||
2067 | // G_STORE %x:gpr(s32) | ||||||
2068 | // | ||||||
2069 | // And then continue the selection process normally. | ||||||
2070 | Register DefDstReg = getSrcRegIgnoringCopies(I.getOperand(0).getReg(), MRI); | ||||||
2071 | if (!DefDstReg.isValid()) | ||||||
2072 | return false; | ||||||
2073 | LLT DefDstTy = MRI.getType(DefDstReg); | ||||||
2074 | Register StoreSrcReg = I.getOperand(0).getReg(); | ||||||
2075 | LLT StoreSrcTy = MRI.getType(StoreSrcReg); | ||||||
2076 | |||||||
2077 | // If we get something strange like a physical register, then we shouldn't | ||||||
2078 | // go any further. | ||||||
2079 | if (!DefDstTy.isValid()) | ||||||
2080 | return false; | ||||||
2081 | |||||||
2082 | // Are the source and dst types the same size? | ||||||
2083 | if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits()) | ||||||
2084 | return false; | ||||||
2085 | |||||||
2086 | if (RBI.getRegBank(StoreSrcReg, MRI, TRI) == | ||||||
2087 | RBI.getRegBank(DefDstReg, MRI, TRI)) | ||||||
2088 | return false; | ||||||
2089 | |||||||
2090 | // We have a cross-bank copy, which is entering a store. Let's fold it. | ||||||
2091 | I.getOperand(0).setReg(DefDstReg); | ||||||
2092 | return true; | ||||||
2093 | } | ||||||
2094 | |||||||
2095 | bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const { | ||||||
2096 | assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!" ) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2096, __PRETTY_FUNCTION__)); | ||||||
2097 | assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!" ) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2097, __PRETTY_FUNCTION__)); | ||||||
2098 | |||||||
2099 | MachineBasicBlock &MBB = *I.getParent(); | ||||||
2100 | MachineFunction &MF = *MBB.getParent(); | ||||||
2101 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||
2102 | |||||||
2103 | switch (I.getOpcode()) { | ||||||
2104 | case TargetOpcode::G_BR: { | ||||||
2105 | // If the branch jumps to the fallthrough block, don't bother emitting it. | ||||||
2106 | // Only do this for -O0 for a good code size improvement, because when | ||||||
2107 | // optimizations are enabled we want to leave this choice to | ||||||
2108 | // MachineBlockPlacement. | ||||||
2109 | bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOpt::None; | ||||||
2110 | if (EnableOpt || !MBB.isLayoutSuccessor(I.getOperand(0).getMBB())) | ||||||
2111 | return false; | ||||||
2112 | I.eraseFromParent(); | ||||||
2113 | return true; | ||||||
2114 | } | ||||||
2115 | case TargetOpcode::G_SHL: | ||||||
2116 | return earlySelectSHL(I, MRI); | ||||||
2117 | case TargetOpcode::G_CONSTANT: { | ||||||
2118 | bool IsZero = false; | ||||||
2119 | if (I.getOperand(1).isCImm()) | ||||||
2120 | IsZero = I.getOperand(1).getCImm()->getZExtValue() == 0; | ||||||
2121 | else if (I.getOperand(1).isImm()) | ||||||
2122 | IsZero = I.getOperand(1).getImm() == 0; | ||||||
2123 | |||||||
2124 | if (!IsZero) | ||||||
2125 | return false; | ||||||
2126 | |||||||
2127 | Register DefReg = I.getOperand(0).getReg(); | ||||||
2128 | LLT Ty = MRI.getType(DefReg); | ||||||
2129 | if (Ty.getSizeInBits() == 64) { | ||||||
2130 | I.getOperand(1).ChangeToRegister(AArch64::XZR, false); | ||||||
2131 | RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI); | ||||||
2132 | } else if (Ty.getSizeInBits() == 32) { | ||||||
2133 | I.getOperand(1).ChangeToRegister(AArch64::WZR, false); | ||||||
2134 | RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI); | ||||||
2135 | } else | ||||||
2136 | return false; | ||||||
2137 | |||||||
2138 | I.setDesc(TII.get(TargetOpcode::COPY)); | ||||||
2139 | return true; | ||||||
2140 | } | ||||||
2141 | default: | ||||||
2142 | return false; | ||||||
2143 | } | ||||||
2144 | } | ||||||
2145 | |||||||
2146 | bool AArch64InstructionSelector::select(MachineInstr &I) { | ||||||
2147 | assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!" ) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2147, __PRETTY_FUNCTION__)); | ||||||
2148 | assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!" ) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2148, __PRETTY_FUNCTION__)); | ||||||
2149 | |||||||
2150 | MachineBasicBlock &MBB = *I.getParent(); | ||||||
2151 | MachineFunction &MF = *MBB.getParent(); | ||||||
2152 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||
2153 | |||||||
2154 | const AArch64Subtarget *Subtarget = | ||||||
2155 | &static_cast<const AArch64Subtarget &>(MF.getSubtarget()); | ||||||
2156 | if (Subtarget->requiresStrictAlign()) { | ||||||
2157 | // We don't support this feature yet. | ||||||
2158 | LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "AArch64 GISel does not support strict-align yet\n" ; } } while (false); | ||||||
2159 | return false; | ||||||
2160 | } | ||||||
2161 | |||||||
2162 | unsigned Opcode = I.getOpcode(); | ||||||
2163 | // G_PHI requires same handling as PHI | ||||||
2164 | if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) { | ||||||
2165 | // Certain non-generic instructions also need some special handling. | ||||||
2166 | |||||||
2167 | if (Opcode == TargetOpcode::LOAD_STACK_GUARD) | ||||||
2168 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2169 | |||||||
2170 | if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) { | ||||||
2171 | const Register DefReg = I.getOperand(0).getReg(); | ||||||
2172 | const LLT DefTy = MRI.getType(DefReg); | ||||||
2173 | |||||||
2174 | const RegClassOrRegBank &RegClassOrBank = | ||||||
2175 | MRI.getRegClassOrRegBank(DefReg); | ||||||
2176 | |||||||
2177 | const TargetRegisterClass *DefRC | ||||||
2178 | = RegClassOrBank.dyn_cast<const TargetRegisterClass *>(); | ||||||
2179 | if (!DefRC) { | ||||||
2180 | if (!DefTy.isValid()) { | ||||||
2181 | LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "PHI operand has no type, not a gvreg?\n" ; } } while (false); | ||||||
2182 | return false; | ||||||
2183 | } | ||||||
2184 | const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>(); | ||||||
2185 | DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI); | ||||||
2186 | if (!DefRC) { | ||||||
2187 | LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "PHI operand has unexpected size/bank\n" ; } } while (false); | ||||||
2188 | return false; | ||||||
2189 | } | ||||||
2190 | } | ||||||
2191 | |||||||
2192 | I.setDesc(TII.get(TargetOpcode::PHI)); | ||||||
2193 | |||||||
2194 | return RBI.constrainGenericRegister(DefReg, *DefRC, MRI); | ||||||
2195 | } | ||||||
2196 | |||||||
2197 | if (I.isCopy()) | ||||||
2198 | return selectCopy(I, TII, MRI, TRI, RBI); | ||||||
2199 | |||||||
2200 | return true; | ||||||
2201 | } | ||||||
2202 | |||||||
2203 | |||||||
2204 | if (I.getNumOperands() != I.getNumExplicitOperands()) { | ||||||
2205 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n" ; } } while (false) | ||||||
2206 | dbgs() << "Generic instruction has unexpected implicit operands\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n" ; } } while (false); | ||||||
2207 | return false; | ||||||
2208 | } | ||||||
2209 | |||||||
2210 | // Try to do some lowering before we start instruction selecting. These | ||||||
2211 | // lowerings are purely transformations on the input G_MIR and so selection | ||||||
2212 | // must continue after any modification of the instruction. | ||||||
2213 | if (preISelLower(I)) { | ||||||
2214 | Opcode = I.getOpcode(); // The opcode may have been modified, refresh it. | ||||||
2215 | } | ||||||
2216 | |||||||
2217 | // There may be patterns where the importer can't deal with them optimally, | ||||||
2218 | // but does select it to a suboptimal sequence so our custom C++ selection | ||||||
2219 | // code later never has a chance to work on it. Therefore, we have an early | ||||||
2220 | // selection attempt here to give priority to certain selection routines | ||||||
2221 | // over the imported ones. | ||||||
2222 | if (earlySelect(I)) | ||||||
2223 | return true; | ||||||
2224 | |||||||
2225 | if (selectImpl(I, *CoverageInfo)) | ||||||
2226 | return true; | ||||||
2227 | |||||||
2228 | LLT Ty = | ||||||
2229 | I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{}; | ||||||
2230 | |||||||
2231 | MachineIRBuilder MIB(I); | ||||||
2232 | |||||||
2233 | switch (Opcode) { | ||||||
2234 | case TargetOpcode::G_BRCOND: | ||||||
2235 | return selectCompareBranch(I, MF, MRI); | ||||||
2236 | |||||||
2237 | case TargetOpcode::G_BRINDIRECT: { | ||||||
2238 | I.setDesc(TII.get(AArch64::BR)); | ||||||
2239 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2240 | } | ||||||
2241 | |||||||
2242 | case TargetOpcode::G_BRJT: | ||||||
2243 | return selectBrJT(I, MRI); | ||||||
2244 | |||||||
2245 | case AArch64::G_ADD_LOW: { | ||||||
2246 | // This op may have been separated from it's ADRP companion by the localizer | ||||||
2247 | // or some other code motion pass. Given that many CPUs will try to | ||||||
2248 | // macro fuse these operations anyway, select this into a MOVaddr pseudo | ||||||
2249 | // which will later be expanded into an ADRP+ADD pair after scheduling. | ||||||
2250 | MachineInstr *BaseMI = MRI.getVRegDef(I.getOperand(1).getReg()); | ||||||
2251 | if (BaseMI->getOpcode() != AArch64::ADRP) { | ||||||
2252 | I.setDesc(TII.get(AArch64::ADDXri)); | ||||||
2253 | I.addOperand(MachineOperand::CreateImm(0)); | ||||||
2254 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2255 | } | ||||||
2256 | assert(TM.getCodeModel() == CodeModel::Small &&((TM.getCodeModel() == CodeModel::Small && "Expected small code model" ) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2257, __PRETTY_FUNCTION__)) | ||||||
2257 | "Expected small code model")((TM.getCodeModel() == CodeModel::Small && "Expected small code model" ) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2257, __PRETTY_FUNCTION__)); | ||||||
2258 | MachineIRBuilder MIB(I); | ||||||
2259 | auto Op1 = BaseMI->getOperand(1); | ||||||
2260 | auto Op2 = I.getOperand(2); | ||||||
2261 | auto MovAddr = MIB.buildInstr(AArch64::MOVaddr, {I.getOperand(0)}, {}) | ||||||
2262 | .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(), | ||||||
2263 | Op1.getTargetFlags()) | ||||||
2264 | .addGlobalAddress(Op2.getGlobal(), Op2.getOffset(), | ||||||
2265 | Op2.getTargetFlags()); | ||||||
2266 | I.eraseFromParent(); | ||||||
2267 | return constrainSelectedInstRegOperands(*MovAddr, TII, TRI, RBI); | ||||||
2268 | } | ||||||
2269 | |||||||
2270 | case TargetOpcode::G_BSWAP: { | ||||||
2271 | // Handle vector types for G_BSWAP directly. | ||||||
2272 | Register DstReg = I.getOperand(0).getReg(); | ||||||
2273 | LLT DstTy = MRI.getType(DstReg); | ||||||
2274 | |||||||
2275 | // We should only get vector types here; everything else is handled by the | ||||||
2276 | // importer right now. | ||||||
2277 | if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) { | ||||||
2278 | LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Dst type for G_BSWAP currently unsupported.\n" ; } } while (false); | ||||||
2279 | return false; | ||||||
2280 | } | ||||||
2281 | |||||||
2282 | // Only handle 4 and 2 element vectors for now. | ||||||
2283 | // TODO: 16-bit elements. | ||||||
2284 | unsigned NumElts = DstTy.getNumElements(); | ||||||
2285 | if (NumElts != 4 && NumElts != 2) { | ||||||
2286 | LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported number of elements for G_BSWAP.\n" ; } } while (false); | ||||||
2287 | return false; | ||||||
2288 | } | ||||||
2289 | |||||||
2290 | // Choose the correct opcode for the supported types. Right now, that's | ||||||
2291 | // v2s32, v4s32, and v2s64. | ||||||
2292 | unsigned Opc = 0; | ||||||
2293 | unsigned EltSize = DstTy.getElementType().getSizeInBits(); | ||||||
2294 | if (EltSize == 32) | ||||||
2295 | Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8 | ||||||
2296 | : AArch64::REV32v16i8; | ||||||
2297 | else if (EltSize == 64) | ||||||
2298 | Opc = AArch64::REV64v16i8; | ||||||
2299 | |||||||
2300 | // We should always get something by the time we get here... | ||||||
2301 | assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?")((Opc != 0 && "Didn't get an opcode for G_BSWAP?") ? static_cast <void> (0) : __assert_fail ("Opc != 0 && \"Didn't get an opcode for G_BSWAP?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2301, __PRETTY_FUNCTION__)); | ||||||
2302 | |||||||
2303 | I.setDesc(TII.get(Opc)); | ||||||
2304 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2305 | } | ||||||
2306 | |||||||
2307 | case TargetOpcode::G_FCONSTANT: | ||||||
2308 | case TargetOpcode::G_CONSTANT: { | ||||||
2309 | const bool isFP = Opcode == TargetOpcode::G_FCONSTANT; | ||||||
2310 | |||||||
2311 | const LLT s8 = LLT::scalar(8); | ||||||
2312 | const LLT s16 = LLT::scalar(16); | ||||||
2313 | const LLT s32 = LLT::scalar(32); | ||||||
2314 | const LLT s64 = LLT::scalar(64); | ||||||
2315 | const LLT s128 = LLT::scalar(128); | ||||||
2316 | const LLT p0 = LLT::pointer(0, 64); | ||||||
2317 | |||||||
2318 | const Register DefReg = I.getOperand(0).getReg(); | ||||||
2319 | const LLT DefTy = MRI.getType(DefReg); | ||||||
2320 | const unsigned DefSize = DefTy.getSizeInBits(); | ||||||
2321 | const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); | ||||||
2322 | |||||||
2323 | // FIXME: Redundant check, but even less readable when factored out. | ||||||
2324 | if (isFP) { | ||||||
2325 | if (Ty != s32 && Ty != s64 && Ty != s128) { | ||||||
2326 | LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize FP " << Ty << " constant, expected: " << s32 << " or " << s64 << " or " << s128 << '\n' ; } } while (false) | ||||||
2327 | << " constant, expected: " << s32 << " or " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize FP " << Ty << " constant, expected: " << s32 << " or " << s64 << " or " << s128 << '\n' ; } } while (false) | ||||||
2328 | << " or " << s128 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize FP " << Ty << " constant, expected: " << s32 << " or " << s64 << " or " << s128 << '\n' ; } } while (false); | ||||||
2329 | return false; | ||||||
2330 | } | ||||||
2331 | |||||||
2332 | if (RB.getID() != AArch64::FPRRegBankID) { | ||||||
2333 | LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize FP " << Ty << " constant on bank: " << RB << ", expected: FPR\n"; } } while (false) | ||||||
2334 | << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize FP " << Ty << " constant on bank: " << RB << ", expected: FPR\n"; } } while (false) | ||||||
2335 | << ", expected: FPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize FP " << Ty << " constant on bank: " << RB << ", expected: FPR\n"; } } while (false); | ||||||
2336 | return false; | ||||||
2337 | } | ||||||
2338 | |||||||
2339 | // The case when we have 0.0 is covered by tablegen. Reject it here so we | ||||||
2340 | // can be sure tablegen works correctly and isn't rescued by this code. | ||||||
2341 | // 0.0 is not covered by tablegen for FP128. So we will handle this | ||||||
2342 | // scenario in the code here. | ||||||
2343 | if (DefSize != 128 && I.getOperand(1).getFPImm()->isExactlyValue(0.0)) | ||||||
2344 | return false; | ||||||
2345 | } else { | ||||||
2346 | // s32 and s64 are covered by tablegen. | ||||||
2347 | if (Ty != p0 && Ty != s8 && Ty != s16) { | ||||||
2348 | LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize integer " << Ty << " constant, expected: " << s32 << ", " << s64 << ", or " << p0 << '\n' ; } } while (false) | ||||||
2349 | << " constant, expected: " << s32 << ", " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize integer " << Ty << " constant, expected: " << s32 << ", " << s64 << ", or " << p0 << '\n' ; } } while (false) | ||||||
2350 | << ", or " << p0 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize integer " << Ty << " constant, expected: " << s32 << ", " << s64 << ", or " << p0 << '\n' ; } } while (false); | ||||||
2351 | return false; | ||||||
2352 | } | ||||||
2353 | |||||||
2354 | if (RB.getID() != AArch64::GPRRegBankID) { | ||||||
2355 | LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize integer " << Ty << " constant on bank: " << RB << ", expected: GPR\n"; } } while (false) | ||||||
2356 | << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize integer " << Ty << " constant on bank: " << RB << ", expected: GPR\n"; } } while (false) | ||||||
2357 | << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unable to materialize integer " << Ty << " constant on bank: " << RB << ", expected: GPR\n"; } } while (false); | ||||||
2358 | return false; | ||||||
2359 | } | ||||||
2360 | } | ||||||
2361 | |||||||
2362 | // We allow G_CONSTANT of types < 32b. | ||||||
2363 | const unsigned MovOpc = | ||||||
2364 | DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm; | ||||||
2365 | |||||||
2366 | if (isFP) { | ||||||
2367 | // Either emit a FMOV, or emit a copy to emit a normal mov. | ||||||
2368 | const TargetRegisterClass &GPRRC = | ||||||
2369 | DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass; | ||||||
2370 | const TargetRegisterClass &FPRRC = | ||||||
2371 | DefSize == 32 ? AArch64::FPR32RegClass | ||||||
2372 | : (DefSize == 64 ? AArch64::FPR64RegClass | ||||||
2373 | : AArch64::FPR128RegClass); | ||||||
2374 | |||||||
2375 | // Can we use a FMOV instruction to represent the immediate? | ||||||
2376 | if (emitFMovForFConstant(I, MRI)) | ||||||
2377 | return true; | ||||||
2378 | |||||||
2379 | // For 64b values, emit a constant pool load instead. | ||||||
2380 | if (DefSize == 64 || DefSize == 128) { | ||||||
2381 | auto *FPImm = I.getOperand(1).getFPImm(); | ||||||
2382 | MachineIRBuilder MIB(I); | ||||||
2383 | auto *LoadMI = emitLoadFromConstantPool(FPImm, MIB); | ||||||
2384 | if (!LoadMI) { | ||||||
2385 | LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to load double constant pool entry\n" ; } } while (false); | ||||||
2386 | return false; | ||||||
2387 | } | ||||||
2388 | MIB.buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()}); | ||||||
2389 | I.eraseFromParent(); | ||||||
2390 | return RBI.constrainGenericRegister(DefReg, FPRRC, MRI); | ||||||
2391 | } | ||||||
2392 | |||||||
2393 | // Nope. Emit a copy and use a normal mov instead. | ||||||
2394 | const Register DefGPRReg = MRI.createVirtualRegister(&GPRRC); | ||||||
2395 | MachineOperand &RegOp = I.getOperand(0); | ||||||
2396 | RegOp.setReg(DefGPRReg); | ||||||
2397 | MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); | ||||||
2398 | MIB.buildCopy({DefReg}, {DefGPRReg}); | ||||||
2399 | |||||||
2400 | if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) { | ||||||
2401 | LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain G_FCONSTANT def operand\n" ; } } while (false); | ||||||
2402 | return false; | ||||||
2403 | } | ||||||
2404 | |||||||
2405 | MachineOperand &ImmOp = I.getOperand(1); | ||||||
2406 | // FIXME: Is going through int64_t always correct? | ||||||
2407 | ImmOp.ChangeToImmediate( | ||||||
2408 | ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue()); | ||||||
2409 | } else if (I.getOperand(1).isCImm()) { | ||||||
2410 | uint64_t Val = I.getOperand(1).getCImm()->getZExtValue(); | ||||||
2411 | I.getOperand(1).ChangeToImmediate(Val); | ||||||
2412 | } else if (I.getOperand(1).isImm()) { | ||||||
2413 | uint64_t Val = I.getOperand(1).getImm(); | ||||||
2414 | I.getOperand(1).ChangeToImmediate(Val); | ||||||
2415 | } | ||||||
2416 | |||||||
2417 | I.setDesc(TII.get(MovOpc)); | ||||||
2418 | constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2419 | return true; | ||||||
2420 | } | ||||||
2421 | case TargetOpcode::G_EXTRACT: { | ||||||
2422 | Register DstReg = I.getOperand(0).getReg(); | ||||||
2423 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
2424 | LLT SrcTy = MRI.getType(SrcReg); | ||||||
2425 | LLT DstTy = MRI.getType(DstReg); | ||||||
2426 | (void)DstTy; | ||||||
2427 | unsigned SrcSize = SrcTy.getSizeInBits(); | ||||||
2428 | |||||||
2429 | if (SrcTy.getSizeInBits() > 64) { | ||||||
2430 | // This should be an extract of an s128, which is like a vector extract. | ||||||
2431 | if (SrcTy.getSizeInBits() != 128) | ||||||
2432 | return false; | ||||||
2433 | // Only support extracting 64 bits from an s128 at the moment. | ||||||
2434 | if (DstTy.getSizeInBits() != 64) | ||||||
2435 | return false; | ||||||
2436 | |||||||
2437 | const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); | ||||||
2438 | const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); | ||||||
2439 | // Check we have the right regbank always. | ||||||
2440 | assert(SrcRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID () == AArch64::FPRRegBankID && "Wrong extract regbank!" ) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2442, __PRETTY_FUNCTION__)) | ||||||
2441 | DstRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID () == AArch64::FPRRegBankID && "Wrong extract regbank!" ) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2442, __PRETTY_FUNCTION__)) | ||||||
2442 | "Wrong extract regbank!")((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID () == AArch64::FPRRegBankID && "Wrong extract regbank!" ) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2442, __PRETTY_FUNCTION__)); | ||||||
2443 | (void)SrcRB; | ||||||
2444 | |||||||
2445 | // Emit the same code as a vector extract. | ||||||
2446 | // Offset must be a multiple of 64. | ||||||
2447 | unsigned Offset = I.getOperand(2).getImm(); | ||||||
2448 | if (Offset % 64 != 0) | ||||||
2449 | return false; | ||||||
2450 | unsigned LaneIdx = Offset / 64; | ||||||
2451 | MachineIRBuilder MIB(I); | ||||||
2452 | MachineInstr *Extract = emitExtractVectorElt( | ||||||
2453 | DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB); | ||||||
2454 | if (!Extract) | ||||||
2455 | return false; | ||||||
2456 | I.eraseFromParent(); | ||||||
2457 | return true; | ||||||
2458 | } | ||||||
2459 | |||||||
2460 | I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri)); | ||||||
2461 | MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() + | ||||||
2462 | Ty.getSizeInBits() - 1); | ||||||
2463 | |||||||
2464 | if (SrcSize < 64) { | ||||||
2465 | assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&((SrcSize == 32 && DstTy.getSizeInBits() == 16 && "unexpected G_EXTRACT types") ? static_cast<void> (0) : __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2466, __PRETTY_FUNCTION__)) | ||||||
2466 | "unexpected G_EXTRACT types")((SrcSize == 32 && DstTy.getSizeInBits() == 16 && "unexpected G_EXTRACT types") ? static_cast<void> (0) : __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2466, __PRETTY_FUNCTION__)); | ||||||
2467 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2468 | } | ||||||
2469 | |||||||
2470 | DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); | ||||||
2471 | MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator())); | ||||||
2472 | MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {}) | ||||||
2473 | .addReg(DstReg, 0, AArch64::sub_32); | ||||||
2474 | RBI.constrainGenericRegister(I.getOperand(0).getReg(), | ||||||
2475 | AArch64::GPR32RegClass, MRI); | ||||||
2476 | I.getOperand(0).setReg(DstReg); | ||||||
2477 | |||||||
2478 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2479 | } | ||||||
2480 | |||||||
2481 | case TargetOpcode::G_INSERT: { | ||||||
2482 | LLT SrcTy = MRI.getType(I.getOperand(2).getReg()); | ||||||
2483 | LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
2484 | unsigned DstSize = DstTy.getSizeInBits(); | ||||||
2485 | // Larger inserts are vectors, same-size ones should be something else by | ||||||
2486 | // now (split up or turned into COPYs). | ||||||
2487 | if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32) | ||||||
2488 | return false; | ||||||
2489 | |||||||
2490 | I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri)); | ||||||
2491 | unsigned LSB = I.getOperand(3).getImm(); | ||||||
2492 | unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits(); | ||||||
2493 | I.getOperand(3).setImm((DstSize - LSB) % DstSize); | ||||||
2494 | MachineInstrBuilder(MF, I).addImm(Width - 1); | ||||||
2495 | |||||||
2496 | if (DstSize < 64) { | ||||||
2497 | assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&((DstSize == 32 && SrcTy.getSizeInBits() == 16 && "unexpected G_INSERT types") ? static_cast<void> (0) : __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2498, __PRETTY_FUNCTION__)) | ||||||
2498 | "unexpected G_INSERT types")((DstSize == 32 && SrcTy.getSizeInBits() == 16 && "unexpected G_INSERT types") ? static_cast<void> (0) : __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2498, __PRETTY_FUNCTION__)); | ||||||
2499 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2500 | } | ||||||
2501 | |||||||
2502 | Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); | ||||||
2503 | BuildMI(MBB, I.getIterator(), I.getDebugLoc(), | ||||||
2504 | TII.get(AArch64::SUBREG_TO_REG)) | ||||||
2505 | .addDef(SrcReg) | ||||||
2506 | .addImm(0) | ||||||
2507 | .addUse(I.getOperand(2).getReg()) | ||||||
2508 | .addImm(AArch64::sub_32); | ||||||
2509 | RBI.constrainGenericRegister(I.getOperand(2).getReg(), | ||||||
2510 | AArch64::GPR32RegClass, MRI); | ||||||
2511 | I.getOperand(2).setReg(SrcReg); | ||||||
2512 | |||||||
2513 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2514 | } | ||||||
2515 | case TargetOpcode::G_FRAME_INDEX: { | ||||||
2516 | // allocas and G_FRAME_INDEX are only supported in addrspace(0). | ||||||
2517 | if (Ty != LLT::pointer(0, 64)) { | ||||||
2518 | LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: " << Ty << ", expected: " << LLT::pointer(0, 64) << '\n'; } } while (false) | ||||||
2519 | << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: " << Ty << ", expected: " << LLT::pointer(0, 64) << '\n'; } } while (false); | ||||||
2520 | return false; | ||||||
2521 | } | ||||||
2522 | I.setDesc(TII.get(AArch64::ADDXri)); | ||||||
2523 | |||||||
2524 | // MOs for a #0 shifted immediate. | ||||||
2525 | I.addOperand(MachineOperand::CreateImm(0)); | ||||||
2526 | I.addOperand(MachineOperand::CreateImm(0)); | ||||||
2527 | |||||||
2528 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2529 | } | ||||||
2530 | |||||||
2531 | case TargetOpcode::G_GLOBAL_VALUE: { | ||||||
2532 | auto GV = I.getOperand(1).getGlobal(); | ||||||
2533 | if (GV->isThreadLocal()) | ||||||
2534 | return selectTLSGlobalValue(I, MRI); | ||||||
2535 | |||||||
2536 | unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM); | ||||||
2537 | if (OpFlags & AArch64II::MO_GOT) { | ||||||
2538 | I.setDesc(TII.get(AArch64::LOADgot)); | ||||||
2539 | I.getOperand(1).setTargetFlags(OpFlags); | ||||||
2540 | } else if (TM.getCodeModel() == CodeModel::Large) { | ||||||
2541 | // Materialize the global using movz/movk instructions. | ||||||
2542 | materializeLargeCMVal(I, GV, OpFlags); | ||||||
2543 | I.eraseFromParent(); | ||||||
2544 | return true; | ||||||
2545 | } else if (TM.getCodeModel() == CodeModel::Tiny) { | ||||||
2546 | I.setDesc(TII.get(AArch64::ADR)); | ||||||
2547 | I.getOperand(1).setTargetFlags(OpFlags); | ||||||
2548 | } else { | ||||||
2549 | I.setDesc(TII.get(AArch64::MOVaddr)); | ||||||
2550 | I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE); | ||||||
2551 | MachineInstrBuilder MIB(MF, I); | ||||||
2552 | MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(), | ||||||
2553 | OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); | ||||||
2554 | } | ||||||
2555 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2556 | } | ||||||
2557 | |||||||
2558 | case TargetOpcode::G_ZEXTLOAD: | ||||||
2559 | case TargetOpcode::G_LOAD: | ||||||
2560 | case TargetOpcode::G_STORE: { | ||||||
2561 | bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD; | ||||||
2562 | MachineIRBuilder MIB(I); | ||||||
2563 | |||||||
2564 | LLT PtrTy = MRI.getType(I.getOperand(1).getReg()); | ||||||
2565 | |||||||
2566 | if (PtrTy != LLT::pointer(0, 64)) { | ||||||
2567 | LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Load/Store pointer has type: " << PtrTy << ", expected: " << LLT::pointer (0, 64) << '\n'; } } while (false) | ||||||
2568 | << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Load/Store pointer has type: " << PtrTy << ", expected: " << LLT::pointer (0, 64) << '\n'; } } while (false); | ||||||
2569 | return false; | ||||||
2570 | } | ||||||
2571 | |||||||
2572 | auto &MemOp = **I.memoperands_begin(); | ||||||
2573 | uint64_t MemSizeInBytes = MemOp.getSize(); | ||||||
2574 | if (MemOp.isAtomic()) { | ||||||
2575 | // For now we just support s8 acquire loads to be able to compile stack | ||||||
2576 | // protector code. | ||||||
2577 | if (MemOp.getOrdering() == AtomicOrdering::Acquire && | ||||||
2578 | MemSizeInBytes == 1) { | ||||||
2579 | I.setDesc(TII.get(AArch64::LDARB)); | ||||||
2580 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2581 | } | ||||||
2582 | LLVM_DEBUG(dbgs() << "Atomic load/store not fully supported yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Atomic load/store not fully supported yet\n" ; } } while (false); | ||||||
2583 | return false; | ||||||
2584 | } | ||||||
2585 | unsigned MemSizeInBits = MemSizeInBytes * 8; | ||||||
2586 | |||||||
2587 | #ifndef NDEBUG | ||||||
2588 | const Register PtrReg = I.getOperand(1).getReg(); | ||||||
2589 | const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI); | ||||||
2590 | // Sanity-check the pointer register. | ||||||
2591 | assert(PtrRB.getID() == AArch64::GPRRegBankID &&((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR" ) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2592, __PRETTY_FUNCTION__)) | ||||||
2592 | "Load/Store pointer operand isn't a GPR")((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR" ) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2592, __PRETTY_FUNCTION__)); | ||||||
2593 | assert(MRI.getType(PtrReg).isPointer() &&((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer" ) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2594, __PRETTY_FUNCTION__)) | ||||||
2594 | "Load/Store pointer operand isn't a pointer")((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer" ) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2594, __PRETTY_FUNCTION__)); | ||||||
2595 | #endif | ||||||
2596 | |||||||
2597 | const Register ValReg = I.getOperand(0).getReg(); | ||||||
2598 | const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI); | ||||||
2599 | |||||||
2600 | // Helper lambda for partially selecting I. Either returns the original | ||||||
2601 | // instruction with an updated opcode, or a new instruction. | ||||||
2602 | auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * { | ||||||
2603 | bool IsStore = I.getOpcode() == TargetOpcode::G_STORE; | ||||||
2604 | const unsigned NewOpc = | ||||||
2605 | selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits); | ||||||
2606 | if (NewOpc == I.getOpcode()) | ||||||
2607 | return nullptr; | ||||||
2608 | // Check if we can fold anything into the addressing mode. | ||||||
2609 | auto AddrModeFns = | ||||||
2610 | selectAddrModeIndexed(I.getOperand(1), MemSizeInBytes); | ||||||
2611 | if (!AddrModeFns) { | ||||||
2612 | // Can't fold anything. Use the original instruction. | ||||||
2613 | I.setDesc(TII.get(NewOpc)); | ||||||
2614 | I.addOperand(MachineOperand::CreateImm(0)); | ||||||
2615 | return &I; | ||||||
2616 | } | ||||||
2617 | |||||||
2618 | // Folded something. Create a new instruction and return it. | ||||||
2619 | auto NewInst = MIB.buildInstr(NewOpc, {}, {}, I.getFlags()); | ||||||
2620 | IsStore ? NewInst.addUse(ValReg) : NewInst.addDef(ValReg); | ||||||
2621 | NewInst.cloneMemRefs(I); | ||||||
2622 | for (auto &Fn : *AddrModeFns) | ||||||
2623 | Fn(NewInst); | ||||||
2624 | I.eraseFromParent(); | ||||||
2625 | return &*NewInst; | ||||||
2626 | }; | ||||||
2627 | |||||||
2628 | MachineInstr *LoadStore = SelectLoadStoreAddressingMode(); | ||||||
2629 | if (!LoadStore) | ||||||
2630 | return false; | ||||||
2631 | |||||||
2632 | // If we're storing a 0, use WZR/XZR. | ||||||
2633 | if (Opcode == TargetOpcode::G_STORE) { | ||||||
2634 | auto CVal = getConstantVRegValWithLookThrough( | ||||||
2635 | LoadStore->getOperand(0).getReg(), MRI, /*LookThroughInstrs = */ true, | ||||||
2636 | /*HandleFConstants = */ false); | ||||||
2637 | if (CVal && CVal->Value == 0) { | ||||||
2638 | switch (LoadStore->getOpcode()) { | ||||||
2639 | case AArch64::STRWui: | ||||||
2640 | case AArch64::STRHHui: | ||||||
2641 | case AArch64::STRBBui: | ||||||
2642 | LoadStore->getOperand(0).setReg(AArch64::WZR); | ||||||
2643 | break; | ||||||
2644 | case AArch64::STRXui: | ||||||
2645 | LoadStore->getOperand(0).setReg(AArch64::XZR); | ||||||
2646 | break; | ||||||
2647 | } | ||||||
2648 | } | ||||||
2649 | } | ||||||
2650 | |||||||
2651 | if (IsZExtLoad) { | ||||||
2652 | // The zextload from a smaller type to i32 should be handled by the | ||||||
2653 | // importer. | ||||||
2654 | if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64) | ||||||
2655 | return false; | ||||||
2656 | // If we have a ZEXTLOAD then change the load's type to be a narrower reg | ||||||
2657 | // and zero_extend with SUBREG_TO_REG. | ||||||
2658 | Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); | ||||||
2659 | Register DstReg = LoadStore->getOperand(0).getReg(); | ||||||
2660 | LoadStore->getOperand(0).setReg(LdReg); | ||||||
2661 | |||||||
2662 | MIB.setInsertPt(MIB.getMBB(), std::next(LoadStore->getIterator())); | ||||||
2663 | MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {}) | ||||||
2664 | .addImm(0) | ||||||
2665 | .addUse(LdReg) | ||||||
2666 | .addImm(AArch64::sub_32); | ||||||
2667 | constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI); | ||||||
2668 | return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass, | ||||||
2669 | MRI); | ||||||
2670 | } | ||||||
2671 | return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI); | ||||||
2672 | } | ||||||
2673 | |||||||
2674 | case TargetOpcode::G_SMULH: | ||||||
2675 | case TargetOpcode::G_UMULH: { | ||||||
2676 | // Reject the various things we don't support yet. | ||||||
2677 | if (unsupportedBinOp(I, RBI, MRI, TRI)) | ||||||
2678 | return false; | ||||||
2679 | |||||||
2680 | const Register DefReg = I.getOperand(0).getReg(); | ||||||
2681 | const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); | ||||||
2682 | |||||||
2683 | if (RB.getID() != AArch64::GPRRegBankID) { | ||||||
2684 | LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n"; } } while (false); | ||||||
2685 | return false; | ||||||
2686 | } | ||||||
2687 | |||||||
2688 | if (Ty != LLT::scalar(64)) { | ||||||
2689 | LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " << Ty << ", expected: " << LLT::scalar(64) << '\n'; } } while (false) | ||||||
2690 | << ", expected: " << LLT::scalar(64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " << Ty << ", expected: " << LLT::scalar(64) << '\n'; } } while (false); | ||||||
2691 | return false; | ||||||
2692 | } | ||||||
2693 | |||||||
2694 | unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr | ||||||
2695 | : AArch64::UMULHrr; | ||||||
2696 | I.setDesc(TII.get(NewOpc)); | ||||||
2697 | |||||||
2698 | // Now that we selected an opcode, we need to constrain the register | ||||||
2699 | // operands to use appropriate classes. | ||||||
2700 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2701 | } | ||||||
2702 | case TargetOpcode::G_LSHR: | ||||||
2703 | case TargetOpcode::G_ASHR: | ||||||
2704 | if (MRI.getType(I.getOperand(0).getReg()).isVector()) | ||||||
2705 | return selectVectorAshrLshr(I, MRI); | ||||||
2706 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||
2707 | case TargetOpcode::G_SHL: | ||||||
2708 | if (Opcode == TargetOpcode::G_SHL && | ||||||
2709 | MRI.getType(I.getOperand(0).getReg()).isVector()) | ||||||
2710 | return selectVectorSHL(I, MRI); | ||||||
2711 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||
2712 | case TargetOpcode::G_FADD: | ||||||
2713 | case TargetOpcode::G_FSUB: | ||||||
2714 | case TargetOpcode::G_FMUL: | ||||||
2715 | case TargetOpcode::G_FDIV: | ||||||
2716 | case TargetOpcode::G_OR: { | ||||||
2717 | // Reject the various things we don't support yet. | ||||||
2718 | if (unsupportedBinOp(I, RBI, MRI, TRI)) | ||||||
2719 | return false; | ||||||
2720 | |||||||
2721 | const unsigned OpSize = Ty.getSizeInBits(); | ||||||
2722 | |||||||
2723 | const Register DefReg = I.getOperand(0).getReg(); | ||||||
2724 | const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); | ||||||
2725 | |||||||
2726 | const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize); | ||||||
2727 | if (NewOpc == I.getOpcode()) | ||||||
2728 | return false; | ||||||
2729 | |||||||
2730 | I.setDesc(TII.get(NewOpc)); | ||||||
2731 | // FIXME: Should the type be always reset in setDesc? | ||||||
2732 | |||||||
2733 | // Now that we selected an opcode, we need to constrain the register | ||||||
2734 | // operands to use appropriate classes. | ||||||
2735 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2736 | } | ||||||
2737 | |||||||
2738 | case TargetOpcode::G_PTR_ADD: { | ||||||
2739 | MachineIRBuilder MIRBuilder(I); | ||||||
2740 | emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2), | ||||||
2741 | MIRBuilder); | ||||||
2742 | I.eraseFromParent(); | ||||||
2743 | return true; | ||||||
2744 | } | ||||||
2745 | case TargetOpcode::G_SADDO: | ||||||
2746 | case TargetOpcode::G_UADDO: | ||||||
2747 | case TargetOpcode::G_SSUBO: { | ||||||
2748 | // Emit the operation and get the correct condition code. | ||||||
2749 | MachineIRBuilder MIRBuilder(I); | ||||||
2750 | auto OpAndCC = emitOverflowOp(Opcode, I.getOperand(0).getReg(), | ||||||
2751 | I.getOperand(2), I.getOperand(3), MIRBuilder); | ||||||
2752 | |||||||
2753 | // Now, put the overflow result in the register given by the first operand | ||||||
2754 | // to the overflow op. CSINC increments the result when the predicate is | ||||||
2755 | // false, so to get the increment when it's true, we need to use the | ||||||
2756 | // inverse. In this case, we want to increment when carry is set. | ||||||
2757 | Register ZReg = AArch64::WZR; | ||||||
2758 | auto CsetMI = MIRBuilder | ||||||
2759 | .buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()}, | ||||||
2760 | {ZReg, ZReg}) | ||||||
2761 | .addImm(getInvertedCondCode(OpAndCC.second)); | ||||||
2762 | constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI); | ||||||
2763 | I.eraseFromParent(); | ||||||
2764 | return true; | ||||||
2765 | } | ||||||
2766 | |||||||
2767 | case TargetOpcode::G_PTRMASK: { | ||||||
2768 | Register MaskReg = I.getOperand(2).getReg(); | ||||||
2769 | Optional<int64_t> MaskVal = getConstantVRegSExtVal(MaskReg, MRI); | ||||||
2770 | // TODO: Implement arbitrary cases | ||||||
2771 | if (!MaskVal || !isShiftedMask_64(*MaskVal)) | ||||||
2772 | return false; | ||||||
2773 | |||||||
2774 | uint64_t Mask = *MaskVal; | ||||||
2775 | I.setDesc(TII.get(AArch64::ANDXri)); | ||||||
2776 | I.getOperand(2).ChangeToImmediate( | ||||||
2777 | AArch64_AM::encodeLogicalImmediate(Mask, 64)); | ||||||
2778 | |||||||
2779 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2780 | } | ||||||
2781 | case TargetOpcode::G_PTRTOINT: | ||||||
2782 | case TargetOpcode::G_TRUNC: { | ||||||
2783 | const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
2784 | const LLT SrcTy = MRI.getType(I.getOperand(1).getReg()); | ||||||
2785 | |||||||
2786 | const Register DstReg = I.getOperand(0).getReg(); | ||||||
2787 | const Register SrcReg = I.getOperand(1).getReg(); | ||||||
2788 | |||||||
2789 | const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); | ||||||
2790 | const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); | ||||||
2791 | |||||||
2792 | if (DstRB.getID() != SrcRB.getID()) { | ||||||
2793 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n" ; } } while (false) | ||||||
2794 | dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n" ; } } while (false); | ||||||
2795 | return false; | ||||||
2796 | } | ||||||
2797 | |||||||
2798 | if (DstRB.getID() == AArch64::GPRRegBankID) { | ||||||
2799 | const TargetRegisterClass *DstRC = | ||||||
2800 | getRegClassForTypeOnBank(DstTy, DstRB, RBI); | ||||||
2801 | if (!DstRC) | ||||||
2802 | return false; | ||||||
2803 | |||||||
2804 | const TargetRegisterClass *SrcRC = | ||||||
2805 | getRegClassForTypeOnBank(SrcTy, SrcRB, RBI); | ||||||
2806 | if (!SrcRC) | ||||||
2807 | return false; | ||||||
2808 | |||||||
2809 | if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || | ||||||
2810 | !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { | ||||||
2811 | LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n" ; } } while (false); | ||||||
2812 | return false; | ||||||
2813 | } | ||||||
2814 | |||||||
2815 | if (DstRC == SrcRC) { | ||||||
2816 | // Nothing to be done | ||||||
2817 | } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) && | ||||||
2818 | SrcTy == LLT::scalar(64)) { | ||||||
2819 | llvm_unreachable("TableGen can import this case")::llvm::llvm_unreachable_internal("TableGen can import this case" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2819); | ||||||
2820 | return false; | ||||||
2821 | } else if (DstRC == &AArch64::GPR32RegClass && | ||||||
2822 | SrcRC == &AArch64::GPR64RegClass) { | ||||||
2823 | I.getOperand(1).setSubReg(AArch64::sub_32); | ||||||
2824 | } else { | ||||||
2825 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n" ; } } while (false) | ||||||
2826 | dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n" ; } } while (false); | ||||||
2827 | return false; | ||||||
2828 | } | ||||||
2829 | |||||||
2830 | I.setDesc(TII.get(TargetOpcode::COPY)); | ||||||
2831 | return true; | ||||||
2832 | } else if (DstRB.getID() == AArch64::FPRRegBankID) { | ||||||
2833 | if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) { | ||||||
2834 | I.setDesc(TII.get(AArch64::XTNv4i16)); | ||||||
2835 | constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
2836 | return true; | ||||||
2837 | } | ||||||
2838 | |||||||
2839 | if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) { | ||||||
2840 | MachineIRBuilder MIB(I); | ||||||
2841 | MachineInstr *Extract = emitExtractVectorElt( | ||||||
2842 | DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB); | ||||||
2843 | if (!Extract) | ||||||
2844 | return false; | ||||||
2845 | I.eraseFromParent(); | ||||||
2846 | return true; | ||||||
2847 | } | ||||||
2848 | |||||||
2849 | // We might have a vector G_PTRTOINT, in which case just emit a COPY. | ||||||
2850 | if (Opcode == TargetOpcode::G_PTRTOINT) { | ||||||
2851 | assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector")((DstTy.isVector() && "Expected an FPR ptrtoint to be a vector" ) ? static_cast<void> (0) : __assert_fail ("DstTy.isVector() && \"Expected an FPR ptrtoint to be a vector\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2851, __PRETTY_FUNCTION__)); | ||||||
2852 | I.setDesc(TII.get(TargetOpcode::COPY)); | ||||||
2853 | return true; | ||||||
2854 | } | ||||||
2855 | } | ||||||
2856 | |||||||
2857 | return false; | ||||||
2858 | } | ||||||
2859 | |||||||
2860 | case TargetOpcode::G_ANYEXT: { | ||||||
2861 | const Register DstReg = I.getOperand(0).getReg(); | ||||||
2862 | const Register SrcReg = I.getOperand(1).getReg(); | ||||||
2863 | |||||||
2864 | const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI); | ||||||
2865 | if (RBDst.getID() != AArch64::GPRRegBankID) { | ||||||
2866 | LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " << RBDst << ", expected: GPR\n"; } } while (false) | ||||||
2867 | << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " << RBDst << ", expected: GPR\n"; } } while (false); | ||||||
2868 | return false; | ||||||
2869 | } | ||||||
2870 | |||||||
2871 | const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI); | ||||||
2872 | if (RBSrc.getID() != AArch64::GPRRegBankID) { | ||||||
2873 | LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrcdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " << RBSrc << ", expected: GPR\n"; } } while (false) | ||||||
2874 | << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " << RBSrc << ", expected: GPR\n"; } } while (false); | ||||||
2875 | return false; | ||||||
2876 | } | ||||||
2877 | |||||||
2878 | const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); | ||||||
2879 | |||||||
2880 | if (DstSize == 0) { | ||||||
2881 | LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n" ; } } while (false); | ||||||
2882 | return false; | ||||||
2883 | } | ||||||
2884 | |||||||
2885 | if (DstSize != 64 && DstSize > 32) { | ||||||
2886 | LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " << DstSize << ", expected: 32 or 64\n"; } } while (false) | ||||||
2887 | << ", expected: 32 or 64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " << DstSize << ", expected: 32 or 64\n"; } } while (false); | ||||||
2888 | return false; | ||||||
2889 | } | ||||||
2890 | // At this point G_ANYEXT is just like a plain COPY, but we need | ||||||
2891 | // to explicitly form the 64-bit value if any. | ||||||
2892 | if (DstSize > 32) { | ||||||
2893 | Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass); | ||||||
2894 | BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG)) | ||||||
2895 | .addDef(ExtSrc) | ||||||
2896 | .addImm(0) | ||||||
2897 | .addUse(SrcReg) | ||||||
2898 | .addImm(AArch64::sub_32); | ||||||
2899 | I.getOperand(1).setReg(ExtSrc); | ||||||
2900 | } | ||||||
2901 | return selectCopy(I, TII, MRI, TRI, RBI); | ||||||
2902 | } | ||||||
2903 | |||||||
2904 | case TargetOpcode::G_ZEXT: | ||||||
2905 | case TargetOpcode::G_SEXT_INREG: | ||||||
2906 | case TargetOpcode::G_SEXT: { | ||||||
2907 | unsigned Opcode = I.getOpcode(); | ||||||
2908 | const bool IsSigned = Opcode != TargetOpcode::G_ZEXT; | ||||||
2909 | const Register DefReg = I.getOperand(0).getReg(); | ||||||
2910 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
2911 | const LLT DstTy = MRI.getType(DefReg); | ||||||
2912 | const LLT SrcTy = MRI.getType(SrcReg); | ||||||
2913 | unsigned DstSize = DstTy.getSizeInBits(); | ||||||
2914 | unsigned SrcSize = SrcTy.getSizeInBits(); | ||||||
2915 | |||||||
2916 | // SEXT_INREG has the same src reg size as dst, the size of the value to be | ||||||
2917 | // extended is encoded in the imm. | ||||||
2918 | if (Opcode == TargetOpcode::G_SEXT_INREG) | ||||||
2919 | SrcSize = I.getOperand(2).getImm(); | ||||||
2920 | |||||||
2921 | if (DstTy.isVector()) | ||||||
2922 | return false; // Should be handled by imported patterns. | ||||||
2923 | |||||||
2924 | assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank") ? static_cast<void> (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2926, __PRETTY_FUNCTION__)) | ||||||
2925 | AArch64::GPRRegBankID &&(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank") ? static_cast<void> (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2926, __PRETTY_FUNCTION__)) | ||||||
2926 | "Unexpected ext regbank")(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank") ? static_cast<void> (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 2926, __PRETTY_FUNCTION__)); | ||||||
2927 | |||||||
2928 | MachineIRBuilder MIB(I); | ||||||
2929 | MachineInstr *ExtI; | ||||||
2930 | |||||||
2931 | // First check if we're extending the result of a load which has a dest type | ||||||
2932 | // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest | ||||||
2933 | // GPR register on AArch64 and all loads which are smaller automatically | ||||||
2934 | // zero-extend the upper bits. E.g. | ||||||
2935 | // %v(s8) = G_LOAD %p, :: (load 1) | ||||||
2936 | // %v2(s32) = G_ZEXT %v(s8) | ||||||
2937 | if (!IsSigned) { | ||||||
2938 | auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI); | ||||||
2939 | bool IsGPR = | ||||||
2940 | RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID; | ||||||
2941 | if (LoadMI && IsGPR) { | ||||||
2942 | const MachineMemOperand *MemOp = *LoadMI->memoperands_begin(); | ||||||
2943 | unsigned BytesLoaded = MemOp->getSize(); | ||||||
2944 | if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded) | ||||||
2945 | return selectCopy(I, TII, MRI, TRI, RBI); | ||||||
2946 | } | ||||||
2947 | |||||||
2948 | // If we are zero extending from 32 bits to 64 bits, it's possible that | ||||||
2949 | // the instruction implicitly does the zero extend for us. In that case, | ||||||
2950 | // we can just emit a SUBREG_TO_REG. | ||||||
2951 | if (IsGPR && SrcSize == 32 && DstSize == 64) { | ||||||
2952 | // Unlike with the G_LOAD case, we don't want to look through copies | ||||||
2953 | // here. | ||||||
2954 | MachineInstr *Def = MRI.getVRegDef(SrcReg); | ||||||
2955 | if (Def && isDef32(*Def)) { | ||||||
2956 | MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {}) | ||||||
2957 | .addImm(0) | ||||||
2958 | .addUse(SrcReg) | ||||||
2959 | .addImm(AArch64::sub_32); | ||||||
2960 | |||||||
2961 | if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, | ||||||
2962 | MRI)) { | ||||||
2963 | LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT destination\n" ; } } while (false); | ||||||
2964 | return false; | ||||||
2965 | } | ||||||
2966 | |||||||
2967 | if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, | ||||||
2968 | MRI)) { | ||||||
2969 | LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT source\n" ; } } while (false); | ||||||
2970 | return false; | ||||||
2971 | } | ||||||
2972 | |||||||
2973 | I.eraseFromParent(); | ||||||
2974 | return true; | ||||||
2975 | } | ||||||
2976 | } | ||||||
2977 | } | ||||||
2978 | |||||||
2979 | if (DstSize == 64) { | ||||||
2980 | if (Opcode != TargetOpcode::G_SEXT_INREG) { | ||||||
2981 | // FIXME: Can we avoid manually doing this? | ||||||
2982 | if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, | ||||||
2983 | MRI)) { | ||||||
2984 | LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain " << TII.getName(Opcode) << " operand\n"; } } while (false) | ||||||
2985 | << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Failed to constrain " << TII.getName(Opcode) << " operand\n"; } } while (false); | ||||||
2986 | return false; | ||||||
2987 | } | ||||||
2988 | SrcReg = MIB.buildInstr(AArch64::SUBREG_TO_REG, | ||||||
2989 | {&AArch64::GPR64RegClass}, {}) | ||||||
2990 | .addImm(0) | ||||||
2991 | .addUse(SrcReg) | ||||||
2992 | .addImm(AArch64::sub_32) | ||||||
2993 | .getReg(0); | ||||||
2994 | } | ||||||
2995 | |||||||
2996 | ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri, | ||||||
2997 | {DefReg}, {SrcReg}) | ||||||
2998 | .addImm(0) | ||||||
2999 | .addImm(SrcSize - 1); | ||||||
3000 | } else if (DstSize <= 32) { | ||||||
3001 | ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri, | ||||||
3002 | {DefReg}, {SrcReg}) | ||||||
3003 | .addImm(0) | ||||||
3004 | .addImm(SrcSize - 1); | ||||||
3005 | } else { | ||||||
3006 | return false; | ||||||
3007 | } | ||||||
3008 | |||||||
3009 | constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI); | ||||||
3010 | I.eraseFromParent(); | ||||||
3011 | return true; | ||||||
3012 | } | ||||||
3013 | |||||||
3014 | case TargetOpcode::G_SITOFP: | ||||||
3015 | case TargetOpcode::G_UITOFP: | ||||||
3016 | case TargetOpcode::G_FPTOSI: | ||||||
3017 | case TargetOpcode::G_FPTOUI: { | ||||||
3018 | const LLT DstTy = MRI.getType(I.getOperand(0).getReg()), | ||||||
3019 | SrcTy = MRI.getType(I.getOperand(1).getReg()); | ||||||
3020 | const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy); | ||||||
3021 | if (NewOpc == Opcode) | ||||||
3022 | return false; | ||||||
3023 | |||||||
3024 | I.setDesc(TII.get(NewOpc)); | ||||||
3025 | constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
3026 | |||||||
3027 | return true; | ||||||
3028 | } | ||||||
3029 | |||||||
3030 | case TargetOpcode::G_FREEZE: | ||||||
3031 | return selectCopy(I, TII, MRI, TRI, RBI); | ||||||
3032 | |||||||
3033 | case TargetOpcode::G_INTTOPTR: | ||||||
3034 | // The importer is currently unable to import pointer types since they | ||||||
3035 | // didn't exist in SelectionDAG. | ||||||
3036 | return selectCopy(I, TII, MRI, TRI, RBI); | ||||||
3037 | |||||||
3038 | case TargetOpcode::G_BITCAST: | ||||||
3039 | // Imported SelectionDAG rules can handle every bitcast except those that | ||||||
3040 | // bitcast from a type to the same type. Ideally, these shouldn't occur | ||||||
3041 | // but we might not run an optimizer that deletes them. The other exception | ||||||
3042 | // is bitcasts involving pointer types, as SelectionDAG has no knowledge | ||||||
3043 | // of them. | ||||||
3044 | return selectCopy(I, TII, MRI, TRI, RBI); | ||||||
3045 | |||||||
3046 | case TargetOpcode::G_SELECT: { | ||||||
3047 | if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) { | ||||||
3048 | LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_SELECT cond has type: " << Ty << ", expected: " << LLT::scalar(1) << '\n'; } } while (false) | ||||||
3049 | << ", expected: " << LLT::scalar(1) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_SELECT cond has type: " << Ty << ", expected: " << LLT::scalar(1) << '\n'; } } while (false); | ||||||
3050 | return false; | ||||||
3051 | } | ||||||
3052 | |||||||
3053 | const Register CondReg = I.getOperand(1).getReg(); | ||||||
3054 | const Register TReg = I.getOperand(2).getReg(); | ||||||
3055 | const Register FReg = I.getOperand(3).getReg(); | ||||||
3056 | |||||||
3057 | if (tryOptSelect(I)) | ||||||
3058 | return true; | ||||||
3059 | |||||||
3060 | // Make sure to use an unused vreg instead of wzr, so that the peephole | ||||||
3061 | // optimizations will be able to optimize these. | ||||||
3062 | MachineIRBuilder MIB(I); | ||||||
3063 | Register DeadVReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); | ||||||
3064 | auto TstMI = MIB.buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg}) | ||||||
3065 | .addImm(AArch64_AM::encodeLogicalImmediate(1, 32)); | ||||||
3066 | constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI); | ||||||
3067 | if (!emitSelect(I.getOperand(0).getReg(), TReg, FReg, AArch64CC::NE, MIB)) | ||||||
3068 | return false; | ||||||
3069 | I.eraseFromParent(); | ||||||
3070 | return true; | ||||||
3071 | } | ||||||
3072 | case TargetOpcode::G_ICMP: { | ||||||
3073 | if (Ty.isVector()) | ||||||
3074 | return selectVectorICmp(I, MRI); | ||||||
3075 | |||||||
3076 | if (Ty != LLT::scalar(32)) { | ||||||
3077 | LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ICMP result has type: " << Ty << ", expected: " << LLT::scalar(32) << '\n'; } } while (false) | ||||||
3078 | << ", expected: " << LLT::scalar(32) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "G_ICMP result has type: " << Ty << ", expected: " << LLT::scalar(32) << '\n'; } } while (false); | ||||||
3079 | return false; | ||||||
3080 | } | ||||||
3081 | |||||||
3082 | MachineIRBuilder MIRBuilder(I); | ||||||
3083 | auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate()); | ||||||
3084 | emitIntegerCompare(I.getOperand(2), I.getOperand(3), I.getOperand(1), | ||||||
3085 | MIRBuilder); | ||||||
3086 | emitCSetForICMP(I.getOperand(0).getReg(), Pred, MIRBuilder); | ||||||
3087 | I.eraseFromParent(); | ||||||
3088 | return true; | ||||||
3089 | } | ||||||
3090 | |||||||
3091 | case TargetOpcode::G_FCMP: { | ||||||
3092 | MachineIRBuilder MIRBuilder(I); | ||||||
3093 | CmpInst::Predicate Pred = | ||||||
3094 | static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate()); | ||||||
3095 | if (!emitFPCompare(I.getOperand(2).getReg(), I.getOperand(3).getReg(), | ||||||
3096 | MIRBuilder, Pred) || | ||||||
3097 | !emitCSetForFCmp(I.getOperand(0).getReg(), Pred, MIRBuilder)) | ||||||
3098 | return false; | ||||||
3099 | I.eraseFromParent(); | ||||||
3100 | return true; | ||||||
3101 | } | ||||||
3102 | case TargetOpcode::G_VASTART: | ||||||
3103 | return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI) | ||||||
3104 | : selectVaStartAAPCS(I, MF, MRI); | ||||||
3105 | case TargetOpcode::G_INTRINSIC: | ||||||
3106 | return selectIntrinsic(I, MRI); | ||||||
3107 | case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: | ||||||
3108 | return selectIntrinsicWithSideEffects(I, MRI); | ||||||
3109 | case TargetOpcode::G_IMPLICIT_DEF: { | ||||||
3110 | I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF)); | ||||||
3111 | const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
3112 | const Register DstReg = I.getOperand(0).getReg(); | ||||||
3113 | const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); | ||||||
3114 | const TargetRegisterClass *DstRC = | ||||||
3115 | getRegClassForTypeOnBank(DstTy, DstRB, RBI); | ||||||
3116 | RBI.constrainGenericRegister(DstReg, *DstRC, MRI); | ||||||
3117 | return true; | ||||||
3118 | } | ||||||
3119 | case TargetOpcode::G_BLOCK_ADDR: { | ||||||
3120 | if (TM.getCodeModel() == CodeModel::Large) { | ||||||
3121 | materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0); | ||||||
3122 | I.eraseFromParent(); | ||||||
3123 | return true; | ||||||
3124 | } else { | ||||||
3125 | I.setDesc(TII.get(AArch64::MOVaddrBA)); | ||||||
3126 | auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA), | ||||||
3127 | I.getOperand(0).getReg()) | ||||||
3128 | .addBlockAddress(I.getOperand(1).getBlockAddress(), | ||||||
3129 | /* Offset */ 0, AArch64II::MO_PAGE) | ||||||
3130 | .addBlockAddress( | ||||||
3131 | I.getOperand(1).getBlockAddress(), /* Offset */ 0, | ||||||
3132 | AArch64II::MO_NC | AArch64II::MO_PAGEOFF); | ||||||
3133 | I.eraseFromParent(); | ||||||
3134 | return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI); | ||||||
3135 | } | ||||||
3136 | } | ||||||
3137 | case AArch64::G_DUP: { | ||||||
3138 | // When the scalar of G_DUP is an s8/s16 gpr, they can't be selected by | ||||||
3139 | // imported patterns. Do it manually here. Avoiding generating s16 gpr is | ||||||
3140 | // difficult because at RBS we may end up pessimizing the fpr case if we | ||||||
3141 | // decided to add an anyextend to fix this. Manual selection is the most | ||||||
3142 | // robust solution for now. | ||||||
3143 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
3144 | if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::GPRRegBankID) | ||||||
3145 | return false; // We expect the fpr regbank case to be imported. | ||||||
3146 | LLT SrcTy = MRI.getType(SrcReg); | ||||||
3147 | if (SrcTy.getSizeInBits() == 16) | ||||||
3148 | I.setDesc(TII.get(AArch64::DUPv8i16gpr)); | ||||||
3149 | else if (SrcTy.getSizeInBits() == 8) | ||||||
3150 | I.setDesc(TII.get(AArch64::DUPv16i8gpr)); | ||||||
3151 | else | ||||||
3152 | return false; | ||||||
3153 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
3154 | } | ||||||
3155 | case TargetOpcode::G_INTRINSIC_TRUNC: | ||||||
3156 | return selectIntrinsicTrunc(I, MRI); | ||||||
3157 | case TargetOpcode::G_INTRINSIC_ROUND: | ||||||
3158 | return selectIntrinsicRound(I, MRI); | ||||||
3159 | case TargetOpcode::G_BUILD_VECTOR: | ||||||
3160 | return selectBuildVector(I, MRI); | ||||||
3161 | case TargetOpcode::G_MERGE_VALUES: | ||||||
3162 | return selectMergeValues(I, MRI); | ||||||
3163 | case TargetOpcode::G_UNMERGE_VALUES: | ||||||
3164 | return selectUnmergeValues(I, MRI); | ||||||
3165 | case TargetOpcode::G_SHUFFLE_VECTOR: | ||||||
3166 | return selectShuffleVector(I, MRI); | ||||||
3167 | case TargetOpcode::G_EXTRACT_VECTOR_ELT: | ||||||
3168 | return selectExtractElt(I, MRI); | ||||||
3169 | case TargetOpcode::G_INSERT_VECTOR_ELT: | ||||||
3170 | return selectInsertElt(I, MRI); | ||||||
3171 | case TargetOpcode::G_CONCAT_VECTORS: | ||||||
3172 | return selectConcatVectors(I, MRI); | ||||||
3173 | case TargetOpcode::G_JUMP_TABLE: | ||||||
3174 | return selectJumpTable(I, MRI); | ||||||
3175 | case TargetOpcode::G_VECREDUCE_FADD: | ||||||
3176 | case TargetOpcode::G_VECREDUCE_ADD: | ||||||
3177 | return selectReduction(I, MRI); | ||||||
3178 | } | ||||||
3179 | |||||||
3180 | return false; | ||||||
3181 | } | ||||||
3182 | |||||||
3183 | bool AArch64InstructionSelector::selectReduction( | ||||||
3184 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3185 | Register VecReg = I.getOperand(1).getReg(); | ||||||
3186 | LLT VecTy = MRI.getType(VecReg); | ||||||
3187 | if (I.getOpcode() == TargetOpcode::G_VECREDUCE_ADD) { | ||||||
3188 | unsigned Opc = 0; | ||||||
3189 | if (VecTy == LLT::vector(16, 8)) | ||||||
3190 | Opc = AArch64::ADDVv16i8v; | ||||||
3191 | else if (VecTy == LLT::vector(8, 16)) | ||||||
3192 | Opc = AArch64::ADDVv8i16v; | ||||||
3193 | else if (VecTy == LLT::vector(4, 32)) | ||||||
3194 | Opc = AArch64::ADDVv4i32v; | ||||||
3195 | else if (VecTy == LLT::vector(2, 64)) | ||||||
3196 | Opc = AArch64::ADDPv2i64p; | ||||||
3197 | else { | ||||||
3198 | LLVM_DEBUG(dbgs() << "Unhandled type for add reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unhandled type for add reduction" ; } } while (false); | ||||||
3199 | return false; | ||||||
3200 | } | ||||||
3201 | I.setDesc(TII.get(Opc)); | ||||||
3202 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
3203 | } | ||||||
3204 | |||||||
3205 | if (I.getOpcode() == TargetOpcode::G_VECREDUCE_FADD) { | ||||||
3206 | unsigned Opc = 0; | ||||||
3207 | if (VecTy == LLT::vector(2, 32)) | ||||||
3208 | Opc = AArch64::FADDPv2i32p; | ||||||
3209 | else if (VecTy == LLT::vector(2, 64)) | ||||||
3210 | Opc = AArch64::FADDPv2i64p; | ||||||
3211 | else { | ||||||
3212 | LLVM_DEBUG(dbgs() << "Unhandled type for fadd reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unhandled type for fadd reduction" ; } } while (false); | ||||||
3213 | return false; | ||||||
3214 | } | ||||||
3215 | I.setDesc(TII.get(Opc)); | ||||||
3216 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
3217 | } | ||||||
3218 | return false; | ||||||
3219 | } | ||||||
3220 | |||||||
3221 | bool AArch64InstructionSelector::selectBrJT(MachineInstr &I, | ||||||
3222 | MachineRegisterInfo &MRI) const { | ||||||
3223 | assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT")((I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRJT && \"Expected G_BRJT\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3223, __PRETTY_FUNCTION__)); | ||||||
3224 | Register JTAddr = I.getOperand(0).getReg(); | ||||||
3225 | unsigned JTI = I.getOperand(1).getIndex(); | ||||||
3226 | Register Index = I.getOperand(2).getReg(); | ||||||
3227 | MachineIRBuilder MIB(I); | ||||||
3228 | |||||||
3229 | Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); | ||||||
3230 | Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); | ||||||
3231 | |||||||
3232 | MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(JTI, 4, nullptr); | ||||||
3233 | auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32, | ||||||
3234 | {TargetReg, ScratchReg}, {JTAddr, Index}) | ||||||
3235 | .addJumpTableIndex(JTI); | ||||||
3236 | // Build the indirect branch. | ||||||
3237 | MIB.buildInstr(AArch64::BR, {}, {TargetReg}); | ||||||
3238 | I.eraseFromParent(); | ||||||
3239 | return constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI); | ||||||
3240 | } | ||||||
3241 | |||||||
3242 | bool AArch64InstructionSelector::selectJumpTable( | ||||||
3243 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3244 | assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table")((I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_JUMP_TABLE && \"Expected jump table\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3244, __PRETTY_FUNCTION__)); | ||||||
3245 | assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!")((I.getOperand(1).isJTI() && "Jump table op should have a JTI!" ) ? static_cast<void> (0) : __assert_fail ("I.getOperand(1).isJTI() && \"Jump table op should have a JTI!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3245, __PRETTY_FUNCTION__)); | ||||||
3246 | |||||||
3247 | Register DstReg = I.getOperand(0).getReg(); | ||||||
3248 | unsigned JTI = I.getOperand(1).getIndex(); | ||||||
3249 | // We generate a MOVaddrJT which will get expanded to an ADRP + ADD later. | ||||||
3250 | MachineIRBuilder MIB(I); | ||||||
3251 | auto MovMI = | ||||||
3252 | MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {}) | ||||||
3253 | .addJumpTableIndex(JTI, AArch64II::MO_PAGE) | ||||||
3254 | .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF); | ||||||
3255 | I.eraseFromParent(); | ||||||
3256 | return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI); | ||||||
3257 | } | ||||||
3258 | |||||||
3259 | bool AArch64InstructionSelector::selectTLSGlobalValue( | ||||||
3260 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3261 | if (!STI.isTargetMachO()) | ||||||
3262 | return false; | ||||||
3263 | MachineFunction &MF = *I.getParent()->getParent(); | ||||||
3264 | MF.getFrameInfo().setAdjustsStack(true); | ||||||
3265 | |||||||
3266 | const GlobalValue &GV = *I.getOperand(1).getGlobal(); | ||||||
3267 | MachineIRBuilder MIB(I); | ||||||
3268 | |||||||
3269 | auto LoadGOT = | ||||||
3270 | MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {}) | ||||||
3271 | .addGlobalAddress(&GV, 0, AArch64II::MO_TLS); | ||||||
3272 | |||||||
3273 | auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass}, | ||||||
3274 | {LoadGOT.getReg(0)}) | ||||||
3275 | .addImm(0); | ||||||
3276 | |||||||
3277 | MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0)); | ||||||
3278 | // TLS calls preserve all registers except those that absolutely must be | ||||||
3279 | // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be | ||||||
3280 | // silly). | ||||||
3281 | MIB.buildInstr(getBLRCallOpcode(MF), {}, {Load}) | ||||||
3282 | .addUse(AArch64::X0, RegState::Implicit) | ||||||
3283 | .addDef(AArch64::X0, RegState::Implicit) | ||||||
3284 | .addRegMask(TRI.getTLSCallPreservedMask()); | ||||||
3285 | |||||||
3286 | MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0)); | ||||||
3287 | RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass, | ||||||
3288 | MRI); | ||||||
3289 | I.eraseFromParent(); | ||||||
3290 | return true; | ||||||
3291 | } | ||||||
3292 | |||||||
3293 | bool AArch64InstructionSelector::selectIntrinsicTrunc( | ||||||
3294 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3295 | const LLT SrcTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
3296 | |||||||
3297 | // Select the correct opcode. | ||||||
3298 | unsigned Opc = 0; | ||||||
3299 | if (!SrcTy.isVector()) { | ||||||
3300 | switch (SrcTy.getSizeInBits()) { | ||||||
3301 | default: | ||||||
3302 | case 16: | ||||||
3303 | Opc = AArch64::FRINTZHr; | ||||||
3304 | break; | ||||||
3305 | case 32: | ||||||
3306 | Opc = AArch64::FRINTZSr; | ||||||
3307 | break; | ||||||
3308 | case 64: | ||||||
3309 | Opc = AArch64::FRINTZDr; | ||||||
3310 | break; | ||||||
3311 | } | ||||||
3312 | } else { | ||||||
3313 | unsigned NumElts = SrcTy.getNumElements(); | ||||||
3314 | switch (SrcTy.getElementType().getSizeInBits()) { | ||||||
3315 | default: | ||||||
3316 | break; | ||||||
3317 | case 16: | ||||||
3318 | if (NumElts == 4) | ||||||
3319 | Opc = AArch64::FRINTZv4f16; | ||||||
3320 | else if (NumElts == 8) | ||||||
3321 | Opc = AArch64::FRINTZv8f16; | ||||||
3322 | break; | ||||||
3323 | case 32: | ||||||
3324 | if (NumElts == 2) | ||||||
3325 | Opc = AArch64::FRINTZv2f32; | ||||||
3326 | else if (NumElts == 4) | ||||||
3327 | Opc = AArch64::FRINTZv4f32; | ||||||
3328 | break; | ||||||
3329 | case 64: | ||||||
3330 | if (NumElts == 2) | ||||||
3331 | Opc = AArch64::FRINTZv2f64; | ||||||
3332 | break; | ||||||
3333 | } | ||||||
3334 | } | ||||||
3335 | |||||||
3336 | if (!Opc) { | ||||||
3337 | // Didn't get an opcode above, bail. | ||||||
3338 | LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n" ; } } while (false); | ||||||
3339 | return false; | ||||||
3340 | } | ||||||
3341 | |||||||
3342 | // Legalization would have set us up perfectly for this; we just need to | ||||||
3343 | // set the opcode and move on. | ||||||
3344 | I.setDesc(TII.get(Opc)); | ||||||
3345 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
3346 | } | ||||||
3347 | |||||||
3348 | bool AArch64InstructionSelector::selectIntrinsicRound( | ||||||
3349 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3350 | const LLT SrcTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
3351 | |||||||
3352 | // Select the correct opcode. | ||||||
3353 | unsigned Opc = 0; | ||||||
3354 | if (!SrcTy.isVector()) { | ||||||
3355 | switch (SrcTy.getSizeInBits()) { | ||||||
3356 | default: | ||||||
3357 | case 16: | ||||||
3358 | Opc = AArch64::FRINTAHr; | ||||||
3359 | break; | ||||||
3360 | case 32: | ||||||
3361 | Opc = AArch64::FRINTASr; | ||||||
3362 | break; | ||||||
3363 | case 64: | ||||||
3364 | Opc = AArch64::FRINTADr; | ||||||
3365 | break; | ||||||
3366 | } | ||||||
3367 | } else { | ||||||
3368 | unsigned NumElts = SrcTy.getNumElements(); | ||||||
3369 | switch (SrcTy.getElementType().getSizeInBits()) { | ||||||
3370 | default: | ||||||
3371 | break; | ||||||
3372 | case 16: | ||||||
3373 | if (NumElts == 4) | ||||||
3374 | Opc = AArch64::FRINTAv4f16; | ||||||
3375 | else if (NumElts == 8) | ||||||
3376 | Opc = AArch64::FRINTAv8f16; | ||||||
3377 | break; | ||||||
3378 | case 32: | ||||||
3379 | if (NumElts == 2) | ||||||
3380 | Opc = AArch64::FRINTAv2f32; | ||||||
3381 | else if (NumElts == 4) | ||||||
3382 | Opc = AArch64::FRINTAv4f32; | ||||||
3383 | break; | ||||||
3384 | case 64: | ||||||
3385 | if (NumElts == 2) | ||||||
3386 | Opc = AArch64::FRINTAv2f64; | ||||||
3387 | break; | ||||||
3388 | } | ||||||
3389 | } | ||||||
3390 | |||||||
3391 | if (!Opc) { | ||||||
3392 | // Didn't get an opcode above, bail. | ||||||
3393 | LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n" ; } } while (false); | ||||||
3394 | return false; | ||||||
3395 | } | ||||||
3396 | |||||||
3397 | // Legalization would have set us up perfectly for this; we just need to | ||||||
3398 | // set the opcode and move on. | ||||||
3399 | I.setDesc(TII.get(Opc)); | ||||||
3400 | return constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
3401 | } | ||||||
3402 | |||||||
3403 | bool AArch64InstructionSelector::selectVectorICmp( | ||||||
3404 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3405 | Register DstReg = I.getOperand(0).getReg(); | ||||||
3406 | LLT DstTy = MRI.getType(DstReg); | ||||||
3407 | Register SrcReg = I.getOperand(2).getReg(); | ||||||
3408 | Register Src2Reg = I.getOperand(3).getReg(); | ||||||
3409 | LLT SrcTy = MRI.getType(SrcReg); | ||||||
3410 | |||||||
3411 | unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits(); | ||||||
3412 | unsigned NumElts = DstTy.getNumElements(); | ||||||
3413 | |||||||
3414 | // First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b | ||||||
3415 | // Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16 | ||||||
3416 | // Third index is cc opcode: | ||||||
3417 | // 0 == eq | ||||||
3418 | // 1 == ugt | ||||||
3419 | // 2 == uge | ||||||
3420 | // 3 == ult | ||||||
3421 | // 4 == ule | ||||||
3422 | // 5 == sgt | ||||||
3423 | // 6 == sge | ||||||
3424 | // 7 == slt | ||||||
3425 | // 8 == sle | ||||||
3426 | // ne is done by negating 'eq' result. | ||||||
3427 | |||||||
3428 | // This table below assumes that for some comparisons the operands will be | ||||||
3429 | // commuted. | ||||||
3430 | // ult op == commute + ugt op | ||||||
3431 | // ule op == commute + uge op | ||||||
3432 | // slt op == commute + sgt op | ||||||
3433 | // sle op == commute + sge op | ||||||
3434 | unsigned PredIdx = 0; | ||||||
3435 | bool SwapOperands = false; | ||||||
3436 | CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate(); | ||||||
3437 | switch (Pred) { | ||||||
3438 | case CmpInst::ICMP_NE: | ||||||
3439 | case CmpInst::ICMP_EQ: | ||||||
3440 | PredIdx = 0; | ||||||
3441 | break; | ||||||
3442 | case CmpInst::ICMP_UGT: | ||||||
3443 | PredIdx = 1; | ||||||
3444 | break; | ||||||
3445 | case CmpInst::ICMP_UGE: | ||||||
3446 | PredIdx = 2; | ||||||
3447 | break; | ||||||
3448 | case CmpInst::ICMP_ULT: | ||||||
3449 | PredIdx = 3; | ||||||
3450 | SwapOperands = true; | ||||||
3451 | break; | ||||||
3452 | case CmpInst::ICMP_ULE: | ||||||
3453 | PredIdx = 4; | ||||||
3454 | SwapOperands = true; | ||||||
3455 | break; | ||||||
3456 | case CmpInst::ICMP_SGT: | ||||||
3457 | PredIdx = 5; | ||||||
3458 | break; | ||||||
3459 | case CmpInst::ICMP_SGE: | ||||||
3460 | PredIdx = 6; | ||||||
3461 | break; | ||||||
3462 | case CmpInst::ICMP_SLT: | ||||||
3463 | PredIdx = 7; | ||||||
3464 | SwapOperands = true; | ||||||
3465 | break; | ||||||
3466 | case CmpInst::ICMP_SLE: | ||||||
3467 | PredIdx = 8; | ||||||
3468 | SwapOperands = true; | ||||||
3469 | break; | ||||||
3470 | default: | ||||||
3471 | llvm_unreachable("Unhandled icmp predicate")::llvm::llvm_unreachable_internal("Unhandled icmp predicate", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3471); | ||||||
3472 | return false; | ||||||
3473 | } | ||||||
3474 | |||||||
3475 | // This table obviously should be tablegen'd when we have our GISel native | ||||||
3476 | // tablegen selector. | ||||||
3477 | |||||||
3478 | static const unsigned OpcTable[4][4][9] = { | ||||||
3479 | { | ||||||
3480 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3481 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3482 | 0 /* invalid */}, | ||||||
3483 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3484 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3485 | 0 /* invalid */}, | ||||||
3486 | {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8, | ||||||
3487 | AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8, | ||||||
3488 | AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8}, | ||||||
3489 | {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8, | ||||||
3490 | AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8, | ||||||
3491 | AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8} | ||||||
3492 | }, | ||||||
3493 | { | ||||||
3494 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3495 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3496 | 0 /* invalid */}, | ||||||
3497 | {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16, | ||||||
3498 | AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16, | ||||||
3499 | AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16}, | ||||||
3500 | {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16, | ||||||
3501 | AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16, | ||||||
3502 | AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16}, | ||||||
3503 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3504 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3505 | 0 /* invalid */} | ||||||
3506 | }, | ||||||
3507 | { | ||||||
3508 | {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32, | ||||||
3509 | AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32, | ||||||
3510 | AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32}, | ||||||
3511 | {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32, | ||||||
3512 | AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32, | ||||||
3513 | AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32}, | ||||||
3514 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3515 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3516 | 0 /* invalid */}, | ||||||
3517 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3518 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3519 | 0 /* invalid */} | ||||||
3520 | }, | ||||||
3521 | { | ||||||
3522 | {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64, | ||||||
3523 | AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64, | ||||||
3524 | AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64}, | ||||||
3525 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3526 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3527 | 0 /* invalid */}, | ||||||
3528 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3529 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3530 | 0 /* invalid */}, | ||||||
3531 | {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3532 | 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */, | ||||||
3533 | 0 /* invalid */} | ||||||
3534 | }, | ||||||
3535 | }; | ||||||
3536 | unsigned EltIdx = Log2_32(SrcEltSize / 8); | ||||||
3537 | unsigned NumEltsIdx = Log2_32(NumElts / 2); | ||||||
3538 | unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx]; | ||||||
3539 | if (!Opc) { | ||||||
3540 | LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not map G_ICMP to cmp opcode" ; } } while (false); | ||||||
3541 | return false; | ||||||
3542 | } | ||||||
3543 | |||||||
3544 | const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI); | ||||||
3545 | const TargetRegisterClass *SrcRC = | ||||||
3546 | getRegClassForTypeOnBank(SrcTy, VecRB, RBI, true); | ||||||
3547 | if (!SrcRC) { | ||||||
3548 | LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not determine source register class.\n" ; } } while (false); | ||||||
3549 | return false; | ||||||
3550 | } | ||||||
3551 | |||||||
3552 | unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0; | ||||||
3553 | if (SrcTy.getSizeInBits() == 128) | ||||||
3554 | NotOpc = NotOpc ? AArch64::NOTv16i8 : 0; | ||||||
3555 | |||||||
3556 | if (SwapOperands) | ||||||
3557 | std::swap(SrcReg, Src2Reg); | ||||||
3558 | |||||||
3559 | MachineIRBuilder MIB(I); | ||||||
3560 | auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg}); | ||||||
3561 | constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI); | ||||||
3562 | |||||||
3563 | // Invert if we had a 'ne' cc. | ||||||
3564 | if (NotOpc) { | ||||||
3565 | Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp}); | ||||||
3566 | constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI); | ||||||
3567 | } else { | ||||||
3568 | MIB.buildCopy(DstReg, Cmp.getReg(0)); | ||||||
3569 | } | ||||||
3570 | RBI.constrainGenericRegister(DstReg, *SrcRC, MRI); | ||||||
3571 | I.eraseFromParent(); | ||||||
3572 | return true; | ||||||
3573 | } | ||||||
3574 | |||||||
3575 | MachineInstr *AArch64InstructionSelector::emitScalarToVector( | ||||||
3576 | unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar, | ||||||
3577 | MachineIRBuilder &MIRBuilder) const { | ||||||
3578 | auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {}); | ||||||
3579 | |||||||
3580 | auto BuildFn = [&](unsigned SubregIndex) { | ||||||
3581 | auto Ins = | ||||||
3582 | MIRBuilder | ||||||
3583 | .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar}) | ||||||
3584 | .addImm(SubregIndex); | ||||||
3585 | constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI); | ||||||
3586 | constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI); | ||||||
3587 | return &*Ins; | ||||||
3588 | }; | ||||||
3589 | |||||||
3590 | switch (EltSize) { | ||||||
3591 | case 16: | ||||||
3592 | return BuildFn(AArch64::hsub); | ||||||
3593 | case 32: | ||||||
3594 | return BuildFn(AArch64::ssub); | ||||||
3595 | case 64: | ||||||
3596 | return BuildFn(AArch64::dsub); | ||||||
3597 | default: | ||||||
3598 | return nullptr; | ||||||
3599 | } | ||||||
3600 | } | ||||||
3601 | |||||||
3602 | bool AArch64InstructionSelector::selectMergeValues( | ||||||
3603 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3604 | assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode")((I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_MERGE_VALUES && \"unexpected opcode\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3604, __PRETTY_FUNCTION__)); | ||||||
3605 | const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
3606 | const LLT SrcTy = MRI.getType(I.getOperand(1).getReg()); | ||||||
3607 | assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation")((!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation" ) ? static_cast<void> (0) : __assert_fail ("!DstTy.isVector() && !SrcTy.isVector() && \"invalid merge operation\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3607, __PRETTY_FUNCTION__)); | ||||||
3608 | const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI); | ||||||
3609 | |||||||
3610 | if (I.getNumOperands() != 3) | ||||||
3611 | return false; | ||||||
3612 | |||||||
3613 | // Merging 2 s64s into an s128. | ||||||
3614 | if (DstTy == LLT::scalar(128)) { | ||||||
3615 | if (SrcTy.getSizeInBits() != 64) | ||||||
3616 | return false; | ||||||
3617 | MachineIRBuilder MIB(I); | ||||||
3618 | Register DstReg = I.getOperand(0).getReg(); | ||||||
3619 | Register Src1Reg = I.getOperand(1).getReg(); | ||||||
3620 | Register Src2Reg = I.getOperand(2).getReg(); | ||||||
3621 | auto Tmp = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {}); | ||||||
3622 | MachineInstr *InsMI = | ||||||
3623 | emitLaneInsert(None, Tmp.getReg(0), Src1Reg, /* LaneIdx */ 0, RB, MIB); | ||||||
3624 | if (!InsMI) | ||||||
3625 | return false; | ||||||
3626 | MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->getOperand(0).getReg(), | ||||||
3627 | Src2Reg, /* LaneIdx */ 1, RB, MIB); | ||||||
3628 | if (!Ins2MI) | ||||||
3629 | return false; | ||||||
3630 | constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI); | ||||||
3631 | constrainSelectedInstRegOperands(*Ins2MI, TII, TRI, RBI); | ||||||
3632 | I.eraseFromParent(); | ||||||
3633 | return true; | ||||||
3634 | } | ||||||
3635 | |||||||
3636 | if (RB.getID() != AArch64::GPRRegBankID) | ||||||
3637 | return false; | ||||||
3638 | |||||||
3639 | if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32) | ||||||
3640 | return false; | ||||||
3641 | |||||||
3642 | auto *DstRC = &AArch64::GPR64RegClass; | ||||||
3643 | Register SubToRegDef = MRI.createVirtualRegister(DstRC); | ||||||
3644 | MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | ||||||
3645 | TII.get(TargetOpcode::SUBREG_TO_REG)) | ||||||
3646 | .addDef(SubToRegDef) | ||||||
3647 | .addImm(0) | ||||||
3648 | .addUse(I.getOperand(1).getReg()) | ||||||
3649 | .addImm(AArch64::sub_32); | ||||||
3650 | Register SubToRegDef2 = MRI.createVirtualRegister(DstRC); | ||||||
3651 | // Need to anyext the second scalar before we can use bfm | ||||||
3652 | MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), | ||||||
3653 | TII.get(TargetOpcode::SUBREG_TO_REG)) | ||||||
3654 | .addDef(SubToRegDef2) | ||||||
3655 | .addImm(0) | ||||||
3656 | .addUse(I.getOperand(2).getReg()) | ||||||
3657 | .addImm(AArch64::sub_32); | ||||||
3658 | MachineInstr &BFM = | ||||||
3659 | *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri)) | ||||||
3660 | .addDef(I.getOperand(0).getReg()) | ||||||
3661 | .addUse(SubToRegDef) | ||||||
3662 | .addUse(SubToRegDef2) | ||||||
3663 | .addImm(32) | ||||||
3664 | .addImm(31); | ||||||
3665 | constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI); | ||||||
3666 | constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI); | ||||||
3667 | constrainSelectedInstRegOperands(BFM, TII, TRI, RBI); | ||||||
3668 | I.eraseFromParent(); | ||||||
3669 | return true; | ||||||
3670 | } | ||||||
3671 | |||||||
3672 | static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg, | ||||||
3673 | const unsigned EltSize) { | ||||||
3674 | // Choose a lane copy opcode and subregister based off of the size of the | ||||||
3675 | // vector's elements. | ||||||
3676 | switch (EltSize) { | ||||||
3677 | case 16: | ||||||
3678 | CopyOpc = AArch64::CPYi16; | ||||||
3679 | ExtractSubReg = AArch64::hsub; | ||||||
3680 | break; | ||||||
3681 | case 32: | ||||||
3682 | CopyOpc = AArch64::CPYi32; | ||||||
3683 | ExtractSubReg = AArch64::ssub; | ||||||
3684 | break; | ||||||
3685 | case 64: | ||||||
3686 | CopyOpc = AArch64::CPYi64; | ||||||
3687 | ExtractSubReg = AArch64::dsub; | ||||||
3688 | break; | ||||||
3689 | default: | ||||||
3690 | // Unknown size, bail out. | ||||||
3691 | LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Elt size '" << EltSize << "' unsupported.\n"; } } while (false); | ||||||
3692 | return false; | ||||||
3693 | } | ||||||
3694 | return true; | ||||||
3695 | } | ||||||
3696 | |||||||
3697 | MachineInstr *AArch64InstructionSelector::emitExtractVectorElt( | ||||||
3698 | Optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy, | ||||||
3699 | Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const { | ||||||
3700 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); | ||||||
3701 | unsigned CopyOpc = 0; | ||||||
3702 | unsigned ExtractSubReg = 0; | ||||||
3703 | if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) { | ||||||
3704 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n" ; } } while (false) | ||||||
3705 | dbgs() << "Couldn't determine lane copy opcode for instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n" ; } } while (false); | ||||||
3706 | return nullptr; | ||||||
3707 | } | ||||||
3708 | |||||||
3709 | const TargetRegisterClass *DstRC = | ||||||
3710 | getRegClassForTypeOnBank(ScalarTy, DstRB, RBI, true); | ||||||
3711 | if (!DstRC) { | ||||||
3712 | LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not determine destination register class.\n" ; } } while (false); | ||||||
3713 | return nullptr; | ||||||
3714 | } | ||||||
3715 | |||||||
3716 | const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI); | ||||||
3717 | const LLT &VecTy = MRI.getType(VecReg); | ||||||
3718 | const TargetRegisterClass *VecRC = | ||||||
3719 | getRegClassForTypeOnBank(VecTy, VecRB, RBI, true); | ||||||
3720 | if (!VecRC) { | ||||||
3721 | LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not determine source register class.\n" ; } } while (false); | ||||||
3722 | return nullptr; | ||||||
3723 | } | ||||||
3724 | |||||||
3725 | // The register that we're going to copy into. | ||||||
3726 | Register InsertReg = VecReg; | ||||||
3727 | if (!DstReg) | ||||||
3728 | DstReg = MRI.createVirtualRegister(DstRC); | ||||||
3729 | // If the lane index is 0, we just use a subregister COPY. | ||||||
3730 | if (LaneIdx == 0) { | ||||||
3731 | auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {}) | ||||||
3732 | .addReg(VecReg, 0, ExtractSubReg); | ||||||
3733 | RBI.constrainGenericRegister(*DstReg, *DstRC, MRI); | ||||||
3734 | return &*Copy; | ||||||
3735 | } | ||||||
3736 | |||||||
3737 | // Lane copies require 128-bit wide registers. If we're dealing with an | ||||||
3738 | // unpacked vector, then we need to move up to that width. Insert an implicit | ||||||
3739 | // def and a subregister insert to get us there. | ||||||
3740 | if (VecTy.getSizeInBits() != 128) { | ||||||
3741 | MachineInstr *ScalarToVector = emitScalarToVector( | ||||||
3742 | VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder); | ||||||
3743 | if (!ScalarToVector) | ||||||
3744 | return nullptr; | ||||||
3745 | InsertReg = ScalarToVector->getOperand(0).getReg(); | ||||||
3746 | } | ||||||
3747 | |||||||
3748 | MachineInstr *LaneCopyMI = | ||||||
3749 | MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx); | ||||||
3750 | constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI); | ||||||
3751 | |||||||
3752 | // Make sure that we actually constrain the initial copy. | ||||||
3753 | RBI.constrainGenericRegister(*DstReg, *DstRC, MRI); | ||||||
3754 | return LaneCopyMI; | ||||||
3755 | } | ||||||
3756 | |||||||
3757 | bool AArch64InstructionSelector::selectExtractElt( | ||||||
3758 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3759 | assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && "unexpected opcode!") ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3760, __PRETTY_FUNCTION__)) | ||||||
3760 | "unexpected opcode!")((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && "unexpected opcode!") ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3760, __PRETTY_FUNCTION__)); | ||||||
3761 | Register DstReg = I.getOperand(0).getReg(); | ||||||
3762 | const LLT NarrowTy = MRI.getType(DstReg); | ||||||
3763 | const Register SrcReg = I.getOperand(1).getReg(); | ||||||
3764 | const LLT WideTy = MRI.getType(SrcReg); | ||||||
3765 | (void)WideTy; | ||||||
3766 | assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && "source register size too small!") ? static_cast<void> (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3767, __PRETTY_FUNCTION__)) | ||||||
3767 | "source register size too small!")((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && "source register size too small!") ? static_cast<void> (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3767, __PRETTY_FUNCTION__)); | ||||||
3768 | assert(!NarrowTy.isVector() && "cannot extract vector into vector!")((!NarrowTy.isVector() && "cannot extract vector into vector!" ) ? static_cast<void> (0) : __assert_fail ("!NarrowTy.isVector() && \"cannot extract vector into vector!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3768, __PRETTY_FUNCTION__)); | ||||||
3769 | |||||||
3770 | // Need the lane index to determine the correct copy opcode. | ||||||
3771 | MachineOperand &LaneIdxOp = I.getOperand(2); | ||||||
3772 | assert(LaneIdxOp.isReg() && "Lane index operand was not a register?")((LaneIdxOp.isReg() && "Lane index operand was not a register?" ) ? static_cast<void> (0) : __assert_fail ("LaneIdxOp.isReg() && \"Lane index operand was not a register?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3772, __PRETTY_FUNCTION__)); | ||||||
3773 | |||||||
3774 | if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) { | ||||||
3775 | LLVM_DEBUG(dbgs() << "Cannot extract into GPR.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Cannot extract into GPR.\n" ; } } while (false); | ||||||
3776 | return false; | ||||||
3777 | } | ||||||
3778 | |||||||
3779 | // Find the index to extract from. | ||||||
3780 | auto VRegAndVal = getConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI); | ||||||
3781 | if (!VRegAndVal) | ||||||
3782 | return false; | ||||||
3783 | unsigned LaneIdx = VRegAndVal->Value.getSExtValue(); | ||||||
3784 | |||||||
3785 | MachineIRBuilder MIRBuilder(I); | ||||||
3786 | |||||||
3787 | const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); | ||||||
3788 | MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg, | ||||||
3789 | LaneIdx, MIRBuilder); | ||||||
3790 | if (!Extract) | ||||||
3791 | return false; | ||||||
3792 | |||||||
3793 | I.eraseFromParent(); | ||||||
3794 | return true; | ||||||
3795 | } | ||||||
3796 | |||||||
3797 | bool AArch64InstructionSelector::selectSplitVectorUnmerge( | ||||||
3798 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3799 | unsigned NumElts = I.getNumOperands() - 1; | ||||||
3800 | Register SrcReg = I.getOperand(NumElts).getReg(); | ||||||
3801 | const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
3802 | const LLT SrcTy = MRI.getType(SrcReg); | ||||||
3803 | |||||||
3804 | assert(NarrowTy.isVector() && "Expected an unmerge into vectors")((NarrowTy.isVector() && "Expected an unmerge into vectors" ) ? static_cast<void> (0) : __assert_fail ("NarrowTy.isVector() && \"Expected an unmerge into vectors\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3804, __PRETTY_FUNCTION__)); | ||||||
3805 | if (SrcTy.getSizeInBits() > 128) { | ||||||
3806 | LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unexpected vector type for vec split unmerge" ; } } while (false); | ||||||
3807 | return false; | ||||||
3808 | } | ||||||
3809 | |||||||
3810 | MachineIRBuilder MIB(I); | ||||||
3811 | |||||||
3812 | // We implement a split vector operation by treating the sub-vectors as | ||||||
3813 | // scalars and extracting them. | ||||||
3814 | const RegisterBank &DstRB = | ||||||
3815 | *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI); | ||||||
3816 | for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) { | ||||||
3817 | Register Dst = I.getOperand(OpIdx).getReg(); | ||||||
3818 | MachineInstr *Extract = | ||||||
3819 | emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB); | ||||||
3820 | if (!Extract) | ||||||
3821 | return false; | ||||||
3822 | } | ||||||
3823 | I.eraseFromParent(); | ||||||
3824 | return true; | ||||||
3825 | } | ||||||
3826 | |||||||
3827 | bool AArch64InstructionSelector::selectUnmergeValues( | ||||||
3828 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3829 | assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3830, __PRETTY_FUNCTION__)) | ||||||
3830 | "unexpected opcode")((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3830, __PRETTY_FUNCTION__)); | ||||||
3831 | |||||||
3832 | // TODO: Handle unmerging into GPRs and from scalars to scalars. | ||||||
3833 | if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() != | ||||||
3834 | AArch64::FPRRegBankID || | ||||||
3835 | RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() != | ||||||
3836 | AArch64::FPRRegBankID) { | ||||||
3837 | LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar " "currently unsupported.\n"; } } while (false) | ||||||
3838 | "currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar " "currently unsupported.\n"; } } while (false); | ||||||
3839 | return false; | ||||||
3840 | } | ||||||
3841 | |||||||
3842 | // The last operand is the vector source register, and every other operand is | ||||||
3843 | // a register to unpack into. | ||||||
3844 | unsigned NumElts = I.getNumOperands() - 1; | ||||||
3845 | Register SrcReg = I.getOperand(NumElts).getReg(); | ||||||
3846 | const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
3847 | const LLT WideTy = MRI.getType(SrcReg); | ||||||
3848 | (void)WideTy; | ||||||
3849 | assert((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&(((WideTy.isVector() || WideTy.getSizeInBits() == 128) && "can only unmerge from vector or s128 types!") ? static_cast <void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3850, __PRETTY_FUNCTION__)) | ||||||
3850 | "can only unmerge from vector or s128 types!")(((WideTy.isVector() || WideTy.getSizeInBits() == 128) && "can only unmerge from vector or s128 types!") ? static_cast <void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3850, __PRETTY_FUNCTION__)); | ||||||
3851 | assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && "source register size too small!") ? static_cast<void> (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3852, __PRETTY_FUNCTION__)) | ||||||
3852 | "source register size too small!")((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && "source register size too small!") ? static_cast<void> (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3852, __PRETTY_FUNCTION__)); | ||||||
3853 | |||||||
3854 | if (!NarrowTy.isScalar()) | ||||||
3855 | return selectSplitVectorUnmerge(I, MRI); | ||||||
3856 | |||||||
3857 | MachineIRBuilder MIB(I); | ||||||
3858 | |||||||
3859 | // Choose a lane copy opcode and subregister based off of the size of the | ||||||
3860 | // vector's elements. | ||||||
3861 | unsigned CopyOpc = 0; | ||||||
3862 | unsigned ExtractSubReg = 0; | ||||||
3863 | if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits())) | ||||||
3864 | return false; | ||||||
3865 | |||||||
3866 | // Set up for the lane copies. | ||||||
3867 | MachineBasicBlock &MBB = *I.getParent(); | ||||||
3868 | |||||||
3869 | // Stores the registers we'll be copying from. | ||||||
3870 | SmallVector<Register, 4> InsertRegs; | ||||||
3871 | |||||||
3872 | // We'll use the first register twice, so we only need NumElts-1 registers. | ||||||
3873 | unsigned NumInsertRegs = NumElts - 1; | ||||||
3874 | |||||||
3875 | // If our elements fit into exactly 128 bits, then we can copy from the source | ||||||
3876 | // directly. Otherwise, we need to do a bit of setup with some subregister | ||||||
3877 | // inserts. | ||||||
3878 | if (NarrowTy.getSizeInBits() * NumElts == 128) { | ||||||
3879 | InsertRegs = SmallVector<Register, 4>(NumInsertRegs, SrcReg); | ||||||
3880 | } else { | ||||||
3881 | // No. We have to perform subregister inserts. For each insert, create an | ||||||
3882 | // implicit def and a subregister insert, and save the register we create. | ||||||
3883 | for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) { | ||||||
3884 | Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass); | ||||||
3885 | MachineInstr &ImpDefMI = | ||||||
3886 | *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF), | ||||||
3887 | ImpDefReg); | ||||||
3888 | |||||||
3889 | // Now, create the subregister insert from SrcReg. | ||||||
3890 | Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass); | ||||||
3891 | MachineInstr &InsMI = | ||||||
3892 | *BuildMI(MBB, I, I.getDebugLoc(), | ||||||
3893 | TII.get(TargetOpcode::INSERT_SUBREG), InsertReg) | ||||||
3894 | .addUse(ImpDefReg) | ||||||
3895 | .addUse(SrcReg) | ||||||
3896 | .addImm(AArch64::dsub); | ||||||
3897 | |||||||
3898 | constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI); | ||||||
3899 | constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI); | ||||||
3900 | |||||||
3901 | // Save the register so that we can copy from it after. | ||||||
3902 | InsertRegs.push_back(InsertReg); | ||||||
3903 | } | ||||||
3904 | } | ||||||
3905 | |||||||
3906 | // Now that we've created any necessary subregister inserts, we can | ||||||
3907 | // create the copies. | ||||||
3908 | // | ||||||
3909 | // Perform the first copy separately as a subregister copy. | ||||||
3910 | Register CopyTo = I.getOperand(0).getReg(); | ||||||
3911 | auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {}) | ||||||
3912 | .addReg(InsertRegs[0], 0, ExtractSubReg); | ||||||
3913 | constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI); | ||||||
3914 | |||||||
3915 | // Now, perform the remaining copies as vector lane copies. | ||||||
3916 | unsigned LaneIdx = 1; | ||||||
3917 | for (Register InsReg : InsertRegs) { | ||||||
3918 | Register CopyTo = I.getOperand(LaneIdx).getReg(); | ||||||
3919 | MachineInstr &CopyInst = | ||||||
3920 | *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo) | ||||||
3921 | .addUse(InsReg) | ||||||
3922 | .addImm(LaneIdx); | ||||||
3923 | constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI); | ||||||
3924 | ++LaneIdx; | ||||||
3925 | } | ||||||
3926 | |||||||
3927 | // Separately constrain the first copy's destination. Because of the | ||||||
3928 | // limitation in constrainOperandRegClass, we can't guarantee that this will | ||||||
3929 | // actually be constrained. So, do it ourselves using the second operand. | ||||||
3930 | const TargetRegisterClass *RC = | ||||||
3931 | MRI.getRegClassOrNull(I.getOperand(1).getReg()); | ||||||
3932 | if (!RC) { | ||||||
3933 | LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Couldn't constrain copy destination.\n" ; } } while (false); | ||||||
3934 | return false; | ||||||
3935 | } | ||||||
3936 | |||||||
3937 | RBI.constrainGenericRegister(CopyTo, *RC, MRI); | ||||||
3938 | I.eraseFromParent(); | ||||||
3939 | return true; | ||||||
3940 | } | ||||||
3941 | |||||||
3942 | bool AArch64InstructionSelector::selectConcatVectors( | ||||||
3943 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
3944 | assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3945, __PRETTY_FUNCTION__)) | ||||||
3945 | "Unexpected opcode")((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 3945, __PRETTY_FUNCTION__)); | ||||||
3946 | Register Dst = I.getOperand(0).getReg(); | ||||||
3947 | Register Op1 = I.getOperand(1).getReg(); | ||||||
3948 | Register Op2 = I.getOperand(2).getReg(); | ||||||
3949 | MachineIRBuilder MIRBuilder(I); | ||||||
3950 | MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder); | ||||||
3951 | if (!ConcatMI) | ||||||
3952 | return false; | ||||||
3953 | I.eraseFromParent(); | ||||||
3954 | return true; | ||||||
3955 | } | ||||||
3956 | |||||||
3957 | unsigned | ||||||
3958 | AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal, | ||||||
3959 | MachineFunction &MF) const { | ||||||
3960 | Type *CPTy = CPVal->getType(); | ||||||
3961 | Align Alignment = MF.getDataLayout().getPrefTypeAlign(CPTy); | ||||||
3962 | |||||||
3963 | MachineConstantPool *MCP = MF.getConstantPool(); | ||||||
3964 | return MCP->getConstantPoolIndex(CPVal, Alignment); | ||||||
3965 | } | ||||||
3966 | |||||||
3967 | MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool( | ||||||
3968 | const Constant *CPVal, MachineIRBuilder &MIRBuilder) const { | ||||||
3969 | unsigned CPIdx = emitConstantPoolEntry(CPVal, MIRBuilder.getMF()); | ||||||
3970 | |||||||
3971 | auto Adrp = | ||||||
3972 | MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {}) | ||||||
3973 | .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE); | ||||||
3974 | |||||||
3975 | MachineInstr *LoadMI = nullptr; | ||||||
3976 | switch (MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType())) { | ||||||
3977 | case 16: | ||||||
3978 | LoadMI = | ||||||
3979 | &*MIRBuilder | ||||||
3980 | .buildInstr(AArch64::LDRQui, {&AArch64::FPR128RegClass}, {Adrp}) | ||||||
3981 | .addConstantPoolIndex(CPIdx, 0, | ||||||
3982 | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); | ||||||
3983 | break; | ||||||
3984 | case 8: | ||||||
3985 | LoadMI = &*MIRBuilder | ||||||
3986 | .buildInstr(AArch64::LDRDui, {&AArch64::FPR64RegClass}, {Adrp}) | ||||||
3987 | .addConstantPoolIndex( | ||||||
3988 | CPIdx, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); | ||||||
3989 | break; | ||||||
3990 | default: | ||||||
3991 | LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not load from constant pool of type " << *CPVal->getType(); } } while (false) | ||||||
3992 | << *CPVal->getType())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not load from constant pool of type " << *CPVal->getType(); } } while (false); | ||||||
3993 | return nullptr; | ||||||
3994 | } | ||||||
3995 | constrainSelectedInstRegOperands(*Adrp, TII, TRI, RBI); | ||||||
3996 | constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI); | ||||||
3997 | return LoadMI; | ||||||
3998 | } | ||||||
3999 | |||||||
4000 | /// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given | ||||||
4001 | /// size and RB. | ||||||
4002 | static std::pair<unsigned, unsigned> | ||||||
4003 | getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) { | ||||||
4004 | unsigned Opc, SubregIdx; | ||||||
4005 | if (RB.getID() == AArch64::GPRRegBankID) { | ||||||
4006 | if (EltSize == 16) { | ||||||
4007 | Opc = AArch64::INSvi16gpr; | ||||||
4008 | SubregIdx = AArch64::ssub; | ||||||
4009 | } else if (EltSize == 32) { | ||||||
4010 | Opc = AArch64::INSvi32gpr; | ||||||
4011 | SubregIdx = AArch64::ssub; | ||||||
4012 | } else if (EltSize == 64) { | ||||||
4013 | Opc = AArch64::INSvi64gpr; | ||||||
4014 | SubregIdx = AArch64::dsub; | ||||||
4015 | } else { | ||||||
4016 | llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4016); | ||||||
4017 | } | ||||||
4018 | } else { | ||||||
4019 | if (EltSize == 8) { | ||||||
4020 | Opc = AArch64::INSvi8lane; | ||||||
4021 | SubregIdx = AArch64::bsub; | ||||||
4022 | } else if (EltSize == 16) { | ||||||
4023 | Opc = AArch64::INSvi16lane; | ||||||
4024 | SubregIdx = AArch64::hsub; | ||||||
4025 | } else if (EltSize == 32) { | ||||||
4026 | Opc = AArch64::INSvi32lane; | ||||||
4027 | SubregIdx = AArch64::ssub; | ||||||
4028 | } else if (EltSize == 64) { | ||||||
4029 | Opc = AArch64::INSvi64lane; | ||||||
4030 | SubregIdx = AArch64::dsub; | ||||||
4031 | } else { | ||||||
4032 | llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4032); | ||||||
4033 | } | ||||||
4034 | } | ||||||
4035 | return std::make_pair(Opc, SubregIdx); | ||||||
4036 | } | ||||||
4037 | |||||||
4038 | MachineInstr *AArch64InstructionSelector::emitInstr( | ||||||
4039 | unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps, | ||||||
4040 | std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder, | ||||||
4041 | const ComplexRendererFns &RenderFns) const { | ||||||
4042 | assert(Opcode && "Expected an opcode?")((Opcode && "Expected an opcode?") ? static_cast<void > (0) : __assert_fail ("Opcode && \"Expected an opcode?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4042, __PRETTY_FUNCTION__)); | ||||||
4043 | assert(!isPreISelGenericOpcode(Opcode) &&((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!" ) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4044, __PRETTY_FUNCTION__)) | ||||||
4044 | "Function should only be used to produce selected instructions!")((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!" ) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4044, __PRETTY_FUNCTION__)); | ||||||
4045 | auto MI = MIRBuilder.buildInstr(Opcode, DstOps, SrcOps); | ||||||
4046 | if (RenderFns) | ||||||
4047 | for (auto &Fn : *RenderFns) | ||||||
4048 | Fn(MI); | ||||||
4049 | constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); | ||||||
4050 | return &*MI; | ||||||
4051 | } | ||||||
4052 | |||||||
4053 | MachineInstr *AArch64InstructionSelector::emitAddSub( | ||||||
4054 | const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode, | ||||||
4055 | Register Dst, MachineOperand &LHS, MachineOperand &RHS, | ||||||
4056 | MachineIRBuilder &MIRBuilder) const { | ||||||
4057 | MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); | ||||||
4058 | assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")((LHS.isReg() && RHS.isReg() && "Expected register operands?" ) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4058, __PRETTY_FUNCTION__)); | ||||||
4059 | auto Ty = MRI.getType(LHS.getReg()); | ||||||
4060 | assert(!Ty.isVector() && "Expected a scalar or pointer?")((!Ty.isVector() && "Expected a scalar or pointer?") ? static_cast<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar or pointer?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4060, __PRETTY_FUNCTION__)); | ||||||
4061 | unsigned Size = Ty.getSizeInBits(); | ||||||
4062 | assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only")(((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only" ) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit type only\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4062, __PRETTY_FUNCTION__)); | ||||||
4063 | bool Is32Bit = Size == 32; | ||||||
4064 | |||||||
4065 | // INSTRri form with positive arithmetic immediate. | ||||||
4066 | if (auto Fns = selectArithImmed(RHS)) | ||||||
4067 | return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {LHS}, | ||||||
4068 | MIRBuilder, Fns); | ||||||
4069 | |||||||
4070 | // INSTRri form with negative arithmetic immediate. | ||||||
4071 | if (auto Fns = selectNegArithImmed(RHS)) | ||||||
4072 | return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {LHS}, | ||||||
4073 | MIRBuilder, Fns); | ||||||
4074 | |||||||
4075 | // INSTRrx form. | ||||||
4076 | if (auto Fns = selectArithExtendedRegister(RHS)) | ||||||
4077 | return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {LHS}, | ||||||
4078 | MIRBuilder, Fns); | ||||||
4079 | |||||||
4080 | // INSTRrs form. | ||||||
4081 | if (auto Fns = selectShiftedRegister(RHS)) | ||||||
4082 | return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {LHS}, | ||||||
4083 | MIRBuilder, Fns); | ||||||
4084 | return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {LHS, RHS}, | ||||||
4085 | MIRBuilder); | ||||||
4086 | } | ||||||
4087 | |||||||
4088 | MachineInstr * | ||||||
4089 | AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS, | ||||||
4090 | MachineOperand &RHS, | ||||||
4091 | MachineIRBuilder &MIRBuilder) const { | ||||||
4092 | const std::array<std::array<unsigned, 2>, 5> OpcTable{ | ||||||
4093 | {{AArch64::ADDXri, AArch64::ADDWri}, | ||||||
4094 | {AArch64::ADDXrs, AArch64::ADDWrs}, | ||||||
4095 | {AArch64::ADDXrr, AArch64::ADDWrr}, | ||||||
4096 | {AArch64::SUBXri, AArch64::SUBWri}, | ||||||
4097 | {AArch64::ADDXrx, AArch64::ADDWrx}}}; | ||||||
4098 | return emitAddSub(OpcTable, DefReg, LHS, RHS, MIRBuilder); | ||||||
| |||||||
4099 | } | ||||||
4100 | |||||||
4101 | MachineInstr * | ||||||
4102 | AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS, | ||||||
4103 | MachineOperand &RHS, | ||||||
4104 | MachineIRBuilder &MIRBuilder) const { | ||||||
4105 | const std::array<std::array<unsigned, 2>, 5> OpcTable{ | ||||||
4106 | {{AArch64::ADDSXri, AArch64::ADDSWri}, | ||||||
4107 | {AArch64::ADDSXrs, AArch64::ADDSWrs}, | ||||||
4108 | {AArch64::ADDSXrr, AArch64::ADDSWrr}, | ||||||
4109 | {AArch64::SUBSXri, AArch64::SUBSWri}, | ||||||
4110 | {AArch64::ADDSXrx, AArch64::ADDSWrx}}}; | ||||||
4111 | return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder); | ||||||
4112 | } | ||||||
4113 | |||||||
4114 | MachineInstr * | ||||||
4115 | AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS, | ||||||
4116 | MachineOperand &RHS, | ||||||
4117 | MachineIRBuilder &MIRBuilder) const { | ||||||
4118 | const std::array<std::array<unsigned, 2>, 5> OpcTable{ | ||||||
4119 | {{AArch64::SUBSXri, AArch64::SUBSWri}, | ||||||
4120 | {AArch64::SUBSXrs, AArch64::SUBSWrs}, | ||||||
4121 | {AArch64::SUBSXrr, AArch64::SUBSWrr}, | ||||||
4122 | {AArch64::ADDSXri, AArch64::ADDSWri}, | ||||||
4123 | {AArch64::SUBSXrx, AArch64::SUBSWrx}}}; | ||||||
4124 | return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder); | ||||||
4125 | } | ||||||
4126 | |||||||
4127 | MachineInstr * | ||||||
4128 | AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS, | ||||||
4129 | MachineIRBuilder &MIRBuilder) const { | ||||||
4130 | MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); | ||||||
4131 | bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32); | ||||||
4132 | auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass; | ||||||
4133 | return emitADDS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder); | ||||||
4134 | } | ||||||
4135 | |||||||
4136 | MachineInstr * | ||||||
4137 | AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS, | ||||||
4138 | MachineIRBuilder &MIRBuilder) const { | ||||||
4139 | assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")((LHS.isReg() && RHS.isReg() && "Expected register operands?" ) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4139, __PRETTY_FUNCTION__)); | ||||||
4140 | MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); | ||||||
4141 | LLT Ty = MRI.getType(LHS.getReg()); | ||||||
4142 | unsigned RegSize = Ty.getSizeInBits(); | ||||||
4143 | bool Is32Bit = (RegSize == 32); | ||||||
4144 | const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri}, | ||||||
4145 | {AArch64::ANDSXrs, AArch64::ANDSWrs}, | ||||||
4146 | {AArch64::ANDSXrr, AArch64::ANDSWrr}}; | ||||||
4147 | // ANDS needs a logical immediate for its immediate form. Check if we can | ||||||
4148 | // fold one in. | ||||||
4149 | if (auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI)) { | ||||||
4150 | int64_t Imm = ValAndVReg->Value.getSExtValue(); | ||||||
4151 | |||||||
4152 | if (AArch64_AM::isLogicalImmediate(Imm, RegSize)) { | ||||||
4153 | auto TstMI = MIRBuilder.buildInstr(OpcTable[0][Is32Bit], {Ty}, {LHS}); | ||||||
4154 | TstMI.addImm(AArch64_AM::encodeLogicalImmediate(Imm, RegSize)); | ||||||
4155 | constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI); | ||||||
4156 | return &*TstMI; | ||||||
4157 | } | ||||||
4158 | } | ||||||
4159 | |||||||
4160 | if (auto Fns = selectLogicalShiftedRegister(RHS)) | ||||||
4161 | return emitInstr(OpcTable[1][Is32Bit], {Ty}, {LHS}, MIRBuilder, Fns); | ||||||
4162 | return emitInstr(OpcTable[2][Is32Bit], {Ty}, {LHS, RHS}, MIRBuilder); | ||||||
4163 | } | ||||||
4164 | |||||||
4165 | MachineInstr *AArch64InstructionSelector::emitIntegerCompare( | ||||||
4166 | MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate, | ||||||
4167 | MachineIRBuilder &MIRBuilder) const { | ||||||
4168 | assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!")((LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!" ) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected LHS and RHS to be registers!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4168, __PRETTY_FUNCTION__)); | ||||||
4169 | assert(Predicate.isPredicate() && "Expected predicate?")((Predicate.isPredicate() && "Expected predicate?") ? static_cast<void> (0) : __assert_fail ("Predicate.isPredicate() && \"Expected predicate?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4169, __PRETTY_FUNCTION__)); | ||||||
4170 | MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); | ||||||
4171 | LLT CmpTy = MRI.getType(LHS.getReg()); | ||||||
4172 | assert(!CmpTy.isVector() && "Expected scalar or pointer")((!CmpTy.isVector() && "Expected scalar or pointer") ? static_cast<void> (0) : __assert_fail ("!CmpTy.isVector() && \"Expected scalar or pointer\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4172, __PRETTY_FUNCTION__)); | ||||||
4173 | unsigned Size = CmpTy.getSizeInBits(); | ||||||
4174 | (void)Size; | ||||||
4175 | assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?")(((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?" ) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit LHS/RHS?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4175, __PRETTY_FUNCTION__)); | ||||||
4176 | // Fold the compare into a cmn or tst if possible. | ||||||
4177 | if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder)) | ||||||
4178 | return FoldCmp; | ||||||
4179 | auto Dst = MRI.cloneVirtualRegister(LHS.getReg()); | ||||||
4180 | return emitSUBS(Dst, LHS, RHS, MIRBuilder); | ||||||
4181 | } | ||||||
4182 | |||||||
4183 | MachineInstr *AArch64InstructionSelector::emitCSetForFCmp( | ||||||
4184 | Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const { | ||||||
4185 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); | ||||||
4186 | #ifndef NDEBUG | ||||||
4187 | LLT Ty = MRI.getType(Dst); | ||||||
4188 | assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&((!Ty.isVector() && Ty.getSizeInBits() == 32 && "Expected a 32-bit scalar register?") ? static_cast<void> (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4189, __PRETTY_FUNCTION__)) | ||||||
4189 | "Expected a 32-bit scalar register?")((!Ty.isVector() && Ty.getSizeInBits() == 32 && "Expected a 32-bit scalar register?") ? static_cast<void> (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4189, __PRETTY_FUNCTION__)); | ||||||
4190 | #endif | ||||||
4191 | const Register ZeroReg = AArch64::WZR; | ||||||
4192 | auto EmitCSet = [&](Register CsetDst, AArch64CC::CondCode CC) { | ||||||
4193 | auto CSet = | ||||||
4194 | MIRBuilder.buildInstr(AArch64::CSINCWr, {CsetDst}, {ZeroReg, ZeroReg}) | ||||||
4195 | .addImm(getInvertedCondCode(CC)); | ||||||
4196 | constrainSelectedInstRegOperands(*CSet, TII, TRI, RBI); | ||||||
4197 | return &*CSet; | ||||||
4198 | }; | ||||||
4199 | |||||||
4200 | AArch64CC::CondCode CC1, CC2; | ||||||
4201 | changeFCMPPredToAArch64CC(Pred, CC1, CC2); | ||||||
4202 | if (CC2 == AArch64CC::AL) | ||||||
4203 | return EmitCSet(Dst, CC1); | ||||||
4204 | |||||||
4205 | const TargetRegisterClass *RC = &AArch64::GPR32RegClass; | ||||||
4206 | Register Def1Reg = MRI.createVirtualRegister(RC); | ||||||
4207 | Register Def2Reg = MRI.createVirtualRegister(RC); | ||||||
4208 | EmitCSet(Def1Reg, CC1); | ||||||
4209 | EmitCSet(Def2Reg, CC2); | ||||||
4210 | auto OrMI = MIRBuilder.buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg}); | ||||||
4211 | constrainSelectedInstRegOperands(*OrMI, TII, TRI, RBI); | ||||||
4212 | return &*OrMI; | ||||||
4213 | } | ||||||
4214 | |||||||
4215 | MachineInstr * | ||||||
4216 | AArch64InstructionSelector::emitFPCompare(Register LHS, Register RHS, | ||||||
4217 | MachineIRBuilder &MIRBuilder, | ||||||
4218 | Optional<CmpInst::Predicate> Pred) const { | ||||||
4219 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); | ||||||
4220 | LLT Ty = MRI.getType(LHS); | ||||||
4221 | if (Ty.isVector()) | ||||||
4222 | return nullptr; | ||||||
4223 | unsigned OpSize = Ty.getSizeInBits(); | ||||||
4224 | if (OpSize != 32 && OpSize != 64) | ||||||
4225 | return nullptr; | ||||||
4226 | |||||||
4227 | // If this is a compare against +0.0, then we don't have | ||||||
4228 | // to explicitly materialize a constant. | ||||||
4229 | const ConstantFP *FPImm = getConstantFPVRegVal(RHS, MRI); | ||||||
4230 | bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative()); | ||||||
4231 | |||||||
4232 | auto IsEqualityPred = [](CmpInst::Predicate P) { | ||||||
4233 | return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE || | ||||||
4234 | P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE; | ||||||
4235 | }; | ||||||
4236 | if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) { | ||||||
4237 | // Try commutating the operands. | ||||||
4238 | const ConstantFP *LHSImm = getConstantFPVRegVal(LHS, MRI); | ||||||
4239 | if (LHSImm && (LHSImm->isZero() && !LHSImm->isNegative())) { | ||||||
4240 | ShouldUseImm = true; | ||||||
4241 | std::swap(LHS, RHS); | ||||||
4242 | } | ||||||
4243 | } | ||||||
4244 | unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr}, | ||||||
4245 | {AArch64::FCMPSri, AArch64::FCMPDri}}; | ||||||
4246 | unsigned CmpOpc = CmpOpcTbl[ShouldUseImm][OpSize == 64]; | ||||||
4247 | |||||||
4248 | // Partially build the compare. Decide if we need to add a use for the | ||||||
4249 | // third operand based off whether or not we're comparing against 0.0. | ||||||
4250 | auto CmpMI = MIRBuilder.buildInstr(CmpOpc).addUse(LHS); | ||||||
4251 | if (!ShouldUseImm) | ||||||
4252 | CmpMI.addUse(RHS); | ||||||
4253 | constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI); | ||||||
4254 | return &*CmpMI; | ||||||
4255 | } | ||||||
4256 | |||||||
4257 | MachineInstr *AArch64InstructionSelector::emitVectorConcat( | ||||||
4258 | Optional<Register> Dst, Register Op1, Register Op2, | ||||||
4259 | MachineIRBuilder &MIRBuilder) const { | ||||||
4260 | // We implement a vector concat by: | ||||||
4261 | // 1. Use scalar_to_vector to insert the lower vector into the larger dest | ||||||
4262 | // 2. Insert the upper vector into the destination's upper element | ||||||
4263 | // TODO: some of this code is common with G_BUILD_VECTOR handling. | ||||||
4264 | MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo(); | ||||||
4265 | |||||||
4266 | const LLT Op1Ty = MRI.getType(Op1); | ||||||
4267 | const LLT Op2Ty = MRI.getType(Op2); | ||||||
4268 | |||||||
4269 | if (Op1Ty != Op2Ty) { | ||||||
4270 | LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not do vector concat of differing vector tys" ; } } while (false); | ||||||
4271 | return nullptr; | ||||||
4272 | } | ||||||
4273 | assert(Op1Ty.isVector() && "Expected a vector for vector concat")((Op1Ty.isVector() && "Expected a vector for vector concat" ) ? static_cast<void> (0) : __assert_fail ("Op1Ty.isVector() && \"Expected a vector for vector concat\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4273, __PRETTY_FUNCTION__)); | ||||||
4274 | |||||||
4275 | if (Op1Ty.getSizeInBits() >= 128) { | ||||||
4276 | LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Vector concat not supported for full size vectors" ; } } while (false); | ||||||
4277 | return nullptr; | ||||||
4278 | } | ||||||
4279 | |||||||
4280 | // At the moment we just support 64 bit vector concats. | ||||||
4281 | if (Op1Ty.getSizeInBits() != 64) { | ||||||
4282 | LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Vector concat supported for 64b vectors" ; } } while (false); | ||||||
4283 | return nullptr; | ||||||
4284 | } | ||||||
4285 | |||||||
4286 | const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits()); | ||||||
4287 | const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI); | ||||||
4288 | const TargetRegisterClass *DstRC = | ||||||
4289 | getMinClassForRegBank(FPRBank, Op1Ty.getSizeInBits() * 2); | ||||||
4290 | |||||||
4291 | MachineInstr *WidenedOp1 = | ||||||
4292 | emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder); | ||||||
4293 | MachineInstr *WidenedOp2 = | ||||||
4294 | emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder); | ||||||
4295 | if (!WidenedOp1 || !WidenedOp2) { | ||||||
4296 | LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not emit a vector from scalar value" ; } } while (false); | ||||||
4297 | return nullptr; | ||||||
4298 | } | ||||||
4299 | |||||||
4300 | // Now do the insert of the upper element. | ||||||
4301 | unsigned InsertOpc, InsSubRegIdx; | ||||||
4302 | std::tie(InsertOpc, InsSubRegIdx) = | ||||||
4303 | getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits()); | ||||||
4304 | |||||||
4305 | if (!Dst) | ||||||
4306 | Dst = MRI.createVirtualRegister(DstRC); | ||||||
4307 | auto InsElt = | ||||||
4308 | MIRBuilder | ||||||
4309 | .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()}) | ||||||
4310 | .addImm(1) /* Lane index */ | ||||||
4311 | .addUse(WidenedOp2->getOperand(0).getReg()) | ||||||
4312 | .addImm(0); | ||||||
4313 | constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI); | ||||||
4314 | return &*InsElt; | ||||||
4315 | } | ||||||
4316 | |||||||
4317 | MachineInstr *AArch64InstructionSelector::emitFMovForFConstant( | ||||||
4318 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
4319 | assert(I.getOpcode() == TargetOpcode::G_FCONSTANT &&((I.getOpcode() == TargetOpcode::G_FCONSTANT && "Expected a G_FCONSTANT!" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_FCONSTANT && \"Expected a G_FCONSTANT!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4320, __PRETTY_FUNCTION__)) | ||||||
4320 | "Expected a G_FCONSTANT!")((I.getOpcode() == TargetOpcode::G_FCONSTANT && "Expected a G_FCONSTANT!" ) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_FCONSTANT && \"Expected a G_FCONSTANT!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4320, __PRETTY_FUNCTION__)); | ||||||
4321 | MachineOperand &ImmOp = I.getOperand(1); | ||||||
4322 | unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits(); | ||||||
4323 | |||||||
4324 | // Only handle 32 and 64 bit defs for now. | ||||||
4325 | if (DefSize != 32 && DefSize != 64) | ||||||
4326 | return nullptr; | ||||||
4327 | |||||||
4328 | // Don't handle null values using FMOV. | ||||||
4329 | if (ImmOp.getFPImm()->isNullValue()) | ||||||
4330 | return nullptr; | ||||||
4331 | |||||||
4332 | // Get the immediate representation for the FMOV. | ||||||
4333 | const APFloat &ImmValAPF = ImmOp.getFPImm()->getValueAPF(); | ||||||
4334 | int Imm = DefSize == 32 ? AArch64_AM::getFP32Imm(ImmValAPF) | ||||||
4335 | : AArch64_AM::getFP64Imm(ImmValAPF); | ||||||
4336 | |||||||
4337 | // If this is -1, it means the immediate can't be represented as the requested | ||||||
4338 | // floating point value. Bail. | ||||||
4339 | if (Imm == -1) | ||||||
4340 | return nullptr; | ||||||
4341 | |||||||
4342 | // Update MI to represent the new FMOV instruction, constrain it, and return. | ||||||
4343 | ImmOp.ChangeToImmediate(Imm); | ||||||
4344 | unsigned MovOpc = DefSize == 32 ? AArch64::FMOVSi : AArch64::FMOVDi; | ||||||
4345 | I.setDesc(TII.get(MovOpc)); | ||||||
4346 | constrainSelectedInstRegOperands(I, TII, TRI, RBI); | ||||||
4347 | return &I; | ||||||
4348 | } | ||||||
4349 | |||||||
4350 | MachineInstr * | ||||||
4351 | AArch64InstructionSelector::emitCSetForICMP(Register DefReg, unsigned Pred, | ||||||
4352 | MachineIRBuilder &MIRBuilder) const { | ||||||
4353 | // CSINC increments the result when the predicate is false. Invert it. | ||||||
4354 | const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC( | ||||||
4355 | CmpInst::getInversePredicate((CmpInst::Predicate)Pred)); | ||||||
4356 | auto I = | ||||||
4357 | MIRBuilder | ||||||
4358 | .buildInstr(AArch64::CSINCWr, {DefReg}, {Register(AArch64::WZR), Register(AArch64::WZR)}) | ||||||
4359 | .addImm(InvCC); | ||||||
4360 | constrainSelectedInstRegOperands(*I, TII, TRI, RBI); | ||||||
4361 | return &*I; | ||||||
4362 | } | ||||||
4363 | |||||||
4364 | std::pair<MachineInstr *, AArch64CC::CondCode> | ||||||
4365 | AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst, | ||||||
4366 | MachineOperand &LHS, | ||||||
4367 | MachineOperand &RHS, | ||||||
4368 | MachineIRBuilder &MIRBuilder) const { | ||||||
4369 | switch (Opcode) { | ||||||
4370 | default: | ||||||
4371 | llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4371); | ||||||
4372 | case TargetOpcode::G_SADDO: | ||||||
4373 | return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS); | ||||||
4374 | case TargetOpcode::G_UADDO: | ||||||
4375 | return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS); | ||||||
4376 | case TargetOpcode::G_SSUBO: | ||||||
4377 | return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS); | ||||||
4378 | } | ||||||
4379 | } | ||||||
4380 | |||||||
4381 | bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) const { | ||||||
4382 | MachineIRBuilder MIB(I); | ||||||
4383 | MachineRegisterInfo &MRI = *MIB.getMRI(); | ||||||
4384 | // We want to recognize this pattern: | ||||||
4385 | // | ||||||
4386 | // $z = G_FCMP pred, $x, $y | ||||||
4387 | // ... | ||||||
4388 | // $w = G_SELECT $z, $a, $b | ||||||
4389 | // | ||||||
4390 | // Where the value of $z is *only* ever used by the G_SELECT (possibly with | ||||||
4391 | // some copies/truncs in between.) | ||||||
4392 | // | ||||||
4393 | // If we see this, then we can emit something like this: | ||||||
4394 | // | ||||||
4395 | // fcmp $x, $y | ||||||
4396 | // fcsel $w, $a, $b, pred | ||||||
4397 | // | ||||||
4398 | // Rather than emitting both of the rather long sequences in the standard | ||||||
4399 | // G_FCMP/G_SELECT select methods. | ||||||
4400 | |||||||
4401 | // First, check if the condition is defined by a compare. | ||||||
4402 | MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg()); | ||||||
4403 | while (CondDef) { | ||||||
4404 | // We can only fold if all of the defs have one use. | ||||||
4405 | Register CondDefReg = CondDef->getOperand(0).getReg(); | ||||||
4406 | if (!MRI.hasOneNonDBGUse(CondDefReg)) { | ||||||
4407 | // Unless it's another select. | ||||||
4408 | for (const MachineInstr &UI : MRI.use_nodbg_instructions(CondDefReg)) { | ||||||
4409 | if (CondDef == &UI) | ||||||
4410 | continue; | ||||||
4411 | if (UI.getOpcode() != TargetOpcode::G_SELECT) | ||||||
4412 | return false; | ||||||
4413 | } | ||||||
4414 | } | ||||||
4415 | |||||||
4416 | // We can skip over G_TRUNC since the condition is 1-bit. | ||||||
4417 | // Truncating/extending can have no impact on the value. | ||||||
4418 | unsigned Opc = CondDef->getOpcode(); | ||||||
4419 | if (Opc != TargetOpcode::COPY && Opc != TargetOpcode::G_TRUNC) | ||||||
4420 | break; | ||||||
4421 | |||||||
4422 | // Can't see past copies from physregs. | ||||||
4423 | if (Opc == TargetOpcode::COPY && | ||||||
4424 | Register::isPhysicalRegister(CondDef->getOperand(1).getReg())) | ||||||
4425 | return false; | ||||||
4426 | |||||||
4427 | CondDef = MRI.getVRegDef(CondDef->getOperand(1).getReg()); | ||||||
4428 | } | ||||||
4429 | |||||||
4430 | // Is the condition defined by a compare? | ||||||
4431 | if (!CondDef) | ||||||
4432 | return false; | ||||||
4433 | |||||||
4434 | unsigned CondOpc = CondDef->getOpcode(); | ||||||
4435 | if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) | ||||||
4436 | return false; | ||||||
4437 | |||||||
4438 | AArch64CC::CondCode CondCode; | ||||||
4439 | if (CondOpc == TargetOpcode::G_ICMP) { | ||||||
4440 | auto Pred = | ||||||
4441 | static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate()); | ||||||
4442 | CondCode = changeICMPPredToAArch64CC(Pred); | ||||||
4443 | emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3), | ||||||
4444 | CondDef->getOperand(1), MIB); | ||||||
4445 | } else { | ||||||
4446 | // Get the condition code for the select. | ||||||
4447 | auto Pred = | ||||||
4448 | static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate()); | ||||||
4449 | AArch64CC::CondCode CondCode2; | ||||||
4450 | changeFCMPPredToAArch64CC(Pred, CondCode, CondCode2); | ||||||
4451 | |||||||
4452 | // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two | ||||||
4453 | // instructions to emit the comparison. | ||||||
4454 | // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be | ||||||
4455 | // unnecessary. | ||||||
4456 | if (CondCode2 != AArch64CC::AL) | ||||||
4457 | return false; | ||||||
4458 | |||||||
4459 | if (!emitFPCompare(CondDef->getOperand(2).getReg(), | ||||||
4460 | CondDef->getOperand(3).getReg(), MIB)) { | ||||||
4461 | LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Couldn't emit compare for select!\n" ; } } while (false); | ||||||
4462 | return false; | ||||||
4463 | } | ||||||
4464 | } | ||||||
4465 | |||||||
4466 | // Emit the select. | ||||||
4467 | emitSelect(I.getOperand(0).getReg(), I.getOperand(2).getReg(), | ||||||
4468 | I.getOperand(3).getReg(), CondCode, MIB); | ||||||
4469 | I.eraseFromParent(); | ||||||
4470 | return true; | ||||||
4471 | } | ||||||
4472 | |||||||
4473 | MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare( | ||||||
4474 | MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate, | ||||||
4475 | MachineIRBuilder &MIRBuilder) const { | ||||||
4476 | assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&((LHS.isReg() && RHS.isReg() && Predicate.isPredicate () && "Unexpected MachineOperand") ? static_cast<void > (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4477, __PRETTY_FUNCTION__)) | ||||||
4477 | "Unexpected MachineOperand")((LHS.isReg() && RHS.isReg() && Predicate.isPredicate () && "Unexpected MachineOperand") ? static_cast<void > (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4477, __PRETTY_FUNCTION__)); | ||||||
4478 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); | ||||||
4479 | // We want to find this sort of thing: | ||||||
4480 | // x = G_SUB 0, y | ||||||
4481 | // G_ICMP z, x | ||||||
4482 | // | ||||||
4483 | // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead. | ||||||
4484 | // e.g: | ||||||
4485 | // | ||||||
4486 | // cmn z, y | ||||||
4487 | |||||||
4488 | // Helper lambda to detect the subtract followed by the compare. | ||||||
4489 | // Takes in the def of the LHS or RHS, and checks if it's a subtract from 0. | ||||||
4490 | auto IsCMN = [&](MachineInstr *DefMI, const AArch64CC::CondCode &CC) { | ||||||
4491 | if (!DefMI || DefMI->getOpcode() != TargetOpcode::G_SUB) | ||||||
4492 | return false; | ||||||
4493 | |||||||
4494 | // Need to make sure NZCV is the same at the end of the transformation. | ||||||
4495 | if (CC != AArch64CC::EQ && CC != AArch64CC::NE) | ||||||
4496 | return false; | ||||||
4497 | |||||||
4498 | // We want to match against SUBs. | ||||||
4499 | if (DefMI->getOpcode() != TargetOpcode::G_SUB) | ||||||
4500 | return false; | ||||||
4501 | |||||||
4502 | // Make sure that we're getting | ||||||
4503 | // x = G_SUB 0, y | ||||||
4504 | auto ValAndVReg = | ||||||
4505 | getConstantVRegValWithLookThrough(DefMI->getOperand(1).getReg(), MRI); | ||||||
4506 | if (!ValAndVReg || ValAndVReg->Value != 0) | ||||||
4507 | return false; | ||||||
4508 | |||||||
4509 | // This can safely be represented as a CMN. | ||||||
4510 | return true; | ||||||
4511 | }; | ||||||
4512 | |||||||
4513 | // Check if the RHS or LHS of the G_ICMP is defined by a SUB | ||||||
4514 | MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI); | ||||||
4515 | MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI); | ||||||
4516 | CmpInst::Predicate P = (CmpInst::Predicate)Predicate.getPredicate(); | ||||||
4517 | const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(P); | ||||||
4518 | |||||||
4519 | // Given this: | ||||||
4520 | // | ||||||
4521 | // x = G_SUB 0, y | ||||||
4522 | // G_ICMP x, z | ||||||
4523 | // | ||||||
4524 | // Produce this: | ||||||
4525 | // | ||||||
4526 | // cmn y, z | ||||||
4527 | if (IsCMN(LHSDef, CC)) | ||||||
4528 | return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder); | ||||||
4529 | |||||||
4530 | // Same idea here, but with the RHS of the compare instead: | ||||||
4531 | // | ||||||
4532 | // Given this: | ||||||
4533 | // | ||||||
4534 | // x = G_SUB 0, y | ||||||
4535 | // G_ICMP z, x | ||||||
4536 | // | ||||||
4537 | // Produce this: | ||||||
4538 | // | ||||||
4539 | // cmn z, y | ||||||
4540 | if (IsCMN(RHSDef, CC)) | ||||||
4541 | return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder); | ||||||
4542 | |||||||
4543 | // Given this: | ||||||
4544 | // | ||||||
4545 | // z = G_AND x, y | ||||||
4546 | // G_ICMP z, 0 | ||||||
4547 | // | ||||||
4548 | // Produce this if the compare is signed: | ||||||
4549 | // | ||||||
4550 | // tst x, y | ||||||
4551 | if (!CmpInst::isUnsigned(P) && LHSDef && | ||||||
4552 | LHSDef->getOpcode() == TargetOpcode::G_AND) { | ||||||
4553 | // Make sure that the RHS is 0. | ||||||
4554 | auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI); | ||||||
4555 | if (!ValAndVReg || ValAndVReg->Value != 0) | ||||||
4556 | return nullptr; | ||||||
4557 | |||||||
4558 | return emitTST(LHSDef->getOperand(1), | ||||||
4559 | LHSDef->getOperand(2), MIRBuilder); | ||||||
4560 | } | ||||||
4561 | |||||||
4562 | return nullptr; | ||||||
4563 | } | ||||||
4564 | |||||||
4565 | bool AArch64InstructionSelector::selectShuffleVector( | ||||||
4566 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
4567 | const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
4568 | Register Src1Reg = I.getOperand(1).getReg(); | ||||||
4569 | const LLT Src1Ty = MRI.getType(Src1Reg); | ||||||
4570 | Register Src2Reg = I.getOperand(2).getReg(); | ||||||
4571 | const LLT Src2Ty = MRI.getType(Src2Reg); | ||||||
4572 | ArrayRef<int> Mask = I.getOperand(3).getShuffleMask(); | ||||||
4573 | |||||||
4574 | MachineBasicBlock &MBB = *I.getParent(); | ||||||
4575 | MachineFunction &MF = *MBB.getParent(); | ||||||
4576 | LLVMContext &Ctx = MF.getFunction().getContext(); | ||||||
4577 | |||||||
4578 | // G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if | ||||||
4579 | // it's originated from a <1 x T> type. Those should have been lowered into | ||||||
4580 | // G_BUILD_VECTOR earlier. | ||||||
4581 | if (!Src1Ty.isVector() || !Src2Ty.isVector()) { | ||||||
4582 | LLVM_DEBUG(dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n" ; } } while (false); | ||||||
4583 | return false; | ||||||
4584 | } | ||||||
4585 | |||||||
4586 | unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8; | ||||||
4587 | |||||||
4588 | SmallVector<Constant *, 64> CstIdxs; | ||||||
4589 | for (int Val : Mask) { | ||||||
4590 | // For now, any undef indexes we'll just assume to be 0. This should be | ||||||
4591 | // optimized in future, e.g. to select DUP etc. | ||||||
4592 | Val = Val < 0 ? 0 : Val; | ||||||
4593 | for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) { | ||||||
4594 | unsigned Offset = Byte + Val * BytesPerElt; | ||||||
4595 | CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset)); | ||||||
4596 | } | ||||||
4597 | } | ||||||
4598 | |||||||
4599 | MachineIRBuilder MIRBuilder(I); | ||||||
4600 | |||||||
4601 | // Use a constant pool to load the index vector for TBL. | ||||||
4602 | Constant *CPVal = ConstantVector::get(CstIdxs); | ||||||
4603 | MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder); | ||||||
4604 | if (!IndexLoad) { | ||||||
4605 | LLVM_DEBUG(dbgs() << "Could not load from a constant pool")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not load from a constant pool" ; } } while (false); | ||||||
4606 | return false; | ||||||
4607 | } | ||||||
4608 | |||||||
4609 | if (DstTy.getSizeInBits() != 128) { | ||||||
4610 | assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty")((DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty" ) ? static_cast<void> (0) : __assert_fail ("DstTy.getSizeInBits() == 64 && \"Unexpected shuffle result ty\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4610, __PRETTY_FUNCTION__)); | ||||||
4611 | // This case can be done with TBL1. | ||||||
4612 | MachineInstr *Concat = emitVectorConcat(None, Src1Reg, Src2Reg, MIRBuilder); | ||||||
4613 | if (!Concat) { | ||||||
4614 | LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not do vector concat for tbl1" ; } } while (false); | ||||||
4615 | return false; | ||||||
4616 | } | ||||||
4617 | |||||||
4618 | // The constant pool load will be 64 bits, so need to convert to FPR128 reg. | ||||||
4619 | IndexLoad = | ||||||
4620 | emitScalarToVector(64, &AArch64::FPR128RegClass, | ||||||
4621 | IndexLoad->getOperand(0).getReg(), MIRBuilder); | ||||||
4622 | |||||||
4623 | auto TBL1 = MIRBuilder.buildInstr( | ||||||
4624 | AArch64::TBLv16i8One, {&AArch64::FPR128RegClass}, | ||||||
4625 | {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()}); | ||||||
4626 | constrainSelectedInstRegOperands(*TBL1, TII, TRI, RBI); | ||||||
4627 | |||||||
4628 | auto Copy = | ||||||
4629 | MIRBuilder | ||||||
4630 | .buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {}) | ||||||
4631 | .addReg(TBL1.getReg(0), 0, AArch64::dsub); | ||||||
4632 | RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI); | ||||||
4633 | I.eraseFromParent(); | ||||||
4634 | return true; | ||||||
4635 | } | ||||||
4636 | |||||||
4637 | // For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive | ||||||
4638 | // Q registers for regalloc. | ||||||
4639 | auto RegSeq = MIRBuilder | ||||||
4640 | .buildInstr(TargetOpcode::REG_SEQUENCE, | ||||||
4641 | {&AArch64::QQRegClass}, {Src1Reg}) | ||||||
4642 | .addImm(AArch64::qsub0) | ||||||
4643 | .addUse(Src2Reg) | ||||||
4644 | .addImm(AArch64::qsub1); | ||||||
4645 | |||||||
4646 | auto TBL2 = MIRBuilder.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0)}, | ||||||
4647 | {RegSeq, IndexLoad->getOperand(0)}); | ||||||
4648 | constrainSelectedInstRegOperands(*RegSeq, TII, TRI, RBI); | ||||||
4649 | constrainSelectedInstRegOperands(*TBL2, TII, TRI, RBI); | ||||||
4650 | I.eraseFromParent(); | ||||||
4651 | return true; | ||||||
4652 | } | ||||||
4653 | |||||||
4654 | MachineInstr *AArch64InstructionSelector::emitLaneInsert( | ||||||
4655 | Optional<Register> DstReg, Register SrcReg, Register EltReg, | ||||||
4656 | unsigned LaneIdx, const RegisterBank &RB, | ||||||
4657 | MachineIRBuilder &MIRBuilder) const { | ||||||
4658 | MachineInstr *InsElt = nullptr; | ||||||
4659 | const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass; | ||||||
4660 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); | ||||||
4661 | |||||||
4662 | // Create a register to define with the insert if one wasn't passed in. | ||||||
4663 | if (!DstReg) | ||||||
4664 | DstReg = MRI.createVirtualRegister(DstRC); | ||||||
4665 | |||||||
4666 | unsigned EltSize = MRI.getType(EltReg).getSizeInBits(); | ||||||
4667 | unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first; | ||||||
4668 | |||||||
4669 | if (RB.getID() == AArch64::FPRRegBankID) { | ||||||
4670 | auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder); | ||||||
4671 | InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg}) | ||||||
4672 | .addImm(LaneIdx) | ||||||
4673 | .addUse(InsSub->getOperand(0).getReg()) | ||||||
4674 | .addImm(0); | ||||||
4675 | } else { | ||||||
4676 | InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg}) | ||||||
4677 | .addImm(LaneIdx) | ||||||
4678 | .addUse(EltReg); | ||||||
4679 | } | ||||||
4680 | |||||||
4681 | constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI); | ||||||
4682 | return InsElt; | ||||||
4683 | } | ||||||
4684 | |||||||
4685 | bool AArch64InstructionSelector::selectInsertElt( | ||||||
4686 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
4687 | assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)((I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT) ? static_cast <void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4687, __PRETTY_FUNCTION__)); | ||||||
4688 | |||||||
4689 | // Get information on the destination. | ||||||
4690 | Register DstReg = I.getOperand(0).getReg(); | ||||||
4691 | const LLT DstTy = MRI.getType(DstReg); | ||||||
4692 | unsigned VecSize = DstTy.getSizeInBits(); | ||||||
4693 | |||||||
4694 | // Get information on the element we want to insert into the destination. | ||||||
4695 | Register EltReg = I.getOperand(2).getReg(); | ||||||
4696 | const LLT EltTy = MRI.getType(EltReg); | ||||||
4697 | unsigned EltSize = EltTy.getSizeInBits(); | ||||||
4698 | if (EltSize < 16 || EltSize > 64) | ||||||
4699 | return false; // Don't support all element types yet. | ||||||
4700 | |||||||
4701 | // Find the definition of the index. Bail out if it's not defined by a | ||||||
4702 | // G_CONSTANT. | ||||||
4703 | Register IdxReg = I.getOperand(3).getReg(); | ||||||
4704 | auto VRegAndVal = getConstantVRegValWithLookThrough(IdxReg, MRI); | ||||||
4705 | if (!VRegAndVal) | ||||||
4706 | return false; | ||||||
4707 | unsigned LaneIdx = VRegAndVal->Value.getSExtValue(); | ||||||
4708 | |||||||
4709 | // Perform the lane insert. | ||||||
4710 | Register SrcReg = I.getOperand(1).getReg(); | ||||||
4711 | const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI); | ||||||
4712 | MachineIRBuilder MIRBuilder(I); | ||||||
4713 | |||||||
4714 | if (VecSize < 128) { | ||||||
4715 | // If the vector we're inserting into is smaller than 128 bits, widen it | ||||||
4716 | // to 128 to do the insert. | ||||||
4717 | MachineInstr *ScalarToVec = emitScalarToVector( | ||||||
4718 | VecSize, &AArch64::FPR128RegClass, SrcReg, MIRBuilder); | ||||||
4719 | if (!ScalarToVec) | ||||||
4720 | return false; | ||||||
4721 | SrcReg = ScalarToVec->getOperand(0).getReg(); | ||||||
4722 | } | ||||||
4723 | |||||||
4724 | // Create an insert into a new FPR128 register. | ||||||
4725 | // Note that if our vector is already 128 bits, we end up emitting an extra | ||||||
4726 | // register. | ||||||
4727 | MachineInstr *InsMI = | ||||||
4728 | emitLaneInsert(None, SrcReg, EltReg, LaneIdx, EltRB, MIRBuilder); | ||||||
4729 | |||||||
4730 | if (VecSize < 128) { | ||||||
4731 | // If we had to widen to perform the insert, then we have to demote back to | ||||||
4732 | // the original size to get the result we want. | ||||||
4733 | Register DemoteVec = InsMI->getOperand(0).getReg(); | ||||||
4734 | const TargetRegisterClass *RC = | ||||||
4735 | getMinClassForRegBank(*RBI.getRegBank(DemoteVec, MRI, TRI), VecSize); | ||||||
4736 | if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) { | ||||||
4737 | LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported register class!\n" ; } } while (false); | ||||||
4738 | return false; | ||||||
4739 | } | ||||||
4740 | unsigned SubReg = 0; | ||||||
4741 | if (!getSubRegForClass(RC, TRI, SubReg)) | ||||||
4742 | return false; | ||||||
4743 | if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) { | ||||||
4744 | LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << VecSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported destination size! (" << VecSize << "\n"; } } while (false) | ||||||
4745 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported destination size! (" << VecSize << "\n"; } } while (false); | ||||||
4746 | return false; | ||||||
4747 | } | ||||||
4748 | MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {}) | ||||||
4749 | .addReg(DemoteVec, 0, SubReg); | ||||||
4750 | RBI.constrainGenericRegister(DstReg, *RC, MRI); | ||||||
4751 | } else { | ||||||
4752 | // No widening needed. | ||||||
4753 | InsMI->getOperand(0).setReg(DstReg); | ||||||
4754 | constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI); | ||||||
4755 | } | ||||||
4756 | |||||||
4757 | I.eraseFromParent(); | ||||||
4758 | return true; | ||||||
4759 | } | ||||||
4760 | |||||||
4761 | bool AArch64InstructionSelector::tryOptConstantBuildVec( | ||||||
4762 | MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) const { | ||||||
4763 | assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast <void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4763, __PRETTY_FUNCTION__)); | ||||||
4764 | unsigned DstSize = DstTy.getSizeInBits(); | ||||||
4765 | assert(DstSize <= 128 && "Unexpected build_vec type!")((DstSize <= 128 && "Unexpected build_vec type!") ? static_cast<void> (0) : __assert_fail ("DstSize <= 128 && \"Unexpected build_vec type!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4765, __PRETTY_FUNCTION__)); | ||||||
4766 | if (DstSize < 32) | ||||||
4767 | return false; | ||||||
4768 | // Check if we're building a constant vector, in which case we want to | ||||||
4769 | // generate a constant pool load instead of a vector insert sequence. | ||||||
4770 | SmallVector<Constant *, 16> Csts; | ||||||
4771 | for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) { | ||||||
4772 | // Try to find G_CONSTANT or G_FCONSTANT | ||||||
4773 | auto *OpMI = | ||||||
4774 | getOpcodeDef(TargetOpcode::G_CONSTANT, I.getOperand(Idx).getReg(), MRI); | ||||||
4775 | if (OpMI) | ||||||
4776 | Csts.emplace_back( | ||||||
4777 | const_cast<ConstantInt *>(OpMI->getOperand(1).getCImm())); | ||||||
4778 | else if ((OpMI = getOpcodeDef(TargetOpcode::G_FCONSTANT, | ||||||
4779 | I.getOperand(Idx).getReg(), MRI))) | ||||||
4780 | Csts.emplace_back( | ||||||
4781 | const_cast<ConstantFP *>(OpMI->getOperand(1).getFPImm())); | ||||||
4782 | else | ||||||
4783 | return false; | ||||||
4784 | } | ||||||
4785 | Constant *CV = ConstantVector::get(Csts); | ||||||
4786 | MachineIRBuilder MIB(I); | ||||||
4787 | if (CV->isNullValue()) { | ||||||
4788 | // Until the importer can support immAllZerosV in pattern leaf nodes, | ||||||
4789 | // select a zero move manually here. | ||||||
4790 | Register DstReg = I.getOperand(0).getReg(); | ||||||
4791 | if (DstSize == 128) { | ||||||
4792 | auto Mov = MIB.buildInstr(AArch64::MOVIv2d_ns, {DstReg}, {}).addImm(0); | ||||||
4793 | I.eraseFromParent(); | ||||||
4794 | return constrainSelectedInstRegOperands(*Mov, TII, TRI, RBI); | ||||||
4795 | } else if (DstSize == 64) { | ||||||
4796 | auto Mov = | ||||||
4797 | MIB.buildInstr(AArch64::MOVIv2d_ns, {&AArch64::FPR128RegClass}, {}) | ||||||
4798 | .addImm(0); | ||||||
4799 | MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {}) | ||||||
4800 | .addReg(Mov.getReg(0), 0, AArch64::dsub); | ||||||
4801 | I.eraseFromParent(); | ||||||
4802 | return RBI.constrainGenericRegister(DstReg, AArch64::FPR64RegClass, MRI); | ||||||
4803 | } | ||||||
4804 | } | ||||||
4805 | auto *CPLoad = emitLoadFromConstantPool(CV, MIB); | ||||||
4806 | if (!CPLoad) { | ||||||
4807 | LLVM_DEBUG(dbgs() << "Could not generate cp load for build_vector")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Could not generate cp load for build_vector" ; } } while (false); | ||||||
4808 | return false; | ||||||
4809 | } | ||||||
4810 | MIB.buildCopy(I.getOperand(0), CPLoad->getOperand(0)); | ||||||
4811 | RBI.constrainGenericRegister(I.getOperand(0).getReg(), | ||||||
4812 | *MRI.getRegClass(CPLoad->getOperand(0).getReg()), | ||||||
4813 | MRI); | ||||||
4814 | I.eraseFromParent(); | ||||||
4815 | return true; | ||||||
4816 | } | ||||||
4817 | |||||||
4818 | bool AArch64InstructionSelector::selectBuildVector( | ||||||
4819 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
4820 | assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast <void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4820, __PRETTY_FUNCTION__)); | ||||||
4821 | // Until we port more of the optimized selections, for now just use a vector | ||||||
4822 | // insert sequence. | ||||||
4823 | const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); | ||||||
4824 | const LLT EltTy = MRI.getType(I.getOperand(1).getReg()); | ||||||
4825 | unsigned EltSize = EltTy.getSizeInBits(); | ||||||
4826 | |||||||
4827 | if (tryOptConstantBuildVec(I, DstTy, MRI)) | ||||||
4828 | return true; | ||||||
4829 | if (EltSize < 16 || EltSize > 64) | ||||||
4830 | return false; // Don't support all element types yet. | ||||||
4831 | const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI); | ||||||
4832 | MachineIRBuilder MIRBuilder(I); | ||||||
4833 | |||||||
4834 | const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass; | ||||||
4835 | MachineInstr *ScalarToVec = | ||||||
4836 | emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC, | ||||||
4837 | I.getOperand(1).getReg(), MIRBuilder); | ||||||
4838 | if (!ScalarToVec) | ||||||
4839 | return false; | ||||||
4840 | |||||||
4841 | Register DstVec = ScalarToVec->getOperand(0).getReg(); | ||||||
4842 | unsigned DstSize = DstTy.getSizeInBits(); | ||||||
4843 | |||||||
4844 | // Keep track of the last MI we inserted. Later on, we might be able to save | ||||||
4845 | // a copy using it. | ||||||
4846 | MachineInstr *PrevMI = nullptr; | ||||||
4847 | for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) { | ||||||
4848 | // Note that if we don't do a subregister copy, we can end up making an | ||||||
4849 | // extra register. | ||||||
4850 | PrevMI = &*emitLaneInsert(None, DstVec, I.getOperand(i).getReg(), i - 1, RB, | ||||||
4851 | MIRBuilder); | ||||||
4852 | DstVec = PrevMI->getOperand(0).getReg(); | ||||||
4853 | } | ||||||
4854 | |||||||
4855 | // If DstTy's size in bits is less than 128, then emit a subregister copy | ||||||
4856 | // from DstVec to the last register we've defined. | ||||||
4857 | if (DstSize < 128) { | ||||||
4858 | // Force this to be FPR using the destination vector. | ||||||
4859 | const TargetRegisterClass *RC = | ||||||
4860 | getMinClassForRegBank(*RBI.getRegBank(DstVec, MRI, TRI), DstSize); | ||||||
4861 | if (!RC) | ||||||
4862 | return false; | ||||||
4863 | if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) { | ||||||
4864 | LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported register class!\n" ; } } while (false); | ||||||
4865 | return false; | ||||||
4866 | } | ||||||
4867 | |||||||
4868 | unsigned SubReg = 0; | ||||||
4869 | if (!getSubRegForClass(RC, TRI, SubReg)) | ||||||
4870 | return false; | ||||||
4871 | if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) { | ||||||
4872 | LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported destination size! (" << DstSize << "\n"; } } while (false) | ||||||
4873 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("aarch64-isel")) { dbgs() << "Unsupported destination size! (" << DstSize << "\n"; } } while (false); | ||||||
4874 | return false; | ||||||
4875 | } | ||||||
4876 | |||||||
4877 | Register Reg = MRI.createVirtualRegister(RC); | ||||||
4878 | Register DstReg = I.getOperand(0).getReg(); | ||||||
4879 | |||||||
4880 | MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {}) | ||||||
4881 | .addReg(DstVec, 0, SubReg); | ||||||
4882 | MachineOperand &RegOp = I.getOperand(1); | ||||||
4883 | RegOp.setReg(Reg); | ||||||
4884 | RBI.constrainGenericRegister(DstReg, *RC, MRI); | ||||||
4885 | } else { | ||||||
4886 | // We don't need a subregister copy. Save a copy by re-using the | ||||||
4887 | // destination register on the final insert. | ||||||
4888 | assert(PrevMI && "PrevMI was null?")((PrevMI && "PrevMI was null?") ? static_cast<void > (0) : __assert_fail ("PrevMI && \"PrevMI was null?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 4888, __PRETTY_FUNCTION__)); | ||||||
4889 | PrevMI->getOperand(0).setReg(I.getOperand(0).getReg()); | ||||||
4890 | constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI); | ||||||
4891 | } | ||||||
4892 | |||||||
4893 | I.eraseFromParent(); | ||||||
4894 | return true; | ||||||
4895 | } | ||||||
4896 | |||||||
4897 | /// Helper function to find an intrinsic ID on an a MachineInstr. Returns the | ||||||
4898 | /// ID if it exists, and 0 otherwise. | ||||||
4899 | static unsigned findIntrinsicID(MachineInstr &I) { | ||||||
4900 | auto IntrinOp = find_if(I.operands(), [&](const MachineOperand &Op) { | ||||||
4901 | return Op.isIntrinsicID(); | ||||||
4902 | }); | ||||||
4903 | if (IntrinOp == I.operands_end()) | ||||||
4904 | return 0; | ||||||
4905 | return IntrinOp->getIntrinsicID(); | ||||||
4906 | } | ||||||
4907 | |||||||
4908 | bool AArch64InstructionSelector::selectIntrinsicWithSideEffects( | ||||||
4909 | MachineInstr &I, MachineRegisterInfo &MRI) const { | ||||||
4910 | // Find the intrinsic ID. | ||||||
4911 | unsigned IntrinID = findIntrinsicID(I); | ||||||
4912 | if (!IntrinID) | ||||||
4913 | return false; | ||||||
4914 | MachineIRBuilder MIRBuilder(I); | ||||||
4915 | |||||||
4916 | // Select the instruction. | ||||||
4917 | switch (IntrinID) { | ||||||
4918 | default: | ||||||
4919 | return false; | ||||||
4920 | case Intrinsic::trap: | ||||||
4921 | MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(1); | ||||||
4922 | break; | ||||||
4923 | case Intrinsic::debugtrap: | ||||||
4924 | MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000); | ||||||
4925 | break; | ||||||
4926 | case Intrinsic::ubsantrap: | ||||||
4927 | MIRBuilder.buildInstr(AArch64::BRK, {}, {}) | ||||||
4928 | .addImm(I.getOperand(1).getImm() | ('U' << 8)); | ||||||
4929 | break; | ||||||
4930 | } | ||||||
4931 | |||||||
4932 | I.eraseFromParent(); | ||||||
4933 | return true; | ||||||
4934 | } | ||||||
4935 | |||||||
4936 | bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I, | ||||||
4937 | MachineRegisterInfo &MRI) { | ||||||
4938 | unsigned IntrinID = findIntrinsicID(I); | ||||||
4939 | if (!IntrinID) | ||||||
4940 | return false; | ||||||
4941 | MachineIRBuilder MIRBuilder(I); | ||||||
4942 | |||||||
4943 | switch (IntrinID) { | ||||||
4944 | default: | ||||||
4945 | break; | ||||||
4946 | case Intrinsic::aarch64_crypto_sha1h: { | ||||||
4947 | Register DstReg = I.getOperand(0).getReg(); | ||||||
4948 | Register SrcReg = I.getOperand(2).getReg(); | ||||||
4949 | |||||||
4950 | // FIXME: Should this be an assert? | ||||||
4951 | if (MRI.getType(DstReg).getSizeInBits() != 32 || | ||||||
4952 | MRI.getType(SrcReg).getSizeInBits() != 32) | ||||||
4953 | return false; | ||||||
4954 | |||||||
4955 | // The operation has to happen on FPRs. Set up some new FPR registers for | ||||||
4956 | // the source and destination if they are on GPRs. | ||||||
4957 | if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) { | ||||||
4958 | SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); | ||||||
4959 | MIRBuilder.buildCopy({SrcReg}, {I.getOperand(2)}); | ||||||
4960 | |||||||
4961 | // Make sure the copy ends up getting constrained properly. | ||||||
4962 | RBI.constrainGenericRegister(I.getOperand(2).getReg(), | ||||||
4963 | AArch64::GPR32RegClass, MRI); | ||||||
4964 | } | ||||||
4965 | |||||||
4966 | if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) | ||||||
4967 | DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); | ||||||
4968 | |||||||
4969 | // Actually insert the instruction. | ||||||
4970 | auto SHA1Inst = MIRBuilder.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg}); | ||||||
4971 | constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI); | ||||||
4972 | |||||||
4973 | // Did we create a new register for the destination? | ||||||
4974 | if (DstReg != I.getOperand(0).getReg()) { | ||||||
4975 | // Yep. Copy the result of the instruction back into the original | ||||||
4976 | // destination. | ||||||
4977 | MIRBuilder.buildCopy({I.getOperand(0)}, {DstReg}); | ||||||
4978 | RBI.constrainGenericRegister(I.getOperand(0).getReg(), | ||||||
4979 | AArch64::GPR32RegClass, MRI); | ||||||
4980 | } | ||||||
4981 | |||||||
4982 | I.eraseFromParent(); | ||||||
4983 | return true; | ||||||
4984 | } | ||||||
4985 | case Intrinsic::frameaddress: | ||||||
4986 | case Intrinsic::returnaddress: { | ||||||
4987 | MachineFunction &MF = *I.getParent()->getParent(); | ||||||
4988 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||||
4989 | |||||||
4990 | unsigned Depth = I.getOperand(2).getImm(); | ||||||
4991 | Register DstReg = I.getOperand(0).getReg(); | ||||||
4992 | RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI); | ||||||
4993 | |||||||
4994 | if (Depth == 0 && IntrinID == Intrinsic::returnaddress) { | ||||||
4995 | if (!MFReturnAddr) { | ||||||
4996 | // Insert the copy from LR/X30 into the entry block, before it can be | ||||||
4997 | // clobbered by anything. | ||||||
4998 | MFI.setReturnAddressIsTaken(true); | ||||||
4999 | MFReturnAddr = getFunctionLiveInPhysReg(MF, TII, AArch64::LR, | ||||||
5000 | AArch64::GPR64RegClass); | ||||||
5001 | } | ||||||
5002 | |||||||
5003 | if (STI.hasPAuth()) { | ||||||
5004 | MIRBuilder.buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr}); | ||||||
5005 | } else { | ||||||
5006 | MIRBuilder.buildCopy({Register(AArch64::LR)}, {MFReturnAddr}); | ||||||
5007 | MIRBuilder.buildInstr(AArch64::XPACLRI); | ||||||
5008 | MIRBuilder.buildCopy({DstReg}, {Register(AArch64::LR)}); | ||||||
5009 | } | ||||||
5010 | |||||||
5011 | I.eraseFromParent(); | ||||||
5012 | return true; | ||||||
5013 | } | ||||||
5014 | |||||||
5015 | MFI.setFrameAddressIsTaken(true); | ||||||
5016 | Register FrameAddr(AArch64::FP); | ||||||
5017 | while (Depth--) { | ||||||
5018 | Register NextFrame = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); | ||||||
5019 | auto Ldr = | ||||||
5020 | MIRBuilder.buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr}) | ||||||
5021 | .addImm(0); | ||||||
5022 | constrainSelectedInstRegOperands(*Ldr, TII, TRI, RBI); | ||||||
5023 | FrameAddr = NextFrame; | ||||||
5024 | } | ||||||
5025 | |||||||
5026 | if (IntrinID == Intrinsic::frameaddress) | ||||||
5027 | MIRBuilder.buildCopy({DstReg}, {FrameAddr}); | ||||||
5028 | else { | ||||||
5029 | MFI.setReturnAddressIsTaken(true); | ||||||
5030 | |||||||
5031 | if (STI.hasPAuth()) { | ||||||
5032 | Register TmpReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); | ||||||
5033 | MIRBuilder.buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1); | ||||||
5034 | MIRBuilder.buildInstr(AArch64::XPACI, {DstReg}, {TmpReg}); | ||||||
5035 | } else { | ||||||
5036 | MIRBuilder.buildInstr(AArch64::LDRXui, {Register(AArch64::LR)}, {FrameAddr}).addImm(1); | ||||||
5037 | MIRBuilder.buildInstr(AArch64::XPACLRI); | ||||||
5038 | MIRBuilder.buildCopy({DstReg}, {Register(AArch64::LR)}); | ||||||
5039 | } | ||||||
5040 | } | ||||||
5041 | |||||||
5042 | I.eraseFromParent(); | ||||||
5043 | return true; | ||||||
5044 | } | ||||||
5045 | } | ||||||
5046 | return false; | ||||||
5047 | } | ||||||
5048 | |||||||
5049 | InstructionSelector::ComplexRendererFns | ||||||
5050 | AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const { | ||||||
5051 | auto MaybeImmed = getImmedFromMO(Root); | ||||||
5052 | if (MaybeImmed == None || *MaybeImmed > 31) | ||||||
5053 | return None; | ||||||
5054 | uint64_t Enc = (32 - *MaybeImmed) & 0x1f; | ||||||
5055 | return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; | ||||||
5056 | } | ||||||
5057 | |||||||
5058 | InstructionSelector::ComplexRendererFns | ||||||
5059 | AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const { | ||||||
5060 | auto MaybeImmed = getImmedFromMO(Root); | ||||||
5061 | if (MaybeImmed == None || *MaybeImmed > 31) | ||||||
5062 | return None; | ||||||
5063 | uint64_t Enc = 31 - *MaybeImmed; | ||||||
5064 | return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; | ||||||
5065 | } | ||||||
5066 | |||||||
5067 | InstructionSelector::ComplexRendererFns | ||||||
5068 | AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const { | ||||||
5069 | auto MaybeImmed = getImmedFromMO(Root); | ||||||
5070 | if (MaybeImmed == None || *MaybeImmed > 63) | ||||||
5071 | return None; | ||||||
5072 | uint64_t Enc = (64 - *MaybeImmed) & 0x3f; | ||||||
5073 | return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; | ||||||
5074 | } | ||||||
5075 | |||||||
5076 | InstructionSelector::ComplexRendererFns | ||||||
5077 | AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const { | ||||||
5078 | auto MaybeImmed = getImmedFromMO(Root); | ||||||
5079 | if (MaybeImmed == None || *MaybeImmed > 63) | ||||||
5080 | return None; | ||||||
5081 | uint64_t Enc = 63 - *MaybeImmed; | ||||||
5082 | return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}}; | ||||||
5083 | } | ||||||
5084 | |||||||
5085 | /// Helper to select an immediate value that can be represented as a 12-bit | ||||||
5086 | /// value shifted left by either 0 or 12. If it is possible to do so, return | ||||||
5087 | /// the immediate and shift value. If not, return None. | ||||||
5088 | /// | ||||||
5089 | /// Used by selectArithImmed and selectNegArithImmed. | ||||||
5090 | InstructionSelector::ComplexRendererFns | ||||||
5091 | AArch64InstructionSelector::select12BitValueWithLeftShift( | ||||||
5092 | uint64_t Immed) const { | ||||||
5093 | unsigned ShiftAmt; | ||||||
5094 | if (Immed >> 12 == 0) { | ||||||
5095 | ShiftAmt = 0; | ||||||
5096 | } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) { | ||||||
5097 | ShiftAmt = 12; | ||||||
5098 | Immed = Immed >> 12; | ||||||
5099 | } else | ||||||
5100 | return None; | ||||||
5101 | |||||||
5102 | unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt); | ||||||
5103 | return {{ | ||||||
5104 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); }, | ||||||
5105 | [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); }, | ||||||
5106 | }}; | ||||||
5107 | } | ||||||
5108 | |||||||
5109 | /// SelectArithImmed - Select an immediate value that can be represented as | ||||||
5110 | /// a 12-bit value shifted left by either 0 or 12. If so, return true with | ||||||
5111 | /// Val set to the 12-bit value and Shift set to the shifter operand. | ||||||
5112 | InstructionSelector::ComplexRendererFns | ||||||
5113 | AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const { | ||||||
5114 | // This function is called from the addsub_shifted_imm ComplexPattern, | ||||||
5115 | // which lists [imm] as the list of opcode it's interested in, however | ||||||
5116 | // we still need to check whether the operand is actually an immediate | ||||||
5117 | // here because the ComplexPattern opcode list is only used in | ||||||
5118 | // root-level opcode matching. | ||||||
5119 | auto MaybeImmed = getImmedFromMO(Root); | ||||||
5120 | if (MaybeImmed == None) | ||||||
5121 | return None; | ||||||
5122 | return select12BitValueWithLeftShift(*MaybeImmed); | ||||||
5123 | } | ||||||
5124 | |||||||
5125 | /// SelectNegArithImmed - As above, but negates the value before trying to | ||||||
5126 | /// select it. | ||||||
5127 | InstructionSelector::ComplexRendererFns | ||||||
5128 | AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const { | ||||||
5129 | // We need a register here, because we need to know if we have a 64 or 32 | ||||||
5130 | // bit immediate. | ||||||
5131 | if (!Root.isReg()) | ||||||
5132 | return None; | ||||||
5133 | auto MaybeImmed = getImmedFromMO(Root); | ||||||
5134 | if (MaybeImmed == None) | ||||||
5135 | return None; | ||||||
5136 | uint64_t Immed = *MaybeImmed; | ||||||
5137 | |||||||
5138 | // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0" | ||||||
5139 | // have the opposite effect on the C flag, so this pattern mustn't match under | ||||||
5140 | // those circumstances. | ||||||
5141 | if (Immed == 0) | ||||||
5142 | return None; | ||||||
5143 | |||||||
5144 | // Check if we're dealing with a 32-bit type on the root or a 64-bit type on | ||||||
5145 | // the root. | ||||||
5146 | MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo(); | ||||||
5147 | if (MRI.getType(Root.getReg()).getSizeInBits() == 32) | ||||||
5148 | Immed = ~((uint32_t)Immed) + 1; | ||||||
5149 | else | ||||||
5150 | Immed = ~Immed + 1ULL; | ||||||
5151 | |||||||
5152 | if (Immed & 0xFFFFFFFFFF000000ULL) | ||||||
5153 | return None; | ||||||
5154 | |||||||
5155 | Immed &= 0xFFFFFFULL; | ||||||
5156 | return select12BitValueWithLeftShift(Immed); | ||||||
5157 | } | ||||||
5158 | |||||||
5159 | /// Return true if it is worth folding MI into an extended register. That is, | ||||||
5160 | /// if it's safe to pull it into the addressing mode of a load or store as a | ||||||
5161 | /// shift. | ||||||
5162 | bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg( | ||||||
5163 | MachineInstr &MI, const MachineRegisterInfo &MRI) const { | ||||||
5164 | // Always fold if there is one use, or if we're optimizing for size. | ||||||
5165 | Register DefReg = MI.getOperand(0).getReg(); | ||||||
5166 | if (MRI.hasOneNonDBGUse(DefReg) || | ||||||
5167 | MI.getParent()->getParent()->getFunction().hasMinSize()) | ||||||
5168 | return true; | ||||||
5169 | |||||||
5170 | // It's better to avoid folding and recomputing shifts when we don't have a | ||||||
5171 | // fastpath. | ||||||
5172 | if (!STI.hasLSLFast()) | ||||||
5173 | return false; | ||||||
5174 | |||||||
5175 | // We have a fastpath, so folding a shift in and potentially computing it | ||||||
5176 | // many times may be beneficial. Check if this is only used in memory ops. | ||||||
5177 | // If it is, then we should fold. | ||||||
5178 | return all_of(MRI.use_nodbg_instructions(DefReg), | ||||||
5179 | [](MachineInstr &Use) { return Use.mayLoadOrStore(); }); | ||||||
5180 | } | ||||||
5181 | |||||||
5182 | static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type) { | ||||||
5183 | switch (Type) { | ||||||
5184 | case AArch64_AM::SXTB: | ||||||
5185 | case AArch64_AM::SXTH: | ||||||
5186 | case AArch64_AM::SXTW: | ||||||
5187 | return true; | ||||||
5188 | default: | ||||||
5189 | return false; | ||||||
5190 | } | ||||||
5191 | } | ||||||
5192 | |||||||
5193 | InstructionSelector::ComplexRendererFns | ||||||
5194 | AArch64InstructionSelector::selectExtendedSHL( | ||||||
5195 | MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset, | ||||||
5196 | unsigned SizeInBytes, bool WantsExt) const { | ||||||
5197 | assert(Base.isReg() && "Expected base to be a register operand")((Base.isReg() && "Expected base to be a register operand" ) ? static_cast<void> (0) : __assert_fail ("Base.isReg() && \"Expected base to be a register operand\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5197, __PRETTY_FUNCTION__)); | ||||||
5198 | assert(Offset.isReg() && "Expected offset to be a register operand")((Offset.isReg() && "Expected offset to be a register operand" ) ? static_cast<void> (0) : __assert_fail ("Offset.isReg() && \"Expected offset to be a register operand\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5198, __PRETTY_FUNCTION__)); | ||||||
5199 | |||||||
5200 | MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo(); | ||||||
5201 | MachineInstr *OffsetInst = MRI.getVRegDef(Offset.getReg()); | ||||||
5202 | if (!OffsetInst) | ||||||
5203 | return None; | ||||||
5204 | |||||||
5205 | unsigned OffsetOpc = OffsetInst->getOpcode(); | ||||||
5206 | bool LookedThroughZExt = false; | ||||||
5207 | if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) { | ||||||
5208 | // Try to look through a ZEXT. | ||||||
5209 | if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt) | ||||||
5210 | return None; | ||||||
5211 | |||||||
5212 | OffsetInst = MRI.getVRegDef(OffsetInst->getOperand(1).getReg()); | ||||||
5213 | OffsetOpc = OffsetInst->getOpcode(); | ||||||
5214 | LookedThroughZExt = true; | ||||||
5215 | |||||||
5216 | if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) | ||||||
5217 | return None; | ||||||
5218 | } | ||||||
5219 | // Make sure that the memory op is a valid size. | ||||||
5220 | int64_t LegalShiftVal = Log2_32(SizeInBytes); | ||||||
5221 | if (LegalShiftVal == 0) | ||||||
5222 | return None; | ||||||
5223 | if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI)) | ||||||
5224 | return None; | ||||||
5225 | |||||||
5226 | // Now, try to find the specific G_CONSTANT. Start by assuming that the | ||||||
5227 | // register we will offset is the LHS, and the register containing the | ||||||
5228 | // constant is the RHS. | ||||||
5229 | Register OffsetReg = OffsetInst->getOperand(1).getReg(); | ||||||
5230 | Register ConstantReg = OffsetInst->getOperand(2).getReg(); | ||||||
5231 | auto ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI); | ||||||
5232 | if (!ValAndVReg) { | ||||||
5233 | // We didn't get a constant on the RHS. If the opcode is a shift, then | ||||||
5234 | // we're done. | ||||||
5235 | if (OffsetOpc == TargetOpcode::G_SHL) | ||||||
5236 | return None; | ||||||
5237 | |||||||
5238 | // If we have a G_MUL, we can use either register. Try looking at the RHS. | ||||||
5239 | std::swap(OffsetReg, ConstantReg); | ||||||
5240 | ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI); | ||||||
5241 | if (!ValAndVReg) | ||||||
5242 | return None; | ||||||
5243 | } | ||||||
5244 | |||||||
5245 | // The value must fit into 3 bits, and must be positive. Make sure that is | ||||||
5246 | // true. | ||||||
5247 | int64_t ImmVal = ValAndVReg->Value.getSExtValue(); | ||||||
5248 | |||||||
5249 | // Since we're going to pull this into a shift, the constant value must be | ||||||
5250 | // a power of 2. If we got a multiply, then we need to check this. | ||||||
5251 | if (OffsetOpc == TargetOpcode::G_MUL) { | ||||||
5252 | if (!isPowerOf2_32(ImmVal)) | ||||||
5253 | return None; | ||||||
5254 | |||||||
5255 | // Got a power of 2. So, the amount we'll shift is the log base-2 of that. | ||||||
5256 | ImmVal = Log2_32(ImmVal); | ||||||
5257 | } | ||||||
5258 | |||||||
5259 | if ((ImmVal & 0x7) != ImmVal) | ||||||
5260 | return None; | ||||||
5261 | |||||||
5262 | // We are only allowed to shift by LegalShiftVal. This shift value is built | ||||||
5263 | // into the instruction, so we can't just use whatever we want. | ||||||
5264 | if (ImmVal != LegalShiftVal) | ||||||
5265 | return None; | ||||||
5266 | |||||||
5267 | unsigned SignExtend = 0; | ||||||
5268 | if (WantsExt) { | ||||||
5269 | // Check if the offset is defined by an extend, unless we looked through a | ||||||
5270 | // G_ZEXT earlier. | ||||||
5271 | if (!LookedThroughZExt) { | ||||||
5272 | MachineInstr *ExtInst = getDefIgnoringCopies(OffsetReg, MRI); | ||||||
5273 | auto Ext = getExtendTypeForInst(*ExtInst, MRI, true); | ||||||
5274 | if (Ext == AArch64_AM::InvalidShiftExtend) | ||||||
5275 | return None; | ||||||
5276 | |||||||
5277 | SignExtend = isSignExtendShiftType(Ext) ? 1 : 0; | ||||||
5278 | // We only support SXTW for signed extension here. | ||||||
5279 | if (SignExtend && Ext != AArch64_AM::SXTW) | ||||||
5280 | return None; | ||||||
5281 | OffsetReg = ExtInst->getOperand(1).getReg(); | ||||||
5282 | } | ||||||
5283 | |||||||
5284 | // Need a 32-bit wide register here. | ||||||
5285 | MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg())); | ||||||
5286 | OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB); | ||||||
5287 | } | ||||||
5288 | |||||||
5289 | // We can use the LHS of the GEP as the base, and the LHS of the shift as an | ||||||
5290 | // offset. Signify that we are shifting by setting the shift flag to 1. | ||||||
5291 | return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base.getReg()); }, | ||||||
5292 | [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); }, | ||||||
5293 | [=](MachineInstrBuilder &MIB) { | ||||||
5294 | // Need to add both immediates here to make sure that they are both | ||||||
5295 | // added to the instruction. | ||||||
5296 | MIB.addImm(SignExtend); | ||||||
5297 | MIB.addImm(1); | ||||||
5298 | }}}; | ||||||
5299 | } | ||||||
5300 | |||||||
5301 | /// This is used for computing addresses like this: | ||||||
5302 | /// | ||||||
5303 | /// ldr x1, [x2, x3, lsl #3] | ||||||
5304 | /// | ||||||
5305 | /// Where x2 is the base register, and x3 is an offset register. The shift-left | ||||||
5306 | /// is a constant value specific to this load instruction. That is, we'll never | ||||||
5307 | /// see anything other than a 3 here (which corresponds to the size of the | ||||||
5308 | /// element being loaded.) | ||||||
5309 | InstructionSelector::ComplexRendererFns | ||||||
5310 | AArch64InstructionSelector::selectAddrModeShiftedExtendXReg( | ||||||
5311 | MachineOperand &Root, unsigned SizeInBytes) const { | ||||||
5312 | if (!Root.isReg()) | ||||||
5313 | return None; | ||||||
5314 | MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo(); | ||||||
5315 | |||||||
5316 | // We want to find something like this: | ||||||
5317 | // | ||||||
5318 | // val = G_CONSTANT LegalShiftVal | ||||||
5319 | // shift = G_SHL off_reg val | ||||||
5320 | // ptr = G_PTR_ADD base_reg shift | ||||||
5321 | // x = G_LOAD ptr | ||||||
5322 | // | ||||||
5323 | // And fold it into this addressing mode: | ||||||
5324 | // | ||||||
5325 | // ldr x, [base_reg, off_reg, lsl #LegalShiftVal] | ||||||
5326 | |||||||
5327 | // Check if we can find the G_PTR_ADD. | ||||||
5328 | MachineInstr *PtrAdd = | ||||||
5329 | getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI); | ||||||
5330 | if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI)) | ||||||
5331 | return None; | ||||||
5332 | |||||||
5333 | // Now, try to match an opcode which will match our specific offset. | ||||||
5334 | // We want a G_SHL or a G_MUL. | ||||||
5335 | MachineInstr *OffsetInst = | ||||||
5336 | getDefIgnoringCopies(PtrAdd->getOperand(2).getReg(), MRI); | ||||||
5337 | return selectExtendedSHL(Root, PtrAdd->getOperand(1), | ||||||
5338 | OffsetInst->getOperand(0), SizeInBytes, | ||||||
5339 | /*WantsExt=*/false); | ||||||
5340 | } | ||||||
5341 | |||||||
5342 | /// This is used for computing addresses like this: | ||||||
5343 | /// | ||||||
5344 | /// ldr x1, [x2, x3] | ||||||
5345 | /// | ||||||
5346 | /// Where x2 is the base register, and x3 is an offset register. | ||||||
5347 | /// | ||||||
5348 | /// When possible (or profitable) to fold a G_PTR_ADD into the address calculation, | ||||||
5349 | /// this will do so. Otherwise, it will return None. | ||||||
5350 | InstructionSelector::ComplexRendererFns | ||||||
5351 | AArch64InstructionSelector::selectAddrModeRegisterOffset( | ||||||
5352 | MachineOperand &Root) const { | ||||||
5353 | MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo(); | ||||||
5354 | |||||||
5355 | // We need a GEP. | ||||||
5356 | MachineInstr *Gep = MRI.getVRegDef(Root.getReg()); | ||||||
5357 | if (!Gep || Gep->getOpcode() != TargetOpcode::G_PTR_ADD) | ||||||
5358 | return None; | ||||||
5359 | |||||||
5360 | // If this is used more than once, let's not bother folding. | ||||||
5361 | // TODO: Check if they are memory ops. If they are, then we can still fold | ||||||
5362 | // without having to recompute anything. | ||||||
5363 | if (!MRI.hasOneNonDBGUse(Gep->getOperand(0).getReg())) | ||||||
5364 | return None; | ||||||
5365 | |||||||
5366 | // Base is the GEP's LHS, offset is its RHS. | ||||||
5367 | return {{[=](MachineInstrBuilder &MIB) { | ||||||
5368 | MIB.addUse(Gep->getOperand(1).getReg()); | ||||||
5369 | }, | ||||||
5370 | [=](MachineInstrBuilder &MIB) { | ||||||
5371 | MIB.addUse(Gep->getOperand(2).getReg()); | ||||||
5372 | }, | ||||||
5373 | [=](MachineInstrBuilder &MIB) { | ||||||
5374 | // Need to add both immediates here to make sure that they are both | ||||||
5375 | // added to the instruction. | ||||||
5376 | MIB.addImm(0); | ||||||
5377 | MIB.addImm(0); | ||||||
5378 | }}}; | ||||||
5379 | } | ||||||
5380 | |||||||
5381 | /// This is intended to be equivalent to selectAddrModeXRO in | ||||||
5382 | /// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads. | ||||||
5383 | InstructionSelector::ComplexRendererFns | ||||||
5384 | AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root, | ||||||
5385 | unsigned SizeInBytes) const { | ||||||
5386 | MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo(); | ||||||
5387 | if (!Root.isReg()) | ||||||
5388 | return None; | ||||||
5389 | MachineInstr *PtrAdd = | ||||||
5390 | getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI); | ||||||
5391 | if (!PtrAdd) | ||||||
5392 | return None; | ||||||
5393 | |||||||
5394 | // Check for an immediates which cannot be encoded in the [base + imm] | ||||||
5395 | // addressing mode, and can't be encoded in an add/sub. If this happens, we'll | ||||||
5396 | // end up with code like: | ||||||
5397 | // | ||||||
5398 | // mov x0, wide | ||||||
5399 | // add x1 base, x0 | ||||||
5400 | // ldr x2, [x1, x0] | ||||||
5401 | // | ||||||
5402 | // In this situation, we can use the [base, xreg] addressing mode to save an | ||||||
5403 | // add/sub: | ||||||
5404 | // | ||||||
5405 | // mov x0, wide | ||||||
5406 | // ldr x2, [base, x0] | ||||||
5407 | auto ValAndVReg = | ||||||
5408 | getConstantVRegValWithLookThrough(PtrAdd->getOperand(2).getReg(), MRI); | ||||||
5409 | if (ValAndVReg) { | ||||||
5410 | unsigned Scale = Log2_32(SizeInBytes); | ||||||
5411 | int64_t ImmOff = ValAndVReg->Value.getSExtValue(); | ||||||
5412 | |||||||
5413 | // Skip immediates that can be selected in the load/store addresing | ||||||
5414 | // mode. | ||||||
5415 | if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 && | ||||||
5416 | ImmOff < (0x1000 << Scale)) | ||||||
5417 | return None; | ||||||
5418 | |||||||
5419 | // Helper lambda to decide whether or not it is preferable to emit an add. | ||||||
5420 | auto isPreferredADD = [](int64_t ImmOff) { | ||||||
5421 | // Constants in [0x0, 0xfff] can be encoded in an add. | ||||||
5422 | if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL) | ||||||
5423 | return true; | ||||||
5424 | |||||||
5425 | // Can it be encoded in an add lsl #12? | ||||||
5426 | if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL) | ||||||
5427 | return false; | ||||||
5428 | |||||||
5429 | // It can be encoded in an add lsl #12, but we may not want to. If it is | ||||||
5430 | // possible to select this as a single movz, then prefer that. A single | ||||||
5431 | // movz is faster than an add with a shift. | ||||||
5432 | return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL && | ||||||
5433 | (ImmOff & 0xffffffffffff0fffLL) != 0x0LL; | ||||||
5434 | }; | ||||||
5435 | |||||||
5436 | // If the immediate can be encoded in a single add/sub, then bail out. | ||||||
5437 | if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff)) | ||||||
5438 | return None; | ||||||
5439 | } | ||||||
5440 | |||||||
5441 | // Try to fold shifts into the addressing mode. | ||||||
5442 | auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes); | ||||||
5443 | if (AddrModeFns) | ||||||
5444 | return AddrModeFns; | ||||||
5445 | |||||||
5446 | // If that doesn't work, see if it's possible to fold in registers from | ||||||
5447 | // a GEP. | ||||||
5448 | return selectAddrModeRegisterOffset(Root); | ||||||
5449 | } | ||||||
5450 | |||||||
5451 | /// This is used for computing addresses like this: | ||||||
5452 | /// | ||||||
5453 | /// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal] | ||||||
5454 | /// | ||||||
5455 | /// Where we have a 64-bit base register, a 32-bit offset register, and an | ||||||
5456 | /// extend (which may or may not be signed). | ||||||
5457 | InstructionSelector::ComplexRendererFns | ||||||
5458 | AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root, | ||||||
5459 | unsigned SizeInBytes) const { | ||||||
5460 | MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo(); | ||||||
5461 | |||||||
5462 | MachineInstr *PtrAdd = | ||||||
5463 | getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI); | ||||||
5464 | if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI)) | ||||||
5465 | return None; | ||||||
5466 | |||||||
5467 | MachineOperand &LHS = PtrAdd->getOperand(1); | ||||||
5468 | MachineOperand &RHS = PtrAdd->getOperand(2); | ||||||
5469 | MachineInstr *OffsetInst = getDefIgnoringCopies(RHS.getReg(), MRI); | ||||||
5470 | |||||||
5471 | // The first case is the same as selectAddrModeXRO, except we need an extend. | ||||||
5472 | // In this case, we try to find a shift and extend, and fold them into the | ||||||
5473 | // addressing mode. | ||||||
5474 | // | ||||||
5475 | // E.g. | ||||||
5476 | // | ||||||
5477 | // off_reg = G_Z/S/ANYEXT ext_reg | ||||||
5478 | // val = G_CONSTANT LegalShiftVal | ||||||
5479 | // shift = G_SHL off_reg val | ||||||
5480 | // ptr = G_PTR_ADD base_reg shift | ||||||
5481 | // x = G_LOAD ptr | ||||||
5482 | // | ||||||
5483 | // In this case we can get a load like this: | ||||||
5484 | // | ||||||
5485 | // ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal] | ||||||
5486 | auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->getOperand(0), | ||||||
5487 | SizeInBytes, /*WantsExt=*/true); | ||||||
5488 | if (ExtendedShl) | ||||||
5489 | return ExtendedShl; | ||||||
5490 | |||||||
5491 | // There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though. | ||||||
5492 | // | ||||||
5493 | // e.g. | ||||||
5494 | // ldr something, [base_reg, ext_reg, sxtw] | ||||||
5495 | if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI)) | ||||||
5496 | return None; | ||||||
5497 | |||||||
5498 | // Check if this is an extend. We'll get an extend type if it is. | ||||||
5499 | AArch64_AM::ShiftExtendType Ext = | ||||||
5500 | getExtendTypeForInst(*OffsetInst, MRI, /*IsLoadStore=*/true); | ||||||
5501 | if (Ext == AArch64_AM::InvalidShiftExtend) | ||||||
5502 | return None; | ||||||
5503 | |||||||
5504 | // Need a 32-bit wide register. | ||||||
5505 | MachineIRBuilder MIB(*PtrAdd); | ||||||
5506 | Register ExtReg = moveScalarRegClass(OffsetInst->getOperand(1).getReg(), | ||||||
5507 | AArch64::GPR32RegClass, MIB); | ||||||
5508 | unsigned SignExtend = Ext == AArch64_AM::SXTW; | ||||||
5509 | |||||||
5510 | // Base is LHS, offset is ExtReg. | ||||||
5511 | return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(LHS.getReg()); }, | ||||||
5512 | [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }, | ||||||
5513 | [=](MachineInstrBuilder &MIB) { | ||||||
5514 | MIB.addImm(SignExtend); | ||||||
5515 | MIB.addImm(0); | ||||||
5516 | }}}; | ||||||
5517 | } | ||||||
5518 | |||||||
5519 | /// Select a "register plus unscaled signed 9-bit immediate" address. This | ||||||
5520 | /// should only match when there is an offset that is not valid for a scaled | ||||||
5521 | /// immediate addressing mode. The "Size" argument is the size in bytes of the | ||||||
5522 | /// memory reference, which is needed here to know what is valid for a scaled | ||||||
5523 | /// immediate. | ||||||
5524 | InstructionSelector::ComplexRendererFns | ||||||
5525 | AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root, | ||||||
5526 | unsigned Size) const { | ||||||
5527 | MachineRegisterInfo &MRI = | ||||||
5528 | Root.getParent()->getParent()->getParent()->getRegInfo(); | ||||||
5529 | |||||||
5530 | if (!Root.isReg()) | ||||||
5531 | return None; | ||||||
5532 | |||||||
5533 | if (!isBaseWithConstantOffset(Root, MRI)) | ||||||
5534 | return None; | ||||||
5535 | |||||||
5536 | MachineInstr *RootDef = MRI.getVRegDef(Root.getReg()); | ||||||
5537 | if (!RootDef) | ||||||
5538 | return None; | ||||||
5539 | |||||||
5540 | MachineOperand &OffImm = RootDef->getOperand(2); | ||||||
5541 | if (!OffImm.isReg()) | ||||||
5542 | return None; | ||||||
5543 | MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg()); | ||||||
5544 | if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT) | ||||||
5545 | return None; | ||||||
5546 | int64_t RHSC; | ||||||
5547 | MachineOperand &RHSOp1 = RHS->getOperand(1); | ||||||
5548 | if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64) | ||||||
5549 | return None; | ||||||
5550 | RHSC = RHSOp1.getCImm()->getSExtValue(); | ||||||
5551 | |||||||
5552 | // If the offset is valid as a scaled immediate, don't match here. | ||||||
5553 | if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size))) | ||||||
5554 | return None; | ||||||
5555 | if (RHSC >= -256 && RHSC < 256) { | ||||||
5556 | MachineOperand &Base = RootDef->getOperand(1); | ||||||
5557 | return {{ | ||||||
5558 | [=](MachineInstrBuilder &MIB) { MIB.add(Base); }, | ||||||
5559 | [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }, | ||||||
5560 | }}; | ||||||
5561 | } | ||||||
5562 | return None; | ||||||
5563 | } | ||||||
5564 | |||||||
5565 | InstructionSelector::ComplexRendererFns | ||||||
5566 | AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef, | ||||||
5567 | unsigned Size, | ||||||
5568 | MachineRegisterInfo &MRI) const { | ||||||
5569 | if (RootDef.getOpcode() != AArch64::G_ADD_LOW) | ||||||
5570 | return None; | ||||||
5571 | MachineInstr &Adrp = *MRI.getVRegDef(RootDef.getOperand(1).getReg()); | ||||||
5572 | if (Adrp.getOpcode() != AArch64::ADRP) | ||||||
5573 | return None; | ||||||
5574 | |||||||
5575 | // TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG. | ||||||
5576 | // TODO: Need to check GV's offset % size if doing offset folding into globals. | ||||||
5577 | assert(Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global")((Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global" ) ? static_cast<void> (0) : __assert_fail ("Adrp.getOperand(1).getOffset() == 0 && \"Unexpected offset in global\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5577, __PRETTY_FUNCTION__)); | ||||||
5578 | auto GV = Adrp.getOperand(1).getGlobal(); | ||||||
5579 | if (GV->isThreadLocal()) | ||||||
5580 | return None; | ||||||
5581 | |||||||
5582 | auto &MF = *RootDef.getParent()->getParent(); | ||||||
5583 | if (GV->getPointerAlignment(MF.getDataLayout()) < Size) | ||||||
5584 | return None; | ||||||
5585 | |||||||
5586 | unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget()); | ||||||
5587 | MachineIRBuilder MIRBuilder(RootDef); | ||||||
5588 | Register AdrpReg = Adrp.getOperand(0).getReg(); | ||||||
5589 | return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); }, | ||||||
5590 | [=](MachineInstrBuilder &MIB) { | ||||||
5591 | MIB.addGlobalAddress(GV, /* Offset */ 0, | ||||||
5592 | OpFlags | AArch64II::MO_PAGEOFF | | ||||||
5593 | AArch64II::MO_NC); | ||||||
5594 | }}}; | ||||||
5595 | } | ||||||
5596 | |||||||
5597 | /// Select a "register plus scaled unsigned 12-bit immediate" address. The | ||||||
5598 | /// "Size" argument is the size in bytes of the memory reference, which | ||||||
5599 | /// determines the scale. | ||||||
5600 | InstructionSelector::ComplexRendererFns | ||||||
5601 | AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root, | ||||||
5602 | unsigned Size) const { | ||||||
5603 | MachineFunction &MF = *Root.getParent()->getParent()->getParent(); | ||||||
5604 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||
5605 | |||||||
5606 | if (!Root.isReg()) | ||||||
5607 | return None; | ||||||
5608 | |||||||
5609 | MachineInstr *RootDef = MRI.getVRegDef(Root.getReg()); | ||||||
5610 | if (!RootDef) | ||||||
5611 | return None; | ||||||
5612 | |||||||
5613 | if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) { | ||||||
5614 | return {{ | ||||||
5615 | [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); }, | ||||||
5616 | [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, | ||||||
5617 | }}; | ||||||
5618 | } | ||||||
5619 | |||||||
5620 | CodeModel::Model CM = MF.getTarget().getCodeModel(); | ||||||
5621 | // Check if we can fold in the ADD of small code model ADRP + ADD address. | ||||||
5622 | if (CM == CodeModel::Small) { | ||||||
5623 | auto OpFns = tryFoldAddLowIntoImm(*RootDef, Size, MRI); | ||||||
5624 | if (OpFns) | ||||||
5625 | return OpFns; | ||||||
5626 | } | ||||||
5627 | |||||||
5628 | if (isBaseWithConstantOffset(Root, MRI)) { | ||||||
5629 | MachineOperand &LHS = RootDef->getOperand(1); | ||||||
5630 | MachineOperand &RHS = RootDef->getOperand(2); | ||||||
5631 | MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg()); | ||||||
5632 | MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg()); | ||||||
5633 | if (LHSDef && RHSDef) { | ||||||
5634 | int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue(); | ||||||
5635 | unsigned Scale = Log2_32(Size); | ||||||
5636 | if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) { | ||||||
5637 | if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) | ||||||
5638 | return {{ | ||||||
5639 | [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); }, | ||||||
5640 | [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); }, | ||||||
5641 | }}; | ||||||
5642 | |||||||
5643 | return {{ | ||||||
5644 | [=](MachineInstrBuilder &MIB) { MIB.add(LHS); }, | ||||||
5645 | [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); }, | ||||||
5646 | }}; | ||||||
5647 | } | ||||||
5648 | } | ||||||
5649 | } | ||||||
5650 | |||||||
5651 | // Before falling back to our general case, check if the unscaled | ||||||
5652 | // instructions can handle this. If so, that's preferable. | ||||||
5653 | if (selectAddrModeUnscaled(Root, Size).hasValue()) | ||||||
5654 | return None; | ||||||
5655 | |||||||
5656 | return {{ | ||||||
5657 | [=](MachineInstrBuilder &MIB) { MIB.add(Root); }, | ||||||
5658 | [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, | ||||||
5659 | }}; | ||||||
5660 | } | ||||||
5661 | |||||||
5662 | /// Given a shift instruction, return the correct shift type for that | ||||||
5663 | /// instruction. | ||||||
5664 | static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) { | ||||||
5665 | // TODO: Handle AArch64_AM::ROR | ||||||
5666 | switch (MI.getOpcode()) { | ||||||
5667 | default: | ||||||
5668 | return AArch64_AM::InvalidShiftExtend; | ||||||
5669 | case TargetOpcode::G_SHL: | ||||||
5670 | return AArch64_AM::LSL; | ||||||
5671 | case TargetOpcode::G_LSHR: | ||||||
5672 | return AArch64_AM::LSR; | ||||||
5673 | case TargetOpcode::G_ASHR: | ||||||
5674 | return AArch64_AM::ASR; | ||||||
5675 | } | ||||||
5676 | } | ||||||
5677 | |||||||
5678 | /// Select a "shifted register" operand. If the value is not shifted, set the | ||||||
5679 | /// shift operand to a default value of "lsl 0". | ||||||
5680 | /// | ||||||
5681 | /// TODO: Allow shifted register to be rotated in logical instructions. | ||||||
5682 | InstructionSelector::ComplexRendererFns | ||||||
5683 | AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root) const { | ||||||
5684 | if (!Root.isReg()) | ||||||
5685 | return None; | ||||||
5686 | MachineRegisterInfo &MRI = | ||||||
5687 | Root.getParent()->getParent()->getParent()->getRegInfo(); | ||||||
5688 | |||||||
5689 | // Check if the operand is defined by an instruction which corresponds to | ||||||
5690 | // a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc. | ||||||
5691 | // | ||||||
5692 | // TODO: Handle AArch64_AM::ROR for logical instructions. | ||||||
5693 | MachineInstr *ShiftInst = MRI.getVRegDef(Root.getReg()); | ||||||
5694 | if (!ShiftInst) | ||||||
5695 | return None; | ||||||
5696 | AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(*ShiftInst); | ||||||
5697 | if (ShType == AArch64_AM::InvalidShiftExtend) | ||||||
5698 | return None; | ||||||
5699 | if (!isWorthFoldingIntoExtendedReg(*ShiftInst, MRI)) | ||||||
5700 | return None; | ||||||
5701 | |||||||
5702 | // Need an immediate on the RHS. | ||||||
5703 | MachineOperand &ShiftRHS = ShiftInst->getOperand(2); | ||||||
5704 | auto Immed = getImmedFromMO(ShiftRHS); | ||||||
5705 | if (!Immed) | ||||||
5706 | return None; | ||||||
5707 | |||||||
5708 | // We have something that we can fold. Fold in the shift's LHS and RHS into | ||||||
5709 | // the instruction. | ||||||
5710 | MachineOperand &ShiftLHS = ShiftInst->getOperand(1); | ||||||
5711 | Register ShiftReg = ShiftLHS.getReg(); | ||||||
5712 | |||||||
5713 | unsigned NumBits = MRI.getType(ShiftReg).getSizeInBits(); | ||||||
5714 | unsigned Val = *Immed & (NumBits - 1); | ||||||
5715 | unsigned ShiftVal = AArch64_AM::getShifterImm(ShType, Val); | ||||||
5716 | |||||||
5717 | return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ShiftReg); }, | ||||||
5718 | [=](MachineInstrBuilder &MIB) { MIB.addImm(ShiftVal); }}}; | ||||||
5719 | } | ||||||
5720 | |||||||
5721 | AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst( | ||||||
5722 | MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const { | ||||||
5723 | unsigned Opc = MI.getOpcode(); | ||||||
5724 | |||||||
5725 | // Handle explicit extend instructions first. | ||||||
5726 | if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) { | ||||||
5727 | unsigned Size; | ||||||
5728 | if (Opc == TargetOpcode::G_SEXT) | ||||||
5729 | Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); | ||||||
5730 | else | ||||||
5731 | Size = MI.getOperand(2).getImm(); | ||||||
5732 | assert(Size != 64 && "Extend from 64 bits?")((Size != 64 && "Extend from 64 bits?") ? static_cast <void> (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5732, __PRETTY_FUNCTION__)); | ||||||
5733 | switch (Size) { | ||||||
5734 | case 8: | ||||||
5735 | return AArch64_AM::SXTB; | ||||||
5736 | case 16: | ||||||
5737 | return AArch64_AM::SXTH; | ||||||
5738 | case 32: | ||||||
5739 | return AArch64_AM::SXTW; | ||||||
5740 | default: | ||||||
5741 | return AArch64_AM::InvalidShiftExtend; | ||||||
5742 | } | ||||||
5743 | } | ||||||
5744 | |||||||
5745 | if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) { | ||||||
5746 | unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); | ||||||
5747 | assert(Size != 64 && "Extend from 64 bits?")((Size != 64 && "Extend from 64 bits?") ? static_cast <void> (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5747, __PRETTY_FUNCTION__)); | ||||||
5748 | switch (Size) { | ||||||
5749 | case 8: | ||||||
5750 | return AArch64_AM::UXTB; | ||||||
5751 | case 16: | ||||||
5752 | return AArch64_AM::UXTH; | ||||||
5753 | case 32: | ||||||
5754 | return AArch64_AM::UXTW; | ||||||
5755 | default: | ||||||
5756 | return AArch64_AM::InvalidShiftExtend; | ||||||
5757 | } | ||||||
5758 | } | ||||||
5759 | |||||||
5760 | // Don't have an explicit extend. Try to handle a G_AND with a constant mask | ||||||
5761 | // on the RHS. | ||||||
5762 | if (Opc != TargetOpcode::G_AND) | ||||||
5763 | return AArch64_AM::InvalidShiftExtend; | ||||||
5764 | |||||||
5765 | Optional<uint64_t> MaybeAndMask = getImmedFromMO(MI.getOperand(2)); | ||||||
5766 | if (!MaybeAndMask) | ||||||
5767 | return AArch64_AM::InvalidShiftExtend; | ||||||
5768 | uint64_t AndMask = *MaybeAndMask; | ||||||
5769 | switch (AndMask) { | ||||||
5770 | default: | ||||||
5771 | return AArch64_AM::InvalidShiftExtend; | ||||||
5772 | case 0xFF: | ||||||
5773 | return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend; | ||||||
5774 | case 0xFFFF: | ||||||
5775 | return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend; | ||||||
5776 | case 0xFFFFFFFF: | ||||||
5777 | return AArch64_AM::UXTW; | ||||||
5778 | } | ||||||
5779 | } | ||||||
5780 | |||||||
5781 | Register AArch64InstructionSelector::moveScalarRegClass( | ||||||
5782 | Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const { | ||||||
5783 | MachineRegisterInfo &MRI = *MIB.getMRI(); | ||||||
5784 | auto Ty = MRI.getType(Reg); | ||||||
5785 | assert(!Ty.isVector() && "Expected scalars only!")((!Ty.isVector() && "Expected scalars only!") ? static_cast <void> (0) : __assert_fail ("!Ty.isVector() && \"Expected scalars only!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5785, __PRETTY_FUNCTION__)); | ||||||
5786 | if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC)) | ||||||
5787 | return Reg; | ||||||
5788 | |||||||
5789 | // Create a copy and immediately select it. | ||||||
5790 | // FIXME: We should have an emitCopy function? | ||||||
5791 | auto Copy = MIB.buildCopy({&RC}, {Reg}); | ||||||
5792 | selectCopy(*Copy, TII, MRI, TRI, RBI); | ||||||
5793 | return Copy.getReg(0); | ||||||
5794 | } | ||||||
5795 | |||||||
5796 | /// Select an "extended register" operand. This operand folds in an extend | ||||||
5797 | /// followed by an optional left shift. | ||||||
5798 | InstructionSelector::ComplexRendererFns | ||||||
5799 | AArch64InstructionSelector::selectArithExtendedRegister( | ||||||
5800 | MachineOperand &Root) const { | ||||||
5801 | if (!Root.isReg()) | ||||||
5802 | return None; | ||||||
5803 | MachineRegisterInfo &MRI = | ||||||
5804 | Root.getParent()->getParent()->getParent()->getRegInfo(); | ||||||
5805 | |||||||
5806 | uint64_t ShiftVal = 0; | ||||||
5807 | Register ExtReg; | ||||||
5808 | AArch64_AM::ShiftExtendType Ext; | ||||||
5809 | MachineInstr *RootDef = getDefIgnoringCopies(Root.getReg(), MRI); | ||||||
5810 | if (!RootDef) | ||||||
5811 | return None; | ||||||
5812 | |||||||
5813 | if (!isWorthFoldingIntoExtendedReg(*RootDef, MRI)) | ||||||
5814 | return None; | ||||||
5815 | |||||||
5816 | // Check if we can fold a shift and an extend. | ||||||
5817 | if (RootDef->getOpcode() == TargetOpcode::G_SHL) { | ||||||
5818 | // Look for a constant on the RHS of the shift. | ||||||
5819 | MachineOperand &RHS = RootDef->getOperand(2); | ||||||
5820 | Optional<uint64_t> MaybeShiftVal = getImmedFromMO(RHS); | ||||||
5821 | if (!MaybeShiftVal) | ||||||
5822 | return None; | ||||||
5823 | ShiftVal = *MaybeShiftVal; | ||||||
5824 | if (ShiftVal > 4) | ||||||
5825 | return None; | ||||||
5826 | // Look for a valid extend instruction on the LHS of the shift. | ||||||
5827 | MachineOperand &LHS = RootDef->getOperand(1); | ||||||
5828 | MachineInstr *ExtDef = getDefIgnoringCopies(LHS.getReg(), MRI); | ||||||
5829 | if (!ExtDef) | ||||||
5830 | return None; | ||||||
5831 | Ext = getExtendTypeForInst(*ExtDef, MRI); | ||||||
5832 | if (Ext == AArch64_AM::InvalidShiftExtend) | ||||||
5833 | return None; | ||||||
5834 | ExtReg = ExtDef->getOperand(1).getReg(); | ||||||
5835 | } else { | ||||||
5836 | // Didn't get a shift. Try just folding an extend. | ||||||
5837 | Ext = getExtendTypeForInst(*RootDef, MRI); | ||||||
5838 | if (Ext
| ||||||
5839 | return None; | ||||||
5840 | ExtReg = RootDef->getOperand(1).getReg(); | ||||||
5841 | |||||||
5842 | // If we have a 32 bit instruction which zeroes out the high half of a | ||||||
5843 | // register, we get an implicit zero extend for free. Check if we have one. | ||||||
5844 | // FIXME: We actually emit the extend right now even though we don't have | ||||||
5845 | // to. | ||||||
5846 | if (Ext
| ||||||
5847 | MachineInstr *ExtInst = MRI.getVRegDef(ExtReg); | ||||||
5848 | if (ExtInst && isDef32(*ExtInst)) | ||||||
5849 | return None; | ||||||
5850 | } | ||||||
5851 | } | ||||||
5852 | |||||||
5853 | // We require a GPR32 here. Narrow the ExtReg if needed using a subregister | ||||||
5854 | // copy. | ||||||
5855 | MachineIRBuilder MIB(*RootDef); | ||||||
5856 | ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB); | ||||||
5857 | |||||||
5858 | return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }, | ||||||
5859 | [=](MachineInstrBuilder &MIB) { | ||||||
5860 | MIB.addImm(getArithExtendImm(Ext, ShiftVal)); | ||||||
5861 | }}}; | ||||||
5862 | } | ||||||
5863 | |||||||
5864 | void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB, | ||||||
5865 | const MachineInstr &MI, | ||||||
5866 | int OpIdx) const { | ||||||
5867 | const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); | ||||||
5868 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&((MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? static_cast<void > (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5869, __PRETTY_FUNCTION__)) | ||||||
5869 | "Expected G_CONSTANT")((MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? static_cast<void > (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5869, __PRETTY_FUNCTION__)); | ||||||
5870 | Optional<int64_t> CstVal = | ||||||
5871 | getConstantVRegSExtVal(MI.getOperand(0).getReg(), MRI); | ||||||
5872 | assert(CstVal && "Expected constant value")((CstVal && "Expected constant value") ? static_cast< void> (0) : __assert_fail ("CstVal && \"Expected constant value\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5872, __PRETTY_FUNCTION__)); | ||||||
5873 | MIB.addImm(CstVal.getValue()); | ||||||
5874 | } | ||||||
5875 | |||||||
5876 | void AArch64InstructionSelector::renderLogicalImm32( | ||||||
5877 | MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const { | ||||||
5878 | assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5879, __PRETTY_FUNCTION__)) | ||||||
5879 | "Expected G_CONSTANT")((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5879, __PRETTY_FUNCTION__)); | ||||||
5880 | uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue(); | ||||||
5881 | uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 32); | ||||||
5882 | MIB.addImm(Enc); | ||||||
5883 | } | ||||||
5884 | |||||||
5885 | void AArch64InstructionSelector::renderLogicalImm64( | ||||||
5886 | MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const { | ||||||
5887 | assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5888, __PRETTY_FUNCTION__)) | ||||||
5888 | "Expected G_CONSTANT")((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT") ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5888, __PRETTY_FUNCTION__)); | ||||||
5889 | uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue(); | ||||||
5890 | uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 64); | ||||||
5891 | MIB.addImm(Enc); | ||||||
5892 | } | ||||||
5893 | |||||||
5894 | bool AArch64InstructionSelector::isLoadStoreOfNumBytes( | ||||||
5895 | const MachineInstr &MI, unsigned NumBytes) const { | ||||||
5896 | if (!MI.mayLoadOrStore()) | ||||||
5897 | return false; | ||||||
5898 | assert(MI.hasOneMemOperand() &&((MI.hasOneMemOperand() && "Expected load/store to have only one mem op!" ) ? static_cast<void> (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5899, __PRETTY_FUNCTION__)) | ||||||
5899 | "Expected load/store to have only one mem op!")((MI.hasOneMemOperand() && "Expected load/store to have only one mem op!" ) ? static_cast<void> (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5899, __PRETTY_FUNCTION__)); | ||||||
5900 | return (*MI.memoperands_begin())->getSize() == NumBytes; | ||||||
5901 | } | ||||||
5902 | |||||||
5903 | bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const { | ||||||
5904 | const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); | ||||||
5905 | if (MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() != 32) | ||||||
5906 | return false; | ||||||
5907 | |||||||
5908 | // Only return true if we know the operation will zero-out the high half of | ||||||
5909 | // the 64-bit register. Truncates can be subregister copies, which don't | ||||||
5910 | // zero out the high bits. Copies and other copy-like instructions can be | ||||||
5911 | // fed by truncates, or could be lowered as subregister copies. | ||||||
5912 | switch (MI.getOpcode()) { | ||||||
5913 | default: | ||||||
5914 | return true; | ||||||
5915 | case TargetOpcode::COPY: | ||||||
5916 | case TargetOpcode::G_BITCAST: | ||||||
5917 | case TargetOpcode::G_TRUNC: | ||||||
5918 | case TargetOpcode::G_PHI: | ||||||
5919 | return false; | ||||||
5920 | } | ||||||
5921 | } | ||||||
5922 | |||||||
5923 | |||||||
5924 | // Perform fixups on the given PHI instruction's operands to force them all | ||||||
5925 | // to be the same as the destination regbank. | ||||||
5926 | static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI, | ||||||
5927 | const AArch64RegisterBankInfo &RBI) { | ||||||
5928 | assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI")((MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI" ) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_PHI && \"Expected a G_PHI\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5928, __PRETTY_FUNCTION__)); | ||||||
5929 | Register DstReg = MI.getOperand(0).getReg(); | ||||||
5930 | const RegisterBank *DstRB = MRI.getRegBankOrNull(DstReg); | ||||||
5931 | assert(DstRB && "Expected PHI dst to have regbank assigned")((DstRB && "Expected PHI dst to have regbank assigned" ) ? static_cast<void> (0) : __assert_fail ("DstRB && \"Expected PHI dst to have regbank assigned\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp" , 5931, __PRETTY_FUNCTION__)); | ||||||
5932 | MachineIRBuilder MIB(MI); | ||||||
5933 | |||||||
5934 | // Go through each operand and ensure it has the same regbank. | ||||||
5935 | for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) { | ||||||
5936 | MachineOperand &MO = MI.getOperand(OpIdx); | ||||||
5937 | if (!MO.isReg()) | ||||||
5938 | continue; | ||||||
5939 | Register OpReg = MO.getReg(); | ||||||
5940 | const RegisterBank *RB = MRI.getRegBankOrNull(OpReg); | ||||||
5941 | if (RB != DstRB) { | ||||||
5942 | // Insert a cross-bank copy. | ||||||
5943 | auto *OpDef = MRI.getVRegDef(OpReg); | ||||||
5944 | const LLT &Ty = MRI.getType(OpReg); | ||||||
5945 | MIB.setInsertPt(*OpDef->getParent(), std::next(OpDef->getIterator())); | ||||||
5946 | auto Copy = MIB.buildCopy(Ty, OpReg); | ||||||
5947 | MRI.setRegBank(Copy.getReg(0), *DstRB); | ||||||
5948 | MO.setReg(Copy.getReg(0)); | ||||||
5949 | } | ||||||
5950 | } | ||||||
5951 | } | ||||||
5952 | |||||||
5953 | void AArch64InstructionSelector::processPHIs(MachineFunction &MF) { | ||||||
5954 | // We're looking for PHIs, build a list so we don't invalidate iterators. | ||||||
5955 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||
5956 | SmallVector<MachineInstr *, 32> Phis; | ||||||
5957 | for (auto &BB : MF) { | ||||||
5958 | for (auto &MI : BB) { | ||||||
5959 | if (MI.getOpcode() == TargetOpcode::G_PHI) | ||||||
5960 | Phis.emplace_back(&MI); | ||||||
5961 | } | ||||||
5962 | } | ||||||
5963 | |||||||
5964 | for (auto *MI : Phis) { | ||||||
5965 | // We need to do some work here if the operand types are < 16 bit and they | ||||||
5966 | // are split across fpr/gpr banks. Since all types <32b on gpr | ||||||
5967 | // end up being assigned gpr32 regclasses, we can end up with PHIs here | ||||||
5968 | // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't | ||||||
5969 | // be selecting heterogenous regbanks for operands if possible, but we | ||||||
5970 | // still need to be able to deal with it here. | ||||||
5971 | // | ||||||
5972 | // To fix this, if we have a gpr-bank operand < 32b in size and at least | ||||||
5973 | // one other operand is on the fpr bank, then we add cross-bank copies | ||||||
5974 | // to homogenize the operand banks. For simplicity the bank that we choose | ||||||
5975 | // to settle on is whatever bank the def operand has. For example: | ||||||
5976 | // | ||||||
5977 | // %endbb: | ||||||
5978 | // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2 | ||||||
5979 | // => | ||||||
5980 | // %bb2: | ||||||
5981 | // ... | ||||||
5982 | // %in2_copy:gpr(s16) = COPY %in2:fpr(s16) | ||||||
5983 | // ... | ||||||
5984 | // %endbb: | ||||||
5985 | // %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2 | ||||||
5986 | bool HasGPROp = false, HasFPROp = false; | ||||||
5987 | for (unsigned OpIdx = 1; OpIdx < MI->getNumOperands(); ++OpIdx) { | ||||||
5988 | const auto &MO = MI->getOperand(OpIdx); | ||||||
5989 | if (!MO.isReg()) | ||||||
5990 | continue; | ||||||
5991 | const LLT &Ty = MRI.getType(MO.getReg()); | ||||||
5992 | if (!Ty.isValid() || !Ty.isScalar()) | ||||||
5993 | break; | ||||||
5994 | if (Ty.getSizeInBits() >= 32) | ||||||
5995 | break; | ||||||
5996 | const RegisterBank *RB = MRI.getRegBankOrNull(MO.getReg()); | ||||||
5997 | // If for some reason we don't have a regbank yet. Don't try anything. | ||||||
5998 | if (!RB) | ||||||
5999 | break; | ||||||
6000 | |||||||
6001 | if (RB->getID() == AArch64::GPRRegBankID) | ||||||
6002 | HasGPROp = true; | ||||||
6003 | else | ||||||
6004 | HasFPROp = true; | ||||||
6005 | } | ||||||
6006 | // We have heterogenous regbanks, need to fixup. | ||||||
6007 | if (HasGPROp && HasFPROp) | ||||||
6008 | fixupPHIOpBanks(*MI, MRI, RBI); | ||||||
6009 | } | ||||||
6010 | } | ||||||
6011 | |||||||
6012 | namespace llvm { | ||||||
6013 | InstructionSelector * | ||||||
6014 | createAArch64InstructionSelector(const AArch64TargetMachine &TM, | ||||||
6015 | AArch64Subtarget &Subtarget, | ||||||
6016 | AArch64RegisterBankInfo &RBI) { | ||||||
6017 | return new AArch64InstructionSelector(TM, Subtarget, RBI); | ||||||
6018 | } | ||||||
6019 | } |
1 | // <tuple> -*- C++ -*- |
2 | |
3 | // Copyright (C) 2007-2016 Free Software Foundation, Inc. |
4 | // |
5 | // This file is part of the GNU ISO C++ Library. This library is free |
6 | // software; you can redistribute it and/or modify it under the |
7 | // terms of the GNU General Public License as published by the |
8 | // Free Software Foundation; either version 3, or (at your option) |
9 | // any later version. |
10 | |
11 | // This library is distributed in the hope that it will be useful, |
12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of |
13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
14 | // GNU General Public License for more details. |
15 | |
16 | // Under Section 7 of GPL version 3, you are granted additional |
17 | // permissions described in the GCC Runtime Library Exception, version |
18 | // 3.1, as published by the Free Software Foundation. |
19 | |
20 | // You should have received a copy of the GNU General Public License and |
21 | // a copy of the GCC Runtime Library Exception along with this program; |
22 | // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
23 | // <http://www.gnu.org/licenses/>. |
24 | |
25 | /** @file include/tuple |
26 | * This is a Standard C++ Library header. |
27 | */ |
28 | |
29 | #ifndef _GLIBCXX_TUPLE1 |
30 | #define _GLIBCXX_TUPLE1 1 |
31 | |
32 | #pragma GCC system_header |
33 | |
34 | #if __cplusplus201402L < 201103L |
35 | # include <bits/c++0x_warning.h> |
36 | #else |
37 | |
38 | #include <utility> |
39 | #include <array> |
40 | #include <bits/uses_allocator.h> |
41 | |
42 | namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default"))) |
43 | { |
44 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
45 | |
46 | /** |
47 | * @addtogroup utilities |
48 | * @{ |
49 | */ |
50 | |
51 | template<std::size_t _Idx, typename _Head, bool _IsEmptyNotFinal> |
52 | struct _Head_base; |
53 | |
54 | template<std::size_t _Idx, typename _Head> |
55 | struct _Head_base<_Idx, _Head, true> |
56 | : public _Head |
57 | { |
58 | constexpr _Head_base() |
59 | : _Head() { } |
60 | |
61 | constexpr _Head_base(const _Head& __h) |
62 | : _Head(__h) { } |
63 | |
64 | constexpr _Head_base(const _Head_base&) = default; |
65 | constexpr _Head_base(_Head_base&&) = default; |
66 | |
67 | template<typename _UHead> |
68 | constexpr _Head_base(_UHead&& __h) |
69 | : _Head(std::forward<_UHead>(__h)) { } |
70 | |
71 | _Head_base(allocator_arg_t, __uses_alloc0) |
72 | : _Head() { } |
73 | |
74 | template<typename _Alloc> |
75 | _Head_base(allocator_arg_t, __uses_alloc1<_Alloc> __a) |
76 | : _Head(allocator_arg, *__a._M_a) { } |
77 | |
78 | template<typename _Alloc> |
79 | _Head_base(allocator_arg_t, __uses_alloc2<_Alloc> __a) |
80 | : _Head(*__a._M_a) { } |
81 | |
82 | template<typename _UHead> |
83 | _Head_base(__uses_alloc0, _UHead&& __uhead) |
84 | : _Head(std::forward<_UHead>(__uhead)) { } |
85 | |
86 | template<typename _Alloc, typename _UHead> |
87 | _Head_base(__uses_alloc1<_Alloc> __a, _UHead&& __uhead) |
88 | : _Head(allocator_arg, *__a._M_a, std::forward<_UHead>(__uhead)) { } |
89 | |
90 | template<typename _Alloc, typename _UHead> |
91 | _Head_base(__uses_alloc2<_Alloc> __a, _UHead&& __uhead) |
92 | : _Head(std::forward<_UHead>(__uhead), *__a._M_a) { } |
93 | |
94 | static constexpr _Head& |
95 | _M_head(_Head_base& __b) noexcept { return __b; } |
96 | |
97 | static constexpr const _Head& |
98 | _M_head(const _Head_base& __b) noexcept { return __b; } |
99 | }; |
100 | |
101 | template<std::size_t _Idx, typename _Head> |
102 | struct _Head_base<_Idx, _Head, false> |
103 | { |
104 | constexpr _Head_base() |
105 | : _M_head_impl() { } |
106 | |
107 | constexpr _Head_base(const _Head& __h) |
108 | : _M_head_impl(__h) { } |
109 | |
110 | constexpr _Head_base(const _Head_base&) = default; |
111 | constexpr _Head_base(_Head_base&&) = default; |
112 | |
113 | template<typename _UHead> |
114 | constexpr _Head_base(_UHead&& __h) |
115 | : _M_head_impl(std::forward<_UHead>(__h)) { } |
116 | |
117 | _Head_base(allocator_arg_t, __uses_alloc0) |
118 | : _M_head_impl() { } |
119 | |
120 | template<typename _Alloc> |
121 | _Head_base(allocator_arg_t, __uses_alloc1<_Alloc> __a) |
122 | : _M_head_impl(allocator_arg, *__a._M_a) { } |
123 | |
124 | template<typename _Alloc> |
125 | _Head_base(allocator_arg_t, __uses_alloc2<_Alloc> __a) |
126 | : _M_head_impl(*__a._M_a) { } |
127 | |
128 | template<typename _UHead> |
129 | _Head_base(__uses_alloc0, _UHead&& __uhead) |
130 | : _M_head_impl(std::forward<_UHead>(__uhead)) { } |
131 | |
132 | template<typename _Alloc, typename _UHead> |
133 | _Head_base(__uses_alloc1<_Alloc> __a, _UHead&& __uhead) |
134 | : _M_head_impl(allocator_arg, *__a._M_a, std::forward<_UHead>(__uhead)) |
135 | { } |
136 | |
137 | template<typename _Alloc, typename _UHead> |
138 | _Head_base(__uses_alloc2<_Alloc> __a, _UHead&& __uhead) |
139 | : _M_head_impl(std::forward<_UHead>(__uhead), *__a._M_a) { } |
140 | |
141 | static constexpr _Head& |
142 | _M_head(_Head_base& __b) noexcept { return __b._M_head_impl; } |
143 | |
144 | static constexpr const _Head& |
145 | _M_head(const _Head_base& __b) noexcept { return __b._M_head_impl; } |
146 | |
147 | _Head _M_head_impl; |
148 | }; |
149 | |
150 | /** |
151 | * Contains the actual implementation of the @c tuple template, stored |
152 | * as a recursive inheritance hierarchy from the first element (most |
153 | * derived class) to the last (least derived class). The @c Idx |
154 | * parameter gives the 0-based index of the element stored at this |
155 | * point in the hierarchy; we use it to implement a constant-time |
156 | * get() operation. |
157 | */ |
158 | template<std::size_t _Idx, typename... _Elements> |
159 | struct _Tuple_impl; |
160 | |
161 | template<typename _Tp> |
162 | struct __is_empty_non_tuple : is_empty<_Tp> { }; |
163 | |
164 | // Using EBO for elements that are tuples causes ambiguous base errors. |
165 | template<typename _El0, typename... _El> |
166 | struct __is_empty_non_tuple<tuple<_El0, _El...>> : false_type { }; |
167 | |
168 | // Use the Empty Base-class Optimization for empty, non-final types. |
169 | template<typename _Tp> |
170 | using __empty_not_final |
171 | = typename conditional<__is_final(_Tp), false_type, |
172 | __is_empty_non_tuple<_Tp>>::type; |
173 | |
174 | /** |
175 | * Recursive tuple implementation. Here we store the @c Head element |
176 | * and derive from a @c Tuple_impl containing the remaining elements |
177 | * (which contains the @c Tail). |
178 | */ |
179 | template<std::size_t _Idx, typename _Head, typename... _Tail> |
180 | struct _Tuple_impl<_Idx, _Head, _Tail...> |
181 | : public _Tuple_impl<_Idx + 1, _Tail...>, |
182 | private _Head_base<_Idx, _Head, __empty_not_final<_Head>::value> |
183 | { |
184 | template<std::size_t, typename...> friend class _Tuple_impl; |
185 | |
186 | typedef _Tuple_impl<_Idx + 1, _Tail...> _Inherited; |
187 | typedef _Head_base<_Idx, _Head, __empty_not_final<_Head>::value> _Base; |
188 | |
189 | static constexpr _Head& |
190 | _M_head(_Tuple_impl& __t) noexcept { return _Base::_M_head(__t); } |
191 | |
192 | static constexpr const _Head& |
193 | _M_head(const _Tuple_impl& __t) noexcept { return _Base::_M_head(__t); } |
194 | |
195 | static constexpr _Inherited& |
196 | _M_tail(_Tuple_impl& __t) noexcept { return __t; } |
197 | |
198 | static constexpr const _Inherited& |
199 | _M_tail(const _Tuple_impl& __t) noexcept { return __t; } |
200 | |
201 | constexpr _Tuple_impl() |
202 | : _Inherited(), _Base() { } |
203 | |
204 | explicit |
205 | constexpr _Tuple_impl(const _Head& __head, const _Tail&... __tail) |
206 | : _Inherited(__tail...), _Base(__head) { } |
207 | |
208 | template<typename _UHead, typename... _UTail, typename = typename |
209 | enable_if<sizeof...(_Tail) == sizeof...(_UTail)>::type> |
210 | explicit |
211 | constexpr _Tuple_impl(_UHead&& __head, _UTail&&... __tail) |
212 | : _Inherited(std::forward<_UTail>(__tail)...), |
213 | _Base(std::forward<_UHead>(__head)) { } |
214 | |
215 | constexpr _Tuple_impl(const _Tuple_impl&) = default; |
216 | |
217 | constexpr |
218 | _Tuple_impl(_Tuple_impl&& __in) |
219 | noexcept(__and_<is_nothrow_move_constructible<_Head>, |
220 | is_nothrow_move_constructible<_Inherited>>::value) |
221 | : _Inherited(std::move(_M_tail(__in))), |
222 | _Base(std::forward<_Head>(_M_head(__in))) { } |
223 | |
224 | template<typename... _UElements> |
225 | constexpr _Tuple_impl(const _Tuple_impl<_Idx, _UElements...>& __in) |
226 | : _Inherited(_Tuple_impl<_Idx, _UElements...>::_M_tail(__in)), |
227 | _Base(_Tuple_impl<_Idx, _UElements...>::_M_head(__in)) { } |
228 | |
229 | template<typename _UHead, typename... _UTails> |
230 | constexpr _Tuple_impl(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in) |
231 | : _Inherited(std::move |
232 | (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in))), |
233 | _Base(std::forward<_UHead> |
234 | (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in))) { } |
235 | |
236 | template<typename _Alloc> |
237 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a) |
238 | : _Inherited(__tag, __a), |
239 | _Base(__tag, __use_alloc<_Head>(__a)) { } |
240 | |
241 | template<typename _Alloc> |
242 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
243 | const _Head& __head, const _Tail&... __tail) |
244 | : _Inherited(__tag, __a, __tail...), |
245 | _Base(__use_alloc<_Head, _Alloc, _Head>(__a), __head) { } |
246 | |
247 | template<typename _Alloc, typename _UHead, typename... _UTail, |
248 | typename = typename enable_if<sizeof...(_Tail) |
249 | == sizeof...(_UTail)>::type> |
250 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
251 | _UHead&& __head, _UTail&&... __tail) |
252 | : _Inherited(__tag, __a, std::forward<_UTail>(__tail)...), |
253 | _Base(__use_alloc<_Head, _Alloc, _UHead>(__a), |
254 | std::forward<_UHead>(__head)) { } |
255 | |
256 | template<typename _Alloc> |
257 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
258 | const _Tuple_impl& __in) |
259 | : _Inherited(__tag, __a, _M_tail(__in)), |
260 | _Base(__use_alloc<_Head, _Alloc, _Head>(__a), _M_head(__in)) { } |
261 | |
262 | template<typename _Alloc> |
263 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
264 | _Tuple_impl&& __in) |
265 | : _Inherited(__tag, __a, std::move(_M_tail(__in))), |
266 | _Base(__use_alloc<_Head, _Alloc, _Head>(__a), |
267 | std::forward<_Head>(_M_head(__in))) { } |
268 | |
269 | template<typename _Alloc, typename... _UElements> |
270 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
271 | const _Tuple_impl<_Idx, _UElements...>& __in) |
272 | : _Inherited(__tag, __a, |
273 | _Tuple_impl<_Idx, _UElements...>::_M_tail(__in)), |
274 | _Base(__use_alloc<_Head, _Alloc, _Head>(__a), |
275 | _Tuple_impl<_Idx, _UElements...>::_M_head(__in)) { } |
276 | |
277 | template<typename _Alloc, typename _UHead, typename... _UTails> |
278 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
279 | _Tuple_impl<_Idx, _UHead, _UTails...>&& __in) |
280 | : _Inherited(__tag, __a, std::move |
281 | (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in))), |
282 | _Base(__use_alloc<_Head, _Alloc, _UHead>(__a), |
283 | std::forward<_UHead> |
284 | (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in))) { } |
285 | |
286 | _Tuple_impl& |
287 | operator=(const _Tuple_impl& __in) |
288 | { |
289 | _M_head(*this) = _M_head(__in); |
290 | _M_tail(*this) = _M_tail(__in); |
291 | return *this; |
292 | } |
293 | |
294 | _Tuple_impl& |
295 | operator=(_Tuple_impl&& __in) |
296 | noexcept(__and_<is_nothrow_move_assignable<_Head>, |
297 | is_nothrow_move_assignable<_Inherited>>::value) |
298 | { |
299 | _M_head(*this) = std::forward<_Head>(_M_head(__in)); |
300 | _M_tail(*this) = std::move(_M_tail(__in)); |
301 | return *this; |
302 | } |
303 | |
304 | template<typename... _UElements> |
305 | _Tuple_impl& |
306 | operator=(const _Tuple_impl<_Idx, _UElements...>& __in) |
307 | { |
308 | _M_head(*this) = _Tuple_impl<_Idx, _UElements...>::_M_head(__in); |
309 | _M_tail(*this) = _Tuple_impl<_Idx, _UElements...>::_M_tail(__in); |
310 | return *this; |
311 | } |
312 | |
313 | template<typename _UHead, typename... _UTails> |
314 | _Tuple_impl& |
315 | operator=(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in) |
316 | { |
317 | _M_head(*this) = std::forward<_UHead> |
318 | (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in)); |
319 | _M_tail(*this) = std::move |
320 | (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in)); |
321 | return *this; |
322 | } |
323 | |
324 | protected: |
325 | void |
326 | _M_swap(_Tuple_impl& __in) |
327 | noexcept(__is_nothrow_swappable<_Head>::value |
328 | && noexcept(_M_tail(__in)._M_swap(_M_tail(__in)))) |
329 | { |
330 | using std::swap; |
331 | swap(_M_head(*this), _M_head(__in)); |
332 | _Inherited::_M_swap(_M_tail(__in)); |
333 | } |
334 | }; |
335 | |
336 | // Basis case of inheritance recursion. |
337 | template<std::size_t _Idx, typename _Head> |
338 | struct _Tuple_impl<_Idx, _Head> |
339 | : private _Head_base<_Idx, _Head, __empty_not_final<_Head>::value> |
340 | { |
341 | template<std::size_t, typename...> friend class _Tuple_impl; |
342 | |
343 | typedef _Head_base<_Idx, _Head, __empty_not_final<_Head>::value> _Base; |
344 | |
345 | static constexpr _Head& |
346 | _M_head(_Tuple_impl& __t) noexcept { return _Base::_M_head(__t); } |
347 | |
348 | static constexpr const _Head& |
349 | _M_head(const _Tuple_impl& __t) noexcept { return _Base::_M_head(__t); } |
350 | |
351 | constexpr _Tuple_impl() |
352 | : _Base() { } |
353 | |
354 | explicit |
355 | constexpr _Tuple_impl(const _Head& __head) |
356 | : _Base(__head) { } |
357 | |
358 | template<typename _UHead> |
359 | explicit |
360 | constexpr _Tuple_impl(_UHead&& __head) |
361 | : _Base(std::forward<_UHead>(__head)) { } |
362 | |
363 | constexpr _Tuple_impl(const _Tuple_impl&) = default; |
364 | |
365 | constexpr |
366 | _Tuple_impl(_Tuple_impl&& __in) |
367 | noexcept(is_nothrow_move_constructible<_Head>::value) |
368 | : _Base(std::forward<_Head>(_M_head(__in))) { } |
369 | |
370 | template<typename _UHead> |
371 | constexpr _Tuple_impl(const _Tuple_impl<_Idx, _UHead>& __in) |
372 | : _Base(_Tuple_impl<_Idx, _UHead>::_M_head(__in)) { } |
373 | |
374 | template<typename _UHead> |
375 | constexpr _Tuple_impl(_Tuple_impl<_Idx, _UHead>&& __in) |
376 | : _Base(std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in))) |
377 | { } |
378 | |
379 | template<typename _Alloc> |
380 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a) |
381 | : _Base(__tag, __use_alloc<_Head>(__a)) { } |
382 | |
383 | template<typename _Alloc> |
384 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
385 | const _Head& __head) |
386 | : _Base(__use_alloc<_Head, _Alloc, _Head>(__a), __head) { } |
387 | |
388 | template<typename _Alloc, typename _UHead> |
389 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
390 | _UHead&& __head) |
391 | : _Base(__use_alloc<_Head, _Alloc, _UHead>(__a), |
392 | std::forward<_UHead>(__head)) { } |
393 | |
394 | template<typename _Alloc> |
395 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
396 | const _Tuple_impl& __in) |
397 | : _Base(__use_alloc<_Head, _Alloc, _Head>(__a), _M_head(__in)) { } |
398 | |
399 | template<typename _Alloc> |
400 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
401 | _Tuple_impl&& __in) |
402 | : _Base(__use_alloc<_Head, _Alloc, _Head>(__a), |
403 | std::forward<_Head>(_M_head(__in))) { } |
404 | |
405 | template<typename _Alloc, typename _UHead> |
406 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
407 | const _Tuple_impl<_Idx, _UHead>& __in) |
408 | : _Base(__use_alloc<_Head, _Alloc, _Head>(__a), |
409 | _Tuple_impl<_Idx, _UHead>::_M_head(__in)) { } |
410 | |
411 | template<typename _Alloc, typename _UHead> |
412 | _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a, |
413 | _Tuple_impl<_Idx, _UHead>&& __in) |
414 | : _Base(__use_alloc<_Head, _Alloc, _UHead>(__a), |
415 | std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in))) |
416 | { } |
417 | |
418 | _Tuple_impl& |
419 | operator=(const _Tuple_impl& __in) |
420 | { |
421 | _M_head(*this) = _M_head(__in); |
422 | return *this; |
423 | } |
424 | |
425 | _Tuple_impl& |
426 | operator=(_Tuple_impl&& __in) |
427 | noexcept(is_nothrow_move_assignable<_Head>::value) |
428 | { |
429 | _M_head(*this) = std::forward<_Head>(_M_head(__in)); |
430 | return *this; |
431 | } |
432 | |
433 | template<typename _UHead> |
434 | _Tuple_impl& |
435 | operator=(const _Tuple_impl<_Idx, _UHead>& __in) |
436 | { |
437 | _M_head(*this) = _Tuple_impl<_Idx, _UHead>::_M_head(__in); |
438 | return *this; |
439 | } |
440 | |
441 | template<typename _UHead> |
442 | _Tuple_impl& |
443 | operator=(_Tuple_impl<_Idx, _UHead>&& __in) |
444 | { |
445 | _M_head(*this) |
446 | = std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in)); |
447 | return *this; |
448 | } |
449 | |
450 | protected: |
451 | void |
452 | _M_swap(_Tuple_impl& __in) |
453 | noexcept(__is_nothrow_swappable<_Head>::value) |
454 | { |
455 | using std::swap; |
456 | swap(_M_head(*this), _M_head(__in)); |
457 | } |
458 | }; |
459 | |
460 | template<typename... _Elements> |
461 | class tuple; |
462 | |
463 | // Concept utility functions, reused in conditionally-explicit |
464 | // constructors. |
465 | template<bool, typename... _Elements> |
466 | struct _TC |
467 | { |
468 | template<typename... _UElements> |
469 | static constexpr bool _ConstructibleTuple() |
470 | { |
471 | return __and_<is_constructible<_Elements, const _UElements&>...>::value; |
472 | } |
473 | |
474 | template<typename... _UElements> |
475 | static constexpr bool _ImplicitlyConvertibleTuple() |
476 | { |
477 | return __and_<is_convertible<const _UElements&, _Elements>...>::value; |
478 | } |
479 | |
480 | template<typename... _UElements> |
481 | static constexpr bool _MoveConstructibleTuple() |
482 | { |
483 | return __and_<is_constructible<_Elements, _UElements&&>...>::value; |
484 | } |
485 | |
486 | template<typename... _UElements> |
487 | static constexpr bool _ImplicitlyMoveConvertibleTuple() |
488 | { |
489 | return __and_<is_convertible<_UElements&&, _Elements>...>::value; |
490 | } |
491 | |
492 | template<typename _SrcTuple> |
493 | static constexpr bool _NonNestedTuple() |
494 | { |
495 | return __and_<__not_<is_same<tuple<_Elements...>, |
496 | typename remove_cv< |
497 | typename remove_reference<_SrcTuple>::type |
498 | >::type>>, |
499 | __not_<is_convertible<_SrcTuple, _Elements...>>, |
500 | __not_<is_constructible<_Elements..., _SrcTuple>> |
501 | >::value; |
502 | } |
503 | template<typename... _UElements> |
504 | static constexpr bool _NotSameTuple() |
505 | { |
506 | return __not_<is_same<tuple<_Elements...>, |
507 | typename remove_const< |
508 | typename remove_reference<_UElements...>::type |
509 | >::type>>::value; |
510 | } |
511 | }; |
512 | |
513 | template<typename... _Elements> |
514 | struct _TC<false, _Elements...> |
515 | { |
516 | template<typename... _UElements> |
517 | static constexpr bool _ConstructibleTuple() |
518 | { |
519 | return false; |
520 | } |
521 | |
522 | template<typename... _UElements> |
523 | static constexpr bool _ImplicitlyConvertibleTuple() |
524 | { |
525 | return false; |
526 | } |
527 | |
528 | template<typename... _UElements> |
529 | static constexpr bool _MoveConstructibleTuple() |
530 | { |
531 | return false; |
532 | } |
533 | |
534 | template<typename... _UElements> |
535 | static constexpr bool _ImplicitlyMoveConvertibleTuple() |
536 | { |
537 | return false; |
538 | } |
539 | |
540 | template<typename... _UElements> |
541 | static constexpr bool _NonNestedTuple() |
542 | { |
543 | return true; |
544 | } |
545 | template<typename... _UElements> |
546 | static constexpr bool _NotSameTuple() |
547 | { |
548 | return true; |
549 | } |
550 | }; |
551 | |
552 | /// Primary class template, tuple |
553 | template<typename... _Elements> |
554 | class tuple : public _Tuple_impl<0, _Elements...> |
555 | { |
556 | typedef _Tuple_impl<0, _Elements...> _Inherited; |
557 | |
558 | // Used for constraining the default constructor so |
559 | // that it becomes dependent on the constraints. |
560 | template<typename _Dummy> |
561 | struct _TC2 |
562 | { |
563 | static constexpr bool _DefaultConstructibleTuple() |
564 | { |
565 | return __and_<is_default_constructible<_Elements>...>::value; |
566 | } |
567 | static constexpr bool _ImplicitlyDefaultConstructibleTuple() |
568 | { |
569 | return __and_<__is_implicitly_default_constructible<_Elements>...> |
570 | ::value; |
571 | } |
572 | }; |
573 | |
574 | public: |
575 | template<typename _Dummy = void, |
576 | typename enable_if<_TC2<_Dummy>:: |
577 | _ImplicitlyDefaultConstructibleTuple(), |
578 | bool>::type = true> |
579 | constexpr tuple() |
580 | : _Inherited() { } |
581 | |
582 | template<typename _Dummy = void, |
583 | typename enable_if<_TC2<_Dummy>:: |
584 | _DefaultConstructibleTuple() |
585 | && |
586 | !_TC2<_Dummy>:: |
587 | _ImplicitlyDefaultConstructibleTuple(), |
588 | bool>::type = false> |
589 | explicit constexpr tuple() |
590 | : _Inherited() { } |
591 | |
592 | // Shortcut for the cases where constructors taking _Elements... |
593 | // need to be constrained. |
594 | template<typename _Dummy> using _TCC = |
595 | _TC<is_same<_Dummy, void>::value, |
596 | _Elements...>; |
597 | |
598 | template<typename _Dummy = void, |
599 | typename enable_if< |
600 | _TCC<_Dummy>::template |
601 | _ConstructibleTuple<_Elements...>() |
602 | && _TCC<_Dummy>::template |
603 | _ImplicitlyConvertibleTuple<_Elements...>() |
604 | && (sizeof...(_Elements) >= 1), |
605 | bool>::type=true> |
606 | constexpr tuple(const _Elements&... __elements) |
607 | : _Inherited(__elements...) { } |
608 | |
609 | template<typename _Dummy = void, |
610 | typename enable_if< |
611 | _TCC<_Dummy>::template |
612 | _ConstructibleTuple<_Elements...>() |
613 | && !_TCC<_Dummy>::template |
614 | _ImplicitlyConvertibleTuple<_Elements...>() |
615 | && (sizeof...(_Elements) >= 1), |
616 | bool>::type=false> |
617 | explicit constexpr tuple(const _Elements&... __elements) |
618 | : _Inherited(__elements...) { } |
619 | |
620 | // Shortcut for the cases where constructors taking _UElements... |
621 | // need to be constrained. |
622 | template<typename... _UElements> using _TMC = |
623 | _TC<(sizeof...(_Elements) == sizeof...(_UElements)), |
624 | _Elements...>; |
625 | |
626 | template<typename... _UElements, typename |
627 | enable_if< |
628 | _TC<sizeof...(_UElements) == 1, _Elements...>::template |
629 | _NotSameTuple<_UElements...>() |
630 | && _TMC<_UElements...>::template |
631 | _MoveConstructibleTuple<_UElements...>() |
632 | && _TMC<_UElements...>::template |
633 | _ImplicitlyMoveConvertibleTuple<_UElements...>() |
634 | && (sizeof...(_Elements) >= 1), |
635 | bool>::type=true> |
636 | constexpr tuple(_UElements&&... __elements) |
637 | : _Inherited(std::forward<_UElements>(__elements)...) { } |
638 | |
639 | template<typename... _UElements, typename |
640 | enable_if< |
641 | _TC<sizeof...(_UElements) == 1, _Elements...>::template |
642 | _NotSameTuple<_UElements...>() |
643 | && _TMC<_UElements...>::template |
644 | _MoveConstructibleTuple<_UElements...>() |
645 | && !_TMC<_UElements...>::template |
646 | _ImplicitlyMoveConvertibleTuple<_UElements...>() |
647 | && (sizeof...(_Elements) >= 1), |
648 | bool>::type=false> |
649 | explicit constexpr tuple(_UElements&&... __elements) |
650 | : _Inherited(std::forward<_UElements>(__elements)...) { } |
651 | |
652 | constexpr tuple(const tuple&) = default; |
653 | |
654 | constexpr tuple(tuple&&) = default; |
655 | |
656 | // Shortcut for the cases where constructors taking tuples |
657 | // must avoid creating temporaries. |
658 | template<typename _Dummy> using _TNTC = |
659 | _TC<is_same<_Dummy, void>::value && sizeof...(_Elements) == 1, |
660 | _Elements...>; |
661 | |
662 | template<typename... _UElements, typename _Dummy = void, typename |
663 | enable_if<_TMC<_UElements...>::template |
664 | _ConstructibleTuple<_UElements...>() |
665 | && _TMC<_UElements...>::template |
666 | _ImplicitlyConvertibleTuple<_UElements...>() |
667 | && _TNTC<_Dummy>::template |
668 | _NonNestedTuple<const tuple<_UElements...>&>(), |
669 | bool>::type=true> |
670 | constexpr tuple(const tuple<_UElements...>& __in) |
671 | : _Inherited(static_cast<const _Tuple_impl<0, _UElements...>&>(__in)) |
672 | { } |
673 | |
674 | template<typename... _UElements, typename _Dummy = void, typename |
675 | enable_if<_TMC<_UElements...>::template |
676 | _ConstructibleTuple<_UElements...>() |
677 | && !_TMC<_UElements...>::template |
678 | _ImplicitlyConvertibleTuple<_UElements...>() |
679 | && _TNTC<_Dummy>::template |
680 | _NonNestedTuple<const tuple<_UElements...>&>(), |
681 | bool>::type=false> |
682 | explicit constexpr tuple(const tuple<_UElements...>& __in) |
683 | : _Inherited(static_cast<const _Tuple_impl<0, _UElements...>&>(__in)) |
684 | { } |
685 | |
686 | template<typename... _UElements, typename _Dummy = void, typename |
687 | enable_if<_TMC<_UElements...>::template |
688 | _MoveConstructibleTuple<_UElements...>() |
689 | && _TMC<_UElements...>::template |
690 | _ImplicitlyMoveConvertibleTuple<_UElements...>() |
691 | && _TNTC<_Dummy>::template |
692 | _NonNestedTuple<tuple<_UElements...>&&>(), |
693 | bool>::type=true> |
694 | constexpr tuple(tuple<_UElements...>&& __in) |
695 | : _Inherited(static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) { } |
696 | |
697 | template<typename... _UElements, typename _Dummy = void, typename |
698 | enable_if<_TMC<_UElements...>::template |
699 | _MoveConstructibleTuple<_UElements...>() |
700 | && !_TMC<_UElements...>::template |
701 | _ImplicitlyMoveConvertibleTuple<_UElements...>() |
702 | && _TNTC<_Dummy>::template |
703 | _NonNestedTuple<tuple<_UElements...>&&>(), |
704 | bool>::type=false> |
705 | explicit constexpr tuple(tuple<_UElements...>&& __in) |
706 | : _Inherited(static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) { } |
707 | |
708 | // Allocator-extended constructors. |
709 | |
710 | template<typename _Alloc> |
711 | tuple(allocator_arg_t __tag, const _Alloc& __a) |
712 | : _Inherited(__tag, __a) { } |
713 | |
714 | template<typename _Alloc, typename _Dummy = void, |
715 | typename enable_if< |
716 | _TCC<_Dummy>::template |
717 | _ConstructibleTuple<_Elements...>() |
718 | && _TCC<_Dummy>::template |
719 | _ImplicitlyConvertibleTuple<_Elements...>(), |
720 | bool>::type=true> |
721 | tuple(allocator_arg_t __tag, const _Alloc& __a, |
722 | const _Elements&... __elements) |
723 | : _Inherited(__tag, __a, __elements...) { } |
724 | |
725 | template<typename _Alloc, typename _Dummy = void, |
726 | typename enable_if< |
727 | _TCC<_Dummy>::template |
728 | _ConstructibleTuple<_Elements...>() |
729 | && !_TCC<_Dummy>::template |
730 | _ImplicitlyConvertibleTuple<_Elements...>(), |
731 | bool>::type=false> |
732 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
733 | const _Elements&... __elements) |
734 | : _Inherited(__tag, __a, __elements...) { } |
735 | |
736 | template<typename _Alloc, typename... _UElements, typename |
737 | enable_if<_TMC<_UElements...>::template |
738 | _MoveConstructibleTuple<_UElements...>() |
739 | && _TMC<_UElements...>::template |
740 | _ImplicitlyMoveConvertibleTuple<_UElements...>(), |
741 | bool>::type=true> |
742 | tuple(allocator_arg_t __tag, const _Alloc& __a, |
743 | _UElements&&... __elements) |
744 | : _Inherited(__tag, __a, std::forward<_UElements>(__elements)...) |
745 | { } |
746 | |
747 | template<typename _Alloc, typename... _UElements, typename |
748 | enable_if<_TMC<_UElements...>::template |
749 | _MoveConstructibleTuple<_UElements...>() |
750 | && !_TMC<_UElements...>::template |
751 | _ImplicitlyMoveConvertibleTuple<_UElements...>(), |
752 | bool>::type=false> |
753 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
754 | _UElements&&... __elements) |
755 | : _Inherited(__tag, __a, std::forward<_UElements>(__elements)...) |
756 | { } |
757 | |
758 | template<typename _Alloc> |
759 | tuple(allocator_arg_t __tag, const _Alloc& __a, const tuple& __in) |
760 | : _Inherited(__tag, __a, static_cast<const _Inherited&>(__in)) { } |
761 | |
762 | template<typename _Alloc> |
763 | tuple(allocator_arg_t __tag, const _Alloc& __a, tuple&& __in) |
764 | : _Inherited(__tag, __a, static_cast<_Inherited&&>(__in)) { } |
765 | |
766 | template<typename _Alloc, typename... _UElements, typename |
767 | enable_if<_TMC<_UElements...>::template |
768 | _ConstructibleTuple<_UElements...>() |
769 | && _TMC<_UElements...>::template |
770 | _ImplicitlyConvertibleTuple<_UElements...>(), |
771 | bool>::type=true> |
772 | tuple(allocator_arg_t __tag, const _Alloc& __a, |
773 | const tuple<_UElements...>& __in) |
774 | : _Inherited(__tag, __a, |
775 | static_cast<const _Tuple_impl<0, _UElements...>&>(__in)) |
776 | { } |
777 | |
778 | template<typename _Alloc, typename... _UElements, typename |
779 | enable_if<_TMC<_UElements...>::template |
780 | _ConstructibleTuple<_UElements...>() |
781 | && !_TMC<_UElements...>::template |
782 | _ImplicitlyConvertibleTuple<_UElements...>(), |
783 | bool>::type=false> |
784 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
785 | const tuple<_UElements...>& __in) |
786 | : _Inherited(__tag, __a, |
787 | static_cast<const _Tuple_impl<0, _UElements...>&>(__in)) |
788 | { } |
789 | |
790 | template<typename _Alloc, typename... _UElements, typename |
791 | enable_if<_TMC<_UElements...>::template |
792 | _MoveConstructibleTuple<_UElements...>() |
793 | && _TMC<_UElements...>::template |
794 | _ImplicitlyMoveConvertibleTuple<_UElements...>(), |
795 | bool>::type=true> |
796 | tuple(allocator_arg_t __tag, const _Alloc& __a, |
797 | tuple<_UElements...>&& __in) |
798 | : _Inherited(__tag, __a, |
799 | static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) |
800 | { } |
801 | |
802 | template<typename _Alloc, typename... _UElements, typename |
803 | enable_if<_TMC<_UElements...>::template |
804 | _MoveConstructibleTuple<_UElements...>() |
805 | && !_TMC<_UElements...>::template |
806 | _ImplicitlyMoveConvertibleTuple<_UElements...>(), |
807 | bool>::type=false> |
808 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
809 | tuple<_UElements...>&& __in) |
810 | : _Inherited(__tag, __a, |
811 | static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) |
812 | { } |
813 | |
814 | tuple& |
815 | operator=(const tuple& __in) |
816 | { |
817 | static_cast<_Inherited&>(*this) = __in; |
818 | return *this; |
819 | } |
820 | |
821 | tuple& |
822 | operator=(tuple&& __in) |
823 | noexcept(is_nothrow_move_assignable<_Inherited>::value) |
824 | { |
825 | static_cast<_Inherited&>(*this) = std::move(__in); |
826 | return *this; |
827 | } |
828 | |
829 | template<typename... _UElements, typename = typename |
830 | enable_if<sizeof...(_UElements) |
831 | == sizeof...(_Elements)>::type> |
832 | tuple& |
833 | operator=(const tuple<_UElements...>& __in) |
834 | { |
835 | static_cast<_Inherited&>(*this) = __in; |
836 | return *this; |
837 | } |
838 | |
839 | template<typename... _UElements, typename = typename |
840 | enable_if<sizeof...(_UElements) |
841 | == sizeof...(_Elements)>::type> |
842 | tuple& |
843 | operator=(tuple<_UElements...>&& __in) |
844 | { |
845 | static_cast<_Inherited&>(*this) = std::move(__in); |
846 | return *this; |
847 | } |
848 | |
849 | void |
850 | swap(tuple& __in) |
851 | noexcept(noexcept(__in._M_swap(__in))) |
852 | { _Inherited::_M_swap(__in); } |
853 | }; |
854 | |
855 | // Explicit specialization, zero-element tuple. |
856 | template<> |
857 | class tuple<> |
858 | { |
859 | public: |
860 | void swap(tuple&) noexcept { /* no-op */ } |
861 | }; |
862 | |
863 | /// Partial specialization, 2-element tuple. |
864 | /// Includes construction and assignment from a pair. |
865 | template<typename _T1, typename _T2> |
866 | class tuple<_T1, _T2> : public _Tuple_impl<0, _T1, _T2> |
867 | { |
868 | typedef _Tuple_impl<0, _T1, _T2> _Inherited; |
869 | |
870 | public: |
871 | template <typename _U1 = _T1, |
872 | typename _U2 = _T2, |
873 | typename enable_if<__and_< |
874 | __is_implicitly_default_constructible<_U1>, |
875 | __is_implicitly_default_constructible<_U2>> |
876 | ::value, bool>::type = true> |
877 | |
878 | constexpr tuple() |
879 | : _Inherited() { } |
880 | |
881 | template <typename _U1 = _T1, |
882 | typename _U2 = _T2, |
883 | typename enable_if< |
884 | __and_< |
885 | is_default_constructible<_U1>, |
886 | is_default_constructible<_U2>, |
887 | __not_< |
888 | __and_<__is_implicitly_default_constructible<_U1>, |
889 | __is_implicitly_default_constructible<_U2>>>> |
890 | ::value, bool>::type = false> |
891 | |
892 | explicit constexpr tuple() |
893 | : _Inherited() { } |
894 | |
895 | // Shortcut for the cases where constructors taking _T1, _T2 |
896 | // need to be constrained. |
897 | template<typename _Dummy> using _TCC = |
898 | _TC<is_same<_Dummy, void>::value, _T1, _T2>; |
899 | |
900 | template<typename _Dummy = void, typename |
901 | enable_if<_TCC<_Dummy>::template |
902 | _ConstructibleTuple<_T1, _T2>() |
903 | && _TCC<_Dummy>::template |
904 | _ImplicitlyConvertibleTuple<_T1, _T2>(), |
905 | bool>::type = true> |
906 | constexpr tuple(const _T1& __a1, const _T2& __a2) |
907 | : _Inherited(__a1, __a2) { } |
908 | |
909 | template<typename _Dummy = void, typename |
910 | enable_if<_TCC<_Dummy>::template |
911 | _ConstructibleTuple<_T1, _T2>() |
912 | && !_TCC<_Dummy>::template |
913 | _ImplicitlyConvertibleTuple<_T1, _T2>(), |
914 | bool>::type = false> |
915 | explicit constexpr tuple(const _T1& __a1, const _T2& __a2) |
916 | : _Inherited(__a1, __a2) { } |
917 | |
918 | // Shortcut for the cases where constructors taking _U1, _U2 |
919 | // need to be constrained. |
920 | using _TMC = _TC<true, _T1, _T2>; |
921 | |
922 | template<typename _U1, typename _U2, typename |
923 | enable_if<_TMC::template |
924 | _MoveConstructibleTuple<_U1, _U2>() |
925 | && _TMC::template |
926 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>() |
927 | && !is_same<typename decay<_U1>::type, |
928 | allocator_arg_t>::value, |
929 | bool>::type = true> |
930 | constexpr tuple(_U1&& __a1, _U2&& __a2) |
931 | : _Inherited(std::forward<_U1>(__a1), std::forward<_U2>(__a2)) { } |
932 | |
933 | template<typename _U1, typename _U2, typename |
934 | enable_if<_TMC::template |
935 | _MoveConstructibleTuple<_U1, _U2>() |
936 | && !_TMC::template |
937 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>() |
938 | && !is_same<typename decay<_U1>::type, |
939 | allocator_arg_t>::value, |
940 | bool>::type = false> |
941 | explicit constexpr tuple(_U1&& __a1, _U2&& __a2) |
942 | : _Inherited(std::forward<_U1>(__a1), std::forward<_U2>(__a2)) { } |
943 | |
944 | constexpr tuple(const tuple&) = default; |
945 | |
946 | constexpr tuple(tuple&&) = default; |
947 | |
948 | template<typename _U1, typename _U2, typename |
949 | enable_if<_TMC::template |
950 | _ConstructibleTuple<_U1, _U2>() |
951 | && _TMC::template |
952 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
953 | bool>::type = true> |
954 | constexpr tuple(const tuple<_U1, _U2>& __in) |
955 | : _Inherited(static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) { } |
956 | |
957 | template<typename _U1, typename _U2, typename |
958 | enable_if<_TMC::template |
959 | _ConstructibleTuple<_U1, _U2>() |
960 | && !_TMC::template |
961 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
962 | bool>::type = false> |
963 | explicit constexpr tuple(const tuple<_U1, _U2>& __in) |
964 | : _Inherited(static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) { } |
965 | |
966 | template<typename _U1, typename _U2, typename |
967 | enable_if<_TMC::template |
968 | _MoveConstructibleTuple<_U1, _U2>() |
969 | && _TMC::template |
970 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
971 | bool>::type = true> |
972 | constexpr tuple(tuple<_U1, _U2>&& __in) |
973 | : _Inherited(static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) { } |
974 | |
975 | template<typename _U1, typename _U2, typename |
976 | enable_if<_TMC::template |
977 | _MoveConstructibleTuple<_U1, _U2>() |
978 | && !_TMC::template |
979 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
980 | bool>::type = false> |
981 | explicit constexpr tuple(tuple<_U1, _U2>&& __in) |
982 | : _Inherited(static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) { } |
983 | |
984 | template<typename _U1, typename _U2, typename |
985 | enable_if<_TMC::template |
986 | _ConstructibleTuple<_U1, _U2>() |
987 | && _TMC::template |
988 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
989 | bool>::type = true> |
990 | constexpr tuple(const pair<_U1, _U2>& __in) |
991 | : _Inherited(__in.first, __in.second) { } |
992 | |
993 | template<typename _U1, typename _U2, typename |
994 | enable_if<_TMC::template |
995 | _ConstructibleTuple<_U1, _U2>() |
996 | && !_TMC::template |
997 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
998 | bool>::type = false> |
999 | explicit constexpr tuple(const pair<_U1, _U2>& __in) |
1000 | : _Inherited(__in.first, __in.second) { } |
1001 | |
1002 | template<typename _U1, typename _U2, typename |
1003 | enable_if<_TMC::template |
1004 | _MoveConstructibleTuple<_U1, _U2>() |
1005 | && _TMC::template |
1006 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1007 | bool>::type = true> |
1008 | constexpr tuple(pair<_U1, _U2>&& __in) |
1009 | : _Inherited(std::forward<_U1>(__in.first), |
1010 | std::forward<_U2>(__in.second)) { } |
1011 | |
1012 | template<typename _U1, typename _U2, typename |
1013 | enable_if<_TMC::template |
1014 | _MoveConstructibleTuple<_U1, _U2>() |
1015 | && !_TMC::template |
1016 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1017 | bool>::type = false> |
1018 | explicit constexpr tuple(pair<_U1, _U2>&& __in) |
1019 | : _Inherited(std::forward<_U1>(__in.first), |
1020 | std::forward<_U2>(__in.second)) { } |
1021 | |
1022 | // Allocator-extended constructors. |
1023 | |
1024 | template<typename _Alloc> |
1025 | tuple(allocator_arg_t __tag, const _Alloc& __a) |
1026 | : _Inherited(__tag, __a) { } |
1027 | |
1028 | template<typename _Alloc, typename _Dummy = void, |
1029 | typename enable_if< |
1030 | _TCC<_Dummy>::template |
1031 | _ConstructibleTuple<_T1, _T2>() |
1032 | && _TCC<_Dummy>::template |
1033 | _ImplicitlyConvertibleTuple<_T1, _T2>(), |
1034 | bool>::type=true> |
1035 | |
1036 | tuple(allocator_arg_t __tag, const _Alloc& __a, |
1037 | const _T1& __a1, const _T2& __a2) |
1038 | : _Inherited(__tag, __a, __a1, __a2) { } |
1039 | |
1040 | template<typename _Alloc, typename _Dummy = void, |
1041 | typename enable_if< |
1042 | _TCC<_Dummy>::template |
1043 | _ConstructibleTuple<_T1, _T2>() |
1044 | && !_TCC<_Dummy>::template |
1045 | _ImplicitlyConvertibleTuple<_T1, _T2>(), |
1046 | bool>::type=false> |
1047 | |
1048 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
1049 | const _T1& __a1, const _T2& __a2) |
1050 | : _Inherited(__tag, __a, __a1, __a2) { } |
1051 | |
1052 | template<typename _Alloc, typename _U1, typename _U2, typename |
1053 | enable_if<_TMC::template |
1054 | _MoveConstructibleTuple<_U1, _U2>() |
1055 | && _TMC::template |
1056 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1057 | bool>::type = true> |
1058 | tuple(allocator_arg_t __tag, const _Alloc& __a, _U1&& __a1, _U2&& __a2) |
1059 | : _Inherited(__tag, __a, std::forward<_U1>(__a1), |
1060 | std::forward<_U2>(__a2)) { } |
1061 | |
1062 | template<typename _Alloc, typename _U1, typename _U2, typename |
1063 | enable_if<_TMC::template |
1064 | _MoveConstructibleTuple<_U1, _U2>() |
1065 | && !_TMC::template |
1066 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1067 | bool>::type = false> |
1068 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
1069 | _U1&& __a1, _U2&& __a2) |
1070 | : _Inherited(__tag, __a, std::forward<_U1>(__a1), |
1071 | std::forward<_U2>(__a2)) { } |
1072 | |
1073 | template<typename _Alloc> |
1074 | tuple(allocator_arg_t __tag, const _Alloc& __a, const tuple& __in) |
1075 | : _Inherited(__tag, __a, static_cast<const _Inherited&>(__in)) { } |
1076 | |
1077 | template<typename _Alloc> |
1078 | tuple(allocator_arg_t __tag, const _Alloc& __a, tuple&& __in) |
1079 | : _Inherited(__tag, __a, static_cast<_Inherited&&>(__in)) { } |
1080 | |
1081 | template<typename _Alloc, typename _U1, typename _U2, typename |
1082 | enable_if<_TMC::template |
1083 | _ConstructibleTuple<_U1, _U2>() |
1084 | && _TMC::template |
1085 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
1086 | bool>::type = true> |
1087 | tuple(allocator_arg_t __tag, const _Alloc& __a, |
1088 | const tuple<_U1, _U2>& __in) |
1089 | : _Inherited(__tag, __a, |
1090 | static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) |
1091 | { } |
1092 | |
1093 | template<typename _Alloc, typename _U1, typename _U2, typename |
1094 | enable_if<_TMC::template |
1095 | _ConstructibleTuple<_U1, _U2>() |
1096 | && !_TMC::template |
1097 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
1098 | bool>::type = false> |
1099 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
1100 | const tuple<_U1, _U2>& __in) |
1101 | : _Inherited(__tag, __a, |
1102 | static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) |
1103 | { } |
1104 | |
1105 | template<typename _Alloc, typename _U1, typename _U2, typename |
1106 | enable_if<_TMC::template |
1107 | _MoveConstructibleTuple<_U1, _U2>() |
1108 | && _TMC::template |
1109 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1110 | bool>::type = true> |
1111 | tuple(allocator_arg_t __tag, const _Alloc& __a, tuple<_U1, _U2>&& __in) |
1112 | : _Inherited(__tag, __a, static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) |
1113 | { } |
1114 | |
1115 | template<typename _Alloc, typename _U1, typename _U2, typename |
1116 | enable_if<_TMC::template |
1117 | _MoveConstructibleTuple<_U1, _U2>() |
1118 | && !_TMC::template |
1119 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1120 | bool>::type = false> |
1121 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
1122 | tuple<_U1, _U2>&& __in) |
1123 | : _Inherited(__tag, __a, static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) |
1124 | { } |
1125 | |
1126 | template<typename _Alloc, typename _U1, typename _U2, typename |
1127 | enable_if<_TMC::template |
1128 | _ConstructibleTuple<_U1, _U2>() |
1129 | && _TMC::template |
1130 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
1131 | bool>::type = true> |
1132 | tuple(allocator_arg_t __tag, const _Alloc& __a, |
1133 | const pair<_U1, _U2>& __in) |
1134 | : _Inherited(__tag, __a, __in.first, __in.second) { } |
1135 | |
1136 | template<typename _Alloc, typename _U1, typename _U2, typename |
1137 | enable_if<_TMC::template |
1138 | _ConstructibleTuple<_U1, _U2>() |
1139 | && !_TMC::template |
1140 | _ImplicitlyConvertibleTuple<_U1, _U2>(), |
1141 | bool>::type = false> |
1142 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
1143 | const pair<_U1, _U2>& __in) |
1144 | : _Inherited(__tag, __a, __in.first, __in.second) { } |
1145 | |
1146 | template<typename _Alloc, typename _U1, typename _U2, typename |
1147 | enable_if<_TMC::template |
1148 | _MoveConstructibleTuple<_U1, _U2>() |
1149 | && _TMC::template |
1150 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1151 | bool>::type = true> |
1152 | tuple(allocator_arg_t __tag, const _Alloc& __a, pair<_U1, _U2>&& __in) |
1153 | : _Inherited(__tag, __a, std::forward<_U1>(__in.first), |
1154 | std::forward<_U2>(__in.second)) { } |
1155 | |
1156 | template<typename _Alloc, typename _U1, typename _U2, typename |
1157 | enable_if<_TMC::template |
1158 | _MoveConstructibleTuple<_U1, _U2>() |
1159 | && !_TMC::template |
1160 | _ImplicitlyMoveConvertibleTuple<_U1, _U2>(), |
1161 | bool>::type = false> |
1162 | explicit tuple(allocator_arg_t __tag, const _Alloc& __a, |
1163 | pair<_U1, _U2>&& __in) |
1164 | : _Inherited(__tag, __a, std::forward<_U1>(__in.first), |
1165 | std::forward<_U2>(__in.second)) { } |
1166 | |
1167 | tuple& |
1168 | operator=(const tuple& __in) |
1169 | { |
1170 | static_cast<_Inherited&>(*this) = __in; |
1171 | return *this; |
1172 | } |
1173 | |
1174 | tuple& |
1175 | operator=(tuple&& __in) |
1176 | noexcept(is_nothrow_move_assignable<_Inherited>::value) |
1177 | { |
1178 | static_cast<_Inherited&>(*this) = std::move(__in); |
1179 | return *this; |
1180 | } |
1181 | |
1182 | template<typename _U1, typename _U2> |
1183 | tuple& |
1184 | operator=(const tuple<_U1, _U2>& __in) |
1185 | { |
1186 | static_cast<_Inherited&>(*this) = __in; |
1187 | return *this; |
1188 | } |
1189 | |
1190 | template<typename _U1, typename _U2> |
1191 | tuple& |
1192 | operator=(tuple<_U1, _U2>&& __in) |
1193 | { |
1194 | static_cast<_Inherited&>(*this) = std::move(__in); |
1195 | return *this; |
1196 | } |
1197 | |
1198 | template<typename _U1, typename _U2> |
1199 | tuple& |
1200 | operator=(const pair<_U1, _U2>& __in) |
1201 | { |
1202 | this->_M_head(*this) = __in.first; |
1203 | this->_M_tail(*this)._M_head(*this) = __in.second; |
1204 | return *this; |
1205 | } |
1206 | |
1207 | template<typename _U1, typename _U2> |
1208 | tuple& |
1209 | operator=(pair<_U1, _U2>&& __in) |
1210 | { |
1211 | this->_M_head(*this) = std::forward<_U1>(__in.first); |
1212 | this->_M_tail(*this)._M_head(*this) = std::forward<_U2>(__in.second); |
1213 | return *this; |
1214 | } |
1215 | |
1216 | void |
1217 | swap(tuple& __in) |
1218 | noexcept(noexcept(__in._M_swap(__in))) |
1219 | { _Inherited::_M_swap(__in); } |
1220 | }; |
1221 | |
1222 | |
1223 | /** |
1224 | * Recursive case for tuple_element: strip off the first element in |
1225 | * the tuple and retrieve the (i-1)th element of the remaining tuple. |
1226 | */ |
1227 | template<std::size_t __i, typename _Head, typename... _Tail> |
1228 | struct tuple_element<__i, tuple<_Head, _Tail...> > |
1229 | : tuple_element<__i - 1, tuple<_Tail...> > { }; |
1230 | |
1231 | /** |
1232 | * Basis case for tuple_element: The first element is the one we're seeking. |
1233 | */ |
1234 | template<typename _Head, typename... _Tail> |
1235 | struct tuple_element<0, tuple<_Head, _Tail...> > |
1236 | { |
1237 | typedef _Head type; |
1238 | }; |
1239 | |
1240 | /// class tuple_size |
1241 | template<typename... _Elements> |
1242 | struct tuple_size<tuple<_Elements...>> |
1243 | : public integral_constant<std::size_t, sizeof...(_Elements)> { }; |
1244 | |
1245 | template<std::size_t __i, typename _Head, typename... _Tail> |
1246 | constexpr _Head& |
1247 | __get_helper(_Tuple_impl<__i, _Head, _Tail...>& __t) noexcept |
1248 | { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); } |
1249 | |
1250 | template<std::size_t __i, typename _Head, typename... _Tail> |
1251 | constexpr const _Head& |
1252 | __get_helper(const _Tuple_impl<__i, _Head, _Tail...>& __t) noexcept |
1253 | { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); } |
1254 | |
1255 | /// Return a reference to the ith element of a tuple. |
1256 | template<std::size_t __i, typename... _Elements> |
1257 | constexpr __tuple_element_t<__i, tuple<_Elements...>>& |
1258 | get(tuple<_Elements...>& __t) noexcept |
1259 | { return std::__get_helper<__i>(__t); } |
1260 | |
1261 | /// Return a const reference to the ith element of a const tuple. |
1262 | template<std::size_t __i, typename... _Elements> |
1263 | constexpr const __tuple_element_t<__i, tuple<_Elements...>>& |
1264 | get(const tuple<_Elements...>& __t) noexcept |
1265 | { return std::__get_helper<__i>(__t); } |
1266 | |
1267 | /// Return an rvalue reference to the ith element of a tuple rvalue. |
1268 | template<std::size_t __i, typename... _Elements> |
1269 | constexpr __tuple_element_t<__i, tuple<_Elements...>>&& |
1270 | get(tuple<_Elements...>&& __t) noexcept |
1271 | { |
1272 | typedef __tuple_element_t<__i, tuple<_Elements...>> __element_type; |
1273 | return std::forward<__element_type&&>(std::get<__i>(__t)); |
1274 | } |
1275 | |
1276 | #if __cplusplus201402L > 201103L |
1277 | |
1278 | #define __cpp_lib_tuples_by_type201304 201304 |
1279 | |
1280 | template<typename _Head, size_t __i, typename... _Tail> |
1281 | constexpr _Head& |
1282 | __get_helper2(_Tuple_impl<__i, _Head, _Tail...>& __t) noexcept |
1283 | { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); } |
1284 | |
1285 | template<typename _Head, size_t __i, typename... _Tail> |
1286 | constexpr const _Head& |
1287 | __get_helper2(const _Tuple_impl<__i, _Head, _Tail...>& __t) noexcept |
1288 | { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); } |
1289 | |
1290 | /// Return a reference to the unique element of type _Tp of a tuple. |
1291 | template <typename _Tp, typename... _Types> |
1292 | constexpr _Tp& |
1293 | get(tuple<_Types...>& __t) noexcept |
1294 | { return std::__get_helper2<_Tp>(__t); } |
1295 | |
1296 | /// Return a reference to the unique element of type _Tp of a tuple rvalue. |
1297 | template <typename _Tp, typename... _Types> |
1298 | constexpr _Tp&& |
1299 | get(tuple<_Types...>&& __t) noexcept |
1300 | { return std::forward<_Tp&&>(std::__get_helper2<_Tp>(__t)); } |
1301 | |
1302 | /// Return a const reference to the unique element of type _Tp of a tuple. |
1303 | template <typename _Tp, typename... _Types> |
1304 | constexpr const _Tp& |
1305 | get(const tuple<_Types...>& __t) noexcept |
1306 | { return std::__get_helper2<_Tp>(__t); } |
1307 | #endif |
1308 | |
1309 | // This class performs the comparison operations on tuples |
1310 | template<typename _Tp, typename _Up, size_t __i, size_t __size> |
1311 | struct __tuple_compare |
1312 | { |
1313 | static constexpr bool |
1314 | __eq(const _Tp& __t, const _Up& __u) |
1315 | { |
1316 | return bool(std::get<__i>(__t) == std::get<__i>(__u)) |
1317 | && __tuple_compare<_Tp, _Up, __i + 1, __size>::__eq(__t, __u); |
1318 | } |
1319 | |
1320 | static constexpr bool |
1321 | __less(const _Tp& __t, const _Up& __u) |
1322 | { |
1323 | return bool(std::get<__i>(__t) < std::get<__i>(__u)) |
1324 | || (!bool(std::get<__i>(__u) < std::get<__i>(__t)) |
1325 | && __tuple_compare<_Tp, _Up, __i + 1, __size>::__less(__t, __u)); |
1326 | } |
1327 | }; |
1328 | |
1329 | template<typename _Tp, typename _Up, size_t __size> |
1330 | struct __tuple_compare<_Tp, _Up, __size, __size> |
1331 | { |
1332 | static constexpr bool |
1333 | __eq(const _Tp&, const _Up&) { return true; } |
1334 | |
1335 | static constexpr bool |
1336 | __less(const _Tp&, const _Up&) { return false; } |
1337 | }; |
1338 | |
1339 | template<typename... _TElements, typename... _UElements> |
1340 | constexpr bool |
1341 | operator==(const tuple<_TElements...>& __t, |
1342 | const tuple<_UElements...>& __u) |
1343 | { |
1344 | static_assert(sizeof...(_TElements) == sizeof...(_UElements), |
1345 | "tuple objects can only be compared if they have equal sizes."); |
1346 | using __compare = __tuple_compare<tuple<_TElements...>, |
1347 | tuple<_UElements...>, |
1348 | 0, sizeof...(_TElements)>; |
1349 | return __compare::__eq(__t, __u); |
1350 | } |
1351 | |
1352 | template<typename... _TElements, typename... _UElements> |
1353 | constexpr bool |
1354 | operator<(const tuple<_TElements...>& __t, |
1355 | const tuple<_UElements...>& __u) |
1356 | { |
1357 | static_assert(sizeof...(_TElements) == sizeof...(_UElements), |
1358 | "tuple objects can only be compared if they have equal sizes."); |
1359 | using __compare = __tuple_compare<tuple<_TElements...>, |
1360 | tuple<_UElements...>, |
1361 | 0, sizeof...(_TElements)>; |
1362 | return __compare::__less(__t, __u); |
1363 | } |
1364 | |
1365 | template<typename... _TElements, typename... _UElements> |
1366 | constexpr bool |
1367 | operator!=(const tuple<_TElements...>& __t, |
1368 | const tuple<_UElements...>& __u) |
1369 | { return !(__t == __u); } |
1370 | |
1371 | template<typename... _TElements, typename... _UElements> |
1372 | constexpr bool |
1373 | operator>(const tuple<_TElements...>& __t, |
1374 | const tuple<_UElements...>& __u) |
1375 | { return __u < __t; } |
1376 | |
1377 | template<typename... _TElements, typename... _UElements> |
1378 | constexpr bool |
1379 | operator<=(const tuple<_TElements...>& __t, |
1380 | const tuple<_UElements...>& __u) |
1381 | { return !(__u < __t); } |
1382 | |
1383 | template<typename... _TElements, typename... _UElements> |
1384 | constexpr bool |
1385 | operator>=(const tuple<_TElements...>& __t, |
1386 | const tuple<_UElements...>& __u) |
1387 | { return !(__t < __u); } |
1388 | |
1389 | // NB: DR 705. |
1390 | template<typename... _Elements> |
1391 | constexpr tuple<typename __decay_and_strip<_Elements>::__type...> |
1392 | make_tuple(_Elements&&... __args) |
1393 | { |
1394 | typedef tuple<typename __decay_and_strip<_Elements>::__type...> |
1395 | __result_type; |
1396 | return __result_type(std::forward<_Elements>(__args)...); |
1397 | } |
1398 | |
1399 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
1400 | // 2275. Why is forward_as_tuple not constexpr? |
1401 | template<typename... _Elements> |
1402 | constexpr tuple<_Elements&&...> |
1403 | forward_as_tuple(_Elements&&... __args) noexcept |
1404 | { return tuple<_Elements&&...>(std::forward<_Elements>(__args)...); } |
1405 | |
1406 | template<typename... _Tps> |
1407 | struct __is_tuple_like_impl<tuple<_Tps...>> : true_type |
1408 | { }; |
1409 | |
1410 | // Internal type trait that allows us to sfinae-protect tuple_cat. |
1411 | template<typename _Tp> |
1412 | struct __is_tuple_like |
1413 | : public __is_tuple_like_impl<typename std::remove_cv |
1414 | <typename std::remove_reference<_Tp>::type>::type>::type |
1415 | { }; |
1416 | |
1417 | template<size_t, typename, typename, size_t> |
1418 | struct __make_tuple_impl; |
1419 | |
1420 | template<size_t _Idx, typename _Tuple, typename... _Tp, size_t _Nm> |
1421 | struct __make_tuple_impl<_Idx, tuple<_Tp...>, _Tuple, _Nm> |
1422 | : __make_tuple_impl<_Idx + 1, |
1423 | tuple<_Tp..., __tuple_element_t<_Idx, _Tuple>>, |
1424 | _Tuple, _Nm> |
1425 | { }; |
1426 | |
1427 | template<std::size_t _Nm, typename _Tuple, typename... _Tp> |
1428 | struct __make_tuple_impl<_Nm, tuple<_Tp...>, _Tuple, _Nm> |
1429 | { |
1430 | typedef tuple<_Tp...> __type; |
1431 | }; |
1432 | |
1433 | template<typename _Tuple> |
1434 | struct __do_make_tuple |
1435 | : __make_tuple_impl<0, tuple<>, _Tuple, std::tuple_size<_Tuple>::value> |
1436 | { }; |
1437 | |
1438 | // Returns the std::tuple equivalent of a tuple-like type. |
1439 | template<typename _Tuple> |
1440 | struct __make_tuple |
1441 | : public __do_make_tuple<typename std::remove_cv |
1442 | <typename std::remove_reference<_Tuple>::type>::type> |
1443 | { }; |
1444 | |
1445 | // Combines several std::tuple's into a single one. |
1446 | template<typename...> |
1447 | struct __combine_tuples; |
1448 | |
1449 | template<> |
1450 | struct __combine_tuples<> |
1451 | { |
1452 | typedef tuple<> __type; |
1453 | }; |
1454 | |
1455 | template<typename... _Ts> |
1456 | struct __combine_tuples<tuple<_Ts...>> |
1457 | { |
1458 | typedef tuple<_Ts...> __type; |
1459 | }; |
1460 | |
1461 | template<typename... _T1s, typename... _T2s, typename... _Rem> |
1462 | struct __combine_tuples<tuple<_T1s...>, tuple<_T2s...>, _Rem...> |
1463 | { |
1464 | typedef typename __combine_tuples<tuple<_T1s..., _T2s...>, |
1465 | _Rem...>::__type __type; |
1466 | }; |
1467 | |
1468 | // Computes the result type of tuple_cat given a set of tuple-like types. |
1469 | template<typename... _Tpls> |
1470 | struct __tuple_cat_result |
1471 | { |
1472 | typedef typename __combine_tuples |
1473 | <typename __make_tuple<_Tpls>::__type...>::__type __type; |
1474 | }; |
1475 | |
1476 | // Helper to determine the index set for the first tuple-like |
1477 | // type of a given set. |
1478 | template<typename...> |
1479 | struct __make_1st_indices; |
1480 | |
1481 | template<> |
1482 | struct __make_1st_indices<> |
1483 | { |
1484 | typedef std::_Index_tuple<> __type; |
1485 | }; |
1486 | |
1487 | template<typename _Tp, typename... _Tpls> |
1488 | struct __make_1st_indices<_Tp, _Tpls...> |
1489 | { |
1490 | typedef typename std::_Build_index_tuple<std::tuple_size< |
1491 | typename std::remove_reference<_Tp>::type>::value>::__type __type; |
1492 | }; |
1493 | |
1494 | // Performs the actual concatenation by step-wise expanding tuple-like |
1495 | // objects into the elements, which are finally forwarded into the |
1496 | // result tuple. |
1497 | template<typename _Ret, typename _Indices, typename... _Tpls> |
1498 | struct __tuple_concater; |
1499 | |
1500 | template<typename _Ret, std::size_t... _Is, typename _Tp, typename... _Tpls> |
1501 | struct __tuple_concater<_Ret, std::_Index_tuple<_Is...>, _Tp, _Tpls...> |
1502 | { |
1503 | template<typename... _Us> |
1504 | static constexpr _Ret |
1505 | _S_do(_Tp&& __tp, _Tpls&&... __tps, _Us&&... __us) |
1506 | { |
1507 | typedef typename __make_1st_indices<_Tpls...>::__type __idx; |
1508 | typedef __tuple_concater<_Ret, __idx, _Tpls...> __next; |
1509 | return __next::_S_do(std::forward<_Tpls>(__tps)..., |
1510 | std::forward<_Us>(__us)..., |
1511 | std::get<_Is>(std::forward<_Tp>(__tp))...); |
1512 | } |
1513 | }; |
1514 | |
1515 | template<typename _Ret> |
1516 | struct __tuple_concater<_Ret, std::_Index_tuple<>> |
1517 | { |
1518 | template<typename... _Us> |
1519 | static constexpr _Ret |
1520 | _S_do(_Us&&... __us) |
1521 | { |
1522 | return _Ret(std::forward<_Us>(__us)...); |
1523 | } |
1524 | }; |
1525 | |
1526 | /// tuple_cat |
1527 | template<typename... _Tpls, typename = typename |
1528 | enable_if<__and_<__is_tuple_like<_Tpls>...>::value>::type> |
1529 | constexpr auto |
1530 | tuple_cat(_Tpls&&... __tpls) |
1531 | -> typename __tuple_cat_result<_Tpls...>::__type |
1532 | { |
1533 | typedef typename __tuple_cat_result<_Tpls...>::__type __ret; |
1534 | typedef typename __make_1st_indices<_Tpls...>::__type __idx; |
1535 | typedef __tuple_concater<__ret, __idx, _Tpls...> __concater; |
1536 | return __concater::_S_do(std::forward<_Tpls>(__tpls)...); |
1537 | } |
1538 | |
1539 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
1540 | // 2301. Why is tie not constexpr? |
1541 | /// tie |
1542 | template<typename... _Elements> |
1543 | constexpr tuple<_Elements&...> |
1544 | tie(_Elements&... __args) noexcept |
1545 | { return tuple<_Elements&...>(__args...); } |
1546 | |
1547 | /// swap |
1548 | template<typename... _Elements> |
1549 | inline void |
1550 | swap(tuple<_Elements...>& __x, tuple<_Elements...>& __y) |
1551 | noexcept(noexcept(__x.swap(__y))) |
1552 | { __x.swap(__y); } |
1553 | |
1554 | // A class (and instance) which can be used in 'tie' when an element |
1555 | // of a tuple is not required |
1556 | struct _Swallow_assign |
1557 | { |
1558 | template<class _Tp> |
1559 | const _Swallow_assign& |
1560 | operator=(const _Tp&) const |
1561 | { return *this; } |
1562 | }; |
1563 | |
1564 | const _Swallow_assign ignore{}; |
1565 | |
1566 | /// Partial specialization for tuples |
1567 | template<typename... _Types, typename _Alloc> |
1568 | struct uses_allocator<tuple<_Types...>, _Alloc> : true_type { }; |
1569 | |
1570 | // See stl_pair.h... |
1571 | template<class _T1, class _T2> |
1572 | template<typename... _Args1, typename... _Args2> |
1573 | inline |
1574 | pair<_T1, _T2>:: |
1575 | pair(piecewise_construct_t, |
1576 | tuple<_Args1...> __first, tuple<_Args2...> __second) |
1577 | : pair(__first, __second, |
1578 | typename _Build_index_tuple<sizeof...(_Args1)>::__type(), |
1579 | typename _Build_index_tuple<sizeof...(_Args2)>::__type()) |
1580 | { } |
1581 | |
1582 | template<class _T1, class _T2> |
1583 | template<typename... _Args1, std::size_t... _Indexes1, |
1584 | typename... _Args2, std::size_t... _Indexes2> |
1585 | inline |
1586 | pair<_T1, _T2>:: |
1587 | pair(tuple<_Args1...>& __tuple1, tuple<_Args2...>& __tuple2, |
1588 | _Index_tuple<_Indexes1...>, _Index_tuple<_Indexes2...>) |
1589 | : first(std::forward<_Args1>(std::get<_Indexes1>(__tuple1))...), |
1590 | second(std::forward<_Args2>(std::get<_Indexes2>(__tuple2))...) |
1591 | { } |
1592 | |
1593 | /// @} |
1594 | |
1595 | _GLIBCXX_END_NAMESPACE_VERSION |
1596 | } // namespace std |
1597 | |
1598 | #endif // C++11 |
1599 | |
1600 | #endif // _GLIBCXX_TUPLE |
1 | //===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains the declaration of the MachineInstr class, which is the |
10 | // basic representation for all target dependent machine instructions used by |
11 | // the back end. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #ifndef LLVM_CODEGEN_MACHINEINSTR_H |
16 | #define LLVM_CODEGEN_MACHINEINSTR_H |
17 | |
18 | #include "llvm/ADT/DenseMapInfo.h" |
19 | #include "llvm/ADT/PointerSumType.h" |
20 | #include "llvm/ADT/ilist.h" |
21 | #include "llvm/ADT/ilist_node.h" |
22 | #include "llvm/ADT/iterator_range.h" |
23 | #include "llvm/CodeGen/MachineMemOperand.h" |
24 | #include "llvm/CodeGen/MachineOperand.h" |
25 | #include "llvm/CodeGen/TargetOpcodes.h" |
26 | #include "llvm/IR/DebugLoc.h" |
27 | #include "llvm/IR/InlineAsm.h" |
28 | #include "llvm/MC/MCInstrDesc.h" |
29 | #include "llvm/MC/MCSymbol.h" |
30 | #include "llvm/Support/ArrayRecycler.h" |
31 | #include "llvm/Support/TrailingObjects.h" |
32 | #include <algorithm> |
33 | #include <cassert> |
34 | #include <cstdint> |
35 | #include <utility> |
36 | |
37 | namespace llvm { |
38 | |
39 | class AAResults; |
40 | template <typename T> class ArrayRef; |
41 | class DIExpression; |
42 | class DILocalVariable; |
43 | class MachineBasicBlock; |
44 | class MachineFunction; |
45 | class MachineRegisterInfo; |
46 | class ModuleSlotTracker; |
47 | class raw_ostream; |
48 | template <typename T> class SmallVectorImpl; |
49 | class SmallBitVector; |
50 | class StringRef; |
51 | class TargetInstrInfo; |
52 | class TargetRegisterClass; |
53 | class TargetRegisterInfo; |
54 | |
55 | //===----------------------------------------------------------------------===// |
56 | /// Representation of each machine instruction. |
57 | /// |
58 | /// This class isn't a POD type, but it must have a trivial destructor. When a |
59 | /// MachineFunction is deleted, all the contained MachineInstrs are deallocated |
60 | /// without having their destructor called. |
61 | /// |
62 | class MachineInstr |
63 | : public ilist_node_with_parent<MachineInstr, MachineBasicBlock, |
64 | ilist_sentinel_tracking<true>> { |
65 | public: |
66 | using mmo_iterator = ArrayRef<MachineMemOperand *>::iterator; |
67 | |
68 | /// Flags to specify different kinds of comments to output in |
69 | /// assembly code. These flags carry semantic information not |
70 | /// otherwise easily derivable from the IR text. |
71 | /// |
72 | enum CommentFlag { |
73 | ReloadReuse = 0x1, // higher bits are reserved for target dep comments. |
74 | NoSchedComment = 0x2, |
75 | TAsmComments = 0x4 // Target Asm comments should start from this value. |
76 | }; |
77 | |
78 | enum MIFlag { |
79 | NoFlags = 0, |
80 | FrameSetup = 1 << 0, // Instruction is used as a part of |
81 | // function frame setup code. |
82 | FrameDestroy = 1 << 1, // Instruction is used as a part of |
83 | // function frame destruction code. |
84 | BundledPred = 1 << 2, // Instruction has bundled predecessors. |
85 | BundledSucc = 1 << 3, // Instruction has bundled successors. |
86 | FmNoNans = 1 << 4, // Instruction does not support Fast |
87 | // math nan values. |
88 | FmNoInfs = 1 << 5, // Instruction does not support Fast |
89 | // math infinity values. |
90 | FmNsz = 1 << 6, // Instruction is not required to retain |
91 | // signed zero values. |
92 | FmArcp = 1 << 7, // Instruction supports Fast math |
93 | // reciprocal approximations. |
94 | FmContract = 1 << 8, // Instruction supports Fast math |
95 | // contraction operations like fma. |
96 | FmAfn = 1 << 9, // Instruction may map to Fast math |
97 | // instrinsic approximation. |
98 | FmReassoc = 1 << 10, // Instruction supports Fast math |
99 | // reassociation of operand order. |
100 | NoUWrap = 1 << 11, // Instruction supports binary operator |
101 | // no unsigned wrap. |
102 | NoSWrap = 1 << 12, // Instruction supports binary operator |
103 | // no signed wrap. |
104 | IsExact = 1 << 13, // Instruction supports division is |
105 | // known to be exact. |
106 | NoFPExcept = 1 << 14, // Instruction does not raise |
107 | // floatint-point exceptions. |
108 | NoMerge = 1 << 15, // Passes that drop source location info |
109 | // (e.g. branch folding) should skip |
110 | // this instruction. |
111 | }; |
112 | |
113 | private: |
114 | const MCInstrDesc *MCID; // Instruction descriptor. |
115 | MachineBasicBlock *Parent = nullptr; // Pointer to the owning basic block. |
116 | |
117 | // Operands are allocated by an ArrayRecycler. |
118 | MachineOperand *Operands = nullptr; // Pointer to the first operand. |
119 | unsigned NumOperands = 0; // Number of operands on instruction. |
120 | |
121 | uint16_t Flags = 0; // Various bits of additional |
122 | // information about machine |
123 | // instruction. |
124 | |
125 | uint8_t AsmPrinterFlags = 0; // Various bits of information used by |
126 | // the AsmPrinter to emit helpful |
127 | // comments. This is *not* semantic |
128 | // information. Do not use this for |
129 | // anything other than to convey comment |
130 | // information to AsmPrinter. |
131 | |
132 | // OperandCapacity has uint8_t size, so it should be next to AsmPrinterFlags |
133 | // to properly pack. |
134 | using OperandCapacity = ArrayRecycler<MachineOperand>::Capacity; |
135 | OperandCapacity CapOperands; // Capacity of the Operands array. |
136 | |
137 | /// Internal implementation detail class that provides out-of-line storage for |
138 | /// extra info used by the machine instruction when this info cannot be stored |
139 | /// in-line within the instruction itself. |
140 | /// |
141 | /// This has to be defined eagerly due to the implementation constraints of |
142 | /// `PointerSumType` where it is used. |
143 | class ExtraInfo final |
144 | : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *> { |
145 | public: |
146 | static ExtraInfo *create(BumpPtrAllocator &Allocator, |
147 | ArrayRef<MachineMemOperand *> MMOs, |
148 | MCSymbol *PreInstrSymbol = nullptr, |
149 | MCSymbol *PostInstrSymbol = nullptr, |
150 | MDNode *HeapAllocMarker = nullptr) { |
151 | bool HasPreInstrSymbol = PreInstrSymbol != nullptr; |
152 | bool HasPostInstrSymbol = PostInstrSymbol != nullptr; |
153 | bool HasHeapAllocMarker = HeapAllocMarker != nullptr; |
154 | auto *Result = new (Allocator.Allocate( |
155 | totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *>( |
156 | MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, |
157 | HasHeapAllocMarker), |
158 | alignof(ExtraInfo))) |
159 | ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, |
160 | HasHeapAllocMarker); |
161 | |
162 | // Copy the actual data into the trailing objects. |
163 | std::copy(MMOs.begin(), MMOs.end(), |
164 | Result->getTrailingObjects<MachineMemOperand *>()); |
165 | |
166 | if (HasPreInstrSymbol) |
167 | Result->getTrailingObjects<MCSymbol *>()[0] = PreInstrSymbol; |
168 | if (HasPostInstrSymbol) |
169 | Result->getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] = |
170 | PostInstrSymbol; |
171 | if (HasHeapAllocMarker) |
172 | Result->getTrailingObjects<MDNode *>()[0] = HeapAllocMarker; |
173 | |
174 | return Result; |
175 | } |
176 | |
177 | ArrayRef<MachineMemOperand *> getMMOs() const { |
178 | return makeArrayRef(getTrailingObjects<MachineMemOperand *>(), NumMMOs); |
179 | } |
180 | |
181 | MCSymbol *getPreInstrSymbol() const { |
182 | return HasPreInstrSymbol ? getTrailingObjects<MCSymbol *>()[0] : nullptr; |
183 | } |
184 | |
185 | MCSymbol *getPostInstrSymbol() const { |
186 | return HasPostInstrSymbol |
187 | ? getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] |
188 | : nullptr; |
189 | } |
190 | |
191 | MDNode *getHeapAllocMarker() const { |
192 | return HasHeapAllocMarker ? getTrailingObjects<MDNode *>()[0] : nullptr; |
193 | } |
194 | |
195 | private: |
196 | friend TrailingObjects; |
197 | |
198 | // Description of the extra info, used to interpret the actual optional |
199 | // data appended. |
200 | // |
201 | // Note that this is not terribly space optimized. This leaves a great deal |
202 | // of flexibility to fit more in here later. |
203 | const int NumMMOs; |
204 | const bool HasPreInstrSymbol; |
205 | const bool HasPostInstrSymbol; |
206 | const bool HasHeapAllocMarker; |
207 | |
208 | // Implement the `TrailingObjects` internal API. |
209 | size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const { |
210 | return NumMMOs; |
211 | } |
212 | size_t numTrailingObjects(OverloadToken<MCSymbol *>) const { |
213 | return HasPreInstrSymbol + HasPostInstrSymbol; |
214 | } |
215 | size_t numTrailingObjects(OverloadToken<MDNode *>) const { |
216 | return HasHeapAllocMarker; |
217 | } |
218 | |
219 | // Just a boring constructor to allow us to initialize the sizes. Always use |
220 | // the `create` routine above. |
221 | ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, |
222 | bool HasHeapAllocMarker) |
223 | : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), |
224 | HasPostInstrSymbol(HasPostInstrSymbol), |
225 | HasHeapAllocMarker(HasHeapAllocMarker) {} |
226 | }; |
227 | |
228 | /// Enumeration of the kinds of inline extra info available. It is important |
229 | /// that the `MachineMemOperand` inline kind has a tag value of zero to make |
230 | /// it accessible as an `ArrayRef`. |
231 | enum ExtraInfoInlineKinds { |
232 | EIIK_MMO = 0, |
233 | EIIK_PreInstrSymbol, |
234 | EIIK_PostInstrSymbol, |
235 | EIIK_OutOfLine |
236 | }; |
237 | |
238 | // We store extra information about the instruction here. The common case is |
239 | // expected to be nothing or a single pointer (typically a MMO or a symbol). |
240 | // We work to optimize this common case by storing it inline here rather than |
241 | // requiring a separate allocation, but we fall back to an allocation when |
242 | // multiple pointers are needed. |
243 | PointerSumType<ExtraInfoInlineKinds, |
244 | PointerSumTypeMember<EIIK_MMO, MachineMemOperand *>, |
245 | PointerSumTypeMember<EIIK_PreInstrSymbol, MCSymbol *>, |
246 | PointerSumTypeMember<EIIK_PostInstrSymbol, MCSymbol *>, |
247 | PointerSumTypeMember<EIIK_OutOfLine, ExtraInfo *>> |
248 | Info; |
249 | |
250 | DebugLoc debugLoc; // Source line information. |
251 | |
252 | /// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values |
253 | /// defined by this instruction. |
254 | unsigned DebugInstrNum; |
255 | |
256 | // Intrusive list support |
257 | friend struct ilist_traits<MachineInstr>; |
258 | friend struct ilist_callback_traits<MachineBasicBlock>; |
259 | void setParent(MachineBasicBlock *P) { Parent = P; } |
260 | |
261 | /// This constructor creates a copy of the given |
262 | /// MachineInstr in the given MachineFunction. |
263 | MachineInstr(MachineFunction &, const MachineInstr &); |
264 | |
265 | /// This constructor create a MachineInstr and add the implicit operands. |
266 | /// It reserves space for number of operands specified by |
267 | /// MCInstrDesc. An explicit DebugLoc is supplied. |
268 | MachineInstr(MachineFunction &, const MCInstrDesc &tid, DebugLoc dl, |
269 | bool NoImp = false); |
270 | |
271 | // MachineInstrs are pool-allocated and owned by MachineFunction. |
272 | friend class MachineFunction; |
273 | |
274 | void |
275 | dumprImpl(const MachineRegisterInfo &MRI, unsigned Depth, unsigned MaxDepth, |
276 | SmallPtrSetImpl<const MachineInstr *> &AlreadySeenInstrs) const; |
277 | |
278 | public: |
279 | MachineInstr(const MachineInstr &) = delete; |
280 | MachineInstr &operator=(const MachineInstr &) = delete; |
281 | // Use MachineFunction::DeleteMachineInstr() instead. |
282 | ~MachineInstr() = delete; |
283 | |
284 | const MachineBasicBlock* getParent() const { return Parent; } |
285 | MachineBasicBlock* getParent() { return Parent; } |
286 | |
287 | /// Move the instruction before \p MovePos. |
288 | void moveBefore(MachineInstr *MovePos); |
289 | |
290 | /// Return the function that contains the basic block that this instruction |
291 | /// belongs to. |
292 | /// |
293 | /// Note: this is undefined behaviour if the instruction does not have a |
294 | /// parent. |
295 | const MachineFunction *getMF() const; |
296 | MachineFunction *getMF() { |
297 | return const_cast<MachineFunction *>( |
298 | static_cast<const MachineInstr *>(this)->getMF()); |
299 | } |
300 | |
301 | /// Return the asm printer flags bitvector. |
302 | uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; } |
303 | |
304 | /// Clear the AsmPrinter bitvector. |
305 | void clearAsmPrinterFlags() { AsmPrinterFlags = 0; } |
306 | |
307 | /// Return whether an AsmPrinter flag is set. |
308 | bool getAsmPrinterFlag(CommentFlag Flag) const { |
309 | return AsmPrinterFlags & Flag; |
310 | } |
311 | |
312 | /// Set a flag for the AsmPrinter. |
313 | void setAsmPrinterFlag(uint8_t Flag) { |
314 | AsmPrinterFlags |= Flag; |
315 | } |
316 | |
317 | /// Clear specific AsmPrinter flags. |
318 | void clearAsmPrinterFlag(CommentFlag Flag) { |
319 | AsmPrinterFlags &= ~Flag; |
320 | } |
321 | |
322 | /// Return the MI flags bitvector. |
323 | uint16_t getFlags() const { |
324 | return Flags; |
325 | } |
326 | |
327 | /// Return whether an MI flag is set. |
328 | bool getFlag(MIFlag Flag) const { |
329 | return Flags & Flag; |
330 | } |
331 | |
332 | /// Set a MI flag. |
333 | void setFlag(MIFlag Flag) { |
334 | Flags |= (uint16_t)Flag; |
335 | } |
336 | |
337 | void setFlags(unsigned flags) { |
338 | // Filter out the automatically maintained flags. |
339 | unsigned Mask = BundledPred | BundledSucc; |
340 | Flags = (Flags & Mask) | (flags & ~Mask); |
341 | } |
342 | |
343 | /// clearFlag - Clear a MI flag. |
344 | void clearFlag(MIFlag Flag) { |
345 | Flags &= ~((uint16_t)Flag); |
346 | } |
347 | |
348 | /// Return true if MI is in a bundle (but not the first MI in a bundle). |
349 | /// |
350 | /// A bundle looks like this before it's finalized: |
351 | /// ---------------- |
352 | /// | MI | |
353 | /// ---------------- |
354 | /// | |
355 | /// ---------------- |
356 | /// | MI * | |
357 | /// ---------------- |
358 | /// | |
359 | /// ---------------- |
360 | /// | MI * | |
361 | /// ---------------- |
362 | /// In this case, the first MI starts a bundle but is not inside a bundle, the |
363 | /// next 2 MIs are considered "inside" the bundle. |
364 | /// |
365 | /// After a bundle is finalized, it looks like this: |
366 | /// ---------------- |
367 | /// | Bundle | |
368 | /// ---------------- |
369 | /// | |
370 | /// ---------------- |
371 | /// | MI * | |
372 | /// ---------------- |
373 | /// | |
374 | /// ---------------- |
375 | /// | MI * | |
376 | /// ---------------- |
377 | /// | |
378 | /// ---------------- |
379 | /// | MI * | |
380 | /// ---------------- |
381 | /// The first instruction has the special opcode "BUNDLE". It's not "inside" |
382 | /// a bundle, but the next three MIs are. |
383 | bool isInsideBundle() const { |
384 | return getFlag(BundledPred); |
385 | } |
386 | |
387 | /// Return true if this instruction part of a bundle. This is true |
388 | /// if either itself or its following instruction is marked "InsideBundle". |
389 | bool isBundled() const { |
390 | return isBundledWithPred() || isBundledWithSucc(); |
391 | } |
392 | |
393 | /// Return true if this instruction is part of a bundle, and it is not the |
394 | /// first instruction in the bundle. |
395 | bool isBundledWithPred() const { return getFlag(BundledPred); } |
396 | |
397 | /// Return true if this instruction is part of a bundle, and it is not the |
398 | /// last instruction in the bundle. |
399 | bool isBundledWithSucc() const { return getFlag(BundledSucc); } |
400 | |
401 | /// Bundle this instruction with its predecessor. This can be an unbundled |
402 | /// instruction, or it can be the first instruction in a bundle. |
403 | void bundleWithPred(); |
404 | |
405 | /// Bundle this instruction with its successor. This can be an unbundled |
406 | /// instruction, or it can be the last instruction in a bundle. |
407 | void bundleWithSucc(); |
408 | |
409 | /// Break bundle above this instruction. |
410 | void unbundleFromPred(); |
411 | |
412 | /// Break bundle below this instruction. |
413 | void unbundleFromSucc(); |
414 | |
415 | /// Returns the debug location id of this MachineInstr. |
416 | const DebugLoc &getDebugLoc() const { return debugLoc; } |
417 | |
418 | /// Return the operand containing the offset to be used if this DBG_VALUE |
419 | /// instruction is indirect; will be an invalid register if this value is |
420 | /// not indirect, and an immediate with value 0 otherwise. |
421 | const MachineOperand &getDebugOffset() const { |
422 | assert(isDebugValue() && "not a DBG_VALUE")((isDebugValue() && "not a DBG_VALUE") ? static_cast< void> (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 422, __PRETTY_FUNCTION__)); |
423 | return getOperand(1); |
424 | } |
425 | MachineOperand &getDebugOffset() { |
426 | assert(isDebugValue() && "not a DBG_VALUE")((isDebugValue() && "not a DBG_VALUE") ? static_cast< void> (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 426, __PRETTY_FUNCTION__)); |
427 | return getOperand(1); |
428 | } |
429 | |
430 | /// Return the operand for the debug variable referenced by |
431 | /// this DBG_VALUE instruction. |
432 | const MachineOperand &getDebugVariableOp() const; |
433 | MachineOperand &getDebugVariableOp(); |
434 | |
435 | /// Return the debug variable referenced by |
436 | /// this DBG_VALUE instruction. |
437 | const DILocalVariable *getDebugVariable() const; |
438 | |
439 | /// Return the operand for the complex address expression referenced by |
440 | /// this DBG_VALUE instruction. |
441 | MachineOperand &getDebugExpressionOp(); |
442 | |
443 | /// Return the complex address expression referenced by |
444 | /// this DBG_VALUE instruction. |
445 | const DIExpression *getDebugExpression() const; |
446 | |
447 | /// Return the debug label referenced by |
448 | /// this DBG_LABEL instruction. |
449 | const DILabel *getDebugLabel() const; |
450 | |
451 | /// Fetch the instruction number of this MachineInstr. If it does not have |
452 | /// one already, a new and unique number will be assigned. |
453 | unsigned getDebugInstrNum(); |
454 | |
455 | /// Examine the instruction number of this MachineInstr. May be zero if |
456 | /// it hasn't been assigned a number yet. |
457 | unsigned peekDebugInstrNum() const { return DebugInstrNum; } |
458 | |
459 | /// Set instruction number of this MachineInstr. Avoid using unless you're |
460 | /// deserializing this information. |
461 | void setDebugInstrNum(unsigned Num) { DebugInstrNum = Num; } |
462 | |
463 | /// Emit an error referring to the source location of this instruction. |
464 | /// This should only be used for inline assembly that is somehow |
465 | /// impossible to compile. Other errors should have been handled much |
466 | /// earlier. |
467 | /// |
468 | /// If this method returns, the caller should try to recover from the error. |
469 | void emitError(StringRef Msg) const; |
470 | |
471 | /// Returns the target instruction descriptor of this MachineInstr. |
472 | const MCInstrDesc &getDesc() const { return *MCID; } |
473 | |
474 | /// Returns the opcode of this MachineInstr. |
475 | unsigned getOpcode() const { return MCID->Opcode; } |
476 | |
477 | /// Retuns the total number of operands. |
478 | unsigned getNumOperands() const { return NumOperands; } |
479 | |
480 | /// Returns the total number of operands which are debug locations. |
481 | unsigned getNumDebugOperands() const { |
482 | return std::distance(debug_operands().begin(), debug_operands().end()); |
483 | } |
484 | |
485 | const MachineOperand& getOperand(unsigned i) const { |
486 | assert(i < getNumOperands() && "getOperand() out of range!")((i < getNumOperands() && "getOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 486, __PRETTY_FUNCTION__)); |
487 | return Operands[i]; |
488 | } |
489 | MachineOperand& getOperand(unsigned i) { |
490 | assert(i < getNumOperands() && "getOperand() out of range!")((i < getNumOperands() && "getOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 490, __PRETTY_FUNCTION__)); |
491 | return Operands[i]; |
492 | } |
493 | |
494 | MachineOperand &getDebugOperand(unsigned Index) { |
495 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")((Index < getNumDebugOperands() && "getDebugOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 495, __PRETTY_FUNCTION__)); |
496 | return *(debug_operands().begin() + Index); |
497 | } |
498 | const MachineOperand &getDebugOperand(unsigned Index) const { |
499 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")((Index < getNumDebugOperands() && "getDebugOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 499, __PRETTY_FUNCTION__)); |
500 | return *(debug_operands().begin() + Index); |
501 | } |
502 | |
503 | /// Returns a pointer to the operand corresponding to a debug use of Reg, or |
504 | /// nullptr if Reg is not used in any debug operand. |
505 | const MachineOperand *getDebugOperandForReg(Register Reg) const { |
506 | const MachineOperand *RegOp = |
507 | find_if(debug_operands(), [Reg](const MachineOperand &Op) { |
508 | return Op.isReg() && Op.getReg() == Reg; |
509 | }); |
510 | return RegOp == adl_end(debug_operands()) ? nullptr : RegOp; |
511 | } |
512 | MachineOperand *getDebugOperandForReg(Register Reg) { |
513 | MachineOperand *RegOp = |
514 | find_if(debug_operands(), [Reg](const MachineOperand &Op) { |
515 | return Op.isReg() && Op.getReg() == Reg; |
516 | }); |
517 | return RegOp == adl_end(debug_operands()) ? nullptr : RegOp; |
518 | } |
519 | |
520 | unsigned getDebugOperandIndex(const MachineOperand *Op) const { |
521 | assert(Op >= adl_begin(debug_operands()) &&((Op >= adl_begin(debug_operands()) && Op <= adl_end (debug_operands()) && "Expected a debug operand.") ? static_cast <void> (0) : __assert_fail ("Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()) && \"Expected a debug operand.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 522, __PRETTY_FUNCTION__)) |
522 | Op <= adl_end(debug_operands()) && "Expected a debug operand.")((Op >= adl_begin(debug_operands()) && Op <= adl_end (debug_operands()) && "Expected a debug operand.") ? static_cast <void> (0) : __assert_fail ("Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()) && \"Expected a debug operand.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 522, __PRETTY_FUNCTION__)); |
523 | return std::distance(adl_begin(debug_operands()), Op); |
524 | } |
525 | |
526 | /// Returns the total number of definitions. |
527 | unsigned getNumDefs() const { |
528 | return getNumExplicitDefs() + MCID->getNumImplicitDefs(); |
529 | } |
530 | |
531 | /// Returns true if the instruction has implicit definition. |
532 | bool hasImplicitDef() const { |
533 | for (unsigned I = getNumExplicitOperands(), E = getNumOperands(); |
534 | I != E; ++I) { |
535 | const MachineOperand &MO = getOperand(I); |
536 | if (MO.isDef() && MO.isImplicit()) |
537 | return true; |
538 | } |
539 | return false; |
540 | } |
541 | |
542 | /// Returns the implicit operands number. |
543 | unsigned getNumImplicitOperands() const { |
544 | return getNumOperands() - getNumExplicitOperands(); |
545 | } |
546 | |
547 | /// Return true if operand \p OpIdx is a subregister index. |
548 | bool isOperandSubregIdx(unsigned OpIdx) const { |
549 | assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate &&((getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && "Expected MO_Immediate operand type.") ? static_cast <void> (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 550, __PRETTY_FUNCTION__)) |
550 | "Expected MO_Immediate operand type.")((getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && "Expected MO_Immediate operand type.") ? static_cast <void> (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 550, __PRETTY_FUNCTION__)); |
551 | if (isExtractSubreg() && OpIdx == 2) |
552 | return true; |
553 | if (isInsertSubreg() && OpIdx == 3) |
554 | return true; |
555 | if (isRegSequence() && OpIdx > 1 && (OpIdx % 2) == 0) |
556 | return true; |
557 | if (isSubregToReg() && OpIdx == 3) |
558 | return true; |
559 | return false; |
560 | } |
561 | |
562 | /// Returns the number of non-implicit operands. |
563 | unsigned getNumExplicitOperands() const; |
564 | |
565 | /// Returns the number of non-implicit definitions. |
566 | unsigned getNumExplicitDefs() const; |
567 | |
568 | /// iterator/begin/end - Iterate over all operands of a machine instruction. |
569 | using mop_iterator = MachineOperand *; |
570 | using const_mop_iterator = const MachineOperand *; |
571 | |
572 | mop_iterator operands_begin() { return Operands; } |
573 | mop_iterator operands_end() { return Operands + NumOperands; } |
574 | |
575 | const_mop_iterator operands_begin() const { return Operands; } |
576 | const_mop_iterator operands_end() const { return Operands + NumOperands; } |
577 | |
578 | iterator_range<mop_iterator> operands() { |
579 | return make_range(operands_begin(), operands_end()); |
580 | } |
581 | iterator_range<const_mop_iterator> operands() const { |
582 | return make_range(operands_begin(), operands_end()); |
583 | } |
584 | iterator_range<mop_iterator> explicit_operands() { |
585 | return make_range(operands_begin(), |
586 | operands_begin() + getNumExplicitOperands()); |
587 | } |
588 | iterator_range<const_mop_iterator> explicit_operands() const { |
589 | return make_range(operands_begin(), |
590 | operands_begin() + getNumExplicitOperands()); |
591 | } |
592 | iterator_range<mop_iterator> implicit_operands() { |
593 | return make_range(explicit_operands().end(), operands_end()); |
594 | } |
595 | iterator_range<const_mop_iterator> implicit_operands() const { |
596 | return make_range(explicit_operands().end(), operands_end()); |
597 | } |
598 | /// Returns a range over all operands that are used to determine the variable |
599 | /// location for this DBG_VALUE instruction. |
600 | iterator_range<mop_iterator> debug_operands() { |
601 | assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction." ) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 601, __PRETTY_FUNCTION__)); |
602 | return make_range(operands_begin(), operands_begin() + 1); |
603 | } |
604 | /// \copydoc debug_operands() |
605 | iterator_range<const_mop_iterator> debug_operands() const { |
606 | assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction." ) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 606, __PRETTY_FUNCTION__)); |
607 | return make_range(operands_begin(), operands_begin() + 1); |
608 | } |
609 | /// Returns a range over all explicit operands that are register definitions. |
610 | /// Implicit definition are not included! |
611 | iterator_range<mop_iterator> defs() { |
612 | return make_range(operands_begin(), |
613 | operands_begin() + getNumExplicitDefs()); |
614 | } |
615 | /// \copydoc defs() |
616 | iterator_range<const_mop_iterator> defs() const { |
617 | return make_range(operands_begin(), |
618 | operands_begin() + getNumExplicitDefs()); |
619 | } |
620 | /// Returns a range that includes all operands that are register uses. |
621 | /// This may include unrelated operands which are not register uses. |
622 | iterator_range<mop_iterator> uses() { |
623 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
624 | } |
625 | /// \copydoc uses() |
626 | iterator_range<const_mop_iterator> uses() const { |
627 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
628 | } |
629 | iterator_range<mop_iterator> explicit_uses() { |
630 | return make_range(operands_begin() + getNumExplicitDefs(), |
631 | operands_begin() + getNumExplicitOperands()); |
632 | } |
633 | iterator_range<const_mop_iterator> explicit_uses() const { |
634 | return make_range(operands_begin() + getNumExplicitDefs(), |
635 | operands_begin() + getNumExplicitOperands()); |
636 | } |
637 | |
638 | /// Returns the number of the operand iterator \p I points to. |
639 | unsigned getOperandNo(const_mop_iterator I) const { |
640 | return I - operands_begin(); |
641 | } |
642 | |
643 | /// Access to memory operands of the instruction. If there are none, that does |
644 | /// not imply anything about whether the function accesses memory. Instead, |
645 | /// the caller must behave conservatively. |
646 | ArrayRef<MachineMemOperand *> memoperands() const { |
647 | if (!Info) |
648 | return {}; |
649 | |
650 | if (Info.is<EIIK_MMO>()) |
651 | return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1); |
652 | |
653 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
654 | return EI->getMMOs(); |
655 | |
656 | return {}; |
657 | } |
658 | |
659 | /// Access to memory operands of the instruction. |
660 | /// |
661 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
662 | /// anything about whether the function accesses memory. Instead, the caller |
663 | /// must behave conservatively. |
664 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } |
665 | |
666 | /// Access to memory operands of the instruction. |
667 | /// |
668 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
669 | /// anything about whether the function accesses memory. Instead, the caller |
670 | /// must behave conservatively. |
671 | mmo_iterator memoperands_end() const { return memoperands().end(); } |
672 | |
673 | /// Return true if we don't have any memory operands which described the |
674 | /// memory access done by this instruction. If this is true, calling code |
675 | /// must be conservative. |
676 | bool memoperands_empty() const { return memoperands().empty(); } |
677 | |
678 | /// Return true if this instruction has exactly one MachineMemOperand. |
679 | bool hasOneMemOperand() const { return memoperands().size() == 1; } |
680 | |
681 | /// Return the number of memory operands. |
682 | unsigned getNumMemOperands() const { return memoperands().size(); } |
683 | |
684 | /// Helper to extract a pre-instruction symbol if one has been added. |
685 | MCSymbol *getPreInstrSymbol() const { |
686 | if (!Info) |
687 | return nullptr; |
688 | if (MCSymbol *S = Info.get<EIIK_PreInstrSymbol>()) |
689 | return S; |
690 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
691 | return EI->getPreInstrSymbol(); |
692 | |
693 | return nullptr; |
694 | } |
695 | |
696 | /// Helper to extract a post-instruction symbol if one has been added. |
697 | MCSymbol *getPostInstrSymbol() const { |
698 | if (!Info) |
699 | return nullptr; |
700 | if (MCSymbol *S = Info.get<EIIK_PostInstrSymbol>()) |
701 | return S; |
702 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
703 | return EI->getPostInstrSymbol(); |
704 | |
705 | return nullptr; |
706 | } |
707 | |
708 | /// Helper to extract a heap alloc marker if one has been added. |
709 | MDNode *getHeapAllocMarker() const { |
710 | if (!Info) |
711 | return nullptr; |
712 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
713 | return EI->getHeapAllocMarker(); |
714 | |
715 | return nullptr; |
716 | } |
717 | |
718 | /// API for querying MachineInstr properties. They are the same as MCInstrDesc |
719 | /// queries but they are bundle aware. |
720 | |
721 | enum QueryType { |
722 | IgnoreBundle, // Ignore bundles |
723 | AnyInBundle, // Return true if any instruction in bundle has property |
724 | AllInBundle // Return true if all instructions in bundle have property |
725 | }; |
726 | |
727 | /// Return true if the instruction (or in the case of a bundle, |
728 | /// the instructions inside the bundle) has the specified property. |
729 | /// The first argument is the property being queried. |
730 | /// The second argument indicates whether the query should look inside |
731 | /// instruction bundles. |
732 | bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const { |
733 | assert(MCFlag < 64 &&((MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? static_cast<void> (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 734, __PRETTY_FUNCTION__)) |
734 | "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.")((MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? static_cast<void> (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 734, __PRETTY_FUNCTION__)); |
735 | // Inline the fast path for unbundled or bundle-internal instructions. |
736 | if (Type == IgnoreBundle || !isBundled() || isBundledWithPred()) |
737 | return getDesc().getFlags() & (1ULL << MCFlag); |
738 | |
739 | // If this is the first instruction in a bundle, take the slow path. |
740 | return hasPropertyInBundle(1ULL << MCFlag, Type); |
741 | } |
742 | |
743 | /// Return true if this is an instruction that should go through the usual |
744 | /// legalization steps. |
745 | bool isPreISelOpcode(QueryType Type = IgnoreBundle) const { |
746 | return hasProperty(MCID::PreISelOpcode, Type); |
747 | } |
748 | |
749 | /// Return true if this instruction can have a variable number of operands. |
750 | /// In this case, the variable operands will be after the normal |
751 | /// operands but before the implicit definitions and uses (if any are |
752 | /// present). |
753 | bool isVariadic(QueryType Type = IgnoreBundle) const { |
754 | return hasProperty(MCID::Variadic, Type); |
755 | } |
756 | |
757 | /// Set if this instruction has an optional definition, e.g. |
758 | /// ARM instructions which can set condition code if 's' bit is set. |
759 | bool hasOptionalDef(QueryType Type = IgnoreBundle) const { |
760 | return hasProperty(MCID::HasOptionalDef, Type); |
761 | } |
762 | |
763 | /// Return true if this is a pseudo instruction that doesn't |
764 | /// correspond to a real machine instruction. |
765 | bool isPseudo(QueryType Type = IgnoreBundle) const { |
766 | return hasProperty(MCID::Pseudo, Type); |
767 | } |
768 | |
769 | bool isReturn(QueryType Type = AnyInBundle) const { |
770 | return hasProperty(MCID::Return, Type); |
771 | } |
772 | |
773 | /// Return true if this is an instruction that marks the end of an EH scope, |
774 | /// i.e., a catchpad or a cleanuppad instruction. |
775 | bool isEHScopeReturn(QueryType Type = AnyInBundle) const { |
776 | return hasProperty(MCID::EHScopeReturn, Type); |
777 | } |
778 | |
779 | bool isCall(QueryType Type = AnyInBundle) const { |
780 | return hasProperty(MCID::Call, Type); |
781 | } |
782 | |
783 | /// Return true if this is a call instruction that may have an associated |
784 | /// call site entry in the debug info. |
785 | bool isCandidateForCallSiteEntry(QueryType Type = IgnoreBundle) const; |
786 | /// Return true if copying, moving, or erasing this instruction requires |
787 | /// updating Call Site Info (see \ref copyCallSiteInfo, \ref moveCallSiteInfo, |
788 | /// \ref eraseCallSiteInfo). |
789 | bool shouldUpdateCallSiteInfo() const; |
790 | |
791 | /// Returns true if the specified instruction stops control flow |
792 | /// from executing the instruction immediately following it. Examples include |
793 | /// unconditional branches and return instructions. |
794 | bool isBarrier(QueryType Type = AnyInBundle) const { |
795 | return hasProperty(MCID::Barrier, Type); |
796 | } |
797 | |
798 | /// Returns true if this instruction part of the terminator for a basic block. |
799 | /// Typically this is things like return and branch instructions. |
800 | /// |
801 | /// Various passes use this to insert code into the bottom of a basic block, |
802 | /// but before control flow occurs. |
803 | bool isTerminator(QueryType Type = AnyInBundle) const { |
804 | return hasProperty(MCID::Terminator, Type); |
805 | } |
806 | |
807 | /// Returns true if this is a conditional, unconditional, or indirect branch. |
808 | /// Predicates below can be used to discriminate between |
809 | /// these cases, and the TargetInstrInfo::analyzeBranch method can be used to |
810 | /// get more information. |
811 | bool isBranch(QueryType Type = AnyInBundle) const { |
812 | return hasProperty(MCID::Branch, Type); |
813 | } |
814 | |
815 | /// Return true if this is an indirect branch, such as a |
816 | /// branch through a register. |
817 | bool isIndirectBranch(QueryType Type = AnyInBundle) const { |
818 | return hasProperty(MCID::IndirectBranch, Type); |
819 | } |
820 | |
821 | /// Return true if this is a branch which may fall |
822 | /// through to the next instruction or may transfer control flow to some other |
823 | /// block. The TargetInstrInfo::analyzeBranch method can be used to get more |
824 | /// information about this branch. |
825 | bool isConditionalBranch(QueryType Type = AnyInBundle) const { |
826 | return isBranch(Type) && !isBarrier(Type) && !isIndirectBranch(Type); |
827 | } |
828 | |
829 | /// Return true if this is a branch which always |
830 | /// transfers control flow to some other block. The |
831 | /// TargetInstrInfo::analyzeBranch method can be used to get more information |
832 | /// about this branch. |
833 | bool isUnconditionalBranch(QueryType Type = AnyInBundle) const { |
834 | return isBranch(Type) && isBarrier(Type) && !isIndirectBranch(Type); |
835 | } |
836 | |
837 | /// Return true if this instruction has a predicate operand that |
838 | /// controls execution. It may be set to 'always', or may be set to other |
839 | /// values. There are various methods in TargetInstrInfo that can be used to |
840 | /// control and modify the predicate in this instruction. |
841 | bool isPredicable(QueryType Type = AllInBundle) const { |
842 | // If it's a bundle than all bundled instructions must be predicable for this |
843 | // to return true. |
844 | return hasProperty(MCID::Predicable, Type); |
845 | } |
846 | |
847 | /// Return true if this instruction is a comparison. |
848 | bool isCompare(QueryType Type = IgnoreBundle) const { |
849 | return hasProperty(MCID::Compare, Type); |
850 | } |
851 | |
852 | /// Return true if this instruction is a move immediate |
853 | /// (including conditional moves) instruction. |
854 | bool isMoveImmediate(QueryType Type = IgnoreBundle) const { |
855 | return hasProperty(MCID::MoveImm, Type); |
856 | } |
857 | |
858 | /// Return true if this instruction is a register move. |
859 | /// (including moving values from subreg to reg) |
860 | bool isMoveReg(QueryType Type = IgnoreBundle) const { |
861 | return hasProperty(MCID::MoveReg, Type); |
862 | } |
863 | |
864 | /// Return true if this instruction is a bitcast instruction. |
865 | bool isBitcast(QueryType Type = IgnoreBundle) const { |
866 | return hasProperty(MCID::Bitcast, Type); |
867 | } |
868 | |
869 | /// Return true if this instruction is a select instruction. |
870 | bool isSelect(QueryType Type = IgnoreBundle) const { |
871 | return hasProperty(MCID::Select, Type); |
872 | } |
873 | |
874 | /// Return true if this instruction cannot be safely duplicated. |
875 | /// For example, if the instruction has a unique labels attached |
876 | /// to it, duplicating it would cause multiple definition errors. |
877 | bool isNotDuplicable(QueryType Type = AnyInBundle) const { |
878 | return hasProperty(MCID::NotDuplicable, Type); |
879 | } |
880 | |
881 | /// Return true if this instruction is convergent. |
882 | /// Convergent instructions can not be made control-dependent on any |
883 | /// additional values. |
884 | bool isConvergent(QueryType Type = AnyInBundle) const { |
885 | if (isInlineAsm()) { |
886 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
887 | if (ExtraInfo & InlineAsm::Extra_IsConvergent) |
888 | return true; |
889 | } |
890 | return hasProperty(MCID::Convergent, Type); |
891 | } |
892 | |
893 | /// Returns true if the specified instruction has a delay slot |
894 | /// which must be filled by the code generator. |
895 | bool hasDelaySlot(QueryType Type = AnyInBundle) const { |
896 | return hasProperty(MCID::DelaySlot, Type); |
897 | } |
898 | |
899 | /// Return true for instructions that can be folded as |
900 | /// memory operands in other instructions. The most common use for this |
901 | /// is instructions that are simple loads from memory that don't modify |
902 | /// the loaded value in any way, but it can also be used for instructions |
903 | /// that can be expressed as constant-pool loads, such as V_SETALLONES |
904 | /// on x86, to allow them to be folded when it is beneficial. |
905 | /// This should only be set on instructions that return a value in their |
906 | /// only virtual register definition. |
907 | bool canFoldAsLoad(QueryType Type = IgnoreBundle) const { |
908 | return hasProperty(MCID::FoldableAsLoad, Type); |
909 | } |
910 | |
911 | /// Return true if this instruction behaves |
912 | /// the same way as the generic REG_SEQUENCE instructions. |
913 | /// E.g., on ARM, |
914 | /// dX VMOVDRR rY, rZ |
915 | /// is equivalent to |
916 | /// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1. |
917 | /// |
918 | /// Note that for the optimizers to be able to take advantage of |
919 | /// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be |
920 | /// override accordingly. |
921 | bool isRegSequenceLike(QueryType Type = IgnoreBundle) const { |
922 | return hasProperty(MCID::RegSequence, Type); |
923 | } |
924 | |
925 | /// Return true if this instruction behaves |
926 | /// the same way as the generic EXTRACT_SUBREG instructions. |
927 | /// E.g., on ARM, |
928 | /// rX, rY VMOVRRD dZ |
929 | /// is equivalent to two EXTRACT_SUBREG: |
930 | /// rX = EXTRACT_SUBREG dZ, ssub_0 |
931 | /// rY = EXTRACT_SUBREG dZ, ssub_1 |
932 | /// |
933 | /// Note that for the optimizers to be able to take advantage of |
934 | /// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be |
935 | /// override accordingly. |
936 | bool isExtractSubregLike(QueryType Type = IgnoreBundle) const { |
937 | return hasProperty(MCID::ExtractSubreg, Type); |
938 | } |
939 | |
940 | /// Return true if this instruction behaves |
941 | /// the same way as the generic INSERT_SUBREG instructions. |
942 | /// E.g., on ARM, |
943 | /// dX = VSETLNi32 dY, rZ, Imm |
944 | /// is equivalent to a INSERT_SUBREG: |
945 | /// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm) |
946 | /// |
947 | /// Note that for the optimizers to be able to take advantage of |
948 | /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be |
949 | /// override accordingly. |
950 | bool isInsertSubregLike(QueryType Type = IgnoreBundle) const { |
951 | return hasProperty(MCID::InsertSubreg, Type); |
952 | } |
953 | |
954 | //===--------------------------------------------------------------------===// |
955 | // Side Effect Analysis |
956 | //===--------------------------------------------------------------------===// |
957 | |
958 | /// Return true if this instruction could possibly read memory. |
959 | /// Instructions with this flag set are not necessarily simple load |
960 | /// instructions, they may load a value and modify it, for example. |
961 | bool mayLoad(QueryType Type = AnyInBundle) const { |
962 | if (isInlineAsm()) { |
963 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
964 | if (ExtraInfo & InlineAsm::Extra_MayLoad) |
965 | return true; |
966 | } |
967 | return hasProperty(MCID::MayLoad, Type); |
968 | } |
969 | |
970 | /// Return true if this instruction could possibly modify memory. |
971 | /// Instructions with this flag set are not necessarily simple store |
972 | /// instructions, they may store a modified value based on their operands, or |
973 | /// may not actually modify anything, for example. |
974 | bool mayStore(QueryType Type = AnyInBundle) const { |
975 | if (isInlineAsm()) { |
976 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
977 | if (ExtraInfo & InlineAsm::Extra_MayStore) |
978 | return true; |
979 | } |
980 | return hasProperty(MCID::MayStore, Type); |
981 | } |
982 | |
983 | /// Return true if this instruction could possibly read or modify memory. |
984 | bool mayLoadOrStore(QueryType Type = AnyInBundle) const { |
985 | return mayLoad(Type) || mayStore(Type); |
986 | } |
987 | |
988 | /// Return true if this instruction could possibly raise a floating-point |
989 | /// exception. This is the case if the instruction is a floating-point |
990 | /// instruction that can in principle raise an exception, as indicated |
991 | /// by the MCID::MayRaiseFPException property, *and* at the same time, |
992 | /// the instruction is used in a context where we expect floating-point |
993 | /// exceptions are not disabled, as indicated by the NoFPExcept MI flag. |
994 | bool mayRaiseFPException() const { |
995 | return hasProperty(MCID::MayRaiseFPException) && |
996 | !getFlag(MachineInstr::MIFlag::NoFPExcept); |
997 | } |
998 | |
999 | //===--------------------------------------------------------------------===// |
1000 | // Flags that indicate whether an instruction can be modified by a method. |
1001 | //===--------------------------------------------------------------------===// |
1002 | |
1003 | /// Return true if this may be a 2- or 3-address |
1004 | /// instruction (of the form "X = op Y, Z, ..."), which produces the same |
1005 | /// result if Y and Z are exchanged. If this flag is set, then the |
1006 | /// TargetInstrInfo::commuteInstruction method may be used to hack on the |
1007 | /// instruction. |
1008 | /// |
1009 | /// Note that this flag may be set on instructions that are only commutable |
1010 | /// sometimes. In these cases, the call to commuteInstruction will fail. |
1011 | /// Also note that some instructions require non-trivial modification to |
1012 | /// commute them. |
1013 | bool isCommutable(QueryType Type = IgnoreBundle) const { |
1014 | return hasProperty(MCID::Commutable, Type); |
1015 | } |
1016 | |
1017 | /// Return true if this is a 2-address instruction |
1018 | /// which can be changed into a 3-address instruction if needed. Doing this |
1019 | /// transformation can be profitable in the register allocator, because it |
1020 | /// means that the instruction can use a 2-address form if possible, but |
1021 | /// degrade into a less efficient form if the source and dest register cannot |
1022 | /// be assigned to the same register. For example, this allows the x86 |
1023 | /// backend to turn a "shl reg, 3" instruction into an LEA instruction, which |
1024 | /// is the same speed as the shift but has bigger code size. |
1025 | /// |
1026 | /// If this returns true, then the target must implement the |
1027 | /// TargetInstrInfo::convertToThreeAddress method for this instruction, which |
1028 | /// is allowed to fail if the transformation isn't valid for this specific |
1029 | /// instruction (e.g. shl reg, 4 on x86). |
1030 | /// |
1031 | bool isConvertibleTo3Addr(QueryType Type = IgnoreBundle) const { |
1032 | return hasProperty(MCID::ConvertibleTo3Addr, Type); |
1033 | } |
1034 | |
1035 | /// Return true if this instruction requires |
1036 | /// custom insertion support when the DAG scheduler is inserting it into a |
1037 | /// machine basic block. If this is true for the instruction, it basically |
1038 | /// means that it is a pseudo instruction used at SelectionDAG time that is |
1039 | /// expanded out into magic code by the target when MachineInstrs are formed. |
1040 | /// |
1041 | /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method |
1042 | /// is used to insert this into the MachineBasicBlock. |
1043 | bool usesCustomInsertionHook(QueryType Type = IgnoreBundle) const { |
1044 | return hasProperty(MCID::UsesCustomInserter, Type); |
1045 | } |
1046 | |
1047 | /// Return true if this instruction requires *adjustment* |
1048 | /// after instruction selection by calling a target hook. For example, this |
1049 | /// can be used to fill in ARM 's' optional operand depending on whether |
1050 | /// the conditional flag register is used. |
1051 | bool hasPostISelHook(QueryType Type = IgnoreBundle) const { |
1052 | return hasProperty(MCID::HasPostISelHook, Type); |
1053 | } |
1054 | |
1055 | /// Returns true if this instruction is a candidate for remat. |
1056 | /// This flag is deprecated, please don't use it anymore. If this |
1057 | /// flag is set, the isReallyTriviallyReMaterializable() method is called to |
1058 | /// verify the instruction is really rematable. |
1059 | bool isRematerializable(QueryType Type = AllInBundle) const { |
1060 | // It's only possible to re-mat a bundle if all bundled instructions are |
1061 | // re-materializable. |
1062 | return hasProperty(MCID::Rematerializable, Type); |
1063 | } |
1064 | |
1065 | /// Returns true if this instruction has the same cost (or less) than a move |
1066 | /// instruction. This is useful during certain types of optimizations |
1067 | /// (e.g., remat during two-address conversion or machine licm) |
1068 | /// where we would like to remat or hoist the instruction, but not if it costs |
1069 | /// more than moving the instruction into the appropriate register. Note, we |
1070 | /// are not marking copies from and to the same register class with this flag. |
1071 | bool isAsCheapAsAMove(QueryType Type = AllInBundle) const { |
1072 | // Only returns true for a bundle if all bundled instructions are cheap. |
1073 | return hasProperty(MCID::CheapAsAMove, Type); |
1074 | } |
1075 | |
1076 | /// Returns true if this instruction source operands |
1077 | /// have special register allocation requirements that are not captured by the |
1078 | /// operand register classes. e.g. ARM::STRD's two source registers must be an |
1079 | /// even / odd pair, ARM::STM registers have to be in ascending order. |
1080 | /// Post-register allocation passes should not attempt to change allocations |
1081 | /// for sources of instructions with this flag. |
1082 | bool hasExtraSrcRegAllocReq(QueryType Type = AnyInBundle) const { |
1083 | return hasProperty(MCID::ExtraSrcRegAllocReq, Type); |
1084 | } |
1085 | |
1086 | /// Returns true if this instruction def operands |
1087 | /// have special register allocation requirements that are not captured by the |
1088 | /// operand register classes. e.g. ARM::LDRD's two def registers must be an |
1089 | /// even / odd pair, ARM::LDM registers have to be in ascending order. |
1090 | /// Post-register allocation passes should not attempt to change allocations |
1091 | /// for definitions of instructions with this flag. |
1092 | bool hasExtraDefRegAllocReq(QueryType Type = AnyInBundle) const { |
1093 | return hasProperty(MCID::ExtraDefRegAllocReq, Type); |
1094 | } |
1095 | |
1096 | enum MICheckType { |
1097 | CheckDefs, // Check all operands for equality |
1098 | CheckKillDead, // Check all operands including kill / dead markers |
1099 | IgnoreDefs, // Ignore all definitions |
1100 | IgnoreVRegDefs // Ignore virtual register definitions |
1101 | }; |
1102 | |
1103 | /// Return true if this instruction is identical to \p Other. |
1104 | /// Two instructions are identical if they have the same opcode and all their |
1105 | /// operands are identical (with respect to MachineOperand::isIdenticalTo()). |
1106 | /// Note that this means liveness related flags (dead, undef, kill) do not |
1107 | /// affect the notion of identical. |
1108 | bool isIdenticalTo(const MachineInstr &Other, |
1109 | MICheckType Check = CheckDefs) const; |
1110 | |
1111 | /// Unlink 'this' from the containing basic block, and return it without |
1112 | /// deleting it. |
1113 | /// |
1114 | /// This function can not be used on bundled instructions, use |
1115 | /// removeFromBundle() to remove individual instructions from a bundle. |
1116 | MachineInstr *removeFromParent(); |
1117 | |
1118 | /// Unlink this instruction from its basic block and return it without |
1119 | /// deleting it. |
1120 | /// |
1121 | /// If the instruction is part of a bundle, the other instructions in the |
1122 | /// bundle remain bundled. |
1123 | MachineInstr *removeFromBundle(); |
1124 | |
1125 | /// Unlink 'this' from the containing basic block and delete it. |
1126 | /// |
1127 | /// If this instruction is the header of a bundle, the whole bundle is erased. |
1128 | /// This function can not be used for instructions inside a bundle, use |
1129 | /// eraseFromBundle() to erase individual bundled instructions. |
1130 | void eraseFromParent(); |
1131 | |
1132 | /// Unlink 'this' from the containing basic block and delete it. |
1133 | /// |
1134 | /// For all definitions mark their uses in DBG_VALUE nodes |
1135 | /// as undefined. Otherwise like eraseFromParent(). |
1136 | void eraseFromParentAndMarkDBGValuesForRemoval(); |
1137 | |
1138 | /// Unlink 'this' form its basic block and delete it. |
1139 | /// |
1140 | /// If the instruction is part of a bundle, the other instructions in the |
1141 | /// bundle remain bundled. |
1142 | void eraseFromBundle(); |
1143 | |
1144 | bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } |
1145 | bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } |
1146 | bool isAnnotationLabel() const { |
1147 | return getOpcode() == TargetOpcode::ANNOTATION_LABEL; |
1148 | } |
1149 | |
1150 | /// Returns true if the MachineInstr represents a label. |
1151 | bool isLabel() const { |
1152 | return isEHLabel() || isGCLabel() || isAnnotationLabel(); |
1153 | } |
1154 | |
1155 | bool isCFIInstruction() const { |
1156 | return getOpcode() == TargetOpcode::CFI_INSTRUCTION; |
1157 | } |
1158 | |
1159 | // True if the instruction represents a position in the function. |
1160 | bool isPosition() const { return isLabel() || isCFIInstruction(); } |
1161 | |
1162 | bool isDebugValue() const { return getOpcode() == TargetOpcode::DBG_VALUE; } |
1163 | bool isDebugLabel() const { return getOpcode() == TargetOpcode::DBG_LABEL; } |
1164 | bool isDebugRef() const { return getOpcode() == TargetOpcode::DBG_INSTR_REF; } |
1165 | bool isDebugInstr() const { |
1166 | return isDebugValue() || isDebugLabel() || isDebugRef(); |
1167 | } |
1168 | |
1169 | bool isDebugOffsetImm() const { return getDebugOffset().isImm(); } |
1170 | |
1171 | /// A DBG_VALUE is indirect iff the location operand is a register and |
1172 | /// the offset operand is an immediate. |
1173 | bool isIndirectDebugValue() const { |
1174 | return isDebugValue() && getDebugOperand(0).isReg() && isDebugOffsetImm(); |
1175 | } |
1176 | |
1177 | /// A DBG_VALUE is an entry value iff its debug expression contains the |
1178 | /// DW_OP_LLVM_entry_value operation. |
1179 | bool isDebugEntryValue() const; |
1180 | |
1181 | /// Return true if the instruction is a debug value which describes a part of |
1182 | /// a variable as unavailable. |
1183 | bool isUndefDebugValue() const { |
1184 | return isDebugValue() && getDebugOperand(0).isReg() && |
1185 | !getDebugOperand(0).getReg().isValid(); |
1186 | } |
1187 | |
1188 | bool isPHI() const { |
1189 | return getOpcode() == TargetOpcode::PHI || |
1190 | getOpcode() == TargetOpcode::G_PHI; |
1191 | } |
1192 | bool isKill() const { return getOpcode() == TargetOpcode::KILL; } |
1193 | bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } |
1194 | bool isInlineAsm() const { |
1195 | return getOpcode() == TargetOpcode::INLINEASM || |
1196 | getOpcode() == TargetOpcode::INLINEASM_BR; |
1197 | } |
1198 | |
1199 | /// FIXME: Seems like a layering violation that the AsmDialect, which is X86 |
1200 | /// specific, be attached to a generic MachineInstr. |
1201 | bool isMSInlineAsm() const { |
1202 | return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel; |
1203 | } |
1204 | |
1205 | bool isStackAligningInlineAsm() const; |
1206 | InlineAsm::AsmDialect getInlineAsmDialect() const; |
1207 | |
1208 | bool isInsertSubreg() const { |
1209 | return getOpcode() == TargetOpcode::INSERT_SUBREG; |
1210 | } |
1211 | |
1212 | bool isSubregToReg() const { |
1213 | return getOpcode() == TargetOpcode::SUBREG_TO_REG; |
1214 | } |
1215 | |
1216 | bool isRegSequence() const { |
1217 | return getOpcode() == TargetOpcode::REG_SEQUENCE; |
1218 | } |
1219 | |
1220 | bool isBundle() const { |
1221 | return getOpcode() == TargetOpcode::BUNDLE; |
1222 | } |
1223 | |
1224 | bool isCopy() const { |
1225 | return getOpcode() == TargetOpcode::COPY; |
1226 | } |
1227 | |
1228 | bool isFullCopy() const { |
1229 | return isCopy() && !getOperand(0).getSubReg() && !getOperand(1).getSubReg(); |
1230 | } |
1231 | |
1232 | bool isExtractSubreg() const { |
1233 | return getOpcode() == TargetOpcode::EXTRACT_SUBREG; |
1234 | } |
1235 | |
1236 | /// Return true if the instruction behaves like a copy. |
1237 | /// This does not include native copy instructions. |
1238 | bool isCopyLike() const { |
1239 | return isCopy() || isSubregToReg(); |
1240 | } |
1241 | |
1242 | /// Return true is the instruction is an identity copy. |
1243 | bool isIdentityCopy() const { |
1244 | return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() && |
1245 | getOperand(0).getSubReg() == getOperand(1).getSubReg(); |
1246 | } |
1247 | |
1248 | /// Return true if this instruction doesn't produce any output in the form of |
1249 | /// executable instructions. |
1250 | bool isMetaInstruction() const { |
1251 | switch (getOpcode()) { |
1252 | default: |
1253 | return false; |
1254 | case TargetOpcode::IMPLICIT_DEF: |
1255 | case TargetOpcode::KILL: |
1256 | case TargetOpcode::CFI_INSTRUCTION: |
1257 | case TargetOpcode::EH_LABEL: |
1258 | case TargetOpcode::GC_LABEL: |
1259 | case TargetOpcode::DBG_VALUE: |
1260 | case TargetOpcode::DBG_INSTR_REF: |
1261 | case TargetOpcode::DBG_LABEL: |
1262 | case TargetOpcode::LIFETIME_START: |
1263 | case TargetOpcode::LIFETIME_END: |
1264 | case TargetOpcode::PSEUDO_PROBE: |
1265 | return true; |
1266 | } |
1267 | } |
1268 | |
1269 | /// Return true if this is a transient instruction that is either very likely |
1270 | /// to be eliminated during register allocation (such as copy-like |
1271 | /// instructions), or if this instruction doesn't have an execution-time cost. |
1272 | bool isTransient() const { |
1273 | switch (getOpcode()) { |
1274 | default: |
1275 | return isMetaInstruction(); |
1276 | // Copy-like instructions are usually eliminated during register allocation. |
1277 | case TargetOpcode::PHI: |
1278 | case TargetOpcode::G_PHI: |
1279 | case TargetOpcode::COPY: |
1280 | case TargetOpcode::INSERT_SUBREG: |
1281 | case TargetOpcode::SUBREG_TO_REG: |
1282 | case TargetOpcode::REG_SEQUENCE: |
1283 | return true; |
1284 | } |
1285 | } |
1286 | |
1287 | /// Return the number of instructions inside the MI bundle, excluding the |
1288 | /// bundle header. |
1289 | /// |
1290 | /// This is the number of instructions that MachineBasicBlock::iterator |
1291 | /// skips, 0 for unbundled instructions. |
1292 | unsigned getBundleSize() const; |
1293 | |
1294 | /// Return true if the MachineInstr reads the specified register. |
1295 | /// If TargetRegisterInfo is passed, then it also checks if there |
1296 | /// is a read of a super-register. |
1297 | /// This does not count partial redefines of virtual registers as reads: |
1298 | /// %reg1024:6 = OP. |
1299 | bool readsRegister(Register Reg, |
1300 | const TargetRegisterInfo *TRI = nullptr) const { |
1301 | return findRegisterUseOperandIdx(Reg, false, TRI) != -1; |
1302 | } |
1303 | |
1304 | /// Return true if the MachineInstr reads the specified virtual register. |
1305 | /// Take into account that a partial define is a |
1306 | /// read-modify-write operation. |
1307 | bool readsVirtualRegister(Register Reg) const { |
1308 | return readsWritesVirtualRegister(Reg).first; |
1309 | } |
1310 | |
1311 | /// Return a pair of bools (reads, writes) indicating if this instruction |
1312 | /// reads or writes Reg. This also considers partial defines. |
1313 | /// If Ops is not null, all operand indices for Reg are added. |
1314 | std::pair<bool,bool> readsWritesVirtualRegister(Register Reg, |
1315 | SmallVectorImpl<unsigned> *Ops = nullptr) const; |
1316 | |
1317 | /// Return true if the MachineInstr kills the specified register. |
1318 | /// If TargetRegisterInfo is passed, then it also checks if there is |
1319 | /// a kill of a super-register. |
1320 | bool killsRegister(Register Reg, |
1321 | const TargetRegisterInfo *TRI = nullptr) const { |
1322 | return findRegisterUseOperandIdx(Reg, true, TRI) != -1; |
1323 | } |
1324 | |
1325 | /// Return true if the MachineInstr fully defines the specified register. |
1326 | /// If TargetRegisterInfo is passed, then it also checks |
1327 | /// if there is a def of a super-register. |
1328 | /// NOTE: It's ignoring subreg indices on virtual registers. |
1329 | bool definesRegister(Register Reg, |
1330 | const TargetRegisterInfo *TRI = nullptr) const { |
1331 | return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1; |
1332 | } |
1333 | |
1334 | /// Return true if the MachineInstr modifies (fully define or partially |
1335 | /// define) the specified register. |
1336 | /// NOTE: It's ignoring subreg indices on virtual registers. |
1337 | bool modifiesRegister(Register Reg, |
1338 | const TargetRegisterInfo *TRI = nullptr) const { |
1339 | return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1; |
1340 | } |
1341 | |
1342 | /// Returns true if the register is dead in this machine instruction. |
1343 | /// If TargetRegisterInfo is passed, then it also checks |
1344 | /// if there is a dead def of a super-register. |
1345 | bool registerDefIsDead(Register Reg, |
1346 | const TargetRegisterInfo *TRI = nullptr) const { |
1347 | return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1; |
1348 | } |
1349 | |
1350 | /// Returns true if the MachineInstr has an implicit-use operand of exactly |
1351 | /// the given register (not considering sub/super-registers). |
1352 | bool hasRegisterImplicitUseOperand(Register Reg) const; |
1353 | |
1354 | /// Returns the operand index that is a use of the specific register or -1 |
1355 | /// if it is not found. It further tightens the search criteria to a use |
1356 | /// that kills the register if isKill is true. |
1357 | int findRegisterUseOperandIdx(Register Reg, bool isKill = false, |
1358 | const TargetRegisterInfo *TRI = nullptr) const; |
1359 | |
1360 | /// Wrapper for findRegisterUseOperandIdx, it returns |
1361 | /// a pointer to the MachineOperand rather than an index. |
1362 | MachineOperand *findRegisterUseOperand(Register Reg, bool isKill = false, |
1363 | const TargetRegisterInfo *TRI = nullptr) { |
1364 | int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI); |
1365 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
1366 | } |
1367 | |
1368 | const MachineOperand *findRegisterUseOperand( |
1369 | Register Reg, bool isKill = false, |
1370 | const TargetRegisterInfo *TRI = nullptr) const { |
1371 | return const_cast<MachineInstr *>(this)-> |
1372 | findRegisterUseOperand(Reg, isKill, TRI); |
1373 | } |
1374 | |
1375 | /// Returns the operand index that is a def of the specified register or |
1376 | /// -1 if it is not found. If isDead is true, defs that are not dead are |
1377 | /// skipped. If Overlap is true, then it also looks for defs that merely |
1378 | /// overlap the specified register. If TargetRegisterInfo is non-null, |
1379 | /// then it also checks if there is a def of a super-register. |
1380 | /// This may also return a register mask operand when Overlap is true. |
1381 | int findRegisterDefOperandIdx(Register Reg, |
1382 | bool isDead = false, bool Overlap = false, |
1383 | const TargetRegisterInfo *TRI = nullptr) const; |
1384 | |
1385 | /// Wrapper for findRegisterDefOperandIdx, it returns |
1386 | /// a pointer to the MachineOperand rather than an index. |
1387 | MachineOperand * |
1388 | findRegisterDefOperand(Register Reg, bool isDead = false, |
1389 | bool Overlap = false, |
1390 | const TargetRegisterInfo *TRI = nullptr) { |
1391 | int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI); |
1392 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
1393 | } |
1394 | |
1395 | const MachineOperand * |
1396 | findRegisterDefOperand(Register Reg, bool isDead = false, |
1397 | bool Overlap = false, |
1398 | const TargetRegisterInfo *TRI = nullptr) const { |
1399 | return const_cast<MachineInstr *>(this)->findRegisterDefOperand( |
1400 | Reg, isDead, Overlap, TRI); |
1401 | } |
1402 | |
1403 | /// Find the index of the first operand in the |
1404 | /// operand list that is used to represent the predicate. It returns -1 if |
1405 | /// none is found. |
1406 | int findFirstPredOperandIdx() const; |
1407 | |
1408 | /// Find the index of the flag word operand that |
1409 | /// corresponds to operand OpIdx on an inline asm instruction. Returns -1 if |
1410 | /// getOperand(OpIdx) does not belong to an inline asm operand group. |
1411 | /// |
1412 | /// If GroupNo is not NULL, it will receive the number of the operand group |
1413 | /// containing OpIdx. |
1414 | /// |
1415 | /// The flag operand is an immediate that can be decoded with methods like |
1416 | /// InlineAsm::hasRegClassConstraint(). |
1417 | int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const; |
1418 | |
1419 | /// Compute the static register class constraint for operand OpIdx. |
1420 | /// For normal instructions, this is derived from the MCInstrDesc. |
1421 | /// For inline assembly it is derived from the flag words. |
1422 | /// |
1423 | /// Returns NULL if the static register class constraint cannot be |
1424 | /// determined. |
1425 | const TargetRegisterClass* |
1426 | getRegClassConstraint(unsigned OpIdx, |
1427 | const TargetInstrInfo *TII, |
1428 | const TargetRegisterInfo *TRI) const; |
1429 | |
1430 | /// Applies the constraints (def/use) implied by this MI on \p Reg to |
1431 | /// the given \p CurRC. |
1432 | /// If \p ExploreBundle is set and MI is part of a bundle, all the |
1433 | /// instructions inside the bundle will be taken into account. In other words, |
1434 | /// this method accumulates all the constraints of the operand of this MI and |
1435 | /// the related bundle if MI is a bundle or inside a bundle. |
1436 | /// |
1437 | /// Returns the register class that satisfies both \p CurRC and the |
1438 | /// constraints set by MI. Returns NULL if such a register class does not |
1439 | /// exist. |
1440 | /// |
1441 | /// \pre CurRC must not be NULL. |
1442 | const TargetRegisterClass *getRegClassConstraintEffectForVReg( |
1443 | Register Reg, const TargetRegisterClass *CurRC, |
1444 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, |
1445 | bool ExploreBundle = false) const; |
1446 | |
1447 | /// Applies the constraints (def/use) implied by the \p OpIdx operand |
1448 | /// to the given \p CurRC. |
1449 | /// |
1450 | /// Returns the register class that satisfies both \p CurRC and the |
1451 | /// constraints set by \p OpIdx MI. Returns NULL if such a register class |
1452 | /// does not exist. |
1453 | /// |
1454 | /// \pre CurRC must not be NULL. |
1455 | /// \pre The operand at \p OpIdx must be a register. |
1456 | const TargetRegisterClass * |
1457 | getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC, |
1458 | const TargetInstrInfo *TII, |
1459 | const TargetRegisterInfo *TRI) const; |
1460 | |
1461 | /// Add a tie between the register operands at DefIdx and UseIdx. |
1462 | /// The tie will cause the register allocator to ensure that the two |
1463 | /// operands are assigned the same physical register. |
1464 | /// |
1465 | /// Tied operands are managed automatically for explicit operands in the |
1466 | /// MCInstrDesc. This method is for exceptional cases like inline asm. |
1467 | void tieOperands(unsigned DefIdx, unsigned UseIdx); |
1468 | |
1469 | /// Given the index of a tied register operand, find the |
1470 | /// operand it is tied to. Defs are tied to uses and vice versa. Returns the |
1471 | /// index of the tied operand which must exist. |
1472 | unsigned findTiedOperandIdx(unsigned OpIdx) const; |
1473 | |
1474 | /// Given the index of a register def operand, |
1475 | /// check if the register def is tied to a source operand, due to either |
1476 | /// two-address elimination or inline assembly constraints. Returns the |
1477 | /// first tied use operand index by reference if UseOpIdx is not null. |
1478 | bool isRegTiedToUseOperand(unsigned DefOpIdx, |
1479 | unsigned *UseOpIdx = nullptr) const { |
1480 | const MachineOperand &MO = getOperand(DefOpIdx); |
1481 | if (!MO.isReg() || !MO.isDef() || !MO.isTied()) |
1482 | return false; |
1483 | if (UseOpIdx) |
1484 | *UseOpIdx = findTiedOperandIdx(DefOpIdx); |
1485 | return true; |
1486 | } |
1487 | |
1488 | /// Return true if the use operand of the specified index is tied to a def |
1489 | /// operand. It also returns the def operand index by reference if DefOpIdx |
1490 | /// is not null. |
1491 | bool isRegTiedToDefOperand(unsigned UseOpIdx, |
1492 | unsigned *DefOpIdx = nullptr) const { |
1493 | const MachineOperand &MO = getOperand(UseOpIdx); |
1494 | if (!MO.isReg() || !MO.isUse() || !MO.isTied()) |
1495 | return false; |
1496 | if (DefOpIdx) |
1497 | *DefOpIdx = findTiedOperandIdx(UseOpIdx); |
1498 | return true; |
1499 | } |
1500 | |
1501 | /// Clears kill flags on all operands. |
1502 | void clearKillInfo(); |
1503 | |
1504 | /// Replace all occurrences of FromReg with ToReg:SubIdx, |
1505 | /// properly composing subreg indices where necessary. |
1506 | void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, |
1507 | const TargetRegisterInfo &RegInfo); |
1508 | |
1509 | /// We have determined MI kills a register. Look for the |
1510 | /// operand that uses it and mark it as IsKill. If AddIfNotFound is true, |
1511 | /// add a implicit operand if it's not found. Returns true if the operand |
1512 | /// exists / is added. |
1513 | bool addRegisterKilled(Register IncomingReg, |
1514 | const TargetRegisterInfo *RegInfo, |
1515 | bool AddIfNotFound = false); |
1516 | |
1517 | /// Clear all kill flags affecting Reg. If RegInfo is provided, this includes |
1518 | /// all aliasing registers. |
1519 | void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo); |
1520 | |
1521 | /// We have determined MI defined a register without a use. |
1522 | /// Look for the operand that defines it and mark it as IsDead. If |
1523 | /// AddIfNotFound is true, add a implicit operand if it's not found. Returns |
1524 | /// true if the operand exists / is added. |
1525 | bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, |
1526 | bool AddIfNotFound = false); |
1527 | |
1528 | /// Clear all dead flags on operands defining register @p Reg. |
1529 | void clearRegisterDeads(Register Reg); |
1530 | |
1531 | /// Mark all subregister defs of register @p Reg with the undef flag. |
1532 | /// This function is used when we determined to have a subregister def in an |
1533 | /// otherwise undefined super register. |
1534 | void setRegisterDefReadUndef(Register Reg, bool IsUndef = true); |
1535 | |
1536 | /// We have determined MI defines a register. Make sure there is an operand |
1537 | /// defining Reg. |
1538 | void addRegisterDefined(Register Reg, |
1539 | const TargetRegisterInfo *RegInfo = nullptr); |
1540 | |
1541 | /// Mark every physreg used by this instruction as |
1542 | /// dead except those in the UsedRegs list. |
1543 | /// |
1544 | /// On instructions with register mask operands, also add implicit-def |
1545 | /// operands for all registers in UsedRegs. |
1546 | void setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs, |
1547 | const TargetRegisterInfo &TRI); |
1548 | |
1549 | /// Return true if it is safe to move this instruction. If |
1550 | /// SawStore is set to true, it means that there is a store (or call) between |
1551 | /// the instruction's location and its intended destination. |
1552 | bool isSafeToMove(AAResults *AA, bool &SawStore) const; |
1553 | |
1554 | /// Returns true if this instruction's memory access aliases the memory |
1555 | /// access of Other. |
1556 | // |
1557 | /// Assumes any physical registers used to compute addresses |
1558 | /// have the same value for both instructions. Returns false if neither |
1559 | /// instruction writes to memory. |
1560 | /// |
1561 | /// @param AA Optional alias analysis, used to compare memory operands. |
1562 | /// @param Other MachineInstr to check aliasing against. |
1563 | /// @param UseTBAA Whether to pass TBAA information to alias analysis. |
1564 | bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const; |
1565 | |
1566 | /// Return true if this instruction may have an ordered |
1567 | /// or volatile memory reference, or if the information describing the memory |
1568 | /// reference is not available. Return false if it is known to have no |
1569 | /// ordered or volatile memory references. |
1570 | bool hasOrderedMemoryRef() const; |
1571 | |
1572 | /// Return true if this load instruction never traps and points to a memory |
1573 | /// location whose value doesn't change during the execution of this function. |
1574 | /// |
1575 | /// Examples include loading a value from the constant pool or from the |
1576 | /// argument area of a function (if it does not change). If the instruction |
1577 | /// does multiple loads, this returns true only if all of the loads are |
1578 | /// dereferenceable and invariant. |
1579 | bool isDereferenceableInvariantLoad(AAResults *AA) const; |
1580 | |
1581 | /// If the specified instruction is a PHI that always merges together the |
1582 | /// same virtual register, return the register, otherwise return 0. |
1583 | unsigned isConstantValuePHI() const; |
1584 | |
1585 | /// Return true if this instruction has side effects that are not modeled |
1586 | /// by mayLoad / mayStore, etc. |
1587 | /// For all instructions, the property is encoded in MCInstrDesc::Flags |
1588 | /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is |
1589 | /// INLINEASM instruction, in which case the side effect property is encoded |
1590 | /// in one of its operands (see InlineAsm::Extra_HasSideEffect). |
1591 | /// |
1592 | bool hasUnmodeledSideEffects() const; |
1593 | |
1594 | /// Returns true if it is illegal to fold a load across this instruction. |
1595 | bool isLoadFoldBarrier() const; |
1596 | |
1597 | /// Return true if all the defs of this instruction are dead. |
1598 | bool allDefsAreDead() const; |
1599 | |
1600 | /// Return a valid size if the instruction is a spill instruction. |
1601 | Optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const; |
1602 | |
1603 | /// Return a valid size if the instruction is a folded spill instruction. |
1604 | Optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const; |
1605 | |
1606 | /// Return a valid size if the instruction is a restore instruction. |
1607 | Optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const; |
1608 | |
1609 | /// Return a valid size if the instruction is a folded restore instruction. |
1610 | Optional<unsigned> |
1611 | getFoldedRestoreSize(const TargetInstrInfo *TII) const; |
1612 | |
1613 | /// Copy implicit register operands from specified |
1614 | /// instruction to this instruction. |
1615 | void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI); |
1616 | |
1617 | /// Debugging support |
1618 | /// @{ |
1619 | /// Determine the generic type to be printed (if needed) on uses and defs. |
1620 | LLT getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes, |
1621 | const MachineRegisterInfo &MRI) const; |
1622 | |
1623 | /// Return true when an instruction has tied register that can't be determined |
1624 | /// by the instruction's descriptor. This is useful for MIR printing, to |
1625 | /// determine whether we need to print the ties or not. |
1626 | bool hasComplexRegisterTies() const; |
1627 | |
1628 | /// Print this MI to \p OS. |
1629 | /// Don't print information that can be inferred from other instructions if |
1630 | /// \p IsStandalone is false. It is usually true when only a fragment of the |
1631 | /// function is printed. |
1632 | /// Only print the defs and the opcode if \p SkipOpers is true. |
1633 | /// Otherwise, also print operands if \p SkipDebugLoc is true. |
1634 | /// Otherwise, also print the debug loc, with a terminating newline. |
1635 | /// \p TII is used to print the opcode name. If it's not present, but the |
1636 | /// MI is in a function, the opcode will be printed using the function's TII. |
1637 | void print(raw_ostream &OS, bool IsStandalone = true, bool SkipOpers = false, |
1638 | bool SkipDebugLoc = false, bool AddNewLine = true, |
1639 | const TargetInstrInfo *TII = nullptr) const; |
1640 | void print(raw_ostream &OS, ModuleSlotTracker &MST, bool IsStandalone = true, |
1641 | bool SkipOpers = false, bool SkipDebugLoc = false, |
1642 | bool AddNewLine = true, |
1643 | const TargetInstrInfo *TII = nullptr) const; |
1644 | void dump() const; |
1645 | /// Print on dbgs() the current instruction and the instructions defining its |
1646 | /// operands and so on until we reach \p MaxDepth. |
1647 | void dumpr(const MachineRegisterInfo &MRI, |
1648 | unsigned MaxDepth = UINT_MAX(2147483647 *2U +1U)) const; |
1649 | /// @} |
1650 | |
1651 | //===--------------------------------------------------------------------===// |
1652 | // Accessors used to build up machine instructions. |
1653 | |
1654 | /// Add the specified operand to the instruction. If it is an implicit |
1655 | /// operand, it is added to the end of the operand list. If it is an |
1656 | /// explicit operand it is added at the end of the explicit operand list |
1657 | /// (before the first implicit operand). |
1658 | /// |
1659 | /// MF must be the machine function that was used to allocate this |
1660 | /// instruction. |
1661 | /// |
1662 | /// MachineInstrBuilder provides a more convenient interface for creating |
1663 | /// instructions and adding operands. |
1664 | void addOperand(MachineFunction &MF, const MachineOperand &Op); |
1665 | |
1666 | /// Add an operand without providing an MF reference. This only works for |
1667 | /// instructions that are inserted in a basic block. |
1668 | /// |
1669 | /// MachineInstrBuilder and the two-argument addOperand(MF, MO) should be |
1670 | /// preferred. |
1671 | void addOperand(const MachineOperand &Op); |
1672 | |
1673 | /// Replace the instruction descriptor (thus opcode) of |
1674 | /// the current instruction with a new one. |
1675 | void setDesc(const MCInstrDesc &tid) { MCID = &tid; } |
1676 | |
1677 | /// Replace current source information with new such. |
1678 | /// Avoid using this, the constructor argument is preferable. |
1679 | void setDebugLoc(DebugLoc dl) { |
1680 | debugLoc = std::move(dl); |
1681 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor" ) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 1681, __PRETTY_FUNCTION__)); |
1682 | } |
1683 | |
1684 | /// Erase an operand from an instruction, leaving it with one |
1685 | /// fewer operand than it started with. |
1686 | void RemoveOperand(unsigned OpNo); |
1687 | |
1688 | /// Clear this MachineInstr's memory reference descriptor list. This resets |
1689 | /// the memrefs to their most conservative state. This should be used only |
1690 | /// as a last resort since it greatly pessimizes our knowledge of the memory |
1691 | /// access performed by the instruction. |
1692 | void dropMemRefs(MachineFunction &MF); |
1693 | |
1694 | /// Assign this MachineInstr's memory reference descriptor list. |
1695 | /// |
1696 | /// Unlike other methods, this *will* allocate them into a new array |
1697 | /// associated with the provided `MachineFunction`. |
1698 | void setMemRefs(MachineFunction &MF, ArrayRef<MachineMemOperand *> MemRefs); |
1699 | |
1700 | /// Add a MachineMemOperand to the machine instruction. |
1701 | /// This function should be used only occasionally. The setMemRefs function |
1702 | /// is the primary method for setting up a MachineInstr's MemRefs list. |
1703 | void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); |
1704 | |
1705 | /// Clone another MachineInstr's memory reference descriptor list and replace |
1706 | /// ours with it. |
1707 | /// |
1708 | /// Note that `*this` may be the incoming MI! |
1709 | /// |
1710 | /// Prefer this API whenever possible as it can avoid allocations in common |
1711 | /// cases. |
1712 | void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI); |
1713 | |
1714 | /// Clone the merge of multiple MachineInstrs' memory reference descriptors |
1715 | /// list and replace ours with it. |
1716 | /// |
1717 | /// Note that `*this` may be one of the incoming MIs! |
1718 | /// |
1719 | /// Prefer this API whenever possible as it can avoid allocations in common |
1720 | /// cases. |
1721 | void cloneMergedMemRefs(MachineFunction &MF, |
1722 | ArrayRef<const MachineInstr *> MIs); |
1723 | |
1724 | /// Set a symbol that will be emitted just prior to the instruction itself. |
1725 | /// |
1726 | /// Setting this to a null pointer will remove any such symbol. |
1727 | /// |
1728 | /// FIXME: This is not fully implemented yet. |
1729 | void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
1730 | |
1731 | /// Set a symbol that will be emitted just after the instruction itself. |
1732 | /// |
1733 | /// Setting this to a null pointer will remove any such symbol. |
1734 | /// |
1735 | /// FIXME: This is not fully implemented yet. |
1736 | void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
1737 | |
1738 | /// Clone another MachineInstr's pre- and post- instruction symbols and |
1739 | /// replace ours with it. |
1740 | void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI); |
1741 | |
1742 | /// Set a marker on instructions that denotes where we should create and emit |
1743 | /// heap alloc site labels. This waits until after instruction selection and |
1744 | /// optimizations to create the label, so it should still work if the |
1745 | /// instruction is removed or duplicated. |
1746 | void setHeapAllocMarker(MachineFunction &MF, MDNode *MD); |
1747 | |
1748 | /// Return the MIFlags which represent both MachineInstrs. This |
1749 | /// should be used when merging two MachineInstrs into one. This routine does |
1750 | /// not modify the MIFlags of this MachineInstr. |
1751 | uint16_t mergeFlagsWith(const MachineInstr& Other) const; |
1752 | |
1753 | static uint16_t copyFlagsFromInstruction(const Instruction &I); |
1754 | |
1755 | /// Copy all flags to MachineInst MIFlags |
1756 | void copyIRFlags(const Instruction &I); |
1757 | |
1758 | /// Break any tie involving OpIdx. |
1759 | void untieRegOperand(unsigned OpIdx) { |
1760 | MachineOperand &MO = getOperand(OpIdx); |
1761 | if (MO.isReg() && MO.isTied()) { |
1762 | getOperand(findTiedOperandIdx(OpIdx)).TiedTo = 0; |
1763 | MO.TiedTo = 0; |
1764 | } |
1765 | } |
1766 | |
1767 | /// Add all implicit def and use operands to this instruction. |
1768 | void addImplicitDefUseOperands(MachineFunction &MF); |
1769 | |
1770 | /// Scan instructions immediately following MI and collect any matching |
1771 | /// DBG_VALUEs. |
1772 | void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues); |
1773 | |
1774 | /// Find all DBG_VALUEs that point to the register def in this instruction |
1775 | /// and point them to \p Reg instead. |
1776 | void changeDebugValuesDefReg(Register Reg); |
1777 | |
1778 | /// Returns the Intrinsic::ID for this instruction. |
1779 | /// \pre Must have an intrinsic ID operand. |
1780 | unsigned getIntrinsicID() const { |
1781 | return getOperand(getNumExplicitDefs()).getIntrinsicID(); |
1782 | } |
1783 | |
1784 | /// Sets all register debug operands in this debug value instruction to be |
1785 | /// undef. |
1786 | void setDebugValueUndef() { |
1787 | assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction." ) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h" , 1787, __PRETTY_FUNCTION__)); |
1788 | for (MachineOperand &MO : debug_operands()) { |
1789 | if (MO.isReg()) { |
1790 | MO.setReg(0); |
1791 | MO.setSubReg(0); |
1792 | } |
1793 | } |
1794 | } |
1795 | |
1796 | private: |
1797 | /// If this instruction is embedded into a MachineFunction, return the |
1798 | /// MachineRegisterInfo object for the current function, otherwise |
1799 | /// return null. |
1800 | MachineRegisterInfo *getRegInfo(); |
1801 | |
1802 | /// Unlink all of the register operands in this instruction from their |
1803 | /// respective use lists. This requires that the operands already be on their |
1804 | /// use lists. |
1805 | void RemoveRegOperandsFromUseLists(MachineRegisterInfo&); |
1806 | |
1807 | /// Add all of the register operands in this instruction from their |
1808 | /// respective use lists. This requires that the operands not be on their |
1809 | /// use lists yet. |
1810 | void AddRegOperandsToUseLists(MachineRegisterInfo&); |
1811 | |
1812 | /// Slow path for hasProperty when we're dealing with a bundle. |
1813 | bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; |
1814 | |
1815 | /// Implements the logic of getRegClassConstraintEffectForVReg for the |
1816 | /// this MI and the given operand index \p OpIdx. |
1817 | /// If the related operand does not constrained Reg, this returns CurRC. |
1818 | const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( |
1819 | unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC, |
1820 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; |
1821 | |
1822 | /// Stores extra instruction information inline or allocates as ExtraInfo |
1823 | /// based on the number of pointers. |
1824 | void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs, |
1825 | MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, |
1826 | MDNode *HeapAllocMarker); |
1827 | }; |
1828 | |
1829 | /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the |
1830 | /// instruction rather than by pointer value. |
1831 | /// The hashing and equality testing functions ignore definitions so this is |
1832 | /// useful for CSE, etc. |
1833 | struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> { |
1834 | static inline MachineInstr *getEmptyKey() { |
1835 | return nullptr; |
1836 | } |
1837 | |
1838 | static inline MachineInstr *getTombstoneKey() { |
1839 | return reinterpret_cast<MachineInstr*>(-1); |
1840 | } |
1841 | |
1842 | static unsigned getHashValue(const MachineInstr* const &MI); |
1843 | |
1844 | static bool isEqual(const MachineInstr* const &LHS, |
1845 | const MachineInstr* const &RHS) { |
1846 | if (RHS == getEmptyKey() || RHS == getTombstoneKey() || |
1847 | LHS == getEmptyKey() || LHS == getTombstoneKey()) |
1848 | return LHS == RHS; |
1849 | return LHS->isIdenticalTo(*RHS, MachineInstr::IgnoreVRegDefs); |
1850 | } |
1851 | }; |
1852 | |
1853 | //===----------------------------------------------------------------------===// |
1854 | // Debugging Support |
1855 | |
1856 | inline raw_ostream& operator<<(raw_ostream &OS, const MachineInstr &MI) { |
1857 | MI.print(OS); |
1858 | return OS; |
1859 | } |
1860 | |
1861 | } // end namespace llvm |
1862 | |
1863 | #endif // LLVM_CODEGEN_MACHINEINSTR_H |