File: | llvm/include/llvm/CodeGen/SelectionDAGNodes.h |
Warning: | line 1114, column 10 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===// | ||||||
2 | // | ||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
6 | // | ||||||
7 | //===----------------------------------------------------------------------===// | ||||||
8 | // | ||||||
9 | // This file implements the interfaces that Hexagon uses to lower LLVM code | ||||||
10 | // into a selection DAG. | ||||||
11 | // | ||||||
12 | //===----------------------------------------------------------------------===// | ||||||
13 | |||||||
14 | #include "HexagonISelLowering.h" | ||||||
15 | #include "Hexagon.h" | ||||||
16 | #include "HexagonMachineFunctionInfo.h" | ||||||
17 | #include "HexagonRegisterInfo.h" | ||||||
18 | #include "HexagonSubtarget.h" | ||||||
19 | #include "HexagonTargetMachine.h" | ||||||
20 | #include "HexagonTargetObjectFile.h" | ||||||
21 | #include "llvm/ADT/APInt.h" | ||||||
22 | #include "llvm/ADT/ArrayRef.h" | ||||||
23 | #include "llvm/ADT/SmallVector.h" | ||||||
24 | #include "llvm/ADT/StringSwitch.h" | ||||||
25 | #include "llvm/CodeGen/CallingConvLower.h" | ||||||
26 | #include "llvm/CodeGen/MachineFrameInfo.h" | ||||||
27 | #include "llvm/CodeGen/MachineFunction.h" | ||||||
28 | #include "llvm/CodeGen/MachineMemOperand.h" | ||||||
29 | #include "llvm/CodeGen/MachineRegisterInfo.h" | ||||||
30 | #include "llvm/CodeGen/RuntimeLibcalls.h" | ||||||
31 | #include "llvm/CodeGen/SelectionDAG.h" | ||||||
32 | #include "llvm/CodeGen/TargetCallingConv.h" | ||||||
33 | #include "llvm/CodeGen/ValueTypes.h" | ||||||
34 | #include "llvm/IR/BasicBlock.h" | ||||||
35 | #include "llvm/IR/CallingConv.h" | ||||||
36 | #include "llvm/IR/DataLayout.h" | ||||||
37 | #include "llvm/IR/DerivedTypes.h" | ||||||
38 | #include "llvm/IR/DiagnosticInfo.h" | ||||||
39 | #include "llvm/IR/DiagnosticPrinter.h" | ||||||
40 | #include "llvm/IR/Function.h" | ||||||
41 | #include "llvm/IR/GlobalValue.h" | ||||||
42 | #include "llvm/IR/InlineAsm.h" | ||||||
43 | #include "llvm/IR/Instructions.h" | ||||||
44 | #include "llvm/IR/IntrinsicInst.h" | ||||||
45 | #include "llvm/IR/Intrinsics.h" | ||||||
46 | #include "llvm/IR/IntrinsicsHexagon.h" | ||||||
47 | #include "llvm/IR/IRBuilder.h" | ||||||
48 | #include "llvm/IR/Module.h" | ||||||
49 | #include "llvm/IR/Type.h" | ||||||
50 | #include "llvm/IR/Value.h" | ||||||
51 | #include "llvm/MC/MCRegisterInfo.h" | ||||||
52 | #include "llvm/Support/Casting.h" | ||||||
53 | #include "llvm/Support/CodeGen.h" | ||||||
54 | #include "llvm/Support/CommandLine.h" | ||||||
55 | #include "llvm/Support/Debug.h" | ||||||
56 | #include "llvm/Support/ErrorHandling.h" | ||||||
57 | #include "llvm/Support/MathExtras.h" | ||||||
58 | #include "llvm/Support/raw_ostream.h" | ||||||
59 | #include "llvm/Target/TargetMachine.h" | ||||||
60 | #include <algorithm> | ||||||
61 | #include <cassert> | ||||||
62 | #include <cstddef> | ||||||
63 | #include <cstdint> | ||||||
64 | #include <limits> | ||||||
65 | #include <utility> | ||||||
66 | |||||||
67 | using namespace llvm; | ||||||
68 | |||||||
69 | #define DEBUG_TYPE"hexagon-lowering" "hexagon-lowering" | ||||||
70 | |||||||
71 | static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables", | ||||||
72 | cl::init(true), cl::Hidden, | ||||||
73 | cl::desc("Control jump table emission on Hexagon target")); | ||||||
74 | |||||||
75 | static cl::opt<bool> EnableHexSDNodeSched("enable-hexagon-sdnode-sched", | ||||||
76 | cl::Hidden, cl::ZeroOrMore, cl::init(false), | ||||||
77 | cl::desc("Enable Hexagon SDNode scheduling")); | ||||||
78 | |||||||
79 | static cl::opt<bool> EnableFastMath("ffast-math", | ||||||
80 | cl::Hidden, cl::ZeroOrMore, cl::init(false), | ||||||
81 | cl::desc("Enable Fast Math processing")); | ||||||
82 | |||||||
83 | static cl::opt<int> MinimumJumpTables("minimum-jump-tables", | ||||||
84 | cl::Hidden, cl::ZeroOrMore, cl::init(5), | ||||||
85 | cl::desc("Set minimum jump tables")); | ||||||
86 | |||||||
87 | static cl::opt<int> MaxStoresPerMemcpyCL("max-store-memcpy", | ||||||
88 | cl::Hidden, cl::ZeroOrMore, cl::init(6), | ||||||
89 | cl::desc("Max #stores to inline memcpy")); | ||||||
90 | |||||||
91 | static cl::opt<int> MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os", | ||||||
92 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | ||||||
93 | cl::desc("Max #stores to inline memcpy")); | ||||||
94 | |||||||
95 | static cl::opt<int> MaxStoresPerMemmoveCL("max-store-memmove", | ||||||
96 | cl::Hidden, cl::ZeroOrMore, cl::init(6), | ||||||
97 | cl::desc("Max #stores to inline memmove")); | ||||||
98 | |||||||
99 | static cl::opt<int> MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os", | ||||||
100 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | ||||||
101 | cl::desc("Max #stores to inline memmove")); | ||||||
102 | |||||||
103 | static cl::opt<int> MaxStoresPerMemsetCL("max-store-memset", | ||||||
104 | cl::Hidden, cl::ZeroOrMore, cl::init(8), | ||||||
105 | cl::desc("Max #stores to inline memset")); | ||||||
106 | |||||||
107 | static cl::opt<int> MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", | ||||||
108 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | ||||||
109 | cl::desc("Max #stores to inline memset")); | ||||||
110 | |||||||
111 | static cl::opt<bool> AlignLoads("hexagon-align-loads", | ||||||
112 | cl::Hidden, cl::init(false), | ||||||
113 | cl::desc("Rewrite unaligned loads as a pair of aligned loads")); | ||||||
114 | |||||||
115 | static cl::opt<bool> | ||||||
116 | DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden, | ||||||
117 | cl::init(false), | ||||||
118 | cl::desc("Disable minimum alignment of 1 for " | ||||||
119 | "arguments passed by value on stack")); | ||||||
120 | |||||||
121 | namespace { | ||||||
122 | |||||||
123 | class HexagonCCState : public CCState { | ||||||
124 | unsigned NumNamedVarArgParams = 0; | ||||||
125 | |||||||
126 | public: | ||||||
127 | HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF, | ||||||
128 | SmallVectorImpl<CCValAssign> &locs, LLVMContext &C, | ||||||
129 | unsigned NumNamedArgs) | ||||||
130 | : CCState(CC, IsVarArg, MF, locs, C), | ||||||
131 | NumNamedVarArgParams(NumNamedArgs) {} | ||||||
132 | unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; } | ||||||
133 | }; | ||||||
134 | |||||||
135 | } // end anonymous namespace | ||||||
136 | |||||||
137 | |||||||
138 | // Implement calling convention for Hexagon. | ||||||
139 | |||||||
140 | static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT, | ||||||
141 | CCValAssign::LocInfo &LocInfo, | ||||||
142 | ISD::ArgFlagsTy &ArgFlags, CCState &State) { | ||||||
143 | static const MCPhysReg ArgRegs[] = { | ||||||
144 | Hexagon::R0, Hexagon::R1, Hexagon::R2, | ||||||
145 | Hexagon::R3, Hexagon::R4, Hexagon::R5 | ||||||
146 | }; | ||||||
147 | const unsigned NumArgRegs = array_lengthof(ArgRegs); | ||||||
148 | unsigned RegNum = State.getFirstUnallocated(ArgRegs); | ||||||
149 | |||||||
150 | // RegNum is an index into ArgRegs: skip a register if RegNum is odd. | ||||||
151 | if (RegNum != NumArgRegs && RegNum % 2 == 1) | ||||||
152 | State.AllocateReg(ArgRegs[RegNum]); | ||||||
153 | |||||||
154 | // Always return false here, as this function only makes sure that the first | ||||||
155 | // unallocated register has an even register number and does not actually | ||||||
156 | // allocate a register for the current argument. | ||||||
157 | return false; | ||||||
158 | } | ||||||
159 | |||||||
160 | #include "HexagonGenCallingConv.inc" | ||||||
161 | |||||||
162 | |||||||
163 | SDValue | ||||||
164 | HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) | ||||||
165 | const { | ||||||
166 | return SDValue(); | ||||||
167 | } | ||||||
168 | |||||||
169 | /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified | ||||||
170 | /// by "Src" to address "Dst" of size "Size". Alignment information is | ||||||
171 | /// specified by the specific parameter attribute. The copy will be passed as | ||||||
172 | /// a byval function parameter. Sometimes what we are copying is the end of a | ||||||
173 | /// larger object, the part that does not fit in registers. | ||||||
174 | static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, | ||||||
175 | SDValue Chain, ISD::ArgFlagsTy Flags, | ||||||
176 | SelectionDAG &DAG, const SDLoc &dl) { | ||||||
177 | SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); | ||||||
178 | return DAG.getMemcpy( | ||||||
179 | Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(), | ||||||
180 | /*isVolatile=*/false, /*AlwaysInline=*/false, | ||||||
181 | /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo()); | ||||||
182 | } | ||||||
183 | |||||||
184 | bool | ||||||
185 | HexagonTargetLowering::CanLowerReturn( | ||||||
186 | CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, | ||||||
187 | const SmallVectorImpl<ISD::OutputArg> &Outs, | ||||||
188 | LLVMContext &Context) const { | ||||||
189 | SmallVector<CCValAssign, 16> RVLocs; | ||||||
190 | CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); | ||||||
191 | |||||||
192 | if (MF.getSubtarget<HexagonSubtarget>().useHVXOps()) | ||||||
193 | return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX); | ||||||
194 | return CCInfo.CheckReturn(Outs, RetCC_Hexagon); | ||||||
195 | } | ||||||
196 | |||||||
197 | // LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is | ||||||
198 | // passed by value, the function prototype is modified to return void and | ||||||
199 | // the value is stored in memory pointed by a pointer passed by caller. | ||||||
200 | SDValue | ||||||
201 | HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, | ||||||
202 | bool IsVarArg, | ||||||
203 | const SmallVectorImpl<ISD::OutputArg> &Outs, | ||||||
204 | const SmallVectorImpl<SDValue> &OutVals, | ||||||
205 | const SDLoc &dl, SelectionDAG &DAG) const { | ||||||
206 | // CCValAssign - represent the assignment of the return value to locations. | ||||||
207 | SmallVector<CCValAssign, 16> RVLocs; | ||||||
208 | |||||||
209 | // CCState - Info about the registers and stack slot. | ||||||
210 | CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, | ||||||
211 | *DAG.getContext()); | ||||||
212 | |||||||
213 | // Analyze return values of ISD::RET | ||||||
214 | if (Subtarget.useHVXOps()) | ||||||
215 | CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX); | ||||||
216 | else | ||||||
217 | CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon); | ||||||
218 | |||||||
219 | SDValue Flag; | ||||||
220 | SmallVector<SDValue, 4> RetOps(1, Chain); | ||||||
221 | |||||||
222 | // Copy the result values into the output registers. | ||||||
223 | for (unsigned i = 0; i != RVLocs.size(); ++i) { | ||||||
224 | CCValAssign &VA = RVLocs[i]; | ||||||
225 | SDValue Val = OutVals[i]; | ||||||
226 | |||||||
227 | switch (VA.getLocInfo()) { | ||||||
228 | default: | ||||||
229 | // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt. | ||||||
230 | llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 230); | ||||||
231 | case CCValAssign::Full: | ||||||
232 | break; | ||||||
233 | case CCValAssign::BCvt: | ||||||
234 | Val = DAG.getBitcast(VA.getLocVT(), Val); | ||||||
235 | break; | ||||||
236 | case CCValAssign::SExt: | ||||||
237 | Val = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Val); | ||||||
238 | break; | ||||||
239 | case CCValAssign::ZExt: | ||||||
240 | Val = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Val); | ||||||
241 | break; | ||||||
242 | case CCValAssign::AExt: | ||||||
243 | Val = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Val); | ||||||
244 | break; | ||||||
245 | } | ||||||
246 | |||||||
247 | Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Val, Flag); | ||||||
248 | |||||||
249 | // Guarantee that all emitted copies are stuck together with flags. | ||||||
250 | Flag = Chain.getValue(1); | ||||||
251 | RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); | ||||||
252 | } | ||||||
253 | |||||||
254 | RetOps[0] = Chain; // Update chain. | ||||||
255 | |||||||
256 | // Add the flag if we have it. | ||||||
257 | if (Flag.getNode()) | ||||||
258 | RetOps.push_back(Flag); | ||||||
259 | |||||||
260 | return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps); | ||||||
261 | } | ||||||
262 | |||||||
263 | bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { | ||||||
264 | // If either no tail call or told not to tail call at all, don't. | ||||||
265 | return CI->isTailCall(); | ||||||
266 | } | ||||||
267 | |||||||
268 | Register HexagonTargetLowering::getRegisterByName( | ||||||
269 | const char* RegName, LLT VT, const MachineFunction &) const { | ||||||
270 | // Just support r19, the linux kernel uses it. | ||||||
271 | Register Reg = StringSwitch<Register>(RegName) | ||||||
272 | .Case("r0", Hexagon::R0) | ||||||
273 | .Case("r1", Hexagon::R1) | ||||||
274 | .Case("r2", Hexagon::R2) | ||||||
275 | .Case("r3", Hexagon::R3) | ||||||
276 | .Case("r4", Hexagon::R4) | ||||||
277 | .Case("r5", Hexagon::R5) | ||||||
278 | .Case("r6", Hexagon::R6) | ||||||
279 | .Case("r7", Hexagon::R7) | ||||||
280 | .Case("r8", Hexagon::R8) | ||||||
281 | .Case("r9", Hexagon::R9) | ||||||
282 | .Case("r10", Hexagon::R10) | ||||||
283 | .Case("r11", Hexagon::R11) | ||||||
284 | .Case("r12", Hexagon::R12) | ||||||
285 | .Case("r13", Hexagon::R13) | ||||||
286 | .Case("r14", Hexagon::R14) | ||||||
287 | .Case("r15", Hexagon::R15) | ||||||
288 | .Case("r16", Hexagon::R16) | ||||||
289 | .Case("r17", Hexagon::R17) | ||||||
290 | .Case("r18", Hexagon::R18) | ||||||
291 | .Case("r19", Hexagon::R19) | ||||||
292 | .Case("r20", Hexagon::R20) | ||||||
293 | .Case("r21", Hexagon::R21) | ||||||
294 | .Case("r22", Hexagon::R22) | ||||||
295 | .Case("r23", Hexagon::R23) | ||||||
296 | .Case("r24", Hexagon::R24) | ||||||
297 | .Case("r25", Hexagon::R25) | ||||||
298 | .Case("r26", Hexagon::R26) | ||||||
299 | .Case("r27", Hexagon::R27) | ||||||
300 | .Case("r28", Hexagon::R28) | ||||||
301 | .Case("r29", Hexagon::R29) | ||||||
302 | .Case("r30", Hexagon::R30) | ||||||
303 | .Case("r31", Hexagon::R31) | ||||||
304 | .Case("r1:0", Hexagon::D0) | ||||||
305 | .Case("r3:2", Hexagon::D1) | ||||||
306 | .Case("r5:4", Hexagon::D2) | ||||||
307 | .Case("r7:6", Hexagon::D3) | ||||||
308 | .Case("r9:8", Hexagon::D4) | ||||||
309 | .Case("r11:10", Hexagon::D5) | ||||||
310 | .Case("r13:12", Hexagon::D6) | ||||||
311 | .Case("r15:14", Hexagon::D7) | ||||||
312 | .Case("r17:16", Hexagon::D8) | ||||||
313 | .Case("r19:18", Hexagon::D9) | ||||||
314 | .Case("r21:20", Hexagon::D10) | ||||||
315 | .Case("r23:22", Hexagon::D11) | ||||||
316 | .Case("r25:24", Hexagon::D12) | ||||||
317 | .Case("r27:26", Hexagon::D13) | ||||||
318 | .Case("r29:28", Hexagon::D14) | ||||||
319 | .Case("r31:30", Hexagon::D15) | ||||||
320 | .Case("sp", Hexagon::R29) | ||||||
321 | .Case("fp", Hexagon::R30) | ||||||
322 | .Case("lr", Hexagon::R31) | ||||||
323 | .Case("p0", Hexagon::P0) | ||||||
324 | .Case("p1", Hexagon::P1) | ||||||
325 | .Case("p2", Hexagon::P2) | ||||||
326 | .Case("p3", Hexagon::P3) | ||||||
327 | .Case("sa0", Hexagon::SA0) | ||||||
328 | .Case("lc0", Hexagon::LC0) | ||||||
329 | .Case("sa1", Hexagon::SA1) | ||||||
330 | .Case("lc1", Hexagon::LC1) | ||||||
331 | .Case("m0", Hexagon::M0) | ||||||
332 | .Case("m1", Hexagon::M1) | ||||||
333 | .Case("usr", Hexagon::USR) | ||||||
334 | .Case("ugp", Hexagon::UGP) | ||||||
335 | .Case("cs0", Hexagon::CS0) | ||||||
336 | .Case("cs1", Hexagon::CS1) | ||||||
337 | .Default(Register()); | ||||||
338 | if (Reg) | ||||||
339 | return Reg; | ||||||
340 | |||||||
341 | report_fatal_error("Invalid register name global variable"); | ||||||
342 | } | ||||||
343 | |||||||
344 | /// LowerCallResult - Lower the result values of an ISD::CALL into the | ||||||
345 | /// appropriate copies out of appropriate physical registers. This assumes that | ||||||
346 | /// Chain/Glue are the input chain/glue to use, and that TheCall is the call | ||||||
347 | /// being lowered. Returns a SDNode with the same number of values as the | ||||||
348 | /// ISD::CALL. | ||||||
349 | SDValue HexagonTargetLowering::LowerCallResult( | ||||||
350 | SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg, | ||||||
351 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, | ||||||
352 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, | ||||||
353 | const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const { | ||||||
354 | // Assign locations to each value returned by this call. | ||||||
355 | SmallVector<CCValAssign, 16> RVLocs; | ||||||
356 | |||||||
357 | CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, | ||||||
358 | *DAG.getContext()); | ||||||
359 | |||||||
360 | if (Subtarget.useHVXOps()) | ||||||
361 | CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX); | ||||||
362 | else | ||||||
363 | CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon); | ||||||
364 | |||||||
365 | // Copy all of the result registers out of their specified physreg. | ||||||
366 | for (unsigned i = 0; i != RVLocs.size(); ++i) { | ||||||
367 | SDValue RetVal; | ||||||
368 | if (RVLocs[i].getValVT() == MVT::i1) { | ||||||
369 | // Return values of type MVT::i1 require special handling. The reason | ||||||
370 | // is that MVT::i1 is associated with the PredRegs register class, but | ||||||
371 | // values of that type are still returned in R0. Generate an explicit | ||||||
372 | // copy into a predicate register from R0, and treat the value of the | ||||||
373 | // predicate register as the call result. | ||||||
374 | auto &MRI = DAG.getMachineFunction().getRegInfo(); | ||||||
375 | SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(), | ||||||
376 | MVT::i32, Glue); | ||||||
377 | // FR0 = (Value, Chain, Glue) | ||||||
378 | Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass); | ||||||
379 | SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR, | ||||||
380 | FR0.getValue(0), FR0.getValue(2)); | ||||||
381 | // TPR = (Chain, Glue) | ||||||
382 | // Don't glue this CopyFromReg, because it copies from a virtual | ||||||
383 | // register. If it is glued to the call, InstrEmitter will add it | ||||||
384 | // as an implicit def to the call (EmitMachineNode). | ||||||
385 | RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1); | ||||||
386 | Glue = TPR.getValue(1); | ||||||
387 | Chain = TPR.getValue(0); | ||||||
388 | } else { | ||||||
389 | RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(), | ||||||
390 | RVLocs[i].getValVT(), Glue); | ||||||
391 | Glue = RetVal.getValue(2); | ||||||
392 | Chain = RetVal.getValue(1); | ||||||
393 | } | ||||||
394 | InVals.push_back(RetVal.getValue(0)); | ||||||
395 | } | ||||||
396 | |||||||
397 | return Chain; | ||||||
398 | } | ||||||
399 | |||||||
400 | /// LowerCall - Functions arguments are copied from virtual regs to | ||||||
401 | /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted. | ||||||
402 | SDValue | ||||||
403 | HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, | ||||||
404 | SmallVectorImpl<SDValue> &InVals) const { | ||||||
405 | SelectionDAG &DAG = CLI.DAG; | ||||||
406 | SDLoc &dl = CLI.DL; | ||||||
407 | SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; | ||||||
408 | SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; | ||||||
409 | SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; | ||||||
410 | SDValue Chain = CLI.Chain; | ||||||
411 | SDValue Callee = CLI.Callee; | ||||||
412 | CallingConv::ID CallConv = CLI.CallConv; | ||||||
413 | bool IsVarArg = CLI.IsVarArg; | ||||||
414 | bool DoesNotReturn = CLI.DoesNotReturn; | ||||||
415 | |||||||
416 | bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet(); | ||||||
417 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||
418 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||||
419 | auto PtrVT = getPointerTy(MF.getDataLayout()); | ||||||
420 | |||||||
421 | unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0; | ||||||
422 | if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee)) | ||||||
423 | Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32); | ||||||
424 | |||||||
425 | // Linux ABI treats var-arg calls the same way as regular ones. | ||||||
426 | bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg; | ||||||
427 | |||||||
428 | // Analyze operands of the call, assigning locations to each operand. | ||||||
429 | SmallVector<CCValAssign, 16> ArgLocs; | ||||||
430 | HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(), | ||||||
431 | NumParams); | ||||||
432 | |||||||
433 | if (Subtarget.useHVXOps()) | ||||||
434 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX); | ||||||
435 | else if (DisableArgsMinAlignment) | ||||||
436 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy); | ||||||
437 | else | ||||||
438 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon); | ||||||
439 | |||||||
440 | if (CLI.IsTailCall) { | ||||||
441 | bool StructAttrFlag = MF.getFunction().hasStructRetAttr(); | ||||||
442 | CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, | ||||||
443 | IsVarArg, IsStructRet, StructAttrFlag, Outs, | ||||||
444 | OutVals, Ins, DAG); | ||||||
445 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | ||||||
446 | CCValAssign &VA = ArgLocs[i]; | ||||||
447 | if (VA.isMemLoc()) { | ||||||
448 | CLI.IsTailCall = false; | ||||||
449 | break; | ||||||
450 | } | ||||||
451 | } | ||||||
452 | LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false) | ||||||
453 | : "Argument must be passed on stack. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false) | ||||||
454 | "Not eligible for Tail Call\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false); | ||||||
455 | } | ||||||
456 | // Get a count of how many bytes are to be pushed on the stack. | ||||||
457 | unsigned NumBytes = CCInfo.getNextStackOffset(); | ||||||
458 | SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass; | ||||||
459 | SmallVector<SDValue, 8> MemOpChains; | ||||||
460 | |||||||
461 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||||
462 | SDValue StackPtr = | ||||||
463 | DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT); | ||||||
464 | |||||||
465 | bool NeedsArgAlign = false; | ||||||
466 | Align LargestAlignSeen; | ||||||
467 | // Walk the register/memloc assignments, inserting copies/loads. | ||||||
468 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | ||||||
469 | CCValAssign &VA = ArgLocs[i]; | ||||||
470 | SDValue Arg = OutVals[i]; | ||||||
471 | ISD::ArgFlagsTy Flags = Outs[i].Flags; | ||||||
472 | // Record if we need > 8 byte alignment on an argument. | ||||||
473 | bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT()); | ||||||
474 | NeedsArgAlign |= ArgAlign; | ||||||
475 | |||||||
476 | // Promote the value if needed. | ||||||
477 | switch (VA.getLocInfo()) { | ||||||
478 | default: | ||||||
479 | // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt. | ||||||
480 | llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 480); | ||||||
481 | case CCValAssign::Full: | ||||||
482 | break; | ||||||
483 | case CCValAssign::BCvt: | ||||||
484 | Arg = DAG.getBitcast(VA.getLocVT(), Arg); | ||||||
485 | break; | ||||||
486 | case CCValAssign::SExt: | ||||||
487 | Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); | ||||||
488 | break; | ||||||
489 | case CCValAssign::ZExt: | ||||||
490 | Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); | ||||||
491 | break; | ||||||
492 | case CCValAssign::AExt: | ||||||
493 | Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); | ||||||
494 | break; | ||||||
495 | } | ||||||
496 | |||||||
497 | if (VA.isMemLoc()) { | ||||||
498 | unsigned LocMemOffset = VA.getLocMemOffset(); | ||||||
499 | SDValue MemAddr = DAG.getConstant(LocMemOffset, dl, | ||||||
500 | StackPtr.getValueType()); | ||||||
501 | MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr); | ||||||
502 | if (ArgAlign) | ||||||
503 | LargestAlignSeen = std::max( | ||||||
504 | LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8)); | ||||||
505 | if (Flags.isByVal()) { | ||||||
506 | // The argument is a struct passed by value. According to LLVM, "Arg" | ||||||
507 | // is a pointer. | ||||||
508 | MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain, | ||||||
509 | Flags, DAG, dl)); | ||||||
510 | } else { | ||||||
511 | MachinePointerInfo LocPI = MachinePointerInfo::getStack( | ||||||
512 | DAG.getMachineFunction(), LocMemOffset); | ||||||
513 | SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI); | ||||||
514 | MemOpChains.push_back(S); | ||||||
515 | } | ||||||
516 | continue; | ||||||
517 | } | ||||||
518 | |||||||
519 | // Arguments that can be passed on register must be kept at RegsToPass | ||||||
520 | // vector. | ||||||
521 | if (VA.isRegLoc()) | ||||||
522 | RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); | ||||||
523 | } | ||||||
524 | |||||||
525 | if (NeedsArgAlign && Subtarget.hasV60Ops()) { | ||||||
526 | LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << "Function needs byte stack align due to call args\n" ; } } while (false); | ||||||
527 | Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass); | ||||||
528 | LargestAlignSeen = std::max(LargestAlignSeen, VecAlign); | ||||||
529 | MFI.ensureMaxAlignment(LargestAlignSeen); | ||||||
530 | } | ||||||
531 | // Transform all store nodes into one single node because all store | ||||||
532 | // nodes are independent of each other. | ||||||
533 | if (!MemOpChains.empty()) | ||||||
534 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); | ||||||
535 | |||||||
536 | SDValue Glue; | ||||||
537 | if (!CLI.IsTailCall) { | ||||||
538 | Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); | ||||||
539 | Glue = Chain.getValue(1); | ||||||
540 | } | ||||||
541 | |||||||
542 | // Build a sequence of copy-to-reg nodes chained together with token | ||||||
543 | // chain and flag operands which copy the outgoing args into registers. | ||||||
544 | // The Glue is necessary since all emitted instructions must be | ||||||
545 | // stuck together. | ||||||
546 | if (!CLI.IsTailCall) { | ||||||
547 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | ||||||
548 | Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, | ||||||
549 | RegsToPass[i].second, Glue); | ||||||
550 | Glue = Chain.getValue(1); | ||||||
551 | } | ||||||
552 | } else { | ||||||
553 | // For tail calls lower the arguments to the 'real' stack slot. | ||||||
554 | // | ||||||
555 | // Force all the incoming stack arguments to be loaded from the stack | ||||||
556 | // before any new outgoing arguments are stored to the stack, because the | ||||||
557 | // outgoing stack slots may alias the incoming argument stack slots, and | ||||||
558 | // the alias isn't otherwise explicit. This is slightly more conservative | ||||||
559 | // than necessary, because it means that each store effectively depends | ||||||
560 | // on every argument instead of just those arguments it would clobber. | ||||||
561 | // | ||||||
562 | // Do not flag preceding copytoreg stuff together with the following stuff. | ||||||
563 | Glue = SDValue(); | ||||||
564 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | ||||||
565 | Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, | ||||||
566 | RegsToPass[i].second, Glue); | ||||||
567 | Glue = Chain.getValue(1); | ||||||
568 | } | ||||||
569 | Glue = SDValue(); | ||||||
570 | } | ||||||
571 | |||||||
572 | bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls(); | ||||||
573 | unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0; | ||||||
574 | |||||||
575 | // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every | ||||||
576 | // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol | ||||||
577 | // node so that legalize doesn't hack it. | ||||||
578 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { | ||||||
579 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags); | ||||||
580 | } else if (ExternalSymbolSDNode *S = | ||||||
581 | dyn_cast<ExternalSymbolSDNode>(Callee)) { | ||||||
582 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags); | ||||||
583 | } | ||||||
584 | |||||||
585 | // Returns a chain & a flag for retval copy to use. | ||||||
586 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||
587 | SmallVector<SDValue, 8> Ops; | ||||||
588 | Ops.push_back(Chain); | ||||||
589 | Ops.push_back(Callee); | ||||||
590 | |||||||
591 | // Add argument registers to the end of the list so that they are | ||||||
592 | // known live into the call. | ||||||
593 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | ||||||
594 | Ops.push_back(DAG.getRegister(RegsToPass[i].first, | ||||||
595 | RegsToPass[i].second.getValueType())); | ||||||
596 | } | ||||||
597 | |||||||
598 | const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv); | ||||||
599 | assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention" ) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 599, __extension__ __PRETTY_FUNCTION__)); | ||||||
600 | Ops.push_back(DAG.getRegisterMask(Mask)); | ||||||
601 | |||||||
602 | if (Glue.getNode()) | ||||||
603 | Ops.push_back(Glue); | ||||||
604 | |||||||
605 | if (CLI.IsTailCall) { | ||||||
606 | MFI.setHasTailCall(); | ||||||
607 | return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops); | ||||||
608 | } | ||||||
609 | |||||||
610 | // Set this here because we need to know this for "hasFP" in frame lowering. | ||||||
611 | // The target-independent code calls getFrameRegister before setting it, and | ||||||
612 | // getFrameRegister uses hasFP to determine whether the function has FP. | ||||||
613 | MFI.setHasCalls(true); | ||||||
614 | |||||||
615 | unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL; | ||||||
616 | Chain = DAG.getNode(OpCode, dl, NodeTys, Ops); | ||||||
617 | Glue = Chain.getValue(1); | ||||||
618 | |||||||
619 | // Create the CALLSEQ_END node. | ||||||
620 | Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), | ||||||
621 | DAG.getIntPtrConstant(0, dl, true), Glue, dl); | ||||||
622 | Glue = Chain.getValue(1); | ||||||
623 | |||||||
624 | // Handle result values, copying them out of physregs into vregs that we | ||||||
625 | // return. | ||||||
626 | return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG, | ||||||
627 | InVals, OutVals, Callee); | ||||||
628 | } | ||||||
629 | |||||||
630 | /// Returns true by value, base pointer and offset pointer and addressing | ||||||
631 | /// mode by reference if this node can be combined with a load / store to | ||||||
632 | /// form a post-indexed load / store. | ||||||
633 | bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op, | ||||||
634 | SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, | ||||||
635 | SelectionDAG &DAG) const { | ||||||
636 | LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N); | ||||||
637 | if (!LSN) | ||||||
638 | return false; | ||||||
639 | EVT VT = LSN->getMemoryVT(); | ||||||
640 | if (!VT.isSimple()) | ||||||
641 | return false; | ||||||
642 | bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || | ||||||
643 | VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 || | ||||||
644 | VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 || | ||||||
645 | VT == MVT::v4i16 || VT == MVT::v8i8 || | ||||||
646 | Subtarget.isHVXVectorType(VT.getSimpleVT()); | ||||||
647 | if (!IsLegalType) | ||||||
648 | return false; | ||||||
649 | |||||||
650 | if (Op->getOpcode() != ISD::ADD) | ||||||
651 | return false; | ||||||
652 | Base = Op->getOperand(0); | ||||||
653 | Offset = Op->getOperand(1); | ||||||
654 | if (!isa<ConstantSDNode>(Offset.getNode())) | ||||||
655 | return false; | ||||||
656 | AM = ISD::POST_INC; | ||||||
657 | |||||||
658 | int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue(); | ||||||
659 | return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V); | ||||||
660 | } | ||||||
661 | |||||||
662 | SDValue | ||||||
663 | HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const { | ||||||
664 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||
665 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); | ||||||
666 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||||
667 | unsigned LR = HRI.getRARegister(); | ||||||
668 | |||||||
669 | if ((Op.getOpcode() != ISD::INLINEASM && | ||||||
670 | Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR()) | ||||||
671 | return Op; | ||||||
672 | |||||||
673 | unsigned NumOps = Op.getNumOperands(); | ||||||
674 | if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue) | ||||||
675 | --NumOps; // Ignore the flag operand. | ||||||
676 | |||||||
677 | for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { | ||||||
678 | unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue(); | ||||||
679 | unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); | ||||||
680 | ++i; // Skip the ID value. | ||||||
681 | |||||||
682 | switch (InlineAsm::getKind(Flags)) { | ||||||
683 | default: | ||||||
684 | llvm_unreachable("Bad flags!")::llvm::llvm_unreachable_internal("Bad flags!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 684); | ||||||
685 | case InlineAsm::Kind_RegUse: | ||||||
686 | case InlineAsm::Kind_Imm: | ||||||
687 | case InlineAsm::Kind_Mem: | ||||||
688 | i += NumVals; | ||||||
689 | break; | ||||||
690 | case InlineAsm::Kind_Clobber: | ||||||
691 | case InlineAsm::Kind_RegDef: | ||||||
692 | case InlineAsm::Kind_RegDefEarlyClobber: { | ||||||
693 | for (; NumVals; --NumVals, ++i) { | ||||||
694 | unsigned Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg(); | ||||||
695 | if (Reg != LR) | ||||||
696 | continue; | ||||||
697 | HMFI.setHasClobberLR(true); | ||||||
698 | return Op; | ||||||
699 | } | ||||||
700 | break; | ||||||
701 | } | ||||||
702 | } | ||||||
703 | } | ||||||
704 | |||||||
705 | return Op; | ||||||
706 | } | ||||||
707 | |||||||
708 | // Need to transform ISD::PREFETCH into something that doesn't inherit | ||||||
709 | // all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and | ||||||
710 | // SDNPMayStore. | ||||||
711 | SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op, | ||||||
712 | SelectionDAG &DAG) const { | ||||||
713 | SDValue Chain = Op.getOperand(0); | ||||||
714 | SDValue Addr = Op.getOperand(1); | ||||||
715 | // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in, | ||||||
716 | // if the "reg" is fed by an "add". | ||||||
717 | SDLoc DL(Op); | ||||||
718 | SDValue Zero = DAG.getConstant(0, DL, MVT::i32); | ||||||
719 | return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero); | ||||||
720 | } | ||||||
721 | |||||||
722 | // Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode | ||||||
723 | // is marked as having side-effects, while the register read on Hexagon does | ||||||
724 | // not have any. TableGen refuses to accept the direct pattern from that node | ||||||
725 | // to the A4_tfrcpp. | ||||||
726 | SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op, | ||||||
727 | SelectionDAG &DAG) const { | ||||||
728 | SDValue Chain = Op.getOperand(0); | ||||||
729 | SDLoc dl(Op); | ||||||
730 | SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other); | ||||||
731 | return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain); | ||||||
732 | } | ||||||
733 | |||||||
734 | SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op, | ||||||
735 | SelectionDAG &DAG) const { | ||||||
736 | SDValue Chain = Op.getOperand(0); | ||||||
737 | unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); | ||||||
738 | // Lower the hexagon_prefetch builtin to DCFETCH, as above. | ||||||
739 | if (IntNo == Intrinsic::hexagon_prefetch) { | ||||||
740 | SDValue Addr = Op.getOperand(2); | ||||||
741 | SDLoc DL(Op); | ||||||
742 | SDValue Zero = DAG.getConstant(0, DL, MVT::i32); | ||||||
743 | return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero); | ||||||
744 | } | ||||||
745 | return SDValue(); | ||||||
746 | } | ||||||
747 | |||||||
748 | SDValue | ||||||
749 | HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, | ||||||
750 | SelectionDAG &DAG) const { | ||||||
751 | SDValue Chain = Op.getOperand(0); | ||||||
752 | SDValue Size = Op.getOperand(1); | ||||||
753 | SDValue Align = Op.getOperand(2); | ||||||
754 | SDLoc dl(Op); | ||||||
755 | |||||||
756 | ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align); | ||||||
757 | assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC")(static_cast <bool> (AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC" ) ? void (0) : __assert_fail ("AlignConst && \"Non-constant Align in LowerDYNAMIC_STACKALLOC\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 757, __extension__ __PRETTY_FUNCTION__)); | ||||||
758 | |||||||
759 | unsigned A = AlignConst->getSExtValue(); | ||||||
760 | auto &HFI = *Subtarget.getFrameLowering(); | ||||||
761 | // "Zero" means natural stack alignment. | ||||||
762 | if (A == 0) | ||||||
763 | A = HFI.getStackAlign().value(); | ||||||
764 | |||||||
765 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||||
766 | dbgs () << __func__ << " Align: " << A << " Size: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||||
767 | Size.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||||
768 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||||
769 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false); | ||||||
770 | |||||||
771 | SDValue AC = DAG.getConstant(A, dl, MVT::i32); | ||||||
772 | SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); | ||||||
773 | SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC); | ||||||
774 | |||||||
775 | DAG.ReplaceAllUsesOfValueWith(Op, AA); | ||||||
776 | return AA; | ||||||
777 | } | ||||||
778 | |||||||
779 | SDValue HexagonTargetLowering::LowerFormalArguments( | ||||||
780 | SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, | ||||||
781 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, | ||||||
782 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { | ||||||
783 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||
784 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||||
785 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||
786 | |||||||
787 | // Linux ABI treats var-arg calls the same way as regular ones. | ||||||
788 | bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg; | ||||||
789 | |||||||
790 | // Assign locations to all of the incoming arguments. | ||||||
791 | SmallVector<CCValAssign, 16> ArgLocs; | ||||||
792 | HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, | ||||||
793 | *DAG.getContext(), | ||||||
794 | MF.getFunction().getFunctionType()->getNumParams()); | ||||||
795 | |||||||
796 | if (Subtarget.useHVXOps()) | ||||||
797 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX); | ||||||
798 | else if (DisableArgsMinAlignment) | ||||||
799 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy); | ||||||
800 | else | ||||||
801 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon); | ||||||
802 | |||||||
803 | // For LLVM, in the case when returning a struct by value (>8byte), | ||||||
804 | // the first argument is a pointer that points to the location on caller's | ||||||
805 | // stack where the return value will be stored. For Hexagon, the location on | ||||||
806 | // caller's stack is passed only when the struct size is smaller than (and | ||||||
807 | // equal to) 8 bytes. If not, no address will be passed into callee and | ||||||
808 | // callee return the result direclty through R0/R1. | ||||||
809 | auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) { | ||||||
810 | switch (RC.getID()) { | ||||||
811 | case Hexagon::IntRegsRegClassID: | ||||||
812 | return Reg - Hexagon::R0 + 1; | ||||||
813 | case Hexagon::DoubleRegsRegClassID: | ||||||
814 | return (Reg - Hexagon::D0 + 1) * 2; | ||||||
815 | case Hexagon::HvxVRRegClassID: | ||||||
816 | return Reg - Hexagon::V0 + 1; | ||||||
817 | case Hexagon::HvxWRRegClassID: | ||||||
818 | return (Reg - Hexagon::W0 + 1) * 2; | ||||||
819 | } | ||||||
820 | llvm_unreachable("Unexpected register class")::llvm::llvm_unreachable_internal("Unexpected register class" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 820); | ||||||
821 | }; | ||||||
822 | |||||||
823 | auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering()); | ||||||
824 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); | ||||||
825 | HFL.FirstVarArgSavedReg = 0; | ||||||
826 | HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects())); | ||||||
827 | |||||||
828 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | ||||||
829 | CCValAssign &VA = ArgLocs[i]; | ||||||
830 | ISD::ArgFlagsTy Flags = Ins[i].Flags; | ||||||
831 | bool ByVal = Flags.isByVal(); | ||||||
832 | |||||||
833 | // Arguments passed in registers: | ||||||
834 | // 1. 32- and 64-bit values and HVX vectors are passed directly, | ||||||
835 | // 2. Large structs are passed via an address, and the address is | ||||||
836 | // passed in a register. | ||||||
837 | if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8) | ||||||
838 | llvm_unreachable("ByValSize must be bigger than 8 bytes")::llvm::llvm_unreachable_internal("ByValSize must be bigger than 8 bytes" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 838); | ||||||
839 | |||||||
840 | bool InReg = VA.isRegLoc() && | ||||||
841 | (!ByVal || (ByVal && Flags.getByValSize() > 8)); | ||||||
842 | |||||||
843 | if (InReg) { | ||||||
844 | MVT RegVT = VA.getLocVT(); | ||||||
845 | if (VA.getLocInfo() == CCValAssign::BCvt) | ||||||
846 | RegVT = VA.getValVT(); | ||||||
847 | |||||||
848 | const TargetRegisterClass *RC = getRegClassFor(RegVT); | ||||||
849 | Register VReg = MRI.createVirtualRegister(RC); | ||||||
850 | SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT); | ||||||
851 | |||||||
852 | // Treat values of type MVT::i1 specially: they are passed in | ||||||
853 | // registers of type i32, but they need to remain as values of | ||||||
854 | // type i1 for consistency of the argument lowering. | ||||||
855 | if (VA.getValVT() == MVT::i1) { | ||||||
856 | assert(RegVT.getSizeInBits() <= 32)(static_cast <bool> (RegVT.getSizeInBits() <= 32) ? void (0) : __assert_fail ("RegVT.getSizeInBits() <= 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 856, __extension__ __PRETTY_FUNCTION__)); | ||||||
857 | SDValue T = DAG.getNode(ISD::AND, dl, RegVT, | ||||||
858 | Copy, DAG.getConstant(1, dl, RegVT)); | ||||||
859 | Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT), | ||||||
860 | ISD::SETNE); | ||||||
861 | } else { | ||||||
862 | #ifndef NDEBUG | ||||||
863 | unsigned RegSize = RegVT.getSizeInBits(); | ||||||
864 | assert(RegSize == 32 || RegSize == 64 ||(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget .isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 865, __extension__ __PRETTY_FUNCTION__)) | ||||||
865 | Subtarget.isHVXVectorType(RegVT))(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget .isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 865, __extension__ __PRETTY_FUNCTION__)); | ||||||
866 | #endif | ||||||
867 | } | ||||||
868 | InVals.push_back(Copy); | ||||||
869 | MRI.addLiveIn(VA.getLocReg(), VReg); | ||||||
870 | HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg()); | ||||||
871 | } else { | ||||||
872 | assert(VA.isMemLoc() && "Argument should be passed in memory")(static_cast <bool> (VA.isMemLoc() && "Argument should be passed in memory" ) ? void (0) : __assert_fail ("VA.isMemLoc() && \"Argument should be passed in memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 872, __extension__ __PRETTY_FUNCTION__)); | ||||||
873 | |||||||
874 | // If it's a byval parameter, then we need to compute the | ||||||
875 | // "real" size, not the size of the pointer. | ||||||
876 | unsigned ObjSize = Flags.isByVal() | ||||||
877 | ? Flags.getByValSize() | ||||||
878 | : VA.getLocVT().getStoreSizeInBits() / 8; | ||||||
879 | |||||||
880 | // Create the frame index object for this incoming parameter. | ||||||
881 | int Offset = HEXAGON_LRFP_SIZE8 + VA.getLocMemOffset(); | ||||||
882 | int FI = MFI.CreateFixedObject(ObjSize, Offset, true); | ||||||
883 | SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); | ||||||
884 | |||||||
885 | if (Flags.isByVal()) { | ||||||
886 | // If it's a pass-by-value aggregate, then do not dereference the stack | ||||||
887 | // location. Instead, we should generate a reference to the stack | ||||||
888 | // location. | ||||||
889 | InVals.push_back(FIN); | ||||||
890 | } else { | ||||||
891 | SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN, | ||||||
892 | MachinePointerInfo::getFixedStack(MF, FI, 0)); | ||||||
893 | InVals.push_back(L); | ||||||
894 | } | ||||||
895 | } | ||||||
896 | } | ||||||
897 | |||||||
898 | if (IsVarArg && Subtarget.isEnvironmentMusl()) { | ||||||
899 | for (int i = HFL.FirstVarArgSavedReg; i < 6; i++) | ||||||
900 | MRI.addLiveIn(Hexagon::R0+i); | ||||||
901 | } | ||||||
902 | |||||||
903 | if (IsVarArg && Subtarget.isEnvironmentMusl()) { | ||||||
904 | HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1); | ||||||
905 | HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects())); | ||||||
906 | |||||||
907 | // Create Frame index for the start of register saved area. | ||||||
908 | int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg; | ||||||
909 | bool RequiresPadding = (NumVarArgRegs & 1); | ||||||
910 | int RegSaveAreaSizePlusPadding = RequiresPadding | ||||||
911 | ? (NumVarArgRegs + 1) * 4 | ||||||
912 | : NumVarArgRegs * 4; | ||||||
913 | |||||||
914 | if (RegSaveAreaSizePlusPadding > 0) { | ||||||
915 | // The offset to saved register area should be 8 byte aligned. | ||||||
916 | int RegAreaStart = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset(); | ||||||
917 | if (!(RegAreaStart % 8)) | ||||||
918 | RegAreaStart = (RegAreaStart + 7) & -8; | ||||||
919 | |||||||
920 | int RegSaveAreaFrameIndex = | ||||||
921 | MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true); | ||||||
922 | HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex); | ||||||
923 | |||||||
924 | // This will point to the next argument passed via stack. | ||||||
925 | int Offset = RegAreaStart + RegSaveAreaSizePlusPadding; | ||||||
926 | int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true); | ||||||
927 | HMFI.setVarArgsFrameIndex(FI); | ||||||
928 | } else { | ||||||
929 | // This will point to the next argument passed via stack, when | ||||||
930 | // there is no saved register area. | ||||||
931 | int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset(); | ||||||
932 | int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true); | ||||||
933 | HMFI.setRegSavedAreaStartFrameIndex(FI); | ||||||
934 | HMFI.setVarArgsFrameIndex(FI); | ||||||
935 | } | ||||||
936 | } | ||||||
937 | |||||||
938 | |||||||
939 | if (IsVarArg && !Subtarget.isEnvironmentMusl()) { | ||||||
940 | // This will point to the next argument passed via stack. | ||||||
941 | int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset(); | ||||||
942 | int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true); | ||||||
943 | HMFI.setVarArgsFrameIndex(FI); | ||||||
944 | } | ||||||
945 | |||||||
946 | return Chain; | ||||||
947 | } | ||||||
948 | |||||||
949 | SDValue | ||||||
950 | HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { | ||||||
951 | // VASTART stores the address of the VarArgsFrameIndex slot into the | ||||||
952 | // memory location argument. | ||||||
953 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||
954 | HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>(); | ||||||
955 | SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32); | ||||||
956 | const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); | ||||||
957 | |||||||
958 | if (!Subtarget.isEnvironmentMusl()) { | ||||||
959 | return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1), | ||||||
960 | MachinePointerInfo(SV)); | ||||||
961 | } | ||||||
962 | auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>(); | ||||||
963 | auto &HFL = *Subtarget.getFrameLowering(); | ||||||
964 | SDLoc DL(Op); | ||||||
965 | SmallVector<SDValue, 8> MemOps; | ||||||
966 | |||||||
967 | // Get frame index of va_list. | ||||||
968 | SDValue FIN = Op.getOperand(1); | ||||||
969 | |||||||
970 | // If first Vararg register is odd, add 4 bytes to start of | ||||||
971 | // saved register area to point to the first register location. | ||||||
972 | // This is because the saved register area has to be 8 byte aligned. | ||||||
973 | // Incase of an odd start register, there will be 4 bytes of padding in | ||||||
974 | // the beginning of saved register area. If all registers area used up, | ||||||
975 | // the following condition will handle it correctly. | ||||||
976 | SDValue SavedRegAreaStartFrameIndex = | ||||||
977 | DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32); | ||||||
978 | |||||||
979 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
980 | |||||||
981 | if (HFL.FirstVarArgSavedReg & 1) | ||||||
982 | SavedRegAreaStartFrameIndex = | ||||||
983 | DAG.getNode(ISD::ADD, DL, PtrVT, | ||||||
984 | DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), | ||||||
985 | MVT::i32), | ||||||
986 | DAG.getIntPtrConstant(4, DL)); | ||||||
987 | |||||||
988 | // Store the saved register area start pointer. | ||||||
989 | SDValue Store = | ||||||
990 | DAG.getStore(Op.getOperand(0), DL, | ||||||
991 | SavedRegAreaStartFrameIndex, | ||||||
992 | FIN, MachinePointerInfo(SV)); | ||||||
993 | MemOps.push_back(Store); | ||||||
994 | |||||||
995 | // Store saved register area end pointer. | ||||||
996 | FIN = DAG.getNode(ISD::ADD, DL, PtrVT, | ||||||
997 | FIN, DAG.getIntPtrConstant(4, DL)); | ||||||
998 | Store = DAG.getStore(Op.getOperand(0), DL, | ||||||
999 | DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(), | ||||||
1000 | PtrVT), | ||||||
1001 | FIN, MachinePointerInfo(SV, 4)); | ||||||
1002 | MemOps.push_back(Store); | ||||||
1003 | |||||||
1004 | // Store overflow area pointer. | ||||||
1005 | FIN = DAG.getNode(ISD::ADD, DL, PtrVT, | ||||||
1006 | FIN, DAG.getIntPtrConstant(4, DL)); | ||||||
1007 | Store = DAG.getStore(Op.getOperand(0), DL, | ||||||
1008 | DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(), | ||||||
1009 | PtrVT), | ||||||
1010 | FIN, MachinePointerInfo(SV, 8)); | ||||||
1011 | MemOps.push_back(Store); | ||||||
1012 | |||||||
1013 | return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); | ||||||
1014 | } | ||||||
1015 | |||||||
1016 | SDValue | ||||||
1017 | HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { | ||||||
1018 | // Assert that the linux ABI is enabled for the current compilation. | ||||||
1019 | assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled")(static_cast <bool> (Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled") ? void (0) : __assert_fail ("Subtarget.isEnvironmentMusl() && \"Linux ABI should be enabled\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1019, __extension__ __PRETTY_FUNCTION__)); | ||||||
1020 | SDValue Chain = Op.getOperand(0); | ||||||
1021 | SDValue DestPtr = Op.getOperand(1); | ||||||
1022 | SDValue SrcPtr = Op.getOperand(2); | ||||||
1023 | const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); | ||||||
1024 | const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); | ||||||
1025 | SDLoc DL(Op); | ||||||
1026 | // Size of the va_list is 12 bytes as it has 3 pointers. Therefore, | ||||||
1027 | // we need to memcopy 12 bytes from va_list to another similar list. | ||||||
1028 | return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr, | ||||||
1029 | DAG.getIntPtrConstant(12, DL), Align(4), | ||||||
1030 | /*isVolatile*/ false, false, false, | ||||||
1031 | MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV)); | ||||||
1032 | } | ||||||
1033 | |||||||
1034 | SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { | ||||||
1035 | const SDLoc &dl(Op); | ||||||
1036 | SDValue LHS = Op.getOperand(0); | ||||||
1037 | SDValue RHS = Op.getOperand(1); | ||||||
1038 | ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); | ||||||
1039 | MVT ResTy = ty(Op); | ||||||
1040 | MVT OpTy = ty(LHS); | ||||||
1041 | |||||||
1042 | if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) { | ||||||
1043 | MVT ElemTy = OpTy.getVectorElementType(); | ||||||
1044 | assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void ( 0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1044, __extension__ __PRETTY_FUNCTION__)); | ||||||
1045 | MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()), | ||||||
1046 | OpTy.getVectorNumElements()); | ||||||
1047 | return DAG.getSetCC(dl, ResTy, | ||||||
1048 | DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy), | ||||||
1049 | DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC); | ||||||
1050 | } | ||||||
1051 | |||||||
1052 | // Treat all other vector types as legal. | ||||||
1053 | if (ResTy.isVector()) | ||||||
1054 | return Op; | ||||||
1055 | |||||||
1056 | // Comparisons of short integers should use sign-extend, not zero-extend, | ||||||
1057 | // since we can represent small negative values in the compare instructions. | ||||||
1058 | // The LLVM default is to use zero-extend arbitrarily in these cases. | ||||||
1059 | auto isSExtFree = [this](SDValue N) { | ||||||
1060 | switch (N.getOpcode()) { | ||||||
1061 | case ISD::TRUNCATE: { | ||||||
1062 | // A sign-extend of a truncate of a sign-extend is free. | ||||||
1063 | SDValue Op = N.getOperand(0); | ||||||
1064 | if (Op.getOpcode() != ISD::AssertSext) | ||||||
1065 | return false; | ||||||
1066 | EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT(); | ||||||
1067 | unsigned ThisBW = ty(N).getSizeInBits(); | ||||||
1068 | unsigned OrigBW = OrigTy.getSizeInBits(); | ||||||
1069 | // The type that was sign-extended to get the AssertSext must be | ||||||
1070 | // narrower than the type of N (so that N has still the same value | ||||||
1071 | // as the original). | ||||||
1072 | return ThisBW >= OrigBW; | ||||||
1073 | } | ||||||
1074 | case ISD::LOAD: | ||||||
1075 | // We have sign-extended loads. | ||||||
1076 | return true; | ||||||
1077 | } | ||||||
1078 | return false; | ||||||
1079 | }; | ||||||
1080 | |||||||
1081 | if (OpTy == MVT::i8 || OpTy == MVT::i16) { | ||||||
1082 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS); | ||||||
1083 | bool IsNegative = C && C->getAPIntValue().isNegative(); | ||||||
1084 | if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS)) | ||||||
1085 | return DAG.getSetCC(dl, ResTy, | ||||||
1086 | DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32), | ||||||
1087 | DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC); | ||||||
1088 | } | ||||||
1089 | |||||||
1090 | return SDValue(); | ||||||
1091 | } | ||||||
1092 | |||||||
1093 | SDValue | ||||||
1094 | HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const { | ||||||
1095 | SDValue PredOp = Op.getOperand(0); | ||||||
1096 | SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2); | ||||||
1097 | MVT OpTy = ty(Op1); | ||||||
1098 | const SDLoc &dl(Op); | ||||||
1099 | |||||||
1100 | if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) { | ||||||
1101 | MVT ElemTy = OpTy.getVectorElementType(); | ||||||
1102 | assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void ( 0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1102, __extension__ __PRETTY_FUNCTION__)); | ||||||
1103 | MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()), | ||||||
1104 | OpTy.getVectorNumElements()); | ||||||
1105 | // Generate (trunc (select (_, sext, sext))). | ||||||
1106 | return DAG.getSExtOrTrunc( | ||||||
1107 | DAG.getSelect(dl, WideTy, PredOp, | ||||||
1108 | DAG.getSExtOrTrunc(Op1, dl, WideTy), | ||||||
1109 | DAG.getSExtOrTrunc(Op2, dl, WideTy)), | ||||||
1110 | dl, OpTy); | ||||||
1111 | } | ||||||
1112 | |||||||
1113 | return SDValue(); | ||||||
1114 | } | ||||||
1115 | |||||||
1116 | SDValue | ||||||
1117 | HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { | ||||||
1118 | EVT ValTy = Op.getValueType(); | ||||||
1119 | ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op); | ||||||
1120 | Constant *CVal = nullptr; | ||||||
1121 | bool isVTi1Type = false; | ||||||
1122 | if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) { | ||||||
1123 | if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) { | ||||||
1124 | IRBuilder<> IRB(CV->getContext()); | ||||||
1125 | SmallVector<Constant*, 128> NewConst; | ||||||
1126 | unsigned VecLen = CV->getNumOperands(); | ||||||
1127 | assert(isPowerOf2_32(VecLen) &&(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize" ) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1128, __extension__ __PRETTY_FUNCTION__)) | ||||||
1128 | "conversion only supported for pow2 VectorSize")(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize" ) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1128, __extension__ __PRETTY_FUNCTION__)); | ||||||
1129 | for (unsigned i = 0; i < VecLen; ++i) | ||||||
1130 | NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue())); | ||||||
1131 | |||||||
1132 | CVal = ConstantVector::get(NewConst); | ||||||
1133 | isVTi1Type = true; | ||||||
1134 | } | ||||||
1135 | } | ||||||
1136 | Align Alignment = CPN->getAlign(); | ||||||
1137 | bool IsPositionIndependent = isPositionIndependent(); | ||||||
1138 | unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0; | ||||||
1139 | |||||||
1140 | unsigned Offset = 0; | ||||||
1141 | SDValue T; | ||||||
1142 | if (CPN->isMachineConstantPoolEntry()) | ||||||
1143 | T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment, | ||||||
1144 | Offset, TF); | ||||||
1145 | else if (isVTi1Type) | ||||||
1146 | T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF); | ||||||
1147 | else | ||||||
1148 | T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset, | ||||||
1149 | TF); | ||||||
1150 | |||||||
1151 | assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&(static_cast <bool> (cast<ConstantPoolSDNode>(T)-> getTargetFlags() == TF && "Inconsistent target flag encountered" ) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1152, __extension__ __PRETTY_FUNCTION__)) | ||||||
1152 | "Inconsistent target flag encountered")(static_cast <bool> (cast<ConstantPoolSDNode>(T)-> getTargetFlags() == TF && "Inconsistent target flag encountered" ) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1152, __extension__ __PRETTY_FUNCTION__)); | ||||||
1153 | |||||||
1154 | if (IsPositionIndependent) | ||||||
1155 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T); | ||||||
1156 | return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T); | ||||||
1157 | } | ||||||
1158 | |||||||
1159 | SDValue | ||||||
1160 | HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { | ||||||
1161 | EVT VT = Op.getValueType(); | ||||||
1162 | int Idx = cast<JumpTableSDNode>(Op)->getIndex(); | ||||||
1163 | if (isPositionIndependent()) { | ||||||
1164 | SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL); | ||||||
1165 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T); | ||||||
1166 | } | ||||||
1167 | |||||||
1168 | SDValue T = DAG.getTargetJumpTable(Idx, VT); | ||||||
1169 | return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T); | ||||||
1170 | } | ||||||
1171 | |||||||
1172 | SDValue | ||||||
1173 | HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const { | ||||||
1174 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||||
1175 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||
1176 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||||
1177 | MFI.setReturnAddressIsTaken(true); | ||||||
1178 | |||||||
1179 | if (verifyReturnAddressArgumentIsConstant(Op, DAG)) | ||||||
1180 | return SDValue(); | ||||||
1181 | |||||||
1182 | EVT VT = Op.getValueType(); | ||||||
1183 | SDLoc dl(Op); | ||||||
1184 | unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); | ||||||
1185 | if (Depth) { | ||||||
1186 | SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); | ||||||
1187 | SDValue Offset = DAG.getConstant(4, dl, MVT::i32); | ||||||
1188 | return DAG.getLoad(VT, dl, DAG.getEntryNode(), | ||||||
1189 | DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), | ||||||
1190 | MachinePointerInfo()); | ||||||
1191 | } | ||||||
1192 | |||||||
1193 | // Return LR, which contains the return address. Mark it an implicit live-in. | ||||||
1194 | unsigned Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32)); | ||||||
1195 | return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); | ||||||
1196 | } | ||||||
1197 | |||||||
1198 | SDValue | ||||||
1199 | HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { | ||||||
1200 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||||
1201 | MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); | ||||||
1202 | MFI.setFrameAddressIsTaken(true); | ||||||
1203 | |||||||
1204 | EVT VT = Op.getValueType(); | ||||||
1205 | SDLoc dl(Op); | ||||||
1206 | unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); | ||||||
1207 | SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, | ||||||
1208 | HRI.getFrameRegister(), VT); | ||||||
1209 | while (Depth--) | ||||||
1210 | FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, | ||||||
1211 | MachinePointerInfo()); | ||||||
1212 | return FrameAddr; | ||||||
1213 | } | ||||||
1214 | |||||||
1215 | SDValue | ||||||
1216 | HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const { | ||||||
1217 | SDLoc dl(Op); | ||||||
1218 | return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0)); | ||||||
1219 | } | ||||||
1220 | |||||||
1221 | SDValue | ||||||
1222 | HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const { | ||||||
1223 | SDLoc dl(Op); | ||||||
1224 | auto *GAN = cast<GlobalAddressSDNode>(Op); | ||||||
1225 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
1226 | auto *GV = GAN->getGlobal(); | ||||||
1227 | int64_t Offset = GAN->getOffset(); | ||||||
1228 | |||||||
1229 | auto &HLOF = *HTM.getObjFileLowering(); | ||||||
1230 | Reloc::Model RM = HTM.getRelocationModel(); | ||||||
1231 | |||||||
1232 | if (RM == Reloc::Static) { | ||||||
1233 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset); | ||||||
1234 | const GlobalObject *GO = GV->getBaseObject(); | ||||||
1235 | if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM)) | ||||||
1236 | return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA); | ||||||
1237 | return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA); | ||||||
1238 | } | ||||||
1239 | |||||||
1240 | bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV); | ||||||
1241 | if (UsePCRel) { | ||||||
1242 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset, | ||||||
1243 | HexagonII::MO_PCREL); | ||||||
1244 | return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA); | ||||||
1245 | } | ||||||
1246 | |||||||
1247 | // Use GOT index. | ||||||
1248 | SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT); | ||||||
1249 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT); | ||||||
1250 | SDValue Off = DAG.getConstant(Offset, dl, MVT::i32); | ||||||
1251 | return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off); | ||||||
1252 | } | ||||||
1253 | |||||||
1254 | // Specifies that for loads and stores VT can be promoted to PromotedLdStVT. | ||||||
1255 | SDValue | ||||||
1256 | HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { | ||||||
1257 | const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); | ||||||
1258 | SDLoc dl(Op); | ||||||
1259 | EVT PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
1260 | |||||||
1261 | Reloc::Model RM = HTM.getRelocationModel(); | ||||||
1262 | if (RM == Reloc::Static) { | ||||||
1263 | SDValue A = DAG.getTargetBlockAddress(BA, PtrVT); | ||||||
1264 | return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A); | ||||||
1265 | } | ||||||
1266 | |||||||
1267 | SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL); | ||||||
1268 | return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A); | ||||||
1269 | } | ||||||
1270 | |||||||
1271 | SDValue | ||||||
1272 | HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) | ||||||
1273 | const { | ||||||
1274 | EVT PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
1275 | SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME"_GLOBAL_OFFSET_TABLE_", PtrVT, | ||||||
1276 | HexagonII::MO_PCREL); | ||||||
1277 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym); | ||||||
1278 | } | ||||||
1279 | |||||||
1280 | SDValue | ||||||
1281 | HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain, | ||||||
1282 | GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg, | ||||||
1283 | unsigned char OperandFlags) const { | ||||||
1284 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||
1285 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||||
1286 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||
1287 | SDLoc dl(GA); | ||||||
1288 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, | ||||||
1289 | GA->getValueType(0), | ||||||
1290 | GA->getOffset(), | ||||||
1291 | OperandFlags); | ||||||
1292 | // Create Operands for the call.The Operands should have the following: | ||||||
1293 | // 1. Chain SDValue | ||||||
1294 | // 2. Callee which in this case is the Global address value. | ||||||
1295 | // 3. Registers live into the call.In this case its R0, as we | ||||||
1296 | // have just one argument to be passed. | ||||||
1297 | // 4. Glue. | ||||||
1298 | // Note: The order is important. | ||||||
1299 | |||||||
1300 | const auto &HRI = *Subtarget.getRegisterInfo(); | ||||||
1301 | const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C); | ||||||
1302 | assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention" ) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1302, __extension__ __PRETTY_FUNCTION__)); | ||||||
1303 | SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT), | ||||||
1304 | DAG.getRegisterMask(Mask), Glue }; | ||||||
1305 | Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops); | ||||||
1306 | |||||||
1307 | // Inform MFI that function has calls. | ||||||
1308 | MFI.setAdjustsStack(true); | ||||||
1309 | |||||||
1310 | Glue = Chain.getValue(1); | ||||||
1311 | return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue); | ||||||
1312 | } | ||||||
1313 | |||||||
1314 | // | ||||||
1315 | // Lower using the intial executable model for TLS addresses | ||||||
1316 | // | ||||||
1317 | SDValue | ||||||
1318 | HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA, | ||||||
1319 | SelectionDAG &DAG) const { | ||||||
1320 | SDLoc dl(GA); | ||||||
1321 | int64_t Offset = GA->getOffset(); | ||||||
1322 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
1323 | |||||||
1324 | // Get the thread pointer. | ||||||
1325 | SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT); | ||||||
1326 | |||||||
1327 | bool IsPositionIndependent = isPositionIndependent(); | ||||||
1328 | unsigned char TF = | ||||||
1329 | IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE; | ||||||
1330 | |||||||
1331 | // First generate the TLS symbol address | ||||||
1332 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, | ||||||
1333 | Offset, TF); | ||||||
1334 | |||||||
1335 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | ||||||
1336 | |||||||
1337 | if (IsPositionIndependent) { | ||||||
1338 | // Generate the GOT pointer in case of position independent code | ||||||
1339 | SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG); | ||||||
1340 | |||||||
1341 | // Add the TLS Symbol address to GOT pointer.This gives | ||||||
1342 | // GOT relative relocation for the symbol. | ||||||
1343 | Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym); | ||||||
1344 | } | ||||||
1345 | |||||||
1346 | // Load the offset value for TLS symbol.This offset is relative to | ||||||
1347 | // thread pointer. | ||||||
1348 | SDValue LoadOffset = | ||||||
1349 | DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo()); | ||||||
1350 | |||||||
1351 | // Address of the thread local variable is the add of thread | ||||||
1352 | // pointer and the offset of the variable. | ||||||
1353 | return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset); | ||||||
1354 | } | ||||||
1355 | |||||||
1356 | // | ||||||
1357 | // Lower using the local executable model for TLS addresses | ||||||
1358 | // | ||||||
1359 | SDValue | ||||||
1360 | HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA, | ||||||
1361 | SelectionDAG &DAG) const { | ||||||
1362 | SDLoc dl(GA); | ||||||
1363 | int64_t Offset = GA->getOffset(); | ||||||
1364 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
1365 | |||||||
1366 | // Get the thread pointer. | ||||||
1367 | SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT); | ||||||
1368 | // Generate the TLS symbol address | ||||||
1369 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset, | ||||||
1370 | HexagonII::MO_TPREL); | ||||||
1371 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | ||||||
1372 | |||||||
1373 | // Address of the thread local variable is the add of thread | ||||||
1374 | // pointer and the offset of the variable. | ||||||
1375 | return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym); | ||||||
1376 | } | ||||||
1377 | |||||||
1378 | // | ||||||
1379 | // Lower using the general dynamic model for TLS addresses | ||||||
1380 | // | ||||||
1381 | SDValue | ||||||
1382 | HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, | ||||||
1383 | SelectionDAG &DAG) const { | ||||||
1384 | SDLoc dl(GA); | ||||||
1385 | int64_t Offset = GA->getOffset(); | ||||||
1386 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
1387 | |||||||
1388 | // First generate the TLS symbol address | ||||||
1389 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset, | ||||||
1390 | HexagonII::MO_GDGOT); | ||||||
1391 | |||||||
1392 | // Then, generate the GOT pointer | ||||||
1393 | SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG); | ||||||
1394 | |||||||
1395 | // Add the TLS symbol and the GOT pointer | ||||||
1396 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | ||||||
1397 | SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym); | ||||||
1398 | |||||||
1399 | // Copy over the argument to R0 | ||||||
1400 | SDValue InFlag; | ||||||
1401 | Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag); | ||||||
1402 | InFlag = Chain.getValue(1); | ||||||
1403 | |||||||
1404 | unsigned Flags = | ||||||
1405 | static_cast<const HexagonSubtarget &>(DAG.getSubtarget()).useLongCalls() | ||||||
1406 | ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended | ||||||
1407 | : HexagonII::MO_GDPLT; | ||||||
1408 | |||||||
1409 | return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT, | ||||||
1410 | Hexagon::R0, Flags); | ||||||
1411 | } | ||||||
1412 | |||||||
1413 | // | ||||||
1414 | // Lower TLS addresses. | ||||||
1415 | // | ||||||
1416 | // For now for dynamic models, we only support the general dynamic model. | ||||||
1417 | // | ||||||
1418 | SDValue | ||||||
1419 | HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op, | ||||||
1420 | SelectionDAG &DAG) const { | ||||||
1421 | GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); | ||||||
1422 | |||||||
1423 | switch (HTM.getTLSModel(GA->getGlobal())) { | ||||||
1424 | case TLSModel::GeneralDynamic: | ||||||
1425 | case TLSModel::LocalDynamic: | ||||||
1426 | return LowerToTLSGeneralDynamicModel(GA, DAG); | ||||||
1427 | case TLSModel::InitialExec: | ||||||
1428 | return LowerToTLSInitialExecModel(GA, DAG); | ||||||
1429 | case TLSModel::LocalExec: | ||||||
1430 | return LowerToTLSLocalExecModel(GA, DAG); | ||||||
1431 | } | ||||||
1432 | llvm_unreachable("Bogus TLS model")::llvm::llvm_unreachable_internal("Bogus TLS model", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1432); | ||||||
1433 | } | ||||||
1434 | |||||||
1435 | //===----------------------------------------------------------------------===// | ||||||
1436 | // TargetLowering Implementation | ||||||
1437 | //===----------------------------------------------------------------------===// | ||||||
1438 | |||||||
1439 | HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, | ||||||
1440 | const HexagonSubtarget &ST) | ||||||
1441 | : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)), | ||||||
1442 | Subtarget(ST) { | ||||||
1443 | auto &HRI = *Subtarget.getRegisterInfo(); | ||||||
1444 | |||||||
1445 | setPrefLoopAlignment(Align(16)); | ||||||
1446 | setMinFunctionAlignment(Align(4)); | ||||||
1447 | setPrefFunctionAlignment(Align(16)); | ||||||
1448 | setStackPointerRegisterToSaveRestore(HRI.getStackRegister()); | ||||||
1449 | setBooleanContents(TargetLoweringBase::UndefinedBooleanContent); | ||||||
1450 | setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent); | ||||||
1451 | |||||||
1452 | setMaxAtomicSizeInBitsSupported(64); | ||||||
1453 | setMinCmpXchgSizeInBits(32); | ||||||
1454 | |||||||
1455 | if (EnableHexSDNodeSched) | ||||||
1456 | setSchedulingPreference(Sched::VLIW); | ||||||
1457 | else | ||||||
1458 | setSchedulingPreference(Sched::Source); | ||||||
1459 | |||||||
1460 | // Limits for inline expansion of memcpy/memmove | ||||||
1461 | MaxStoresPerMemcpy = MaxStoresPerMemcpyCL; | ||||||
1462 | MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL; | ||||||
1463 | MaxStoresPerMemmove = MaxStoresPerMemmoveCL; | ||||||
1464 | MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL; | ||||||
1465 | MaxStoresPerMemset = MaxStoresPerMemsetCL; | ||||||
1466 | MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL; | ||||||
1467 | |||||||
1468 | // | ||||||
1469 | // Set up register classes. | ||||||
1470 | // | ||||||
1471 | |||||||
1472 | addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass); | ||||||
1473 | addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass); // bbbbaaaa | ||||||
1474 | addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass); // ddccbbaa | ||||||
1475 | addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass); // hgfedcba | ||||||
1476 | addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass); | ||||||
1477 | addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass); | ||||||
1478 | addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass); | ||||||
1479 | addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass); | ||||||
1480 | addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass); | ||||||
1481 | addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass); | ||||||
1482 | addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass); | ||||||
1483 | |||||||
1484 | addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass); | ||||||
1485 | addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass); | ||||||
1486 | |||||||
1487 | // | ||||||
1488 | // Handling of scalar operations. | ||||||
1489 | // | ||||||
1490 | // All operations default to "legal", except: | ||||||
1491 | // - indexed loads and stores (pre-/post-incremented), | ||||||
1492 | // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS, | ||||||
1493 | // ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN, | ||||||
1494 | // FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP, | ||||||
1495 | // FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG, | ||||||
1496 | // which default to "expand" for at least one type. | ||||||
1497 | |||||||
1498 | // Misc operations. | ||||||
1499 | setOperationAction(ISD::ConstantFP, MVT::f32, Legal); | ||||||
1500 | setOperationAction(ISD::ConstantFP, MVT::f64, Legal); | ||||||
1501 | setOperationAction(ISD::TRAP, MVT::Other, Legal); | ||||||
1502 | setOperationAction(ISD::ConstantPool, MVT::i32, Custom); | ||||||
1503 | setOperationAction(ISD::JumpTable, MVT::i32, Custom); | ||||||
1504 | setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); | ||||||
1505 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); | ||||||
1506 | setOperationAction(ISD::INLINEASM, MVT::Other, Custom); | ||||||
1507 | setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom); | ||||||
1508 | setOperationAction(ISD::PREFETCH, MVT::Other, Custom); | ||||||
1509 | setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); | ||||||
1510 | setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); | ||||||
1511 | setOperationAction(ISD::EH_RETURN, MVT::Other, Custom); | ||||||
1512 | setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom); | ||||||
1513 | setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); | ||||||
1514 | setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); | ||||||
1515 | |||||||
1516 | // Custom legalize GlobalAddress nodes into CONST32. | ||||||
1517 | setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); | ||||||
1518 | setOperationAction(ISD::GlobalAddress, MVT::i8, Custom); | ||||||
1519 | setOperationAction(ISD::BlockAddress, MVT::i32, Custom); | ||||||
1520 | |||||||
1521 | // Hexagon needs to optimize cases with negative constants. | ||||||
1522 | setOperationAction(ISD::SETCC, MVT::i8, Custom); | ||||||
1523 | setOperationAction(ISD::SETCC, MVT::i16, Custom); | ||||||
1524 | setOperationAction(ISD::SETCC, MVT::v4i8, Custom); | ||||||
1525 | setOperationAction(ISD::SETCC, MVT::v2i16, Custom); | ||||||
1526 | |||||||
1527 | // VASTART needs to be custom lowered to use the VarArgsFrameIndex. | ||||||
1528 | setOperationAction(ISD::VASTART, MVT::Other, Custom); | ||||||
1529 | setOperationAction(ISD::VAEND, MVT::Other, Expand); | ||||||
1530 | setOperationAction(ISD::VAARG, MVT::Other, Expand); | ||||||
1531 | if (Subtarget.isEnvironmentMusl()) | ||||||
1532 | setOperationAction(ISD::VACOPY, MVT::Other, Custom); | ||||||
1533 | else | ||||||
1534 | setOperationAction(ISD::VACOPY, MVT::Other, Expand); | ||||||
1535 | |||||||
1536 | setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); | ||||||
1537 | setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); | ||||||
1538 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); | ||||||
1539 | |||||||
1540 | if (EmitJumpTables) | ||||||
1541 | setMinimumJumpTableEntries(MinimumJumpTables); | ||||||
1542 | else | ||||||
1543 | setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max()); | ||||||
1544 | setOperationAction(ISD::BR_JT, MVT::Other, Expand); | ||||||
1545 | |||||||
1546 | for (unsigned LegalIntOp : | ||||||
1547 | {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) { | ||||||
1548 | setOperationAction(LegalIntOp, MVT::i32, Legal); | ||||||
1549 | setOperationAction(LegalIntOp, MVT::i64, Legal); | ||||||
1550 | } | ||||||
1551 | |||||||
1552 | // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit, | ||||||
1553 | // but they only operate on i64. | ||||||
1554 | for (MVT VT : MVT::integer_valuetypes()) { | ||||||
1555 | setOperationAction(ISD::UADDO, VT, Custom); | ||||||
1556 | setOperationAction(ISD::USUBO, VT, Custom); | ||||||
1557 | setOperationAction(ISD::SADDO, VT, Expand); | ||||||
1558 | setOperationAction(ISD::SSUBO, VT, Expand); | ||||||
1559 | setOperationAction(ISD::ADDCARRY, VT, Expand); | ||||||
1560 | setOperationAction(ISD::SUBCARRY, VT, Expand); | ||||||
1561 | } | ||||||
1562 | setOperationAction(ISD::ADDCARRY, MVT::i64, Custom); | ||||||
1563 | setOperationAction(ISD::SUBCARRY, MVT::i64, Custom); | ||||||
1564 | |||||||
1565 | setOperationAction(ISD::CTLZ, MVT::i8, Promote); | ||||||
1566 | setOperationAction(ISD::CTLZ, MVT::i16, Promote); | ||||||
1567 | setOperationAction(ISD::CTTZ, MVT::i8, Promote); | ||||||
1568 | setOperationAction(ISD::CTTZ, MVT::i16, Promote); | ||||||
1569 | |||||||
1570 | // Popcount can count # of 1s in i64 but returns i32. | ||||||
1571 | setOperationAction(ISD::CTPOP, MVT::i8, Promote); | ||||||
1572 | setOperationAction(ISD::CTPOP, MVT::i16, Promote); | ||||||
1573 | setOperationAction(ISD::CTPOP, MVT::i32, Promote); | ||||||
1574 | setOperationAction(ISD::CTPOP, MVT::i64, Legal); | ||||||
1575 | |||||||
1576 | setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); | ||||||
1577 | setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); | ||||||
1578 | setOperationAction(ISD::BSWAP, MVT::i32, Legal); | ||||||
1579 | setOperationAction(ISD::BSWAP, MVT::i64, Legal); | ||||||
1580 | |||||||
1581 | setOperationAction(ISD::FSHL, MVT::i32, Legal); | ||||||
1582 | setOperationAction(ISD::FSHL, MVT::i64, Legal); | ||||||
1583 | setOperationAction(ISD::FSHR, MVT::i32, Legal); | ||||||
1584 | setOperationAction(ISD::FSHR, MVT::i64, Legal); | ||||||
1585 | |||||||
1586 | for (unsigned IntExpOp : | ||||||
1587 | {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, | ||||||
1588 | ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR, | ||||||
1589 | ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS, | ||||||
1590 | ISD::SMUL_LOHI, ISD::UMUL_LOHI}) { | ||||||
1591 | for (MVT VT : MVT::integer_valuetypes()) | ||||||
1592 | setOperationAction(IntExpOp, VT, Expand); | ||||||
1593 | } | ||||||
1594 | |||||||
1595 | for (unsigned FPExpOp : | ||||||
1596 | {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS, | ||||||
1597 | ISD::FPOW, ISD::FCOPYSIGN}) { | ||||||
1598 | for (MVT VT : MVT::fp_valuetypes()) | ||||||
1599 | setOperationAction(FPExpOp, VT, Expand); | ||||||
1600 | } | ||||||
1601 | |||||||
1602 | // No extending loads from i32. | ||||||
1603 | for (MVT VT : MVT::integer_valuetypes()) { | ||||||
1604 | setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand); | ||||||
1605 | setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); | ||||||
1606 | setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand); | ||||||
1607 | } | ||||||
1608 | // Turn FP truncstore into trunc + store. | ||||||
1609 | setTruncStoreAction(MVT::f64, MVT::f32, Expand); | ||||||
1610 | // Turn FP extload into load/fpextend. | ||||||
1611 | for (MVT VT : MVT::fp_valuetypes()) | ||||||
1612 | setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); | ||||||
1613 | |||||||
1614 | // Expand BR_CC and SELECT_CC for all integer and fp types. | ||||||
1615 | for (MVT VT : MVT::integer_valuetypes()) { | ||||||
1616 | setOperationAction(ISD::BR_CC, VT, Expand); | ||||||
1617 | setOperationAction(ISD::SELECT_CC, VT, Expand); | ||||||
1618 | } | ||||||
1619 | for (MVT VT : MVT::fp_valuetypes()) { | ||||||
1620 | setOperationAction(ISD::BR_CC, VT, Expand); | ||||||
1621 | setOperationAction(ISD::SELECT_CC, VT, Expand); | ||||||
1622 | } | ||||||
1623 | setOperationAction(ISD::BR_CC, MVT::Other, Expand); | ||||||
1624 | |||||||
1625 | // | ||||||
1626 | // Handling of vector operations. | ||||||
1627 | // | ||||||
1628 | |||||||
1629 | // Set the action for vector operations to "expand", then override it with | ||||||
1630 | // either "custom" or "legal" for specific cases. | ||||||
1631 | static const unsigned VectExpOps[] = { | ||||||
1632 | // Integer arithmetic: | ||||||
1633 | ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV, | ||||||
1634 | ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO, | ||||||
1635 | ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI, | ||||||
1636 | // Logical/bit: | ||||||
1637 | ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR, | ||||||
1638 | ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, | ||||||
1639 | // Floating point arithmetic/math functions: | ||||||
1640 | ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV, | ||||||
1641 | ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN, | ||||||
1642 | ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2, | ||||||
1643 | ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC, | ||||||
1644 | ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR, | ||||||
1645 | ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS, | ||||||
1646 | // Misc: | ||||||
1647 | ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool, | ||||||
1648 | // Vector: | ||||||
1649 | ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR, | ||||||
1650 | ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT, | ||||||
1651 | ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR, | ||||||
1652 | ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE, | ||||||
1653 | ISD::SPLAT_VECTOR, | ||||||
1654 | }; | ||||||
1655 | |||||||
1656 | for (MVT VT : MVT::fixedlen_vector_valuetypes()) { | ||||||
1657 | for (unsigned VectExpOp : VectExpOps) | ||||||
1658 | setOperationAction(VectExpOp, VT, Expand); | ||||||
1659 | |||||||
1660 | // Expand all extending loads and truncating stores: | ||||||
1661 | for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) { | ||||||
1662 | if (TargetVT == VT) | ||||||
1663 | continue; | ||||||
1664 | setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand); | ||||||
1665 | setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand); | ||||||
1666 | setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand); | ||||||
1667 | setTruncStoreAction(VT, TargetVT, Expand); | ||||||
1668 | } | ||||||
1669 | |||||||
1670 | // Normalize all inputs to SELECT to be vectors of i32. | ||||||
1671 | if (VT.getVectorElementType() != MVT::i32) { | ||||||
1672 | MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32); | ||||||
1673 | setOperationAction(ISD::SELECT, VT, Promote); | ||||||
1674 | AddPromotedToType(ISD::SELECT, VT, VT32); | ||||||
1675 | } | ||||||
1676 | setOperationAction(ISD::SRA, VT, Custom); | ||||||
1677 | setOperationAction(ISD::SHL, VT, Custom); | ||||||
1678 | setOperationAction(ISD::SRL, VT, Custom); | ||||||
1679 | } | ||||||
1680 | |||||||
1681 | // Extending loads from (native) vectors of i8 into (native) vectors of i16 | ||||||
1682 | // are legal. | ||||||
1683 | setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | ||||||
1684 | setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | ||||||
1685 | setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | ||||||
1686 | setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | ||||||
1687 | setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | ||||||
1688 | setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | ||||||
1689 | |||||||
1690 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal); | ||||||
1691 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal); | ||||||
1692 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); | ||||||
1693 | |||||||
1694 | // Types natively supported: | ||||||
1695 | for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8, | ||||||
1696 | MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) { | ||||||
1697 | setOperationAction(ISD::BUILD_VECTOR, NativeVT, Custom); | ||||||
1698 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom); | ||||||
1699 | setOperationAction(ISD::INSERT_VECTOR_ELT, NativeVT, Custom); | ||||||
1700 | setOperationAction(ISD::EXTRACT_SUBVECTOR, NativeVT, Custom); | ||||||
1701 | setOperationAction(ISD::INSERT_SUBVECTOR, NativeVT, Custom); | ||||||
1702 | setOperationAction(ISD::CONCAT_VECTORS, NativeVT, Custom); | ||||||
1703 | |||||||
1704 | setOperationAction(ISD::ADD, NativeVT, Legal); | ||||||
1705 | setOperationAction(ISD::SUB, NativeVT, Legal); | ||||||
1706 | setOperationAction(ISD::MUL, NativeVT, Legal); | ||||||
1707 | setOperationAction(ISD::AND, NativeVT, Legal); | ||||||
1708 | setOperationAction(ISD::OR, NativeVT, Legal); | ||||||
1709 | setOperationAction(ISD::XOR, NativeVT, Legal); | ||||||
1710 | |||||||
1711 | if (NativeVT.getVectorElementType() != MVT::i1) | ||||||
1712 | setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal); | ||||||
1713 | } | ||||||
1714 | |||||||
1715 | for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) { | ||||||
1716 | setOperationAction(ISD::SMIN, VT, Legal); | ||||||
1717 | setOperationAction(ISD::SMAX, VT, Legal); | ||||||
1718 | setOperationAction(ISD::UMIN, VT, Legal); | ||||||
1719 | setOperationAction(ISD::UMAX, VT, Legal); | ||||||
1720 | } | ||||||
1721 | |||||||
1722 | // Custom lower unaligned loads. | ||||||
1723 | // Also, for both loads and stores, verify the alignment of the address | ||||||
1724 | // in case it is a compile-time constant. This is a usability feature to | ||||||
1725 | // provide a meaningful error message to users. | ||||||
1726 | for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8, | ||||||
1727 | MVT::v2i16, MVT::v4i16, MVT::v2i32}) { | ||||||
1728 | setOperationAction(ISD::LOAD, VT, Custom); | ||||||
1729 | setOperationAction(ISD::STORE, VT, Custom); | ||||||
1730 | } | ||||||
1731 | |||||||
1732 | // Custom-lower load/stores of boolean vectors. | ||||||
1733 | for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) { | ||||||
1734 | setOperationAction(ISD::LOAD, VT, Custom); | ||||||
1735 | setOperationAction(ISD::STORE, VT, Custom); | ||||||
1736 | } | ||||||
1737 | |||||||
1738 | for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16, | ||||||
1739 | MVT::v2i32}) { | ||||||
1740 | setCondCodeAction(ISD::SETNE, VT, Expand); | ||||||
1741 | setCondCodeAction(ISD::SETLE, VT, Expand); | ||||||
1742 | setCondCodeAction(ISD::SETGE, VT, Expand); | ||||||
1743 | setCondCodeAction(ISD::SETLT, VT, Expand); | ||||||
1744 | setCondCodeAction(ISD::SETULE, VT, Expand); | ||||||
1745 | setCondCodeAction(ISD::SETUGE, VT, Expand); | ||||||
1746 | setCondCodeAction(ISD::SETULT, VT, Expand); | ||||||
1747 | } | ||||||
1748 | |||||||
1749 | // Custom-lower bitcasts from i8 to v8i1. | ||||||
1750 | setOperationAction(ISD::BITCAST, MVT::i8, Custom); | ||||||
1751 | setOperationAction(ISD::SETCC, MVT::v2i16, Custom); | ||||||
1752 | setOperationAction(ISD::VSELECT, MVT::v4i8, Custom); | ||||||
1753 | setOperationAction(ISD::VSELECT, MVT::v2i16, Custom); | ||||||
1754 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom); | ||||||
1755 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); | ||||||
1756 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); | ||||||
1757 | |||||||
1758 | // V5+. | ||||||
1759 | setOperationAction(ISD::FMA, MVT::f64, Expand); | ||||||
1760 | setOperationAction(ISD::FADD, MVT::f64, Expand); | ||||||
1761 | setOperationAction(ISD::FSUB, MVT::f64, Expand); | ||||||
1762 | setOperationAction(ISD::FMUL, MVT::f64, Expand); | ||||||
1763 | |||||||
1764 | setOperationAction(ISD::FMINNUM, MVT::f32, Legal); | ||||||
1765 | setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); | ||||||
1766 | |||||||
1767 | setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote); | ||||||
1768 | setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote); | ||||||
1769 | setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote); | ||||||
1770 | setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote); | ||||||
1771 | setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote); | ||||||
1772 | setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote); | ||||||
1773 | setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); | ||||||
1774 | setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote); | ||||||
1775 | setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote); | ||||||
1776 | setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); | ||||||
1777 | setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote); | ||||||
1778 | setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote); | ||||||
1779 | |||||||
1780 | // Handling of indexed loads/stores: default is "expand". | ||||||
1781 | // | ||||||
1782 | for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64, | ||||||
1783 | MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) { | ||||||
1784 | setIndexedLoadAction(ISD::POST_INC, VT, Legal); | ||||||
1785 | setIndexedStoreAction(ISD::POST_INC, VT, Legal); | ||||||
1786 | } | ||||||
1787 | |||||||
1788 | // Subtarget-specific operation actions. | ||||||
1789 | // | ||||||
1790 | if (Subtarget.hasV60Ops()) { | ||||||
1791 | setOperationAction(ISD::ROTL, MVT::i32, Legal); | ||||||
1792 | setOperationAction(ISD::ROTL, MVT::i64, Legal); | ||||||
1793 | setOperationAction(ISD::ROTR, MVT::i32, Legal); | ||||||
1794 | setOperationAction(ISD::ROTR, MVT::i64, Legal); | ||||||
1795 | } | ||||||
1796 | if (Subtarget.hasV66Ops()) { | ||||||
1797 | setOperationAction(ISD::FADD, MVT::f64, Legal); | ||||||
1798 | setOperationAction(ISD::FSUB, MVT::f64, Legal); | ||||||
1799 | } | ||||||
1800 | if (Subtarget.hasV67Ops()) { | ||||||
1801 | setOperationAction(ISD::FMINNUM, MVT::f64, Legal); | ||||||
1802 | setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); | ||||||
1803 | setOperationAction(ISD::FMUL, MVT::f64, Legal); | ||||||
1804 | } | ||||||
1805 | |||||||
1806 | setTargetDAGCombine(ISD::VSELECT); | ||||||
1807 | |||||||
1808 | if (Subtarget.useHVXOps()) | ||||||
1809 | initializeHVXLowering(); | ||||||
1810 | |||||||
1811 | computeRegisterProperties(&HRI); | ||||||
1812 | |||||||
1813 | // | ||||||
1814 | // Library calls for unsupported operations | ||||||
1815 | // | ||||||
1816 | bool FastMath = EnableFastMath; | ||||||
1817 | |||||||
1818 | setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3"); | ||||||
1819 | setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3"); | ||||||
1820 | setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3"); | ||||||
1821 | setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3"); | ||||||
1822 | setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3"); | ||||||
1823 | setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3"); | ||||||
1824 | setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3"); | ||||||
1825 | setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3"); | ||||||
1826 | |||||||
1827 | setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf"); | ||||||
1828 | setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf"); | ||||||
1829 | setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti"); | ||||||
1830 | setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti"); | ||||||
1831 | setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti"); | ||||||
1832 | setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti"); | ||||||
1833 | |||||||
1834 | // This is the only fast library function for sqrtd. | ||||||
1835 | if (FastMath) | ||||||
1836 | setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2"); | ||||||
1837 | |||||||
1838 | // Prefix is: nothing for "slow-math", | ||||||
1839 | // "fast2_" for V5+ fast-math double-precision | ||||||
1840 | // (actually, keep fast-math and fast-math2 separate for now) | ||||||
1841 | if (FastMath) { | ||||||
1842 | setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3"); | ||||||
1843 | setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3"); | ||||||
1844 | setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3"); | ||||||
1845 | setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3"); | ||||||
1846 | setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3"); | ||||||
1847 | } else { | ||||||
1848 | setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3"); | ||||||
1849 | setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3"); | ||||||
1850 | setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3"); | ||||||
1851 | setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3"); | ||||||
1852 | setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3"); | ||||||
1853 | } | ||||||
1854 | |||||||
1855 | if (FastMath) | ||||||
1856 | setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf"); | ||||||
1857 | else | ||||||
1858 | setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf"); | ||||||
1859 | |||||||
1860 | // These cause problems when the shift amount is non-constant. | ||||||
1861 | setLibcallName(RTLIB::SHL_I128, nullptr); | ||||||
1862 | setLibcallName(RTLIB::SRL_I128, nullptr); | ||||||
1863 | setLibcallName(RTLIB::SRA_I128, nullptr); | ||||||
1864 | } | ||||||
1865 | |||||||
1866 | const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const { | ||||||
1867 | switch ((HexagonISD::NodeType)Opcode) { | ||||||
1868 | case HexagonISD::ADDC: return "HexagonISD::ADDC"; | ||||||
1869 | case HexagonISD::SUBC: return "HexagonISD::SUBC"; | ||||||
1870 | case HexagonISD::ALLOCA: return "HexagonISD::ALLOCA"; | ||||||
1871 | case HexagonISD::AT_GOT: return "HexagonISD::AT_GOT"; | ||||||
1872 | case HexagonISD::AT_PCREL: return "HexagonISD::AT_PCREL"; | ||||||
1873 | case HexagonISD::BARRIER: return "HexagonISD::BARRIER"; | ||||||
1874 | case HexagonISD::CALL: return "HexagonISD::CALL"; | ||||||
1875 | case HexagonISD::CALLnr: return "HexagonISD::CALLnr"; | ||||||
1876 | case HexagonISD::CALLR: return "HexagonISD::CALLR"; | ||||||
1877 | case HexagonISD::COMBINE: return "HexagonISD::COMBINE"; | ||||||
1878 | case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP"; | ||||||
1879 | case HexagonISD::CONST32: return "HexagonISD::CONST32"; | ||||||
1880 | case HexagonISD::CP: return "HexagonISD::CP"; | ||||||
1881 | case HexagonISD::DCFETCH: return "HexagonISD::DCFETCH"; | ||||||
1882 | case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN"; | ||||||
1883 | case HexagonISD::TSTBIT: return "HexagonISD::TSTBIT"; | ||||||
1884 | case HexagonISD::EXTRACTU: return "HexagonISD::EXTRACTU"; | ||||||
1885 | case HexagonISD::INSERT: return "HexagonISD::INSERT"; | ||||||
1886 | case HexagonISD::JT: return "HexagonISD::JT"; | ||||||
1887 | case HexagonISD::RET_FLAG: return "HexagonISD::RET_FLAG"; | ||||||
1888 | case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN"; | ||||||
1889 | case HexagonISD::VASL: return "HexagonISD::VASL"; | ||||||
1890 | case HexagonISD::VASR: return "HexagonISD::VASR"; | ||||||
1891 | case HexagonISD::VLSR: return "HexagonISD::VLSR"; | ||||||
1892 | case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW"; | ||||||
1893 | case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0"; | ||||||
1894 | case HexagonISD::VROR: return "HexagonISD::VROR"; | ||||||
1895 | case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE"; | ||||||
1896 | case HexagonISD::PTRUE: return "HexagonISD::PTRUE"; | ||||||
1897 | case HexagonISD::PFALSE: return "HexagonISD::PFALSE"; | ||||||
1898 | case HexagonISD::D2P: return "HexagonISD::D2P"; | ||||||
1899 | case HexagonISD::P2D: return "HexagonISD::P2D"; | ||||||
1900 | case HexagonISD::V2Q: return "HexagonISD::V2Q"; | ||||||
1901 | case HexagonISD::Q2V: return "HexagonISD::Q2V"; | ||||||
1902 | case HexagonISD::QCAT: return "HexagonISD::QCAT"; | ||||||
1903 | case HexagonISD::QTRUE: return "HexagonISD::QTRUE"; | ||||||
1904 | case HexagonISD::QFALSE: return "HexagonISD::QFALSE"; | ||||||
1905 | case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST"; | ||||||
1906 | case HexagonISD::VALIGN: return "HexagonISD::VALIGN"; | ||||||
1907 | case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR"; | ||||||
1908 | case HexagonISD::VPACKL: return "HexagonISD::VPACKL"; | ||||||
1909 | case HexagonISD::VUNPACK: return "HexagonISD::VUNPACK"; | ||||||
1910 | case HexagonISD::VUNPACKU: return "HexagonISD::VUNPACKU"; | ||||||
1911 | case HexagonISD::ISEL: return "HexagonISD::ISEL"; | ||||||
1912 | case HexagonISD::OP_END: break; | ||||||
1913 | } | ||||||
1914 | return nullptr; | ||||||
1915 | } | ||||||
1916 | |||||||
1917 | bool | ||||||
1918 | HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign, | ||||||
1919 | const SDLoc &dl, SelectionDAG &DAG) const { | ||||||
1920 | auto *CA = dyn_cast<ConstantSDNode>(Ptr); | ||||||
1921 | if (!CA) | ||||||
1922 | return true; | ||||||
1923 | unsigned Addr = CA->getZExtValue(); | ||||||
1924 | Align HaveAlign = | ||||||
1925 | Addr != 0 ? Align(1ull << countTrailingZeros(Addr)) : NeedAlign; | ||||||
1926 | if (HaveAlign >= NeedAlign) | ||||||
1927 | return true; | ||||||
1928 | |||||||
1929 | static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind(); | ||||||
1930 | |||||||
1931 | struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo { | ||||||
1932 | DiagnosticInfoMisalignedTrap(StringRef M) | ||||||
1933 | : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {} | ||||||
1934 | void print(DiagnosticPrinter &DP) const override { | ||||||
1935 | DP << Msg; | ||||||
1936 | } | ||||||
1937 | static bool classof(const DiagnosticInfo *DI) { | ||||||
1938 | return DI->getKind() == DK_MisalignedTrap; | ||||||
1939 | } | ||||||
1940 | StringRef Msg; | ||||||
1941 | }; | ||||||
1942 | |||||||
1943 | std::string ErrMsg; | ||||||
1944 | raw_string_ostream O(ErrMsg); | ||||||
1945 | O << "Misaligned constant address: " << format_hex(Addr, 10) | ||||||
1946 | << " has alignment " << HaveAlign.value() | ||||||
1947 | << ", but the memory access requires " << NeedAlign.value(); | ||||||
1948 | if (DebugLoc DL = dl.getDebugLoc()) | ||||||
1949 | DL.print(O << ", at "); | ||||||
1950 | O << ". The instruction has been replaced with a trap."; | ||||||
1951 | |||||||
1952 | DAG.getContext()->diagnose(DiagnosticInfoMisalignedTrap(O.str())); | ||||||
1953 | return false; | ||||||
1954 | } | ||||||
1955 | |||||||
1956 | SDValue | ||||||
1957 | HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG) | ||||||
1958 | const { | ||||||
1959 | const SDLoc &dl(Op); | ||||||
1960 | auto *LS = cast<LSBaseSDNode>(Op.getNode()); | ||||||
1961 | assert(!LS->isIndexed() && "Not expecting indexed ops on constant address")(static_cast <bool> (!LS->isIndexed() && "Not expecting indexed ops on constant address" ) ? void (0) : __assert_fail ("!LS->isIndexed() && \"Not expecting indexed ops on constant address\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1961, __extension__ __PRETTY_FUNCTION__)); | ||||||
1962 | |||||||
1963 | SDValue Chain = LS->getChain(); | ||||||
1964 | SDValue Trap = DAG.getNode(ISD::TRAP, dl, MVT::Other, Chain); | ||||||
1965 | if (LS->getOpcode() == ISD::LOAD) | ||||||
1966 | return DAG.getMergeValues({DAG.getUNDEF(ty(Op)), Trap}, dl); | ||||||
1967 | return Trap; | ||||||
1968 | } | ||||||
1969 | |||||||
1970 | // Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load | ||||||
1971 | // intrinsic. | ||||||
1972 | static bool isBrevLdIntrinsic(const Value *Inst) { | ||||||
1973 | unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID(); | ||||||
1974 | return (ID == Intrinsic::hexagon_L2_loadrd_pbr || | ||||||
1975 | ID == Intrinsic::hexagon_L2_loadri_pbr || | ||||||
1976 | ID == Intrinsic::hexagon_L2_loadrh_pbr || | ||||||
1977 | ID == Intrinsic::hexagon_L2_loadruh_pbr || | ||||||
1978 | ID == Intrinsic::hexagon_L2_loadrb_pbr || | ||||||
1979 | ID == Intrinsic::hexagon_L2_loadrub_pbr); | ||||||
1980 | } | ||||||
1981 | |||||||
1982 | // Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous | ||||||
1983 | // instruction. So far we only handle bitcast, extract value and bit reverse | ||||||
1984 | // load intrinsic instructions. Should we handle CGEP ? | ||||||
1985 | static Value *getBrevLdObject(Value *V) { | ||||||
1986 | if (Operator::getOpcode(V) == Instruction::ExtractValue || | ||||||
1987 | Operator::getOpcode(V) == Instruction::BitCast) | ||||||
1988 | V = cast<Operator>(V)->getOperand(0); | ||||||
1989 | else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V)) | ||||||
1990 | V = cast<Instruction>(V)->getOperand(0); | ||||||
1991 | return V; | ||||||
1992 | } | ||||||
1993 | |||||||
1994 | // Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or | ||||||
1995 | // a back edge. If the back edge comes from the intrinsic itself, the incoming | ||||||
1996 | // edge is returned. | ||||||
1997 | static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) { | ||||||
1998 | const BasicBlock *Parent = PN->getParent(); | ||||||
1999 | int Idx = -1; | ||||||
2000 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) { | ||||||
2001 | BasicBlock *Blk = PN->getIncomingBlock(i); | ||||||
2002 | // Determine if the back edge is originated from intrinsic. | ||||||
2003 | if (Blk == Parent) { | ||||||
2004 | Value *BackEdgeVal = PN->getIncomingValue(i); | ||||||
2005 | Value *BaseVal; | ||||||
2006 | // Loop over till we return the same Value or we hit the IntrBaseVal. | ||||||
2007 | do { | ||||||
2008 | BaseVal = BackEdgeVal; | ||||||
2009 | BackEdgeVal = getBrevLdObject(BackEdgeVal); | ||||||
2010 | } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal)); | ||||||
2011 | // If the getBrevLdObject returns IntrBaseVal, we should return the | ||||||
2012 | // incoming edge. | ||||||
2013 | if (IntrBaseVal == BackEdgeVal) | ||||||
2014 | continue; | ||||||
2015 | Idx = i; | ||||||
2016 | break; | ||||||
2017 | } else // Set the node to incoming edge. | ||||||
2018 | Idx = i; | ||||||
2019 | } | ||||||
2020 | assert(Idx >= 0 && "Unexpected index to incoming argument in PHI")(static_cast <bool> (Idx >= 0 && "Unexpected index to incoming argument in PHI" ) ? void (0) : __assert_fail ("Idx >= 0 && \"Unexpected index to incoming argument in PHI\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2020, __extension__ __PRETTY_FUNCTION__)); | ||||||
2021 | return PN->getIncomingValue(Idx); | ||||||
2022 | } | ||||||
2023 | |||||||
2024 | // Bit-reverse Load Intrinsic: Figure out the underlying object the base | ||||||
2025 | // pointer points to, for the bit-reverse load intrinsic. Setting this to | ||||||
2026 | // memoperand might help alias analysis to figure out the dependencies. | ||||||
2027 | static Value *getUnderLyingObjectForBrevLdIntr(Value *V) { | ||||||
2028 | Value *IntrBaseVal = V; | ||||||
2029 | Value *BaseVal; | ||||||
2030 | // Loop over till we return the same Value, implies we either figure out | ||||||
2031 | // the object or we hit a PHI | ||||||
2032 | do { | ||||||
2033 | BaseVal = V; | ||||||
2034 | V = getBrevLdObject(V); | ||||||
2035 | } while (BaseVal != V); | ||||||
2036 | |||||||
2037 | // Identify the object from PHINode. | ||||||
2038 | if (const PHINode *PN = dyn_cast<PHINode>(V)) | ||||||
2039 | return returnEdge(PN, IntrBaseVal); | ||||||
2040 | // For non PHI nodes, the object is the last value returned by getBrevLdObject | ||||||
2041 | else | ||||||
2042 | return V; | ||||||
2043 | } | ||||||
2044 | |||||||
2045 | /// Given an intrinsic, checks if on the target the intrinsic will need to map | ||||||
2046 | /// to a MemIntrinsicNode (touches memory). If this is the case, it returns | ||||||
2047 | /// true and store the intrinsic information into the IntrinsicInfo that was | ||||||
2048 | /// passed to the function. | ||||||
2049 | bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, | ||||||
2050 | const CallInst &I, | ||||||
2051 | MachineFunction &MF, | ||||||
2052 | unsigned Intrinsic) const { | ||||||
2053 | switch (Intrinsic) { | ||||||
2054 | case Intrinsic::hexagon_L2_loadrd_pbr: | ||||||
2055 | case Intrinsic::hexagon_L2_loadri_pbr: | ||||||
2056 | case Intrinsic::hexagon_L2_loadrh_pbr: | ||||||
2057 | case Intrinsic::hexagon_L2_loadruh_pbr: | ||||||
2058 | case Intrinsic::hexagon_L2_loadrb_pbr: | ||||||
2059 | case Intrinsic::hexagon_L2_loadrub_pbr: { | ||||||
2060 | Info.opc = ISD::INTRINSIC_W_CHAIN; | ||||||
2061 | auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); | ||||||
2062 | auto &Cont = I.getCalledFunction()->getParent()->getContext(); | ||||||
2063 | // The intrinsic function call is of the form { ElTy, i8* } | ||||||
2064 | // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type | ||||||
2065 | // should be derived from ElTy. | ||||||
2066 | Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0); | ||||||
2067 | Info.memVT = MVT::getVT(ElTy); | ||||||
2068 | llvm::Value *BasePtrVal = I.getOperand(0); | ||||||
2069 | Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal); | ||||||
2070 | // The offset value comes through Modifier register. For now, assume the | ||||||
2071 | // offset is 0. | ||||||
2072 | Info.offset = 0; | ||||||
2073 | Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont)); | ||||||
2074 | Info.flags = MachineMemOperand::MOLoad; | ||||||
2075 | return true; | ||||||
2076 | } | ||||||
2077 | case Intrinsic::hexagon_V6_vgathermw: | ||||||
2078 | case Intrinsic::hexagon_V6_vgathermw_128B: | ||||||
2079 | case Intrinsic::hexagon_V6_vgathermh: | ||||||
2080 | case Intrinsic::hexagon_V6_vgathermh_128B: | ||||||
2081 | case Intrinsic::hexagon_V6_vgathermhw: | ||||||
2082 | case Intrinsic::hexagon_V6_vgathermhw_128B: | ||||||
2083 | case Intrinsic::hexagon_V6_vgathermwq: | ||||||
2084 | case Intrinsic::hexagon_V6_vgathermwq_128B: | ||||||
2085 | case Intrinsic::hexagon_V6_vgathermhq: | ||||||
2086 | case Intrinsic::hexagon_V6_vgathermhq_128B: | ||||||
2087 | case Intrinsic::hexagon_V6_vgathermhwq: | ||||||
2088 | case Intrinsic::hexagon_V6_vgathermhwq_128B: { | ||||||
2089 | const Module &M = *I.getParent()->getParent()->getParent(); | ||||||
2090 | Info.opc = ISD::INTRINSIC_W_CHAIN; | ||||||
2091 | Type *VecTy = I.getArgOperand(1)->getType(); | ||||||
2092 | Info.memVT = MVT::getVT(VecTy); | ||||||
2093 | Info.ptrVal = I.getArgOperand(0); | ||||||
2094 | Info.offset = 0; | ||||||
2095 | Info.align = | ||||||
2096 | MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8); | ||||||
2097 | Info.flags = MachineMemOperand::MOLoad | | ||||||
2098 | MachineMemOperand::MOStore | | ||||||
2099 | MachineMemOperand::MOVolatile; | ||||||
2100 | return true; | ||||||
2101 | } | ||||||
2102 | default: | ||||||
2103 | break; | ||||||
2104 | } | ||||||
2105 | return false; | ||||||
2106 | } | ||||||
2107 | |||||||
2108 | bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const { | ||||||
2109 | return X.getValueType().isScalarInteger(); // 'tstbit' | ||||||
2110 | } | ||||||
2111 | |||||||
2112 | bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { | ||||||
2113 | return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2)); | ||||||
2114 | } | ||||||
2115 | |||||||
2116 | bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { | ||||||
2117 | if (!VT1.isSimple() || !VT2.isSimple()) | ||||||
2118 | return false; | ||||||
2119 | return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32; | ||||||
2120 | } | ||||||
2121 | |||||||
2122 | bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd( | ||||||
2123 | const MachineFunction &MF, EVT VT) const { | ||||||
2124 | return isOperationLegalOrCustom(ISD::FMA, VT); | ||||||
2125 | } | ||||||
2126 | |||||||
2127 | // Should we expand the build vector with shuffles? | ||||||
2128 | bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT, | ||||||
2129 | unsigned DefinedValues) const { | ||||||
2130 | return false; | ||||||
2131 | } | ||||||
2132 | |||||||
2133 | bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, | ||||||
2134 | EVT VT) const { | ||||||
2135 | return true; | ||||||
2136 | } | ||||||
2137 | |||||||
2138 | TargetLoweringBase::LegalizeTypeAction | ||||||
2139 | HexagonTargetLowering::getPreferredVectorAction(MVT VT) const { | ||||||
2140 | unsigned VecLen = VT.getVectorMinNumElements(); | ||||||
2141 | MVT ElemTy = VT.getVectorElementType(); | ||||||
2142 | |||||||
2143 | if (VecLen == 1 || VT.isScalableVector()) | ||||||
2144 | return TargetLoweringBase::TypeScalarizeVector; | ||||||
2145 | |||||||
2146 | if (Subtarget.useHVXOps()) { | ||||||
2147 | unsigned Action = getPreferredHvxVectorAction(VT); | ||||||
2148 | if (Action != ~0u) | ||||||
2149 | return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action); | ||||||
2150 | } | ||||||
2151 | |||||||
2152 | // Always widen (remaining) vectors of i1. | ||||||
2153 | if (ElemTy == MVT::i1) | ||||||
2154 | return TargetLoweringBase::TypeWidenVector; | ||||||
2155 | |||||||
2156 | return TargetLoweringBase::TypeSplitVector; | ||||||
2157 | } | ||||||
2158 | |||||||
2159 | std::pair<SDValue, int> | ||||||
2160 | HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const { | ||||||
2161 | if (Addr.getOpcode() == ISD::ADD) { | ||||||
2162 | SDValue Op1 = Addr.getOperand(1); | ||||||
2163 | if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode())) | ||||||
2164 | return { Addr.getOperand(0), CN->getSExtValue() }; | ||||||
2165 | } | ||||||
2166 | return { Addr, 0 }; | ||||||
2167 | } | ||||||
2168 | |||||||
2169 | // Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors | ||||||
2170 | // to select data from, V3 is the permutation. | ||||||
2171 | SDValue | ||||||
2172 | HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) | ||||||
2173 | const { | ||||||
2174 | const auto *SVN = cast<ShuffleVectorSDNode>(Op); | ||||||
2175 | ArrayRef<int> AM = SVN->getMask(); | ||||||
2176 | assert(AM.size() <= 8 && "Unexpected shuffle mask")(static_cast <bool> (AM.size() <= 8 && "Unexpected shuffle mask" ) ? void (0) : __assert_fail ("AM.size() <= 8 && \"Unexpected shuffle mask\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2176, __extension__ __PRETTY_FUNCTION__)); | ||||||
2177 | unsigned VecLen = AM.size(); | ||||||
2178 | |||||||
2179 | MVT VecTy = ty(Op); | ||||||
2180 | assert(!Subtarget.isHVXVectorType(VecTy, true) &&(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true ) && "HVX shuffles should be legal") ? void (0) : __assert_fail ("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2181, __extension__ __PRETTY_FUNCTION__)) | ||||||
2181 | "HVX shuffles should be legal")(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true ) && "HVX shuffles should be legal") ? void (0) : __assert_fail ("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2181, __extension__ __PRETTY_FUNCTION__)); | ||||||
2182 | assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length")(static_cast <bool> (VecTy.getSizeInBits() <= 64 && "Unexpected vector length") ? void (0) : __assert_fail ("VecTy.getSizeInBits() <= 64 && \"Unexpected vector length\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2182, __extension__ __PRETTY_FUNCTION__)); | ||||||
2183 | |||||||
2184 | SDValue Op0 = Op.getOperand(0); | ||||||
2185 | SDValue Op1 = Op.getOperand(1); | ||||||
2186 | const SDLoc &dl(Op); | ||||||
2187 | |||||||
2188 | // If the inputs are not the same as the output, bail. This is not an | ||||||
2189 | // error situation, but complicates the handling and the default expansion | ||||||
2190 | // (into BUILD_VECTOR) should be adequate. | ||||||
2191 | if (ty(Op0) != VecTy || ty(Op1) != VecTy) | ||||||
2192 | return SDValue(); | ||||||
2193 | |||||||
2194 | // Normalize the mask so that the first non-negative index comes from | ||||||
2195 | // the first operand. | ||||||
2196 | SmallVector<int,8> Mask(AM.begin(), AM.end()); | ||||||
2197 | unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data(); | ||||||
2198 | if (F == AM.size()) | ||||||
2199 | return DAG.getUNDEF(VecTy); | ||||||
2200 | if (AM[F] >= int(VecLen)) { | ||||||
2201 | ShuffleVectorSDNode::commuteMask(Mask); | ||||||
2202 | std::swap(Op0, Op1); | ||||||
2203 | } | ||||||
2204 | |||||||
2205 | // Express the shuffle mask in terms of bytes. | ||||||
2206 | SmallVector<int,8> ByteMask; | ||||||
2207 | unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8; | ||||||
2208 | for (unsigned i = 0, e = Mask.size(); i != e; ++i) { | ||||||
2209 | int M = Mask[i]; | ||||||
2210 | if (M < 0) { | ||||||
2211 | for (unsigned j = 0; j != ElemBytes; ++j) | ||||||
2212 | ByteMask.push_back(-1); | ||||||
2213 | } else { | ||||||
2214 | for (unsigned j = 0; j != ElemBytes; ++j) | ||||||
2215 | ByteMask.push_back(M*ElemBytes + j); | ||||||
2216 | } | ||||||
2217 | } | ||||||
2218 | assert(ByteMask.size() <= 8)(static_cast <bool> (ByteMask.size() <= 8) ? void (0 ) : __assert_fail ("ByteMask.size() <= 8", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2218, __extension__ __PRETTY_FUNCTION__)); | ||||||
2219 | |||||||
2220 | // All non-undef (non-negative) indexes are well within [0..127], so they | ||||||
2221 | // fit in a single byte. Build two 64-bit words: | ||||||
2222 | // - MaskIdx where each byte is the corresponding index (for non-negative | ||||||
2223 | // indexes), and 0xFF for negative indexes, and | ||||||
2224 | // - MaskUnd that has 0xFF for each negative index. | ||||||
2225 | uint64_t MaskIdx = 0; | ||||||
2226 | uint64_t MaskUnd = 0; | ||||||
2227 | for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) { | ||||||
2228 | unsigned S = 8*i; | ||||||
2229 | uint64_t M = ByteMask[i] & 0xFF; | ||||||
2230 | if (M == 0xFF) | ||||||
2231 | MaskUnd |= M << S; | ||||||
2232 | MaskIdx |= M << S; | ||||||
2233 | } | ||||||
2234 | |||||||
2235 | if (ByteMask.size() == 4) { | ||||||
2236 | // Identity. | ||||||
2237 | if (MaskIdx == (0x03020100 | MaskUnd)) | ||||||
2238 | return Op0; | ||||||
2239 | // Byte swap. | ||||||
2240 | if (MaskIdx == (0x00010203 | MaskUnd)) { | ||||||
2241 | SDValue T0 = DAG.getBitcast(MVT::i32, Op0); | ||||||
2242 | SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0); | ||||||
2243 | return DAG.getBitcast(VecTy, T1); | ||||||
2244 | } | ||||||
2245 | |||||||
2246 | // Byte packs. | ||||||
2247 | SDValue Concat10 = DAG.getNode(HexagonISD::COMBINE, dl, | ||||||
2248 | typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0}); | ||||||
2249 | if (MaskIdx == (0x06040200 | MaskUnd)) | ||||||
2250 | return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG); | ||||||
2251 | if (MaskIdx == (0x07050301 | MaskUnd)) | ||||||
2252 | return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG); | ||||||
2253 | |||||||
2254 | SDValue Concat01 = DAG.getNode(HexagonISD::COMBINE, dl, | ||||||
2255 | typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1}); | ||||||
2256 | if (MaskIdx == (0x02000604 | MaskUnd)) | ||||||
2257 | return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG); | ||||||
2258 | if (MaskIdx == (0x03010705 | MaskUnd)) | ||||||
2259 | return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG); | ||||||
2260 | } | ||||||
2261 | |||||||
2262 | if (ByteMask.size() == 8) { | ||||||
2263 | // Identity. | ||||||
2264 | if (MaskIdx == (0x0706050403020100ull | MaskUnd)) | ||||||
2265 | return Op0; | ||||||
2266 | // Byte swap. | ||||||
2267 | if (MaskIdx == (0x0001020304050607ull | MaskUnd)) { | ||||||
2268 | SDValue T0 = DAG.getBitcast(MVT::i64, Op0); | ||||||
2269 | SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0); | ||||||
2270 | return DAG.getBitcast(VecTy, T1); | ||||||
2271 | } | ||||||
2272 | |||||||
2273 | // Halfword picks. | ||||||
2274 | if (MaskIdx == (0x0d0c050409080100ull | MaskUnd)) | ||||||
2275 | return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG); | ||||||
2276 | if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd)) | ||||||
2277 | return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG); | ||||||
2278 | if (MaskIdx == (0x0d0c090805040100ull | MaskUnd)) | ||||||
2279 | return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG); | ||||||
2280 | if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd)) | ||||||
2281 | return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG); | ||||||
2282 | if (MaskIdx == (0x0706030205040100ull | MaskUnd)) { | ||||||
2283 | VectorPair P = opSplit(Op0, dl, DAG); | ||||||
2284 | return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG); | ||||||
2285 | } | ||||||
2286 | |||||||
2287 | // Byte packs. | ||||||
2288 | if (MaskIdx == (0x0e060c040a020800ull | MaskUnd)) | ||||||
2289 | return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG); | ||||||
2290 | if (MaskIdx == (0x0f070d050b030901ull | MaskUnd)) | ||||||
2291 | return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG); | ||||||
2292 | } | ||||||
2293 | |||||||
2294 | return SDValue(); | ||||||
2295 | } | ||||||
2296 | |||||||
2297 | // Create a Hexagon-specific node for shifting a vector by an integer. | ||||||
2298 | SDValue | ||||||
2299 | HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) | ||||||
2300 | const { | ||||||
2301 | unsigned NewOpc; | ||||||
2302 | switch (Op.getOpcode()) { | ||||||
2303 | case ISD::SHL: | ||||||
2304 | NewOpc = HexagonISD::VASL; | ||||||
2305 | break; | ||||||
2306 | case ISD::SRA: | ||||||
2307 | NewOpc = HexagonISD::VASR; | ||||||
2308 | break; | ||||||
2309 | case ISD::SRL: | ||||||
2310 | NewOpc = HexagonISD::VLSR; | ||||||
2311 | break; | ||||||
2312 | default: | ||||||
2313 | llvm_unreachable("Unexpected shift opcode")::llvm::llvm_unreachable_internal("Unexpected shift opcode", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2313); | ||||||
2314 | } | ||||||
2315 | |||||||
2316 | SDValue Op0 = Op.getOperand(0); | ||||||
2317 | SDValue Op1 = Op.getOperand(1); | ||||||
2318 | const SDLoc &dl(Op); | ||||||
2319 | |||||||
2320 | switch (Op1.getOpcode()) { | ||||||
2321 | case ISD::BUILD_VECTOR: | ||||||
2322 | if (SDValue S = cast<BuildVectorSDNode>(Op1)->getSplatValue()) | ||||||
2323 | return DAG.getNode(NewOpc, dl, ty(Op), Op0, S); | ||||||
2324 | break; | ||||||
2325 | case ISD::SPLAT_VECTOR: | ||||||
2326 | return DAG.getNode(NewOpc, dl, ty(Op), Op0, Op1.getOperand(0)); | ||||||
2327 | } | ||||||
2328 | return SDValue(); | ||||||
2329 | } | ||||||
2330 | |||||||
2331 | SDValue | ||||||
2332 | HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { | ||||||
2333 | return getVectorShiftByInt(Op, DAG); | ||||||
2334 | } | ||||||
2335 | |||||||
2336 | SDValue | ||||||
2337 | HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const { | ||||||
2338 | if (isa<ConstantSDNode>(Op.getOperand(1).getNode())) | ||||||
2339 | return Op; | ||||||
2340 | return SDValue(); | ||||||
2341 | } | ||||||
2342 | |||||||
2343 | SDValue | ||||||
2344 | HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { | ||||||
2345 | MVT ResTy = ty(Op); | ||||||
2346 | SDValue InpV = Op.getOperand(0); | ||||||
2347 | MVT InpTy = ty(InpV); | ||||||
2348 | assert(ResTy.getSizeInBits() == InpTy.getSizeInBits())(static_cast <bool> (ResTy.getSizeInBits() == InpTy.getSizeInBits ()) ? void (0) : __assert_fail ("ResTy.getSizeInBits() == InpTy.getSizeInBits()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2348, __extension__ __PRETTY_FUNCTION__)); | ||||||
2349 | const SDLoc &dl(Op); | ||||||
2350 | |||||||
2351 | // Handle conversion from i8 to v8i1. | ||||||
2352 | if (InpTy == MVT::i8) { | ||||||
2353 | if (ResTy == MVT::v8i1) { | ||||||
2354 | SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV); | ||||||
2355 | SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32); | ||||||
2356 | return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG); | ||||||
2357 | } | ||||||
2358 | return SDValue(); | ||||||
2359 | } | ||||||
2360 | |||||||
2361 | return Op; | ||||||
2362 | } | ||||||
2363 | |||||||
2364 | bool | ||||||
2365 | HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values, | ||||||
2366 | MVT VecTy, SelectionDAG &DAG, | ||||||
2367 | MutableArrayRef<ConstantInt*> Consts) const { | ||||||
2368 | MVT ElemTy = VecTy.getVectorElementType(); | ||||||
2369 | unsigned ElemWidth = ElemTy.getSizeInBits(); | ||||||
2370 | IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth); | ||||||
2371 | bool AllConst = true; | ||||||
2372 | |||||||
2373 | for (unsigned i = 0, e = Values.size(); i != e; ++i) { | ||||||
2374 | SDValue V = Values[i]; | ||||||
2375 | if (V.isUndef()) { | ||||||
2376 | Consts[i] = ConstantInt::get(IntTy, 0); | ||||||
2377 | continue; | ||||||
2378 | } | ||||||
2379 | // Make sure to always cast to IntTy. | ||||||
2380 | if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) { | ||||||
2381 | const ConstantInt *CI = CN->getConstantIntValue(); | ||||||
2382 | Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue()); | ||||||
2383 | } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) { | ||||||
2384 | const ConstantFP *CF = CN->getConstantFPValue(); | ||||||
2385 | APInt A = CF->getValueAPF().bitcastToAPInt(); | ||||||
2386 | Consts[i] = ConstantInt::get(IntTy, A.getZExtValue()); | ||||||
2387 | } else { | ||||||
2388 | AllConst = false; | ||||||
2389 | } | ||||||
2390 | } | ||||||
2391 | return AllConst; | ||||||
2392 | } | ||||||
2393 | |||||||
2394 | SDValue | ||||||
2395 | HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, | ||||||
2396 | MVT VecTy, SelectionDAG &DAG) const { | ||||||
2397 | MVT ElemTy = VecTy.getVectorElementType(); | ||||||
2398 | assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem .size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2398, __extension__ __PRETTY_FUNCTION__)); | ||||||
2399 | |||||||
2400 | SmallVector<ConstantInt*,4> Consts(Elem.size()); | ||||||
2401 | bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); | ||||||
2402 | |||||||
2403 | unsigned First, Num = Elem.size(); | ||||||
2404 | for (First = 0; First != Num; ++First) { | ||||||
2405 | if (!isUndef(Elem[First])) | ||||||
2406 | break; | ||||||
2407 | } | ||||||
2408 | if (First == Num) | ||||||
2409 | return DAG.getUNDEF(VecTy); | ||||||
2410 | |||||||
2411 | if (AllConst && | ||||||
2412 | llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); })) | ||||||
2413 | return getZero(dl, VecTy, DAG); | ||||||
2414 | |||||||
2415 | if (ElemTy == MVT::i16) { | ||||||
2416 | assert(Elem.size() == 2)(static_cast <bool> (Elem.size() == 2) ? void (0) : __assert_fail ("Elem.size() == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2416, __extension__ __PRETTY_FUNCTION__)); | ||||||
2417 | if (AllConst) { | ||||||
2418 | uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) | | ||||||
2419 | Consts[1]->getZExtValue() << 16; | ||||||
2420 | return DAG.getBitcast(MVT::v2i16, DAG.getConstant(V, dl, MVT::i32)); | ||||||
2421 | } | ||||||
2422 | SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, | ||||||
2423 | {Elem[1], Elem[0]}, DAG); | ||||||
2424 | return DAG.getBitcast(MVT::v2i16, N); | ||||||
2425 | } | ||||||
2426 | |||||||
2427 | if (ElemTy == MVT::i8) { | ||||||
2428 | // First try generating a constant. | ||||||
2429 | if (AllConst) { | ||||||
2430 | int32_t V = (Consts[0]->getZExtValue() & 0xFF) | | ||||||
2431 | (Consts[1]->getZExtValue() & 0xFF) << 8 | | ||||||
2432 | (Consts[1]->getZExtValue() & 0xFF) << 16 | | ||||||
2433 | Consts[2]->getZExtValue() << 24; | ||||||
2434 | return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32)); | ||||||
2435 | } | ||||||
2436 | |||||||
2437 | // Then try splat. | ||||||
2438 | bool IsSplat = true; | ||||||
2439 | for (unsigned i = First+1; i != Num; ++i) { | ||||||
2440 | if (Elem[i] == Elem[First] || isUndef(Elem[i])) | ||||||
2441 | continue; | ||||||
2442 | IsSplat = false; | ||||||
2443 | break; | ||||||
2444 | } | ||||||
2445 | if (IsSplat) { | ||||||
2446 | // Legalize the operand of SPLAT_VECTOR. | ||||||
2447 | SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); | ||||||
2448 | return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext); | ||||||
2449 | } | ||||||
2450 | |||||||
2451 | // Generate | ||||||
2452 | // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) | | ||||||
2453 | // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16 | ||||||
2454 | assert(Elem.size() == 4)(static_cast <bool> (Elem.size() == 4) ? void (0) : __assert_fail ("Elem.size() == 4", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2454, __extension__ __PRETTY_FUNCTION__)); | ||||||
2455 | SDValue Vs[4]; | ||||||
2456 | for (unsigned i = 0; i != 4; ++i) { | ||||||
2457 | Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32); | ||||||
2458 | Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8); | ||||||
2459 | } | ||||||
2460 | SDValue S8 = DAG.getConstant(8, dl, MVT::i32); | ||||||
2461 | SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8}); | ||||||
2462 | SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8}); | ||||||
2463 | SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0}); | ||||||
2464 | SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1}); | ||||||
2465 | |||||||
2466 | SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG); | ||||||
2467 | return DAG.getBitcast(MVT::v4i8, R); | ||||||
2468 | } | ||||||
2469 | |||||||
2470 | #ifndef NDEBUG | ||||||
2471 | dbgs() << "VecTy: " << EVT(VecTy).getEVTString() << '\n'; | ||||||
2472 | #endif | ||||||
2473 | llvm_unreachable("Unexpected vector element type")::llvm::llvm_unreachable_internal("Unexpected vector element type" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2473); | ||||||
2474 | } | ||||||
2475 | |||||||
2476 | SDValue | ||||||
2477 | HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, | ||||||
2478 | MVT VecTy, SelectionDAG &DAG) const { | ||||||
2479 | MVT ElemTy = VecTy.getVectorElementType(); | ||||||
2480 | assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem .size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2480, __extension__ __PRETTY_FUNCTION__)); | ||||||
2481 | |||||||
2482 | SmallVector<ConstantInt*,8> Consts(Elem.size()); | ||||||
2483 | bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); | ||||||
2484 | |||||||
2485 | unsigned First, Num = Elem.size(); | ||||||
2486 | for (First = 0; First != Num; ++First) { | ||||||
2487 | if (!isUndef(Elem[First])) | ||||||
2488 | break; | ||||||
2489 | } | ||||||
2490 | if (First == Num) | ||||||
2491 | return DAG.getUNDEF(VecTy); | ||||||
2492 | |||||||
2493 | if (AllConst && | ||||||
2494 | llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); })) | ||||||
2495 | return getZero(dl, VecTy, DAG); | ||||||
2496 | |||||||
2497 | // First try splat if possible. | ||||||
2498 | if (ElemTy == MVT::i16) { | ||||||
2499 | bool IsSplat = true; | ||||||
2500 | for (unsigned i = First+1; i != Num; ++i) { | ||||||
2501 | if (Elem[i] == Elem[First] || isUndef(Elem[i])) | ||||||
2502 | continue; | ||||||
2503 | IsSplat = false; | ||||||
2504 | break; | ||||||
2505 | } | ||||||
2506 | if (IsSplat) { | ||||||
2507 | // Legalize the operand of SPLAT_VECTOR | ||||||
2508 | SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); | ||||||
2509 | return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext); | ||||||
2510 | } | ||||||
2511 | } | ||||||
2512 | |||||||
2513 | // Then try constant. | ||||||
2514 | if (AllConst) { | ||||||
2515 | uint64_t Val = 0; | ||||||
2516 | unsigned W = ElemTy.getSizeInBits(); | ||||||
2517 | uint64_t Mask = (ElemTy == MVT::i8) ? 0xFFull | ||||||
2518 | : (ElemTy == MVT::i16) ? 0xFFFFull : 0xFFFFFFFFull; | ||||||
2519 | for (unsigned i = 0; i != Num; ++i) | ||||||
2520 | Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask); | ||||||
2521 | SDValue V0 = DAG.getConstant(Val, dl, MVT::i64); | ||||||
2522 | return DAG.getBitcast(VecTy, V0); | ||||||
2523 | } | ||||||
2524 | |||||||
2525 | // Build two 32-bit vectors and concatenate. | ||||||
2526 | MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2); | ||||||
2527 | SDValue L = (ElemTy == MVT::i32) | ||||||
2528 | ? Elem[0] | ||||||
2529 | : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG); | ||||||
2530 | SDValue H = (ElemTy == MVT::i32) | ||||||
2531 | ? Elem[1] | ||||||
2532 | : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG); | ||||||
2533 | return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, {H, L}); | ||||||
2534 | } | ||||||
2535 | |||||||
2536 | SDValue | ||||||
2537 | HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV, | ||||||
2538 | const SDLoc &dl, MVT ValTy, MVT ResTy, | ||||||
2539 | SelectionDAG &DAG) const { | ||||||
2540 | MVT VecTy = ty(VecV); | ||||||
2541 | assert(!ValTy.isVector() ||(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType () == ValTy.getVectorElementType()) ? void (0) : __assert_fail ("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2542, __extension__ __PRETTY_FUNCTION__)) | ||||||
2542 | VecTy.getVectorElementType() == ValTy.getVectorElementType())(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType () == ValTy.getVectorElementType()) ? void (0) : __assert_fail ("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2542, __extension__ __PRETTY_FUNCTION__)); | ||||||
2543 | unsigned VecWidth = VecTy.getSizeInBits(); | ||||||
2544 | unsigned ValWidth = ValTy.getSizeInBits(); | ||||||
2545 | unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits(); | ||||||
2546 | assert((VecWidth % ElemWidth) == 0)(static_cast <bool> ((VecWidth % ElemWidth) == 0) ? void (0) : __assert_fail ("(VecWidth % ElemWidth) == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2546, __extension__ __PRETTY_FUNCTION__)); | ||||||
2547 | auto *IdxN = dyn_cast<ConstantSDNode>(IdxV); | ||||||
2548 | |||||||
2549 | // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon | ||||||
2550 | // without any coprocessors). | ||||||
2551 | if (ElemWidth == 1) { | ||||||
2552 | assert(VecWidth == VecTy.getVectorNumElements() && "Sanity failure")(static_cast <bool> (VecWidth == VecTy.getVectorNumElements () && "Sanity failure") ? void (0) : __assert_fail ("VecWidth == VecTy.getVectorNumElements() && \"Sanity failure\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2552, __extension__ __PRETTY_FUNCTION__)); | ||||||
2553 | assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2)(static_cast <bool> (VecWidth == 8 || VecWidth == 4 || VecWidth == 2) ? void (0) : __assert_fail ("VecWidth == 8 || VecWidth == 4 || VecWidth == 2" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2553, __extension__ __PRETTY_FUNCTION__)); | ||||||
2554 | // Check if this is an extract of the lowest bit. | ||||||
2555 | if (IdxN) { | ||||||
2556 | // Extracting the lowest bit is a no-op, but it changes the type, | ||||||
2557 | // so it must be kept as an operation to avoid errors related to | ||||||
2558 | // type mismatches. | ||||||
2559 | if (IdxN->isNullValue() && ValTy.getSizeInBits() == 1) | ||||||
2560 | return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV); | ||||||
2561 | } | ||||||
2562 | |||||||
2563 | // If the value extracted is a single bit, use tstbit. | ||||||
2564 | if (ValWidth == 1) { | ||||||
2565 | SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG); | ||||||
2566 | SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32); | ||||||
2567 | SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0); | ||||||
2568 | return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0); | ||||||
2569 | } | ||||||
2570 | |||||||
2571 | // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in | ||||||
2572 | // a predicate register. The elements of the vector are repeated | ||||||
2573 | // in the register (if necessary) so that the total number is 8. | ||||||
2574 | // The extracted subvector will need to be expanded in such a way. | ||||||
2575 | unsigned Scale = VecWidth / ValWidth; | ||||||
2576 | |||||||
2577 | // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to | ||||||
2578 | // position 0. | ||||||
2579 | assert(ty(IdxV) == MVT::i32)(static_cast <bool> (ty(IdxV) == MVT::i32) ? void (0) : __assert_fail ("ty(IdxV) == MVT::i32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2579, __extension__ __PRETTY_FUNCTION__)); | ||||||
2580 | unsigned VecRep = 8 / VecWidth; | ||||||
2581 | SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | ||||||
2582 | DAG.getConstant(8*VecRep, dl, MVT::i32)); | ||||||
2583 | SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV); | ||||||
2584 | SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0); | ||||||
2585 | while (Scale > 1) { | ||||||
2586 | // The longest possible subvector is at most 32 bits, so it is always | ||||||
2587 | // contained in the low subregister. | ||||||
2588 | T1 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, T1); | ||||||
2589 | T1 = expandPredicate(T1, dl, DAG); | ||||||
2590 | Scale /= 2; | ||||||
2591 | } | ||||||
2592 | |||||||
2593 | return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1); | ||||||
2594 | } | ||||||
2595 | |||||||
2596 | assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ? void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2596, __extension__ __PRETTY_FUNCTION__)); | ||||||
2597 | |||||||
2598 | // Cast everything to scalar integer types. | ||||||
2599 | MVT ScalarTy = tyScalar(VecTy); | ||||||
2600 | VecV = DAG.getBitcast(ScalarTy, VecV); | ||||||
2601 | |||||||
2602 | SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32); | ||||||
2603 | SDValue ExtV; | ||||||
2604 | |||||||
2605 | if (IdxN) { | ||||||
2606 | unsigned Off = IdxN->getZExtValue() * ElemWidth; | ||||||
2607 | if (VecWidth == 64 && ValWidth == 32) { | ||||||
2608 | assert(Off == 0 || Off == 32)(static_cast <bool> (Off == 0 || Off == 32) ? void (0) : __assert_fail ("Off == 0 || Off == 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2608, __extension__ __PRETTY_FUNCTION__)); | ||||||
2609 | unsigned SubIdx = Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi; | ||||||
2610 | ExtV = DAG.getTargetExtractSubreg(SubIdx, dl, MVT::i32, VecV); | ||||||
2611 | } else if (Off == 0 && (ValWidth % 8) == 0) { | ||||||
2612 | ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy)); | ||||||
2613 | } else { | ||||||
2614 | SDValue OffV = DAG.getConstant(Off, dl, MVT::i32); | ||||||
2615 | // The return type of EXTRACTU must be the same as the type of the | ||||||
2616 | // input vector. | ||||||
2617 | ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy, | ||||||
2618 | {VecV, WidthV, OffV}); | ||||||
2619 | } | ||||||
2620 | } else { | ||||||
2621 | if (ty(IdxV) != MVT::i32) | ||||||
2622 | IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32); | ||||||
2623 | SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | ||||||
2624 | DAG.getConstant(ElemWidth, dl, MVT::i32)); | ||||||
2625 | ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy, | ||||||
2626 | {VecV, WidthV, OffV}); | ||||||
2627 | } | ||||||
2628 | |||||||
2629 | // Cast ExtV to the requested result type. | ||||||
2630 | ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy)); | ||||||
2631 | ExtV = DAG.getBitcast(ResTy, ExtV); | ||||||
2632 | return ExtV; | ||||||
2633 | } | ||||||
2634 | |||||||
2635 | SDValue | ||||||
2636 | HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV, | ||||||
2637 | const SDLoc &dl, MVT ValTy, | ||||||
2638 | SelectionDAG &DAG) const { | ||||||
2639 | MVT VecTy = ty(VecV); | ||||||
2640 | if (VecTy.getVectorElementType() == MVT::i1) { | ||||||
2641 | MVT ValTy = ty(ValV); | ||||||
2642 | assert(ValTy.getVectorElementType() == MVT::i1)(static_cast <bool> (ValTy.getVectorElementType() == MVT ::i1) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == MVT::i1" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2642, __extension__ __PRETTY_FUNCTION__)); | ||||||
2643 | SDValue ValR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV); | ||||||
2644 | unsigned VecLen = VecTy.getVectorNumElements(); | ||||||
2645 | unsigned Scale = VecLen / ValTy.getVectorNumElements(); | ||||||
2646 | assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail ("Scale > 1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2646, __extension__ __PRETTY_FUNCTION__)); | ||||||
2647 | |||||||
2648 | for (unsigned R = Scale; R > 1; R /= 2) { | ||||||
2649 | ValR = contractPredicate(ValR, dl, DAG); | ||||||
2650 | ValR = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | ||||||
2651 | DAG.getUNDEF(MVT::i32), ValR); | ||||||
2652 | } | ||||||
2653 | // The longest possible subvector is at most 32 bits, so it is always | ||||||
2654 | // contained in the low subregister. | ||||||
2655 | ValR = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, ValR); | ||||||
2656 | |||||||
2657 | unsigned ValBytes = 64 / Scale; | ||||||
2658 | SDValue Width = DAG.getConstant(ValBytes*8, dl, MVT::i32); | ||||||
2659 | SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | ||||||
2660 | DAG.getConstant(8, dl, MVT::i32)); | ||||||
2661 | SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV); | ||||||
2662 | SDValue Ins = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, | ||||||
2663 | {VecR, ValR, Width, Idx}); | ||||||
2664 | return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins); | ||||||
2665 | } | ||||||
2666 | |||||||
2667 | unsigned VecWidth = VecTy.getSizeInBits(); | ||||||
2668 | unsigned ValWidth = ValTy.getSizeInBits(); | ||||||
2669 | assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ? void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2669, __extension__ __PRETTY_FUNCTION__)); | ||||||
2670 | assert((VecWidth % ValWidth) == 0)(static_cast <bool> ((VecWidth % ValWidth) == 0) ? void (0) : __assert_fail ("(VecWidth % ValWidth) == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2670, __extension__ __PRETTY_FUNCTION__)); | ||||||
2671 | |||||||
2672 | // Cast everything to scalar integer types. | ||||||
2673 | MVT ScalarTy = MVT::getIntegerVT(VecWidth); | ||||||
2674 | // The actual type of ValV may be different than ValTy (which is related | ||||||
2675 | // to the vector type). | ||||||
2676 | unsigned VW = ty(ValV).getSizeInBits(); | ||||||
2677 | ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV); | ||||||
2678 | VecV = DAG.getBitcast(ScalarTy, VecV); | ||||||
2679 | if (VW
| ||||||
2680 | ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy); | ||||||
2681 | |||||||
2682 | SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32); | ||||||
2683 | SDValue InsV; | ||||||
2684 | |||||||
2685 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) { | ||||||
2686 | unsigned W = C->getZExtValue() * ValWidth; | ||||||
2687 | SDValue OffV = DAG.getConstant(W, dl, MVT::i32); | ||||||
2688 | InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy, | ||||||
2689 | {VecV, ValV, WidthV, OffV}); | ||||||
2690 | } else { | ||||||
2691 | if (ty(IdxV) != MVT::i32) | ||||||
2692 | IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32); | ||||||
2693 | SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV); | ||||||
2694 | InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy, | ||||||
2695 | {VecV, ValV, WidthV, OffV}); | ||||||
2696 | } | ||||||
2697 | |||||||
2698 | return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV); | ||||||
2699 | } | ||||||
2700 | |||||||
2701 | SDValue | ||||||
2702 | HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl, | ||||||
2703 | SelectionDAG &DAG) const { | ||||||
2704 | assert(ty(Vec32).getSizeInBits() == 32)(static_cast <bool> (ty(Vec32).getSizeInBits() == 32) ? void (0) : __assert_fail ("ty(Vec32).getSizeInBits() == 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2704, __extension__ __PRETTY_FUNCTION__)); | ||||||
2705 | if (isUndef(Vec32)) | ||||||
2706 | return DAG.getUNDEF(MVT::i64); | ||||||
2707 | return getInstr(Hexagon::S2_vsxtbh, dl, MVT::i64, {Vec32}, DAG); | ||||||
2708 | } | ||||||
2709 | |||||||
2710 | SDValue | ||||||
2711 | HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl, | ||||||
2712 | SelectionDAG &DAG) const { | ||||||
2713 | assert(ty(Vec64).getSizeInBits() == 64)(static_cast <bool> (ty(Vec64).getSizeInBits() == 64) ? void (0) : __assert_fail ("ty(Vec64).getSizeInBits() == 64", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2713, __extension__ __PRETTY_FUNCTION__)); | ||||||
2714 | if (isUndef(Vec64)) | ||||||
2715 | return DAG.getUNDEF(MVT::i32); | ||||||
2716 | return getInstr(Hexagon::S2_vtrunehb, dl, MVT::i32, {Vec64}, DAG); | ||||||
2717 | } | ||||||
2718 | |||||||
2719 | SDValue | ||||||
2720 | HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) | ||||||
2721 | const { | ||||||
2722 | if (Ty.isVector()) { | ||||||
2723 | assert(Ty.isInteger() && "Only integer vectors are supported here")(static_cast <bool> (Ty.isInteger() && "Only integer vectors are supported here" ) ? void (0) : __assert_fail ("Ty.isInteger() && \"Only integer vectors are supported here\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2723, __extension__ __PRETTY_FUNCTION__)); | ||||||
2724 | unsigned W = Ty.getSizeInBits(); | ||||||
2725 | if (W <= 64) | ||||||
2726 | return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W))); | ||||||
2727 | return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG)); | ||||||
2728 | } | ||||||
2729 | |||||||
2730 | if (Ty.isInteger()) | ||||||
2731 | return DAG.getConstant(0, dl, Ty); | ||||||
2732 | if (Ty.isFloatingPoint()) | ||||||
2733 | return DAG.getConstantFP(0.0, dl, Ty); | ||||||
2734 | llvm_unreachable("Invalid type for zero")::llvm::llvm_unreachable_internal("Invalid type for zero", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2734); | ||||||
2735 | } | ||||||
2736 | |||||||
2737 | SDValue | ||||||
2738 | HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG) | ||||||
2739 | const { | ||||||
2740 | MVT ValTy = ty(Val); | ||||||
2741 | assert(ValTy.getVectorElementType() == ResTy.getVectorElementType())(static_cast <bool> (ValTy.getVectorElementType() == ResTy .getVectorElementType()) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == ResTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2741, __extension__ __PRETTY_FUNCTION__)); | ||||||
2742 | |||||||
2743 | unsigned ValLen = ValTy.getVectorNumElements(); | ||||||
2744 | unsigned ResLen = ResTy.getVectorNumElements(); | ||||||
2745 | if (ValLen == ResLen) | ||||||
2746 | return Val; | ||||||
2747 | |||||||
2748 | const SDLoc &dl(Val); | ||||||
2749 | assert(ValLen < ResLen)(static_cast <bool> (ValLen < ResLen) ? void (0) : __assert_fail ("ValLen < ResLen", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2749, __extension__ __PRETTY_FUNCTION__)); | ||||||
2750 | assert(ResLen % ValLen == 0)(static_cast <bool> (ResLen % ValLen == 0) ? void (0) : __assert_fail ("ResLen % ValLen == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2750, __extension__ __PRETTY_FUNCTION__)); | ||||||
2751 | |||||||
2752 | SmallVector<SDValue, 4> Concats = {Val}; | ||||||
2753 | for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i) | ||||||
2754 | Concats.push_back(DAG.getUNDEF(ValTy)); | ||||||
2755 | |||||||
2756 | return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats); | ||||||
2757 | } | ||||||
2758 | |||||||
2759 | SDValue | ||||||
2760 | HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { | ||||||
2761 | MVT VecTy = ty(Op); | ||||||
2762 | unsigned BW = VecTy.getSizeInBits(); | ||||||
2763 | const SDLoc &dl(Op); | ||||||
2764 | SmallVector<SDValue,8> Ops; | ||||||
2765 | for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) | ||||||
2766 | Ops.push_back(Op.getOperand(i)); | ||||||
2767 | |||||||
2768 | if (BW == 32) | ||||||
2769 | return buildVector32(Ops, dl, VecTy, DAG); | ||||||
2770 | if (BW == 64) | ||||||
2771 | return buildVector64(Ops, dl, VecTy, DAG); | ||||||
2772 | |||||||
2773 | if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) { | ||||||
2774 | // Check if this is a special case or all-0 or all-1. | ||||||
2775 | bool All0 = true, All1 = true; | ||||||
2776 | for (SDValue P : Ops) { | ||||||
2777 | auto *CN = dyn_cast<ConstantSDNode>(P.getNode()); | ||||||
2778 | if (CN == nullptr) { | ||||||
2779 | All0 = All1 = false; | ||||||
2780 | break; | ||||||
2781 | } | ||||||
2782 | uint32_t C = CN->getZExtValue(); | ||||||
2783 | All0 &= (C == 0); | ||||||
2784 | All1 &= (C == 1); | ||||||
2785 | } | ||||||
2786 | if (All0) | ||||||
2787 | return DAG.getNode(HexagonISD::PFALSE, dl, VecTy); | ||||||
2788 | if (All1) | ||||||
2789 | return DAG.getNode(HexagonISD::PTRUE, dl, VecTy); | ||||||
2790 | |||||||
2791 | // For each i1 element in the resulting predicate register, put 1 | ||||||
2792 | // shifted by the index of the element into a general-purpose register, | ||||||
2793 | // then or them together and transfer it back into a predicate register. | ||||||
2794 | SDValue Rs[8]; | ||||||
2795 | SDValue Z = getZero(dl, MVT::i32, DAG); | ||||||
2796 | // Always produce 8 bits, repeat inputs if necessary. | ||||||
2797 | unsigned Rep = 8 / VecTy.getVectorNumElements(); | ||||||
2798 | for (unsigned i = 0; i != 8; ++i) { | ||||||
2799 | SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32); | ||||||
2800 | Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z); | ||||||
2801 | } | ||||||
2802 | for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) { | ||||||
2803 | for (unsigned i = 0, e = A.size()/2; i != e; ++i) | ||||||
2804 | Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]); | ||||||
2805 | } | ||||||
2806 | // Move the value directly to a predicate register. | ||||||
2807 | return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG); | ||||||
2808 | } | ||||||
2809 | |||||||
2810 | return SDValue(); | ||||||
2811 | } | ||||||
2812 | |||||||
2813 | SDValue | ||||||
2814 | HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op, | ||||||
2815 | SelectionDAG &DAG) const { | ||||||
2816 | MVT VecTy = ty(Op); | ||||||
2817 | const SDLoc &dl(Op); | ||||||
2818 | if (VecTy.getSizeInBits() == 64) { | ||||||
2819 | assert(Op.getNumOperands() == 2)(static_cast <bool> (Op.getNumOperands() == 2) ? void ( 0) : __assert_fail ("Op.getNumOperands() == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2819, __extension__ __PRETTY_FUNCTION__)); | ||||||
2820 | return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, Op.getOperand(1), | ||||||
2821 | Op.getOperand(0)); | ||||||
2822 | } | ||||||
2823 | |||||||
2824 | MVT ElemTy = VecTy.getVectorElementType(); | ||||||
2825 | if (ElemTy == MVT::i1) { | ||||||
2826 | assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1)(static_cast <bool> (VecTy == MVT::v2i1 || VecTy == MVT ::v4i1 || VecTy == MVT::v8i1) ? void (0) : __assert_fail ("VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2826, __extension__ __PRETTY_FUNCTION__)); | ||||||
2827 | MVT OpTy = ty(Op.getOperand(0)); | ||||||
2828 | // Scale is how many times the operands need to be contracted to match | ||||||
2829 | // the representation in the target register. | ||||||
2830 | unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements(); | ||||||
2831 | assert(Scale == Op.getNumOperands() && Scale > 1)(static_cast <bool> (Scale == Op.getNumOperands() && Scale > 1) ? void (0) : __assert_fail ("Scale == Op.getNumOperands() && Scale > 1" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2831, __extension__ __PRETTY_FUNCTION__)); | ||||||
2832 | |||||||
2833 | // First, convert all bool vectors to integers, then generate pairwise | ||||||
2834 | // inserts to form values of doubled length. Up until there are only | ||||||
2835 | // two values left to concatenate, all of these values will fit in a | ||||||
2836 | // 32-bit integer, so keep them as i32 to use 32-bit inserts. | ||||||
2837 | SmallVector<SDValue,4> Words[2]; | ||||||
2838 | unsigned IdxW = 0; | ||||||
2839 | |||||||
2840 | for (SDValue P : Op.getNode()->op_values()) { | ||||||
2841 | SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P); | ||||||
2842 | for (unsigned R = Scale; R > 1; R /= 2) { | ||||||
2843 | W = contractPredicate(W, dl, DAG); | ||||||
2844 | W = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | ||||||
2845 | DAG.getUNDEF(MVT::i32), W); | ||||||
2846 | } | ||||||
2847 | W = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, W); | ||||||
2848 | Words[IdxW].push_back(W); | ||||||
2849 | } | ||||||
2850 | |||||||
2851 | while (Scale > 2) { | ||||||
2852 | SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32); | ||||||
2853 | Words[IdxW ^ 1].clear(); | ||||||
2854 | |||||||
2855 | for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) { | ||||||
2856 | SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1]; | ||||||
2857 | // Insert W1 into W0 right next to the significant bits of W0. | ||||||
2858 | SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, | ||||||
2859 | {W0, W1, WidthV, WidthV}); | ||||||
2860 | Words[IdxW ^ 1].push_back(T); | ||||||
2861 | } | ||||||
2862 | IdxW ^= 1; | ||||||
2863 | Scale /= 2; | ||||||
2864 | } | ||||||
2865 | |||||||
2866 | // Another sanity check. At this point there should only be two words | ||||||
2867 | // left, and Scale should be 2. | ||||||
2868 | assert(Scale == 2 && Words[IdxW].size() == 2)(static_cast <bool> (Scale == 2 && Words[IdxW]. size() == 2) ? void (0) : __assert_fail ("Scale == 2 && Words[IdxW].size() == 2" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2868, __extension__ __PRETTY_FUNCTION__)); | ||||||
2869 | |||||||
2870 | SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | ||||||
2871 | Words[IdxW][1], Words[IdxW][0]); | ||||||
2872 | return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW); | ||||||
2873 | } | ||||||
2874 | |||||||
2875 | return SDValue(); | ||||||
2876 | } | ||||||
2877 | |||||||
2878 | SDValue | ||||||
2879 | HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, | ||||||
2880 | SelectionDAG &DAG) const { | ||||||
2881 | SDValue Vec = Op.getOperand(0); | ||||||
2882 | MVT ElemTy = ty(Vec).getVectorElementType(); | ||||||
2883 | return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG); | ||||||
2884 | } | ||||||
2885 | |||||||
2886 | SDValue | ||||||
2887 | HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, | ||||||
2888 | SelectionDAG &DAG) const { | ||||||
2889 | return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op), | ||||||
2890 | ty(Op), ty(Op), DAG); | ||||||
2891 | } | ||||||
2892 | |||||||
2893 | SDValue | ||||||
2894 | HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, | ||||||
2895 | SelectionDAG &DAG) const { | ||||||
2896 | return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2), | ||||||
2897 | SDLoc(Op), ty(Op).getVectorElementType(), DAG); | ||||||
2898 | } | ||||||
2899 | |||||||
2900 | SDValue | ||||||
2901 | HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, | ||||||
2902 | SelectionDAG &DAG) const { | ||||||
2903 | SDValue ValV = Op.getOperand(1); | ||||||
2904 | return insertVector(Op.getOperand(0), ValV, Op.getOperand(2), | ||||||
2905 | SDLoc(Op), ty(ValV), DAG); | ||||||
2906 | } | ||||||
2907 | |||||||
2908 | bool | ||||||
2909 | HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const { | ||||||
2910 | // Assuming the caller does not have either a signext or zeroext modifier, and | ||||||
2911 | // only one value is accepted, any reasonable truncation is allowed. | ||||||
2912 | if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) | ||||||
2913 | return false; | ||||||
2914 | |||||||
2915 | // FIXME: in principle up to 64-bit could be made safe, but it would be very | ||||||
2916 | // fragile at the moment: any support for multiple value returns would be | ||||||
2917 | // liable to disallow tail calls involving i64 -> iN truncation in many cases. | ||||||
2918 | return Ty1->getPrimitiveSizeInBits() <= 32; | ||||||
2919 | } | ||||||
2920 | |||||||
2921 | SDValue | ||||||
2922 | HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const { | ||||||
2923 | MVT Ty = ty(Op); | ||||||
2924 | const SDLoc &dl(Op); | ||||||
2925 | // Lower loads of scalar predicate vectors (v2i1, v4i1, v8i1) to loads of i1 | ||||||
2926 | // followed by a TYPECAST. | ||||||
2927 | LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); | ||||||
2928 | bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1); | ||||||
2929 | if (DoCast) { | ||||||
2930 | SDValue NL = DAG.getLoad( | ||||||
2931 | LN->getAddressingMode(), LN->getExtensionType(), MVT::i1, dl, | ||||||
2932 | LN->getChain(), LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(), | ||||||
2933 | /*MemoryVT*/ MVT::i1, LN->getAlign(), LN->getMemOperand()->getFlags(), | ||||||
2934 | LN->getAAInfo(), LN->getRanges()); | ||||||
2935 | LN = cast<LoadSDNode>(NL.getNode()); | ||||||
2936 | } | ||||||
2937 | |||||||
2938 | Align ClaimAlign = LN->getAlign(); | ||||||
2939 | if (!validateConstPtrAlignment(LN->getBasePtr(), ClaimAlign, dl, DAG)) | ||||||
2940 | return replaceMemWithUndef(Op, DAG); | ||||||
2941 | |||||||
2942 | // Call LowerUnalignedLoad for all loads, it recognizes loads that | ||||||
2943 | // don't need extra aligning. | ||||||
2944 | SDValue LU = LowerUnalignedLoad(SDValue(LN, 0), DAG); | ||||||
2945 | if (DoCast) { | ||||||
2946 | SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, Ty, LU); | ||||||
2947 | SDValue Ch = cast<LoadSDNode>(LU.getNode())->getChain(); | ||||||
2948 | return DAG.getMergeValues({TC, Ch}, dl); | ||||||
2949 | } | ||||||
2950 | return LU; | ||||||
2951 | } | ||||||
2952 | |||||||
2953 | SDValue | ||||||
2954 | HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const { | ||||||
2955 | const SDLoc &dl(Op); | ||||||
2956 | StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); | ||||||
2957 | SDValue Val = SN->getValue(); | ||||||
2958 | MVT Ty = ty(Val); | ||||||
2959 | |||||||
2960 | bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1); | ||||||
2961 | if (DoCast) { | ||||||
2962 | SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, Val); | ||||||
2963 | SDValue NS = DAG.getStore(SN->getChain(), dl, TC, SN->getBasePtr(), | ||||||
2964 | SN->getMemOperand()); | ||||||
2965 | if (SN->isIndexed()) { | ||||||
2966 | NS = DAG.getIndexedStore(NS, dl, SN->getBasePtr(), SN->getOffset(), | ||||||
2967 | SN->getAddressingMode()); | ||||||
2968 | } | ||||||
2969 | SN = cast<StoreSDNode>(NS.getNode()); | ||||||
2970 | } | ||||||
2971 | |||||||
2972 | Align ClaimAlign = SN->getAlign(); | ||||||
2973 | if (!validateConstPtrAlignment(SN->getBasePtr(), ClaimAlign, dl, DAG)) | ||||||
2974 | return replaceMemWithUndef(Op, DAG); | ||||||
2975 | |||||||
2976 | MVT StoreTy = SN->getMemoryVT().getSimpleVT(); | ||||||
2977 | Align NeedAlign = Subtarget.getTypeAlignment(StoreTy); | ||||||
2978 | if (ClaimAlign < NeedAlign) | ||||||
2979 | return expandUnalignedStore(SN, DAG); | ||||||
2980 | return SDValue(SN, 0); | ||||||
2981 | } | ||||||
2982 | |||||||
2983 | SDValue | ||||||
2984 | HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) | ||||||
2985 | const { | ||||||
2986 | LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); | ||||||
2987 | MVT LoadTy = ty(Op); | ||||||
2988 | unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy).value(); | ||||||
2989 | unsigned HaveAlign = LN->getAlign().value(); | ||||||
2990 | if (HaveAlign >= NeedAlign) | ||||||
2991 | return Op; | ||||||
2992 | |||||||
2993 | const SDLoc &dl(Op); | ||||||
2994 | const DataLayout &DL = DAG.getDataLayout(); | ||||||
2995 | LLVMContext &Ctx = *DAG.getContext(); | ||||||
2996 | |||||||
2997 | // If the load aligning is disabled or the load can be broken up into two | ||||||
2998 | // smaller legal loads, do the default (target-independent) expansion. | ||||||
2999 | bool DoDefault = false; | ||||||
3000 | // Handle it in the default way if this is an indexed load. | ||||||
3001 | if (!LN->isUnindexed()) | ||||||
3002 | DoDefault = true; | ||||||
3003 | |||||||
3004 | if (!AlignLoads) { | ||||||
3005 | if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(), | ||||||
3006 | *LN->getMemOperand())) | ||||||
3007 | return Op; | ||||||
3008 | DoDefault = true; | ||||||
3009 | } | ||||||
3010 | if (!DoDefault && (2 * HaveAlign) == NeedAlign) { | ||||||
3011 | // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)". | ||||||
3012 | MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign) | ||||||
3013 | : MVT::getVectorVT(MVT::i8, HaveAlign); | ||||||
3014 | DoDefault = | ||||||
3015 | allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand()); | ||||||
3016 | } | ||||||
3017 | if (DoDefault) { | ||||||
3018 | std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG); | ||||||
3019 | return DAG.getMergeValues({P.first, P.second}, dl); | ||||||
3020 | } | ||||||
3021 | |||||||
3022 | // The code below generates two loads, both aligned as NeedAlign, and | ||||||
3023 | // with the distance of NeedAlign between them. For that to cover the | ||||||
3024 | // bits that need to be loaded (and without overlapping), the size of | ||||||
3025 | // the loads should be equal to NeedAlign. This is true for all loadable | ||||||
3026 | // types, but add an assertion in case something changes in the future. | ||||||
3027 | assert(LoadTy.getSizeInBits() == 8*NeedAlign)(static_cast <bool> (LoadTy.getSizeInBits() == 8*NeedAlign ) ? void (0) : __assert_fail ("LoadTy.getSizeInBits() == 8*NeedAlign" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3027, __extension__ __PRETTY_FUNCTION__)); | ||||||
3028 | |||||||
3029 | unsigned LoadLen = NeedAlign; | ||||||
3030 | SDValue Base = LN->getBasePtr(); | ||||||
3031 | SDValue Chain = LN->getChain(); | ||||||
3032 | auto BO = getBaseAndOffset(Base); | ||||||
3033 | unsigned BaseOpc = BO.first.getOpcode(); | ||||||
3034 | if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0) | ||||||
3035 | return Op; | ||||||
3036 | |||||||
3037 | if (BO.second % LoadLen != 0) { | ||||||
3038 | BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first, | ||||||
3039 | DAG.getConstant(BO.second % LoadLen, dl, MVT::i32)); | ||||||
3040 | BO.second -= BO.second % LoadLen; | ||||||
3041 | } | ||||||
3042 | SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR) | ||||||
3043 | ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first, | ||||||
3044 | DAG.getConstant(NeedAlign, dl, MVT::i32)) | ||||||
3045 | : BO.first; | ||||||
3046 | SDValue Base0 = | ||||||
3047 | DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl); | ||||||
3048 | SDValue Base1 = DAG.getMemBasePlusOffset( | ||||||
3049 | BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl); | ||||||
3050 | |||||||
3051 | MachineMemOperand *WideMMO = nullptr; | ||||||
3052 | if (MachineMemOperand *MMO = LN->getMemOperand()) { | ||||||
3053 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||
3054 | WideMMO = MF.getMachineMemOperand( | ||||||
3055 | MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen), | ||||||
3056 | MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(), | ||||||
3057 | MMO->getSuccessOrdering(), MMO->getFailureOrdering()); | ||||||
3058 | } | ||||||
3059 | |||||||
3060 | SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO); | ||||||
3061 | SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO); | ||||||
3062 | |||||||
3063 | SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy, | ||||||
3064 | {Load1, Load0, BaseNoOff.getOperand(0)}); | ||||||
3065 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||
3066 | Load0.getValue(1), Load1.getValue(1)); | ||||||
3067 | SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl); | ||||||
3068 | return M; | ||||||
3069 | } | ||||||
3070 | |||||||
3071 | SDValue | ||||||
3072 | HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const { | ||||||
3073 | SDValue X = Op.getOperand(0), Y = Op.getOperand(1); | ||||||
3074 | auto *CY = dyn_cast<ConstantSDNode>(Y); | ||||||
3075 | if (!CY) | ||||||
3076 | return SDValue(); | ||||||
3077 | |||||||
3078 | const SDLoc &dl(Op); | ||||||
3079 | SDVTList VTs = Op.getNode()->getVTList(); | ||||||
3080 | assert(VTs.NumVTs == 2)(static_cast <bool> (VTs.NumVTs == 2) ? void (0) : __assert_fail ("VTs.NumVTs == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3080, __extension__ __PRETTY_FUNCTION__)); | ||||||
3081 | assert(VTs.VTs[1] == MVT::i1)(static_cast <bool> (VTs.VTs[1] == MVT::i1) ? void (0) : __assert_fail ("VTs.VTs[1] == MVT::i1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3081, __extension__ __PRETTY_FUNCTION__)); | ||||||
3082 | unsigned Opc = Op.getOpcode(); | ||||||
3083 | |||||||
3084 | if (CY) { | ||||||
3085 | uint32_t VY = CY->getZExtValue(); | ||||||
3086 | assert(VY != 0 && "This should have been folded")(static_cast <bool> (VY != 0 && "This should have been folded" ) ? void (0) : __assert_fail ("VY != 0 && \"This should have been folded\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3086, __extension__ __PRETTY_FUNCTION__)); | ||||||
3087 | // X +/- 1 | ||||||
3088 | if (VY != 1) | ||||||
3089 | return SDValue(); | ||||||
3090 | |||||||
3091 | if (Opc == ISD::UADDO) { | ||||||
3092 | SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y}); | ||||||
3093 | SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG), | ||||||
3094 | ISD::SETEQ); | ||||||
3095 | return DAG.getMergeValues({Op, Ov}, dl); | ||||||
3096 | } | ||||||
3097 | if (Opc == ISD::USUBO) { | ||||||
3098 | SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y}); | ||||||
3099 | SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, | ||||||
3100 | DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ); | ||||||
3101 | return DAG.getMergeValues({Op, Ov}, dl); | ||||||
3102 | } | ||||||
3103 | } | ||||||
3104 | |||||||
3105 | return SDValue(); | ||||||
3106 | } | ||||||
3107 | |||||||
3108 | SDValue | ||||||
3109 | HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const { | ||||||
3110 | const SDLoc &dl(Op); | ||||||
3111 | unsigned Opc = Op.getOpcode(); | ||||||
3112 | SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2); | ||||||
3113 | |||||||
3114 | if (Opc == ISD::ADDCARRY) | ||||||
3115 | return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(), | ||||||
3116 | { X, Y, C }); | ||||||
3117 | |||||||
3118 | EVT CarryTy = C.getValueType(); | ||||||
3119 | SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(), | ||||||
3120 | { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) }); | ||||||
3121 | SDValue Out[] = { SubC.getValue(0), | ||||||
3122 | DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) }; | ||||||
3123 | return DAG.getMergeValues(Out, dl); | ||||||
3124 | } | ||||||
3125 | |||||||
3126 | SDValue | ||||||
3127 | HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const { | ||||||
3128 | SDValue Chain = Op.getOperand(0); | ||||||
3129 | SDValue Offset = Op.getOperand(1); | ||||||
3130 | SDValue Handler = Op.getOperand(2); | ||||||
3131 | SDLoc dl(Op); | ||||||
3132 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||||
3133 | |||||||
3134 | // Mark function as containing a call to EH_RETURN. | ||||||
3135 | HexagonMachineFunctionInfo *FuncInfo = | ||||||
3136 | DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>(); | ||||||
3137 | FuncInfo->setHasEHReturn(); | ||||||
3138 | |||||||
3139 | unsigned OffsetReg = Hexagon::R28; | ||||||
3140 | |||||||
3141 | SDValue StoreAddr = | ||||||
3142 | DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT), | ||||||
3143 | DAG.getIntPtrConstant(4, dl)); | ||||||
3144 | Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo()); | ||||||
3145 | Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset); | ||||||
3146 | |||||||
3147 | // Not needed we already use it as explict input to EH_RETURN. | ||||||
3148 | // MF.getRegInfo().addLiveOut(OffsetReg); | ||||||
3149 | |||||||
3150 | return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain); | ||||||
3151 | } | ||||||
3152 | |||||||
3153 | SDValue | ||||||
3154 | HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { | ||||||
3155 | unsigned Opc = Op.getOpcode(); | ||||||
3156 | |||||||
3157 | // Handle INLINEASM first. | ||||||
3158 | if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR) | ||||||
| |||||||
3159 | return LowerINLINEASM(Op, DAG); | ||||||
3160 | |||||||
3161 | if (isHvxOperation(Op.getNode(), DAG)) { | ||||||
3162 | // If HVX lowering returns nothing, try the default lowering. | ||||||
3163 | if (SDValue V = LowerHvxOperation(Op, DAG)) | ||||||
3164 | return V; | ||||||
3165 | } | ||||||
3166 | |||||||
3167 | switch (Opc) { | ||||||
3168 | default: | ||||||
3169 | #ifndef NDEBUG | ||||||
3170 | Op.getNode()->dumpr(&DAG); | ||||||
3171 | if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END) | ||||||
3172 | errs() << "Error: check for a non-legal type in this operation\n"; | ||||||
3173 | #endif | ||||||
3174 | llvm_unreachable("Should not custom lower this!")::llvm::llvm_unreachable_internal("Should not custom lower this!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3174); | ||||||
3175 | case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); | ||||||
3176 | case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG); | ||||||
3177 | case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); | ||||||
3178 | case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); | ||||||
3179 | case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); | ||||||
3180 | case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); | ||||||
3181 | case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); | ||||||
3182 | case ISD::BITCAST: return LowerBITCAST(Op, DAG); | ||||||
3183 | case ISD::LOAD: return LowerLoad(Op, DAG); | ||||||
3184 | case ISD::STORE: return LowerStore(Op, DAG); | ||||||
3185 | case ISD::UADDO: | ||||||
3186 | case ISD::USUBO: return LowerUAddSubO(Op, DAG); | ||||||
3187 | case ISD::ADDCARRY: | ||||||
3188 | case ISD::SUBCARRY: return LowerAddSubCarry(Op, DAG); | ||||||
3189 | case ISD::SRA: | ||||||
3190 | case ISD::SHL: | ||||||
3191 | case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG); | ||||||
3192 | case ISD::ROTL: return LowerROTL(Op, DAG); | ||||||
3193 | case ISD::ConstantPool: return LowerConstantPool(Op, DAG); | ||||||
3194 | case ISD::JumpTable: return LowerJumpTable(Op, DAG); | ||||||
3195 | case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); | ||||||
3196 | case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); | ||||||
3197 | case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); | ||||||
3198 | case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); | ||||||
3199 | case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG); | ||||||
3200 | case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG); | ||||||
3201 | case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); | ||||||
3202 | case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG); | ||||||
3203 | case ISD::VACOPY: return LowerVACOPY(Op, DAG); | ||||||
3204 | case ISD::VASTART: return LowerVASTART(Op, DAG); | ||||||
3205 | case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); | ||||||
3206 | case ISD::SETCC: return LowerSETCC(Op, DAG); | ||||||
3207 | case ISD::VSELECT: return LowerVSELECT(Op, DAG); | ||||||
3208 | case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); | ||||||
3209 | case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG); | ||||||
3210 | case ISD::PREFETCH: return LowerPREFETCH(Op, DAG); | ||||||
3211 | case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); | ||||||
3212 | break; | ||||||
3213 | } | ||||||
3214 | |||||||
3215 | return SDValue(); | ||||||
3216 | } | ||||||
3217 | |||||||
3218 | void | ||||||
3219 | HexagonTargetLowering::LowerOperationWrapper(SDNode *N, | ||||||
3220 | SmallVectorImpl<SDValue> &Results, | ||||||
3221 | SelectionDAG &DAG) const { | ||||||
3222 | if (isHvxOperation(N, DAG)) { | ||||||
3223 | LowerHvxOperationWrapper(N, Results, DAG); | ||||||
3224 | if (!Results.empty()) | ||||||
3225 | return; | ||||||
3226 | } | ||||||
3227 | |||||||
3228 | // We are only custom-lowering stores to verify the alignment of the | ||||||
3229 | // address if it is a compile-time constant. Since a store can be modified | ||||||
3230 | // during type-legalization (the value being stored may need legalization), | ||||||
3231 | // return empty Results here to indicate that we don't really make any | ||||||
3232 | // changes in the custom lowering. | ||||||
3233 | if (N->getOpcode() != ISD::STORE) | ||||||
3234 | return TargetLowering::LowerOperationWrapper(N, Results, DAG); | ||||||
3235 | } | ||||||
3236 | |||||||
3237 | void | ||||||
3238 | HexagonTargetLowering::ReplaceNodeResults(SDNode *N, | ||||||
3239 | SmallVectorImpl<SDValue> &Results, | ||||||
3240 | SelectionDAG &DAG) const { | ||||||
3241 | if (isHvxOperation(N, DAG)) { | ||||||
3242 | ReplaceHvxNodeResults(N, Results, DAG); | ||||||
3243 | if (!Results.empty()) | ||||||
3244 | return; | ||||||
3245 | } | ||||||
3246 | |||||||
3247 | const SDLoc &dl(N); | ||||||
3248 | switch (N->getOpcode()) { | ||||||
3249 | case ISD::SRL: | ||||||
3250 | case ISD::SRA: | ||||||
3251 | case ISD::SHL: | ||||||
3252 | return; | ||||||
3253 | case ISD::BITCAST: | ||||||
3254 | // Handle a bitcast from v8i1 to i8. | ||||||
3255 | if (N->getValueType(0) == MVT::i8) { | ||||||
3256 | if (N->getOperand(0).getValueType() == MVT::v8i1) { | ||||||
3257 | SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, | ||||||
3258 | N->getOperand(0), DAG); | ||||||
3259 | SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8); | ||||||
3260 | Results.push_back(T); | ||||||
3261 | } | ||||||
3262 | } | ||||||
3263 | break; | ||||||
3264 | } | ||||||
3265 | } | ||||||
3266 | |||||||
3267 | SDValue | ||||||
3268 | HexagonTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) | ||||||
3269 | const { | ||||||
3270 | if (isHvxOperation(N, DCI.DAG)) { | ||||||
3271 | if (SDValue V = PerformHvxDAGCombine(N, DCI)) | ||||||
3272 | return V; | ||||||
3273 | return SDValue(); | ||||||
3274 | } | ||||||
3275 | |||||||
3276 | if (DCI.isBeforeLegalizeOps()) | ||||||
3277 | return SDValue(); | ||||||
3278 | |||||||
3279 | SDValue Op(N, 0); | ||||||
3280 | const SDLoc &dl(Op); | ||||||
3281 | unsigned Opc = Op.getOpcode(); | ||||||
3282 | |||||||
3283 | if (Opc == HexagonISD::P2D) { | ||||||
3284 | SDValue P = Op.getOperand(0); | ||||||
3285 | switch (P.getOpcode()) { | ||||||
3286 | case HexagonISD::PTRUE: | ||||||
3287 | return DCI.DAG.getConstant(-1, dl, ty(Op)); | ||||||
3288 | case HexagonISD::PFALSE: | ||||||
3289 | return getZero(dl, ty(Op), DCI.DAG); | ||||||
3290 | default: | ||||||
3291 | break; | ||||||
3292 | } | ||||||
3293 | } else if (Opc == ISD::VSELECT) { | ||||||
3294 | // This is pretty much duplicated in HexagonISelLoweringHVX... | ||||||
3295 | // | ||||||
3296 | // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0) | ||||||
3297 | SDValue Cond = Op.getOperand(0); | ||||||
3298 | if (Cond->getOpcode() == ISD::XOR) { | ||||||
3299 | SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1); | ||||||
3300 | if (C1->getOpcode() == HexagonISD::PTRUE) { | ||||||
3301 | SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, | ||||||
3302 | Op.getOperand(2), Op.getOperand(1)); | ||||||
3303 | return VSel; | ||||||
3304 | } | ||||||
3305 | } | ||||||
3306 | } | ||||||
3307 | |||||||
3308 | return SDValue(); | ||||||
3309 | } | ||||||
3310 | |||||||
3311 | /// Returns relocation base for the given PIC jumptable. | ||||||
3312 | SDValue | ||||||
3313 | HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table, | ||||||
3314 | SelectionDAG &DAG) const { | ||||||
3315 | int Idx = cast<JumpTableSDNode>(Table)->getIndex(); | ||||||
3316 | EVT VT = Table.getValueType(); | ||||||
3317 | SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL); | ||||||
3318 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T); | ||||||
3319 | } | ||||||
3320 | |||||||
3321 | //===----------------------------------------------------------------------===// | ||||||
3322 | // Inline Assembly Support | ||||||
3323 | //===----------------------------------------------------------------------===// | ||||||
3324 | |||||||
3325 | TargetLowering::ConstraintType | ||||||
3326 | HexagonTargetLowering::getConstraintType(StringRef Constraint) const { | ||||||
3327 | if (Constraint.size() == 1) { | ||||||
3328 | switch (Constraint[0]) { | ||||||
3329 | case 'q': | ||||||
3330 | case 'v': | ||||||
3331 | if (Subtarget.useHVXOps()) | ||||||
3332 | return C_RegisterClass; | ||||||
3333 | break; | ||||||
3334 | case 'a': | ||||||
3335 | return C_RegisterClass; | ||||||
3336 | default: | ||||||
3337 | break; | ||||||
3338 | } | ||||||
3339 | } | ||||||
3340 | return TargetLowering::getConstraintType(Constraint); | ||||||
3341 | } | ||||||
3342 | |||||||
3343 | std::pair<unsigned, const TargetRegisterClass*> | ||||||
3344 | HexagonTargetLowering::getRegForInlineAsmConstraint( | ||||||
3345 | const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { | ||||||
3346 | |||||||
3347 | if (Constraint.size() == 1) { | ||||||
3348 | switch (Constraint[0]) { | ||||||
3349 | case 'r': // R0-R31 | ||||||
3350 | switch (VT.SimpleTy) { | ||||||
3351 | default: | ||||||
3352 | return {0u, nullptr}; | ||||||
3353 | case MVT::i1: | ||||||
3354 | case MVT::i8: | ||||||
3355 | case MVT::i16: | ||||||
3356 | case MVT::i32: | ||||||
3357 | case MVT::f32: | ||||||
3358 | return {0u, &Hexagon::IntRegsRegClass}; | ||||||
3359 | case MVT::i64: | ||||||
3360 | case MVT::f64: | ||||||
3361 | return {0u, &Hexagon::DoubleRegsRegClass}; | ||||||
3362 | } | ||||||
3363 | break; | ||||||
3364 | case 'a': // M0-M1 | ||||||
3365 | if (VT != MVT::i32) | ||||||
3366 | return {0u, nullptr}; | ||||||
3367 | return {0u, &Hexagon::ModRegsRegClass}; | ||||||
3368 | case 'q': // q0-q3 | ||||||
3369 | switch (VT.getSizeInBits()) { | ||||||
3370 | default: | ||||||
3371 | return {0u, nullptr}; | ||||||
3372 | case 64: | ||||||
3373 | case 128: | ||||||
3374 | return {0u, &Hexagon::HvxQRRegClass}; | ||||||
3375 | } | ||||||
3376 | break; | ||||||
3377 | case 'v': // V0-V31 | ||||||
3378 | switch (VT.getSizeInBits()) { | ||||||
3379 | default: | ||||||
3380 | return {0u, nullptr}; | ||||||
3381 | case 512: | ||||||
3382 | return {0u, &Hexagon::HvxVRRegClass}; | ||||||
3383 | case 1024: | ||||||
3384 | if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps()) | ||||||
3385 | return {0u, &Hexagon::HvxVRRegClass}; | ||||||
3386 | return {0u, &Hexagon::HvxWRRegClass}; | ||||||
3387 | case 2048: | ||||||
3388 | return {0u, &Hexagon::HvxWRRegClass}; | ||||||
3389 | } | ||||||
3390 | break; | ||||||
3391 | default: | ||||||
3392 | return {0u, nullptr}; | ||||||
3393 | } | ||||||
3394 | } | ||||||
3395 | |||||||
3396 | return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); | ||||||
3397 | } | ||||||
3398 | |||||||
3399 | /// isFPImmLegal - Returns true if the target can instruction select the | ||||||
3400 | /// specified FP immediate natively. If false, the legalizer will | ||||||
3401 | /// materialize the FP immediate as a load from a constant pool. | ||||||
3402 | bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, | ||||||
3403 | bool ForCodeSize) const { | ||||||
3404 | return true; | ||||||
3405 | } | ||||||
3406 | |||||||
3407 | /// isLegalAddressingMode - Return true if the addressing mode represented by | ||||||
3408 | /// AM is legal for this target, for a load/store of the specified type. | ||||||
3409 | bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL, | ||||||
3410 | const AddrMode &AM, Type *Ty, | ||||||
3411 | unsigned AS, Instruction *I) const { | ||||||
3412 | if (Ty->isSized()) { | ||||||
3413 | // When LSR detects uses of the same base address to access different | ||||||
3414 | // types (e.g. unions), it will assume a conservative type for these | ||||||
3415 | // uses: | ||||||
3416 | // LSR Use: Kind=Address of void in addrspace(4294967295), ... | ||||||
3417 | // The type Ty passed here would then be "void". Skip the alignment | ||||||
3418 | // checks, but do not return false right away, since that confuses | ||||||
3419 | // LSR into crashing. | ||||||
3420 | Align A = DL.getABITypeAlign(Ty); | ||||||
3421 | // The base offset must be a multiple of the alignment. | ||||||
3422 | if (!isAligned(A, AM.BaseOffs)) | ||||||
3423 | return false; | ||||||
3424 | // The shifted offset must fit in 11 bits. | ||||||
3425 | if (!isInt<11>(AM.BaseOffs >> Log2(A))) | ||||||
3426 | return false; | ||||||
3427 | } | ||||||
3428 | |||||||
3429 | // No global is ever allowed as a base. | ||||||
3430 | if (AM.BaseGV) | ||||||
3431 | return false; | ||||||
3432 | |||||||
3433 | int Scale = AM.Scale; | ||||||
3434 | if (Scale < 0) | ||||||
3435 | Scale = -Scale; | ||||||
3436 | switch (Scale) { | ||||||
3437 | case 0: // No scale reg, "r+i", "r", or just "i". | ||||||
3438 | break; | ||||||
3439 | default: // No scaled addressing mode. | ||||||
3440 | return false; | ||||||
3441 | } | ||||||
3442 | return true; | ||||||
3443 | } | ||||||
3444 | |||||||
3445 | /// Return true if folding a constant offset with the given GlobalAddress is | ||||||
3446 | /// legal. It is frequently not legal in PIC relocation models. | ||||||
3447 | bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) | ||||||
3448 | const { | ||||||
3449 | return HTM.getRelocationModel() == Reloc::Static; | ||||||
3450 | } | ||||||
3451 | |||||||
3452 | /// isLegalICmpImmediate - Return true if the specified immediate is legal | ||||||
3453 | /// icmp immediate, that is the target has icmp instructions which can compare | ||||||
3454 | /// a register against the immediate without having to materialize the | ||||||
3455 | /// immediate into a register. | ||||||
3456 | bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const { | ||||||
3457 | return Imm >= -512 && Imm <= 511; | ||||||
3458 | } | ||||||
3459 | |||||||
3460 | /// IsEligibleForTailCallOptimization - Check whether the call is eligible | ||||||
3461 | /// for tail call optimization. Targets which want to do tail call | ||||||
3462 | /// optimization should implement this function. | ||||||
3463 | bool HexagonTargetLowering::IsEligibleForTailCallOptimization( | ||||||
3464 | SDValue Callee, | ||||||
3465 | CallingConv::ID CalleeCC, | ||||||
3466 | bool IsVarArg, | ||||||
3467 | bool IsCalleeStructRet, | ||||||
3468 | bool IsCallerStructRet, | ||||||
3469 | const SmallVectorImpl<ISD::OutputArg> &Outs, | ||||||
3470 | const SmallVectorImpl<SDValue> &OutVals, | ||||||
3471 | const SmallVectorImpl<ISD::InputArg> &Ins, | ||||||
3472 | SelectionDAG& DAG) const { | ||||||
3473 | const Function &CallerF = DAG.getMachineFunction().getFunction(); | ||||||
3474 | CallingConv::ID CallerCC = CallerF.getCallingConv(); | ||||||
3475 | bool CCMatch = CallerCC == CalleeCC; | ||||||
3476 | |||||||
3477 | // *************************************************************************** | ||||||
3478 | // Look for obvious safe cases to perform tail call optimization that do not | ||||||
3479 | // require ABI changes. | ||||||
3480 | // *************************************************************************** | ||||||
3481 | |||||||
3482 | // If this is a tail call via a function pointer, then don't do it! | ||||||
3483 | if (!isa<GlobalAddressSDNode>(Callee) && | ||||||
3484 | !isa<ExternalSymbolSDNode>(Callee)) { | ||||||
3485 | return false; | ||||||
3486 | } | ||||||
3487 | |||||||
3488 | // Do not optimize if the calling conventions do not match and the conventions | ||||||
3489 | // used are not C or Fast. | ||||||
3490 | if (!CCMatch) { | ||||||
3491 | bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast); | ||||||
3492 | bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast); | ||||||
3493 | // If R & E, then ok. | ||||||
3494 | if (!R || !E) | ||||||
3495 | return false; | ||||||
3496 | } | ||||||
3497 | |||||||
3498 | // Do not tail call optimize vararg calls. | ||||||
3499 | if (IsVarArg) | ||||||
3500 | return false; | ||||||
3501 | |||||||
3502 | // Also avoid tail call optimization if either caller or callee uses struct | ||||||
3503 | // return semantics. | ||||||
3504 | if (IsCalleeStructRet || IsCallerStructRet) | ||||||
3505 | return false; | ||||||
3506 | |||||||
3507 | // In addition to the cases above, we also disable Tail Call Optimization if | ||||||
3508 | // the calling convention code that at least one outgoing argument needs to | ||||||
3509 | // go on the stack. We cannot check that here because at this point that | ||||||
3510 | // information is not available. | ||||||
3511 | return true; | ||||||
3512 | } | ||||||
3513 | |||||||
3514 | /// Returns the target specific optimal type for load and store operations as | ||||||
3515 | /// a result of memset, memcpy, and memmove lowering. | ||||||
3516 | /// | ||||||
3517 | /// If DstAlign is zero that means it's safe to destination alignment can | ||||||
3518 | /// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't | ||||||
3519 | /// a need to check it against alignment requirement, probably because the | ||||||
3520 | /// source does not need to be loaded. If 'IsMemset' is true, that means it's | ||||||
3521 | /// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of | ||||||
3522 | /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it | ||||||
3523 | /// does not need to be loaded. It returns EVT::Other if the type should be | ||||||
3524 | /// determined using generic target-independent logic. | ||||||
3525 | EVT HexagonTargetLowering::getOptimalMemOpType( | ||||||
3526 | const MemOp &Op, const AttributeList &FuncAttributes) const { | ||||||
3527 | if (Op.size() >= 8 && Op.isAligned(Align(8))) | ||||||
3528 | return MVT::i64; | ||||||
3529 | if (Op.size() >= 4 && Op.isAligned(Align(4))) | ||||||
3530 | return MVT::i32; | ||||||
3531 | if (Op.size() >= 2 && Op.isAligned(Align(2))) | ||||||
3532 | return MVT::i16; | ||||||
3533 | return MVT::Other; | ||||||
3534 | } | ||||||
3535 | |||||||
3536 | bool HexagonTargetLowering::allowsMemoryAccess( | ||||||
3537 | LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, | ||||||
3538 | Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const { | ||||||
3539 | MVT SVT = VT.getSimpleVT(); | ||||||
3540 | if (Subtarget.isHVXVectorType(SVT, true)) | ||||||
3541 | return allowsHvxMemoryAccess(SVT, Flags, Fast); | ||||||
3542 | return TargetLoweringBase::allowsMemoryAccess( | ||||||
3543 | Context, DL, VT, AddrSpace, Alignment, Flags, Fast); | ||||||
3544 | } | ||||||
3545 | |||||||
3546 | bool HexagonTargetLowering::allowsMisalignedMemoryAccesses( | ||||||
3547 | EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, | ||||||
3548 | bool *Fast) const { | ||||||
3549 | MVT SVT = VT.getSimpleVT(); | ||||||
3550 | if (Subtarget.isHVXVectorType(SVT, true)) | ||||||
3551 | return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast); | ||||||
3552 | if (Fast) | ||||||
3553 | *Fast = false; | ||||||
3554 | return false; | ||||||
3555 | } | ||||||
3556 | |||||||
3557 | std::pair<const TargetRegisterClass*, uint8_t> | ||||||
3558 | HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, | ||||||
3559 | MVT VT) const { | ||||||
3560 | if (Subtarget.isHVXVectorType(VT, true)) { | ||||||
3561 | unsigned BitWidth = VT.getSizeInBits(); | ||||||
3562 | unsigned VecWidth = Subtarget.getVectorLength() * 8; | ||||||
3563 | |||||||
3564 | if (VT.getVectorElementType() == MVT::i1) | ||||||
3565 | return std::make_pair(&Hexagon::HvxQRRegClass, 1); | ||||||
3566 | if (BitWidth == VecWidth) | ||||||
3567 | return std::make_pair(&Hexagon::HvxVRRegClass, 1); | ||||||
3568 | assert(BitWidth == 2 * VecWidth)(static_cast <bool> (BitWidth == 2 * VecWidth) ? void ( 0) : __assert_fail ("BitWidth == 2 * VecWidth", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3568, __extension__ __PRETTY_FUNCTION__)); | ||||||
3569 | return std::make_pair(&Hexagon::HvxWRRegClass, 1); | ||||||
3570 | } | ||||||
3571 | |||||||
3572 | return TargetLowering::findRepresentativeClass(TRI, VT); | ||||||
3573 | } | ||||||
3574 | |||||||
3575 | bool HexagonTargetLowering::shouldReduceLoadWidth(SDNode *Load, | ||||||
3576 | ISD::LoadExtType ExtTy, EVT NewVT) const { | ||||||
3577 | // TODO: This may be worth removing. Check regression tests for diffs. | ||||||
3578 | if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT)) | ||||||
3579 | return false; | ||||||
3580 | |||||||
3581 | auto *L = cast<LoadSDNode>(Load); | ||||||
3582 | std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr()); | ||||||
3583 | // Small-data object, do not shrink. | ||||||
3584 | if (BO.first.getOpcode() == HexagonISD::CONST32_GP) | ||||||
3585 | return false; | ||||||
3586 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) { | ||||||
3587 | auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine()); | ||||||
3588 | const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal()); | ||||||
3589 | return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM); | ||||||
3590 | } | ||||||
3591 | return true; | ||||||
3592 | } | ||||||
3593 | |||||||
3594 | Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder, | ||||||
3595 | Type *ValueTy, Value *Addr, | ||||||
3596 | AtomicOrdering Ord) const { | ||||||
3597 | BasicBlock *BB = Builder.GetInsertBlock(); | ||||||
3598 | Module *M = BB->getParent()->getParent(); | ||||||
3599 | unsigned SZ = ValueTy->getPrimitiveSizeInBits(); | ||||||
3600 | assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported" ) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic loads supported\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3600, __extension__ __PRETTY_FUNCTION__)); | ||||||
3601 | Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked | ||||||
3602 | : Intrinsic::hexagon_L4_loadd_locked; | ||||||
3603 | Function *Fn = Intrinsic::getDeclaration(M, IntID); | ||||||
3604 | |||||||
3605 | auto PtrTy = cast<PointerType>(Addr->getType()); | ||||||
3606 | PointerType *NewPtrTy = | ||||||
3607 | Builder.getIntNTy(SZ)->getPointerTo(PtrTy->getAddressSpace()); | ||||||
3608 | Addr = Builder.CreateBitCast(Addr, NewPtrTy); | ||||||
3609 | |||||||
3610 | Value *Call = Builder.CreateCall(Fn, Addr, "larx"); | ||||||
3611 | |||||||
3612 | return Builder.CreateBitCast(Call, ValueTy); | ||||||
3613 | } | ||||||
3614 | |||||||
3615 | /// Perform a store-conditional operation to Addr. Return the status of the | ||||||
3616 | /// store. This should be 0 if the store succeeded, non-zero otherwise. | ||||||
3617 | Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder, | ||||||
3618 | Value *Val, Value *Addr, | ||||||
3619 | AtomicOrdering Ord) const { | ||||||
3620 | BasicBlock *BB = Builder.GetInsertBlock(); | ||||||
3621 | Module *M = BB->getParent()->getParent(); | ||||||
3622 | Type *Ty = Val->getType(); | ||||||
3623 | unsigned SZ = Ty->getPrimitiveSizeInBits(); | ||||||
3624 | |||||||
3625 | Type *CastTy = Builder.getIntNTy(SZ); | ||||||
3626 | assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported" ) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic stores supported\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3626, __extension__ __PRETTY_FUNCTION__)); | ||||||
3627 | Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked | ||||||
3628 | : Intrinsic::hexagon_S4_stored_locked; | ||||||
3629 | Function *Fn = Intrinsic::getDeclaration(M, IntID); | ||||||
3630 | |||||||
3631 | unsigned AS = Addr->getType()->getPointerAddressSpace(); | ||||||
3632 | Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS)); | ||||||
3633 | Val = Builder.CreateBitCast(Val, CastTy); | ||||||
3634 | |||||||
3635 | Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx"); | ||||||
3636 | Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), ""); | ||||||
3637 | Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext())); | ||||||
3638 | return Ext; | ||||||
3639 | } | ||||||
3640 | |||||||
3641 | TargetLowering::AtomicExpansionKind | ||||||
3642 | HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { | ||||||
3643 | // Do not expand loads and stores that don't exceed 64 bits. | ||||||
3644 | return LI->getType()->getPrimitiveSizeInBits() > 64 | ||||||
3645 | ? AtomicExpansionKind::LLOnly | ||||||
3646 | : AtomicExpansionKind::None; | ||||||
3647 | } | ||||||
3648 | |||||||
3649 | bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const { | ||||||
3650 | // Do not expand loads and stores that don't exceed 64 bits. | ||||||
3651 | return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64; | ||||||
3652 | } | ||||||
3653 | |||||||
3654 | TargetLowering::AtomicExpansionKind | ||||||
3655 | HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR( | ||||||
3656 | AtomicCmpXchgInst *AI) const { | ||||||
3657 | return AtomicExpansionKind::LLSC; | ||||||
3658 | } |
1 | //===-- HexagonISelLowering.h - Hexagon DAG Lowering Interface --*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file defines the interfaces that Hexagon uses to lower LLVM code into a |
10 | // selection DAG. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGONISELLOWERING_H |
15 | #define LLVM_LIB_TARGET_HEXAGON_HEXAGONISELLOWERING_H |
16 | |
17 | #include "Hexagon.h" |
18 | #include "MCTargetDesc/HexagonMCTargetDesc.h" |
19 | #include "llvm/ADT/StringRef.h" |
20 | #include "llvm/CodeGen/ISDOpcodes.h" |
21 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
22 | #include "llvm/CodeGen/TargetLowering.h" |
23 | #include "llvm/CodeGen/ValueTypes.h" |
24 | #include "llvm/IR/CallingConv.h" |
25 | #include "llvm/IR/InlineAsm.h" |
26 | #include "llvm/Support/MachineValueType.h" |
27 | #include <cstdint> |
28 | #include <utility> |
29 | |
30 | namespace llvm { |
31 | |
32 | namespace HexagonISD { |
33 | |
34 | enum NodeType : unsigned { |
35 | OP_BEGIN = ISD::BUILTIN_OP_END, |
36 | |
37 | CONST32 = OP_BEGIN, |
38 | CONST32_GP, // For marking data present in GP. |
39 | ADDC, // Add with carry: (X, Y, Cin) -> (X+Y, Cout). |
40 | SUBC, // Sub with carry: (X, Y, Cin) -> (X+~Y+Cin, Cout). |
41 | ALLOCA, |
42 | |
43 | AT_GOT, // Index in GOT. |
44 | AT_PCREL, // Offset relative to PC. |
45 | |
46 | CALL, // Function call. |
47 | CALLnr, // Function call that does not return. |
48 | CALLR, |
49 | |
50 | RET_FLAG, // Return with a flag operand. |
51 | BARRIER, // Memory barrier. |
52 | JT, // Jump table. |
53 | CP, // Constant pool. |
54 | |
55 | COMBINE, |
56 | VASL, |
57 | VASR, |
58 | VLSR, |
59 | |
60 | TSTBIT, |
61 | INSERT, |
62 | EXTRACTU, |
63 | VEXTRACTW, |
64 | VINSERTW0, |
65 | VROR, |
66 | TC_RETURN, |
67 | EH_RETURN, |
68 | DCFETCH, |
69 | READCYCLE, |
70 | PTRUE, |
71 | PFALSE, |
72 | D2P, // Convert 8-byte value to 8-bit predicate register. [*] |
73 | P2D, // Convert 8-bit predicate register to 8-byte value. [*] |
74 | V2Q, // Convert HVX vector to a vector predicate reg. [*] |
75 | Q2V, // Convert vector predicate to an HVX vector. [*] |
76 | // [*] The equivalence is defined as "Q <=> (V != 0)", |
77 | // where the != operation compares bytes. |
78 | // Note: V != 0 is implemented as V >u 0. |
79 | QCAT, |
80 | QTRUE, |
81 | QFALSE, |
82 | TYPECAST, // No-op that's used to convert between different legal |
83 | // types in a register. |
84 | VALIGN, // Align two vectors (in Op0, Op1) to one that would have |
85 | // been loaded from address in Op2. |
86 | VALIGNADDR, // Align vector address: Op0 & -Op1, except when it is |
87 | // an address in a vector load, then it's a no-op. |
88 | VPACKL, // Pack low parts of the input vector to the front of the |
89 | // output. For example v64i16 VPACKL(v32i32) will pick |
90 | // the low halfwords and pack them into the first 32 |
91 | // halfwords of the output. The rest of the output is |
92 | // unspecified. |
93 | VUNPACK, // Unpacking into low elements with sign extension. |
94 | VUNPACKU, // Unpacking into low elements with zero extension. |
95 | ISEL, // Marker for nodes that were created during ISel, and |
96 | // which need explicit selection (would have been left |
97 | // unselected otherwise). |
98 | OP_END |
99 | }; |
100 | |
101 | } // end namespace HexagonISD |
102 | |
103 | class HexagonSubtarget; |
104 | |
105 | class HexagonTargetLowering : public TargetLowering { |
106 | int VarArgsFrameOffset; // Frame offset to start of varargs area. |
107 | const HexagonTargetMachine &HTM; |
108 | const HexagonSubtarget &Subtarget; |
109 | |
110 | bool CanReturnSmallStruct(const Function* CalleeFn, unsigned& RetSize) |
111 | const; |
112 | |
113 | public: |
114 | explicit HexagonTargetLowering(const TargetMachine &TM, |
115 | const HexagonSubtarget &ST); |
116 | |
117 | bool isHVXVectorType(MVT Ty) const; |
118 | |
119 | /// IsEligibleForTailCallOptimization - Check whether the call is eligible |
120 | /// for tail call optimization. Targets which want to do tail call |
121 | /// optimization should implement this function. |
122 | bool IsEligibleForTailCallOptimization(SDValue Callee, |
123 | CallingConv::ID CalleeCC, bool isVarArg, bool isCalleeStructRet, |
124 | bool isCallerStructRet, const SmallVectorImpl<ISD::OutputArg> &Outs, |
125 | const SmallVectorImpl<SDValue> &OutVals, |
126 | const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const; |
127 | |
128 | bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, |
129 | MachineFunction &MF, |
130 | unsigned Intrinsic) const override; |
131 | |
132 | bool isTruncateFree(Type *Ty1, Type *Ty2) const override; |
133 | bool isTruncateFree(EVT VT1, EVT VT2) const override; |
134 | |
135 | bool isCheapToSpeculateCttz() const override { return true; } |
136 | bool isCheapToSpeculateCtlz() const override { return true; } |
137 | bool isCtlzFast() const override { return true; } |
138 | |
139 | bool hasBitTest(SDValue X, SDValue Y) const override; |
140 | |
141 | bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override; |
142 | |
143 | /// Return true if an FMA operation is faster than a pair of mul and add |
144 | /// instructions. fmuladd intrinsics will be expanded to FMAs when this |
145 | /// method returns true (and FMAs are legal), otherwise fmuladd is |
146 | /// expanded to mul + add. |
147 | bool isFMAFasterThanFMulAndFAdd(const MachineFunction &, |
148 | EVT) const override; |
149 | |
150 | // Should we expand the build vector with shuffles? |
151 | bool shouldExpandBuildVectorWithShuffles(EVT VT, |
152 | unsigned DefinedValues) const override; |
153 | |
154 | bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override; |
155 | TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) |
156 | const override; |
157 | |
158 | SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; |
159 | void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, |
160 | SelectionDAG &DAG) const override; |
161 | void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, |
162 | SelectionDAG &DAG) const override; |
163 | |
164 | const char *getTargetNodeName(unsigned Opcode) const override; |
165 | |
166 | SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const; |
167 | SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; |
168 | SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; |
169 | SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; |
170 | SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; |
171 | SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; |
172 | SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const; |
173 | SDValue LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const; |
174 | SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const; |
175 | SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const; |
176 | SDValue LowerANY_EXTEND(SDValue Op, SelectionDAG &DAG) const; |
177 | SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const; |
178 | SDValue LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const; |
179 | SDValue LowerLoad(SDValue Op, SelectionDAG &DAG) const; |
180 | SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const; |
181 | SDValue LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) const; |
182 | SDValue LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const; |
183 | SDValue LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const; |
184 | |
185 | SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const; |
186 | SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const; |
187 | SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) const; |
188 | SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; |
189 | SDValue LowerEH_LABEL(SDValue Op, SelectionDAG &DAG) const; |
190 | SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const; |
191 | SDValue |
192 | LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, |
193 | const SmallVectorImpl<ISD::InputArg> &Ins, |
194 | const SDLoc &dl, SelectionDAG &DAG, |
195 | SmallVectorImpl<SDValue> &InVals) const override; |
196 | SDValue LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const; |
197 | SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; |
198 | SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; |
199 | SDValue LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, |
200 | SelectionDAG &DAG) const; |
201 | SDValue LowerToTLSInitialExecModel(GlobalAddressSDNode *GA, |
202 | SelectionDAG &DAG) const; |
203 | SDValue LowerToTLSLocalExecModel(GlobalAddressSDNode *GA, |
204 | SelectionDAG &DAG) const; |
205 | SDValue GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain, |
206 | GlobalAddressSDNode *GA, SDValue InFlag, EVT PtrVT, |
207 | unsigned ReturnReg, unsigned char OperandFlags) const; |
208 | SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const; |
209 | |
210 | SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, |
211 | SmallVectorImpl<SDValue> &InVals) const override; |
212 | SDValue LowerCallResult(SDValue Chain, SDValue InFlag, |
213 | CallingConv::ID CallConv, bool isVarArg, |
214 | const SmallVectorImpl<ISD::InputArg> &Ins, |
215 | const SDLoc &dl, SelectionDAG &DAG, |
216 | SmallVectorImpl<SDValue> &InVals, |
217 | const SmallVectorImpl<SDValue> &OutVals, |
218 | SDValue Callee) const; |
219 | |
220 | SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; |
221 | SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const; |
222 | SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const; |
223 | SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const; |
224 | SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const; |
225 | |
226 | bool CanLowerReturn(CallingConv::ID CallConv, |
227 | MachineFunction &MF, bool isVarArg, |
228 | const SmallVectorImpl<ISD::OutputArg> &Outs, |
229 | LLVMContext &Context) const override; |
230 | |
231 | SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, |
232 | const SmallVectorImpl<ISD::OutputArg> &Outs, |
233 | const SmallVectorImpl<SDValue> &OutVals, |
234 | const SDLoc &dl, SelectionDAG &DAG) const override; |
235 | |
236 | SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; |
237 | |
238 | bool mayBeEmittedAsTailCall(const CallInst *CI) const override; |
239 | |
240 | Register getRegisterByName(const char* RegName, LLT VT, |
241 | const MachineFunction &MF) const override; |
242 | |
243 | /// If a physical register, this returns the register that receives the |
244 | /// exception address on entry to an EH pad. |
245 | Register |
246 | getExceptionPointerRegister(const Constant *PersonalityFn) const override { |
247 | return Hexagon::R0; |
248 | } |
249 | |
250 | /// If a physical register, this returns the register that receives the |
251 | /// exception typeid on entry to a landing pad. |
252 | Register |
253 | getExceptionSelectorRegister(const Constant *PersonalityFn) const override { |
254 | return Hexagon::R1; |
255 | } |
256 | |
257 | SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const; |
258 | SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const; |
259 | SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; |
260 | SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const; |
261 | |
262 | EVT getSetCCResultType(const DataLayout &, LLVMContext &C, |
263 | EVT VT) const override { |
264 | if (!VT.isVector()) |
265 | return MVT::i1; |
266 | else |
267 | return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); |
268 | } |
269 | |
270 | bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, |
271 | SDValue &Base, SDValue &Offset, |
272 | ISD::MemIndexedMode &AM, |
273 | SelectionDAG &DAG) const override; |
274 | |
275 | ConstraintType getConstraintType(StringRef Constraint) const override; |
276 | |
277 | std::pair<unsigned, const TargetRegisterClass *> |
278 | getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, |
279 | StringRef Constraint, MVT VT) const override; |
280 | |
281 | // Intrinsics |
282 | SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; |
283 | SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; |
284 | /// isLegalAddressingMode - Return true if the addressing mode represented |
285 | /// by AM is legal for this target, for a load/store of the specified type. |
286 | /// The type may be VoidTy, in which case only return true if the addressing |
287 | /// mode is legal for a load/store of any legal type. |
288 | /// TODO: Handle pre/postinc as well. |
289 | bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, |
290 | Type *Ty, unsigned AS, |
291 | Instruction *I = nullptr) const override; |
292 | /// Return true if folding a constant offset with the given GlobalAddress |
293 | /// is legal. It is frequently not legal in PIC relocation models. |
294 | bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; |
295 | |
296 | bool isFPImmLegal(const APFloat &Imm, EVT VT, |
297 | bool ForCodeSize) const override; |
298 | |
299 | /// isLegalICmpImmediate - Return true if the specified immediate is legal |
300 | /// icmp immediate, that is the target has icmp instructions which can |
301 | /// compare a register against the immediate without having to materialize |
302 | /// the immediate into a register. |
303 | bool isLegalICmpImmediate(int64_t Imm) const override; |
304 | |
305 | EVT getOptimalMemOpType(const MemOp &Op, |
306 | const AttributeList &FuncAttributes) const override; |
307 | |
308 | bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, |
309 | unsigned AddrSpace, Align Alignment, |
310 | MachineMemOperand::Flags Flags, |
311 | bool *Fast) const override; |
312 | |
313 | bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, |
314 | Align Alignment, |
315 | MachineMemOperand::Flags Flags, |
316 | bool *Fast) const override; |
317 | |
318 | /// Returns relocation base for the given PIC jumptable. |
319 | SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) |
320 | const override; |
321 | |
322 | bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, |
323 | EVT NewVT) const override; |
324 | |
325 | // Handling of atomic RMW instructions. |
326 | Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, |
327 | AtomicOrdering Ord) const override; |
328 | Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, |
329 | AtomicOrdering Ord) const override; |
330 | AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override; |
331 | bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; |
332 | AtomicExpansionKind |
333 | shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; |
334 | |
335 | AtomicExpansionKind |
336 | shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override { |
337 | return AtomicExpansionKind::LLSC; |
338 | } |
339 | |
340 | private: |
341 | void initializeHVXLowering(); |
342 | unsigned getPreferredHvxVectorAction(MVT VecTy) const; |
343 | |
344 | bool validateConstPtrAlignment(SDValue Ptr, Align NeedAlign, const SDLoc &dl, |
345 | SelectionDAG &DAG) const; |
346 | SDValue replaceMemWithUndef(SDValue Op, SelectionDAG &DAG) const; |
347 | |
348 | std::pair<SDValue,int> getBaseAndOffset(SDValue Addr) const; |
349 | |
350 | bool getBuildVectorConstInts(ArrayRef<SDValue> Values, MVT VecTy, |
351 | SelectionDAG &DAG, |
352 | MutableArrayRef<ConstantInt*> Consts) const; |
353 | SDValue buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, |
354 | SelectionDAG &DAG) const; |
355 | SDValue buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, |
356 | SelectionDAG &DAG) const; |
357 | SDValue extractVector(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
358 | MVT ValTy, MVT ResTy, SelectionDAG &DAG) const; |
359 | SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV, |
360 | const SDLoc &dl, MVT ValTy, SelectionDAG &DAG) const; |
361 | SDValue expandPredicate(SDValue Vec32, const SDLoc &dl, |
362 | SelectionDAG &DAG) const; |
363 | SDValue contractPredicate(SDValue Vec64, const SDLoc &dl, |
364 | SelectionDAG &DAG) const; |
365 | SDValue getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const; |
366 | SDValue appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG) const; |
367 | |
368 | bool isUndef(SDValue Op) const { |
369 | if (Op.isMachineOpcode()) |
370 | return Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF; |
371 | return Op.getOpcode() == ISD::UNDEF; |
372 | } |
373 | SDValue getInstr(unsigned MachineOpc, const SDLoc &dl, MVT Ty, |
374 | ArrayRef<SDValue> Ops, SelectionDAG &DAG) const { |
375 | SDNode *N = DAG.getMachineNode(MachineOpc, dl, Ty, Ops); |
376 | return SDValue(N, 0); |
377 | } |
378 | SDValue getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) const; |
379 | |
380 | using VectorPair = std::pair<SDValue, SDValue>; |
381 | using TypePair = std::pair<MVT, MVT>; |
382 | |
383 | SDValue getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, |
384 | const SDLoc &dl, SelectionDAG &DAG) const; |
385 | |
386 | MVT ty(SDValue Op) const { |
387 | return Op.getValueType().getSimpleVT(); |
388 | } |
389 | TypePair ty(const VectorPair &Ops) const { |
390 | return { Ops.first.getValueType().getSimpleVT(), |
391 | Ops.second.getValueType().getSimpleVT() }; |
392 | } |
393 | MVT tyScalar(MVT Ty) const { |
394 | if (!Ty.isVector()) |
395 | return Ty; |
396 | return MVT::getIntegerVT(Ty.getSizeInBits()); |
397 | } |
398 | MVT tyVector(MVT Ty, MVT ElemTy) const { |
399 | if (Ty.isVector() && Ty.getVectorElementType() == ElemTy) |
400 | return Ty; |
401 | unsigned TyWidth = Ty.getSizeInBits(); |
402 | unsigned ElemWidth = ElemTy.getSizeInBits(); |
403 | assert((TyWidth % ElemWidth) == 0)(static_cast <bool> ((TyWidth % ElemWidth) == 0) ? void (0) : __assert_fail ("(TyWidth % ElemWidth) == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.h" , 403, __extension__ __PRETTY_FUNCTION__)); |
404 | return MVT::getVectorVT(ElemTy, TyWidth/ElemWidth); |
405 | } |
406 | |
407 | MVT typeJoin(const TypePair &Tys) const; |
408 | TypePair typeSplit(MVT Ty) const; |
409 | MVT typeExtElem(MVT VecTy, unsigned Factor) const; |
410 | MVT typeTruncElem(MVT VecTy, unsigned Factor) const; |
411 | |
412 | SDValue opJoin(const VectorPair &Ops, const SDLoc &dl, |
413 | SelectionDAG &DAG) const; |
414 | VectorPair opSplit(SDValue Vec, const SDLoc &dl, SelectionDAG &DAG) const; |
415 | SDValue opCastElem(SDValue Vec, MVT ElemTy, SelectionDAG &DAG) const; |
416 | |
417 | bool allowsHvxMemoryAccess(MVT VecTy, MachineMemOperand::Flags Flags, |
418 | bool *Fast) const; |
419 | bool allowsHvxMisalignedMemoryAccesses(MVT VecTy, |
420 | MachineMemOperand::Flags Flags, |
421 | bool *Fast) const; |
422 | |
423 | bool isHvxSingleTy(MVT Ty) const; |
424 | bool isHvxPairTy(MVT Ty) const; |
425 | bool isHvxBoolTy(MVT Ty) const; |
426 | SDValue convertToByteIndex(SDValue ElemIdx, MVT ElemTy, |
427 | SelectionDAG &DAG) const; |
428 | SDValue getIndexInWord32(SDValue Idx, MVT ElemTy, SelectionDAG &DAG) const; |
429 | SDValue getByteShuffle(const SDLoc &dl, SDValue Op0, SDValue Op1, |
430 | ArrayRef<int> Mask, SelectionDAG &DAG) const; |
431 | |
432 | SDValue buildHvxVectorReg(ArrayRef<SDValue> Values, const SDLoc &dl, |
433 | MVT VecTy, SelectionDAG &DAG) const; |
434 | SDValue buildHvxVectorPred(ArrayRef<SDValue> Values, const SDLoc &dl, |
435 | MVT VecTy, SelectionDAG &DAG) const; |
436 | SDValue createHvxPrefixPred(SDValue PredV, const SDLoc &dl, |
437 | unsigned BitBytes, bool ZeroFill, |
438 | SelectionDAG &DAG) const; |
439 | SDValue extractHvxElementReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
440 | MVT ResTy, SelectionDAG &DAG) const; |
441 | SDValue extractHvxElementPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
442 | MVT ResTy, SelectionDAG &DAG) const; |
443 | SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV, |
444 | const SDLoc &dl, SelectionDAG &DAG) const; |
445 | SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV, |
446 | const SDLoc &dl, SelectionDAG &DAG) const; |
447 | SDValue extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
448 | MVT ResTy, SelectionDAG &DAG) const; |
449 | SDValue extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
450 | MVT ResTy, SelectionDAG &DAG) const; |
451 | SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV, |
452 | const SDLoc &dl, SelectionDAG &DAG) const; |
453 | SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV, |
454 | const SDLoc &dl, SelectionDAG &DAG) const; |
455 | SDValue extendHvxVectorPred(SDValue VecV, const SDLoc &dl, MVT ResTy, |
456 | bool ZeroExt, SelectionDAG &DAG) const; |
457 | SDValue compressHvxPred(SDValue VecQ, const SDLoc &dl, MVT ResTy, |
458 | SelectionDAG &DAG) const; |
459 | |
460 | SDValue LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) const; |
461 | SDValue LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) const; |
462 | SDValue LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) const; |
463 | SDValue LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) const; |
464 | SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const; |
465 | SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const; |
466 | SDValue LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const; |
467 | SDValue LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const; |
468 | SDValue LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const; |
469 | SDValue LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const; |
470 | SDValue LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const; |
471 | SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const; |
472 | SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const; |
473 | SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const; |
474 | SDValue LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const; |
475 | SDValue LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const; |
476 | SDValue LowerHvxShift(SDValue Op, SelectionDAG &DAG) const; |
477 | SDValue LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const; |
478 | SDValue LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const; |
479 | |
480 | SDValue SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const; |
481 | SDValue SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const; |
482 | SDValue WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const; |
483 | SDValue WidenHvxStore(SDValue Op, SelectionDAG &DAG) const; |
484 | SDValue WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const; |
485 | SDValue WidenHvxExtend(SDValue Op, SelectionDAG &DAG) const; |
486 | SDValue WidenHvxTruncate(SDValue Op, SelectionDAG &DAG) const; |
487 | |
488 | std::pair<const TargetRegisterClass*, uint8_t> |
489 | findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) |
490 | const override; |
491 | |
492 | bool shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const; |
493 | bool isHvxOperation(SDNode *N, SelectionDAG &DAG) const; |
494 | SDValue LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const; |
495 | void LowerHvxOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, |
496 | SelectionDAG &DAG) const; |
497 | void ReplaceHvxNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, |
498 | SelectionDAG &DAG) const; |
499 | SDValue PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; |
500 | }; |
501 | |
502 | } // end namespace llvm |
503 | |
504 | #endif // LLVM_LIB_TARGET_HEXAGON_HEXAGONISELLOWERING_H |
1 | //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file declares the SDNode class and derived classes, which are used to | |||
10 | // represent the nodes and operations present in a SelectionDAG. These nodes | |||
11 | // and operations are machine code level operations, with some similarities to | |||
12 | // the GCC RTL representation. | |||
13 | // | |||
14 | // Clients should include the SelectionDAG.h file instead of this file directly. | |||
15 | // | |||
16 | //===----------------------------------------------------------------------===// | |||
17 | ||||
18 | #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
19 | #define LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
20 | ||||
21 | #include "llvm/ADT/APFloat.h" | |||
22 | #include "llvm/ADT/ArrayRef.h" | |||
23 | #include "llvm/ADT/BitVector.h" | |||
24 | #include "llvm/ADT/FoldingSet.h" | |||
25 | #include "llvm/ADT/GraphTraits.h" | |||
26 | #include "llvm/ADT/SmallPtrSet.h" | |||
27 | #include "llvm/ADT/SmallVector.h" | |||
28 | #include "llvm/ADT/ilist_node.h" | |||
29 | #include "llvm/ADT/iterator.h" | |||
30 | #include "llvm/ADT/iterator_range.h" | |||
31 | #include "llvm/CodeGen/ISDOpcodes.h" | |||
32 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
33 | #include "llvm/CodeGen/Register.h" | |||
34 | #include "llvm/CodeGen/ValueTypes.h" | |||
35 | #include "llvm/IR/Constants.h" | |||
36 | #include "llvm/IR/DebugLoc.h" | |||
37 | #include "llvm/IR/Instruction.h" | |||
38 | #include "llvm/IR/Instructions.h" | |||
39 | #include "llvm/IR/Metadata.h" | |||
40 | #include "llvm/IR/Operator.h" | |||
41 | #include "llvm/Support/AlignOf.h" | |||
42 | #include "llvm/Support/AtomicOrdering.h" | |||
43 | #include "llvm/Support/Casting.h" | |||
44 | #include "llvm/Support/ErrorHandling.h" | |||
45 | #include "llvm/Support/MachineValueType.h" | |||
46 | #include "llvm/Support/TypeSize.h" | |||
47 | #include <algorithm> | |||
48 | #include <cassert> | |||
49 | #include <climits> | |||
50 | #include <cstddef> | |||
51 | #include <cstdint> | |||
52 | #include <cstring> | |||
53 | #include <iterator> | |||
54 | #include <string> | |||
55 | #include <tuple> | |||
56 | ||||
57 | namespace llvm { | |||
58 | ||||
59 | class APInt; | |||
60 | class Constant; | |||
61 | template <typename T> struct DenseMapInfo; | |||
62 | class GlobalValue; | |||
63 | class MachineBasicBlock; | |||
64 | class MachineConstantPoolValue; | |||
65 | class MCSymbol; | |||
66 | class raw_ostream; | |||
67 | class SDNode; | |||
68 | class SelectionDAG; | |||
69 | class Type; | |||
70 | class Value; | |||
71 | ||||
72 | void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, | |||
73 | bool force = false); | |||
74 | ||||
75 | /// This represents a list of ValueType's that has been intern'd by | |||
76 | /// a SelectionDAG. Instances of this simple value class are returned by | |||
77 | /// SelectionDAG::getVTList(...). | |||
78 | /// | |||
79 | struct SDVTList { | |||
80 | const EVT *VTs; | |||
81 | unsigned int NumVTs; | |||
82 | }; | |||
83 | ||||
84 | namespace ISD { | |||
85 | ||||
86 | /// Node predicates | |||
87 | ||||
88 | /// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the | |||
89 | /// same constant or undefined, return true and return the constant value in | |||
90 | /// \p SplatValue. | |||
91 | bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); | |||
92 | ||||
93 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
94 | /// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to | |||
95 | /// true, it only checks BUILD_VECTOR. | |||
96 | bool isConstantSplatVectorAllOnes(const SDNode *N, | |||
97 | bool BuildVectorOnly = false); | |||
98 | ||||
99 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
100 | /// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it | |||
101 | /// only checks BUILD_VECTOR. | |||
102 | bool isConstantSplatVectorAllZeros(const SDNode *N, | |||
103 | bool BuildVectorOnly = false); | |||
104 | ||||
105 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
106 | /// elements are ~0 or undef. | |||
107 | bool isBuildVectorAllOnes(const SDNode *N); | |||
108 | ||||
109 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
110 | /// elements are 0 or undef. | |||
111 | bool isBuildVectorAllZeros(const SDNode *N); | |||
112 | ||||
113 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
114 | /// ConstantSDNode or undef. | |||
115 | bool isBuildVectorOfConstantSDNodes(const SDNode *N); | |||
116 | ||||
117 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
118 | /// ConstantFPSDNode or undef. | |||
119 | bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); | |||
120 | ||||
121 | /// Return true if the node has at least one operand and all operands of the | |||
122 | /// specified node are ISD::UNDEF. | |||
123 | bool allOperandsUndef(const SDNode *N); | |||
124 | ||||
125 | } // end namespace ISD | |||
126 | ||||
127 | //===----------------------------------------------------------------------===// | |||
128 | /// Unlike LLVM values, Selection DAG nodes may return multiple | |||
129 | /// values as the result of a computation. Many nodes return multiple values, | |||
130 | /// from loads (which define a token and a return value) to ADDC (which returns | |||
131 | /// a result and a carry value), to calls (which may return an arbitrary number | |||
132 | /// of values). | |||
133 | /// | |||
134 | /// As such, each use of a SelectionDAG computation must indicate the node that | |||
135 | /// computes it as well as which return value to use from that node. This pair | |||
136 | /// of information is represented with the SDValue value type. | |||
137 | /// | |||
138 | class SDValue { | |||
139 | friend struct DenseMapInfo<SDValue>; | |||
140 | ||||
141 | SDNode *Node = nullptr; // The node defining the value we are using. | |||
142 | unsigned ResNo = 0; // Which return value of the node we are using. | |||
143 | ||||
144 | public: | |||
145 | SDValue() = default; | |||
146 | SDValue(SDNode *node, unsigned resno); | |||
147 | ||||
148 | /// get the index which selects a specific result in the SDNode | |||
149 | unsigned getResNo() const { return ResNo; } | |||
150 | ||||
151 | /// get the SDNode which holds the desired result | |||
152 | SDNode *getNode() const { return Node; } | |||
153 | ||||
154 | /// set the SDNode | |||
155 | void setNode(SDNode *N) { Node = N; } | |||
156 | ||||
157 | inline SDNode *operator->() const { return Node; } | |||
158 | ||||
159 | bool operator==(const SDValue &O) const { | |||
160 | return Node == O.Node && ResNo == O.ResNo; | |||
161 | } | |||
162 | bool operator!=(const SDValue &O) const { | |||
163 | return !operator==(O); | |||
164 | } | |||
165 | bool operator<(const SDValue &O) const { | |||
166 | return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); | |||
167 | } | |||
168 | explicit operator bool() const { | |||
169 | return Node != nullptr; | |||
170 | } | |||
171 | ||||
172 | SDValue getValue(unsigned R) const { | |||
173 | return SDValue(Node, R); | |||
174 | } | |||
175 | ||||
176 | /// Return true if this node is an operand of N. | |||
177 | bool isOperandOf(const SDNode *N) const; | |||
178 | ||||
179 | /// Return the ValueType of the referenced return value. | |||
180 | inline EVT getValueType() const; | |||
181 | ||||
182 | /// Return the simple ValueType of the referenced return value. | |||
183 | MVT getSimpleValueType() const { | |||
184 | return getValueType().getSimpleVT(); | |||
185 | } | |||
186 | ||||
187 | /// Returns the size of the value in bits. | |||
188 | /// | |||
189 | /// If the value type is a scalable vector type, the scalable property will | |||
190 | /// be set and the runtime size will be a positive integer multiple of the | |||
191 | /// base size. | |||
192 | TypeSize getValueSizeInBits() const { | |||
193 | return getValueType().getSizeInBits(); | |||
194 | } | |||
195 | ||||
196 | uint64_t getScalarValueSizeInBits() const { | |||
197 | return getValueType().getScalarType().getFixedSizeInBits(); | |||
198 | } | |||
199 | ||||
200 | // Forwarding methods - These forward to the corresponding methods in SDNode. | |||
201 | inline unsigned getOpcode() const; | |||
202 | inline unsigned getNumOperands() const; | |||
203 | inline const SDValue &getOperand(unsigned i) const; | |||
204 | inline uint64_t getConstantOperandVal(unsigned i) const; | |||
205 | inline const APInt &getConstantOperandAPInt(unsigned i) const; | |||
206 | inline bool isTargetMemoryOpcode() const; | |||
207 | inline bool isTargetOpcode() const; | |||
208 | inline bool isMachineOpcode() const; | |||
209 | inline bool isUndef() const; | |||
210 | inline unsigned getMachineOpcode() const; | |||
211 | inline const DebugLoc &getDebugLoc() const; | |||
212 | inline void dump() const; | |||
213 | inline void dump(const SelectionDAG *G) const; | |||
214 | inline void dumpr() const; | |||
215 | inline void dumpr(const SelectionDAG *G) const; | |||
216 | ||||
217 | /// Return true if this operand (which must be a chain) reaches the | |||
218 | /// specified operand without crossing any side-effecting instructions. | |||
219 | /// In practice, this looks through token factors and non-volatile loads. | |||
220 | /// In order to remain efficient, this only | |||
221 | /// looks a couple of nodes in, it does not do an exhaustive search. | |||
222 | bool reachesChainWithoutSideEffects(SDValue Dest, | |||
223 | unsigned Depth = 2) const; | |||
224 | ||||
225 | /// Return true if there are no nodes using value ResNo of Node. | |||
226 | inline bool use_empty() const; | |||
227 | ||||
228 | /// Return true if there is exactly one node using value ResNo of Node. | |||
229 | inline bool hasOneUse() const; | |||
230 | }; | |||
231 | ||||
232 | template<> struct DenseMapInfo<SDValue> { | |||
233 | static inline SDValue getEmptyKey() { | |||
234 | SDValue V; | |||
235 | V.ResNo = -1U; | |||
236 | return V; | |||
237 | } | |||
238 | ||||
239 | static inline SDValue getTombstoneKey() { | |||
240 | SDValue V; | |||
241 | V.ResNo = -2U; | |||
242 | return V; | |||
243 | } | |||
244 | ||||
245 | static unsigned getHashValue(const SDValue &Val) { | |||
246 | return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ | |||
247 | (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo(); | |||
248 | } | |||
249 | ||||
250 | static bool isEqual(const SDValue &LHS, const SDValue &RHS) { | |||
251 | return LHS == RHS; | |||
252 | } | |||
253 | }; | |||
254 | ||||
255 | /// Allow casting operators to work directly on | |||
256 | /// SDValues as if they were SDNode*'s. | |||
257 | template<> struct simplify_type<SDValue> { | |||
258 | using SimpleType = SDNode *; | |||
259 | ||||
260 | static SimpleType getSimplifiedValue(SDValue &Val) { | |||
261 | return Val.getNode(); | |||
262 | } | |||
263 | }; | |||
264 | template<> struct simplify_type<const SDValue> { | |||
265 | using SimpleType = /*const*/ SDNode *; | |||
266 | ||||
267 | static SimpleType getSimplifiedValue(const SDValue &Val) { | |||
268 | return Val.getNode(); | |||
269 | } | |||
270 | }; | |||
271 | ||||
272 | /// Represents a use of a SDNode. This class holds an SDValue, | |||
273 | /// which records the SDNode being used and the result number, a | |||
274 | /// pointer to the SDNode using the value, and Next and Prev pointers, | |||
275 | /// which link together all the uses of an SDNode. | |||
276 | /// | |||
277 | class SDUse { | |||
278 | /// Val - The value being used. | |||
279 | SDValue Val; | |||
280 | /// User - The user of this value. | |||
281 | SDNode *User = nullptr; | |||
282 | /// Prev, Next - Pointers to the uses list of the SDNode referred by | |||
283 | /// this operand. | |||
284 | SDUse **Prev = nullptr; | |||
285 | SDUse *Next = nullptr; | |||
286 | ||||
287 | public: | |||
288 | SDUse() = default; | |||
289 | SDUse(const SDUse &U) = delete; | |||
290 | SDUse &operator=(const SDUse &) = delete; | |||
291 | ||||
292 | /// Normally SDUse will just implicitly convert to an SDValue that it holds. | |||
293 | operator const SDValue&() const { return Val; } | |||
294 | ||||
295 | /// If implicit conversion to SDValue doesn't work, the get() method returns | |||
296 | /// the SDValue. | |||
297 | const SDValue &get() const { return Val; } | |||
298 | ||||
299 | /// This returns the SDNode that contains this Use. | |||
300 | SDNode *getUser() { return User; } | |||
301 | ||||
302 | /// Get the next SDUse in the use list. | |||
303 | SDUse *getNext() const { return Next; } | |||
304 | ||||
305 | /// Convenience function for get().getNode(). | |||
306 | SDNode *getNode() const { return Val.getNode(); } | |||
307 | /// Convenience function for get().getResNo(). | |||
308 | unsigned getResNo() const { return Val.getResNo(); } | |||
309 | /// Convenience function for get().getValueType(). | |||
310 | EVT getValueType() const { return Val.getValueType(); } | |||
311 | ||||
312 | /// Convenience function for get().operator== | |||
313 | bool operator==(const SDValue &V) const { | |||
314 | return Val == V; | |||
315 | } | |||
316 | ||||
317 | /// Convenience function for get().operator!= | |||
318 | bool operator!=(const SDValue &V) const { | |||
319 | return Val != V; | |||
320 | } | |||
321 | ||||
322 | /// Convenience function for get().operator< | |||
323 | bool operator<(const SDValue &V) const { | |||
324 | return Val < V; | |||
325 | } | |||
326 | ||||
327 | private: | |||
328 | friend class SelectionDAG; | |||
329 | friend class SDNode; | |||
330 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
331 | friend class HandleSDNode; | |||
332 | ||||
333 | void setUser(SDNode *p) { User = p; } | |||
334 | ||||
335 | /// Remove this use from its existing use list, assign it the | |||
336 | /// given value, and add it to the new value's node's use list. | |||
337 | inline void set(const SDValue &V); | |||
338 | /// Like set, but only supports initializing a newly-allocated | |||
339 | /// SDUse with a non-null value. | |||
340 | inline void setInitial(const SDValue &V); | |||
341 | /// Like set, but only sets the Node portion of the value, | |||
342 | /// leaving the ResNo portion unmodified. | |||
343 | inline void setNode(SDNode *N); | |||
344 | ||||
345 | void addToList(SDUse **List) { | |||
346 | Next = *List; | |||
347 | if (Next) Next->Prev = &Next; | |||
348 | Prev = List; | |||
349 | *List = this; | |||
350 | } | |||
351 | ||||
352 | void removeFromList() { | |||
353 | *Prev = Next; | |||
354 | if (Next) Next->Prev = Prev; | |||
355 | } | |||
356 | }; | |||
357 | ||||
358 | /// simplify_type specializations - Allow casting operators to work directly on | |||
359 | /// SDValues as if they were SDNode*'s. | |||
360 | template<> struct simplify_type<SDUse> { | |||
361 | using SimpleType = SDNode *; | |||
362 | ||||
363 | static SimpleType getSimplifiedValue(SDUse &Val) { | |||
364 | return Val.getNode(); | |||
365 | } | |||
366 | }; | |||
367 | ||||
368 | /// These are IR-level optimization flags that may be propagated to SDNodes. | |||
369 | /// TODO: This data structure should be shared by the IR optimizer and the | |||
370 | /// the backend. | |||
371 | struct SDNodeFlags { | |||
372 | private: | |||
373 | bool NoUnsignedWrap : 1; | |||
374 | bool NoSignedWrap : 1; | |||
375 | bool Exact : 1; | |||
376 | bool NoNaNs : 1; | |||
377 | bool NoInfs : 1; | |||
378 | bool NoSignedZeros : 1; | |||
379 | bool AllowReciprocal : 1; | |||
380 | bool AllowContract : 1; | |||
381 | bool ApproximateFuncs : 1; | |||
382 | bool AllowReassociation : 1; | |||
383 | ||||
384 | // We assume instructions do not raise floating-point exceptions by default, | |||
385 | // and only those marked explicitly may do so. We could choose to represent | |||
386 | // this via a positive "FPExcept" flags like on the MI level, but having a | |||
387 | // negative "NoFPExcept" flag here (that defaults to true) makes the flag | |||
388 | // intersection logic more straightforward. | |||
389 | bool NoFPExcept : 1; | |||
390 | ||||
391 | public: | |||
392 | /// Default constructor turns off all optimization flags. | |||
393 | SDNodeFlags() | |||
394 | : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false), | |||
395 | NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), | |||
396 | AllowContract(false), ApproximateFuncs(false), | |||
397 | AllowReassociation(false), NoFPExcept(false) {} | |||
398 | ||||
399 | /// Propagate the fast-math-flags from an IR FPMathOperator. | |||
400 | void copyFMF(const FPMathOperator &FPMO) { | |||
401 | setNoNaNs(FPMO.hasNoNaNs()); | |||
402 | setNoInfs(FPMO.hasNoInfs()); | |||
403 | setNoSignedZeros(FPMO.hasNoSignedZeros()); | |||
404 | setAllowReciprocal(FPMO.hasAllowReciprocal()); | |||
405 | setAllowContract(FPMO.hasAllowContract()); | |||
406 | setApproximateFuncs(FPMO.hasApproxFunc()); | |||
407 | setAllowReassociation(FPMO.hasAllowReassoc()); | |||
408 | } | |||
409 | ||||
410 | // These are mutators for each flag. | |||
411 | void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } | |||
412 | void setNoSignedWrap(bool b) { NoSignedWrap = b; } | |||
413 | void setExact(bool b) { Exact = b; } | |||
414 | void setNoNaNs(bool b) { NoNaNs = b; } | |||
415 | void setNoInfs(bool b) { NoInfs = b; } | |||
416 | void setNoSignedZeros(bool b) { NoSignedZeros = b; } | |||
417 | void setAllowReciprocal(bool b) { AllowReciprocal = b; } | |||
418 | void setAllowContract(bool b) { AllowContract = b; } | |||
419 | void setApproximateFuncs(bool b) { ApproximateFuncs = b; } | |||
420 | void setAllowReassociation(bool b) { AllowReassociation = b; } | |||
421 | void setNoFPExcept(bool b) { NoFPExcept = b; } | |||
422 | ||||
423 | // These are accessors for each flag. | |||
424 | bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } | |||
425 | bool hasNoSignedWrap() const { return NoSignedWrap; } | |||
426 | bool hasExact() const { return Exact; } | |||
427 | bool hasNoNaNs() const { return NoNaNs; } | |||
428 | bool hasNoInfs() const { return NoInfs; } | |||
429 | bool hasNoSignedZeros() const { return NoSignedZeros; } | |||
430 | bool hasAllowReciprocal() const { return AllowReciprocal; } | |||
431 | bool hasAllowContract() const { return AllowContract; } | |||
432 | bool hasApproximateFuncs() const { return ApproximateFuncs; } | |||
433 | bool hasAllowReassociation() const { return AllowReassociation; } | |||
434 | bool hasNoFPExcept() const { return NoFPExcept; } | |||
435 | ||||
436 | /// Clear any flags in this flag set that aren't also set in Flags. All | |||
437 | /// flags will be cleared if Flags are undefined. | |||
438 | void intersectWith(const SDNodeFlags Flags) { | |||
439 | NoUnsignedWrap &= Flags.NoUnsignedWrap; | |||
440 | NoSignedWrap &= Flags.NoSignedWrap; | |||
441 | Exact &= Flags.Exact; | |||
442 | NoNaNs &= Flags.NoNaNs; | |||
443 | NoInfs &= Flags.NoInfs; | |||
444 | NoSignedZeros &= Flags.NoSignedZeros; | |||
445 | AllowReciprocal &= Flags.AllowReciprocal; | |||
446 | AllowContract &= Flags.AllowContract; | |||
447 | ApproximateFuncs &= Flags.ApproximateFuncs; | |||
448 | AllowReassociation &= Flags.AllowReassociation; | |||
449 | NoFPExcept &= Flags.NoFPExcept; | |||
450 | } | |||
451 | }; | |||
452 | ||||
453 | /// Represents one node in the SelectionDAG. | |||
454 | /// | |||
455 | class SDNode : public FoldingSetNode, public ilist_node<SDNode> { | |||
456 | private: | |||
457 | /// The operation that this node performs. | |||
458 | int16_t NodeType; | |||
459 | ||||
460 | protected: | |||
461 | // We define a set of mini-helper classes to help us interpret the bits in our | |||
462 | // SubclassData. These are designed to fit within a uint16_t so they pack | |||
463 | // with NodeType. | |||
464 | ||||
465 | #if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1)) | |||
466 | // Except for GCC; by default, AIX compilers store bit-fields in 4-byte words | |||
467 | // and give the `pack` pragma push semantics. | |||
468 | #define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2) | |||
469 | #define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop) | |||
470 | #else | |||
471 | #define BEGIN_TWO_BYTE_PACK() | |||
472 | #define END_TWO_BYTE_PACK() | |||
473 | #endif | |||
474 | ||||
475 | BEGIN_TWO_BYTE_PACK() | |||
476 | class SDNodeBitfields { | |||
477 | friend class SDNode; | |||
478 | friend class MemIntrinsicSDNode; | |||
479 | friend class MemSDNode; | |||
480 | friend class SelectionDAG; | |||
481 | ||||
482 | uint16_t HasDebugValue : 1; | |||
483 | uint16_t IsMemIntrinsic : 1; | |||
484 | uint16_t IsDivergent : 1; | |||
485 | }; | |||
486 | enum { NumSDNodeBits = 3 }; | |||
487 | ||||
488 | class ConstantSDNodeBitfields { | |||
489 | friend class ConstantSDNode; | |||
490 | ||||
491 | uint16_t : NumSDNodeBits; | |||
492 | ||||
493 | uint16_t IsOpaque : 1; | |||
494 | }; | |||
495 | ||||
496 | class MemSDNodeBitfields { | |||
497 | friend class MemSDNode; | |||
498 | friend class MemIntrinsicSDNode; | |||
499 | friend class AtomicSDNode; | |||
500 | ||||
501 | uint16_t : NumSDNodeBits; | |||
502 | ||||
503 | uint16_t IsVolatile : 1; | |||
504 | uint16_t IsNonTemporal : 1; | |||
505 | uint16_t IsDereferenceable : 1; | |||
506 | uint16_t IsInvariant : 1; | |||
507 | }; | |||
508 | enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; | |||
509 | ||||
510 | class LSBaseSDNodeBitfields { | |||
511 | friend class LSBaseSDNode; | |||
512 | friend class MaskedLoadStoreSDNode; | |||
513 | friend class MaskedGatherScatterSDNode; | |||
514 | ||||
515 | uint16_t : NumMemSDNodeBits; | |||
516 | ||||
517 | // This storage is shared between disparate class hierarchies to hold an | |||
518 | // enumeration specific to the class hierarchy in use. | |||
519 | // LSBaseSDNode => enum ISD::MemIndexedMode | |||
520 | // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode | |||
521 | // MaskedGatherScatterSDNode => enum ISD::MemIndexType | |||
522 | uint16_t AddressingMode : 3; | |||
523 | }; | |||
524 | enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; | |||
525 | ||||
526 | class LoadSDNodeBitfields { | |||
527 | friend class LoadSDNode; | |||
528 | friend class MaskedLoadSDNode; | |||
529 | friend class MaskedGatherSDNode; | |||
530 | ||||
531 | uint16_t : NumLSBaseSDNodeBits; | |||
532 | ||||
533 | uint16_t ExtTy : 2; // enum ISD::LoadExtType | |||
534 | uint16_t IsExpanding : 1; | |||
535 | }; | |||
536 | ||||
537 | class StoreSDNodeBitfields { | |||
538 | friend class StoreSDNode; | |||
539 | friend class MaskedStoreSDNode; | |||
540 | friend class MaskedScatterSDNode; | |||
541 | ||||
542 | uint16_t : NumLSBaseSDNodeBits; | |||
543 | ||||
544 | uint16_t IsTruncating : 1; | |||
545 | uint16_t IsCompressing : 1; | |||
546 | }; | |||
547 | ||||
548 | union { | |||
549 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
550 | SDNodeBitfields SDNodeBits; | |||
551 | ConstantSDNodeBitfields ConstantSDNodeBits; | |||
552 | MemSDNodeBitfields MemSDNodeBits; | |||
553 | LSBaseSDNodeBitfields LSBaseSDNodeBits; | |||
554 | LoadSDNodeBitfields LoadSDNodeBits; | |||
555 | StoreSDNodeBitfields StoreSDNodeBits; | |||
556 | }; | |||
557 | END_TWO_BYTE_PACK() | |||
558 | #undef BEGIN_TWO_BYTE_PACK | |||
559 | #undef END_TWO_BYTE_PACK | |||
560 | ||||
561 | // RawSDNodeBits must cover the entirety of the union. This means that all of | |||
562 | // the union's members must have size <= RawSDNodeBits. We write the RHS as | |||
563 | // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter. | |||
564 | static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide"); | |||
565 | static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide"); | |||
566 | static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide"); | |||
567 | static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide"); | |||
568 | static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide"); | |||
569 | static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide"); | |||
570 | ||||
571 | private: | |||
572 | friend class SelectionDAG; | |||
573 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
574 | friend class HandleSDNode; | |||
575 | ||||
576 | /// Unique id per SDNode in the DAG. | |||
577 | int NodeId = -1; | |||
578 | ||||
579 | /// The values that are used by this operation. | |||
580 | SDUse *OperandList = nullptr; | |||
581 | ||||
582 | /// The types of the values this node defines. SDNode's may | |||
583 | /// define multiple values simultaneously. | |||
584 | const EVT *ValueList; | |||
585 | ||||
586 | /// List of uses for this SDNode. | |||
587 | SDUse *UseList = nullptr; | |||
588 | ||||
589 | /// The number of entries in the Operand/Value list. | |||
590 | unsigned short NumOperands = 0; | |||
591 | unsigned short NumValues; | |||
592 | ||||
593 | // The ordering of the SDNodes. It roughly corresponds to the ordering of the | |||
594 | // original LLVM instructions. | |||
595 | // This is used for turning off scheduling, because we'll forgo | |||
596 | // the normal scheduling algorithms and output the instructions according to | |||
597 | // this ordering. | |||
598 | unsigned IROrder; | |||
599 | ||||
600 | /// Source line information. | |||
601 | DebugLoc debugLoc; | |||
602 | ||||
603 | /// Return a pointer to the specified value type. | |||
604 | static const EVT *getValueTypeList(EVT VT); | |||
605 | ||||
606 | SDNodeFlags Flags; | |||
607 | ||||
608 | public: | |||
609 | /// Unique and persistent id per SDNode in the DAG. | |||
610 | /// Used for debug printing. | |||
611 | uint16_t PersistentId; | |||
612 | ||||
613 | //===--------------------------------------------------------------------===// | |||
614 | // Accessors | |||
615 | // | |||
616 | ||||
617 | /// Return the SelectionDAG opcode value for this node. For | |||
618 | /// pre-isel nodes (those for which isMachineOpcode returns false), these | |||
619 | /// are the opcode values in the ISD and <target>ISD namespaces. For | |||
620 | /// post-isel opcodes, see getMachineOpcode. | |||
621 | unsigned getOpcode() const { return (unsigned short)NodeType; } | |||
622 | ||||
623 | /// Test if this node has a target-specific opcode (in the | |||
624 | /// \<target\>ISD namespace). | |||
625 | bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } | |||
626 | ||||
627 | /// Test if this node has a target-specific opcode that may raise | |||
628 | /// FP exceptions (in the \<target\>ISD namespace and greater than | |||
629 | /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory | |||
630 | /// opcode are currently automatically considered to possibly raise | |||
631 | /// FP exceptions as well. | |||
632 | bool isTargetStrictFPOpcode() const { | |||
633 | return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE; | |||
634 | } | |||
635 | ||||
636 | /// Test if this node has a target-specific | |||
637 | /// memory-referencing opcode (in the \<target\>ISD namespace and | |||
638 | /// greater than FIRST_TARGET_MEMORY_OPCODE). | |||
639 | bool isTargetMemoryOpcode() const { | |||
640 | return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; | |||
641 | } | |||
642 | ||||
643 | /// Return true if the type of the node type undefined. | |||
644 | bool isUndef() const { return NodeType == ISD::UNDEF; } | |||
645 | ||||
646 | /// Test if this node is a memory intrinsic (with valid pointer information). | |||
647 | /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for | |||
648 | /// non-memory intrinsics (with chains) that are not really instances of | |||
649 | /// MemSDNode. For such nodes, we need some extra state to determine the | |||
650 | /// proper classof relationship. | |||
651 | bool isMemIntrinsic() const { | |||
652 | return (NodeType == ISD::INTRINSIC_W_CHAIN || | |||
653 | NodeType == ISD::INTRINSIC_VOID) && | |||
654 | SDNodeBits.IsMemIntrinsic; | |||
655 | } | |||
656 | ||||
657 | /// Test if this node is a strict floating point pseudo-op. | |||
658 | bool isStrictFPOpcode() { | |||
659 | switch (NodeType) { | |||
660 | default: | |||
661 | return false; | |||
662 | case ISD::STRICT_FP16_TO_FP: | |||
663 | case ISD::STRICT_FP_TO_FP16: | |||
664 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | |||
665 | case ISD::STRICT_##DAGN: | |||
666 | #include "llvm/IR/ConstrainedOps.def" | |||
667 | return true; | |||
668 | } | |||
669 | } | |||
670 | ||||
671 | /// Test if this node has a post-isel opcode, directly | |||
672 | /// corresponding to a MachineInstr opcode. | |||
673 | bool isMachineOpcode() const { return NodeType < 0; } | |||
674 | ||||
675 | /// This may only be called if isMachineOpcode returns | |||
676 | /// true. It returns the MachineInstr opcode value that the node's opcode | |||
677 | /// corresponds to. | |||
678 | unsigned getMachineOpcode() const { | |||
679 | assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!" ) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 679, __extension__ __PRETTY_FUNCTION__)); | |||
680 | return ~NodeType; | |||
681 | } | |||
682 | ||||
683 | bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } | |||
684 | void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } | |||
685 | ||||
686 | bool isDivergent() const { return SDNodeBits.IsDivergent; } | |||
687 | ||||
688 | /// Return true if there are no uses of this node. | |||
689 | bool use_empty() const { return UseList == nullptr; } | |||
690 | ||||
691 | /// Return true if there is exactly one use of this node. | |||
692 | bool hasOneUse() const { return hasSingleElement(uses()); } | |||
693 | ||||
694 | /// Return the number of uses of this node. This method takes | |||
695 | /// time proportional to the number of uses. | |||
696 | size_t use_size() const { return std::distance(use_begin(), use_end()); } | |||
697 | ||||
698 | /// Return the unique node id. | |||
699 | int getNodeId() const { return NodeId; } | |||
700 | ||||
701 | /// Set unique node id. | |||
702 | void setNodeId(int Id) { NodeId = Id; } | |||
703 | ||||
704 | /// Return the node ordering. | |||
705 | unsigned getIROrder() const { return IROrder; } | |||
706 | ||||
707 | /// Set the node ordering. | |||
708 | void setIROrder(unsigned Order) { IROrder = Order; } | |||
709 | ||||
710 | /// Return the source location info. | |||
711 | const DebugLoc &getDebugLoc() const { return debugLoc; } | |||
712 | ||||
713 | /// Set source location info. Try to avoid this, putting | |||
714 | /// it in the constructor is preferable. | |||
715 | void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } | |||
716 | ||||
717 | /// This class provides iterator support for SDUse | |||
718 | /// operands that use a specific SDNode. | |||
719 | class use_iterator { | |||
720 | friend class SDNode; | |||
721 | ||||
722 | SDUse *Op = nullptr; | |||
723 | ||||
724 | explicit use_iterator(SDUse *op) : Op(op) {} | |||
725 | ||||
726 | public: | |||
727 | using iterator_category = std::forward_iterator_tag; | |||
728 | using value_type = SDUse; | |||
729 | using difference_type = std::ptrdiff_t; | |||
730 | using pointer = value_type *; | |||
731 | using reference = value_type &; | |||
732 | ||||
733 | use_iterator() = default; | |||
734 | use_iterator(const use_iterator &I) : Op(I.Op) {} | |||
735 | ||||
736 | bool operator==(const use_iterator &x) const { | |||
737 | return Op == x.Op; | |||
738 | } | |||
739 | bool operator!=(const use_iterator &x) const { | |||
740 | return !operator==(x); | |||
741 | } | |||
742 | ||||
743 | /// Return true if this iterator is at the end of uses list. | |||
744 | bool atEnd() const { return Op == nullptr; } | |||
745 | ||||
746 | // Iterator traversal: forward iteration only. | |||
747 | use_iterator &operator++() { // Preincrement | |||
748 | assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 748, __extension__ __PRETTY_FUNCTION__)); | |||
749 | Op = Op->getNext(); | |||
750 | return *this; | |||
751 | } | |||
752 | ||||
753 | use_iterator operator++(int) { // Postincrement | |||
754 | use_iterator tmp = *this; ++*this; return tmp; | |||
755 | } | |||
756 | ||||
757 | /// Retrieve a pointer to the current user node. | |||
758 | SDNode *operator*() const { | |||
759 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 759, __extension__ __PRETTY_FUNCTION__)); | |||
760 | return Op->getUser(); | |||
761 | } | |||
762 | ||||
763 | SDNode *operator->() const { return operator*(); } | |||
764 | ||||
765 | SDUse &getUse() const { return *Op; } | |||
766 | ||||
767 | /// Retrieve the operand # of this use in its user. | |||
768 | unsigned getOperandNo() const { | |||
769 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 769, __extension__ __PRETTY_FUNCTION__)); | |||
770 | return (unsigned)(Op - Op->getUser()->OperandList); | |||
771 | } | |||
772 | }; | |||
773 | ||||
774 | /// Provide iteration support to walk over all uses of an SDNode. | |||
775 | use_iterator use_begin() const { | |||
776 | return use_iterator(UseList); | |||
777 | } | |||
778 | ||||
779 | static use_iterator use_end() { return use_iterator(nullptr); } | |||
780 | ||||
781 | inline iterator_range<use_iterator> uses() { | |||
782 | return make_range(use_begin(), use_end()); | |||
783 | } | |||
784 | inline iterator_range<use_iterator> uses() const { | |||
785 | return make_range(use_begin(), use_end()); | |||
786 | } | |||
787 | ||||
788 | /// Return true if there are exactly NUSES uses of the indicated value. | |||
789 | /// This method ignores uses of other values defined by this operation. | |||
790 | bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; | |||
791 | ||||
792 | /// Return true if there are any use of the indicated value. | |||
793 | /// This method ignores uses of other values defined by this operation. | |||
794 | bool hasAnyUseOfValue(unsigned Value) const; | |||
795 | ||||
796 | /// Return true if this node is the only use of N. | |||
797 | bool isOnlyUserOf(const SDNode *N) const; | |||
798 | ||||
799 | /// Return true if this node is an operand of N. | |||
800 | bool isOperandOf(const SDNode *N) const; | |||
801 | ||||
802 | /// Return true if this node is a predecessor of N. | |||
803 | /// NOTE: Implemented on top of hasPredecessor and every bit as | |||
804 | /// expensive. Use carefully. | |||
805 | bool isPredecessorOf(const SDNode *N) const { | |||
806 | return N->hasPredecessor(this); | |||
807 | } | |||
808 | ||||
809 | /// Return true if N is a predecessor of this node. | |||
810 | /// N is either an operand of this node, or can be reached by recursively | |||
811 | /// traversing up the operands. | |||
812 | /// NOTE: This is an expensive method. Use it carefully. | |||
813 | bool hasPredecessor(const SDNode *N) const; | |||
814 | ||||
815 | /// Returns true if N is a predecessor of any node in Worklist. This | |||
816 | /// helper keeps Visited and Worklist sets externally to allow unions | |||
817 | /// searches to be performed in parallel, caching of results across | |||
818 | /// queries and incremental addition to Worklist. Stops early if N is | |||
819 | /// found but will resume. Remember to clear Visited and Worklists | |||
820 | /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before | |||
821 | /// giving up. The TopologicalPrune flag signals that positive NodeIds are | |||
822 | /// topologically ordered (Operands have strictly smaller node id) and search | |||
823 | /// can be pruned leveraging this. | |||
824 | static bool hasPredecessorHelper(const SDNode *N, | |||
825 | SmallPtrSetImpl<const SDNode *> &Visited, | |||
826 | SmallVectorImpl<const SDNode *> &Worklist, | |||
827 | unsigned int MaxSteps = 0, | |||
828 | bool TopologicalPrune = false) { | |||
829 | SmallVector<const SDNode *, 8> DeferredNodes; | |||
830 | if (Visited.count(N)) | |||
831 | return true; | |||
832 | ||||
833 | // Node Id's are assigned in three places: As a topological | |||
834 | // ordering (> 0), during legalization (results in values set to | |||
835 | // 0), new nodes (set to -1). If N has a topolgical id then we | |||
836 | // know that all nodes with ids smaller than it cannot be | |||
837 | // successors and we need not check them. Filter out all node | |||
838 | // that can't be matches. We add them to the worklist before exit | |||
839 | // in case of multiple calls. Note that during selection the topological id | |||
840 | // may be violated if a node's predecessor is selected before it. We mark | |||
841 | // this at selection negating the id of unselected successors and | |||
842 | // restricting topological pruning to positive ids. | |||
843 | ||||
844 | int NId = N->getNodeId(); | |||
845 | // If we Invalidated the Id, reconstruct original NId. | |||
846 | if (NId < -1) | |||
847 | NId = -(NId + 1); | |||
848 | ||||
849 | bool Found = false; | |||
850 | while (!Worklist.empty()) { | |||
851 | const SDNode *M = Worklist.pop_back_val(); | |||
852 | int MId = M->getNodeId(); | |||
853 | if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) && | |||
854 | (MId > 0) && (MId < NId)) { | |||
855 | DeferredNodes.push_back(M); | |||
856 | continue; | |||
857 | } | |||
858 | for (const SDValue &OpV : M->op_values()) { | |||
859 | SDNode *Op = OpV.getNode(); | |||
860 | if (Visited.insert(Op).second) | |||
861 | Worklist.push_back(Op); | |||
862 | if (Op == N) | |||
863 | Found = true; | |||
864 | } | |||
865 | if (Found) | |||
866 | break; | |||
867 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
868 | break; | |||
869 | } | |||
870 | // Push deferred nodes back on worklist. | |||
871 | Worklist.append(DeferredNodes.begin(), DeferredNodes.end()); | |||
872 | // If we bailed early, conservatively return found. | |||
873 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
874 | return true; | |||
875 | return Found; | |||
876 | } | |||
877 | ||||
878 | /// Return true if all the users of N are contained in Nodes. | |||
879 | /// NOTE: Requires at least one match, but doesn't require them all. | |||
880 | static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N); | |||
881 | ||||
882 | /// Return the number of values used by this operation. | |||
883 | unsigned getNumOperands() const { return NumOperands; } | |||
884 | ||||
885 | /// Return the maximum number of operands that a SDNode can hold. | |||
886 | static constexpr size_t getMaxNumOperands() { | |||
887 | return std::numeric_limits<decltype(SDNode::NumOperands)>::max(); | |||
888 | } | |||
889 | ||||
890 | /// Helper method returns the integer value of a ConstantSDNode operand. | |||
891 | inline uint64_t getConstantOperandVal(unsigned Num) const; | |||
892 | ||||
893 | /// Helper method returns the APInt of a ConstantSDNode operand. | |||
894 | inline const APInt &getConstantOperandAPInt(unsigned Num) const; | |||
895 | ||||
896 | const SDValue &getOperand(unsigned Num) const { | |||
897 | assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!" ) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 897, __extension__ __PRETTY_FUNCTION__)); | |||
898 | return OperandList[Num]; | |||
899 | } | |||
900 | ||||
901 | using op_iterator = SDUse *; | |||
902 | ||||
903 | op_iterator op_begin() const { return OperandList; } | |||
904 | op_iterator op_end() const { return OperandList+NumOperands; } | |||
905 | ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); } | |||
906 | ||||
907 | /// Iterator for directly iterating over the operand SDValue's. | |||
908 | struct value_op_iterator | |||
909 | : iterator_adaptor_base<value_op_iterator, op_iterator, | |||
910 | std::random_access_iterator_tag, SDValue, | |||
911 | ptrdiff_t, value_op_iterator *, | |||
912 | value_op_iterator *> { | |||
913 | explicit value_op_iterator(SDUse *U = nullptr) | |||
914 | : iterator_adaptor_base(U) {} | |||
915 | ||||
916 | const SDValue &operator*() const { return I->get(); } | |||
917 | }; | |||
918 | ||||
919 | iterator_range<value_op_iterator> op_values() const { | |||
920 | return make_range(value_op_iterator(op_begin()), | |||
921 | value_op_iterator(op_end())); | |||
922 | } | |||
923 | ||||
924 | SDVTList getVTList() const { | |||
925 | SDVTList X = { ValueList, NumValues }; | |||
926 | return X; | |||
927 | } | |||
928 | ||||
929 | /// If this node has a glue operand, return the node | |||
930 | /// to which the glue operand points. Otherwise return NULL. | |||
931 | SDNode *getGluedNode() const { | |||
932 | if (getNumOperands() != 0 && | |||
933 | getOperand(getNumOperands()-1).getValueType() == MVT::Glue) | |||
934 | return getOperand(getNumOperands()-1).getNode(); | |||
935 | return nullptr; | |||
936 | } | |||
937 | ||||
938 | /// If this node has a glue value with a user, return | |||
939 | /// the user (there is at most one). Otherwise return NULL. | |||
940 | SDNode *getGluedUser() const { | |||
941 | for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) | |||
942 | if (UI.getUse().get().getValueType() == MVT::Glue) | |||
943 | return *UI; | |||
944 | return nullptr; | |||
945 | } | |||
946 | ||||
947 | SDNodeFlags getFlags() const { return Flags; } | |||
948 | void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; } | |||
949 | ||||
950 | /// Clear any flags in this node that aren't also set in Flags. | |||
951 | /// If Flags is not in a defined state then this has no effect. | |||
952 | void intersectFlagsWith(const SDNodeFlags Flags); | |||
953 | ||||
954 | /// Return the number of values defined/returned by this operator. | |||
955 | unsigned getNumValues() const { return NumValues; } | |||
956 | ||||
957 | /// Return the type of a specified result. | |||
958 | EVT getValueType(unsigned ResNo) const { | |||
959 | assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!" ) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 959, __extension__ __PRETTY_FUNCTION__)); | |||
960 | return ValueList[ResNo]; | |||
961 | } | |||
962 | ||||
963 | /// Return the type of a specified result as a simple type. | |||
964 | MVT getSimpleValueType(unsigned ResNo) const { | |||
965 | return getValueType(ResNo).getSimpleVT(); | |||
966 | } | |||
967 | ||||
968 | /// Returns MVT::getSizeInBits(getValueType(ResNo)). | |||
969 | /// | |||
970 | /// If the value type is a scalable vector type, the scalable property will | |||
971 | /// be set and the runtime size will be a positive integer multiple of the | |||
972 | /// base size. | |||
973 | TypeSize getValueSizeInBits(unsigned ResNo) const { | |||
974 | return getValueType(ResNo).getSizeInBits(); | |||
975 | } | |||
976 | ||||
977 | using value_iterator = const EVT *; | |||
978 | ||||
979 | value_iterator value_begin() const { return ValueList; } | |||
980 | value_iterator value_end() const { return ValueList+NumValues; } | |||
981 | iterator_range<value_iterator> values() const { | |||
982 | return llvm::make_range(value_begin(), value_end()); | |||
983 | } | |||
984 | ||||
985 | /// Return the opcode of this operation for printing. | |||
986 | std::string getOperationName(const SelectionDAG *G = nullptr) const; | |||
987 | static const char* getIndexedModeName(ISD::MemIndexedMode AM); | |||
988 | void print_types(raw_ostream &OS, const SelectionDAG *G) const; | |||
989 | void print_details(raw_ostream &OS, const SelectionDAG *G) const; | |||
990 | void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
991 | void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
992 | ||||
993 | /// Print a SelectionDAG node and all children down to | |||
994 | /// the leaves. The given SelectionDAG allows target-specific nodes | |||
995 | /// to be printed in human-readable form. Unlike printr, this will | |||
996 | /// print the whole DAG, including children that appear multiple | |||
997 | /// times. | |||
998 | /// | |||
999 | void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; | |||
1000 | ||||
1001 | /// Print a SelectionDAG node and children up to | |||
1002 | /// depth "depth." The given SelectionDAG allows target-specific | |||
1003 | /// nodes to be printed in human-readable form. Unlike printr, this | |||
1004 | /// will print children that appear multiple times wherever they are | |||
1005 | /// used. | |||
1006 | /// | |||
1007 | void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, | |||
1008 | unsigned depth = 100) const; | |||
1009 | ||||
1010 | /// Dump this node, for debugging. | |||
1011 | void dump() const; | |||
1012 | ||||
1013 | /// Dump (recursively) this node and its use-def subgraph. | |||
1014 | void dumpr() const; | |||
1015 | ||||
1016 | /// Dump this node, for debugging. | |||
1017 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
1018 | /// in human-readable form. | |||
1019 | void dump(const SelectionDAG *G) const; | |||
1020 | ||||
1021 | /// Dump (recursively) this node and its use-def subgraph. | |||
1022 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
1023 | /// in human-readable form. | |||
1024 | void dumpr(const SelectionDAG *G) const; | |||
1025 | ||||
1026 | /// printrFull to dbgs(). The given SelectionDAG allows | |||
1027 | /// target-specific nodes to be printed in human-readable form. | |||
1028 | /// Unlike dumpr, this will print the whole DAG, including children | |||
1029 | /// that appear multiple times. | |||
1030 | void dumprFull(const SelectionDAG *G = nullptr) const; | |||
1031 | ||||
1032 | /// printrWithDepth to dbgs(). The given | |||
1033 | /// SelectionDAG allows target-specific nodes to be printed in | |||
1034 | /// human-readable form. Unlike dumpr, this will print children | |||
1035 | /// that appear multiple times wherever they are used. | |||
1036 | /// | |||
1037 | void dumprWithDepth(const SelectionDAG *G = nullptr, | |||
1038 | unsigned depth = 100) const; | |||
1039 | ||||
1040 | /// Gather unique data for the node. | |||
1041 | void Profile(FoldingSetNodeID &ID) const; | |||
1042 | ||||
1043 | /// This method should only be used by the SDUse class. | |||
1044 | void addUse(SDUse &U) { U.addToList(&UseList); } | |||
1045 | ||||
1046 | protected: | |||
1047 | static SDVTList getSDVTList(EVT VT) { | |||
1048 | SDVTList Ret = { getValueTypeList(VT), 1 }; | |||
1049 | return Ret; | |||
1050 | } | |||
1051 | ||||
1052 | /// Create an SDNode. | |||
1053 | /// | |||
1054 | /// SDNodes are created without any operands, and never own the operand | |||
1055 | /// storage. To add operands, see SelectionDAG::createOperands. | |||
1056 | SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) | |||
1057 | : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs), | |||
1058 | IROrder(Order), debugLoc(std::move(dl)) { | |||
1059 | memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits)); | |||
1060 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() && "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1060, __extension__ __PRETTY_FUNCTION__)); | |||
1061 | assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1062, __extension__ __PRETTY_FUNCTION__)) | |||
1062 | "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1062, __extension__ __PRETTY_FUNCTION__)); | |||
1063 | } | |||
1064 | ||||
1065 | /// Release the operands and set this node to have zero operands. | |||
1066 | void DropOperands(); | |||
1067 | }; | |||
1068 | ||||
1069 | /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed | |||
1070 | /// into SDNode creation functions. | |||
1071 | /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted | |||
1072 | /// from the original Instruction, and IROrder is the ordinal position of | |||
1073 | /// the instruction. | |||
1074 | /// When an SDNode is created after the DAG is being built, both DebugLoc and | |||
1075 | /// the IROrder are propagated from the original SDNode. | |||
1076 | /// So SDLoc class provides two constructors besides the default one, one to | |||
1077 | /// be used by the DAGBuilder, the other to be used by others. | |||
1078 | class SDLoc { | |||
1079 | private: | |||
1080 | DebugLoc DL; | |||
1081 | int IROrder = 0; | |||
1082 | ||||
1083 | public: | |||
1084 | SDLoc() = default; | |||
1085 | SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {} | |||
1086 | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} | |||
1087 | SDLoc(const Instruction *I, int Order) : IROrder(Order) { | |||
1088 | assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder" ) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1088, __extension__ __PRETTY_FUNCTION__)); | |||
1089 | if (I) | |||
1090 | DL = I->getDebugLoc(); | |||
1091 | } | |||
1092 | ||||
1093 | unsigned getIROrder() const { return IROrder; } | |||
1094 | const DebugLoc &getDebugLoc() const { return DL; } | |||
1095 | }; | |||
1096 | ||||
1097 | // Define inline functions from the SDValue class. | |||
1098 | ||||
1099 | inline SDValue::SDValue(SDNode *node, unsigned resno) | |||
1100 | : Node(node), ResNo(resno) { | |||
1101 | // Explicitly check for !ResNo to avoid use-after-free, because there are | |||
1102 | // callers that use SDValue(N, 0) with a deleted N to indicate successful | |||
1103 | // combines. | |||
1104 | assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1105, __extension__ __PRETTY_FUNCTION__)) | |||
1105 | "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1105, __extension__ __PRETTY_FUNCTION__)); | |||
1106 | assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps." ) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1106, __extension__ __PRETTY_FUNCTION__)); | |||
1107 | } | |||
1108 | ||||
1109 | inline unsigned SDValue::getOpcode() const { | |||
1110 | return Node->getOpcode(); | |||
1111 | } | |||
1112 | ||||
1113 | inline EVT SDValue::getValueType() const { | |||
1114 | return Node->getValueType(ResNo); | |||
| ||||
1115 | } | |||
1116 | ||||
1117 | inline unsigned SDValue::getNumOperands() const { | |||
1118 | return Node->getNumOperands(); | |||
1119 | } | |||
1120 | ||||
1121 | inline const SDValue &SDValue::getOperand(unsigned i) const { | |||
1122 | return Node->getOperand(i); | |||
1123 | } | |||
1124 | ||||
1125 | inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { | |||
1126 | return Node->getConstantOperandVal(i); | |||
1127 | } | |||
1128 | ||||
1129 | inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { | |||
1130 | return Node->getConstantOperandAPInt(i); | |||
1131 | } | |||
1132 | ||||
1133 | inline bool SDValue::isTargetOpcode() const { | |||
1134 | return Node->isTargetOpcode(); | |||
1135 | } | |||
1136 | ||||
1137 | inline bool SDValue::isTargetMemoryOpcode() const { | |||
1138 | return Node->isTargetMemoryOpcode(); | |||
1139 | } | |||
1140 | ||||
1141 | inline bool SDValue::isMachineOpcode() const { | |||
1142 | return Node->isMachineOpcode(); | |||
1143 | } | |||
1144 | ||||
1145 | inline unsigned SDValue::getMachineOpcode() const { | |||
1146 | return Node->getMachineOpcode(); | |||
1147 | } | |||
1148 | ||||
1149 | inline bool SDValue::isUndef() const { | |||
1150 | return Node->isUndef(); | |||
1151 | } | |||
1152 | ||||
1153 | inline bool SDValue::use_empty() const { | |||
1154 | return !Node->hasAnyUseOfValue(ResNo); | |||
1155 | } | |||
1156 | ||||
1157 | inline bool SDValue::hasOneUse() const { | |||
1158 | return Node->hasNUsesOfValue(1, ResNo); | |||
1159 | } | |||
1160 | ||||
1161 | inline const DebugLoc &SDValue::getDebugLoc() const { | |||
1162 | return Node->getDebugLoc(); | |||
1163 | } | |||
1164 | ||||
1165 | inline void SDValue::dump() const { | |||
1166 | return Node->dump(); | |||
1167 | } | |||
1168 | ||||
1169 | inline void SDValue::dump(const SelectionDAG *G) const { | |||
1170 | return Node->dump(G); | |||
1171 | } | |||
1172 | ||||
1173 | inline void SDValue::dumpr() const { | |||
1174 | return Node->dumpr(); | |||
1175 | } | |||
1176 | ||||
1177 | inline void SDValue::dumpr(const SelectionDAG *G) const { | |||
1178 | return Node->dumpr(G); | |||
1179 | } | |||
1180 | ||||
1181 | // Define inline functions from the SDUse class. | |||
1182 | ||||
1183 | inline void SDUse::set(const SDValue &V) { | |||
1184 | if (Val.getNode()) removeFromList(); | |||
1185 | Val = V; | |||
1186 | if (V.getNode()) V.getNode()->addUse(*this); | |||
1187 | } | |||
1188 | ||||
1189 | inline void SDUse::setInitial(const SDValue &V) { | |||
1190 | Val = V; | |||
1191 | V.getNode()->addUse(*this); | |||
1192 | } | |||
1193 | ||||
1194 | inline void SDUse::setNode(SDNode *N) { | |||
1195 | if (Val.getNode()) removeFromList(); | |||
1196 | Val.setNode(N); | |||
1197 | if (N) N->addUse(*this); | |||
1198 | } | |||
1199 | ||||
1200 | /// This class is used to form a handle around another node that | |||
1201 | /// is persistent and is updated across invocations of replaceAllUsesWith on its | |||
1202 | /// operand. This node should be directly created by end-users and not added to | |||
1203 | /// the AllNodes list. | |||
1204 | class HandleSDNode : public SDNode { | |||
1205 | SDUse Op; | |||
1206 | ||||
1207 | public: | |||
1208 | explicit HandleSDNode(SDValue X) | |||
1209 | : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { | |||
1210 | // HandleSDNodes are never inserted into the DAG, so they won't be | |||
1211 | // auto-numbered. Use ID 65535 as a sentinel. | |||
1212 | PersistentId = 0xffff; | |||
1213 | ||||
1214 | // Manually set up the operand list. This node type is special in that it's | |||
1215 | // always stack allocated and SelectionDAG does not manage its operands. | |||
1216 | // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not | |||
1217 | // be so special. | |||
1218 | Op.setUser(this); | |||
1219 | Op.setInitial(X); | |||
1220 | NumOperands = 1; | |||
1221 | OperandList = &Op; | |||
1222 | } | |||
1223 | ~HandleSDNode(); | |||
1224 | ||||
1225 | const SDValue &getValue() const { return Op; } | |||
1226 | }; | |||
1227 | ||||
1228 | class AddrSpaceCastSDNode : public SDNode { | |||
1229 | private: | |||
1230 | unsigned SrcAddrSpace; | |||
1231 | unsigned DestAddrSpace; | |||
1232 | ||||
1233 | public: | |||
1234 | AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, | |||
1235 | unsigned SrcAS, unsigned DestAS); | |||
1236 | ||||
1237 | unsigned getSrcAddressSpace() const { return SrcAddrSpace; } | |||
1238 | unsigned getDestAddressSpace() const { return DestAddrSpace; } | |||
1239 | ||||
1240 | static bool classof(const SDNode *N) { | |||
1241 | return N->getOpcode() == ISD::ADDRSPACECAST; | |||
1242 | } | |||
1243 | }; | |||
1244 | ||||
1245 | /// This is an abstract virtual class for memory operations. | |||
1246 | class MemSDNode : public SDNode { | |||
1247 | private: | |||
1248 | // VT of in-memory value. | |||
1249 | EVT MemoryVT; | |||
1250 | ||||
1251 | protected: | |||
1252 | /// Memory reference information. | |||
1253 | MachineMemOperand *MMO; | |||
1254 | ||||
1255 | public: | |||
1256 | MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
1257 | EVT memvt, MachineMemOperand *MMO); | |||
1258 | ||||
1259 | bool readMem() const { return MMO->isLoad(); } | |||
1260 | bool writeMem() const { return MMO->isStore(); } | |||
1261 | ||||
1262 | /// Returns alignment and volatility of the memory access | |||
1263 | Align getOriginalAlign() const { return MMO->getBaseAlign(); } | |||
1264 | Align getAlign() const { return MMO->getAlign(); } | |||
1265 | // FIXME: Remove once transition to getAlign is over. | |||
1266 | unsigned getAlignment() const { return MMO->getAlign().value(); } | |||
1267 | ||||
1268 | /// Return the SubclassData value, without HasDebugValue. This contains an | |||
1269 | /// encoding of the volatile flag, as well as bits used by subclasses. This | |||
1270 | /// function should only be used to compute a FoldingSetNodeID value. | |||
1271 | /// The HasDebugValue bit is masked out because CSE map needs to match | |||
1272 | /// nodes with debug info with nodes without debug info. Same is about | |||
1273 | /// isDivergent bit. | |||
1274 | unsigned getRawSubclassData() const { | |||
1275 | uint16_t Data; | |||
1276 | union { | |||
1277 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
1278 | SDNodeBitfields SDNodeBits; | |||
1279 | }; | |||
1280 | memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits)); | |||
1281 | SDNodeBits.HasDebugValue = 0; | |||
1282 | SDNodeBits.IsDivergent = false; | |||
1283 | memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits)); | |||
1284 | return Data; | |||
1285 | } | |||
1286 | ||||
1287 | bool isVolatile() const { return MemSDNodeBits.IsVolatile; } | |||
1288 | bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } | |||
1289 | bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } | |||
1290 | bool isInvariant() const { return MemSDNodeBits.IsInvariant; } | |||
1291 | ||||
1292 | // Returns the offset from the location of the access. | |||
1293 | int64_t getSrcValueOffset() const { return MMO->getOffset(); } | |||
1294 | ||||
1295 | /// Returns the AA info that describes the dereference. | |||
1296 | AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } | |||
1297 | ||||
1298 | /// Returns the Ranges that describes the dereference. | |||
1299 | const MDNode *getRanges() const { return MMO->getRanges(); } | |||
1300 | ||||
1301 | /// Returns the synchronization scope ID for this memory operation. | |||
1302 | SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } | |||
1303 | ||||
1304 | /// Return the atomic ordering requirements for this memory operation. For | |||
1305 | /// cmpxchg atomic operations, return the atomic ordering requirements when | |||
1306 | /// store occurs. | |||
1307 | AtomicOrdering getSuccessOrdering() const { | |||
1308 | return MMO->getSuccessOrdering(); | |||
1309 | } | |||
1310 | ||||
1311 | /// Return a single atomic ordering that is at least as strong as both the | |||
1312 | /// success and failure orderings for an atomic operation. (For operations | |||
1313 | /// other than cmpxchg, this is equivalent to getSuccessOrdering().) | |||
1314 | AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); } | |||
1315 | ||||
1316 | /// Return true if the memory operation ordering is Unordered or higher. | |||
1317 | bool isAtomic() const { return MMO->isAtomic(); } | |||
1318 | ||||
1319 | /// Returns true if the memory operation doesn't imply any ordering | |||
1320 | /// constraints on surrounding memory operations beyond the normal memory | |||
1321 | /// aliasing rules. | |||
1322 | bool isUnordered() const { return MMO->isUnordered(); } | |||
1323 | ||||
1324 | /// Returns true if the memory operation is neither atomic or volatile. | |||
1325 | bool isSimple() const { return !isAtomic() && !isVolatile(); } | |||
1326 | ||||
1327 | /// Return the type of the in-memory value. | |||
1328 | EVT getMemoryVT() const { return MemoryVT; } | |||
1329 | ||||
1330 | /// Return a MachineMemOperand object describing the memory | |||
1331 | /// reference performed by operation. | |||
1332 | MachineMemOperand *getMemOperand() const { return MMO; } | |||
1333 | ||||
1334 | const MachinePointerInfo &getPointerInfo() const { | |||
1335 | return MMO->getPointerInfo(); | |||
1336 | } | |||
1337 | ||||
1338 | /// Return the address space for the associated pointer | |||
1339 | unsigned getAddressSpace() const { | |||
1340 | return getPointerInfo().getAddrSpace(); | |||
1341 | } | |||
1342 | ||||
1343 | /// Update this MemSDNode's MachineMemOperand information | |||
1344 | /// to reflect the alignment of NewMMO, if it has a greater alignment. | |||
1345 | /// This must only be used when the new alignment applies to all users of | |||
1346 | /// this MachineMemOperand. | |||
1347 | void refineAlignment(const MachineMemOperand *NewMMO) { | |||
1348 | MMO->refineAlignment(NewMMO); | |||
1349 | } | |||
1350 | ||||
1351 | const SDValue &getChain() const { return getOperand(0); } | |||
1352 | ||||
1353 | const SDValue &getBasePtr() const { | |||
1354 | switch (getOpcode()) { | |||
1355 | case ISD::STORE: | |||
1356 | case ISD::MSTORE: | |||
1357 | return getOperand(2); | |||
1358 | case ISD::MGATHER: | |||
1359 | case ISD::MSCATTER: | |||
1360 | return getOperand(3); | |||
1361 | default: | |||
1362 | return getOperand(1); | |||
1363 | } | |||
1364 | } | |||
1365 | ||||
1366 | // Methods to support isa and dyn_cast | |||
1367 | static bool classof(const SDNode *N) { | |||
1368 | // For some targets, we lower some target intrinsics to a MemIntrinsicNode | |||
1369 | // with either an intrinsic or a target opcode. | |||
1370 | switch (N->getOpcode()) { | |||
1371 | case ISD::LOAD: | |||
1372 | case ISD::STORE: | |||
1373 | case ISD::PREFETCH: | |||
1374 | case ISD::ATOMIC_CMP_SWAP: | |||
1375 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: | |||
1376 | case ISD::ATOMIC_SWAP: | |||
1377 | case ISD::ATOMIC_LOAD_ADD: | |||
1378 | case ISD::ATOMIC_LOAD_SUB: | |||
1379 | case ISD::ATOMIC_LOAD_AND: | |||
1380 | case ISD::ATOMIC_LOAD_CLR: | |||
1381 | case ISD::ATOMIC_LOAD_OR: | |||
1382 | case ISD::ATOMIC_LOAD_XOR: | |||
1383 | case ISD::ATOMIC_LOAD_NAND: | |||
1384 | case ISD::ATOMIC_LOAD_MIN: | |||
1385 | case ISD::ATOMIC_LOAD_MAX: | |||
1386 | case ISD::ATOMIC_LOAD_UMIN: | |||
1387 | case ISD::ATOMIC_LOAD_UMAX: | |||
1388 | case ISD::ATOMIC_LOAD_FADD: | |||
1389 | case ISD::ATOMIC_LOAD_FSUB: | |||
1390 | case ISD::ATOMIC_LOAD: | |||
1391 | case ISD::ATOMIC_STORE: | |||
1392 | case ISD::MLOAD: | |||
1393 | case ISD::MSTORE: | |||
1394 | case ISD::MGATHER: | |||
1395 | case ISD::MSCATTER: | |||
1396 | return true; | |||
1397 | default: | |||
1398 | return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); | |||
1399 | } | |||
1400 | } | |||
1401 | }; | |||
1402 | ||||
1403 | /// This is an SDNode representing atomic operations. | |||
1404 | class AtomicSDNode : public MemSDNode { | |||
1405 | public: | |||
1406 | AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, | |||
1407 | EVT MemVT, MachineMemOperand *MMO) | |||
1408 | : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { | |||
1409 | assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1410, __extension__ __PRETTY_FUNCTION__)) | |||
1410 | MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1410, __extension__ __PRETTY_FUNCTION__)); | |||
1411 | } | |||
1412 | ||||
1413 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
1414 | const SDValue &getVal() const { return getOperand(2); } | |||
1415 | ||||
1416 | /// Returns true if this SDNode represents cmpxchg atomic operation, false | |||
1417 | /// otherwise. | |||
1418 | bool isCompareAndSwap() const { | |||
1419 | unsigned Op = getOpcode(); | |||
1420 | return Op == ISD::ATOMIC_CMP_SWAP || | |||
1421 | Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; | |||
1422 | } | |||
1423 | ||||
1424 | /// For cmpxchg atomic operations, return the atomic ordering requirements | |||
1425 | /// when store does not occur. | |||
1426 | AtomicOrdering getFailureOrdering() const { | |||
1427 | assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation" ) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1427, __extension__ __PRETTY_FUNCTION__)); | |||
1428 | return MMO->getFailureOrdering(); | |||
1429 | } | |||
1430 | ||||
1431 | // Methods to support isa and dyn_cast | |||
1432 | static bool classof(const SDNode *N) { | |||
1433 | return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || | |||
1434 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || | |||
1435 | N->getOpcode() == ISD::ATOMIC_SWAP || | |||
1436 | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || | |||
1437 | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || | |||
1438 | N->getOpcode() == ISD::ATOMIC_LOAD_AND || | |||
1439 | N->getOpcode() == ISD::ATOMIC_LOAD_CLR || | |||
1440 | N->getOpcode() == ISD::ATOMIC_LOAD_OR || | |||
1441 | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || | |||
1442 | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || | |||
1443 | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || | |||
1444 | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || | |||
1445 | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || | |||
1446 | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || | |||
1447 | N->getOpcode() == ISD::ATOMIC_LOAD_FADD || | |||
1448 | N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || | |||
1449 | N->getOpcode() == ISD::ATOMIC_LOAD || | |||
1450 | N->getOpcode() == ISD::ATOMIC_STORE; | |||
1451 | } | |||
1452 | }; | |||
1453 | ||||
1454 | /// This SDNode is used for target intrinsics that touch | |||
1455 | /// memory and need an associated MachineMemOperand. Its opcode may be | |||
1456 | /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode | |||
1457 | /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. | |||
1458 | class MemIntrinsicSDNode : public MemSDNode { | |||
1459 | public: | |||
1460 | MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, | |||
1461 | SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) | |||
1462 | : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { | |||
1463 | SDNodeBits.IsMemIntrinsic = true; | |||
1464 | } | |||
1465 | ||||
1466 | // Methods to support isa and dyn_cast | |||
1467 | static bool classof(const SDNode *N) { | |||
1468 | // We lower some target intrinsics to their target opcode | |||
1469 | // early a node with a target opcode can be of this class | |||
1470 | return N->isMemIntrinsic() || | |||
1471 | N->getOpcode() == ISD::PREFETCH || | |||
1472 | N->isTargetMemoryOpcode(); | |||
1473 | } | |||
1474 | }; | |||
1475 | ||||
1476 | /// This SDNode is used to implement the code generator | |||
1477 | /// support for the llvm IR shufflevector instruction. It combines elements | |||
1478 | /// from two input vectors into a new input vector, with the selection and | |||
1479 | /// ordering of elements determined by an array of integers, referred to as | |||
1480 | /// the shuffle mask. For input vectors of width N, mask indices of 0..N-1 | |||
1481 | /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS. | |||
1482 | /// An index of -1 is treated as undef, such that the code generator may put | |||
1483 | /// any value in the corresponding element of the result. | |||
1484 | class ShuffleVectorSDNode : public SDNode { | |||
1485 | // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and | |||
1486 | // is freed when the SelectionDAG object is destroyed. | |||
1487 | const int *Mask; | |||
1488 | ||||
1489 | protected: | |||
1490 | friend class SelectionDAG; | |||
1491 | ||||
1492 | ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M) | |||
1493 | : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} | |||
1494 | ||||
1495 | public: | |||
1496 | ArrayRef<int> getMask() const { | |||
1497 | EVT VT = getValueType(0); | |||
1498 | return makeArrayRef(Mask, VT.getVectorNumElements()); | |||
1499 | } | |||
1500 | ||||
1501 | int getMaskElt(unsigned Idx) const { | |||
1502 | assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements () && "Idx out of range!") ? void (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1502, __extension__ __PRETTY_FUNCTION__)); | |||
1503 | return Mask[Idx]; | |||
1504 | } | |||
1505 | ||||
1506 | bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } | |||
1507 | ||||
1508 | int getSplatIndex() const { | |||
1509 | assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!" ) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1509, __extension__ __PRETTY_FUNCTION__)); | |||
1510 | EVT VT = getValueType(0); | |||
1511 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) | |||
1512 | if (Mask[i] >= 0) | |||
1513 | return Mask[i]; | |||
1514 | ||||
1515 | // We can choose any index value here and be correct because all elements | |||
1516 | // are undefined. Return 0 for better potential for callers to simplify. | |||
1517 | return 0; | |||
1518 | } | |||
1519 | ||||
1520 | static bool isSplatMask(const int *Mask, EVT VT); | |||
1521 | ||||
1522 | /// Change values in a shuffle permute mask assuming | |||
1523 | /// the two vector operands have swapped position. | |||
1524 | static void commuteMask(MutableArrayRef<int> Mask) { | |||
1525 | unsigned NumElems = Mask.size(); | |||
1526 | for (unsigned i = 0; i != NumElems; ++i) { | |||
1527 | int idx = Mask[i]; | |||
1528 | if (idx < 0) | |||
1529 | continue; | |||
1530 | else if (idx < (int)NumElems) | |||
1531 | Mask[i] = idx + NumElems; | |||
1532 | else | |||
1533 | Mask[i] = idx - NumElems; | |||
1534 | } | |||
1535 | } | |||
1536 | ||||
1537 | static bool classof(const SDNode *N) { | |||
1538 | return N->getOpcode() == ISD::VECTOR_SHUFFLE; | |||
1539 | } | |||
1540 | }; | |||
1541 | ||||
1542 | class ConstantSDNode : public SDNode { | |||
1543 | friend class SelectionDAG; | |||
1544 | ||||
1545 | const ConstantInt *Value; | |||
1546 | ||||
1547 | ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) | |||
1548 | : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(), | |||
1549 | getSDVTList(VT)), | |||
1550 | Value(val) { | |||
1551 | ConstantSDNodeBits.IsOpaque = isOpaque; | |||
1552 | } | |||
1553 | ||||
1554 | public: | |||
1555 | const ConstantInt *getConstantIntValue() const { return Value; } | |||
1556 | const APInt &getAPIntValue() const { return Value->getValue(); } | |||
1557 | uint64_t getZExtValue() const { return Value->getZExtValue(); } | |||
1558 | int64_t getSExtValue() const { return Value->getSExtValue(); } | |||
1559 | uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) { | |||
1560 | return Value->getLimitedValue(Limit); | |||
1561 | } | |||
1562 | MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); } | |||
1563 | Align getAlignValue() const { return Value->getAlignValue(); } | |||
1564 | ||||
1565 | bool isOne() const { return Value->isOne(); } | |||
1566 | bool isNullValue() const { return Value->isZero(); } | |||
1567 | bool isAllOnesValue() const { return Value->isMinusOne(); } | |||
1568 | bool isMaxSignedValue() const { return Value->isMaxValue(true); } | |||
1569 | bool isMinSignedValue() const { return Value->isMinValue(true); } | |||
1570 | ||||
1571 | bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } | |||
1572 | ||||
1573 | static bool classof(const SDNode *N) { | |||
1574 | return N->getOpcode() == ISD::Constant || | |||
1575 | N->getOpcode() == ISD::TargetConstant; | |||
1576 | } | |||
1577 | }; | |||
1578 | ||||
1579 | uint64_t SDNode::getConstantOperandVal(unsigned Num) const { | |||
1580 | return cast<ConstantSDNode>(getOperand(Num))->getZExtValue(); | |||
1581 | } | |||
1582 | ||||
1583 | const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { | |||
1584 | return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue(); | |||
1585 | } | |||
1586 | ||||
1587 | class ConstantFPSDNode : public SDNode { | |||
1588 | friend class SelectionDAG; | |||
1589 | ||||
1590 | const ConstantFP *Value; | |||
1591 | ||||
1592 | ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) | |||
1593 | : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, | |||
1594 | DebugLoc(), getSDVTList(VT)), | |||
1595 | Value(val) {} | |||
1596 | ||||
1597 | public: | |||
1598 | const APFloat& getValueAPF() const { return Value->getValueAPF(); } | |||
1599 | const ConstantFP *getConstantFPValue() const { return Value; } | |||
1600 | ||||
1601 | /// Return true if the value is positive or negative zero. | |||
1602 | bool isZero() const { return Value->isZero(); } | |||
1603 | ||||
1604 | /// Return true if the value is a NaN. | |||
1605 | bool isNaN() const { return Value->isNaN(); } | |||
1606 | ||||
1607 | /// Return true if the value is an infinity | |||
1608 | bool isInfinity() const { return Value->isInfinity(); } | |||
1609 | ||||
1610 | /// Return true if the value is negative. | |||
1611 | bool isNegative() const { return Value->isNegative(); } | |||
1612 | ||||
1613 | /// We don't rely on operator== working on double values, as | |||
1614 | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. | |||
1615 | /// As such, this method can be used to do an exact bit-for-bit comparison of | |||
1616 | /// two floating point values. | |||
1617 | ||||
1618 | /// We leave the version with the double argument here because it's just so | |||
1619 | /// convenient to write "2.0" and the like. Without this function we'd | |||
1620 | /// have to duplicate its logic everywhere it's called. | |||
1621 | bool isExactlyValue(double V) const { | |||
1622 | return Value->getValueAPF().isExactlyValue(V); | |||
1623 | } | |||
1624 | bool isExactlyValue(const APFloat& V) const; | |||
1625 | ||||
1626 | static bool isValueValidForType(EVT VT, const APFloat& Val); | |||
1627 | ||||
1628 | static bool classof(const SDNode *N) { | |||
1629 | return N->getOpcode() == ISD::ConstantFP || | |||
1630 | N->getOpcode() == ISD::TargetConstantFP; | |||
1631 | } | |||
1632 | }; | |||
1633 | ||||
1634 | /// Returns true if \p V is a constant integer zero. | |||
1635 | bool isNullConstant(SDValue V); | |||
1636 | ||||
1637 | /// Returns true if \p V is an FP constant with a value of positive zero. | |||
1638 | bool isNullFPConstant(SDValue V); | |||
1639 | ||||
1640 | /// Returns true if \p V is an integer constant with all bits set. | |||
1641 | bool isAllOnesConstant(SDValue V); | |||
1642 | ||||
1643 | /// Returns true if \p V is a constant integer one. | |||
1644 | bool isOneConstant(SDValue V); | |||
1645 | ||||
1646 | /// Return the non-bitcasted source operand of \p V if it exists. | |||
1647 | /// If \p V is not a bitcasted value, it is returned as-is. | |||
1648 | SDValue peekThroughBitcasts(SDValue V); | |||
1649 | ||||
1650 | /// Return the non-bitcasted and one-use source operand of \p V if it exists. | |||
1651 | /// If \p V is not a bitcasted one-use value, it is returned as-is. | |||
1652 | SDValue peekThroughOneUseBitcasts(SDValue V); | |||
1653 | ||||
1654 | /// Return the non-extracted vector source operand of \p V if it exists. | |||
1655 | /// If \p V is not an extracted subvector, it is returned as-is. | |||
1656 | SDValue peekThroughExtractSubvectors(SDValue V); | |||
1657 | ||||
1658 | /// Returns true if \p V is a bitwise not operation. Assumes that an all ones | |||
1659 | /// constant is canonicalized to be operand 1. | |||
1660 | bool isBitwiseNot(SDValue V, bool AllowUndefs = false); | |||
1661 | ||||
1662 | /// Returns the SDNode if it is a constant splat BuildVector or constant int. | |||
1663 | ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, | |||
1664 | bool AllowTruncation = false); | |||
1665 | ||||
1666 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
1667 | /// constant int. | |||
1668 | ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, | |||
1669 | bool AllowUndefs = false, | |||
1670 | bool AllowTruncation = false); | |||
1671 | ||||
1672 | /// Returns the SDNode if it is a constant splat BuildVector or constant float. | |||
1673 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); | |||
1674 | ||||
1675 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
1676 | /// constant float. | |||
1677 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, | |||
1678 | bool AllowUndefs = false); | |||
1679 | ||||
1680 | /// Return true if the value is a constant 0 integer or a splatted vector of | |||
1681 | /// a constant 0 integer (with no undefs by default). | |||
1682 | /// Build vector implicit truncation is not an issue for null values. | |||
1683 | bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); | |||
1684 | ||||
1685 | /// Return true if the value is a constant 1 integer or a splatted vector of a | |||
1686 | /// constant 1 integer (with no undefs). | |||
1687 | /// Does not permit build vector implicit truncation. | |||
1688 | bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false); | |||
1689 | ||||
1690 | /// Return true if the value is a constant -1 integer or a splatted vector of a | |||
1691 | /// constant -1 integer (with no undefs). | |||
1692 | /// Does not permit build vector implicit truncation. | |||
1693 | bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false); | |||
1694 | ||||
1695 | /// Return true if \p V is either a integer or FP constant. | |||
1696 | inline bool isIntOrFPConstant(SDValue V) { | |||
1697 | return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V); | |||
1698 | } | |||
1699 | ||||
1700 | class GlobalAddressSDNode : public SDNode { | |||
1701 | friend class SelectionDAG; | |||
1702 | ||||
1703 | const GlobalValue *TheGlobal; | |||
1704 | int64_t Offset; | |||
1705 | unsigned TargetFlags; | |||
1706 | ||||
1707 | GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, | |||
1708 | const GlobalValue *GA, EVT VT, int64_t o, | |||
1709 | unsigned TF); | |||
1710 | ||||
1711 | public: | |||
1712 | const GlobalValue *getGlobal() const { return TheGlobal; } | |||
1713 | int64_t getOffset() const { return Offset; } | |||
1714 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1715 | // Return the address space this GlobalAddress belongs to. | |||
1716 | unsigned getAddressSpace() const; | |||
1717 | ||||
1718 | static bool classof(const SDNode *N) { | |||
1719 | return N->getOpcode() == ISD::GlobalAddress || | |||
1720 | N->getOpcode() == ISD::TargetGlobalAddress || | |||
1721 | N->getOpcode() == ISD::GlobalTLSAddress || | |||
1722 | N->getOpcode() == ISD::TargetGlobalTLSAddress; | |||
1723 | } | |||
1724 | }; | |||
1725 | ||||
1726 | class FrameIndexSDNode : public SDNode { | |||
1727 | friend class SelectionDAG; | |||
1728 | ||||
1729 | int FI; | |||
1730 | ||||
1731 | FrameIndexSDNode(int fi, EVT VT, bool isTarg) | |||
1732 | : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, | |||
1733 | 0, DebugLoc(), getSDVTList(VT)), FI(fi) { | |||
1734 | } | |||
1735 | ||||
1736 | public: | |||
1737 | int getIndex() const { return FI; } | |||
1738 | ||||
1739 | static bool classof(const SDNode *N) { | |||
1740 | return N->getOpcode() == ISD::FrameIndex || | |||
1741 | N->getOpcode() == ISD::TargetFrameIndex; | |||
1742 | } | |||
1743 | }; | |||
1744 | ||||
1745 | /// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate | |||
1746 | /// the offet and size that are started/ended in the underlying FrameIndex. | |||
1747 | class LifetimeSDNode : public SDNode { | |||
1748 | friend class SelectionDAG; | |||
1749 | int64_t Size; | |||
1750 | int64_t Offset; // -1 if offset is unknown. | |||
1751 | ||||
1752 | LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, | |||
1753 | SDVTList VTs, int64_t Size, int64_t Offset) | |||
1754 | : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} | |||
1755 | public: | |||
1756 | int64_t getFrameIndex() const { | |||
1757 | return cast<FrameIndexSDNode>(getOperand(1))->getIndex(); | |||
1758 | } | |||
1759 | ||||
1760 | bool hasOffset() const { return Offset >= 0; } | |||
1761 | int64_t getOffset() const { | |||
1762 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1762, __extension__ __PRETTY_FUNCTION__)); | |||
1763 | return Offset; | |||
1764 | } | |||
1765 | int64_t getSize() const { | |||
1766 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1766, __extension__ __PRETTY_FUNCTION__)); | |||
1767 | return Size; | |||
1768 | } | |||
1769 | ||||
1770 | // Methods to support isa and dyn_cast | |||
1771 | static bool classof(const SDNode *N) { | |||
1772 | return N->getOpcode() == ISD::LIFETIME_START || | |||
1773 | N->getOpcode() == ISD::LIFETIME_END; | |||
1774 | } | |||
1775 | }; | |||
1776 | ||||
1777 | /// This SDNode is used for PSEUDO_PROBE values, which are the function guid and | |||
1778 | /// the index of the basic block being probed. A pseudo probe serves as a place | |||
1779 | /// holder and will be removed at the end of compilation. It does not have any | |||
1780 | /// operand because we do not want the instruction selection to deal with any. | |||
1781 | class PseudoProbeSDNode : public SDNode { | |||
1782 | friend class SelectionDAG; | |||
1783 | uint64_t Guid; | |||
1784 | uint64_t Index; | |||
1785 | uint32_t Attributes; | |||
1786 | ||||
1787 | PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl, | |||
1788 | SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr) | |||
1789 | : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index), | |||
1790 | Attributes(Attr) {} | |||
1791 | ||||
1792 | public: | |||
1793 | uint64_t getGuid() const { return Guid; } | |||
1794 | uint64_t getIndex() const { return Index; } | |||
1795 | uint32_t getAttributes() const { return Attributes; } | |||
1796 | ||||
1797 | // Methods to support isa and dyn_cast | |||
1798 | static bool classof(const SDNode *N) { | |||
1799 | return N->getOpcode() == ISD::PSEUDO_PROBE; | |||
1800 | } | |||
1801 | }; | |||
1802 | ||||
1803 | class JumpTableSDNode : public SDNode { | |||
1804 | friend class SelectionDAG; | |||
1805 | ||||
1806 | int JTI; | |||
1807 | unsigned TargetFlags; | |||
1808 | ||||
1809 | JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF) | |||
1810 | : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, | |||
1811 | 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { | |||
1812 | } | |||
1813 | ||||
1814 | public: | |||
1815 | int getIndex() const { return JTI; } | |||
1816 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1817 | ||||
1818 | static bool classof(const SDNode *N) { | |||
1819 | return N->getOpcode() == ISD::JumpTable || | |||
1820 | N->getOpcode() == ISD::TargetJumpTable; | |||
1821 | } | |||
1822 | }; | |||
1823 | ||||
1824 | class ConstantPoolSDNode : public SDNode { | |||
1825 | friend class SelectionDAG; | |||
1826 | ||||
1827 | union { | |||
1828 | const Constant *ConstVal; | |||
1829 | MachineConstantPoolValue *MachineCPVal; | |||
1830 | } Val; | |||
1831 | int Offset; // It's a MachineConstantPoolValue if top bit is set. | |||
1832 | Align Alignment; // Minimum alignment requirement of CP. | |||
1833 | unsigned TargetFlags; | |||
1834 | ||||
1835 | ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, | |||
1836 | Align Alignment, unsigned TF) | |||
1837 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
1838 | DebugLoc(), getSDVTList(VT)), | |||
1839 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
1840 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1840, __extension__ __PRETTY_FUNCTION__)); | |||
1841 | Val.ConstVal = c; | |||
1842 | } | |||
1843 | ||||
1844 | ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o, | |||
1845 | Align Alignment, unsigned TF) | |||
1846 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
1847 | DebugLoc(), getSDVTList(VT)), | |||
1848 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
1849 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1849, __extension__ __PRETTY_FUNCTION__)); | |||
1850 | Val.MachineCPVal = v; | |||
1851 | Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1); | |||
1852 | } | |||
1853 | ||||
1854 | public: | |||
1855 | bool isMachineConstantPoolEntry() const { | |||
1856 | return Offset < 0; | |||
1857 | } | |||
1858 | ||||
1859 | const Constant *getConstVal() const { | |||
1860 | assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1860, __extension__ __PRETTY_FUNCTION__)); | |||
1861 | return Val.ConstVal; | |||
1862 | } | |||
1863 | ||||
1864 | MachineConstantPoolValue *getMachineCPVal() const { | |||
1865 | assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1865, __extension__ __PRETTY_FUNCTION__)); | |||
1866 | return Val.MachineCPVal; | |||
1867 | } | |||
1868 | ||||
1869 | int getOffset() const { | |||
1870 | return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1)); | |||
1871 | } | |||
1872 | ||||
1873 | // Return the alignment of this constant pool object, which is either 0 (for | |||
1874 | // default alignment) or the desired value. | |||
1875 | Align getAlign() const { return Alignment; } | |||
1876 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1877 | ||||
1878 | Type *getType() const; | |||
1879 | ||||
1880 | static bool classof(const SDNode *N) { | |||
1881 | return N->getOpcode() == ISD::ConstantPool || | |||
1882 | N->getOpcode() == ISD::TargetConstantPool; | |||
1883 | } | |||
1884 | }; | |||
1885 | ||||
1886 | /// Completely target-dependent object reference. | |||
1887 | class TargetIndexSDNode : public SDNode { | |||
1888 | friend class SelectionDAG; | |||
1889 | ||||
1890 | unsigned TargetFlags; | |||
1891 | int Index; | |||
1892 | int64_t Offset; | |||
1893 | ||||
1894 | public: | |||
1895 | TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF) | |||
1896 | : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), | |||
1897 | TargetFlags(TF), Index(Idx), Offset(Ofs) {} | |||
1898 | ||||
1899 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1900 | int getIndex() const { return Index; } | |||
1901 | int64_t getOffset() const { return Offset; } | |||
1902 | ||||
1903 | static bool classof(const SDNode *N) { | |||
1904 | return N->getOpcode() == ISD::TargetIndex; | |||
1905 | } | |||
1906 | }; | |||
1907 | ||||
1908 | class BasicBlockSDNode : public SDNode { | |||
1909 | friend class SelectionDAG; | |||
1910 | ||||
1911 | MachineBasicBlock *MBB; | |||
1912 | ||||
1913 | /// Debug info is meaningful and potentially useful here, but we create | |||
1914 | /// blocks out of order when they're jumped to, which makes it a bit | |||
1915 | /// harder. Let's see if we need it first. | |||
1916 | explicit BasicBlockSDNode(MachineBasicBlock *mbb) | |||
1917 | : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) | |||
1918 | {} | |||
1919 | ||||
1920 | public: | |||
1921 | MachineBasicBlock *getBasicBlock() const { return MBB; } | |||
1922 | ||||
1923 | static bool classof(const SDNode *N) { | |||
1924 | return N->getOpcode() == ISD::BasicBlock; | |||
1925 | } | |||
1926 | }; | |||
1927 | ||||
1928 | /// A "pseudo-class" with methods for operating on BUILD_VECTORs. | |||
1929 | class BuildVectorSDNode : public SDNode { | |||
1930 | public: | |||
1931 | // These are constructed as SDNodes and then cast to BuildVectorSDNodes. | |||
1932 | explicit BuildVectorSDNode() = delete; | |||
1933 | ||||
1934 | /// Check if this is a constant splat, and if so, find the | |||
1935 | /// smallest element size that splats the vector. If MinSplatBits is | |||
1936 | /// nonzero, the element size must be at least that large. Note that the | |||
1937 | /// splat element may be the entire vector (i.e., a one element vector). | |||
1938 | /// Returns the splat element value in SplatValue. Any undefined bits in | |||
1939 | /// that value are zero, and the corresponding bits in the SplatUndef mask | |||
1940 | /// are set. The SplatBitSize value is set to the splat element size in | |||
1941 | /// bits. HasAnyUndefs is set to true if any bits in the vector are | |||
1942 | /// undefined. isBigEndian describes the endianness of the target. | |||
1943 | bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, | |||
1944 | unsigned &SplatBitSize, bool &HasAnyUndefs, | |||
1945 | unsigned MinSplatBits = 0, | |||
1946 | bool isBigEndian = false) const; | |||
1947 | ||||
1948 | /// Returns the demanded splatted value or a null value if this is not a | |||
1949 | /// splat. | |||
1950 | /// | |||
1951 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
1952 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1953 | /// the vector width and set the bits where elements are undef. | |||
1954 | SDValue getSplatValue(const APInt &DemandedElts, | |||
1955 | BitVector *UndefElements = nullptr) const; | |||
1956 | ||||
1957 | /// Returns the splatted value or a null value if this is not a splat. | |||
1958 | /// | |||
1959 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1960 | /// the vector width and set the bits where elements are undef. | |||
1961 | SDValue getSplatValue(BitVector *UndefElements = nullptr) const; | |||
1962 | ||||
1963 | /// Find the shortest repeating sequence of values in the build vector. | |||
1964 | /// | |||
1965 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
1966 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
1967 | /// | |||
1968 | /// Currently this must be a power-of-2 build vector. | |||
1969 | /// The DemandedElts mask indicates the elements that must be present, | |||
1970 | /// undemanded elements in Sequence may be null (SDValue()). If passed a | |||
1971 | /// non-null UndefElements bitvector, it will resize it to match the original | |||
1972 | /// vector width and set the bits where elements are undef. If result is | |||
1973 | /// false, Sequence will be empty. | |||
1974 | bool getRepeatedSequence(const APInt &DemandedElts, | |||
1975 | SmallVectorImpl<SDValue> &Sequence, | |||
1976 | BitVector *UndefElements = nullptr) const; | |||
1977 | ||||
1978 | /// Find the shortest repeating sequence of values in the build vector. | |||
1979 | /// | |||
1980 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
1981 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
1982 | /// | |||
1983 | /// Currently this must be a power-of-2 build vector. | |||
1984 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1985 | /// the original vector width and set the bits where elements are undef. | |||
1986 | /// If result is false, Sequence will be empty. | |||
1987 | bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, | |||
1988 | BitVector *UndefElements = nullptr) const; | |||
1989 | ||||
1990 | /// Returns the demanded splatted constant or null if this is not a constant | |||
1991 | /// splat. | |||
1992 | /// | |||
1993 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
1994 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1995 | /// the vector width and set the bits where elements are undef. | |||
1996 | ConstantSDNode * | |||
1997 | getConstantSplatNode(const APInt &DemandedElts, | |||
1998 | BitVector *UndefElements = nullptr) const; | |||
1999 | ||||
2000 | /// Returns the splatted constant or null if this is not a constant | |||
2001 | /// splat. | |||
2002 | /// | |||
2003 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
2004 | /// the vector width and set the bits where elements are undef. | |||
2005 | ConstantSDNode * | |||
2006 | getConstantSplatNode(BitVector *UndefElements = nullptr) const; | |||
2007 | ||||
2008 | /// Returns the demanded splatted constant FP or null if this is not a | |||
2009 | /// constant FP splat. | |||
2010 | /// | |||
2011 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
2012 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
2013 | /// the vector width and set the bits where elements are undef. | |||
2014 | ConstantFPSDNode * | |||
2015 | getConstantFPSplatNode(const APInt &DemandedElts, | |||
2016 | BitVector *UndefElements = nullptr) const; | |||
2017 | ||||
2018 | /// Returns the splatted constant FP or null if this is not a constant | |||
2019 | /// FP splat. | |||
2020 | /// | |||
2021 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
2022 | /// the vector width and set the bits where elements are undef. | |||
2023 | ConstantFPSDNode * | |||
2024 | getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; | |||
2025 | ||||
2026 | /// If this is a constant FP splat and the splatted constant FP is an | |||
2027 | /// exact power or 2, return the log base 2 integer value. Otherwise, | |||
2028 | /// return -1. | |||
2029 | /// | |||
2030 | /// The BitWidth specifies the necessary bit precision. | |||
2031 | int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, | |||
2032 | uint32_t BitWidth) const; | |||
2033 | ||||
2034 | bool isConstant() const; | |||
2035 | ||||
2036 | static bool classof(const SDNode *N) { | |||
2037 | return N->getOpcode() == ISD::BUILD_VECTOR; | |||
2038 | } | |||
2039 | }; | |||
2040 | ||||
2041 | /// An SDNode that holds an arbitrary LLVM IR Value. This is | |||
2042 | /// used when the SelectionDAG needs to make a simple reference to something | |||
2043 | /// in the LLVM IR representation. | |||
2044 | /// | |||
2045 | class SrcValueSDNode : public SDNode { | |||
2046 | friend class SelectionDAG; | |||
2047 | ||||
2048 | const Value *V; | |||
2049 | ||||
2050 | /// Create a SrcValue for a general value. | |||
2051 | explicit SrcValueSDNode(const Value *v) | |||
2052 | : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} | |||
2053 | ||||
2054 | public: | |||
2055 | /// Return the contained Value. | |||
2056 | const Value *getValue() const { return V; } | |||
2057 | ||||
2058 | static bool classof(const SDNode *N) { | |||
2059 | return N->getOpcode() == ISD::SRCVALUE; | |||
2060 | } | |||
2061 | }; | |||
2062 | ||||
2063 | class MDNodeSDNode : public SDNode { | |||
2064 | friend class SelectionDAG; | |||
2065 | ||||
2066 | const MDNode *MD; | |||
2067 | ||||
2068 | explicit MDNodeSDNode(const MDNode *md) | |||
2069 | : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) | |||
2070 | {} | |||
2071 | ||||
2072 | public: | |||
2073 | const MDNode *getMD() const { return MD; } | |||
2074 | ||||
2075 | static bool classof(const SDNode *N) { | |||
2076 | return N->getOpcode() == ISD::MDNODE_SDNODE; | |||
2077 | } | |||
2078 | }; | |||
2079 | ||||
2080 | class RegisterSDNode : public SDNode { | |||
2081 | friend class SelectionDAG; | |||
2082 | ||||
2083 | Register Reg; | |||
2084 | ||||
2085 | RegisterSDNode(Register reg, EVT VT) | |||
2086 | : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {} | |||
2087 | ||||
2088 | public: | |||
2089 | Register getReg() const { return Reg; } | |||
2090 | ||||
2091 | static bool classof(const SDNode *N) { | |||
2092 | return N->getOpcode() == ISD::Register; | |||
2093 | } | |||
2094 | }; | |||
2095 | ||||
2096 | class RegisterMaskSDNode : public SDNode { | |||
2097 | friend class SelectionDAG; | |||
2098 | ||||
2099 | // The memory for RegMask is not owned by the node. | |||
2100 | const uint32_t *RegMask; | |||
2101 | ||||
2102 | RegisterMaskSDNode(const uint32_t *mask) | |||
2103 | : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)), | |||
2104 | RegMask(mask) {} | |||
2105 | ||||
2106 | public: | |||
2107 | const uint32_t *getRegMask() const { return RegMask; } | |||
2108 | ||||
2109 | static bool classof(const SDNode *N) { | |||
2110 | return N->getOpcode() == ISD::RegisterMask; | |||
2111 | } | |||
2112 | }; | |||
2113 | ||||
2114 | class BlockAddressSDNode : public SDNode { | |||
2115 | friend class SelectionDAG; | |||
2116 | ||||
2117 | const BlockAddress *BA; | |||
2118 | int64_t Offset; | |||
2119 | unsigned TargetFlags; | |||
2120 | ||||
2121 | BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, | |||
2122 | int64_t o, unsigned Flags) | |||
2123 | : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), | |||
2124 | BA(ba), Offset(o), TargetFlags(Flags) {} | |||
2125 | ||||
2126 | public: | |||
2127 | const BlockAddress *getBlockAddress() const { return BA; } | |||
2128 | int64_t getOffset() const { return Offset; } | |||
2129 | unsigned getTargetFlags() const { return TargetFlags; } | |||
2130 | ||||
2131 | static bool classof(const SDNode *N) { | |||
2132 | return N->getOpcode() == ISD::BlockAddress || | |||
2133 | N->getOpcode() == ISD::TargetBlockAddress; | |||
2134 | } | |||
2135 | }; | |||
2136 | ||||
2137 | class LabelSDNode : public SDNode { | |||
2138 | friend class SelectionDAG; | |||
2139 | ||||
2140 | MCSymbol *Label; | |||
2141 | ||||
2142 | LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L) | |||
2143 | : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) { | |||
2144 | assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) && "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2144, __extension__ __PRETTY_FUNCTION__)); | |||
2145 | } | |||
2146 | ||||
2147 | public: | |||
2148 | MCSymbol *getLabel() const { return Label; } | |||
2149 | ||||
2150 | static bool classof(const SDNode *N) { | |||
2151 | return N->getOpcode() == ISD::EH_LABEL || | |||
2152 | N->getOpcode() == ISD::ANNOTATION_LABEL; | |||
2153 | } | |||
2154 | }; | |||
2155 | ||||
2156 | class ExternalSymbolSDNode : public SDNode { | |||
2157 | friend class SelectionDAG; | |||
2158 | ||||
2159 | const char *Symbol; | |||
2160 | unsigned TargetFlags; | |||
2161 | ||||
2162 | ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT) | |||
2163 | : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0, | |||
2164 | DebugLoc(), getSDVTList(VT)), | |||
2165 | Symbol(Sym), TargetFlags(TF) {} | |||
2166 | ||||
2167 | public: | |||
2168 | const char *getSymbol() const { return Symbol; } | |||
2169 | unsigned getTargetFlags() const { return TargetFlags; } | |||
2170 | ||||
2171 | static bool classof(const SDNode *N) { | |||
2172 | return N->getOpcode() == ISD::ExternalSymbol || | |||
2173 | N->getOpcode() == ISD::TargetExternalSymbol; | |||
2174 | } | |||
2175 | }; | |||
2176 | ||||
2177 | class MCSymbolSDNode : public SDNode { | |||
2178 | friend class SelectionDAG; | |||
2179 | ||||
2180 | MCSymbol *Symbol; | |||
2181 | ||||
2182 | MCSymbolSDNode(MCSymbol *Symbol, EVT VT) | |||
2183 | : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {} | |||
2184 | ||||
2185 | public: | |||
2186 | MCSymbol *getMCSymbol() const { return Symbol; } | |||
2187 | ||||
2188 | static bool classof(const SDNode *N) { | |||
2189 | return N->getOpcode() == ISD::MCSymbol; | |||
2190 | } | |||
2191 | }; | |||
2192 | ||||
2193 | class CondCodeSDNode : public SDNode { | |||
2194 | friend class SelectionDAG; | |||
2195 | ||||
2196 | ISD::CondCode Condition; | |||
2197 | ||||
2198 | explicit CondCodeSDNode(ISD::CondCode Cond) | |||
2199 | : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
2200 | Condition(Cond) {} | |||
2201 | ||||
2202 | public: | |||
2203 | ISD::CondCode get() const { return Condition; } | |||
2204 | ||||
2205 | static bool classof(const SDNode *N) { | |||
2206 | return N->getOpcode() == ISD::CONDCODE; | |||
2207 | } | |||
2208 | }; | |||
2209 | ||||
2210 | /// This class is used to represent EVT's, which are used | |||
2211 | /// to parameterize some operations. | |||
2212 | class VTSDNode : public SDNode { | |||
2213 | friend class SelectionDAG; | |||
2214 | ||||
2215 | EVT ValueType; | |||
2216 | ||||
2217 | explicit VTSDNode(EVT VT) | |||
2218 | : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
2219 | ValueType(VT) {} | |||
2220 | ||||
2221 | public: | |||
2222 | EVT getVT() const { return ValueType; } | |||
2223 | ||||
2224 | static bool classof(const SDNode *N) { | |||
2225 | return N->getOpcode() == ISD::VALUETYPE; | |||
2226 | } | |||
2227 | }; | |||
2228 | ||||
2229 | /// Base class for LoadSDNode and StoreSDNode | |||
2230 | class LSBaseSDNode : public MemSDNode { | |||
2231 | public: | |||
2232 | LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, | |||
2233 | SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, | |||
2234 | MachineMemOperand *MMO) | |||
2235 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
2236 | LSBaseSDNodeBits.AddressingMode = AM; | |||
2237 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2237, __extension__ __PRETTY_FUNCTION__)); | |||
2238 | } | |||
2239 | ||||
2240 | const SDValue &getOffset() const { | |||
2241 | return getOperand(getOpcode() == ISD::LOAD ? 2 : 3); | |||
2242 | } | |||
2243 | ||||
2244 | /// Return the addressing mode for this load or store: | |||
2245 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
2246 | ISD::MemIndexedMode getAddressingMode() const { | |||
2247 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
2248 | } | |||
2249 | ||||
2250 | /// Return true if this is a pre/post inc/dec load/store. | |||
2251 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
2252 | ||||
2253 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
2254 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
2255 | ||||
2256 | static bool classof(const SDNode *N) { | |||
2257 | return N->getOpcode() == ISD::LOAD || | |||
2258 | N->getOpcode() == ISD::STORE; | |||
2259 | } | |||
2260 | }; | |||
2261 | ||||
2262 | /// This class is used to represent ISD::LOAD nodes. | |||
2263 | class LoadSDNode : public LSBaseSDNode { | |||
2264 | friend class SelectionDAG; | |||
2265 | ||||
2266 | LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2267 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, | |||
2268 | MachineMemOperand *MMO) | |||
2269 | : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
2270 | LoadSDNodeBits.ExtTy = ETy; | |||
2271 | assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!" ) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2271, __extension__ __PRETTY_FUNCTION__)); | |||
2272 | assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!" ) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2272, __extension__ __PRETTY_FUNCTION__)); | |||
2273 | } | |||
2274 | ||||
2275 | public: | |||
2276 | /// Return whether this is a plain node, | |||
2277 | /// or one of the varieties of value-extending loads. | |||
2278 | ISD::LoadExtType getExtensionType() const { | |||
2279 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
2280 | } | |||
2281 | ||||
2282 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
2283 | const SDValue &getOffset() const { return getOperand(2); } | |||
2284 | ||||
2285 | static bool classof(const SDNode *N) { | |||
2286 | return N->getOpcode() == ISD::LOAD; | |||
2287 | } | |||
2288 | }; | |||
2289 | ||||
2290 | /// This class is used to represent ISD::STORE nodes. | |||
2291 | class StoreSDNode : public LSBaseSDNode { | |||
2292 | friend class SelectionDAG; | |||
2293 | ||||
2294 | StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2295 | ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, | |||
2296 | MachineMemOperand *MMO) | |||
2297 | : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
2298 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
2299 | assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!" ) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2299, __extension__ __PRETTY_FUNCTION__)); | |||
2300 | assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!" ) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2300, __extension__ __PRETTY_FUNCTION__)); | |||
2301 | } | |||
2302 | ||||
2303 | public: | |||
2304 | /// Return true if the op does a truncation before store. | |||
2305 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
2306 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
2307 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
2308 | void setTruncatingStore(bool Truncating) { | |||
2309 | StoreSDNodeBits.IsTruncating = Truncating; | |||
2310 | } | |||
2311 | ||||
2312 | const SDValue &getValue() const { return getOperand(1); } | |||
2313 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
2314 | const SDValue &getOffset() const { return getOperand(3); } | |||
2315 | ||||
2316 | static bool classof(const SDNode *N) { | |||
2317 | return N->getOpcode() == ISD::STORE; | |||
2318 | } | |||
2319 | }; | |||
2320 | ||||
2321 | /// This base class is used to represent MLOAD and MSTORE nodes | |||
2322 | class MaskedLoadStoreSDNode : public MemSDNode { | |||
2323 | public: | |||
2324 | friend class SelectionDAG; | |||
2325 | ||||
2326 | MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
2327 | const DebugLoc &dl, SDVTList VTs, | |||
2328 | ISD::MemIndexedMode AM, EVT MemVT, | |||
2329 | MachineMemOperand *MMO) | |||
2330 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
2331 | LSBaseSDNodeBits.AddressingMode = AM; | |||
2332 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2332, __extension__ __PRETTY_FUNCTION__)); | |||
2333 | } | |||
2334 | ||||
2335 | // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru) | |||
2336 | // MaskedStoreSDNode (Chain, data, ptr, offset, mask) | |||
2337 | // Mask is a vector of i1 elements | |||
2338 | const SDValue &getOffset() const { | |||
2339 | return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3); | |||
2340 | } | |||
2341 | const SDValue &getMask() const { | |||
2342 | return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4); | |||
2343 | } | |||
2344 | ||||
2345 | /// Return the addressing mode for this load or store: | |||
2346 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
2347 | ISD::MemIndexedMode getAddressingMode() const { | |||
2348 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
2349 | } | |||
2350 | ||||
2351 | /// Return true if this is a pre/post inc/dec load/store. | |||
2352 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
2353 | ||||
2354 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
2355 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
2356 | ||||
2357 | static bool classof(const SDNode *N) { | |||
2358 | return N->getOpcode() == ISD::MLOAD || | |||
2359 | N->getOpcode() == ISD::MSTORE; | |||
2360 | } | |||
2361 | }; | |||
2362 | ||||
2363 | /// This class is used to represent an MLOAD node | |||
2364 | class MaskedLoadSDNode : public MaskedLoadStoreSDNode { | |||
2365 | public: | |||
2366 | friend class SelectionDAG; | |||
2367 | ||||
2368 | MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2369 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, | |||
2370 | bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) | |||
2371 | : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
2372 | LoadSDNodeBits.ExtTy = ETy; | |||
2373 | LoadSDNodeBits.IsExpanding = IsExpanding; | |||
2374 | } | |||
2375 | ||||
2376 | ISD::LoadExtType getExtensionType() const { | |||
2377 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
2378 | } | |||
2379 | ||||
2380 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
2381 | const SDValue &getOffset() const { return getOperand(2); } | |||
2382 | const SDValue &getMask() const { return getOperand(3); } | |||
2383 | const SDValue &getPassThru() const { return getOperand(4); } | |||
2384 | ||||
2385 | static bool classof(const SDNode *N) { | |||
2386 | return N->getOpcode() == ISD::MLOAD; | |||
2387 | } | |||
2388 | ||||
2389 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } | |||
2390 | }; | |||
2391 | ||||
2392 | /// This class is used to represent an MSTORE node | |||
2393 | class MaskedStoreSDNode : public MaskedLoadStoreSDNode { | |||
2394 | public: | |||
2395 | friend class SelectionDAG; | |||
2396 | ||||
2397 | MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2398 | ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, | |||
2399 | EVT MemVT, MachineMemOperand *MMO) | |||
2400 | : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
2401 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
2402 | StoreSDNodeBits.IsCompressing = isCompressing; | |||
2403 | } | |||
2404 | ||||
2405 | /// Return true if the op does a truncation before store. | |||
2406 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
2407 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
2408 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
2409 | ||||
2410 | /// Returns true if the op does a compression to the vector before storing. | |||
2411 | /// The node contiguously stores the active elements (integers or floats) | |||
2412 | /// in src (those with their respective bit set in writemask k) to unaligned | |||
2413 | /// memory at base_addr. | |||
2414 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } | |||
2415 | ||||
2416 | const SDValue &getValue() const { return getOperand(1); } | |||
2417 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
2418 | const SDValue &getOffset() const { return getOperand(3); } | |||
2419 | const SDValue &getMask() const { return getOperand(4); } | |||
2420 | ||||
2421 | static bool classof(const SDNode *N) { | |||
2422 | return N->getOpcode() == ISD::MSTORE; | |||
2423 | } | |||
2424 | }; | |||
2425 | ||||
2426 | /// This is a base class used to represent | |||
2427 | /// MGATHER and MSCATTER nodes | |||
2428 | /// | |||
2429 | class MaskedGatherScatterSDNode : public MemSDNode { | |||
2430 | public: | |||
2431 | friend class SelectionDAG; | |||
2432 | ||||
2433 | MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
2434 | const DebugLoc &dl, SDVTList VTs, EVT MemVT, | |||
2435 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) | |||
2436 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
2437 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
2438 | assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType && "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2438, __extension__ __PRETTY_FUNCTION__)); | |||
2439 | } | |||
2440 | ||||
2441 | /// How is Index applied to BasePtr when computing addresses. | |||
2442 | ISD::MemIndexType getIndexType() const { | |||
2443 | return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); | |||
2444 | } | |||
2445 | void setIndexType(ISD::MemIndexType IndexType) { | |||
2446 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
2447 | } | |||
2448 | bool isIndexScaled() const { | |||
2449 | return (getIndexType() == ISD::SIGNED_SCALED) || | |||
2450 | (getIndexType() == ISD::UNSIGNED_SCALED); | |||
2451 | } | |||
2452 | bool isIndexSigned() const { | |||
2453 | return (getIndexType() == ISD::SIGNED_SCALED) || | |||
2454 | (getIndexType() == ISD::SIGNED_UNSCALED); | |||
2455 | } | |||
2456 | ||||
2457 | // In the both nodes address is Op1, mask is Op2: | |||
2458 | // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale) | |||
2459 | // MaskedScatterSDNode (Chain, value, mask, base, index, scale) | |||
2460 | // Mask is a vector of i1 elements | |||
2461 | const SDValue &getBasePtr() const { return getOperand(3); } | |||
2462 | const SDValue &getIndex() const { return getOperand(4); } | |||
2463 | const SDValue &getMask() const { return getOperand(2); } | |||
2464 | const SDValue &getScale() const { return getOperand(5); } | |||
2465 | ||||
2466 | static bool classof(const SDNode *N) { | |||
2467 | return N->getOpcode() == ISD::MGATHER || | |||
2468 | N->getOpcode() == ISD::MSCATTER; | |||
2469 | } | |||
2470 | }; | |||
2471 | ||||
2472 | /// This class is used to represent an MGATHER node | |||
2473 | /// | |||
2474 | class MaskedGatherSDNode : public MaskedGatherScatterSDNode { | |||
2475 | public: | |||
2476 | friend class SelectionDAG; | |||
2477 | ||||
2478 | MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2479 | EVT MemVT, MachineMemOperand *MMO, | |||
2480 | ISD::MemIndexType IndexType, ISD::LoadExtType ETy) | |||
2481 | : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO, | |||
2482 | IndexType) { | |||
2483 | LoadSDNodeBits.ExtTy = ETy; | |||
2484 | } | |||
2485 | ||||
2486 | const SDValue &getPassThru() const { return getOperand(1); } | |||
2487 | ||||
2488 | ISD::LoadExtType getExtensionType() const { | |||
2489 | return ISD::LoadExtType(LoadSDNodeBits.ExtTy); | |||
2490 | } | |||
2491 | ||||
2492 | static bool classof(const SDNode *N) { | |||
2493 | return N->getOpcode() == ISD::MGATHER; | |||
2494 | } | |||
2495 | }; | |||
2496 | ||||
2497 | /// This class is used to represent an MSCATTER node | |||
2498 | /// | |||
2499 | class MaskedScatterSDNode : public MaskedGatherScatterSDNode { | |||
2500 | public: | |||
2501 | friend class SelectionDAG; | |||
2502 | ||||
2503 | MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
2504 | EVT MemVT, MachineMemOperand *MMO, | |||
2505 | ISD::MemIndexType IndexType, bool IsTrunc) | |||
2506 | : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO, | |||
2507 | IndexType) { | |||
2508 | StoreSDNodeBits.IsTruncating = IsTrunc; | |||
2509 | } | |||
2510 | ||||
2511 | /// Return true if the op does a truncation before store. | |||
2512 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
2513 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
2514 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
2515 | ||||
2516 | const SDValue &getValue() const { return getOperand(1); } | |||
2517 | ||||
2518 | static bool classof(const SDNode *N) { | |||
2519 | return N->getOpcode() == ISD::MSCATTER; | |||
2520 | } | |||
2521 | }; | |||
2522 | ||||
2523 | /// An SDNode that represents everything that will be needed | |||
2524 | /// to construct a MachineInstr. These nodes are created during the | |||
2525 | /// instruction selection proper phase. | |||
2526 | /// | |||
2527 | /// Note that the only supported way to set the `memoperands` is by calling the | |||
2528 | /// `SelectionDAG::setNodeMemRefs` function as the memory management happens | |||
2529 | /// inside the DAG rather than in the node. | |||
2530 | class MachineSDNode : public SDNode { | |||
2531 | private: | |||
2532 | friend class SelectionDAG; | |||
2533 | ||||
2534 | MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) | |||
2535 | : SDNode(Opc, Order, DL, VTs) {} | |||
2536 | ||||
2537 | // We use a pointer union between a single `MachineMemOperand` pointer and | |||
2538 | // a pointer to an array of `MachineMemOperand` pointers. This is null when | |||
2539 | // the number of these is zero, the single pointer variant used when the | |||
2540 | // number is one, and the array is used for larger numbers. | |||
2541 | // | |||
2542 | // The array is allocated via the `SelectionDAG`'s allocator and so will | |||
2543 | // always live until the DAG is cleaned up and doesn't require ownership here. | |||
2544 | // | |||
2545 | // We can't use something simpler like `TinyPtrVector` here because `SDNode` | |||
2546 | // subclasses aren't managed in a conforming C++ manner. See the comments on | |||
2547 | // `SelectionDAG::MorphNodeTo` which details what all goes on, but the | |||
2548 | // constraint here is that these don't manage memory with their constructor or | |||
2549 | // destructor and can be initialized to a good state even if they start off | |||
2550 | // uninitialized. | |||
2551 | PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {}; | |||
2552 | ||||
2553 | // Note that this could be folded into the above `MemRefs` member if doing so | |||
2554 | // is advantageous at some point. We don't need to store this in most cases. | |||
2555 | // However, at the moment this doesn't appear to make the allocation any | |||
2556 | // smaller and makes the code somewhat simpler to read. | |||
2557 | int NumMemRefs = 0; | |||
2558 | ||||
2559 | public: | |||
2560 | using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator; | |||
2561 | ||||
2562 | ArrayRef<MachineMemOperand *> memoperands() const { | |||
2563 | // Special case the common cases. | |||
2564 | if (NumMemRefs == 0) | |||
2565 | return {}; | |||
2566 | if (NumMemRefs == 1) | |||
2567 | return makeArrayRef(MemRefs.getAddrOfPtr1(), 1); | |||
2568 | ||||
2569 | // Otherwise we have an actual array. | |||
2570 | return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs); | |||
2571 | } | |||
2572 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } | |||
2573 | mmo_iterator memoperands_end() const { return memoperands().end(); } | |||
2574 | bool memoperands_empty() const { return memoperands().empty(); } | |||
2575 | ||||
2576 | /// Clear out the memory reference descriptor list. | |||
2577 | void clearMemRefs() { | |||
2578 | MemRefs = nullptr; | |||
2579 | NumMemRefs = 0; | |||
2580 | } | |||
2581 | ||||
2582 | static bool classof(const SDNode *N) { | |||
2583 | return N->isMachineOpcode(); | |||
2584 | } | |||
2585 | }; | |||
2586 | ||||
2587 | /// An SDNode that records if a register contains a value that is guaranteed to | |||
2588 | /// be aligned accordingly. | |||
2589 | class AssertAlignSDNode : public SDNode { | |||
2590 | Align Alignment; | |||
2591 | ||||
2592 | public: | |||
2593 | AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A) | |||
2594 | : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {} | |||
2595 | ||||
2596 | Align getAlign() const { return Alignment; } | |||
2597 | ||||
2598 | static bool classof(const SDNode *N) { | |||
2599 | return N->getOpcode() == ISD::AssertAlign; | |||
2600 | } | |||
2601 | }; | |||
2602 | ||||
2603 | class SDNodeIterator { | |||
2604 | const SDNode *Node; | |||
2605 | unsigned Operand; | |||
2606 | ||||
2607 | SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {} | |||
2608 | ||||
2609 | public: | |||
2610 | using iterator_category = std::forward_iterator_tag; | |||
2611 | using value_type = SDNode; | |||
2612 | using difference_type = std::ptrdiff_t; | |||
2613 | using pointer = value_type *; | |||
2614 | using reference = value_type &; | |||
2615 | ||||
2616 | bool operator==(const SDNodeIterator& x) const { | |||
2617 | return Operand == x.Operand; | |||
2618 | } | |||
2619 | bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } | |||
2620 | ||||
2621 | pointer operator*() const { | |||
2622 | return Node->getOperand(Operand).getNode(); | |||
2623 | } | |||
2624 | pointer operator->() const { return operator*(); } | |||
2625 | ||||
2626 | SDNodeIterator& operator++() { // Preincrement | |||
2627 | ++Operand; | |||
2628 | return *this; | |||
2629 | } | |||
2630 | SDNodeIterator operator++(int) { // Postincrement | |||
2631 | SDNodeIterator tmp = *this; ++*this; return tmp; | |||
2632 | } | |||
2633 | size_t operator-(SDNodeIterator Other) const { | |||
2634 | assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2635, __extension__ __PRETTY_FUNCTION__)) | |||
2635 | "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2635, __extension__ __PRETTY_FUNCTION__)); | |||
2636 | return Operand - Other.Operand; | |||
2637 | } | |||
2638 | ||||
2639 | static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); } | |||
2640 | static SDNodeIterator end (const SDNode *N) { | |||
2641 | return SDNodeIterator(N, N->getNumOperands()); | |||
2642 | } | |||
2643 | ||||
2644 | unsigned getOperand() const { return Operand; } | |||
2645 | const SDNode *getNode() const { return Node; } | |||
2646 | }; | |||
2647 | ||||
2648 | template <> struct GraphTraits<SDNode*> { | |||
2649 | using NodeRef = SDNode *; | |||
2650 | using ChildIteratorType = SDNodeIterator; | |||
2651 | ||||
2652 | static NodeRef getEntryNode(SDNode *N) { return N; } | |||
2653 | ||||
2654 | static ChildIteratorType child_begin(NodeRef N) { | |||
2655 | return SDNodeIterator::begin(N); | |||
2656 | } | |||
2657 | ||||
2658 | static ChildIteratorType child_end(NodeRef N) { | |||
2659 | return SDNodeIterator::end(N); | |||
2660 | } | |||
2661 | }; | |||
2662 | ||||
2663 | /// A representation of the largest SDNode, for use in sizeof(). | |||
2664 | /// | |||
2665 | /// This needs to be a union because the largest node differs on 32 bit systems | |||
2666 | /// with 4 and 8 byte pointer alignment, respectively. | |||
2667 | using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, | |||
2668 | BlockAddressSDNode, | |||
2669 | GlobalAddressSDNode, | |||
2670 | PseudoProbeSDNode>; | |||
2671 | ||||
2672 | /// The SDNode class with the greatest alignment requirement. | |||
2673 | using MostAlignedSDNode = GlobalAddressSDNode; | |||
2674 | ||||
2675 | namespace ISD { | |||
2676 | ||||
2677 | /// Returns true if the specified node is a non-extending and unindexed load. | |||
2678 | inline bool isNormalLoad(const SDNode *N) { | |||
2679 | const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N); | |||
2680 | return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && | |||
2681 | Ld->getAddressingMode() == ISD::UNINDEXED; | |||
2682 | } | |||
2683 | ||||
2684 | /// Returns true if the specified node is a non-extending load. | |||
2685 | inline bool isNON_EXTLoad(const SDNode *N) { | |||
2686 | return isa<LoadSDNode>(N) && | |||
2687 | cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; | |||
2688 | } | |||
2689 | ||||
2690 | /// Returns true if the specified node is a EXTLOAD. | |||
2691 | inline bool isEXTLoad(const SDNode *N) { | |||
2692 | return isa<LoadSDNode>(N) && | |||
2693 | cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; | |||
2694 | } | |||
2695 | ||||
2696 | /// Returns true if the specified node is a SEXTLOAD. | |||
2697 | inline bool isSEXTLoad(const SDNode *N) { | |||
2698 | return isa<LoadSDNode>(N) && | |||
2699 | cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; | |||
2700 | } | |||
2701 | ||||
2702 | /// Returns true if the specified node is a ZEXTLOAD. | |||
2703 | inline bool isZEXTLoad(const SDNode *N) { | |||
2704 | return isa<LoadSDNode>(N) && | |||
2705 | cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; | |||
2706 | } | |||
2707 | ||||
2708 | /// Returns true if the specified node is an unindexed load. | |||
2709 | inline bool isUNINDEXEDLoad(const SDNode *N) { | |||
2710 | return isa<LoadSDNode>(N) && | |||
2711 | cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
2712 | } | |||
2713 | ||||
2714 | /// Returns true if the specified node is a non-truncating | |||
2715 | /// and unindexed store. | |||
2716 | inline bool isNormalStore(const SDNode *N) { | |||
2717 | const StoreSDNode *St = dyn_cast<StoreSDNode>(N); | |||
2718 | return St && !St->isTruncatingStore() && | |||
2719 | St->getAddressingMode() == ISD::UNINDEXED; | |||
2720 | } | |||
2721 | ||||
2722 | /// Returns true if the specified node is an unindexed store. | |||
2723 | inline bool isUNINDEXEDStore(const SDNode *N) { | |||
2724 | return isa<StoreSDNode>(N) && | |||
2725 | cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
2726 | } | |||
2727 | ||||
2728 | /// Attempt to match a unary predicate against a scalar/splat constant or | |||
2729 | /// every element of a constant BUILD_VECTOR. | |||
2730 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
2731 | bool matchUnaryPredicate(SDValue Op, | |||
2732 | std::function<bool(ConstantSDNode *)> Match, | |||
2733 | bool AllowUndefs = false); | |||
2734 | ||||
2735 | /// Attempt to match a binary predicate against a pair of scalar/splat | |||
2736 | /// constants or every element of a pair of constant BUILD_VECTORs. | |||
2737 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
2738 | /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match. | |||
2739 | bool matchBinaryPredicate( | |||
2740 | SDValue LHS, SDValue RHS, | |||
2741 | std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, | |||
2742 | bool AllowUndefs = false, bool AllowTypeMismatch = false); | |||
2743 | ||||
2744 | /// Returns true if the specified value is the overflow result from one | |||
2745 | /// of the overflow intrinsic nodes. | |||
2746 | inline bool isOverflowIntrOpRes(SDValue Op) { | |||
2747 | unsigned Opc = Op.getOpcode(); | |||
2748 | return (Op.getResNo() == 1 && | |||
2749 | (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || | |||
2750 | Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)); | |||
2751 | } | |||
2752 | ||||
2753 | } // end namespace ISD | |||
2754 | ||||
2755 | } // end namespace llvm | |||
2756 | ||||
2757 | #endif // LLVM_CODEGEN_SELECTIONDAGNODES_H |