File: | llvm/include/llvm/CodeGen/SelectionDAGNodes.h |
Warning: | line 1163, column 10 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the interfaces that Hexagon uses to lower LLVM code | |||
10 | // into a selection DAG. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "HexagonISelLowering.h" | |||
15 | #include "Hexagon.h" | |||
16 | #include "HexagonMachineFunctionInfo.h" | |||
17 | #include "HexagonRegisterInfo.h" | |||
18 | #include "HexagonSubtarget.h" | |||
19 | #include "HexagonTargetMachine.h" | |||
20 | #include "HexagonTargetObjectFile.h" | |||
21 | #include "llvm/ADT/APInt.h" | |||
22 | #include "llvm/ADT/ArrayRef.h" | |||
23 | #include "llvm/ADT/SmallVector.h" | |||
24 | #include "llvm/ADT/StringSwitch.h" | |||
25 | #include "llvm/CodeGen/CallingConvLower.h" | |||
26 | #include "llvm/CodeGen/MachineFrameInfo.h" | |||
27 | #include "llvm/CodeGen/MachineFunction.h" | |||
28 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
29 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
30 | #include "llvm/CodeGen/RuntimeLibcalls.h" | |||
31 | #include "llvm/CodeGen/SelectionDAG.h" | |||
32 | #include "llvm/CodeGen/TargetCallingConv.h" | |||
33 | #include "llvm/CodeGen/ValueTypes.h" | |||
34 | #include "llvm/IR/BasicBlock.h" | |||
35 | #include "llvm/IR/CallingConv.h" | |||
36 | #include "llvm/IR/DataLayout.h" | |||
37 | #include "llvm/IR/DerivedTypes.h" | |||
38 | #include "llvm/IR/Function.h" | |||
39 | #include "llvm/IR/GlobalValue.h" | |||
40 | #include "llvm/IR/InlineAsm.h" | |||
41 | #include "llvm/IR/Instructions.h" | |||
42 | #include "llvm/IR/Intrinsics.h" | |||
43 | #include "llvm/IR/IntrinsicInst.h" | |||
44 | #include "llvm/IR/Module.h" | |||
45 | #include "llvm/IR/Type.h" | |||
46 | #include "llvm/IR/Value.h" | |||
47 | #include "llvm/MC/MCRegisterInfo.h" | |||
48 | #include "llvm/Support/Casting.h" | |||
49 | #include "llvm/Support/CodeGen.h" | |||
50 | #include "llvm/Support/CommandLine.h" | |||
51 | #include "llvm/Support/Debug.h" | |||
52 | #include "llvm/Support/ErrorHandling.h" | |||
53 | #include "llvm/Support/MathExtras.h" | |||
54 | #include "llvm/Support/raw_ostream.h" | |||
55 | #include "llvm/Target/TargetMachine.h" | |||
56 | #include <algorithm> | |||
57 | #include <cassert> | |||
58 | #include <cstddef> | |||
59 | #include <cstdint> | |||
60 | #include <limits> | |||
61 | #include <utility> | |||
62 | ||||
63 | using namespace llvm; | |||
64 | ||||
65 | #define DEBUG_TYPE"hexagon-lowering" "hexagon-lowering" | |||
66 | ||||
67 | static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables", | |||
68 | cl::init(true), cl::Hidden, | |||
69 | cl::desc("Control jump table emission on Hexagon target")); | |||
70 | ||||
71 | static cl::opt<bool> EnableHexSDNodeSched("enable-hexagon-sdnode-sched", | |||
72 | cl::Hidden, cl::ZeroOrMore, cl::init(false), | |||
73 | cl::desc("Enable Hexagon SDNode scheduling")); | |||
74 | ||||
75 | static cl::opt<bool> EnableFastMath("ffast-math", | |||
76 | cl::Hidden, cl::ZeroOrMore, cl::init(false), | |||
77 | cl::desc("Enable Fast Math processing")); | |||
78 | ||||
79 | static cl::opt<int> MinimumJumpTables("minimum-jump-tables", | |||
80 | cl::Hidden, cl::ZeroOrMore, cl::init(5), | |||
81 | cl::desc("Set minimum jump tables")); | |||
82 | ||||
83 | static cl::opt<int> MaxStoresPerMemcpyCL("max-store-memcpy", | |||
84 | cl::Hidden, cl::ZeroOrMore, cl::init(6), | |||
85 | cl::desc("Max #stores to inline memcpy")); | |||
86 | ||||
87 | static cl::opt<int> MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os", | |||
88 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | |||
89 | cl::desc("Max #stores to inline memcpy")); | |||
90 | ||||
91 | static cl::opt<int> MaxStoresPerMemmoveCL("max-store-memmove", | |||
92 | cl::Hidden, cl::ZeroOrMore, cl::init(6), | |||
93 | cl::desc("Max #stores to inline memmove")); | |||
94 | ||||
95 | static cl::opt<int> MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os", | |||
96 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | |||
97 | cl::desc("Max #stores to inline memmove")); | |||
98 | ||||
99 | static cl::opt<int> MaxStoresPerMemsetCL("max-store-memset", | |||
100 | cl::Hidden, cl::ZeroOrMore, cl::init(8), | |||
101 | cl::desc("Max #stores to inline memset")); | |||
102 | ||||
103 | static cl::opt<int> MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", | |||
104 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | |||
105 | cl::desc("Max #stores to inline memset")); | |||
106 | ||||
107 | static cl::opt<bool> AlignLoads("hexagon-align-loads", | |||
108 | cl::Hidden, cl::init(false), | |||
109 | cl::desc("Rewrite unaligned loads as a pair of aligned loads")); | |||
110 | ||||
111 | ||||
112 | namespace { | |||
113 | ||||
114 | class HexagonCCState : public CCState { | |||
115 | unsigned NumNamedVarArgParams = 0; | |||
116 | ||||
117 | public: | |||
118 | HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF, | |||
119 | SmallVectorImpl<CCValAssign> &locs, LLVMContext &C, | |||
120 | unsigned NumNamedArgs) | |||
121 | : CCState(CC, IsVarArg, MF, locs, C), | |||
122 | NumNamedVarArgParams(NumNamedArgs) {} | |||
123 | unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; } | |||
124 | }; | |||
125 | ||||
126 | } // end anonymous namespace | |||
127 | ||||
128 | ||||
129 | // Implement calling convention for Hexagon. | |||
130 | ||||
131 | static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT, | |||
132 | CCValAssign::LocInfo &LocInfo, | |||
133 | ISD::ArgFlagsTy &ArgFlags, CCState &State) { | |||
134 | static const MCPhysReg ArgRegs[] = { | |||
135 | Hexagon::R0, Hexagon::R1, Hexagon::R2, | |||
136 | Hexagon::R3, Hexagon::R4, Hexagon::R5 | |||
137 | }; | |||
138 | const unsigned NumArgRegs = array_lengthof(ArgRegs); | |||
139 | unsigned RegNum = State.getFirstUnallocated(ArgRegs); | |||
140 | ||||
141 | // RegNum is an index into ArgRegs: skip a register if RegNum is odd. | |||
142 | if (RegNum != NumArgRegs && RegNum % 2 == 1) | |||
143 | State.AllocateReg(ArgRegs[RegNum]); | |||
144 | ||||
145 | // Always return false here, as this function only makes sure that the first | |||
146 | // unallocated register has an even register number and does not actually | |||
147 | // allocate a register for the current argument. | |||
148 | return false; | |||
149 | } | |||
150 | ||||
151 | #include "HexagonGenCallingConv.inc" | |||
152 | ||||
153 | ||||
154 | SDValue | |||
155 | HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) | |||
156 | const { | |||
157 | return SDValue(); | |||
158 | } | |||
159 | ||||
160 | /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified | |||
161 | /// by "Src" to address "Dst" of size "Size". Alignment information is | |||
162 | /// specified by the specific parameter attribute. The copy will be passed as | |||
163 | /// a byval function parameter. Sometimes what we are copying is the end of a | |||
164 | /// larger object, the part that does not fit in registers. | |||
165 | static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, | |||
166 | SDValue Chain, ISD::ArgFlagsTy Flags, | |||
167 | SelectionDAG &DAG, const SDLoc &dl) { | |||
168 | SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); | |||
169 | return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), | |||
170 | /*isVolatile=*/false, /*AlwaysInline=*/false, | |||
171 | /*isTailCall=*/false, | |||
172 | MachinePointerInfo(), MachinePointerInfo()); | |||
173 | } | |||
174 | ||||
175 | bool | |||
176 | HexagonTargetLowering::CanLowerReturn( | |||
177 | CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, | |||
178 | const SmallVectorImpl<ISD::OutputArg> &Outs, | |||
179 | LLVMContext &Context) const { | |||
180 | SmallVector<CCValAssign, 16> RVLocs; | |||
181 | CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); | |||
182 | ||||
183 | if (MF.getSubtarget<HexagonSubtarget>().useHVXOps()) | |||
184 | return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX); | |||
185 | return CCInfo.CheckReturn(Outs, RetCC_Hexagon); | |||
186 | } | |||
187 | ||||
188 | // LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is | |||
189 | // passed by value, the function prototype is modified to return void and | |||
190 | // the value is stored in memory pointed by a pointer passed by caller. | |||
191 | SDValue | |||
192 | HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, | |||
193 | bool IsVarArg, | |||
194 | const SmallVectorImpl<ISD::OutputArg> &Outs, | |||
195 | const SmallVectorImpl<SDValue> &OutVals, | |||
196 | const SDLoc &dl, SelectionDAG &DAG) const { | |||
197 | // CCValAssign - represent the assignment of the return value to locations. | |||
198 | SmallVector<CCValAssign, 16> RVLocs; | |||
199 | ||||
200 | // CCState - Info about the registers and stack slot. | |||
201 | CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, | |||
202 | *DAG.getContext()); | |||
203 | ||||
204 | // Analyze return values of ISD::RET | |||
205 | if (Subtarget.useHVXOps()) | |||
206 | CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX); | |||
207 | else | |||
208 | CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon); | |||
209 | ||||
210 | SDValue Flag; | |||
211 | SmallVector<SDValue, 4> RetOps(1, Chain); | |||
212 | ||||
213 | // Copy the result values into the output registers. | |||
214 | for (unsigned i = 0; i != RVLocs.size(); ++i) { | |||
215 | CCValAssign &VA = RVLocs[i]; | |||
216 | ||||
217 | Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), OutVals[i], Flag); | |||
218 | ||||
219 | // Guarantee that all emitted copies are stuck together with flags. | |||
220 | Flag = Chain.getValue(1); | |||
221 | RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); | |||
222 | } | |||
223 | ||||
224 | RetOps[0] = Chain; // Update chain. | |||
225 | ||||
226 | // Add the flag if we have it. | |||
227 | if (Flag.getNode()) | |||
228 | RetOps.push_back(Flag); | |||
229 | ||||
230 | return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps); | |||
231 | } | |||
232 | ||||
233 | bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { | |||
234 | // If either no tail call or told not to tail call at all, don't. | |||
235 | auto Attr = | |||
236 | CI->getParent()->getParent()->getFnAttribute("disable-tail-calls"); | |||
237 | if (!CI->isTailCall() || Attr.getValueAsString() == "true") | |||
238 | return false; | |||
239 | ||||
240 | return true; | |||
241 | } | |||
242 | ||||
243 | Register HexagonTargetLowering::getRegisterByName( | |||
244 | const char* RegName, EVT VT, const MachineFunction &) const { | |||
245 | // Just support r19, the linux kernel uses it. | |||
246 | Register Reg = StringSwitch<Register>(RegName) | |||
247 | .Case("r0", Hexagon::R0) | |||
248 | .Case("r1", Hexagon::R1) | |||
249 | .Case("r2", Hexagon::R2) | |||
250 | .Case("r3", Hexagon::R3) | |||
251 | .Case("r4", Hexagon::R4) | |||
252 | .Case("r5", Hexagon::R5) | |||
253 | .Case("r6", Hexagon::R6) | |||
254 | .Case("r7", Hexagon::R7) | |||
255 | .Case("r8", Hexagon::R8) | |||
256 | .Case("r9", Hexagon::R9) | |||
257 | .Case("r10", Hexagon::R10) | |||
258 | .Case("r11", Hexagon::R11) | |||
259 | .Case("r12", Hexagon::R12) | |||
260 | .Case("r13", Hexagon::R13) | |||
261 | .Case("r14", Hexagon::R14) | |||
262 | .Case("r15", Hexagon::R15) | |||
263 | .Case("r16", Hexagon::R16) | |||
264 | .Case("r17", Hexagon::R17) | |||
265 | .Case("r18", Hexagon::R18) | |||
266 | .Case("r19", Hexagon::R19) | |||
267 | .Case("r20", Hexagon::R20) | |||
268 | .Case("r21", Hexagon::R21) | |||
269 | .Case("r22", Hexagon::R22) | |||
270 | .Case("r23", Hexagon::R23) | |||
271 | .Case("r24", Hexagon::R24) | |||
272 | .Case("r25", Hexagon::R25) | |||
273 | .Case("r26", Hexagon::R26) | |||
274 | .Case("r27", Hexagon::R27) | |||
275 | .Case("r28", Hexagon::R28) | |||
276 | .Case("r29", Hexagon::R29) | |||
277 | .Case("r30", Hexagon::R30) | |||
278 | .Case("r31", Hexagon::R31) | |||
279 | .Case("r1:0", Hexagon::D0) | |||
280 | .Case("r3:2", Hexagon::D1) | |||
281 | .Case("r5:4", Hexagon::D2) | |||
282 | .Case("r7:6", Hexagon::D3) | |||
283 | .Case("r9:8", Hexagon::D4) | |||
284 | .Case("r11:10", Hexagon::D5) | |||
285 | .Case("r13:12", Hexagon::D6) | |||
286 | .Case("r15:14", Hexagon::D7) | |||
287 | .Case("r17:16", Hexagon::D8) | |||
288 | .Case("r19:18", Hexagon::D9) | |||
289 | .Case("r21:20", Hexagon::D10) | |||
290 | .Case("r23:22", Hexagon::D11) | |||
291 | .Case("r25:24", Hexagon::D12) | |||
292 | .Case("r27:26", Hexagon::D13) | |||
293 | .Case("r29:28", Hexagon::D14) | |||
294 | .Case("r31:30", Hexagon::D15) | |||
295 | .Case("sp", Hexagon::R29) | |||
296 | .Case("fp", Hexagon::R30) | |||
297 | .Case("lr", Hexagon::R31) | |||
298 | .Case("p0", Hexagon::P0) | |||
299 | .Case("p1", Hexagon::P1) | |||
300 | .Case("p2", Hexagon::P2) | |||
301 | .Case("p3", Hexagon::P3) | |||
302 | .Case("sa0", Hexagon::SA0) | |||
303 | .Case("lc0", Hexagon::LC0) | |||
304 | .Case("sa1", Hexagon::SA1) | |||
305 | .Case("lc1", Hexagon::LC1) | |||
306 | .Case("m0", Hexagon::M0) | |||
307 | .Case("m1", Hexagon::M1) | |||
308 | .Case("usr", Hexagon::USR) | |||
309 | .Case("ugp", Hexagon::UGP) | |||
310 | .Default(Register()); | |||
311 | if (Reg) | |||
312 | return Reg; | |||
313 | ||||
314 | report_fatal_error("Invalid register name global variable"); | |||
315 | } | |||
316 | ||||
317 | /// LowerCallResult - Lower the result values of an ISD::CALL into the | |||
318 | /// appropriate copies out of appropriate physical registers. This assumes that | |||
319 | /// Chain/Glue are the input chain/glue to use, and that TheCall is the call | |||
320 | /// being lowered. Returns a SDNode with the same number of values as the | |||
321 | /// ISD::CALL. | |||
322 | SDValue HexagonTargetLowering::LowerCallResult( | |||
323 | SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg, | |||
324 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, | |||
325 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, | |||
326 | const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const { | |||
327 | // Assign locations to each value returned by this call. | |||
328 | SmallVector<CCValAssign, 16> RVLocs; | |||
329 | ||||
330 | CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, | |||
331 | *DAG.getContext()); | |||
332 | ||||
333 | if (Subtarget.useHVXOps()) | |||
334 | CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX); | |||
335 | else | |||
336 | CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon); | |||
337 | ||||
338 | // Copy all of the result registers out of their specified physreg. | |||
339 | for (unsigned i = 0; i != RVLocs.size(); ++i) { | |||
340 | SDValue RetVal; | |||
341 | if (RVLocs[i].getValVT() == MVT::i1) { | |||
342 | // Return values of type MVT::i1 require special handling. The reason | |||
343 | // is that MVT::i1 is associated with the PredRegs register class, but | |||
344 | // values of that type are still returned in R0. Generate an explicit | |||
345 | // copy into a predicate register from R0, and treat the value of the | |||
346 | // predicate register as the call result. | |||
347 | auto &MRI = DAG.getMachineFunction().getRegInfo(); | |||
348 | SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(), | |||
349 | MVT::i32, Glue); | |||
350 | // FR0 = (Value, Chain, Glue) | |||
351 | Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass); | |||
352 | SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR, | |||
353 | FR0.getValue(0), FR0.getValue(2)); | |||
354 | // TPR = (Chain, Glue) | |||
355 | // Don't glue this CopyFromReg, because it copies from a virtual | |||
356 | // register. If it is glued to the call, InstrEmitter will add it | |||
357 | // as an implicit def to the call (EmitMachineNode). | |||
358 | RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1); | |||
359 | Glue = TPR.getValue(1); | |||
360 | Chain = TPR.getValue(0); | |||
361 | } else { | |||
362 | RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(), | |||
363 | RVLocs[i].getValVT(), Glue); | |||
364 | Glue = RetVal.getValue(2); | |||
365 | Chain = RetVal.getValue(1); | |||
366 | } | |||
367 | InVals.push_back(RetVal.getValue(0)); | |||
368 | } | |||
369 | ||||
370 | return Chain; | |||
371 | } | |||
372 | ||||
373 | /// LowerCall - Functions arguments are copied from virtual regs to | |||
374 | /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted. | |||
375 | SDValue | |||
376 | HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, | |||
377 | SmallVectorImpl<SDValue> &InVals) const { | |||
378 | SelectionDAG &DAG = CLI.DAG; | |||
379 | SDLoc &dl = CLI.DL; | |||
380 | SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; | |||
381 | SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; | |||
382 | SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; | |||
383 | SDValue Chain = CLI.Chain; | |||
384 | SDValue Callee = CLI.Callee; | |||
385 | CallingConv::ID CallConv = CLI.CallConv; | |||
386 | bool IsVarArg = CLI.IsVarArg; | |||
387 | bool DoesNotReturn = CLI.DoesNotReturn; | |||
388 | ||||
389 | bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet(); | |||
390 | MachineFunction &MF = DAG.getMachineFunction(); | |||
391 | MachineFrameInfo &MFI = MF.getFrameInfo(); | |||
392 | auto PtrVT = getPointerTy(MF.getDataLayout()); | |||
393 | ||||
394 | unsigned NumParams = CLI.CS.getInstruction() | |||
395 | ? CLI.CS.getFunctionType()->getNumParams() | |||
396 | : 0; | |||
397 | if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee)) | |||
398 | Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32); | |||
399 | ||||
400 | // Analyze operands of the call, assigning locations to each operand. | |||
401 | SmallVector<CCValAssign, 16> ArgLocs; | |||
402 | HexagonCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext(), | |||
403 | NumParams); | |||
404 | ||||
405 | if (Subtarget.useHVXOps()) | |||
406 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX); | |||
407 | else | |||
408 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon); | |||
409 | ||||
410 | auto Attr = MF.getFunction().getFnAttribute("disable-tail-calls"); | |||
411 | if (Attr.getValueAsString() == "true") | |||
412 | CLI.IsTailCall = false; | |||
413 | ||||
414 | if (CLI.IsTailCall) { | |||
415 | bool StructAttrFlag = MF.getFunction().hasStructRetAttr(); | |||
416 | CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, | |||
417 | IsVarArg, IsStructRet, StructAttrFlag, Outs, | |||
418 | OutVals, Ins, DAG); | |||
419 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | |||
420 | CCValAssign &VA = ArgLocs[i]; | |||
421 | if (VA.isMemLoc()) { | |||
422 | CLI.IsTailCall = false; | |||
423 | break; | |||
424 | } | |||
425 | } | |||
426 | LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false) | |||
427 | : "Argument must be passed on stack. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false) | |||
428 | "Not eligible for Tail Call\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false); | |||
429 | } | |||
430 | // Get a count of how many bytes are to be pushed on the stack. | |||
431 | unsigned NumBytes = CCInfo.getNextStackOffset(); | |||
432 | SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass; | |||
433 | SmallVector<SDValue, 8> MemOpChains; | |||
434 | ||||
435 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | |||
436 | SDValue StackPtr = | |||
437 | DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT); | |||
438 | ||||
439 | bool NeedsArgAlign = false; | |||
440 | unsigned LargestAlignSeen = 0; | |||
441 | // Walk the register/memloc assignments, inserting copies/loads. | |||
442 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | |||
443 | CCValAssign &VA = ArgLocs[i]; | |||
444 | SDValue Arg = OutVals[i]; | |||
445 | ISD::ArgFlagsTy Flags = Outs[i].Flags; | |||
446 | // Record if we need > 8 byte alignment on an argument. | |||
447 | bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT()); | |||
448 | NeedsArgAlign |= ArgAlign; | |||
449 | ||||
450 | // Promote the value if needed. | |||
451 | switch (VA.getLocInfo()) { | |||
452 | default: | |||
453 | // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt. | |||
454 | llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 454); | |||
455 | case CCValAssign::Full: | |||
456 | break; | |||
457 | case CCValAssign::BCvt: | |||
458 | Arg = DAG.getBitcast(VA.getLocVT(), Arg); | |||
459 | break; | |||
460 | case CCValAssign::SExt: | |||
461 | Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); | |||
462 | break; | |||
463 | case CCValAssign::ZExt: | |||
464 | Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); | |||
465 | break; | |||
466 | case CCValAssign::AExt: | |||
467 | Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); | |||
468 | break; | |||
469 | } | |||
470 | ||||
471 | if (VA.isMemLoc()) { | |||
472 | unsigned LocMemOffset = VA.getLocMemOffset(); | |||
473 | SDValue MemAddr = DAG.getConstant(LocMemOffset, dl, | |||
474 | StackPtr.getValueType()); | |||
475 | MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr); | |||
476 | if (ArgAlign) | |||
477 | LargestAlignSeen = std::max(LargestAlignSeen, | |||
478 | VA.getLocVT().getStoreSizeInBits() >> 3); | |||
479 | if (Flags.isByVal()) { | |||
480 | // The argument is a struct passed by value. According to LLVM, "Arg" | |||
481 | // is a pointer. | |||
482 | MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain, | |||
483 | Flags, DAG, dl)); | |||
484 | } else { | |||
485 | MachinePointerInfo LocPI = MachinePointerInfo::getStack( | |||
486 | DAG.getMachineFunction(), LocMemOffset); | |||
487 | SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI); | |||
488 | MemOpChains.push_back(S); | |||
489 | } | |||
490 | continue; | |||
491 | } | |||
492 | ||||
493 | // Arguments that can be passed on register must be kept at RegsToPass | |||
494 | // vector. | |||
495 | if (VA.isRegLoc()) | |||
496 | RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); | |||
497 | } | |||
498 | ||||
499 | if (NeedsArgAlign && Subtarget.hasV60Ops()) { | |||
500 | LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << "Function needs byte stack align due to call args\n" ; } } while (false); | |||
501 | unsigned VecAlign = HRI.getSpillAlignment(Hexagon::HvxVRRegClass); | |||
502 | LargestAlignSeen = std::max(LargestAlignSeen, VecAlign); | |||
503 | MFI.ensureMaxAlignment(LargestAlignSeen); | |||
504 | } | |||
505 | // Transform all store nodes into one single node because all store | |||
506 | // nodes are independent of each other. | |||
507 | if (!MemOpChains.empty()) | |||
508 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); | |||
509 | ||||
510 | SDValue Glue; | |||
511 | if (!CLI.IsTailCall) { | |||
512 | Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); | |||
513 | Glue = Chain.getValue(1); | |||
514 | } | |||
515 | ||||
516 | // Build a sequence of copy-to-reg nodes chained together with token | |||
517 | // chain and flag operands which copy the outgoing args into registers. | |||
518 | // The Glue is necessary since all emitted instructions must be | |||
519 | // stuck together. | |||
520 | if (!CLI.IsTailCall) { | |||
521 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | |||
522 | Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, | |||
523 | RegsToPass[i].second, Glue); | |||
524 | Glue = Chain.getValue(1); | |||
525 | } | |||
526 | } else { | |||
527 | // For tail calls lower the arguments to the 'real' stack slot. | |||
528 | // | |||
529 | // Force all the incoming stack arguments to be loaded from the stack | |||
530 | // before any new outgoing arguments are stored to the stack, because the | |||
531 | // outgoing stack slots may alias the incoming argument stack slots, and | |||
532 | // the alias isn't otherwise explicit. This is slightly more conservative | |||
533 | // than necessary, because it means that each store effectively depends | |||
534 | // on every argument instead of just those arguments it would clobber. | |||
535 | // | |||
536 | // Do not flag preceding copytoreg stuff together with the following stuff. | |||
537 | Glue = SDValue(); | |||
538 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | |||
539 | Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, | |||
540 | RegsToPass[i].second, Glue); | |||
541 | Glue = Chain.getValue(1); | |||
542 | } | |||
543 | Glue = SDValue(); | |||
544 | } | |||
545 | ||||
546 | bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls(); | |||
547 | unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0; | |||
548 | ||||
549 | // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every | |||
550 | // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol | |||
551 | // node so that legalize doesn't hack it. | |||
552 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { | |||
553 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags); | |||
554 | } else if (ExternalSymbolSDNode *S = | |||
555 | dyn_cast<ExternalSymbolSDNode>(Callee)) { | |||
556 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags); | |||
557 | } | |||
558 | ||||
559 | // Returns a chain & a flag for retval copy to use. | |||
560 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | |||
561 | SmallVector<SDValue, 8> Ops; | |||
562 | Ops.push_back(Chain); | |||
563 | Ops.push_back(Callee); | |||
564 | ||||
565 | // Add argument registers to the end of the list so that they are | |||
566 | // known live into the call. | |||
567 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | |||
568 | Ops.push_back(DAG.getRegister(RegsToPass[i].first, | |||
569 | RegsToPass[i].second.getValueType())); | |||
570 | } | |||
571 | ||||
572 | const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv); | |||
573 | assert(Mask && "Missing call preserved mask for calling convention")((Mask && "Missing call preserved mask for calling convention" ) ? static_cast<void> (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 573, __PRETTY_FUNCTION__)); | |||
574 | Ops.push_back(DAG.getRegisterMask(Mask)); | |||
575 | ||||
576 | if (Glue.getNode()) | |||
577 | Ops.push_back(Glue); | |||
578 | ||||
579 | if (CLI.IsTailCall) { | |||
580 | MFI.setHasTailCall(); | |||
581 | return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops); | |||
582 | } | |||
583 | ||||
584 | // Set this here because we need to know this for "hasFP" in frame lowering. | |||
585 | // The target-independent code calls getFrameRegister before setting it, and | |||
586 | // getFrameRegister uses hasFP to determine whether the function has FP. | |||
587 | MFI.setHasCalls(true); | |||
588 | ||||
589 | unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL; | |||
590 | Chain = DAG.getNode(OpCode, dl, NodeTys, Ops); | |||
591 | Glue = Chain.getValue(1); | |||
592 | ||||
593 | // Create the CALLSEQ_END node. | |||
594 | Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), | |||
595 | DAG.getIntPtrConstant(0, dl, true), Glue, dl); | |||
596 | Glue = Chain.getValue(1); | |||
597 | ||||
598 | // Handle result values, copying them out of physregs into vregs that we | |||
599 | // return. | |||
600 | return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG, | |||
601 | InVals, OutVals, Callee); | |||
602 | } | |||
603 | ||||
604 | /// Returns true by value, base pointer and offset pointer and addressing | |||
605 | /// mode by reference if this node can be combined with a load / store to | |||
606 | /// form a post-indexed load / store. | |||
607 | bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op, | |||
608 | SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, | |||
609 | SelectionDAG &DAG) const { | |||
610 | LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N); | |||
611 | if (!LSN) | |||
612 | return false; | |||
613 | EVT VT = LSN->getMemoryVT(); | |||
614 | if (!VT.isSimple()) | |||
615 | return false; | |||
616 | bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || | |||
617 | VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 || | |||
618 | VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 || | |||
619 | VT == MVT::v4i16 || VT == MVT::v8i8 || | |||
620 | Subtarget.isHVXVectorType(VT.getSimpleVT()); | |||
621 | if (!IsLegalType) | |||
622 | return false; | |||
623 | ||||
624 | if (Op->getOpcode() != ISD::ADD) | |||
625 | return false; | |||
626 | Base = Op->getOperand(0); | |||
627 | Offset = Op->getOperand(1); | |||
628 | if (!isa<ConstantSDNode>(Offset.getNode())) | |||
629 | return false; | |||
630 | AM = ISD::POST_INC; | |||
631 | ||||
632 | int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue(); | |||
633 | return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V); | |||
634 | } | |||
635 | ||||
636 | SDValue | |||
637 | HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const { | |||
638 | MachineFunction &MF = DAG.getMachineFunction(); | |||
639 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); | |||
640 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | |||
641 | unsigned LR = HRI.getRARegister(); | |||
642 | ||||
643 | if ((Op.getOpcode() != ISD::INLINEASM && | |||
644 | Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR()) | |||
645 | return Op; | |||
646 | ||||
647 | unsigned NumOps = Op.getNumOperands(); | |||
648 | if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue) | |||
649 | --NumOps; // Ignore the flag operand. | |||
650 | ||||
651 | for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { | |||
652 | unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue(); | |||
653 | unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); | |||
654 | ++i; // Skip the ID value. | |||
655 | ||||
656 | switch (InlineAsm::getKind(Flags)) { | |||
657 | default: | |||
658 | llvm_unreachable("Bad flags!")::llvm::llvm_unreachable_internal("Bad flags!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 658); | |||
659 | case InlineAsm::Kind_RegUse: | |||
660 | case InlineAsm::Kind_Imm: | |||
661 | case InlineAsm::Kind_Mem: | |||
662 | i += NumVals; | |||
663 | break; | |||
664 | case InlineAsm::Kind_Clobber: | |||
665 | case InlineAsm::Kind_RegDef: | |||
666 | case InlineAsm::Kind_RegDefEarlyClobber: { | |||
667 | for (; NumVals; --NumVals, ++i) { | |||
668 | unsigned Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg(); | |||
669 | if (Reg != LR) | |||
670 | continue; | |||
671 | HMFI.setHasClobberLR(true); | |||
672 | return Op; | |||
673 | } | |||
674 | break; | |||
675 | } | |||
676 | } | |||
677 | } | |||
678 | ||||
679 | return Op; | |||
680 | } | |||
681 | ||||
682 | // Need to transform ISD::PREFETCH into something that doesn't inherit | |||
683 | // all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and | |||
684 | // SDNPMayStore. | |||
685 | SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op, | |||
686 | SelectionDAG &DAG) const { | |||
687 | SDValue Chain = Op.getOperand(0); | |||
688 | SDValue Addr = Op.getOperand(1); | |||
689 | // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in, | |||
690 | // if the "reg" is fed by an "add". | |||
691 | SDLoc DL(Op); | |||
692 | SDValue Zero = DAG.getConstant(0, DL, MVT::i32); | |||
693 | return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero); | |||
694 | } | |||
695 | ||||
696 | // Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode | |||
697 | // is marked as having side-effects, while the register read on Hexagon does | |||
698 | // not have any. TableGen refuses to accept the direct pattern from that node | |||
699 | // to the A4_tfrcpp. | |||
700 | SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op, | |||
701 | SelectionDAG &DAG) const { | |||
702 | SDValue Chain = Op.getOperand(0); | |||
703 | SDLoc dl(Op); | |||
704 | SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); | |||
705 | return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain); | |||
706 | } | |||
707 | ||||
708 | SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op, | |||
709 | SelectionDAG &DAG) const { | |||
710 | SDValue Chain = Op.getOperand(0); | |||
711 | unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); | |||
712 | // Lower the hexagon_prefetch builtin to DCFETCH, as above. | |||
713 | if (IntNo == Intrinsic::hexagon_prefetch) { | |||
714 | SDValue Addr = Op.getOperand(2); | |||
715 | SDLoc DL(Op); | |||
716 | SDValue Zero = DAG.getConstant(0, DL, MVT::i32); | |||
717 | return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero); | |||
718 | } | |||
719 | return SDValue(); | |||
720 | } | |||
721 | ||||
722 | SDValue | |||
723 | HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, | |||
724 | SelectionDAG &DAG) const { | |||
725 | SDValue Chain = Op.getOperand(0); | |||
726 | SDValue Size = Op.getOperand(1); | |||
727 | SDValue Align = Op.getOperand(2); | |||
728 | SDLoc dl(Op); | |||
729 | ||||
730 | ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align); | |||
731 | assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC")((AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC" ) ? static_cast<void> (0) : __assert_fail ("AlignConst && \"Non-constant Align in LowerDYNAMIC_STACKALLOC\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 731, __PRETTY_FUNCTION__)); | |||
732 | ||||
733 | unsigned A = AlignConst->getSExtValue(); | |||
734 | auto &HFI = *Subtarget.getFrameLowering(); | |||
735 | // "Zero" means natural stack alignment. | |||
736 | if (A == 0) | |||
737 | A = HFI.getStackAlignment(); | |||
738 | ||||
739 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | |||
740 | dbgs () << __func__ << " Align: " << A << " Size: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | |||
741 | Size.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | |||
742 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | |||
743 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false); | |||
744 | ||||
745 | SDValue AC = DAG.getConstant(A, dl, MVT::i32); | |||
746 | SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); | |||
747 | SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC); | |||
748 | ||||
749 | DAG.ReplaceAllUsesOfValueWith(Op, AA); | |||
750 | return AA; | |||
751 | } | |||
752 | ||||
753 | SDValue HexagonTargetLowering::LowerFormalArguments( | |||
754 | SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, | |||
755 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, | |||
756 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { | |||
757 | MachineFunction &MF = DAG.getMachineFunction(); | |||
758 | MachineFrameInfo &MFI = MF.getFrameInfo(); | |||
759 | MachineRegisterInfo &MRI = MF.getRegInfo(); | |||
760 | ||||
761 | // Assign locations to all of the incoming arguments. | |||
762 | SmallVector<CCValAssign, 16> ArgLocs; | |||
763 | HexagonCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext(), | |||
764 | MF.getFunction().getFunctionType()->getNumParams()); | |||
765 | ||||
766 | if (Subtarget.useHVXOps()) | |||
767 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX); | |||
768 | else | |||
769 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon); | |||
770 | ||||
771 | // For LLVM, in the case when returning a struct by value (>8byte), | |||
772 | // the first argument is a pointer that points to the location on caller's | |||
773 | // stack where the return value will be stored. For Hexagon, the location on | |||
774 | // caller's stack is passed only when the struct size is smaller than (and | |||
775 | // equal to) 8 bytes. If not, no address will be passed into callee and | |||
776 | // callee return the result direclty through R0/R1. | |||
777 | ||||
778 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); | |||
779 | ||||
780 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | |||
781 | CCValAssign &VA = ArgLocs[i]; | |||
782 | ISD::ArgFlagsTy Flags = Ins[i].Flags; | |||
783 | bool ByVal = Flags.isByVal(); | |||
784 | ||||
785 | // Arguments passed in registers: | |||
786 | // 1. 32- and 64-bit values and HVX vectors are passed directly, | |||
787 | // 2. Large structs are passed via an address, and the address is | |||
788 | // passed in a register. | |||
789 | if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8) | |||
790 | llvm_unreachable("ByValSize must be bigger than 8 bytes")::llvm::llvm_unreachable_internal("ByValSize must be bigger than 8 bytes" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 790); | |||
791 | ||||
792 | bool InReg = VA.isRegLoc() && | |||
793 | (!ByVal || (ByVal && Flags.getByValSize() > 8)); | |||
794 | ||||
795 | if (InReg) { | |||
796 | MVT RegVT = VA.getLocVT(); | |||
797 | if (VA.getLocInfo() == CCValAssign::BCvt) | |||
798 | RegVT = VA.getValVT(); | |||
799 | ||||
800 | const TargetRegisterClass *RC = getRegClassFor(RegVT); | |||
801 | Register VReg = MRI.createVirtualRegister(RC); | |||
802 | SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT); | |||
803 | ||||
804 | // Treat values of type MVT::i1 specially: they are passed in | |||
805 | // registers of type i32, but they need to remain as values of | |||
806 | // type i1 for consistency of the argument lowering. | |||
807 | if (VA.getValVT() == MVT::i1) { | |||
808 | assert(RegVT.getSizeInBits() <= 32)((RegVT.getSizeInBits() <= 32) ? static_cast<void> ( 0) : __assert_fail ("RegVT.getSizeInBits() <= 32", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 808, __PRETTY_FUNCTION__)); | |||
809 | SDValue T = DAG.getNode(ISD::AND, dl, RegVT, | |||
810 | Copy, DAG.getConstant(1, dl, RegVT)); | |||
811 | Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT), | |||
812 | ISD::SETNE); | |||
813 | } else { | |||
814 | #ifndef NDEBUG | |||
815 | unsigned RegSize = RegVT.getSizeInBits(); | |||
816 | assert(RegSize == 32 || RegSize == 64 ||((RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType (RegVT)) ? static_cast<void> (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 817, __PRETTY_FUNCTION__)) | |||
817 | Subtarget.isHVXVectorType(RegVT))((RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType (RegVT)) ? static_cast<void> (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 817, __PRETTY_FUNCTION__)); | |||
818 | #endif | |||
819 | } | |||
820 | InVals.push_back(Copy); | |||
821 | MRI.addLiveIn(VA.getLocReg(), VReg); | |||
822 | } else { | |||
823 | assert(VA.isMemLoc() && "Argument should be passed in memory")((VA.isMemLoc() && "Argument should be passed in memory" ) ? static_cast<void> (0) : __assert_fail ("VA.isMemLoc() && \"Argument should be passed in memory\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 823, __PRETTY_FUNCTION__)); | |||
824 | ||||
825 | // If it's a byval parameter, then we need to compute the | |||
826 | // "real" size, not the size of the pointer. | |||
827 | unsigned ObjSize = Flags.isByVal() | |||
828 | ? Flags.getByValSize() | |||
829 | : VA.getLocVT().getStoreSizeInBits() / 8; | |||
830 | ||||
831 | // Create the frame index object for this incoming parameter. | |||
832 | int Offset = HEXAGON_LRFP_SIZE8 + VA.getLocMemOffset(); | |||
833 | int FI = MFI.CreateFixedObject(ObjSize, Offset, true); | |||
834 | SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); | |||
835 | ||||
836 | if (Flags.isByVal()) { | |||
837 | // If it's a pass-by-value aggregate, then do not dereference the stack | |||
838 | // location. Instead, we should generate a reference to the stack | |||
839 | // location. | |||
840 | InVals.push_back(FIN); | |||
841 | } else { | |||
842 | SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN, | |||
843 | MachinePointerInfo::getFixedStack(MF, FI, 0)); | |||
844 | InVals.push_back(L); | |||
845 | } | |||
846 | } | |||
847 | } | |||
848 | ||||
849 | ||||
850 | if (IsVarArg) { | |||
851 | // This will point to the next argument passed via stack. | |||
852 | int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset(); | |||
853 | int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true); | |||
854 | HMFI.setVarArgsFrameIndex(FI); | |||
855 | } | |||
856 | ||||
857 | return Chain; | |||
858 | } | |||
859 | ||||
860 | SDValue | |||
861 | HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { | |||
862 | // VASTART stores the address of the VarArgsFrameIndex slot into the | |||
863 | // memory location argument. | |||
864 | MachineFunction &MF = DAG.getMachineFunction(); | |||
865 | HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>(); | |||
866 | SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32); | |||
867 | const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); | |||
868 | return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1), | |||
869 | MachinePointerInfo(SV)); | |||
870 | } | |||
871 | ||||
872 | SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { | |||
873 | const SDLoc &dl(Op); | |||
874 | SDValue LHS = Op.getOperand(0); | |||
875 | SDValue RHS = Op.getOperand(1); | |||
876 | ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); | |||
877 | MVT ResTy = ty(Op); | |||
878 | MVT OpTy = ty(LHS); | |||
879 | ||||
880 | if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) { | |||
881 | MVT ElemTy = OpTy.getVectorElementType(); | |||
882 | assert(ElemTy.isScalarInteger())((ElemTy.isScalarInteger()) ? static_cast<void> (0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 882, __PRETTY_FUNCTION__)); | |||
883 | MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()), | |||
884 | OpTy.getVectorNumElements()); | |||
885 | return DAG.getSetCC(dl, ResTy, | |||
886 | DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy), | |||
887 | DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC); | |||
888 | } | |||
889 | ||||
890 | // Treat all other vector types as legal. | |||
891 | if (ResTy.isVector()) | |||
892 | return Op; | |||
893 | ||||
894 | // Comparisons of short integers should use sign-extend, not zero-extend, | |||
895 | // since we can represent small negative values in the compare instructions. | |||
896 | // The LLVM default is to use zero-extend arbitrarily in these cases. | |||
897 | auto isSExtFree = [this](SDValue N) { | |||
898 | switch (N.getOpcode()) { | |||
899 | case ISD::TRUNCATE: { | |||
900 | // A sign-extend of a truncate of a sign-extend is free. | |||
901 | SDValue Op = N.getOperand(0); | |||
902 | if (Op.getOpcode() != ISD::AssertSext) | |||
903 | return false; | |||
904 | EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT(); | |||
905 | unsigned ThisBW = ty(N).getSizeInBits(); | |||
906 | unsigned OrigBW = OrigTy.getSizeInBits(); | |||
907 | // The type that was sign-extended to get the AssertSext must be | |||
908 | // narrower than the type of N (so that N has still the same value | |||
909 | // as the original). | |||
910 | return ThisBW >= OrigBW; | |||
911 | } | |||
912 | case ISD::LOAD: | |||
913 | // We have sign-extended loads. | |||
914 | return true; | |||
915 | } | |||
916 | return false; | |||
917 | }; | |||
918 | ||||
919 | if (OpTy == MVT::i8 || OpTy == MVT::i16) { | |||
920 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS); | |||
921 | bool IsNegative = C && C->getAPIntValue().isNegative(); | |||
922 | if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS)) | |||
923 | return DAG.getSetCC(dl, ResTy, | |||
924 | DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32), | |||
925 | DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC); | |||
926 | } | |||
927 | ||||
928 | return SDValue(); | |||
929 | } | |||
930 | ||||
931 | SDValue | |||
932 | HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const { | |||
933 | SDValue PredOp = Op.getOperand(0); | |||
934 | SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2); | |||
935 | MVT OpTy = ty(Op1); | |||
936 | const SDLoc &dl(Op); | |||
937 | ||||
938 | if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) { | |||
939 | MVT ElemTy = OpTy.getVectorElementType(); | |||
940 | assert(ElemTy.isScalarInteger())((ElemTy.isScalarInteger()) ? static_cast<void> (0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 940, __PRETTY_FUNCTION__)); | |||
941 | MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()), | |||
942 | OpTy.getVectorNumElements()); | |||
943 | // Generate (trunc (select (_, sext, sext))). | |||
944 | return DAG.getSExtOrTrunc( | |||
945 | DAG.getSelect(dl, WideTy, PredOp, | |||
946 | DAG.getSExtOrTrunc(Op1, dl, WideTy), | |||
947 | DAG.getSExtOrTrunc(Op2, dl, WideTy)), | |||
948 | dl, OpTy); | |||
949 | } | |||
950 | ||||
951 | return SDValue(); | |||
952 | } | |||
953 | ||||
954 | static Constant *convert_i1_to_i8(const Constant *ConstVal) { | |||
955 | SmallVector<Constant *, 128> NewConst; | |||
956 | const ConstantVector *CV = dyn_cast<ConstantVector>(ConstVal); | |||
957 | if (!CV) | |||
958 | return nullptr; | |||
959 | ||||
960 | LLVMContext &Ctx = ConstVal->getContext(); | |||
961 | IRBuilder<> IRB(Ctx); | |||
962 | unsigned NumVectorElements = CV->getNumOperands(); | |||
963 | assert(isPowerOf2_32(NumVectorElements) &&((isPowerOf2_32(NumVectorElements) && "conversion only supported for pow2 VectorSize!" ) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(NumVectorElements) && \"conversion only supported for pow2 VectorSize!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 964, __PRETTY_FUNCTION__)) | |||
964 | "conversion only supported for pow2 VectorSize!")((isPowerOf2_32(NumVectorElements) && "conversion only supported for pow2 VectorSize!" ) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(NumVectorElements) && \"conversion only supported for pow2 VectorSize!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 964, __PRETTY_FUNCTION__)); | |||
965 | ||||
966 | for (unsigned i = 0; i < NumVectorElements / 8; ++i) { | |||
967 | uint8_t x = 0; | |||
968 | for (unsigned j = 0; j < 8; ++j) { | |||
969 | uint8_t y = CV->getOperand(i * 8 + j)->getUniqueInteger().getZExtValue(); | |||
970 | x |= y << (7 - j); | |||
971 | } | |||
972 | assert((x == 0 || x == 255) && "Either all 0's or all 1's expected!")(((x == 0 || x == 255) && "Either all 0's or all 1's expected!" ) ? static_cast<void> (0) : __assert_fail ("(x == 0 || x == 255) && \"Either all 0's or all 1's expected!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 972, __PRETTY_FUNCTION__)); | |||
973 | NewConst.push_back(IRB.getInt8(x)); | |||
974 | } | |||
975 | return ConstantVector::get(NewConst); | |||
976 | } | |||
977 | ||||
978 | SDValue | |||
979 | HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { | |||
980 | EVT ValTy = Op.getValueType(); | |||
981 | ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op); | |||
982 | Constant *CVal = nullptr; | |||
983 | bool isVTi1Type = false; | |||
984 | if (const Constant *ConstVal = dyn_cast<Constant>(CPN->getConstVal())) { | |||
985 | Type *CValTy = ConstVal->getType(); | |||
986 | if (CValTy->isVectorTy() && | |||
987 | CValTy->getVectorElementType()->isIntegerTy(1)) { | |||
988 | CVal = convert_i1_to_i8(ConstVal); | |||
989 | isVTi1Type = (CVal != nullptr); | |||
990 | } | |||
991 | } | |||
992 | unsigned Align = CPN->getAlignment(); | |||
993 | bool IsPositionIndependent = isPositionIndependent(); | |||
994 | unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0; | |||
995 | ||||
996 | unsigned Offset = 0; | |||
997 | SDValue T; | |||
998 | if (CPN->isMachineConstantPoolEntry()) | |||
999 | T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Align, Offset, | |||
1000 | TF); | |||
1001 | else if (isVTi1Type) | |||
1002 | T = DAG.getTargetConstantPool(CVal, ValTy, Align, Offset, TF); | |||
1003 | else | |||
1004 | T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Align, Offset, TF); | |||
1005 | ||||
1006 | assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&((cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && "Inconsistent target flag encountered") ? static_cast <void> (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1007, __PRETTY_FUNCTION__)) | |||
1007 | "Inconsistent target flag encountered")((cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && "Inconsistent target flag encountered") ? static_cast <void> (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1007, __PRETTY_FUNCTION__)); | |||
1008 | ||||
1009 | if (IsPositionIndependent) | |||
1010 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T); | |||
1011 | return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T); | |||
1012 | } | |||
1013 | ||||
1014 | SDValue | |||
1015 | HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { | |||
1016 | EVT VT = Op.getValueType(); | |||
1017 | int Idx = cast<JumpTableSDNode>(Op)->getIndex(); | |||
1018 | if (isPositionIndependent()) { | |||
1019 | SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL); | |||
1020 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T); | |||
1021 | } | |||
1022 | ||||
1023 | SDValue T = DAG.getTargetJumpTable(Idx, VT); | |||
1024 | return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T); | |||
1025 | } | |||
1026 | ||||
1027 | SDValue | |||
1028 | HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const { | |||
1029 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | |||
1030 | MachineFunction &MF = DAG.getMachineFunction(); | |||
1031 | MachineFrameInfo &MFI = MF.getFrameInfo(); | |||
1032 | MFI.setReturnAddressIsTaken(true); | |||
1033 | ||||
1034 | if (verifyReturnAddressArgumentIsConstant(Op, DAG)) | |||
1035 | return SDValue(); | |||
1036 | ||||
1037 | EVT VT = Op.getValueType(); | |||
1038 | SDLoc dl(Op); | |||
1039 | unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); | |||
1040 | if (Depth) { | |||
1041 | SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); | |||
1042 | SDValue Offset = DAG.getConstant(4, dl, MVT::i32); | |||
1043 | return DAG.getLoad(VT, dl, DAG.getEntryNode(), | |||
1044 | DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), | |||
1045 | MachinePointerInfo()); | |||
1046 | } | |||
1047 | ||||
1048 | // Return LR, which contains the return address. Mark it an implicit live-in. | |||
1049 | unsigned Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32)); | |||
1050 | return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); | |||
1051 | } | |||
1052 | ||||
1053 | SDValue | |||
1054 | HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { | |||
1055 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | |||
1056 | MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); | |||
1057 | MFI.setFrameAddressIsTaken(true); | |||
1058 | ||||
1059 | EVT VT = Op.getValueType(); | |||
1060 | SDLoc dl(Op); | |||
1061 | unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); | |||
1062 | SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, | |||
1063 | HRI.getFrameRegister(), VT); | |||
1064 | while (Depth--) | |||
1065 | FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, | |||
1066 | MachinePointerInfo()); | |||
1067 | return FrameAddr; | |||
1068 | } | |||
1069 | ||||
1070 | SDValue | |||
1071 | HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const { | |||
1072 | SDLoc dl(Op); | |||
1073 | return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0)); | |||
1074 | } | |||
1075 | ||||
1076 | SDValue | |||
1077 | HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const { | |||
1078 | SDLoc dl(Op); | |||
1079 | auto *GAN = cast<GlobalAddressSDNode>(Op); | |||
1080 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | |||
1081 | auto *GV = GAN->getGlobal(); | |||
1082 | int64_t Offset = GAN->getOffset(); | |||
1083 | ||||
1084 | auto &HLOF = *HTM.getObjFileLowering(); | |||
1085 | Reloc::Model RM = HTM.getRelocationModel(); | |||
1086 | ||||
1087 | if (RM == Reloc::Static) { | |||
1088 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset); | |||
1089 | const GlobalObject *GO = GV->getBaseObject(); | |||
1090 | if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM)) | |||
1091 | return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA); | |||
1092 | return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA); | |||
1093 | } | |||
1094 | ||||
1095 | bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV); | |||
1096 | if (UsePCRel) { | |||
1097 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset, | |||
1098 | HexagonII::MO_PCREL); | |||
1099 | return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA); | |||
1100 | } | |||
1101 | ||||
1102 | // Use GOT index. | |||
1103 | SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT); | |||
1104 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT); | |||
1105 | SDValue Off = DAG.getConstant(Offset, dl, MVT::i32); | |||
1106 | return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off); | |||
1107 | } | |||
1108 | ||||
1109 | // Specifies that for loads and stores VT can be promoted to PromotedLdStVT. | |||
1110 | SDValue | |||
1111 | HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { | |||
1112 | const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); | |||
1113 | SDLoc dl(Op); | |||
1114 | EVT PtrVT = getPointerTy(DAG.getDataLayout()); | |||
1115 | ||||
1116 | Reloc::Model RM = HTM.getRelocationModel(); | |||
1117 | if (RM == Reloc::Static) { | |||
1118 | SDValue A = DAG.getTargetBlockAddress(BA, PtrVT); | |||
1119 | return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A); | |||
1120 | } | |||
1121 | ||||
1122 | SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL); | |||
1123 | return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A); | |||
1124 | } | |||
1125 | ||||
1126 | SDValue | |||
1127 | HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) | |||
1128 | const { | |||
1129 | EVT PtrVT = getPointerTy(DAG.getDataLayout()); | |||
1130 | SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME"_GLOBAL_OFFSET_TABLE_", PtrVT, | |||
1131 | HexagonII::MO_PCREL); | |||
1132 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym); | |||
1133 | } | |||
1134 | ||||
1135 | SDValue | |||
1136 | HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain, | |||
1137 | GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg, | |||
1138 | unsigned char OperandFlags) const { | |||
1139 | MachineFunction &MF = DAG.getMachineFunction(); | |||
1140 | MachineFrameInfo &MFI = MF.getFrameInfo(); | |||
1141 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | |||
1142 | SDLoc dl(GA); | |||
1143 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, | |||
1144 | GA->getValueType(0), | |||
1145 | GA->getOffset(), | |||
1146 | OperandFlags); | |||
1147 | // Create Operands for the call.The Operands should have the following: | |||
1148 | // 1. Chain SDValue | |||
1149 | // 2. Callee which in this case is the Global address value. | |||
1150 | // 3. Registers live into the call.In this case its R0, as we | |||
1151 | // have just one argument to be passed. | |||
1152 | // 4. Glue. | |||
1153 | // Note: The order is important. | |||
1154 | ||||
1155 | const auto &HRI = *Subtarget.getRegisterInfo(); | |||
1156 | const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C); | |||
1157 | assert(Mask && "Missing call preserved mask for calling convention")((Mask && "Missing call preserved mask for calling convention" ) ? static_cast<void> (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1157, __PRETTY_FUNCTION__)); | |||
1158 | SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT), | |||
1159 | DAG.getRegisterMask(Mask), Glue }; | |||
1160 | Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops); | |||
1161 | ||||
1162 | // Inform MFI that function has calls. | |||
1163 | MFI.setAdjustsStack(true); | |||
1164 | ||||
1165 | Glue = Chain.getValue(1); | |||
1166 | return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue); | |||
1167 | } | |||
1168 | ||||
1169 | // | |||
1170 | // Lower using the intial executable model for TLS addresses | |||
1171 | // | |||
1172 | SDValue | |||
1173 | HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA, | |||
1174 | SelectionDAG &DAG) const { | |||
1175 | SDLoc dl(GA); | |||
1176 | int64_t Offset = GA->getOffset(); | |||
1177 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | |||
1178 | ||||
1179 | // Get the thread pointer. | |||
1180 | SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT); | |||
1181 | ||||
1182 | bool IsPositionIndependent = isPositionIndependent(); | |||
1183 | unsigned char TF = | |||
1184 | IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE; | |||
1185 | ||||
1186 | // First generate the TLS symbol address | |||
1187 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, | |||
1188 | Offset, TF); | |||
1189 | ||||
1190 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | |||
1191 | ||||
1192 | if (IsPositionIndependent) { | |||
1193 | // Generate the GOT pointer in case of position independent code | |||
1194 | SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG); | |||
1195 | ||||
1196 | // Add the TLS Symbol address to GOT pointer.This gives | |||
1197 | // GOT relative relocation for the symbol. | |||
1198 | Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym); | |||
1199 | } | |||
1200 | ||||
1201 | // Load the offset value for TLS symbol.This offset is relative to | |||
1202 | // thread pointer. | |||
1203 | SDValue LoadOffset = | |||
1204 | DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo()); | |||
1205 | ||||
1206 | // Address of the thread local variable is the add of thread | |||
1207 | // pointer and the offset of the variable. | |||
1208 | return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset); | |||
1209 | } | |||
1210 | ||||
1211 | // | |||
1212 | // Lower using the local executable model for TLS addresses | |||
1213 | // | |||
1214 | SDValue | |||
1215 | HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA, | |||
1216 | SelectionDAG &DAG) const { | |||
1217 | SDLoc dl(GA); | |||
1218 | int64_t Offset = GA->getOffset(); | |||
1219 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | |||
1220 | ||||
1221 | // Get the thread pointer. | |||
1222 | SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT); | |||
1223 | // Generate the TLS symbol address | |||
1224 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset, | |||
1225 | HexagonII::MO_TPREL); | |||
1226 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | |||
1227 | ||||
1228 | // Address of the thread local variable is the add of thread | |||
1229 | // pointer and the offset of the variable. | |||
1230 | return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym); | |||
1231 | } | |||
1232 | ||||
1233 | // | |||
1234 | // Lower using the general dynamic model for TLS addresses | |||
1235 | // | |||
1236 | SDValue | |||
1237 | HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, | |||
1238 | SelectionDAG &DAG) const { | |||
1239 | SDLoc dl(GA); | |||
1240 | int64_t Offset = GA->getOffset(); | |||
1241 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | |||
1242 | ||||
1243 | // First generate the TLS symbol address | |||
1244 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset, | |||
1245 | HexagonII::MO_GDGOT); | |||
1246 | ||||
1247 | // Then, generate the GOT pointer | |||
1248 | SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG); | |||
1249 | ||||
1250 | // Add the TLS symbol and the GOT pointer | |||
1251 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | |||
1252 | SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym); | |||
1253 | ||||
1254 | // Copy over the argument to R0 | |||
1255 | SDValue InFlag; | |||
1256 | Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag); | |||
1257 | InFlag = Chain.getValue(1); | |||
1258 | ||||
1259 | unsigned Flags = | |||
1260 | static_cast<const HexagonSubtarget &>(DAG.getSubtarget()).useLongCalls() | |||
1261 | ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended | |||
1262 | : HexagonII::MO_GDPLT; | |||
1263 | ||||
1264 | return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT, | |||
1265 | Hexagon::R0, Flags); | |||
1266 | } | |||
1267 | ||||
1268 | // | |||
1269 | // Lower TLS addresses. | |||
1270 | // | |||
1271 | // For now for dynamic models, we only support the general dynamic model. | |||
1272 | // | |||
1273 | SDValue | |||
1274 | HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op, | |||
1275 | SelectionDAG &DAG) const { | |||
1276 | GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); | |||
1277 | ||||
1278 | switch (HTM.getTLSModel(GA->getGlobal())) { | |||
1279 | case TLSModel::GeneralDynamic: | |||
1280 | case TLSModel::LocalDynamic: | |||
1281 | return LowerToTLSGeneralDynamicModel(GA, DAG); | |||
1282 | case TLSModel::InitialExec: | |||
1283 | return LowerToTLSInitialExecModel(GA, DAG); | |||
1284 | case TLSModel::LocalExec: | |||
1285 | return LowerToTLSLocalExecModel(GA, DAG); | |||
1286 | } | |||
1287 | llvm_unreachable("Bogus TLS model")::llvm::llvm_unreachable_internal("Bogus TLS model", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1287); | |||
1288 | } | |||
1289 | ||||
1290 | //===----------------------------------------------------------------------===// | |||
1291 | // TargetLowering Implementation | |||
1292 | //===----------------------------------------------------------------------===// | |||
1293 | ||||
1294 | HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, | |||
1295 | const HexagonSubtarget &ST) | |||
1296 | : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)), | |||
1297 | Subtarget(ST) { | |||
1298 | auto &HRI = *Subtarget.getRegisterInfo(); | |||
1299 | ||||
1300 | setPrefLoopAlignment(Align(16)); | |||
1301 | setMinFunctionAlignment(Align(4)); | |||
1302 | setPrefFunctionAlignment(Align(16)); | |||
1303 | setStackPointerRegisterToSaveRestore(HRI.getStackRegister()); | |||
1304 | setBooleanContents(TargetLoweringBase::UndefinedBooleanContent); | |||
1305 | setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent); | |||
1306 | ||||
1307 | setMaxAtomicSizeInBitsSupported(64); | |||
1308 | setMinCmpXchgSizeInBits(32); | |||
1309 | ||||
1310 | if (EnableHexSDNodeSched) | |||
1311 | setSchedulingPreference(Sched::VLIW); | |||
1312 | else | |||
1313 | setSchedulingPreference(Sched::Source); | |||
1314 | ||||
1315 | // Limits for inline expansion of memcpy/memmove | |||
1316 | MaxStoresPerMemcpy = MaxStoresPerMemcpyCL; | |||
1317 | MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL; | |||
1318 | MaxStoresPerMemmove = MaxStoresPerMemmoveCL; | |||
1319 | MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL; | |||
1320 | MaxStoresPerMemset = MaxStoresPerMemsetCL; | |||
1321 | MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL; | |||
1322 | ||||
1323 | // | |||
1324 | // Set up register classes. | |||
1325 | // | |||
1326 | ||||
1327 | addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass); | |||
1328 | addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass); // bbbbaaaa | |||
1329 | addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass); // ddccbbaa | |||
1330 | addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass); // hgfedcba | |||
1331 | addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass); | |||
1332 | addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass); | |||
1333 | addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass); | |||
1334 | addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass); | |||
1335 | addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass); | |||
1336 | addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass); | |||
1337 | addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass); | |||
1338 | ||||
1339 | addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass); | |||
1340 | addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass); | |||
1341 | ||||
1342 | // | |||
1343 | // Handling of scalar operations. | |||
1344 | // | |||
1345 | // All operations default to "legal", except: | |||
1346 | // - indexed loads and stores (pre-/post-incremented), | |||
1347 | // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS, | |||
1348 | // ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN, | |||
1349 | // FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP, | |||
1350 | // FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG, | |||
1351 | // which default to "expand" for at least one type. | |||
1352 | ||||
1353 | // Misc operations. | |||
1354 | setOperationAction(ISD::ConstantFP, MVT::f32, Legal); | |||
1355 | setOperationAction(ISD::ConstantFP, MVT::f64, Legal); | |||
1356 | setOperationAction(ISD::TRAP, MVT::Other, Legal); | |||
1357 | setOperationAction(ISD::ConstantPool, MVT::i32, Custom); | |||
1358 | setOperationAction(ISD::JumpTable, MVT::i32, Custom); | |||
1359 | setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); | |||
1360 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); | |||
1361 | setOperationAction(ISD::INLINEASM, MVT::Other, Custom); | |||
1362 | setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom); | |||
1363 | setOperationAction(ISD::PREFETCH, MVT::Other, Custom); | |||
1364 | setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); | |||
1365 | setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); | |||
1366 | setOperationAction(ISD::EH_RETURN, MVT::Other, Custom); | |||
1367 | setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom); | |||
1368 | setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); | |||
1369 | setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); | |||
1370 | ||||
1371 | // Custom legalize GlobalAddress nodes into CONST32. | |||
1372 | setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); | |||
1373 | setOperationAction(ISD::GlobalAddress, MVT::i8, Custom); | |||
1374 | setOperationAction(ISD::BlockAddress, MVT::i32, Custom); | |||
1375 | ||||
1376 | // Hexagon needs to optimize cases with negative constants. | |||
1377 | setOperationAction(ISD::SETCC, MVT::i8, Custom); | |||
1378 | setOperationAction(ISD::SETCC, MVT::i16, Custom); | |||
1379 | setOperationAction(ISD::SETCC, MVT::v4i8, Custom); | |||
1380 | setOperationAction(ISD::SETCC, MVT::v2i16, Custom); | |||
1381 | ||||
1382 | // VASTART needs to be custom lowered to use the VarArgsFrameIndex. | |||
1383 | setOperationAction(ISD::VASTART, MVT::Other, Custom); | |||
1384 | setOperationAction(ISD::VAEND, MVT::Other, Expand); | |||
1385 | setOperationAction(ISD::VAARG, MVT::Other, Expand); | |||
1386 | setOperationAction(ISD::VACOPY, MVT::Other, Expand); | |||
1387 | ||||
1388 | setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); | |||
1389 | setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); | |||
1390 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); | |||
1391 | ||||
1392 | if (EmitJumpTables) | |||
1393 | setMinimumJumpTableEntries(MinimumJumpTables); | |||
1394 | else | |||
1395 | setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max()); | |||
1396 | setOperationAction(ISD::BR_JT, MVT::Other, Expand); | |||
1397 | ||||
1398 | setOperationAction(ISD::ABS, MVT::i32, Legal); | |||
1399 | setOperationAction(ISD::ABS, MVT::i64, Legal); | |||
1400 | ||||
1401 | // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit, | |||
1402 | // but they only operate on i64. | |||
1403 | for (MVT VT : MVT::integer_valuetypes()) { | |||
1404 | setOperationAction(ISD::UADDO, VT, Custom); | |||
1405 | setOperationAction(ISD::USUBO, VT, Custom); | |||
1406 | setOperationAction(ISD::SADDO, VT, Expand); | |||
1407 | setOperationAction(ISD::SSUBO, VT, Expand); | |||
1408 | setOperationAction(ISD::ADDCARRY, VT, Expand); | |||
1409 | setOperationAction(ISD::SUBCARRY, VT, Expand); | |||
1410 | } | |||
1411 | setOperationAction(ISD::ADDCARRY, MVT::i64, Custom); | |||
1412 | setOperationAction(ISD::SUBCARRY, MVT::i64, Custom); | |||
1413 | ||||
1414 | setOperationAction(ISD::CTLZ, MVT::i8, Promote); | |||
1415 | setOperationAction(ISD::CTLZ, MVT::i16, Promote); | |||
1416 | setOperationAction(ISD::CTTZ, MVT::i8, Promote); | |||
1417 | setOperationAction(ISD::CTTZ, MVT::i16, Promote); | |||
1418 | ||||
1419 | // Popcount can count # of 1s in i64 but returns i32. | |||
1420 | setOperationAction(ISD::CTPOP, MVT::i8, Promote); | |||
1421 | setOperationAction(ISD::CTPOP, MVT::i16, Promote); | |||
1422 | setOperationAction(ISD::CTPOP, MVT::i32, Promote); | |||
1423 | setOperationAction(ISD::CTPOP, MVT::i64, Legal); | |||
1424 | ||||
1425 | setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); | |||
1426 | setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); | |||
1427 | setOperationAction(ISD::BSWAP, MVT::i32, Legal); | |||
1428 | setOperationAction(ISD::BSWAP, MVT::i64, Legal); | |||
1429 | ||||
1430 | setOperationAction(ISD::FSHL, MVT::i32, Legal); | |||
1431 | setOperationAction(ISD::FSHL, MVT::i64, Legal); | |||
1432 | setOperationAction(ISD::FSHR, MVT::i32, Legal); | |||
1433 | setOperationAction(ISD::FSHR, MVT::i64, Legal); | |||
1434 | ||||
1435 | for (unsigned IntExpOp : | |||
1436 | {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, | |||
1437 | ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR, | |||
1438 | ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS, | |||
1439 | ISD::SMUL_LOHI, ISD::UMUL_LOHI}) { | |||
1440 | for (MVT VT : MVT::integer_valuetypes()) | |||
1441 | setOperationAction(IntExpOp, VT, Expand); | |||
1442 | } | |||
1443 | ||||
1444 | for (unsigned FPExpOp : | |||
1445 | {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS, | |||
1446 | ISD::FPOW, ISD::FCOPYSIGN}) { | |||
1447 | for (MVT VT : MVT::fp_valuetypes()) | |||
1448 | setOperationAction(FPExpOp, VT, Expand); | |||
1449 | } | |||
1450 | ||||
1451 | // No extending loads from i32. | |||
1452 | for (MVT VT : MVT::integer_valuetypes()) { | |||
1453 | setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand); | |||
1454 | setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); | |||
1455 | setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand); | |||
1456 | } | |||
1457 | // Turn FP truncstore into trunc + store. | |||
1458 | setTruncStoreAction(MVT::f64, MVT::f32, Expand); | |||
1459 | // Turn FP extload into load/fpextend. | |||
1460 | for (MVT VT : MVT::fp_valuetypes()) | |||
1461 | setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); | |||
1462 | ||||
1463 | // Expand BR_CC and SELECT_CC for all integer and fp types. | |||
1464 | for (MVT VT : MVT::integer_valuetypes()) { | |||
1465 | setOperationAction(ISD::BR_CC, VT, Expand); | |||
1466 | setOperationAction(ISD::SELECT_CC, VT, Expand); | |||
1467 | } | |||
1468 | for (MVT VT : MVT::fp_valuetypes()) { | |||
1469 | setOperationAction(ISD::BR_CC, VT, Expand); | |||
1470 | setOperationAction(ISD::SELECT_CC, VT, Expand); | |||
1471 | } | |||
1472 | setOperationAction(ISD::BR_CC, MVT::Other, Expand); | |||
1473 | ||||
1474 | // | |||
1475 | // Handling of vector operations. | |||
1476 | // | |||
1477 | ||||
1478 | // Set the action for vector operations to "expand", then override it with | |||
1479 | // either "custom" or "legal" for specific cases. | |||
1480 | static const unsigned VectExpOps[] = { | |||
1481 | // Integer arithmetic: | |||
1482 | ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV, | |||
1483 | ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO, | |||
1484 | ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI, | |||
1485 | // Logical/bit: | |||
1486 | ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR, | |||
1487 | ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, | |||
1488 | // Floating point arithmetic/math functions: | |||
1489 | ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV, | |||
1490 | ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN, | |||
1491 | ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2, | |||
1492 | ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC, | |||
1493 | ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR, | |||
1494 | ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS, | |||
1495 | // Misc: | |||
1496 | ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool, | |||
1497 | // Vector: | |||
1498 | ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR, | |||
1499 | ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT, | |||
1500 | ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR, | |||
1501 | ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE | |||
1502 | }; | |||
1503 | ||||
1504 | for (MVT VT : MVT::fixedlen_vector_valuetypes()) { | |||
1505 | for (unsigned VectExpOp : VectExpOps) | |||
1506 | setOperationAction(VectExpOp, VT, Expand); | |||
1507 | ||||
1508 | // Expand all extending loads and truncating stores: | |||
1509 | for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) { | |||
1510 | if (TargetVT == VT) | |||
1511 | continue; | |||
1512 | setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand); | |||
1513 | setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand); | |||
1514 | setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand); | |||
1515 | setTruncStoreAction(VT, TargetVT, Expand); | |||
1516 | } | |||
1517 | ||||
1518 | // Normalize all inputs to SELECT to be vectors of i32. | |||
1519 | if (VT.getVectorElementType() != MVT::i32) { | |||
1520 | MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32); | |||
1521 | setOperationAction(ISD::SELECT, VT, Promote); | |||
1522 | AddPromotedToType(ISD::SELECT, VT, VT32); | |||
1523 | } | |||
1524 | setOperationAction(ISD::SRA, VT, Custom); | |||
1525 | setOperationAction(ISD::SHL, VT, Custom); | |||
1526 | setOperationAction(ISD::SRL, VT, Custom); | |||
1527 | } | |||
1528 | ||||
1529 | // Extending loads from (native) vectors of i8 into (native) vectors of i16 | |||
1530 | // are legal. | |||
1531 | setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | |||
1532 | setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | |||
1533 | setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | |||
1534 | setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | |||
1535 | setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | |||
1536 | setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | |||
1537 | ||||
1538 | // Types natively supported: | |||
1539 | for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8, | |||
1540 | MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) { | |||
1541 | setOperationAction(ISD::BUILD_VECTOR, NativeVT, Custom); | |||
1542 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom); | |||
1543 | setOperationAction(ISD::INSERT_VECTOR_ELT, NativeVT, Custom); | |||
1544 | setOperationAction(ISD::EXTRACT_SUBVECTOR, NativeVT, Custom); | |||
1545 | setOperationAction(ISD::INSERT_SUBVECTOR, NativeVT, Custom); | |||
1546 | setOperationAction(ISD::CONCAT_VECTORS, NativeVT, Custom); | |||
1547 | ||||
1548 | setOperationAction(ISD::ADD, NativeVT, Legal); | |||
1549 | setOperationAction(ISD::SUB, NativeVT, Legal); | |||
1550 | setOperationAction(ISD::MUL, NativeVT, Legal); | |||
1551 | setOperationAction(ISD::AND, NativeVT, Legal); | |||
1552 | setOperationAction(ISD::OR, NativeVT, Legal); | |||
1553 | setOperationAction(ISD::XOR, NativeVT, Legal); | |||
1554 | } | |||
1555 | ||||
1556 | // Custom lower unaligned loads. | |||
1557 | // Also, for both loads and stores, verify the alignment of the address | |||
1558 | // in case it is a compile-time constant. This is a usability feature to | |||
1559 | // provide a meaningful error message to users. | |||
1560 | for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8, | |||
1561 | MVT::v2i16, MVT::v4i16, MVT::v2i32}) { | |||
1562 | setOperationAction(ISD::LOAD, VT, Custom); | |||
1563 | setOperationAction(ISD::STORE, VT, Custom); | |||
1564 | } | |||
1565 | ||||
1566 | for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16, | |||
1567 | MVT::v2i32}) { | |||
1568 | setCondCodeAction(ISD::SETNE, VT, Expand); | |||
1569 | setCondCodeAction(ISD::SETLE, VT, Expand); | |||
1570 | setCondCodeAction(ISD::SETGE, VT, Expand); | |||
1571 | setCondCodeAction(ISD::SETLT, VT, Expand); | |||
1572 | setCondCodeAction(ISD::SETULE, VT, Expand); | |||
1573 | setCondCodeAction(ISD::SETUGE, VT, Expand); | |||
1574 | setCondCodeAction(ISD::SETULT, VT, Expand); | |||
1575 | } | |||
1576 | ||||
1577 | // Custom-lower bitcasts from i8 to v8i1. | |||
1578 | setOperationAction(ISD::BITCAST, MVT::i8, Custom); | |||
1579 | setOperationAction(ISD::SETCC, MVT::v2i16, Custom); | |||
1580 | setOperationAction(ISD::VSELECT, MVT::v4i8, Custom); | |||
1581 | setOperationAction(ISD::VSELECT, MVT::v2i16, Custom); | |||
1582 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom); | |||
1583 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); | |||
1584 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); | |||
1585 | ||||
1586 | // V5+. | |||
1587 | setOperationAction(ISD::FMA, MVT::f64, Expand); | |||
1588 | setOperationAction(ISD::FADD, MVT::f64, Expand); | |||
1589 | setOperationAction(ISD::FSUB, MVT::f64, Expand); | |||
1590 | setOperationAction(ISD::FMUL, MVT::f64, Expand); | |||
1591 | ||||
1592 | setOperationAction(ISD::FMINNUM, MVT::f32, Legal); | |||
1593 | setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); | |||
1594 | ||||
1595 | setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote); | |||
1596 | setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote); | |||
1597 | setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote); | |||
1598 | setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote); | |||
1599 | setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote); | |||
1600 | setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote); | |||
1601 | setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); | |||
1602 | setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote); | |||
1603 | setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote); | |||
1604 | setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); | |||
1605 | setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote); | |||
1606 | setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote); | |||
1607 | ||||
1608 | // Handling of indexed loads/stores: default is "expand". | |||
1609 | // | |||
1610 | for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64, | |||
1611 | MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) { | |||
1612 | setIndexedLoadAction(ISD::POST_INC, VT, Legal); | |||
1613 | setIndexedStoreAction(ISD::POST_INC, VT, Legal); | |||
1614 | } | |||
1615 | ||||
1616 | // Subtarget-specific operation actions. | |||
1617 | // | |||
1618 | if (Subtarget.hasV60Ops()) { | |||
1619 | setOperationAction(ISD::ROTL, MVT::i32, Legal); | |||
1620 | setOperationAction(ISD::ROTL, MVT::i64, Legal); | |||
1621 | setOperationAction(ISD::ROTR, MVT::i32, Legal); | |||
1622 | setOperationAction(ISD::ROTR, MVT::i64, Legal); | |||
1623 | } | |||
1624 | if (Subtarget.hasV66Ops()) { | |||
1625 | setOperationAction(ISD::FADD, MVT::f64, Legal); | |||
1626 | setOperationAction(ISD::FSUB, MVT::f64, Legal); | |||
1627 | } | |||
1628 | ||||
1629 | setTargetDAGCombine(ISD::VSELECT); | |||
1630 | ||||
1631 | if (Subtarget.useHVXOps()) | |||
1632 | initializeHVXLowering(); | |||
1633 | ||||
1634 | computeRegisterProperties(&HRI); | |||
1635 | ||||
1636 | // | |||
1637 | // Library calls for unsupported operations | |||
1638 | // | |||
1639 | bool FastMath = EnableFastMath; | |||
1640 | ||||
1641 | setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3"); | |||
1642 | setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3"); | |||
1643 | setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3"); | |||
1644 | setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3"); | |||
1645 | setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3"); | |||
1646 | setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3"); | |||
1647 | setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3"); | |||
1648 | setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3"); | |||
1649 | ||||
1650 | setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf"); | |||
1651 | setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf"); | |||
1652 | setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti"); | |||
1653 | setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti"); | |||
1654 | setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti"); | |||
1655 | setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti"); | |||
1656 | ||||
1657 | // This is the only fast library function for sqrtd. | |||
1658 | if (FastMath) | |||
1659 | setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2"); | |||
1660 | ||||
1661 | // Prefix is: nothing for "slow-math", | |||
1662 | // "fast2_" for V5+ fast-math double-precision | |||
1663 | // (actually, keep fast-math and fast-math2 separate for now) | |||
1664 | if (FastMath) { | |||
1665 | setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3"); | |||
1666 | setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3"); | |||
1667 | setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3"); | |||
1668 | setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3"); | |||
1669 | setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3"); | |||
1670 | } else { | |||
1671 | setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3"); | |||
1672 | setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3"); | |||
1673 | setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3"); | |||
1674 | setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3"); | |||
1675 | setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3"); | |||
1676 | } | |||
1677 | ||||
1678 | if (FastMath) | |||
1679 | setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf"); | |||
1680 | else | |||
1681 | setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf"); | |||
1682 | ||||
1683 | // These cause problems when the shift amount is non-constant. | |||
1684 | setLibcallName(RTLIB::SHL_I128, nullptr); | |||
1685 | setLibcallName(RTLIB::SRL_I128, nullptr); | |||
1686 | setLibcallName(RTLIB::SRA_I128, nullptr); | |||
1687 | } | |||
1688 | ||||
1689 | const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const { | |||
1690 | switch ((HexagonISD::NodeType)Opcode) { | |||
1691 | case HexagonISD::ADDC: return "HexagonISD::ADDC"; | |||
1692 | case HexagonISD::SUBC: return "HexagonISD::SUBC"; | |||
1693 | case HexagonISD::ALLOCA: return "HexagonISD::ALLOCA"; | |||
1694 | case HexagonISD::AT_GOT: return "HexagonISD::AT_GOT"; | |||
1695 | case HexagonISD::AT_PCREL: return "HexagonISD::AT_PCREL"; | |||
1696 | case HexagonISD::BARRIER: return "HexagonISD::BARRIER"; | |||
1697 | case HexagonISD::CALL: return "HexagonISD::CALL"; | |||
1698 | case HexagonISD::CALLnr: return "HexagonISD::CALLnr"; | |||
1699 | case HexagonISD::CALLR: return "HexagonISD::CALLR"; | |||
1700 | case HexagonISD::COMBINE: return "HexagonISD::COMBINE"; | |||
1701 | case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP"; | |||
1702 | case HexagonISD::CONST32: return "HexagonISD::CONST32"; | |||
1703 | case HexagonISD::CP: return "HexagonISD::CP"; | |||
1704 | case HexagonISD::DCFETCH: return "HexagonISD::DCFETCH"; | |||
1705 | case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN"; | |||
1706 | case HexagonISD::TSTBIT: return "HexagonISD::TSTBIT"; | |||
1707 | case HexagonISD::EXTRACTU: return "HexagonISD::EXTRACTU"; | |||
1708 | case HexagonISD::INSERT: return "HexagonISD::INSERT"; | |||
1709 | case HexagonISD::JT: return "HexagonISD::JT"; | |||
1710 | case HexagonISD::RET_FLAG: return "HexagonISD::RET_FLAG"; | |||
1711 | case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN"; | |||
1712 | case HexagonISD::VASL: return "HexagonISD::VASL"; | |||
1713 | case HexagonISD::VASR: return "HexagonISD::VASR"; | |||
1714 | case HexagonISD::VLSR: return "HexagonISD::VLSR"; | |||
1715 | case HexagonISD::VSPLAT: return "HexagonISD::VSPLAT"; | |||
1716 | case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW"; | |||
1717 | case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0"; | |||
1718 | case HexagonISD::VROR: return "HexagonISD::VROR"; | |||
1719 | case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE"; | |||
1720 | case HexagonISD::PTRUE: return "HexagonISD::PTRUE"; | |||
1721 | case HexagonISD::PFALSE: return "HexagonISD::PFALSE"; | |||
1722 | case HexagonISD::VZERO: return "HexagonISD::VZERO"; | |||
1723 | case HexagonISD::VSPLATW: return "HexagonISD::VSPLATW"; | |||
1724 | case HexagonISD::D2P: return "HexagonISD::D2P"; | |||
1725 | case HexagonISD::P2D: return "HexagonISD::P2D"; | |||
1726 | case HexagonISD::V2Q: return "HexagonISD::V2Q"; | |||
1727 | case HexagonISD::Q2V: return "HexagonISD::Q2V"; | |||
1728 | case HexagonISD::QCAT: return "HexagonISD::QCAT"; | |||
1729 | case HexagonISD::QTRUE: return "HexagonISD::QTRUE"; | |||
1730 | case HexagonISD::QFALSE: return "HexagonISD::QFALSE"; | |||
1731 | case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST"; | |||
1732 | case HexagonISD::VALIGN: return "HexagonISD::VALIGN"; | |||
1733 | case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR"; | |||
1734 | case HexagonISD::OP_END: break; | |||
1735 | } | |||
1736 | return nullptr; | |||
1737 | } | |||
1738 | ||||
1739 | void | |||
1740 | HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl, | |||
1741 | unsigned NeedAlign) const { | |||
1742 | auto *CA = dyn_cast<ConstantSDNode>(Ptr); | |||
1743 | if (!CA) | |||
1744 | return; | |||
1745 | unsigned Addr = CA->getZExtValue(); | |||
1746 | unsigned HaveAlign = Addr != 0 ? 1u << countTrailingZeros(Addr) : NeedAlign; | |||
1747 | if (HaveAlign < NeedAlign) { | |||
1748 | std::string ErrMsg; | |||
1749 | raw_string_ostream O(ErrMsg); | |||
1750 | O << "Misaligned constant address: " << format_hex(Addr, 10) | |||
1751 | << " has alignment " << HaveAlign | |||
1752 | << ", but the memory access requires " << NeedAlign; | |||
1753 | if (DebugLoc DL = dl.getDebugLoc()) | |||
1754 | DL.print(O << ", at "); | |||
1755 | report_fatal_error(O.str()); | |||
1756 | } | |||
1757 | } | |||
1758 | ||||
1759 | // Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load | |||
1760 | // intrinsic. | |||
1761 | static bool isBrevLdIntrinsic(const Value *Inst) { | |||
1762 | unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID(); | |||
1763 | return (ID == Intrinsic::hexagon_L2_loadrd_pbr || | |||
1764 | ID == Intrinsic::hexagon_L2_loadri_pbr || | |||
1765 | ID == Intrinsic::hexagon_L2_loadrh_pbr || | |||
1766 | ID == Intrinsic::hexagon_L2_loadruh_pbr || | |||
1767 | ID == Intrinsic::hexagon_L2_loadrb_pbr || | |||
1768 | ID == Intrinsic::hexagon_L2_loadrub_pbr); | |||
1769 | } | |||
1770 | ||||
1771 | // Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous | |||
1772 | // instruction. So far we only handle bitcast, extract value and bit reverse | |||
1773 | // load intrinsic instructions. Should we handle CGEP ? | |||
1774 | static Value *getBrevLdObject(Value *V) { | |||
1775 | if (Operator::getOpcode(V) == Instruction::ExtractValue || | |||
1776 | Operator::getOpcode(V) == Instruction::BitCast) | |||
1777 | V = cast<Operator>(V)->getOperand(0); | |||
1778 | else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V)) | |||
1779 | V = cast<Instruction>(V)->getOperand(0); | |||
1780 | return V; | |||
1781 | } | |||
1782 | ||||
1783 | // Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or | |||
1784 | // a back edge. If the back edge comes from the intrinsic itself, the incoming | |||
1785 | // edge is returned. | |||
1786 | static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) { | |||
1787 | const BasicBlock *Parent = PN->getParent(); | |||
1788 | int Idx = -1; | |||
1789 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) { | |||
1790 | BasicBlock *Blk = PN->getIncomingBlock(i); | |||
1791 | // Determine if the back edge is originated from intrinsic. | |||
1792 | if (Blk == Parent) { | |||
1793 | Value *BackEdgeVal = PN->getIncomingValue(i); | |||
1794 | Value *BaseVal; | |||
1795 | // Loop over till we return the same Value or we hit the IntrBaseVal. | |||
1796 | do { | |||
1797 | BaseVal = BackEdgeVal; | |||
1798 | BackEdgeVal = getBrevLdObject(BackEdgeVal); | |||
1799 | } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal)); | |||
1800 | // If the getBrevLdObject returns IntrBaseVal, we should return the | |||
1801 | // incoming edge. | |||
1802 | if (IntrBaseVal == BackEdgeVal) | |||
1803 | continue; | |||
1804 | Idx = i; | |||
1805 | break; | |||
1806 | } else // Set the node to incoming edge. | |||
1807 | Idx = i; | |||
1808 | } | |||
1809 | assert(Idx >= 0 && "Unexpected index to incoming argument in PHI")((Idx >= 0 && "Unexpected index to incoming argument in PHI" ) ? static_cast<void> (0) : __assert_fail ("Idx >= 0 && \"Unexpected index to incoming argument in PHI\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1809, __PRETTY_FUNCTION__)); | |||
1810 | return PN->getIncomingValue(Idx); | |||
1811 | } | |||
1812 | ||||
1813 | // Bit-reverse Load Intrinsic: Figure out the underlying object the base | |||
1814 | // pointer points to, for the bit-reverse load intrinsic. Setting this to | |||
1815 | // memoperand might help alias analysis to figure out the dependencies. | |||
1816 | static Value *getUnderLyingObjectForBrevLdIntr(Value *V) { | |||
1817 | Value *IntrBaseVal = V; | |||
1818 | Value *BaseVal; | |||
1819 | // Loop over till we return the same Value, implies we either figure out | |||
1820 | // the object or we hit a PHI | |||
1821 | do { | |||
1822 | BaseVal = V; | |||
1823 | V = getBrevLdObject(V); | |||
1824 | } while (BaseVal != V); | |||
1825 | ||||
1826 | // Identify the object from PHINode. | |||
1827 | if (const PHINode *PN = dyn_cast<PHINode>(V)) | |||
1828 | return returnEdge(PN, IntrBaseVal); | |||
1829 | // For non PHI nodes, the object is the last value returned by getBrevLdObject | |||
1830 | else | |||
1831 | return V; | |||
1832 | } | |||
1833 | ||||
1834 | /// Given an intrinsic, checks if on the target the intrinsic will need to map | |||
1835 | /// to a MemIntrinsicNode (touches memory). If this is the case, it returns | |||
1836 | /// true and store the intrinsic information into the IntrinsicInfo that was | |||
1837 | /// passed to the function. | |||
1838 | bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, | |||
1839 | const CallInst &I, | |||
1840 | MachineFunction &MF, | |||
1841 | unsigned Intrinsic) const { | |||
1842 | switch (Intrinsic) { | |||
1843 | case Intrinsic::hexagon_L2_loadrd_pbr: | |||
1844 | case Intrinsic::hexagon_L2_loadri_pbr: | |||
1845 | case Intrinsic::hexagon_L2_loadrh_pbr: | |||
1846 | case Intrinsic::hexagon_L2_loadruh_pbr: | |||
1847 | case Intrinsic::hexagon_L2_loadrb_pbr: | |||
1848 | case Intrinsic::hexagon_L2_loadrub_pbr: { | |||
1849 | Info.opc = ISD::INTRINSIC_W_CHAIN; | |||
1850 | auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); | |||
1851 | auto &Cont = I.getCalledFunction()->getParent()->getContext(); | |||
1852 | // The intrinsic function call is of the form { ElTy, i8* } | |||
1853 | // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type | |||
1854 | // should be derived from ElTy. | |||
1855 | Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0); | |||
1856 | Info.memVT = MVT::getVT(ElTy); | |||
1857 | llvm::Value *BasePtrVal = I.getOperand(0); | |||
1858 | Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal); | |||
1859 | // The offset value comes through Modifier register. For now, assume the | |||
1860 | // offset is 0. | |||
1861 | Info.offset = 0; | |||
1862 | Info.align = | |||
1863 | MaybeAlign(DL.getABITypeAlignment(Info.memVT.getTypeForEVT(Cont))); | |||
1864 | Info.flags = MachineMemOperand::MOLoad; | |||
1865 | return true; | |||
1866 | } | |||
1867 | case Intrinsic::hexagon_V6_vgathermw: | |||
1868 | case Intrinsic::hexagon_V6_vgathermw_128B: | |||
1869 | case Intrinsic::hexagon_V6_vgathermh: | |||
1870 | case Intrinsic::hexagon_V6_vgathermh_128B: | |||
1871 | case Intrinsic::hexagon_V6_vgathermhw: | |||
1872 | case Intrinsic::hexagon_V6_vgathermhw_128B: | |||
1873 | case Intrinsic::hexagon_V6_vgathermwq: | |||
1874 | case Intrinsic::hexagon_V6_vgathermwq_128B: | |||
1875 | case Intrinsic::hexagon_V6_vgathermhq: | |||
1876 | case Intrinsic::hexagon_V6_vgathermhq_128B: | |||
1877 | case Intrinsic::hexagon_V6_vgathermhwq: | |||
1878 | case Intrinsic::hexagon_V6_vgathermhwq_128B: { | |||
1879 | const Module &M = *I.getParent()->getParent()->getParent(); | |||
1880 | Info.opc = ISD::INTRINSIC_W_CHAIN; | |||
1881 | Type *VecTy = I.getArgOperand(1)->getType(); | |||
1882 | Info.memVT = MVT::getVT(VecTy); | |||
1883 | Info.ptrVal = I.getArgOperand(0); | |||
1884 | Info.offset = 0; | |||
1885 | Info.align = | |||
1886 | MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8); | |||
1887 | Info.flags = MachineMemOperand::MOLoad | | |||
1888 | MachineMemOperand::MOStore | | |||
1889 | MachineMemOperand::MOVolatile; | |||
1890 | return true; | |||
1891 | } | |||
1892 | default: | |||
1893 | break; | |||
1894 | } | |||
1895 | return false; | |||
1896 | } | |||
1897 | ||||
1898 | bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const { | |||
1899 | return X.getValueType().isScalarInteger(); // 'tstbit' | |||
1900 | } | |||
1901 | ||||
1902 | bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { | |||
1903 | return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2)); | |||
1904 | } | |||
1905 | ||||
1906 | bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { | |||
1907 | if (!VT1.isSimple() || !VT2.isSimple()) | |||
1908 | return false; | |||
1909 | return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32; | |||
1910 | } | |||
1911 | ||||
1912 | bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { | |||
1913 | return isOperationLegalOrCustom(ISD::FMA, VT); | |||
1914 | } | |||
1915 | ||||
1916 | // Should we expand the build vector with shuffles? | |||
1917 | bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT, | |||
1918 | unsigned DefinedValues) const { | |||
1919 | return false; | |||
1920 | } | |||
1921 | ||||
1922 | bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, | |||
1923 | EVT VT) const { | |||
1924 | return true; | |||
1925 | } | |||
1926 | ||||
1927 | TargetLoweringBase::LegalizeTypeAction | |||
1928 | HexagonTargetLowering::getPreferredVectorAction(MVT VT) const { | |||
1929 | unsigned VecLen = VT.getVectorNumElements(); | |||
1930 | MVT ElemTy = VT.getVectorElementType(); | |||
1931 | ||||
1932 | if (VecLen == 1 || VT.isScalableVector()) | |||
1933 | return TargetLoweringBase::TypeScalarizeVector; | |||
1934 | ||||
1935 | if (Subtarget.useHVXOps()) { | |||
1936 | unsigned HwLen = Subtarget.getVectorLength(); | |||
1937 | // If the size of VT is at least half of the vector length, | |||
1938 | // widen the vector. Note: the threshold was not selected in | |||
1939 | // any scientific way. | |||
1940 | ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes(); | |||
1941 | if (llvm::find(Tys, ElemTy) != Tys.end()) { | |||
1942 | unsigned HwWidth = 8*HwLen; | |||
1943 | unsigned VecWidth = VT.getSizeInBits(); | |||
1944 | if (VecWidth >= HwWidth/2 && VecWidth < HwWidth) | |||
1945 | return TargetLoweringBase::TypeWidenVector; | |||
1946 | } | |||
1947 | // Split vectors of i1 that correspond to (byte) vector pairs. | |||
1948 | if (ElemTy == MVT::i1 && VecLen == 2*HwLen) | |||
1949 | return TargetLoweringBase::TypeSplitVector; | |||
1950 | } | |||
1951 | ||||
1952 | // Always widen (remaining) vectors of i1. | |||
1953 | if (ElemTy == MVT::i1) | |||
1954 | return TargetLoweringBase::TypeWidenVector; | |||
1955 | ||||
1956 | return TargetLoweringBase::TypeSplitVector; | |||
1957 | } | |||
1958 | ||||
1959 | std::pair<SDValue, int> | |||
1960 | HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const { | |||
1961 | if (Addr.getOpcode() == ISD::ADD) { | |||
1962 | SDValue Op1 = Addr.getOperand(1); | |||
1963 | if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode())) | |||
1964 | return { Addr.getOperand(0), CN->getSExtValue() }; | |||
1965 | } | |||
1966 | return { Addr, 0 }; | |||
1967 | } | |||
1968 | ||||
1969 | // Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors | |||
1970 | // to select data from, V3 is the permutation. | |||
1971 | SDValue | |||
1972 | HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) | |||
1973 | const { | |||
1974 | const auto *SVN = cast<ShuffleVectorSDNode>(Op); | |||
1975 | ArrayRef<int> AM = SVN->getMask(); | |||
1976 | assert(AM.size() <= 8 && "Unexpected shuffle mask")((AM.size() <= 8 && "Unexpected shuffle mask") ? static_cast <void> (0) : __assert_fail ("AM.size() <= 8 && \"Unexpected shuffle mask\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1976, __PRETTY_FUNCTION__)); | |||
1977 | unsigned VecLen = AM.size(); | |||
1978 | ||||
1979 | MVT VecTy = ty(Op); | |||
1980 | assert(!Subtarget.isHVXVectorType(VecTy, true) &&((!Subtarget.isHVXVectorType(VecTy, true) && "HVX shuffles should be legal" ) ? static_cast<void> (0) : __assert_fail ("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1981, __PRETTY_FUNCTION__)) | |||
1981 | "HVX shuffles should be legal")((!Subtarget.isHVXVectorType(VecTy, true) && "HVX shuffles should be legal" ) ? static_cast<void> (0) : __assert_fail ("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1981, __PRETTY_FUNCTION__)); | |||
1982 | assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length")((VecTy.getSizeInBits() <= 64 && "Unexpected vector length" ) ? static_cast<void> (0) : __assert_fail ("VecTy.getSizeInBits() <= 64 && \"Unexpected vector length\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1982, __PRETTY_FUNCTION__)); | |||
1983 | ||||
1984 | SDValue Op0 = Op.getOperand(0); | |||
1985 | SDValue Op1 = Op.getOperand(1); | |||
1986 | const SDLoc &dl(Op); | |||
1987 | ||||
1988 | // If the inputs are not the same as the output, bail. This is not an | |||
1989 | // error situation, but complicates the handling and the default expansion | |||
1990 | // (into BUILD_VECTOR) should be adequate. | |||
1991 | if (ty(Op0) != VecTy || ty(Op1) != VecTy) | |||
1992 | return SDValue(); | |||
1993 | ||||
1994 | // Normalize the mask so that the first non-negative index comes from | |||
1995 | // the first operand. | |||
1996 | SmallVector<int,8> Mask(AM.begin(), AM.end()); | |||
1997 | unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data(); | |||
1998 | if (F == AM.size()) | |||
1999 | return DAG.getUNDEF(VecTy); | |||
2000 | if (AM[F] >= int(VecLen)) { | |||
2001 | ShuffleVectorSDNode::commuteMask(Mask); | |||
2002 | std::swap(Op0, Op1); | |||
2003 | } | |||
2004 | ||||
2005 | // Express the shuffle mask in terms of bytes. | |||
2006 | SmallVector<int,8> ByteMask; | |||
2007 | unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8; | |||
2008 | for (unsigned i = 0, e = Mask.size(); i != e; ++i) { | |||
2009 | int M = Mask[i]; | |||
2010 | if (M < 0) { | |||
2011 | for (unsigned j = 0; j != ElemBytes; ++j) | |||
2012 | ByteMask.push_back(-1); | |||
2013 | } else { | |||
2014 | for (unsigned j = 0; j != ElemBytes; ++j) | |||
2015 | ByteMask.push_back(M*ElemBytes + j); | |||
2016 | } | |||
2017 | } | |||
2018 | assert(ByteMask.size() <= 8)((ByteMask.size() <= 8) ? static_cast<void> (0) : __assert_fail ("ByteMask.size() <= 8", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2018, __PRETTY_FUNCTION__)); | |||
2019 | ||||
2020 | // All non-undef (non-negative) indexes are well within [0..127], so they | |||
2021 | // fit in a single byte. Build two 64-bit words: | |||
2022 | // - MaskIdx where each byte is the corresponding index (for non-negative | |||
2023 | // indexes), and 0xFF for negative indexes, and | |||
2024 | // - MaskUnd that has 0xFF for each negative index. | |||
2025 | uint64_t MaskIdx = 0; | |||
2026 | uint64_t MaskUnd = 0; | |||
2027 | for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) { | |||
2028 | unsigned S = 8*i; | |||
2029 | uint64_t M = ByteMask[i] & 0xFF; | |||
2030 | if (M == 0xFF) | |||
2031 | MaskUnd |= M << S; | |||
2032 | MaskIdx |= M << S; | |||
2033 | } | |||
2034 | ||||
2035 | if (ByteMask.size() == 4) { | |||
2036 | // Identity. | |||
2037 | if (MaskIdx == (0x03020100 | MaskUnd)) | |||
2038 | return Op0; | |||
2039 | // Byte swap. | |||
2040 | if (MaskIdx == (0x00010203 | MaskUnd)) { | |||
2041 | SDValue T0 = DAG.getBitcast(MVT::i32, Op0); | |||
2042 | SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0); | |||
2043 | return DAG.getBitcast(VecTy, T1); | |||
2044 | } | |||
2045 | ||||
2046 | // Byte packs. | |||
2047 | SDValue Concat10 = DAG.getNode(HexagonISD::COMBINE, dl, | |||
2048 | typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0}); | |||
2049 | if (MaskIdx == (0x06040200 | MaskUnd)) | |||
2050 | return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG); | |||
2051 | if (MaskIdx == (0x07050301 | MaskUnd)) | |||
2052 | return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG); | |||
2053 | ||||
2054 | SDValue Concat01 = DAG.getNode(HexagonISD::COMBINE, dl, | |||
2055 | typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1}); | |||
2056 | if (MaskIdx == (0x02000604 | MaskUnd)) | |||
2057 | return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG); | |||
2058 | if (MaskIdx == (0x03010705 | MaskUnd)) | |||
2059 | return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG); | |||
2060 | } | |||
2061 | ||||
2062 | if (ByteMask.size() == 8) { | |||
2063 | // Identity. | |||
2064 | if (MaskIdx == (0x0706050403020100ull | MaskUnd)) | |||
2065 | return Op0; | |||
2066 | // Byte swap. | |||
2067 | if (MaskIdx == (0x0001020304050607ull | MaskUnd)) { | |||
2068 | SDValue T0 = DAG.getBitcast(MVT::i64, Op0); | |||
2069 | SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0); | |||
2070 | return DAG.getBitcast(VecTy, T1); | |||
2071 | } | |||
2072 | ||||
2073 | // Halfword picks. | |||
2074 | if (MaskIdx == (0x0d0c050409080100ull | MaskUnd)) | |||
2075 | return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG); | |||
2076 | if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd)) | |||
2077 | return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG); | |||
2078 | if (MaskIdx == (0x0d0c090805040100ull | MaskUnd)) | |||
2079 | return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG); | |||
2080 | if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd)) | |||
2081 | return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG); | |||
2082 | if (MaskIdx == (0x0706030205040100ull | MaskUnd)) { | |||
2083 | VectorPair P = opSplit(Op0, dl, DAG); | |||
2084 | return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG); | |||
2085 | } | |||
2086 | ||||
2087 | // Byte packs. | |||
2088 | if (MaskIdx == (0x0e060c040a020800ull | MaskUnd)) | |||
2089 | return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG); | |||
2090 | if (MaskIdx == (0x0f070d050b030901ull | MaskUnd)) | |||
2091 | return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG); | |||
2092 | } | |||
2093 | ||||
2094 | return SDValue(); | |||
2095 | } | |||
2096 | ||||
2097 | // Create a Hexagon-specific node for shifting a vector by an integer. | |||
2098 | SDValue | |||
2099 | HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) | |||
2100 | const { | |||
2101 | if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) { | |||
2102 | if (SDValue S = BVN->getSplatValue()) { | |||
2103 | unsigned NewOpc; | |||
2104 | switch (Op.getOpcode()) { | |||
2105 | case ISD::SHL: | |||
2106 | NewOpc = HexagonISD::VASL; | |||
2107 | break; | |||
2108 | case ISD::SRA: | |||
2109 | NewOpc = HexagonISD::VASR; | |||
2110 | break; | |||
2111 | case ISD::SRL: | |||
2112 | NewOpc = HexagonISD::VLSR; | |||
2113 | break; | |||
2114 | default: | |||
2115 | llvm_unreachable("Unexpected shift opcode")::llvm::llvm_unreachable_internal("Unexpected shift opcode", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2115); | |||
2116 | } | |||
2117 | return DAG.getNode(NewOpc, SDLoc(Op), ty(Op), Op.getOperand(0), S); | |||
2118 | } | |||
2119 | } | |||
2120 | ||||
2121 | return SDValue(); | |||
2122 | } | |||
2123 | ||||
2124 | SDValue | |||
2125 | HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { | |||
2126 | return getVectorShiftByInt(Op, DAG); | |||
2127 | } | |||
2128 | ||||
2129 | SDValue | |||
2130 | HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const { | |||
2131 | if (isa<ConstantSDNode>(Op.getOperand(1).getNode())) | |||
2132 | return Op; | |||
2133 | return SDValue(); | |||
2134 | } | |||
2135 | ||||
2136 | SDValue | |||
2137 | HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { | |||
2138 | MVT ResTy = ty(Op); | |||
2139 | SDValue InpV = Op.getOperand(0); | |||
2140 | MVT InpTy = ty(InpV); | |||
2141 | assert(ResTy.getSizeInBits() == InpTy.getSizeInBits())((ResTy.getSizeInBits() == InpTy.getSizeInBits()) ? static_cast <void> (0) : __assert_fail ("ResTy.getSizeInBits() == InpTy.getSizeInBits()" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2141, __PRETTY_FUNCTION__)); | |||
2142 | const SDLoc &dl(Op); | |||
2143 | ||||
2144 | // Handle conversion from i8 to v8i1. | |||
2145 | if (ResTy == MVT::v8i1) { | |||
2146 | SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV); | |||
2147 | SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32); | |||
2148 | return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG); | |||
2149 | } | |||
2150 | ||||
2151 | return SDValue(); | |||
2152 | } | |||
2153 | ||||
2154 | bool | |||
2155 | HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values, | |||
2156 | MVT VecTy, SelectionDAG &DAG, | |||
2157 | MutableArrayRef<ConstantInt*> Consts) const { | |||
2158 | MVT ElemTy = VecTy.getVectorElementType(); | |||
2159 | unsigned ElemWidth = ElemTy.getSizeInBits(); | |||
2160 | IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth); | |||
2161 | bool AllConst = true; | |||
2162 | ||||
2163 | for (unsigned i = 0, e = Values.size(); i != e; ++i) { | |||
2164 | SDValue V = Values[i]; | |||
2165 | if (V.isUndef()) { | |||
2166 | Consts[i] = ConstantInt::get(IntTy, 0); | |||
2167 | continue; | |||
2168 | } | |||
2169 | // Make sure to always cast to IntTy. | |||
2170 | if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) { | |||
2171 | const ConstantInt *CI = CN->getConstantIntValue(); | |||
2172 | Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue()); | |||
2173 | } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) { | |||
2174 | const ConstantFP *CF = CN->getConstantFPValue(); | |||
2175 | APInt A = CF->getValueAPF().bitcastToAPInt(); | |||
2176 | Consts[i] = ConstantInt::get(IntTy, A.getZExtValue()); | |||
2177 | } else { | |||
2178 | AllConst = false; | |||
2179 | } | |||
2180 | } | |||
2181 | return AllConst; | |||
2182 | } | |||
2183 | ||||
2184 | SDValue | |||
2185 | HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, | |||
2186 | MVT VecTy, SelectionDAG &DAG) const { | |||
2187 | MVT ElemTy = VecTy.getVectorElementType(); | |||
2188 | assert(VecTy.getVectorNumElements() == Elem.size())((VecTy.getVectorNumElements() == Elem.size()) ? static_cast< void> (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2188, __PRETTY_FUNCTION__)); | |||
2189 | ||||
2190 | SmallVector<ConstantInt*,4> Consts(Elem.size()); | |||
2191 | bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); | |||
2192 | ||||
2193 | unsigned First, Num = Elem.size(); | |||
2194 | for (First = 0; First != Num; ++First) | |||
2195 | if (!isUndef(Elem[First])) | |||
2196 | break; | |||
2197 | if (First == Num) | |||
2198 | return DAG.getUNDEF(VecTy); | |||
2199 | ||||
2200 | if (AllConst && | |||
2201 | llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); })) | |||
2202 | return getZero(dl, VecTy, DAG); | |||
2203 | ||||
2204 | if (ElemTy == MVT::i16) { | |||
2205 | assert(Elem.size() == 2)((Elem.size() == 2) ? static_cast<void> (0) : __assert_fail ("Elem.size() == 2", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2205, __PRETTY_FUNCTION__)); | |||
2206 | if (AllConst) { | |||
2207 | uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) | | |||
2208 | Consts[1]->getZExtValue() << 16; | |||
2209 | return DAG.getBitcast(MVT::v2i16, DAG.getConstant(V, dl, MVT::i32)); | |||
2210 | } | |||
2211 | SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, | |||
2212 | {Elem[1], Elem[0]}, DAG); | |||
2213 | return DAG.getBitcast(MVT::v2i16, N); | |||
2214 | } | |||
2215 | ||||
2216 | if (ElemTy == MVT::i8) { | |||
2217 | // First try generating a constant. | |||
2218 | if (AllConst) { | |||
2219 | int32_t V = (Consts[0]->getZExtValue() & 0xFF) | | |||
2220 | (Consts[1]->getZExtValue() & 0xFF) << 8 | | |||
2221 | (Consts[1]->getZExtValue() & 0xFF) << 16 | | |||
2222 | Consts[2]->getZExtValue() << 24; | |||
2223 | return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32)); | |||
2224 | } | |||
2225 | ||||
2226 | // Then try splat. | |||
2227 | bool IsSplat = true; | |||
2228 | for (unsigned i = 0; i != Num; ++i) { | |||
2229 | if (i == First) | |||
2230 | continue; | |||
2231 | if (Elem[i] == Elem[First] || isUndef(Elem[i])) | |||
2232 | continue; | |||
2233 | IsSplat = false; | |||
2234 | break; | |||
2235 | } | |||
2236 | if (IsSplat) { | |||
2237 | // Legalize the operand to VSPLAT. | |||
2238 | SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); | |||
2239 | return DAG.getNode(HexagonISD::VSPLAT, dl, VecTy, Ext); | |||
2240 | } | |||
2241 | ||||
2242 | // Generate | |||
2243 | // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) | | |||
2244 | // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16 | |||
2245 | assert(Elem.size() == 4)((Elem.size() == 4) ? static_cast<void> (0) : __assert_fail ("Elem.size() == 4", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2245, __PRETTY_FUNCTION__)); | |||
2246 | SDValue Vs[4]; | |||
2247 | for (unsigned i = 0; i != 4; ++i) { | |||
2248 | Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32); | |||
2249 | Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8); | |||
2250 | } | |||
2251 | SDValue S8 = DAG.getConstant(8, dl, MVT::i32); | |||
2252 | SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8}); | |||
2253 | SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8}); | |||
2254 | SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0}); | |||
2255 | SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1}); | |||
2256 | ||||
2257 | SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG); | |||
2258 | return DAG.getBitcast(MVT::v4i8, R); | |||
2259 | } | |||
2260 | ||||
2261 | #ifndef NDEBUG | |||
2262 | dbgs() << "VecTy: " << EVT(VecTy).getEVTString() << '\n'; | |||
2263 | #endif | |||
2264 | llvm_unreachable("Unexpected vector element type")::llvm::llvm_unreachable_internal("Unexpected vector element type" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2264); | |||
2265 | } | |||
2266 | ||||
2267 | SDValue | |||
2268 | HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, | |||
2269 | MVT VecTy, SelectionDAG &DAG) const { | |||
2270 | MVT ElemTy = VecTy.getVectorElementType(); | |||
2271 | assert(VecTy.getVectorNumElements() == Elem.size())((VecTy.getVectorNumElements() == Elem.size()) ? static_cast< void> (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2271, __PRETTY_FUNCTION__)); | |||
2272 | ||||
2273 | SmallVector<ConstantInt*,8> Consts(Elem.size()); | |||
2274 | bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); | |||
2275 | ||||
2276 | unsigned First, Num = Elem.size(); | |||
2277 | for (First = 0; First != Num; ++First) | |||
2278 | if (!isUndef(Elem[First])) | |||
2279 | break; | |||
2280 | if (First == Num) | |||
2281 | return DAG.getUNDEF(VecTy); | |||
2282 | ||||
2283 | if (AllConst && | |||
2284 | llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); })) | |||
2285 | return getZero(dl, VecTy, DAG); | |||
2286 | ||||
2287 | // First try splat if possible. | |||
2288 | if (ElemTy == MVT::i16) { | |||
2289 | bool IsSplat = true; | |||
2290 | for (unsigned i = 0; i != Num; ++i) { | |||
2291 | if (i == First) | |||
2292 | continue; | |||
2293 | if (Elem[i] == Elem[First] || isUndef(Elem[i])) | |||
2294 | continue; | |||
2295 | IsSplat = false; | |||
2296 | break; | |||
2297 | } | |||
2298 | if (IsSplat) { | |||
2299 | // Legalize the operand to VSPLAT. | |||
2300 | SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); | |||
2301 | return DAG.getNode(HexagonISD::VSPLAT, dl, VecTy, Ext); | |||
2302 | } | |||
2303 | } | |||
2304 | ||||
2305 | // Then try constant. | |||
2306 | if (AllConst) { | |||
2307 | uint64_t Val = 0; | |||
2308 | unsigned W = ElemTy.getSizeInBits(); | |||
2309 | uint64_t Mask = (ElemTy == MVT::i8) ? 0xFFull | |||
2310 | : (ElemTy == MVT::i16) ? 0xFFFFull : 0xFFFFFFFFull; | |||
2311 | for (unsigned i = 0; i != Num; ++i) | |||
2312 | Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask); | |||
2313 | SDValue V0 = DAG.getConstant(Val, dl, MVT::i64); | |||
2314 | return DAG.getBitcast(VecTy, V0); | |||
2315 | } | |||
2316 | ||||
2317 | // Build two 32-bit vectors and concatenate. | |||
2318 | MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2); | |||
2319 | SDValue L = (ElemTy == MVT::i32) | |||
2320 | ? Elem[0] | |||
2321 | : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG); | |||
2322 | SDValue H = (ElemTy == MVT::i32) | |||
2323 | ? Elem[1] | |||
2324 | : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG); | |||
2325 | return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, {H, L}); | |||
2326 | } | |||
2327 | ||||
2328 | SDValue | |||
2329 | HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV, | |||
2330 | const SDLoc &dl, MVT ValTy, MVT ResTy, | |||
2331 | SelectionDAG &DAG) const { | |||
2332 | MVT VecTy = ty(VecV); | |||
2333 | assert(!ValTy.isVector() ||((!ValTy.isVector() || VecTy.getVectorElementType() == ValTy. getVectorElementType()) ? static_cast<void> (0) : __assert_fail ("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2334, __PRETTY_FUNCTION__)) | |||
2334 | VecTy.getVectorElementType() == ValTy.getVectorElementType())((!ValTy.isVector() || VecTy.getVectorElementType() == ValTy. getVectorElementType()) ? static_cast<void> (0) : __assert_fail ("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2334, __PRETTY_FUNCTION__)); | |||
2335 | unsigned VecWidth = VecTy.getSizeInBits(); | |||
2336 | unsigned ValWidth = ValTy.getSizeInBits(); | |||
2337 | unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits(); | |||
2338 | assert((VecWidth % ElemWidth) == 0)(((VecWidth % ElemWidth) == 0) ? static_cast<void> (0) : __assert_fail ("(VecWidth % ElemWidth) == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2338, __PRETTY_FUNCTION__)); | |||
2339 | auto *IdxN = dyn_cast<ConstantSDNode>(IdxV); | |||
2340 | ||||
2341 | // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon | |||
2342 | // without any coprocessors). | |||
2343 | if (ElemWidth == 1) { | |||
2344 | assert(VecWidth == VecTy.getVectorNumElements() && "Sanity failure")((VecWidth == VecTy.getVectorNumElements() && "Sanity failure" ) ? static_cast<void> (0) : __assert_fail ("VecWidth == VecTy.getVectorNumElements() && \"Sanity failure\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2344, __PRETTY_FUNCTION__)); | |||
2345 | assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2)((VecWidth == 8 || VecWidth == 4 || VecWidth == 2) ? static_cast <void> (0) : __assert_fail ("VecWidth == 8 || VecWidth == 4 || VecWidth == 2" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2345, __PRETTY_FUNCTION__)); | |||
2346 | // Check if this is an extract of the lowest bit. | |||
2347 | if (IdxN) { | |||
2348 | // Extracting the lowest bit is a no-op, but it changes the type, | |||
2349 | // so it must be kept as an operation to avoid errors related to | |||
2350 | // type mismatches. | |||
2351 | if (IdxN->isNullValue() && ValTy.getSizeInBits() == 1) | |||
2352 | return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV); | |||
2353 | } | |||
2354 | ||||
2355 | // If the value extracted is a single bit, use tstbit. | |||
2356 | if (ValWidth == 1) { | |||
2357 | SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG); | |||
2358 | SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32); | |||
2359 | SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0); | |||
2360 | return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0); | |||
2361 | } | |||
2362 | ||||
2363 | // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in | |||
2364 | // a predicate register. The elements of the vector are repeated | |||
2365 | // in the register (if necessary) so that the total number is 8. | |||
2366 | // The extracted subvector will need to be expanded in such a way. | |||
2367 | unsigned Scale = VecWidth / ValWidth; | |||
2368 | ||||
2369 | // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to | |||
2370 | // position 0. | |||
2371 | assert(ty(IdxV) == MVT::i32)((ty(IdxV) == MVT::i32) ? static_cast<void> (0) : __assert_fail ("ty(IdxV) == MVT::i32", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2371, __PRETTY_FUNCTION__)); | |||
2372 | unsigned VecRep = 8 / VecWidth; | |||
2373 | SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | |||
2374 | DAG.getConstant(8*VecRep, dl, MVT::i32)); | |||
2375 | SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV); | |||
2376 | SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0); | |||
2377 | while (Scale > 1) { | |||
2378 | // The longest possible subvector is at most 32 bits, so it is always | |||
2379 | // contained in the low subregister. | |||
2380 | T1 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, T1); | |||
2381 | T1 = expandPredicate(T1, dl, DAG); | |||
2382 | Scale /= 2; | |||
2383 | } | |||
2384 | ||||
2385 | return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1); | |||
2386 | } | |||
2387 | ||||
2388 | assert(VecWidth == 32 || VecWidth == 64)((VecWidth == 32 || VecWidth == 64) ? static_cast<void> (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2388, __PRETTY_FUNCTION__)); | |||
2389 | ||||
2390 | // Cast everything to scalar integer types. | |||
2391 | MVT ScalarTy = tyScalar(VecTy); | |||
2392 | VecV = DAG.getBitcast(ScalarTy, VecV); | |||
2393 | ||||
2394 | SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32); | |||
2395 | SDValue ExtV; | |||
2396 | ||||
2397 | if (IdxN) { | |||
2398 | unsigned Off = IdxN->getZExtValue() * ElemWidth; | |||
2399 | if (VecWidth == 64 && ValWidth == 32) { | |||
2400 | assert(Off == 0 || Off == 32)((Off == 0 || Off == 32) ? static_cast<void> (0) : __assert_fail ("Off == 0 || Off == 32", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2400, __PRETTY_FUNCTION__)); | |||
2401 | unsigned SubIdx = Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi; | |||
2402 | ExtV = DAG.getTargetExtractSubreg(SubIdx, dl, MVT::i32, VecV); | |||
2403 | } else if (Off == 0 && (ValWidth % 8) == 0) { | |||
2404 | ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy)); | |||
2405 | } else { | |||
2406 | SDValue OffV = DAG.getConstant(Off, dl, MVT::i32); | |||
2407 | // The return type of EXTRACTU must be the same as the type of the | |||
2408 | // input vector. | |||
2409 | ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy, | |||
2410 | {VecV, WidthV, OffV}); | |||
2411 | } | |||
2412 | } else { | |||
2413 | if (ty(IdxV) != MVT::i32) | |||
2414 | IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32); | |||
2415 | SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | |||
2416 | DAG.getConstant(ElemWidth, dl, MVT::i32)); | |||
2417 | ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy, | |||
2418 | {VecV, WidthV, OffV}); | |||
2419 | } | |||
2420 | ||||
2421 | // Cast ExtV to the requested result type. | |||
2422 | ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy)); | |||
2423 | ExtV = DAG.getBitcast(ResTy, ExtV); | |||
2424 | return ExtV; | |||
2425 | } | |||
2426 | ||||
2427 | SDValue | |||
2428 | HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV, | |||
2429 | const SDLoc &dl, MVT ValTy, | |||
2430 | SelectionDAG &DAG) const { | |||
2431 | MVT VecTy = ty(VecV); | |||
2432 | if (VecTy.getVectorElementType() == MVT::i1) { | |||
2433 | MVT ValTy = ty(ValV); | |||
2434 | assert(ValTy.getVectorElementType() == MVT::i1)((ValTy.getVectorElementType() == MVT::i1) ? static_cast<void > (0) : __assert_fail ("ValTy.getVectorElementType() == MVT::i1" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2434, __PRETTY_FUNCTION__)); | |||
2435 | SDValue ValR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV); | |||
2436 | unsigned VecLen = VecTy.getVectorNumElements(); | |||
2437 | unsigned Scale = VecLen / ValTy.getVectorNumElements(); | |||
2438 | assert(Scale > 1)((Scale > 1) ? static_cast<void> (0) : __assert_fail ("Scale > 1", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2438, __PRETTY_FUNCTION__)); | |||
2439 | ||||
2440 | for (unsigned R = Scale; R > 1; R /= 2) { | |||
2441 | ValR = contractPredicate(ValR, dl, DAG); | |||
2442 | ValR = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | |||
2443 | DAG.getUNDEF(MVT::i32), ValR); | |||
2444 | } | |||
2445 | // The longest possible subvector is at most 32 bits, so it is always | |||
2446 | // contained in the low subregister. | |||
2447 | ValR = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, ValR); | |||
2448 | ||||
2449 | unsigned ValBytes = 64 / Scale; | |||
2450 | SDValue Width = DAG.getConstant(ValBytes*8, dl, MVT::i32); | |||
2451 | SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | |||
2452 | DAG.getConstant(8, dl, MVT::i32)); | |||
2453 | SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV); | |||
2454 | SDValue Ins = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, | |||
2455 | {VecR, ValR, Width, Idx}); | |||
2456 | return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins); | |||
2457 | } | |||
2458 | ||||
2459 | unsigned VecWidth = VecTy.getSizeInBits(); | |||
2460 | unsigned ValWidth = ValTy.getSizeInBits(); | |||
2461 | assert(VecWidth == 32 || VecWidth == 64)((VecWidth == 32 || VecWidth == 64) ? static_cast<void> (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2461, __PRETTY_FUNCTION__)); | |||
2462 | assert((VecWidth % ValWidth) == 0)(((VecWidth % ValWidth) == 0) ? static_cast<void> (0) : __assert_fail ("(VecWidth % ValWidth) == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2462, __PRETTY_FUNCTION__)); | |||
2463 | ||||
2464 | // Cast everything to scalar integer types. | |||
2465 | MVT ScalarTy = MVT::getIntegerVT(VecWidth); | |||
2466 | // The actual type of ValV may be different than ValTy (which is related | |||
2467 | // to the vector type). | |||
2468 | unsigned VW = ty(ValV).getSizeInBits(); | |||
2469 | ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV); | |||
2470 | VecV = DAG.getBitcast(ScalarTy, VecV); | |||
2471 | if (VW != VecWidth) | |||
2472 | ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy); | |||
2473 | ||||
2474 | SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32); | |||
2475 | SDValue InsV; | |||
2476 | ||||
2477 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) { | |||
2478 | unsigned W = C->getZExtValue() * ValWidth; | |||
2479 | SDValue OffV = DAG.getConstant(W, dl, MVT::i32); | |||
2480 | InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy, | |||
2481 | {VecV, ValV, WidthV, OffV}); | |||
2482 | } else { | |||
2483 | if (ty(IdxV) != MVT::i32) | |||
2484 | IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32); | |||
2485 | SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV); | |||
2486 | InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy, | |||
2487 | {VecV, ValV, WidthV, OffV}); | |||
2488 | } | |||
2489 | ||||
2490 | return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV); | |||
2491 | } | |||
2492 | ||||
2493 | SDValue | |||
2494 | HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl, | |||
2495 | SelectionDAG &DAG) const { | |||
2496 | assert(ty(Vec32).getSizeInBits() == 32)((ty(Vec32).getSizeInBits() == 32) ? static_cast<void> ( 0) : __assert_fail ("ty(Vec32).getSizeInBits() == 32", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2496, __PRETTY_FUNCTION__)); | |||
2497 | if (isUndef(Vec32)) | |||
2498 | return DAG.getUNDEF(MVT::i64); | |||
2499 | return getInstr(Hexagon::S2_vsxtbh, dl, MVT::i64, {Vec32}, DAG); | |||
2500 | } | |||
2501 | ||||
2502 | SDValue | |||
2503 | HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl, | |||
2504 | SelectionDAG &DAG) const { | |||
2505 | assert(ty(Vec64).getSizeInBits() == 64)((ty(Vec64).getSizeInBits() == 64) ? static_cast<void> ( 0) : __assert_fail ("ty(Vec64).getSizeInBits() == 64", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2505, __PRETTY_FUNCTION__)); | |||
2506 | if (isUndef(Vec64)) | |||
2507 | return DAG.getUNDEF(MVT::i32); | |||
2508 | return getInstr(Hexagon::S2_vtrunehb, dl, MVT::i32, {Vec64}, DAG); | |||
2509 | } | |||
2510 | ||||
2511 | SDValue | |||
2512 | HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) | |||
2513 | const { | |||
2514 | if (Ty.isVector()) { | |||
2515 | assert(Ty.isInteger() && "Only integer vectors are supported here")((Ty.isInteger() && "Only integer vectors are supported here" ) ? static_cast<void> (0) : __assert_fail ("Ty.isInteger() && \"Only integer vectors are supported here\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2515, __PRETTY_FUNCTION__)); | |||
2516 | unsigned W = Ty.getSizeInBits(); | |||
2517 | if (W <= 64) | |||
2518 | return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W))); | |||
2519 | return DAG.getNode(HexagonISD::VZERO, dl, Ty); | |||
2520 | } | |||
2521 | ||||
2522 | if (Ty.isInteger()) | |||
2523 | return DAG.getConstant(0, dl, Ty); | |||
2524 | if (Ty.isFloatingPoint()) | |||
2525 | return DAG.getConstantFP(0.0, dl, Ty); | |||
2526 | llvm_unreachable("Invalid type for zero")::llvm::llvm_unreachable_internal("Invalid type for zero", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2526); | |||
2527 | } | |||
2528 | ||||
2529 | SDValue | |||
2530 | HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { | |||
2531 | MVT VecTy = ty(Op); | |||
2532 | unsigned BW = VecTy.getSizeInBits(); | |||
2533 | const SDLoc &dl(Op); | |||
2534 | SmallVector<SDValue,8> Ops; | |||
2535 | for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) | |||
2536 | Ops.push_back(Op.getOperand(i)); | |||
2537 | ||||
2538 | if (BW == 32) | |||
2539 | return buildVector32(Ops, dl, VecTy, DAG); | |||
2540 | if (BW == 64) | |||
2541 | return buildVector64(Ops, dl, VecTy, DAG); | |||
2542 | ||||
2543 | if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) { | |||
2544 | // Check if this is a special case or all-0 or all-1. | |||
2545 | bool All0 = true, All1 = true; | |||
2546 | for (SDValue P : Ops) { | |||
2547 | auto *CN = dyn_cast<ConstantSDNode>(P.getNode()); | |||
2548 | if (CN == nullptr) { | |||
2549 | All0 = All1 = false; | |||
2550 | break; | |||
2551 | } | |||
2552 | uint32_t C = CN->getZExtValue(); | |||
2553 | All0 &= (C == 0); | |||
2554 | All1 &= (C == 1); | |||
2555 | } | |||
2556 | if (All0) | |||
2557 | return DAG.getNode(HexagonISD::PFALSE, dl, VecTy); | |||
2558 | if (All1) | |||
2559 | return DAG.getNode(HexagonISD::PTRUE, dl, VecTy); | |||
2560 | ||||
2561 | // For each i1 element in the resulting predicate register, put 1 | |||
2562 | // shifted by the index of the element into a general-purpose register, | |||
2563 | // then or them together and transfer it back into a predicate register. | |||
2564 | SDValue Rs[8]; | |||
2565 | SDValue Z = getZero(dl, MVT::i32, DAG); | |||
2566 | // Always produce 8 bits, repeat inputs if necessary. | |||
2567 | unsigned Rep = 8 / VecTy.getVectorNumElements(); | |||
2568 | for (unsigned i = 0; i != 8; ++i) { | |||
2569 | SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32); | |||
2570 | Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z); | |||
2571 | } | |||
2572 | for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) { | |||
2573 | for (unsigned i = 0, e = A.size()/2; i != e; ++i) | |||
2574 | Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]); | |||
2575 | } | |||
2576 | // Move the value directly to a predicate register. | |||
2577 | return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG); | |||
2578 | } | |||
2579 | ||||
2580 | return SDValue(); | |||
2581 | } | |||
2582 | ||||
2583 | SDValue | |||
2584 | HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op, | |||
2585 | SelectionDAG &DAG) const { | |||
2586 | MVT VecTy = ty(Op); | |||
2587 | const SDLoc &dl(Op); | |||
2588 | if (VecTy.getSizeInBits() == 64) { | |||
2589 | assert(Op.getNumOperands() == 2)((Op.getNumOperands() == 2) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2589, __PRETTY_FUNCTION__)); | |||
2590 | return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, Op.getOperand(1), | |||
2591 | Op.getOperand(0)); | |||
2592 | } | |||
2593 | ||||
2594 | MVT ElemTy = VecTy.getVectorElementType(); | |||
2595 | if (ElemTy == MVT::i1) { | |||
2596 | assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1)((VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1 ) ? static_cast<void> (0) : __assert_fail ("VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2596, __PRETTY_FUNCTION__)); | |||
2597 | MVT OpTy = ty(Op.getOperand(0)); | |||
2598 | // Scale is how many times the operands need to be contracted to match | |||
2599 | // the representation in the target register. | |||
2600 | unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements(); | |||
2601 | assert(Scale == Op.getNumOperands() && Scale > 1)((Scale == Op.getNumOperands() && Scale > 1) ? static_cast <void> (0) : __assert_fail ("Scale == Op.getNumOperands() && Scale > 1" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2601, __PRETTY_FUNCTION__)); | |||
2602 | ||||
2603 | // First, convert all bool vectors to integers, then generate pairwise | |||
2604 | // inserts to form values of doubled length. Up until there are only | |||
2605 | // two values left to concatenate, all of these values will fit in a | |||
2606 | // 32-bit integer, so keep them as i32 to use 32-bit inserts. | |||
2607 | SmallVector<SDValue,4> Words[2]; | |||
2608 | unsigned IdxW = 0; | |||
2609 | ||||
2610 | for (SDValue P : Op.getNode()->op_values()) { | |||
2611 | SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P); | |||
2612 | for (unsigned R = Scale; R > 1; R /= 2) { | |||
2613 | W = contractPredicate(W, dl, DAG); | |||
2614 | W = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | |||
2615 | DAG.getUNDEF(MVT::i32), W); | |||
2616 | } | |||
2617 | W = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, W); | |||
2618 | Words[IdxW].push_back(W); | |||
2619 | } | |||
2620 | ||||
2621 | while (Scale > 2) { | |||
2622 | SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32); | |||
2623 | Words[IdxW ^ 1].clear(); | |||
2624 | ||||
2625 | for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) { | |||
2626 | SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1]; | |||
2627 | // Insert W1 into W0 right next to the significant bits of W0. | |||
2628 | SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, | |||
2629 | {W0, W1, WidthV, WidthV}); | |||
2630 | Words[IdxW ^ 1].push_back(T); | |||
2631 | } | |||
2632 | IdxW ^= 1; | |||
2633 | Scale /= 2; | |||
2634 | } | |||
2635 | ||||
2636 | // Another sanity check. At this point there should only be two words | |||
2637 | // left, and Scale should be 2. | |||
2638 | assert(Scale == 2 && Words[IdxW].size() == 2)((Scale == 2 && Words[IdxW].size() == 2) ? static_cast <void> (0) : __assert_fail ("Scale == 2 && Words[IdxW].size() == 2" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2638, __PRETTY_FUNCTION__)); | |||
2639 | ||||
2640 | SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | |||
2641 | Words[IdxW][1], Words[IdxW][0]); | |||
2642 | return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW); | |||
2643 | } | |||
2644 | ||||
2645 | return SDValue(); | |||
2646 | } | |||
2647 | ||||
2648 | SDValue | |||
2649 | HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, | |||
2650 | SelectionDAG &DAG) const { | |||
2651 | SDValue Vec = Op.getOperand(0); | |||
2652 | MVT ElemTy = ty(Vec).getVectorElementType(); | |||
2653 | return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG); | |||
2654 | } | |||
2655 | ||||
2656 | SDValue | |||
2657 | HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, | |||
2658 | SelectionDAG &DAG) const { | |||
2659 | return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op), | |||
2660 | ty(Op), ty(Op), DAG); | |||
2661 | } | |||
2662 | ||||
2663 | SDValue | |||
2664 | HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, | |||
2665 | SelectionDAG &DAG) const { | |||
2666 | return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2), | |||
2667 | SDLoc(Op), ty(Op).getVectorElementType(), DAG); | |||
2668 | } | |||
2669 | ||||
2670 | SDValue | |||
2671 | HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, | |||
2672 | SelectionDAG &DAG) const { | |||
2673 | SDValue ValV = Op.getOperand(1); | |||
2674 | return insertVector(Op.getOperand(0), ValV, Op.getOperand(2), | |||
2675 | SDLoc(Op), ty(ValV), DAG); | |||
2676 | } | |||
2677 | ||||
2678 | bool | |||
2679 | HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const { | |||
2680 | // Assuming the caller does not have either a signext or zeroext modifier, and | |||
2681 | // only one value is accepted, any reasonable truncation is allowed. | |||
2682 | if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) | |||
2683 | return false; | |||
2684 | ||||
2685 | // FIXME: in principle up to 64-bit could be made safe, but it would be very | |||
2686 | // fragile at the moment: any support for multiple value returns would be | |||
2687 | // liable to disallow tail calls involving i64 -> iN truncation in many cases. | |||
2688 | return Ty1->getPrimitiveSizeInBits() <= 32; | |||
2689 | } | |||
2690 | ||||
2691 | SDValue | |||
2692 | HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const { | |||
2693 | LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); | |||
2694 | unsigned ClaimAlign = LN->getAlignment(); | |||
2695 | validateConstPtrAlignment(LN->getBasePtr(), SDLoc(Op), ClaimAlign); | |||
2696 | // Call LowerUnalignedLoad for all loads, it recognizes loads that | |||
2697 | // don't need extra aligning. | |||
2698 | return LowerUnalignedLoad(Op, DAG); | |||
2699 | } | |||
2700 | ||||
2701 | SDValue | |||
2702 | HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const { | |||
2703 | StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); | |||
2704 | unsigned ClaimAlign = SN->getAlignment(); | |||
2705 | SDValue Ptr = SN->getBasePtr(); | |||
2706 | const SDLoc &dl(Op); | |||
2707 | validateConstPtrAlignment(Ptr, dl, ClaimAlign); | |||
2708 | ||||
2709 | MVT StoreTy = SN->getMemoryVT().getSimpleVT(); | |||
2710 | unsigned NeedAlign = Subtarget.getTypeAlignment(StoreTy); | |||
2711 | if (ClaimAlign < NeedAlign) | |||
2712 | return expandUnalignedStore(SN, DAG); | |||
2713 | return Op; | |||
2714 | } | |||
2715 | ||||
2716 | SDValue | |||
2717 | HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) | |||
2718 | const { | |||
2719 | LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); | |||
2720 | MVT LoadTy = ty(Op); | |||
2721 | unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy); | |||
2722 | unsigned HaveAlign = LN->getAlignment(); | |||
2723 | if (HaveAlign >= NeedAlign) | |||
2724 | return Op; | |||
2725 | ||||
2726 | const SDLoc &dl(Op); | |||
2727 | const DataLayout &DL = DAG.getDataLayout(); | |||
2728 | LLVMContext &Ctx = *DAG.getContext(); | |||
2729 | ||||
2730 | // If the load aligning is disabled or the load can be broken up into two | |||
2731 | // smaller legal loads, do the default (target-independent) expansion. | |||
2732 | bool DoDefault = false; | |||
2733 | // Handle it in the default way if this is an indexed load. | |||
2734 | if (!LN->isUnindexed()) | |||
2735 | DoDefault = true; | |||
2736 | ||||
2737 | if (!AlignLoads) { | |||
2738 | if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(), | |||
2739 | *LN->getMemOperand())) | |||
2740 | return Op; | |||
2741 | DoDefault = true; | |||
2742 | } | |||
2743 | if (!DoDefault && (2 * HaveAlign) == NeedAlign) { | |||
2744 | // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)". | |||
2745 | MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign) | |||
2746 | : MVT::getVectorVT(MVT::i8, HaveAlign); | |||
2747 | DoDefault = | |||
2748 | allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand()); | |||
2749 | } | |||
2750 | if (DoDefault) { | |||
2751 | std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG); | |||
2752 | return DAG.getMergeValues({P.first, P.second}, dl); | |||
2753 | } | |||
2754 | ||||
2755 | // The code below generates two loads, both aligned as NeedAlign, and | |||
2756 | // with the distance of NeedAlign between them. For that to cover the | |||
2757 | // bits that need to be loaded (and without overlapping), the size of | |||
2758 | // the loads should be equal to NeedAlign. This is true for all loadable | |||
2759 | // types, but add an assertion in case something changes in the future. | |||
2760 | assert(LoadTy.getSizeInBits() == 8*NeedAlign)((LoadTy.getSizeInBits() == 8*NeedAlign) ? static_cast<void > (0) : __assert_fail ("LoadTy.getSizeInBits() == 8*NeedAlign" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2760, __PRETTY_FUNCTION__)); | |||
2761 | ||||
2762 | unsigned LoadLen = NeedAlign; | |||
2763 | SDValue Base = LN->getBasePtr(); | |||
2764 | SDValue Chain = LN->getChain(); | |||
2765 | auto BO = getBaseAndOffset(Base); | |||
2766 | unsigned BaseOpc = BO.first.getOpcode(); | |||
2767 | if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0) | |||
2768 | return Op; | |||
2769 | ||||
2770 | if (BO.second % LoadLen != 0) { | |||
2771 | BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first, | |||
2772 | DAG.getConstant(BO.second % LoadLen, dl, MVT::i32)); | |||
2773 | BO.second -= BO.second % LoadLen; | |||
2774 | } | |||
2775 | SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR) | |||
2776 | ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first, | |||
2777 | DAG.getConstant(NeedAlign, dl, MVT::i32)) | |||
2778 | : BO.first; | |||
2779 | SDValue Base0 = DAG.getMemBasePlusOffset(BaseNoOff, BO.second, dl); | |||
2780 | SDValue Base1 = DAG.getMemBasePlusOffset(BaseNoOff, BO.second+LoadLen, dl); | |||
2781 | ||||
2782 | MachineMemOperand *WideMMO = nullptr; | |||
2783 | if (MachineMemOperand *MMO = LN->getMemOperand()) { | |||
2784 | MachineFunction &MF = DAG.getMachineFunction(); | |||
2785 | WideMMO = MF.getMachineMemOperand(MMO->getPointerInfo(), MMO->getFlags(), | |||
2786 | 2*LoadLen, LoadLen, MMO->getAAInfo(), MMO->getRanges(), | |||
2787 | MMO->getSyncScopeID(), MMO->getOrdering(), | |||
2788 | MMO->getFailureOrdering()); | |||
2789 | } | |||
2790 | ||||
2791 | SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO); | |||
2792 | SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO); | |||
2793 | ||||
2794 | SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy, | |||
2795 | {Load1, Load0, BaseNoOff.getOperand(0)}); | |||
2796 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | |||
2797 | Load0.getValue(1), Load1.getValue(1)); | |||
2798 | SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl); | |||
2799 | return M; | |||
2800 | } | |||
2801 | ||||
2802 | SDValue | |||
2803 | HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const { | |||
2804 | SDValue X = Op.getOperand(0), Y = Op.getOperand(1); | |||
2805 | auto *CY = dyn_cast<ConstantSDNode>(Y); | |||
2806 | if (!CY) | |||
2807 | return SDValue(); | |||
2808 | ||||
2809 | const SDLoc &dl(Op); | |||
2810 | SDVTList VTs = Op.getNode()->getVTList(); | |||
2811 | assert(VTs.NumVTs == 2)((VTs.NumVTs == 2) ? static_cast<void> (0) : __assert_fail ("VTs.NumVTs == 2", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2811, __PRETTY_FUNCTION__)); | |||
2812 | assert(VTs.VTs[1] == MVT::i1)((VTs.VTs[1] == MVT::i1) ? static_cast<void> (0) : __assert_fail ("VTs.VTs[1] == MVT::i1", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2812, __PRETTY_FUNCTION__)); | |||
2813 | unsigned Opc = Op.getOpcode(); | |||
2814 | ||||
2815 | if (CY) { | |||
2816 | uint32_t VY = CY->getZExtValue(); | |||
2817 | assert(VY != 0 && "This should have been folded")((VY != 0 && "This should have been folded") ? static_cast <void> (0) : __assert_fail ("VY != 0 && \"This should have been folded\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2817, __PRETTY_FUNCTION__)); | |||
2818 | // X +/- 1 | |||
2819 | if (VY != 1) | |||
2820 | return SDValue(); | |||
2821 | ||||
2822 | if (Opc == ISD::UADDO) { | |||
2823 | SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y}); | |||
2824 | SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG), | |||
2825 | ISD::SETEQ); | |||
2826 | return DAG.getMergeValues({Op, Ov}, dl); | |||
2827 | } | |||
2828 | if (Opc == ISD::USUBO) { | |||
2829 | SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y}); | |||
2830 | SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, | |||
2831 | DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ); | |||
2832 | return DAG.getMergeValues({Op, Ov}, dl); | |||
2833 | } | |||
2834 | } | |||
2835 | ||||
2836 | return SDValue(); | |||
2837 | } | |||
2838 | ||||
2839 | SDValue | |||
2840 | HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const { | |||
2841 | const SDLoc &dl(Op); | |||
2842 | unsigned Opc = Op.getOpcode(); | |||
2843 | SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2); | |||
2844 | ||||
2845 | if (Opc == ISD::ADDCARRY) | |||
2846 | return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(), | |||
2847 | { X, Y, C }); | |||
2848 | ||||
2849 | EVT CarryTy = C.getValueType(); | |||
2850 | SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(), | |||
2851 | { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) }); | |||
2852 | SDValue Out[] = { SubC.getValue(0), | |||
2853 | DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) }; | |||
2854 | return DAG.getMergeValues(Out, dl); | |||
2855 | } | |||
2856 | ||||
2857 | SDValue | |||
2858 | HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const { | |||
2859 | SDValue Chain = Op.getOperand(0); | |||
2860 | SDValue Offset = Op.getOperand(1); | |||
2861 | SDValue Handler = Op.getOperand(2); | |||
2862 | SDLoc dl(Op); | |||
2863 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | |||
2864 | ||||
2865 | // Mark function as containing a call to EH_RETURN. | |||
2866 | HexagonMachineFunctionInfo *FuncInfo = | |||
2867 | DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>(); | |||
2868 | FuncInfo->setHasEHReturn(); | |||
2869 | ||||
2870 | unsigned OffsetReg = Hexagon::R28; | |||
2871 | ||||
2872 | SDValue StoreAddr = | |||
2873 | DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT), | |||
2874 | DAG.getIntPtrConstant(4, dl)); | |||
2875 | Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo()); | |||
2876 | Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset); | |||
2877 | ||||
2878 | // Not needed we already use it as explict input to EH_RETURN. | |||
2879 | // MF.getRegInfo().addLiveOut(OffsetReg); | |||
2880 | ||||
2881 | return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain); | |||
2882 | } | |||
2883 | ||||
2884 | SDValue | |||
2885 | HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { | |||
2886 | unsigned Opc = Op.getOpcode(); | |||
2887 | ||||
2888 | // Handle INLINEASM first. | |||
2889 | if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR) | |||
| ||||
2890 | return LowerINLINEASM(Op, DAG); | |||
2891 | ||||
2892 | if (isHvxOperation(Op)) { | |||
2893 | // If HVX lowering returns nothing, try the default lowering. | |||
2894 | if (SDValue V = LowerHvxOperation(Op, DAG)) | |||
2895 | return V; | |||
2896 | } | |||
2897 | ||||
2898 | switch (Opc) { | |||
2899 | default: | |||
2900 | #ifndef NDEBUG | |||
2901 | Op.getNode()->dumpr(&DAG); | |||
2902 | if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END) | |||
2903 | errs() << "Error: check for a non-legal type in this operation\n"; | |||
2904 | #endif | |||
2905 | llvm_unreachable("Should not custom lower this!")::llvm::llvm_unreachable_internal("Should not custom lower this!" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2905); | |||
2906 | case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); | |||
2907 | case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG); | |||
2908 | case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); | |||
2909 | case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); | |||
2910 | case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); | |||
2911 | case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); | |||
2912 | case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); | |||
2913 | case ISD::BITCAST: return LowerBITCAST(Op, DAG); | |||
2914 | case ISD::LOAD: return LowerLoad(Op, DAG); | |||
2915 | case ISD::STORE: return LowerStore(Op, DAG); | |||
2916 | case ISD::UADDO: | |||
2917 | case ISD::USUBO: return LowerUAddSubO(Op, DAG); | |||
2918 | case ISD::ADDCARRY: | |||
2919 | case ISD::SUBCARRY: return LowerAddSubCarry(Op, DAG); | |||
2920 | case ISD::SRA: | |||
2921 | case ISD::SHL: | |||
2922 | case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG); | |||
2923 | case ISD::ROTL: return LowerROTL(Op, DAG); | |||
2924 | case ISD::ConstantPool: return LowerConstantPool(Op, DAG); | |||
2925 | case ISD::JumpTable: return LowerJumpTable(Op, DAG); | |||
2926 | case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); | |||
2927 | case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); | |||
2928 | case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); | |||
2929 | case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); | |||
2930 | case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG); | |||
2931 | case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG); | |||
2932 | case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); | |||
2933 | case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG); | |||
2934 | case ISD::VASTART: return LowerVASTART(Op, DAG); | |||
2935 | case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); | |||
2936 | case ISD::SETCC: return LowerSETCC(Op, DAG); | |||
2937 | case ISD::VSELECT: return LowerVSELECT(Op, DAG); | |||
2938 | case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); | |||
2939 | case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG); | |||
2940 | case ISD::PREFETCH: return LowerPREFETCH(Op, DAG); | |||
2941 | case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); | |||
2942 | break; | |||
2943 | } | |||
2944 | ||||
2945 | return SDValue(); | |||
2946 | } | |||
2947 | ||||
2948 | void | |||
2949 | HexagonTargetLowering::LowerOperationWrapper(SDNode *N, | |||
2950 | SmallVectorImpl<SDValue> &Results, | |||
2951 | SelectionDAG &DAG) const { | |||
2952 | // We are only custom-lowering stores to verify the alignment of the | |||
2953 | // address if it is a compile-time constant. Since a store can be modified | |||
2954 | // during type-legalization (the value being stored may need legalization), | |||
2955 | // return empty Results here to indicate that we don't really make any | |||
2956 | // changes in the custom lowering. | |||
2957 | if (N->getOpcode() != ISD::STORE) | |||
2958 | return TargetLowering::LowerOperationWrapper(N, Results, DAG); | |||
2959 | } | |||
2960 | ||||
2961 | void | |||
2962 | HexagonTargetLowering::ReplaceNodeResults(SDNode *N, | |||
2963 | SmallVectorImpl<SDValue> &Results, | |||
2964 | SelectionDAG &DAG) const { | |||
2965 | const SDLoc &dl(N); | |||
2966 | switch (N->getOpcode()) { | |||
2967 | case ISD::SRL: | |||
2968 | case ISD::SRA: | |||
2969 | case ISD::SHL: | |||
2970 | return; | |||
2971 | case ISD::BITCAST: | |||
2972 | // Handle a bitcast from v8i1 to i8. | |||
2973 | if (N->getValueType(0) == MVT::i8) { | |||
2974 | SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, | |||
2975 | N->getOperand(0), DAG); | |||
2976 | SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8); | |||
2977 | Results.push_back(T); | |||
2978 | } | |||
2979 | break; | |||
2980 | } | |||
2981 | } | |||
2982 | ||||
2983 | SDValue | |||
2984 | HexagonTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) | |||
2985 | const { | |||
2986 | SDValue Op(N, 0); | |||
2987 | if (isHvxOperation(Op)) { | |||
2988 | if (SDValue V = PerformHvxDAGCombine(N, DCI)) | |||
2989 | return V; | |||
2990 | return SDValue(); | |||
2991 | } | |||
2992 | ||||
2993 | const SDLoc &dl(Op); | |||
2994 | unsigned Opc = Op.getOpcode(); | |||
2995 | ||||
2996 | if (Opc == HexagonISD::P2D) { | |||
2997 | SDValue P = Op.getOperand(0); | |||
2998 | switch (P.getOpcode()) { | |||
2999 | case HexagonISD::PTRUE: | |||
3000 | return DCI.DAG.getConstant(-1, dl, ty(Op)); | |||
3001 | case HexagonISD::PFALSE: | |||
3002 | return getZero(dl, ty(Op), DCI.DAG); | |||
3003 | default: | |||
3004 | break; | |||
3005 | } | |||
3006 | } else if (Opc == ISD::VSELECT) { | |||
3007 | // This is pretty much duplicated in HexagonISelLoweringHVX... | |||
3008 | // | |||
3009 | // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0) | |||
3010 | SDValue Cond = Op.getOperand(0); | |||
3011 | if (Cond->getOpcode() == ISD::XOR) { | |||
3012 | SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1); | |||
3013 | if (C1->getOpcode() == HexagonISD::PTRUE) { | |||
3014 | SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, | |||
3015 | Op.getOperand(2), Op.getOperand(1)); | |||
3016 | return VSel; | |||
3017 | } | |||
3018 | } | |||
3019 | } | |||
3020 | ||||
3021 | return SDValue(); | |||
3022 | } | |||
3023 | ||||
3024 | /// Returns relocation base for the given PIC jumptable. | |||
3025 | SDValue | |||
3026 | HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table, | |||
3027 | SelectionDAG &DAG) const { | |||
3028 | int Idx = cast<JumpTableSDNode>(Table)->getIndex(); | |||
3029 | EVT VT = Table.getValueType(); | |||
3030 | SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL); | |||
3031 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T); | |||
3032 | } | |||
3033 | ||||
3034 | //===----------------------------------------------------------------------===// | |||
3035 | // Inline Assembly Support | |||
3036 | //===----------------------------------------------------------------------===// | |||
3037 | ||||
3038 | TargetLowering::ConstraintType | |||
3039 | HexagonTargetLowering::getConstraintType(StringRef Constraint) const { | |||
3040 | if (Constraint.size() == 1) { | |||
3041 | switch (Constraint[0]) { | |||
3042 | case 'q': | |||
3043 | case 'v': | |||
3044 | if (Subtarget.useHVXOps()) | |||
3045 | return C_RegisterClass; | |||
3046 | break; | |||
3047 | case 'a': | |||
3048 | return C_RegisterClass; | |||
3049 | default: | |||
3050 | break; | |||
3051 | } | |||
3052 | } | |||
3053 | return TargetLowering::getConstraintType(Constraint); | |||
3054 | } | |||
3055 | ||||
3056 | std::pair<unsigned, const TargetRegisterClass*> | |||
3057 | HexagonTargetLowering::getRegForInlineAsmConstraint( | |||
3058 | const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { | |||
3059 | ||||
3060 | if (Constraint.size() == 1) { | |||
3061 | switch (Constraint[0]) { | |||
3062 | case 'r': // R0-R31 | |||
3063 | switch (VT.SimpleTy) { | |||
3064 | default: | |||
3065 | return {0u, nullptr}; | |||
3066 | case MVT::i1: | |||
3067 | case MVT::i8: | |||
3068 | case MVT::i16: | |||
3069 | case MVT::i32: | |||
3070 | case MVT::f32: | |||
3071 | return {0u, &Hexagon::IntRegsRegClass}; | |||
3072 | case MVT::i64: | |||
3073 | case MVT::f64: | |||
3074 | return {0u, &Hexagon::DoubleRegsRegClass}; | |||
3075 | } | |||
3076 | break; | |||
3077 | case 'a': // M0-M1 | |||
3078 | if (VT != MVT::i32) | |||
3079 | return {0u, nullptr}; | |||
3080 | return {0u, &Hexagon::ModRegsRegClass}; | |||
3081 | case 'q': // q0-q3 | |||
3082 | switch (VT.getSizeInBits()) { | |||
3083 | default: | |||
3084 | return {0u, nullptr}; | |||
3085 | case 512: | |||
3086 | case 1024: | |||
3087 | return {0u, &Hexagon::HvxQRRegClass}; | |||
3088 | } | |||
3089 | break; | |||
3090 | case 'v': // V0-V31 | |||
3091 | switch (VT.getSizeInBits()) { | |||
3092 | default: | |||
3093 | return {0u, nullptr}; | |||
3094 | case 512: | |||
3095 | return {0u, &Hexagon::HvxVRRegClass}; | |||
3096 | case 1024: | |||
3097 | if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps()) | |||
3098 | return {0u, &Hexagon::HvxVRRegClass}; | |||
3099 | return {0u, &Hexagon::HvxWRRegClass}; | |||
3100 | case 2048: | |||
3101 | return {0u, &Hexagon::HvxWRRegClass}; | |||
3102 | } | |||
3103 | break; | |||
3104 | default: | |||
3105 | return {0u, nullptr}; | |||
3106 | } | |||
3107 | } | |||
3108 | ||||
3109 | return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); | |||
3110 | } | |||
3111 | ||||
3112 | /// isFPImmLegal - Returns true if the target can instruction select the | |||
3113 | /// specified FP immediate natively. If false, the legalizer will | |||
3114 | /// materialize the FP immediate as a load from a constant pool. | |||
3115 | bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, | |||
3116 | bool ForCodeSize) const { | |||
3117 | return true; | |||
3118 | } | |||
3119 | ||||
3120 | /// isLegalAddressingMode - Return true if the addressing mode represented by | |||
3121 | /// AM is legal for this target, for a load/store of the specified type. | |||
3122 | bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL, | |||
3123 | const AddrMode &AM, Type *Ty, | |||
3124 | unsigned AS, Instruction *I) const { | |||
3125 | if (Ty->isSized()) { | |||
3126 | // When LSR detects uses of the same base address to access different | |||
3127 | // types (e.g. unions), it will assume a conservative type for these | |||
3128 | // uses: | |||
3129 | // LSR Use: Kind=Address of void in addrspace(4294967295), ... | |||
3130 | // The type Ty passed here would then be "void". Skip the alignment | |||
3131 | // checks, but do not return false right away, since that confuses | |||
3132 | // LSR into crashing. | |||
3133 | unsigned A = DL.getABITypeAlignment(Ty); | |||
3134 | // The base offset must be a multiple of the alignment. | |||
3135 | if ((AM.BaseOffs % A) != 0) | |||
3136 | return false; | |||
3137 | // The shifted offset must fit in 11 bits. | |||
3138 | if (!isInt<11>(AM.BaseOffs >> Log2_32(A))) | |||
3139 | return false; | |||
3140 | } | |||
3141 | ||||
3142 | // No global is ever allowed as a base. | |||
3143 | if (AM.BaseGV) | |||
3144 | return false; | |||
3145 | ||||
3146 | int Scale = AM.Scale; | |||
3147 | if (Scale < 0) | |||
3148 | Scale = -Scale; | |||
3149 | switch (Scale) { | |||
3150 | case 0: // No scale reg, "r+i", "r", or just "i". | |||
3151 | break; | |||
3152 | default: // No scaled addressing mode. | |||
3153 | return false; | |||
3154 | } | |||
3155 | return true; | |||
3156 | } | |||
3157 | ||||
3158 | /// Return true if folding a constant offset with the given GlobalAddress is | |||
3159 | /// legal. It is frequently not legal in PIC relocation models. | |||
3160 | bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) | |||
3161 | const { | |||
3162 | return HTM.getRelocationModel() == Reloc::Static; | |||
3163 | } | |||
3164 | ||||
3165 | /// isLegalICmpImmediate - Return true if the specified immediate is legal | |||
3166 | /// icmp immediate, that is the target has icmp instructions which can compare | |||
3167 | /// a register against the immediate without having to materialize the | |||
3168 | /// immediate into a register. | |||
3169 | bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const { | |||
3170 | return Imm >= -512 && Imm <= 511; | |||
3171 | } | |||
3172 | ||||
3173 | /// IsEligibleForTailCallOptimization - Check whether the call is eligible | |||
3174 | /// for tail call optimization. Targets which want to do tail call | |||
3175 | /// optimization should implement this function. | |||
3176 | bool HexagonTargetLowering::IsEligibleForTailCallOptimization( | |||
3177 | SDValue Callee, | |||
3178 | CallingConv::ID CalleeCC, | |||
3179 | bool IsVarArg, | |||
3180 | bool IsCalleeStructRet, | |||
3181 | bool IsCallerStructRet, | |||
3182 | const SmallVectorImpl<ISD::OutputArg> &Outs, | |||
3183 | const SmallVectorImpl<SDValue> &OutVals, | |||
3184 | const SmallVectorImpl<ISD::InputArg> &Ins, | |||
3185 | SelectionDAG& DAG) const { | |||
3186 | const Function &CallerF = DAG.getMachineFunction().getFunction(); | |||
3187 | CallingConv::ID CallerCC = CallerF.getCallingConv(); | |||
3188 | bool CCMatch = CallerCC == CalleeCC; | |||
3189 | ||||
3190 | // *************************************************************************** | |||
3191 | // Look for obvious safe cases to perform tail call optimization that do not | |||
3192 | // require ABI changes. | |||
3193 | // *************************************************************************** | |||
3194 | ||||
3195 | // If this is a tail call via a function pointer, then don't do it! | |||
3196 | if (!isa<GlobalAddressSDNode>(Callee) && | |||
3197 | !isa<ExternalSymbolSDNode>(Callee)) { | |||
3198 | return false; | |||
3199 | } | |||
3200 | ||||
3201 | // Do not optimize if the calling conventions do not match and the conventions | |||
3202 | // used are not C or Fast. | |||
3203 | if (!CCMatch) { | |||
3204 | bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast); | |||
3205 | bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast); | |||
3206 | // If R & E, then ok. | |||
3207 | if (!R || !E) | |||
3208 | return false; | |||
3209 | } | |||
3210 | ||||
3211 | // Do not tail call optimize vararg calls. | |||
3212 | if (IsVarArg) | |||
3213 | return false; | |||
3214 | ||||
3215 | // Also avoid tail call optimization if either caller or callee uses struct | |||
3216 | // return semantics. | |||
3217 | if (IsCalleeStructRet || IsCallerStructRet) | |||
3218 | return false; | |||
3219 | ||||
3220 | // In addition to the cases above, we also disable Tail Call Optimization if | |||
3221 | // the calling convention code that at least one outgoing argument needs to | |||
3222 | // go on the stack. We cannot check that here because at this point that | |||
3223 | // information is not available. | |||
3224 | return true; | |||
3225 | } | |||
3226 | ||||
3227 | /// Returns the target specific optimal type for load and store operations as | |||
3228 | /// a result of memset, memcpy, and memmove lowering. | |||
3229 | /// | |||
3230 | /// If DstAlign is zero that means it's safe to destination alignment can | |||
3231 | /// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't | |||
3232 | /// a need to check it against alignment requirement, probably because the | |||
3233 | /// source does not need to be loaded. If 'IsMemset' is true, that means it's | |||
3234 | /// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of | |||
3235 | /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it | |||
3236 | /// does not need to be loaded. It returns EVT::Other if the type should be | |||
3237 | /// determined using generic target-independent logic. | |||
3238 | EVT HexagonTargetLowering::getOptimalMemOpType(uint64_t Size, | |||
3239 | unsigned DstAlign, unsigned SrcAlign, bool IsMemset, bool ZeroMemset, | |||
3240 | bool MemcpyStrSrc, const AttributeList &FuncAttributes) const { | |||
3241 | ||||
3242 | auto Aligned = [](unsigned GivenA, unsigned MinA) -> bool { | |||
3243 | return (GivenA % MinA) == 0; | |||
3244 | }; | |||
3245 | ||||
3246 | if (Size >= 8 && Aligned(DstAlign, 8) && (IsMemset || Aligned(SrcAlign, 8))) | |||
3247 | return MVT::i64; | |||
3248 | if (Size >= 4 && Aligned(DstAlign, 4) && (IsMemset || Aligned(SrcAlign, 4))) | |||
3249 | return MVT::i32; | |||
3250 | if (Size >= 2 && Aligned(DstAlign, 2) && (IsMemset || Aligned(SrcAlign, 2))) | |||
3251 | return MVT::i16; | |||
3252 | ||||
3253 | return MVT::Other; | |||
3254 | } | |||
3255 | ||||
3256 | bool HexagonTargetLowering::allowsMisalignedMemoryAccesses( | |||
3257 | EVT VT, unsigned AS, unsigned Align, MachineMemOperand::Flags Flags, | |||
3258 | bool *Fast) const { | |||
3259 | if (Fast) | |||
3260 | *Fast = false; | |||
3261 | return Subtarget.isHVXVectorType(VT.getSimpleVT()); | |||
3262 | } | |||
3263 | ||||
3264 | std::pair<const TargetRegisterClass*, uint8_t> | |||
3265 | HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, | |||
3266 | MVT VT) const { | |||
3267 | if (Subtarget.isHVXVectorType(VT, true)) { | |||
3268 | unsigned BitWidth = VT.getSizeInBits(); | |||
3269 | unsigned VecWidth = Subtarget.getVectorLength() * 8; | |||
3270 | ||||
3271 | if (VT.getVectorElementType() == MVT::i1) | |||
3272 | return std::make_pair(&Hexagon::HvxQRRegClass, 1); | |||
3273 | if (BitWidth == VecWidth) | |||
3274 | return std::make_pair(&Hexagon::HvxVRRegClass, 1); | |||
3275 | assert(BitWidth == 2 * VecWidth)((BitWidth == 2 * VecWidth) ? static_cast<void> (0) : __assert_fail ("BitWidth == 2 * VecWidth", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3275, __PRETTY_FUNCTION__)); | |||
3276 | return std::make_pair(&Hexagon::HvxWRRegClass, 1); | |||
3277 | } | |||
3278 | ||||
3279 | return TargetLowering::findRepresentativeClass(TRI, VT); | |||
3280 | } | |||
3281 | ||||
3282 | bool HexagonTargetLowering::shouldReduceLoadWidth(SDNode *Load, | |||
3283 | ISD::LoadExtType ExtTy, EVT NewVT) const { | |||
3284 | // TODO: This may be worth removing. Check regression tests for diffs. | |||
3285 | if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT)) | |||
3286 | return false; | |||
3287 | ||||
3288 | auto *L = cast<LoadSDNode>(Load); | |||
3289 | std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr()); | |||
3290 | // Small-data object, do not shrink. | |||
3291 | if (BO.first.getOpcode() == HexagonISD::CONST32_GP) | |||
3292 | return false; | |||
3293 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) { | |||
3294 | auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine()); | |||
3295 | const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal()); | |||
3296 | return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM); | |||
3297 | } | |||
3298 | return true; | |||
3299 | } | |||
3300 | ||||
3301 | Value *HexagonTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, | |||
3302 | AtomicOrdering Ord) const { | |||
3303 | BasicBlock *BB = Builder.GetInsertBlock(); | |||
3304 | Module *M = BB->getParent()->getParent(); | |||
3305 | auto PT = cast<PointerType>(Addr->getType()); | |||
3306 | Type *Ty = PT->getElementType(); | |||
3307 | unsigned SZ = Ty->getPrimitiveSizeInBits(); | |||
3308 | assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported")(((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported" ) ? static_cast<void> (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic loads supported\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3308, __PRETTY_FUNCTION__)); | |||
3309 | Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked | |||
3310 | : Intrinsic::hexagon_L4_loadd_locked; | |||
3311 | Function *Fn = Intrinsic::getDeclaration(M, IntID); | |||
3312 | ||||
3313 | PointerType *NewPtrTy | |||
3314 | = Builder.getIntNTy(SZ)->getPointerTo(PT->getAddressSpace()); | |||
3315 | Addr = Builder.CreateBitCast(Addr, NewPtrTy); | |||
3316 | ||||
3317 | Value *Call = Builder.CreateCall(Fn, Addr, "larx"); | |||
3318 | ||||
3319 | return Builder.CreateBitCast(Call, Ty); | |||
3320 | } | |||
3321 | ||||
3322 | /// Perform a store-conditional operation to Addr. Return the status of the | |||
3323 | /// store. This should be 0 if the store succeeded, non-zero otherwise. | |||
3324 | Value *HexagonTargetLowering::emitStoreConditional(IRBuilder<> &Builder, | |||
3325 | Value *Val, Value *Addr, AtomicOrdering Ord) const { | |||
3326 | BasicBlock *BB = Builder.GetInsertBlock(); | |||
3327 | Module *M = BB->getParent()->getParent(); | |||
3328 | Type *Ty = Val->getType(); | |||
3329 | unsigned SZ = Ty->getPrimitiveSizeInBits(); | |||
3330 | ||||
3331 | Type *CastTy = Builder.getIntNTy(SZ); | |||
3332 | assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported")(((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported" ) ? static_cast<void> (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic stores supported\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3332, __PRETTY_FUNCTION__)); | |||
3333 | Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked | |||
3334 | : Intrinsic::hexagon_S4_stored_locked; | |||
3335 | Function *Fn = Intrinsic::getDeclaration(M, IntID); | |||
3336 | ||||
3337 | unsigned AS = Addr->getType()->getPointerAddressSpace(); | |||
3338 | Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS)); | |||
3339 | Val = Builder.CreateBitCast(Val, CastTy); | |||
3340 | ||||
3341 | Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx"); | |||
3342 | Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), ""); | |||
3343 | Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext())); | |||
3344 | return Ext; | |||
3345 | } | |||
3346 | ||||
3347 | TargetLowering::AtomicExpansionKind | |||
3348 | HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { | |||
3349 | // Do not expand loads and stores that don't exceed 64 bits. | |||
3350 | return LI->getType()->getPrimitiveSizeInBits() > 64 | |||
3351 | ? AtomicExpansionKind::LLOnly | |||
3352 | : AtomicExpansionKind::None; | |||
3353 | } | |||
3354 | ||||
3355 | bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const { | |||
3356 | // Do not expand loads and stores that don't exceed 64 bits. | |||
3357 | return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64; | |||
3358 | } | |||
3359 | ||||
3360 | TargetLowering::AtomicExpansionKind | |||
3361 | HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR( | |||
3362 | AtomicCmpXchgInst *AI) const { | |||
3363 | const DataLayout &DL = AI->getModule()->getDataLayout(); | |||
3364 | unsigned Size = DL.getTypeStoreSize(AI->getCompareOperand()->getType()); | |||
3365 | if (Size >= 4 && Size <= 8) | |||
3366 | return AtomicExpansionKind::LLSC; | |||
3367 | return AtomicExpansionKind::None; | |||
3368 | } |
1 | //===-- HexagonISelLowering.h - Hexagon DAG Lowering Interface --*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file defines the interfaces that Hexagon uses to lower LLVM code into a |
10 | // selection DAG. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGONISELLOWERING_H |
15 | #define LLVM_LIB_TARGET_HEXAGON_HEXAGONISELLOWERING_H |
16 | |
17 | #include "Hexagon.h" |
18 | #include "llvm/ADT/StringRef.h" |
19 | #include "llvm/CodeGen/ISDOpcodes.h" |
20 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
21 | #include "llvm/CodeGen/TargetLowering.h" |
22 | #include "llvm/CodeGen/ValueTypes.h" |
23 | #include "llvm/IR/CallingConv.h" |
24 | #include "llvm/IR/InlineAsm.h" |
25 | #include "llvm/Support/MachineValueType.h" |
26 | #include <cstdint> |
27 | #include <utility> |
28 | |
29 | namespace llvm { |
30 | |
31 | namespace HexagonISD { |
32 | |
33 | enum NodeType : unsigned { |
34 | OP_BEGIN = ISD::BUILTIN_OP_END, |
35 | |
36 | CONST32 = OP_BEGIN, |
37 | CONST32_GP, // For marking data present in GP. |
38 | ADDC, // Add with carry: (X, Y, Cin) -> (X+Y, Cout). |
39 | SUBC, // Sub with carry: (X, Y, Cin) -> (X+~Y+Cin, Cout). |
40 | ALLOCA, |
41 | |
42 | AT_GOT, // Index in GOT. |
43 | AT_PCREL, // Offset relative to PC. |
44 | |
45 | CALL, // Function call. |
46 | CALLnr, // Function call that does not return. |
47 | CALLR, |
48 | |
49 | RET_FLAG, // Return with a flag operand. |
50 | BARRIER, // Memory barrier. |
51 | JT, // Jump table. |
52 | CP, // Constant pool. |
53 | |
54 | COMBINE, |
55 | VSPLAT, // Generic splat, selection depends on argument/return |
56 | // types. |
57 | VASL, |
58 | VASR, |
59 | VLSR, |
60 | |
61 | TSTBIT, |
62 | INSERT, |
63 | EXTRACTU, |
64 | VEXTRACTW, |
65 | VINSERTW0, |
66 | VROR, |
67 | TC_RETURN, |
68 | EH_RETURN, |
69 | DCFETCH, |
70 | READCYCLE, |
71 | PTRUE, |
72 | PFALSE, |
73 | D2P, // Convert 8-byte value to 8-bit predicate register. [*] |
74 | P2D, // Convert 8-bit predicate register to 8-byte value. [*] |
75 | V2Q, // Convert HVX vector to a vector predicate reg. [*] |
76 | Q2V, // Convert vector predicate to an HVX vector. [*] |
77 | // [*] The equivalence is defined as "Q <=> (V != 0)", |
78 | // where the != operation compares bytes. |
79 | // Note: V != 0 is implemented as V >u 0. |
80 | QCAT, |
81 | QTRUE, |
82 | QFALSE, |
83 | VZERO, |
84 | VSPLATW, // HVX splat of a 32-bit word with an arbitrary result type. |
85 | TYPECAST, // No-op that's used to convert between different legal |
86 | // types in a register. |
87 | VALIGN, // Align two vectors (in Op0, Op1) to one that would have |
88 | // been loaded from address in Op2. |
89 | VALIGNADDR, // Align vector address: Op0 & -Op1, except when it is |
90 | // an address in a vector load, then it's a no-op. |
91 | OP_END |
92 | }; |
93 | |
94 | } // end namespace HexagonISD |
95 | |
96 | class HexagonSubtarget; |
97 | |
98 | class HexagonTargetLowering : public TargetLowering { |
99 | int VarArgsFrameOffset; // Frame offset to start of varargs area. |
100 | const HexagonTargetMachine &HTM; |
101 | const HexagonSubtarget &Subtarget; |
102 | |
103 | bool CanReturnSmallStruct(const Function* CalleeFn, unsigned& RetSize) |
104 | const; |
105 | |
106 | public: |
107 | explicit HexagonTargetLowering(const TargetMachine &TM, |
108 | const HexagonSubtarget &ST); |
109 | |
110 | bool isHVXVectorType(MVT Ty) const; |
111 | |
112 | /// IsEligibleForTailCallOptimization - Check whether the call is eligible |
113 | /// for tail call optimization. Targets which want to do tail call |
114 | /// optimization should implement this function. |
115 | bool IsEligibleForTailCallOptimization(SDValue Callee, |
116 | CallingConv::ID CalleeCC, bool isVarArg, bool isCalleeStructRet, |
117 | bool isCallerStructRet, const SmallVectorImpl<ISD::OutputArg> &Outs, |
118 | const SmallVectorImpl<SDValue> &OutVals, |
119 | const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const; |
120 | |
121 | bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, |
122 | MachineFunction &MF, |
123 | unsigned Intrinsic) const override; |
124 | |
125 | bool isTruncateFree(Type *Ty1, Type *Ty2) const override; |
126 | bool isTruncateFree(EVT VT1, EVT VT2) const override; |
127 | |
128 | bool isCheapToSpeculateCttz() const override { return true; } |
129 | bool isCheapToSpeculateCtlz() const override { return true; } |
130 | bool isCtlzFast() const override { return true; } |
131 | |
132 | bool hasBitTest(SDValue X, SDValue Y) const override; |
133 | |
134 | bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override; |
135 | |
136 | /// Return true if an FMA operation is faster than a pair of mul and add |
137 | /// instructions. fmuladd intrinsics will be expanded to FMAs when this |
138 | /// method returns true (and FMAs are legal), otherwise fmuladd is |
139 | /// expanded to mul + add. |
140 | bool isFMAFasterThanFMulAndFAdd(EVT) const override; |
141 | |
142 | // Should we expand the build vector with shuffles? |
143 | bool shouldExpandBuildVectorWithShuffles(EVT VT, |
144 | unsigned DefinedValues) const override; |
145 | |
146 | bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override; |
147 | TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) |
148 | const override; |
149 | |
150 | SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; |
151 | void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, |
152 | SelectionDAG &DAG) const override; |
153 | void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, |
154 | SelectionDAG &DAG) const override; |
155 | |
156 | const char *getTargetNodeName(unsigned Opcode) const override; |
157 | |
158 | SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const; |
159 | SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; |
160 | SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; |
161 | SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; |
162 | SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; |
163 | SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; |
164 | SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const; |
165 | SDValue LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const; |
166 | SDValue LowerROTL(SDValue Op, SelectionDAG &DAG) const; |
167 | SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const; |
168 | SDValue LowerANY_EXTEND(SDValue Op, SelectionDAG &DAG) const; |
169 | SDValue LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const; |
170 | SDValue LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const; |
171 | SDValue LowerLoad(SDValue Op, SelectionDAG &DAG) const; |
172 | SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const; |
173 | SDValue LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) const; |
174 | SDValue LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const; |
175 | SDValue LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const; |
176 | |
177 | SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const; |
178 | SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const; |
179 | SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) const; |
180 | SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; |
181 | SDValue LowerEH_LABEL(SDValue Op, SelectionDAG &DAG) const; |
182 | SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const; |
183 | SDValue |
184 | LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, |
185 | const SmallVectorImpl<ISD::InputArg> &Ins, |
186 | const SDLoc &dl, SelectionDAG &DAG, |
187 | SmallVectorImpl<SDValue> &InVals) const override; |
188 | SDValue LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const; |
189 | SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; |
190 | SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; |
191 | SDValue LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, |
192 | SelectionDAG &DAG) const; |
193 | SDValue LowerToTLSInitialExecModel(GlobalAddressSDNode *GA, |
194 | SelectionDAG &DAG) const; |
195 | SDValue LowerToTLSLocalExecModel(GlobalAddressSDNode *GA, |
196 | SelectionDAG &DAG) const; |
197 | SDValue GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain, |
198 | GlobalAddressSDNode *GA, SDValue InFlag, EVT PtrVT, |
199 | unsigned ReturnReg, unsigned char OperandFlags) const; |
200 | SDValue LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) const; |
201 | |
202 | SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, |
203 | SmallVectorImpl<SDValue> &InVals) const override; |
204 | SDValue LowerCallResult(SDValue Chain, SDValue InFlag, |
205 | CallingConv::ID CallConv, bool isVarArg, |
206 | const SmallVectorImpl<ISD::InputArg> &Ins, |
207 | const SDLoc &dl, SelectionDAG &DAG, |
208 | SmallVectorImpl<SDValue> &InVals, |
209 | const SmallVectorImpl<SDValue> &OutVals, |
210 | SDValue Callee) const; |
211 | |
212 | SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; |
213 | SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const; |
214 | SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const; |
215 | SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const; |
216 | SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const; |
217 | |
218 | bool CanLowerReturn(CallingConv::ID CallConv, |
219 | MachineFunction &MF, bool isVarArg, |
220 | const SmallVectorImpl<ISD::OutputArg> &Outs, |
221 | LLVMContext &Context) const override; |
222 | |
223 | SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, |
224 | const SmallVectorImpl<ISD::OutputArg> &Outs, |
225 | const SmallVectorImpl<SDValue> &OutVals, |
226 | const SDLoc &dl, SelectionDAG &DAG) const override; |
227 | |
228 | SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; |
229 | |
230 | bool mayBeEmittedAsTailCall(const CallInst *CI) const override; |
231 | |
232 | Register getRegisterByName(const char* RegName, EVT VT, |
233 | const MachineFunction &MF) const override; |
234 | |
235 | /// If a physical register, this returns the register that receives the |
236 | /// exception address on entry to an EH pad. |
237 | unsigned |
238 | getExceptionPointerRegister(const Constant *PersonalityFn) const override { |
239 | return Hexagon::R0; |
240 | } |
241 | |
242 | /// If a physical register, this returns the register that receives the |
243 | /// exception typeid on entry to a landing pad. |
244 | unsigned |
245 | getExceptionSelectorRegister(const Constant *PersonalityFn) const override { |
246 | return Hexagon::R1; |
247 | } |
248 | |
249 | SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const; |
250 | SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; |
251 | SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const; |
252 | |
253 | EVT getSetCCResultType(const DataLayout &, LLVMContext &C, |
254 | EVT VT) const override { |
255 | if (!VT.isVector()) |
256 | return MVT::i1; |
257 | else |
258 | return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); |
259 | } |
260 | |
261 | bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, |
262 | SDValue &Base, SDValue &Offset, |
263 | ISD::MemIndexedMode &AM, |
264 | SelectionDAG &DAG) const override; |
265 | |
266 | ConstraintType getConstraintType(StringRef Constraint) const override; |
267 | |
268 | std::pair<unsigned, const TargetRegisterClass *> |
269 | getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, |
270 | StringRef Constraint, MVT VT) const override; |
271 | |
272 | unsigned |
273 | getInlineAsmMemConstraint(StringRef ConstraintCode) const override { |
274 | if (ConstraintCode == "o") |
275 | return InlineAsm::Constraint_o; |
276 | return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); |
277 | } |
278 | |
279 | // Intrinsics |
280 | SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; |
281 | SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; |
282 | /// isLegalAddressingMode - Return true if the addressing mode represented |
283 | /// by AM is legal for this target, for a load/store of the specified type. |
284 | /// The type may be VoidTy, in which case only return true if the addressing |
285 | /// mode is legal for a load/store of any legal type. |
286 | /// TODO: Handle pre/postinc as well. |
287 | bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, |
288 | Type *Ty, unsigned AS, |
289 | Instruction *I = nullptr) const override; |
290 | /// Return true if folding a constant offset with the given GlobalAddress |
291 | /// is legal. It is frequently not legal in PIC relocation models. |
292 | bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; |
293 | |
294 | bool isFPImmLegal(const APFloat &Imm, EVT VT, |
295 | bool ForCodeSize) const override; |
296 | |
297 | /// isLegalICmpImmediate - Return true if the specified immediate is legal |
298 | /// icmp immediate, that is the target has icmp instructions which can |
299 | /// compare a register against the immediate without having to materialize |
300 | /// the immediate into a register. |
301 | bool isLegalICmpImmediate(int64_t Imm) const override; |
302 | |
303 | EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign, |
304 | unsigned SrcAlign, bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc, |
305 | const AttributeList &FuncAttributes) const override; |
306 | |
307 | bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, |
308 | unsigned Align, MachineMemOperand::Flags Flags, bool *Fast) |
309 | const override; |
310 | |
311 | /// Returns relocation base for the given PIC jumptable. |
312 | SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) |
313 | const override; |
314 | |
315 | bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, |
316 | EVT NewVT) const override; |
317 | |
318 | // Handling of atomic RMW instructions. |
319 | Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr, |
320 | AtomicOrdering Ord) const override; |
321 | Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, |
322 | Value *Addr, AtomicOrdering Ord) const override; |
323 | AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override; |
324 | bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; |
325 | AtomicExpansionKind |
326 | shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; |
327 | |
328 | AtomicExpansionKind |
329 | shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override { |
330 | return AtomicExpansionKind::LLSC; |
331 | } |
332 | |
333 | private: |
334 | void initializeHVXLowering(); |
335 | void validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl, |
336 | unsigned NeedAlign) const; |
337 | |
338 | std::pair<SDValue,int> getBaseAndOffset(SDValue Addr) const; |
339 | |
340 | bool getBuildVectorConstInts(ArrayRef<SDValue> Values, MVT VecTy, |
341 | SelectionDAG &DAG, |
342 | MutableArrayRef<ConstantInt*> Consts) const; |
343 | SDValue buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, |
344 | SelectionDAG &DAG) const; |
345 | SDValue buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, MVT VecTy, |
346 | SelectionDAG &DAG) const; |
347 | SDValue extractVector(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
348 | MVT ValTy, MVT ResTy, SelectionDAG &DAG) const; |
349 | SDValue insertVector(SDValue VecV, SDValue ValV, SDValue IdxV, |
350 | const SDLoc &dl, MVT ValTy, SelectionDAG &DAG) const; |
351 | SDValue expandPredicate(SDValue Vec32, const SDLoc &dl, |
352 | SelectionDAG &DAG) const; |
353 | SDValue contractPredicate(SDValue Vec64, const SDLoc &dl, |
354 | SelectionDAG &DAG) const; |
355 | SDValue getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const; |
356 | |
357 | bool isUndef(SDValue Op) const { |
358 | if (Op.isMachineOpcode()) |
359 | return Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF; |
360 | return Op.getOpcode() == ISD::UNDEF; |
361 | } |
362 | SDValue getInstr(unsigned MachineOpc, const SDLoc &dl, MVT Ty, |
363 | ArrayRef<SDValue> Ops, SelectionDAG &DAG) const { |
364 | SDNode *N = DAG.getMachineNode(MachineOpc, dl, Ty, Ops); |
365 | return SDValue(N, 0); |
366 | } |
367 | SDValue getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) const; |
368 | |
369 | using VectorPair = std::pair<SDValue, SDValue>; |
370 | using TypePair = std::pair<MVT, MVT>; |
371 | |
372 | SDValue getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, |
373 | const SDLoc &dl, SelectionDAG &DAG) const; |
374 | |
375 | MVT ty(SDValue Op) const { |
376 | return Op.getValueType().getSimpleVT(); |
377 | } |
378 | TypePair ty(const VectorPair &Ops) const { |
379 | return { Ops.first.getValueType().getSimpleVT(), |
380 | Ops.second.getValueType().getSimpleVT() }; |
381 | } |
382 | MVT tyScalar(MVT Ty) const { |
383 | if (!Ty.isVector()) |
384 | return Ty; |
385 | return MVT::getIntegerVT(Ty.getSizeInBits()); |
386 | } |
387 | MVT tyVector(MVT Ty, MVT ElemTy) const { |
388 | if (Ty.isVector() && Ty.getVectorElementType() == ElemTy) |
389 | return Ty; |
390 | unsigned TyWidth = Ty.getSizeInBits(); |
391 | unsigned ElemWidth = ElemTy.getSizeInBits(); |
392 | assert((TyWidth % ElemWidth) == 0)(((TyWidth % ElemWidth) == 0) ? static_cast<void> (0) : __assert_fail ("(TyWidth % ElemWidth) == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/Hexagon/HexagonISelLowering.h" , 392, __PRETTY_FUNCTION__)); |
393 | return MVT::getVectorVT(ElemTy, TyWidth/ElemWidth); |
394 | } |
395 | |
396 | MVT typeJoin(const TypePair &Tys) const; |
397 | TypePair typeSplit(MVT Ty) const; |
398 | MVT typeExtElem(MVT VecTy, unsigned Factor) const; |
399 | MVT typeTruncElem(MVT VecTy, unsigned Factor) const; |
400 | |
401 | SDValue opJoin(const VectorPair &Ops, const SDLoc &dl, |
402 | SelectionDAG &DAG) const; |
403 | VectorPair opSplit(SDValue Vec, const SDLoc &dl, SelectionDAG &DAG) const; |
404 | SDValue opCastElem(SDValue Vec, MVT ElemTy, SelectionDAG &DAG) const; |
405 | |
406 | bool isHvxSingleTy(MVT Ty) const; |
407 | bool isHvxPairTy(MVT Ty) const; |
408 | SDValue convertToByteIndex(SDValue ElemIdx, MVT ElemTy, |
409 | SelectionDAG &DAG) const; |
410 | SDValue getIndexInWord32(SDValue Idx, MVT ElemTy, SelectionDAG &DAG) const; |
411 | SDValue getByteShuffle(const SDLoc &dl, SDValue Op0, SDValue Op1, |
412 | ArrayRef<int> Mask, SelectionDAG &DAG) const; |
413 | |
414 | SDValue buildHvxVectorReg(ArrayRef<SDValue> Values, const SDLoc &dl, |
415 | MVT VecTy, SelectionDAG &DAG) const; |
416 | SDValue buildHvxVectorPred(ArrayRef<SDValue> Values, const SDLoc &dl, |
417 | MVT VecTy, SelectionDAG &DAG) const; |
418 | SDValue createHvxPrefixPred(SDValue PredV, const SDLoc &dl, |
419 | unsigned BitBytes, bool ZeroFill, |
420 | SelectionDAG &DAG) const; |
421 | SDValue extractHvxElementReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
422 | MVT ResTy, SelectionDAG &DAG) const; |
423 | SDValue extractHvxElementPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
424 | MVT ResTy, SelectionDAG &DAG) const; |
425 | SDValue insertHvxElementReg(SDValue VecV, SDValue IdxV, SDValue ValV, |
426 | const SDLoc &dl, SelectionDAG &DAG) const; |
427 | SDValue insertHvxElementPred(SDValue VecV, SDValue IdxV, SDValue ValV, |
428 | const SDLoc &dl, SelectionDAG &DAG) const; |
429 | SDValue extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
430 | MVT ResTy, SelectionDAG &DAG) const; |
431 | SDValue extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, const SDLoc &dl, |
432 | MVT ResTy, SelectionDAG &DAG) const; |
433 | SDValue insertHvxSubvectorReg(SDValue VecV, SDValue SubV, SDValue IdxV, |
434 | const SDLoc &dl, SelectionDAG &DAG) const; |
435 | SDValue insertHvxSubvectorPred(SDValue VecV, SDValue SubV, SDValue IdxV, |
436 | const SDLoc &dl, SelectionDAG &DAG) const; |
437 | SDValue extendHvxVectorPred(SDValue VecV, const SDLoc &dl, MVT ResTy, |
438 | bool ZeroExt, SelectionDAG &DAG) const; |
439 | |
440 | SDValue LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) const; |
441 | SDValue LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) const; |
442 | SDValue LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) const; |
443 | SDValue LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) const; |
444 | SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const; |
445 | SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const; |
446 | |
447 | SDValue LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const; |
448 | SDValue LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const; |
449 | SDValue LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const; |
450 | SDValue LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const; |
451 | SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const; |
452 | SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const; |
453 | SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const; |
454 | SDValue LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const; |
455 | SDValue LowerHvxShift(SDValue Op, SelectionDAG &DAG) const; |
456 | |
457 | SDValue SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const; |
458 | SDValue SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const; |
459 | |
460 | std::pair<const TargetRegisterClass*, uint8_t> |
461 | findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) |
462 | const override; |
463 | |
464 | bool isHvxOperation(SDValue Op) const; |
465 | SDValue LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const; |
466 | |
467 | SDValue PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; |
468 | }; |
469 | |
470 | } // end namespace llvm |
471 | |
472 | #endif // LLVM_LIB_TARGET_HEXAGON_HEXAGONISELLOWERING_H |
1 | //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file declares the SDNode class and derived classes, which are used to | |||
10 | // represent the nodes and operations present in a SelectionDAG. These nodes | |||
11 | // and operations are machine code level operations, with some similarities to | |||
12 | // the GCC RTL representation. | |||
13 | // | |||
14 | // Clients should include the SelectionDAG.h file instead of this file directly. | |||
15 | // | |||
16 | //===----------------------------------------------------------------------===// | |||
17 | ||||
18 | #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
19 | #define LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
20 | ||||
21 | #include "llvm/ADT/APFloat.h" | |||
22 | #include "llvm/ADT/ArrayRef.h" | |||
23 | #include "llvm/ADT/BitVector.h" | |||
24 | #include "llvm/ADT/FoldingSet.h" | |||
25 | #include "llvm/ADT/GraphTraits.h" | |||
26 | #include "llvm/ADT/SmallPtrSet.h" | |||
27 | #include "llvm/ADT/SmallVector.h" | |||
28 | #include "llvm/ADT/ilist_node.h" | |||
29 | #include "llvm/ADT/iterator.h" | |||
30 | #include "llvm/ADT/iterator_range.h" | |||
31 | #include "llvm/CodeGen/ISDOpcodes.h" | |||
32 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
33 | #include "llvm/CodeGen/ValueTypes.h" | |||
34 | #include "llvm/IR/Constants.h" | |||
35 | #include "llvm/IR/DebugLoc.h" | |||
36 | #include "llvm/IR/Instruction.h" | |||
37 | #include "llvm/IR/Instructions.h" | |||
38 | #include "llvm/IR/Metadata.h" | |||
39 | #include "llvm/IR/Operator.h" | |||
40 | #include "llvm/Support/AlignOf.h" | |||
41 | #include "llvm/Support/AtomicOrdering.h" | |||
42 | #include "llvm/Support/Casting.h" | |||
43 | #include "llvm/Support/ErrorHandling.h" | |||
44 | #include "llvm/Support/MachineValueType.h" | |||
45 | #include <algorithm> | |||
46 | #include <cassert> | |||
47 | #include <climits> | |||
48 | #include <cstddef> | |||
49 | #include <cstdint> | |||
50 | #include <cstring> | |||
51 | #include <iterator> | |||
52 | #include <string> | |||
53 | #include <tuple> | |||
54 | ||||
55 | namespace llvm { | |||
56 | ||||
57 | class APInt; | |||
58 | class Constant; | |||
59 | template <typename T> struct DenseMapInfo; | |||
60 | class GlobalValue; | |||
61 | class MachineBasicBlock; | |||
62 | class MachineConstantPoolValue; | |||
63 | class MCSymbol; | |||
64 | class raw_ostream; | |||
65 | class SDNode; | |||
66 | class SelectionDAG; | |||
67 | class Type; | |||
68 | class Value; | |||
69 | ||||
70 | void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, | |||
71 | bool force = false); | |||
72 | ||||
73 | /// This represents a list of ValueType's that has been intern'd by | |||
74 | /// a SelectionDAG. Instances of this simple value class are returned by | |||
75 | /// SelectionDAG::getVTList(...). | |||
76 | /// | |||
77 | struct SDVTList { | |||
78 | const EVT *VTs; | |||
79 | unsigned int NumVTs; | |||
80 | }; | |||
81 | ||||
82 | namespace ISD { | |||
83 | ||||
84 | /// Node predicates | |||
85 | ||||
86 | /// If N is a BUILD_VECTOR node whose elements are all the same constant or | |||
87 | /// undefined, return true and return the constant value in \p SplatValue. | |||
88 | bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); | |||
89 | ||||
90 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
91 | /// elements are ~0 or undef. | |||
92 | bool isBuildVectorAllOnes(const SDNode *N); | |||
93 | ||||
94 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
95 | /// elements are 0 or undef. | |||
96 | bool isBuildVectorAllZeros(const SDNode *N); | |||
97 | ||||
98 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
99 | /// ConstantSDNode or undef. | |||
100 | bool isBuildVectorOfConstantSDNodes(const SDNode *N); | |||
101 | ||||
102 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
103 | /// ConstantFPSDNode or undef. | |||
104 | bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); | |||
105 | ||||
106 | /// Return true if the node has at least one operand and all operands of the | |||
107 | /// specified node are ISD::UNDEF. | |||
108 | bool allOperandsUndef(const SDNode *N); | |||
109 | ||||
110 | } // end namespace ISD | |||
111 | ||||
112 | //===----------------------------------------------------------------------===// | |||
113 | /// Unlike LLVM values, Selection DAG nodes may return multiple | |||
114 | /// values as the result of a computation. Many nodes return multiple values, | |||
115 | /// from loads (which define a token and a return value) to ADDC (which returns | |||
116 | /// a result and a carry value), to calls (which may return an arbitrary number | |||
117 | /// of values). | |||
118 | /// | |||
119 | /// As such, each use of a SelectionDAG computation must indicate the node that | |||
120 | /// computes it as well as which return value to use from that node. This pair | |||
121 | /// of information is represented with the SDValue value type. | |||
122 | /// | |||
123 | class SDValue { | |||
124 | friend struct DenseMapInfo<SDValue>; | |||
125 | ||||
126 | SDNode *Node = nullptr; // The node defining the value we are using. | |||
127 | unsigned ResNo = 0; // Which return value of the node we are using. | |||
128 | ||||
129 | public: | |||
130 | SDValue() = default; | |||
131 | SDValue(SDNode *node, unsigned resno); | |||
132 | ||||
133 | /// get the index which selects a specific result in the SDNode | |||
134 | unsigned getResNo() const { return ResNo; } | |||
135 | ||||
136 | /// get the SDNode which holds the desired result | |||
137 | SDNode *getNode() const { return Node; } | |||
138 | ||||
139 | /// set the SDNode | |||
140 | void setNode(SDNode *N) { Node = N; } | |||
141 | ||||
142 | inline SDNode *operator->() const { return Node; } | |||
143 | ||||
144 | bool operator==(const SDValue &O) const { | |||
145 | return Node == O.Node && ResNo == O.ResNo; | |||
146 | } | |||
147 | bool operator!=(const SDValue &O) const { | |||
148 | return !operator==(O); | |||
149 | } | |||
150 | bool operator<(const SDValue &O) const { | |||
151 | return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); | |||
152 | } | |||
153 | explicit operator bool() const { | |||
154 | return Node != nullptr; | |||
155 | } | |||
156 | ||||
157 | SDValue getValue(unsigned R) const { | |||
158 | return SDValue(Node, R); | |||
159 | } | |||
160 | ||||
161 | /// Return true if this node is an operand of N. | |||
162 | bool isOperandOf(const SDNode *N) const; | |||
163 | ||||
164 | /// Return the ValueType of the referenced return value. | |||
165 | inline EVT getValueType() const; | |||
166 | ||||
167 | /// Return the simple ValueType of the referenced return value. | |||
168 | MVT getSimpleValueType() const { | |||
169 | return getValueType().getSimpleVT(); | |||
170 | } | |||
171 | ||||
172 | /// Returns the size of the value in bits. | |||
173 | unsigned getValueSizeInBits() const { | |||
174 | return getValueType().getSizeInBits(); | |||
175 | } | |||
176 | ||||
177 | unsigned getScalarValueSizeInBits() const { | |||
178 | return getValueType().getScalarType().getSizeInBits(); | |||
179 | } | |||
180 | ||||
181 | // Forwarding methods - These forward to the corresponding methods in SDNode. | |||
182 | inline unsigned getOpcode() const; | |||
183 | inline unsigned getNumOperands() const; | |||
184 | inline const SDValue &getOperand(unsigned i) const; | |||
185 | inline uint64_t getConstantOperandVal(unsigned i) const; | |||
186 | inline const APInt &getConstantOperandAPInt(unsigned i) const; | |||
187 | inline bool isTargetMemoryOpcode() const; | |||
188 | inline bool isTargetOpcode() const; | |||
189 | inline bool isMachineOpcode() const; | |||
190 | inline bool isUndef() const; | |||
191 | inline unsigned getMachineOpcode() const; | |||
192 | inline const DebugLoc &getDebugLoc() const; | |||
193 | inline void dump() const; | |||
194 | inline void dump(const SelectionDAG *G) const; | |||
195 | inline void dumpr() const; | |||
196 | inline void dumpr(const SelectionDAG *G) const; | |||
197 | ||||
198 | /// Return true if this operand (which must be a chain) reaches the | |||
199 | /// specified operand without crossing any side-effecting instructions. | |||
200 | /// In practice, this looks through token factors and non-volatile loads. | |||
201 | /// In order to remain efficient, this only | |||
202 | /// looks a couple of nodes in, it does not do an exhaustive search. | |||
203 | bool reachesChainWithoutSideEffects(SDValue Dest, | |||
204 | unsigned Depth = 2) const; | |||
205 | ||||
206 | /// Return true if there are no nodes using value ResNo of Node. | |||
207 | inline bool use_empty() const; | |||
208 | ||||
209 | /// Return true if there is exactly one node using value ResNo of Node. | |||
210 | inline bool hasOneUse() const; | |||
211 | }; | |||
212 | ||||
213 | template<> struct DenseMapInfo<SDValue> { | |||
214 | static inline SDValue getEmptyKey() { | |||
215 | SDValue V; | |||
216 | V.ResNo = -1U; | |||
217 | return V; | |||
218 | } | |||
219 | ||||
220 | static inline SDValue getTombstoneKey() { | |||
221 | SDValue V; | |||
222 | V.ResNo = -2U; | |||
223 | return V; | |||
224 | } | |||
225 | ||||
226 | static unsigned getHashValue(const SDValue &Val) { | |||
227 | return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ | |||
228 | (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo(); | |||
229 | } | |||
230 | ||||
231 | static bool isEqual(const SDValue &LHS, const SDValue &RHS) { | |||
232 | return LHS == RHS; | |||
233 | } | |||
234 | }; | |||
235 | ||||
236 | /// Allow casting operators to work directly on | |||
237 | /// SDValues as if they were SDNode*'s. | |||
238 | template<> struct simplify_type<SDValue> { | |||
239 | using SimpleType = SDNode *; | |||
240 | ||||
241 | static SimpleType getSimplifiedValue(SDValue &Val) { | |||
242 | return Val.getNode(); | |||
243 | } | |||
244 | }; | |||
245 | template<> struct simplify_type<const SDValue> { | |||
246 | using SimpleType = /*const*/ SDNode *; | |||
247 | ||||
248 | static SimpleType getSimplifiedValue(const SDValue &Val) { | |||
249 | return Val.getNode(); | |||
250 | } | |||
251 | }; | |||
252 | ||||
253 | /// Represents a use of a SDNode. This class holds an SDValue, | |||
254 | /// which records the SDNode being used and the result number, a | |||
255 | /// pointer to the SDNode using the value, and Next and Prev pointers, | |||
256 | /// which link together all the uses of an SDNode. | |||
257 | /// | |||
258 | class SDUse { | |||
259 | /// Val - The value being used. | |||
260 | SDValue Val; | |||
261 | /// User - The user of this value. | |||
262 | SDNode *User = nullptr; | |||
263 | /// Prev, Next - Pointers to the uses list of the SDNode referred by | |||
264 | /// this operand. | |||
265 | SDUse **Prev = nullptr; | |||
266 | SDUse *Next = nullptr; | |||
267 | ||||
268 | public: | |||
269 | SDUse() = default; | |||
270 | SDUse(const SDUse &U) = delete; | |||
271 | SDUse &operator=(const SDUse &) = delete; | |||
272 | ||||
273 | /// Normally SDUse will just implicitly convert to an SDValue that it holds. | |||
274 | operator const SDValue&() const { return Val; } | |||
275 | ||||
276 | /// If implicit conversion to SDValue doesn't work, the get() method returns | |||
277 | /// the SDValue. | |||
278 | const SDValue &get() const { return Val; } | |||
279 | ||||
280 | /// This returns the SDNode that contains this Use. | |||
281 | SDNode *getUser() { return User; } | |||
282 | ||||
283 | /// Get the next SDUse in the use list. | |||
284 | SDUse *getNext() const { return Next; } | |||
285 | ||||
286 | /// Convenience function for get().getNode(). | |||
287 | SDNode *getNode() const { return Val.getNode(); } | |||
288 | /// Convenience function for get().getResNo(). | |||
289 | unsigned getResNo() const { return Val.getResNo(); } | |||
290 | /// Convenience function for get().getValueType(). | |||
291 | EVT getValueType() const { return Val.getValueType(); } | |||
292 | ||||
293 | /// Convenience function for get().operator== | |||
294 | bool operator==(const SDValue &V) const { | |||
295 | return Val == V; | |||
296 | } | |||
297 | ||||
298 | /// Convenience function for get().operator!= | |||
299 | bool operator!=(const SDValue &V) const { | |||
300 | return Val != V; | |||
301 | } | |||
302 | ||||
303 | /// Convenience function for get().operator< | |||
304 | bool operator<(const SDValue &V) const { | |||
305 | return Val < V; | |||
306 | } | |||
307 | ||||
308 | private: | |||
309 | friend class SelectionDAG; | |||
310 | friend class SDNode; | |||
311 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
312 | friend class HandleSDNode; | |||
313 | ||||
314 | void setUser(SDNode *p) { User = p; } | |||
315 | ||||
316 | /// Remove this use from its existing use list, assign it the | |||
317 | /// given value, and add it to the new value's node's use list. | |||
318 | inline void set(const SDValue &V); | |||
319 | /// Like set, but only supports initializing a newly-allocated | |||
320 | /// SDUse with a non-null value. | |||
321 | inline void setInitial(const SDValue &V); | |||
322 | /// Like set, but only sets the Node portion of the value, | |||
323 | /// leaving the ResNo portion unmodified. | |||
324 | inline void setNode(SDNode *N); | |||
325 | ||||
326 | void addToList(SDUse **List) { | |||
327 | Next = *List; | |||
328 | if (Next) Next->Prev = &Next; | |||
329 | Prev = List; | |||
330 | *List = this; | |||
331 | } | |||
332 | ||||
333 | void removeFromList() { | |||
334 | *Prev = Next; | |||
335 | if (Next) Next->Prev = Prev; | |||
336 | } | |||
337 | }; | |||
338 | ||||
339 | /// simplify_type specializations - Allow casting operators to work directly on | |||
340 | /// SDValues as if they were SDNode*'s. | |||
341 | template<> struct simplify_type<SDUse> { | |||
342 | using SimpleType = SDNode *; | |||
343 | ||||
344 | static SimpleType getSimplifiedValue(SDUse &Val) { | |||
345 | return Val.getNode(); | |||
346 | } | |||
347 | }; | |||
348 | ||||
349 | /// These are IR-level optimization flags that may be propagated to SDNodes. | |||
350 | /// TODO: This data structure should be shared by the IR optimizer and the | |||
351 | /// the backend. | |||
352 | struct SDNodeFlags { | |||
353 | private: | |||
354 | // This bit is used to determine if the flags are in a defined state. | |||
355 | // Flag bits can only be masked out during intersection if the masking flags | |||
356 | // are defined. | |||
357 | bool AnyDefined : 1; | |||
358 | ||||
359 | bool NoUnsignedWrap : 1; | |||
360 | bool NoSignedWrap : 1; | |||
361 | bool Exact : 1; | |||
362 | bool NoNaNs : 1; | |||
363 | bool NoInfs : 1; | |||
364 | bool NoSignedZeros : 1; | |||
365 | bool AllowReciprocal : 1; | |||
366 | bool VectorReduction : 1; | |||
367 | bool AllowContract : 1; | |||
368 | bool ApproximateFuncs : 1; | |||
369 | bool AllowReassociation : 1; | |||
370 | ||||
371 | // We assume instructions do not raise floating-point exceptions by default, | |||
372 | // and only those marked explicitly may do so. We could choose to represent | |||
373 | // this via a positive "FPExcept" flags like on the MI level, but having a | |||
374 | // negative "NoFPExcept" flag here (that defaults to true) makes the flag | |||
375 | // intersection logic more straightforward. | |||
376 | bool NoFPExcept : 1; | |||
377 | ||||
378 | public: | |||
379 | /// Default constructor turns off all optimization flags. | |||
380 | SDNodeFlags() | |||
381 | : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false), | |||
382 | Exact(false), NoNaNs(false), NoInfs(false), | |||
383 | NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false), | |||
384 | AllowContract(false), ApproximateFuncs(false), | |||
385 | AllowReassociation(false), NoFPExcept(true) {} | |||
386 | ||||
387 | /// Propagate the fast-math-flags from an IR FPMathOperator. | |||
388 | void copyFMF(const FPMathOperator &FPMO) { | |||
389 | setNoNaNs(FPMO.hasNoNaNs()); | |||
390 | setNoInfs(FPMO.hasNoInfs()); | |||
391 | setNoSignedZeros(FPMO.hasNoSignedZeros()); | |||
392 | setAllowReciprocal(FPMO.hasAllowReciprocal()); | |||
393 | setAllowContract(FPMO.hasAllowContract()); | |||
394 | setApproximateFuncs(FPMO.hasApproxFunc()); | |||
395 | setAllowReassociation(FPMO.hasAllowReassoc()); | |||
396 | } | |||
397 | ||||
398 | /// Sets the state of the flags to the defined state. | |||
399 | void setDefined() { AnyDefined = true; } | |||
400 | /// Returns true if the flags are in a defined state. | |||
401 | bool isDefined() const { return AnyDefined; } | |||
402 | ||||
403 | // These are mutators for each flag. | |||
404 | void setNoUnsignedWrap(bool b) { | |||
405 | setDefined(); | |||
406 | NoUnsignedWrap = b; | |||
407 | } | |||
408 | void setNoSignedWrap(bool b) { | |||
409 | setDefined(); | |||
410 | NoSignedWrap = b; | |||
411 | } | |||
412 | void setExact(bool b) { | |||
413 | setDefined(); | |||
414 | Exact = b; | |||
415 | } | |||
416 | void setNoNaNs(bool b) { | |||
417 | setDefined(); | |||
418 | NoNaNs = b; | |||
419 | } | |||
420 | void setNoInfs(bool b) { | |||
421 | setDefined(); | |||
422 | NoInfs = b; | |||
423 | } | |||
424 | void setNoSignedZeros(bool b) { | |||
425 | setDefined(); | |||
426 | NoSignedZeros = b; | |||
427 | } | |||
428 | void setAllowReciprocal(bool b) { | |||
429 | setDefined(); | |||
430 | AllowReciprocal = b; | |||
431 | } | |||
432 | void setVectorReduction(bool b) { | |||
433 | setDefined(); | |||
434 | VectorReduction = b; | |||
435 | } | |||
436 | void setAllowContract(bool b) { | |||
437 | setDefined(); | |||
438 | AllowContract = b; | |||
439 | } | |||
440 | void setApproximateFuncs(bool b) { | |||
441 | setDefined(); | |||
442 | ApproximateFuncs = b; | |||
443 | } | |||
444 | void setAllowReassociation(bool b) { | |||
445 | setDefined(); | |||
446 | AllowReassociation = b; | |||
447 | } | |||
448 | void setFPExcept(bool b) { | |||
449 | setDefined(); | |||
450 | NoFPExcept = !b; | |||
451 | } | |||
452 | ||||
453 | // These are accessors for each flag. | |||
454 | bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } | |||
455 | bool hasNoSignedWrap() const { return NoSignedWrap; } | |||
456 | bool hasExact() const { return Exact; } | |||
457 | bool hasNoNaNs() const { return NoNaNs; } | |||
458 | bool hasNoInfs() const { return NoInfs; } | |||
459 | bool hasNoSignedZeros() const { return NoSignedZeros; } | |||
460 | bool hasAllowReciprocal() const { return AllowReciprocal; } | |||
461 | bool hasVectorReduction() const { return VectorReduction; } | |||
462 | bool hasAllowContract() const { return AllowContract; } | |||
463 | bool hasApproximateFuncs() const { return ApproximateFuncs; } | |||
464 | bool hasAllowReassociation() const { return AllowReassociation; } | |||
465 | bool hasFPExcept() const { return !NoFPExcept; } | |||
466 | ||||
467 | bool isFast() const { | |||
468 | return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept && | |||
469 | AllowContract && ApproximateFuncs && AllowReassociation; | |||
470 | } | |||
471 | ||||
472 | /// Clear any flags in this flag set that aren't also set in Flags. | |||
473 | /// If the given Flags are undefined then don't do anything. | |||
474 | void intersectWith(const SDNodeFlags Flags) { | |||
475 | if (!Flags.isDefined()) | |||
476 | return; | |||
477 | NoUnsignedWrap &= Flags.NoUnsignedWrap; | |||
478 | NoSignedWrap &= Flags.NoSignedWrap; | |||
479 | Exact &= Flags.Exact; | |||
480 | NoNaNs &= Flags.NoNaNs; | |||
481 | NoInfs &= Flags.NoInfs; | |||
482 | NoSignedZeros &= Flags.NoSignedZeros; | |||
483 | AllowReciprocal &= Flags.AllowReciprocal; | |||
484 | VectorReduction &= Flags.VectorReduction; | |||
485 | AllowContract &= Flags.AllowContract; | |||
486 | ApproximateFuncs &= Flags.ApproximateFuncs; | |||
487 | AllowReassociation &= Flags.AllowReassociation; | |||
488 | NoFPExcept &= Flags.NoFPExcept; | |||
489 | } | |||
490 | }; | |||
491 | ||||
492 | /// Represents one node in the SelectionDAG. | |||
493 | /// | |||
494 | class SDNode : public FoldingSetNode, public ilist_node<SDNode> { | |||
495 | private: | |||
496 | /// The operation that this node performs. | |||
497 | int16_t NodeType; | |||
498 | ||||
499 | protected: | |||
500 | // We define a set of mini-helper classes to help us interpret the bits in our | |||
501 | // SubclassData. These are designed to fit within a uint16_t so they pack | |||
502 | // with NodeType. | |||
503 | ||||
504 | #if defined(_AIX) && (!defined(__GNUC__4) || defined(__ibmxl__)) | |||
505 | // Except for GCC; by default, AIX compilers store bit-fields in 4-byte words | |||
506 | // and give the `pack` pragma push semantics. | |||
507 | #define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2) | |||
508 | #define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop) | |||
509 | #else | |||
510 | #define BEGIN_TWO_BYTE_PACK() | |||
511 | #define END_TWO_BYTE_PACK() | |||
512 | #endif | |||
513 | ||||
514 | BEGIN_TWO_BYTE_PACK() | |||
515 | class SDNodeBitfields { | |||
516 | friend class SDNode; | |||
517 | friend class MemIntrinsicSDNode; | |||
518 | friend class MemSDNode; | |||
519 | friend class SelectionDAG; | |||
520 | ||||
521 | uint16_t HasDebugValue : 1; | |||
522 | uint16_t IsMemIntrinsic : 1; | |||
523 | uint16_t IsDivergent : 1; | |||
524 | }; | |||
525 | enum { NumSDNodeBits = 3 }; | |||
526 | ||||
527 | class ConstantSDNodeBitfields { | |||
528 | friend class ConstantSDNode; | |||
529 | ||||
530 | uint16_t : NumSDNodeBits; | |||
531 | ||||
532 | uint16_t IsOpaque : 1; | |||
533 | }; | |||
534 | ||||
535 | class MemSDNodeBitfields { | |||
536 | friend class MemSDNode; | |||
537 | friend class MemIntrinsicSDNode; | |||
538 | friend class AtomicSDNode; | |||
539 | ||||
540 | uint16_t : NumSDNodeBits; | |||
541 | ||||
542 | uint16_t IsVolatile : 1; | |||
543 | uint16_t IsNonTemporal : 1; | |||
544 | uint16_t IsDereferenceable : 1; | |||
545 | uint16_t IsInvariant : 1; | |||
546 | }; | |||
547 | enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; | |||
548 | ||||
549 | class LSBaseSDNodeBitfields { | |||
550 | friend class LSBaseSDNode; | |||
551 | friend class MaskedGatherScatterSDNode; | |||
552 | ||||
553 | uint16_t : NumMemSDNodeBits; | |||
554 | ||||
555 | // This storage is shared between disparate class hierarchies to hold an | |||
556 | // enumeration specific to the class hierarchy in use. | |||
557 | // LSBaseSDNode => enum ISD::MemIndexedMode | |||
558 | // MaskedGatherScatterSDNode => enum ISD::MemIndexType | |||
559 | uint16_t AddressingMode : 3; | |||
560 | }; | |||
561 | enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; | |||
562 | ||||
563 | class LoadSDNodeBitfields { | |||
564 | friend class LoadSDNode; | |||
565 | friend class MaskedLoadSDNode; | |||
566 | ||||
567 | uint16_t : NumLSBaseSDNodeBits; | |||
568 | ||||
569 | uint16_t ExtTy : 2; // enum ISD::LoadExtType | |||
570 | uint16_t IsExpanding : 1; | |||
571 | }; | |||
572 | ||||
573 | class StoreSDNodeBitfields { | |||
574 | friend class StoreSDNode; | |||
575 | friend class MaskedStoreSDNode; | |||
576 | ||||
577 | uint16_t : NumLSBaseSDNodeBits; | |||
578 | ||||
579 | uint16_t IsTruncating : 1; | |||
580 | uint16_t IsCompressing : 1; | |||
581 | }; | |||
582 | ||||
583 | union { | |||
584 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
585 | SDNodeBitfields SDNodeBits; | |||
586 | ConstantSDNodeBitfields ConstantSDNodeBits; | |||
587 | MemSDNodeBitfields MemSDNodeBits; | |||
588 | LSBaseSDNodeBitfields LSBaseSDNodeBits; | |||
589 | LoadSDNodeBitfields LoadSDNodeBits; | |||
590 | StoreSDNodeBitfields StoreSDNodeBits; | |||
591 | }; | |||
592 | END_TWO_BYTE_PACK() | |||
593 | #undef BEGIN_TWO_BYTE_PACK | |||
594 | #undef END_TWO_BYTE_PACK | |||
595 | ||||
596 | // RawSDNodeBits must cover the entirety of the union. This means that all of | |||
597 | // the union's members must have size <= RawSDNodeBits. We write the RHS as | |||
598 | // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter. | |||
599 | static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide"); | |||
600 | static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide"); | |||
601 | static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide"); | |||
602 | static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide"); | |||
603 | static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide"); | |||
604 | static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide"); | |||
605 | ||||
606 | private: | |||
607 | friend class SelectionDAG; | |||
608 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
609 | friend class HandleSDNode; | |||
610 | ||||
611 | /// Unique id per SDNode in the DAG. | |||
612 | int NodeId = -1; | |||
613 | ||||
614 | /// The values that are used by this operation. | |||
615 | SDUse *OperandList = nullptr; | |||
616 | ||||
617 | /// The types of the values this node defines. SDNode's may | |||
618 | /// define multiple values simultaneously. | |||
619 | const EVT *ValueList; | |||
620 | ||||
621 | /// List of uses for this SDNode. | |||
622 | SDUse *UseList = nullptr; | |||
623 | ||||
624 | /// The number of entries in the Operand/Value list. | |||
625 | unsigned short NumOperands = 0; | |||
626 | unsigned short NumValues; | |||
627 | ||||
628 | // The ordering of the SDNodes. It roughly corresponds to the ordering of the | |||
629 | // original LLVM instructions. | |||
630 | // This is used for turning off scheduling, because we'll forgo | |||
631 | // the normal scheduling algorithms and output the instructions according to | |||
632 | // this ordering. | |||
633 | unsigned IROrder; | |||
634 | ||||
635 | /// Source line information. | |||
636 | DebugLoc debugLoc; | |||
637 | ||||
638 | /// Return a pointer to the specified value type. | |||
639 | static const EVT *getValueTypeList(EVT VT); | |||
640 | ||||
641 | SDNodeFlags Flags; | |||
642 | ||||
643 | public: | |||
644 | /// Unique and persistent id per SDNode in the DAG. | |||
645 | /// Used for debug printing. | |||
646 | uint16_t PersistentId; | |||
647 | ||||
648 | //===--------------------------------------------------------------------===// | |||
649 | // Accessors | |||
650 | // | |||
651 | ||||
652 | /// Return the SelectionDAG opcode value for this node. For | |||
653 | /// pre-isel nodes (those for which isMachineOpcode returns false), these | |||
654 | /// are the opcode values in the ISD and <target>ISD namespaces. For | |||
655 | /// post-isel opcodes, see getMachineOpcode. | |||
656 | unsigned getOpcode() const { return (unsigned short)NodeType; } | |||
657 | ||||
658 | /// Test if this node has a target-specific opcode (in the | |||
659 | /// \<target\>ISD namespace). | |||
660 | bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } | |||
661 | ||||
662 | /// Test if this node has a target-specific | |||
663 | /// memory-referencing opcode (in the \<target\>ISD namespace and | |||
664 | /// greater than FIRST_TARGET_MEMORY_OPCODE). | |||
665 | bool isTargetMemoryOpcode() const { | |||
666 | return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; | |||
667 | } | |||
668 | ||||
669 | /// Return true if the type of the node type undefined. | |||
670 | bool isUndef() const { return NodeType == ISD::UNDEF; } | |||
671 | ||||
672 | /// Test if this node is a memory intrinsic (with valid pointer information). | |||
673 | /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for | |||
674 | /// non-memory intrinsics (with chains) that are not really instances of | |||
675 | /// MemSDNode. For such nodes, we need some extra state to determine the | |||
676 | /// proper classof relationship. | |||
677 | bool isMemIntrinsic() const { | |||
678 | return (NodeType == ISD::INTRINSIC_W_CHAIN || | |||
679 | NodeType == ISD::INTRINSIC_VOID) && | |||
680 | SDNodeBits.IsMemIntrinsic; | |||
681 | } | |||
682 | ||||
683 | /// Test if this node is a strict floating point pseudo-op. | |||
684 | bool isStrictFPOpcode() { | |||
685 | switch (NodeType) { | |||
686 | default: | |||
687 | return false; | |||
688 | case ISD::STRICT_FADD: | |||
689 | case ISD::STRICT_FSUB: | |||
690 | case ISD::STRICT_FMUL: | |||
691 | case ISD::STRICT_FDIV: | |||
692 | case ISD::STRICT_FREM: | |||
693 | case ISD::STRICT_FMA: | |||
694 | case ISD::STRICT_FSQRT: | |||
695 | case ISD::STRICT_FPOW: | |||
696 | case ISD::STRICT_FPOWI: | |||
697 | case ISD::STRICT_FSIN: | |||
698 | case ISD::STRICT_FCOS: | |||
699 | case ISD::STRICT_FEXP: | |||
700 | case ISD::STRICT_FEXP2: | |||
701 | case ISD::STRICT_FLOG: | |||
702 | case ISD::STRICT_FLOG10: | |||
703 | case ISD::STRICT_FLOG2: | |||
704 | case ISD::STRICT_LRINT: | |||
705 | case ISD::STRICT_LLRINT: | |||
706 | case ISD::STRICT_FRINT: | |||
707 | case ISD::STRICT_FNEARBYINT: | |||
708 | case ISD::STRICT_FMAXNUM: | |||
709 | case ISD::STRICT_FMINNUM: | |||
710 | case ISD::STRICT_FCEIL: | |||
711 | case ISD::STRICT_FFLOOR: | |||
712 | case ISD::STRICT_LROUND: | |||
713 | case ISD::STRICT_LLROUND: | |||
714 | case ISD::STRICT_FROUND: | |||
715 | case ISD::STRICT_FTRUNC: | |||
716 | case ISD::STRICT_FP_TO_SINT: | |||
717 | case ISD::STRICT_FP_TO_UINT: | |||
718 | case ISD::STRICT_FP_ROUND: | |||
719 | case ISD::STRICT_FP_EXTEND: | |||
720 | return true; | |||
721 | } | |||
722 | } | |||
723 | ||||
724 | /// Test if this node has a post-isel opcode, directly | |||
725 | /// corresponding to a MachineInstr opcode. | |||
726 | bool isMachineOpcode() const { return NodeType < 0; } | |||
727 | ||||
728 | /// This may only be called if isMachineOpcode returns | |||
729 | /// true. It returns the MachineInstr opcode value that the node's opcode | |||
730 | /// corresponds to. | |||
731 | unsigned getMachineOpcode() const { | |||
732 | assert(isMachineOpcode() && "Not a MachineInstr opcode!")((isMachineOpcode() && "Not a MachineInstr opcode!") ? static_cast<void> (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 732, __PRETTY_FUNCTION__)); | |||
733 | return ~NodeType; | |||
734 | } | |||
735 | ||||
736 | bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } | |||
737 | void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } | |||
738 | ||||
739 | bool isDivergent() const { return SDNodeBits.IsDivergent; } | |||
740 | ||||
741 | /// Return true if there are no uses of this node. | |||
742 | bool use_empty() const { return UseList == nullptr; } | |||
743 | ||||
744 | /// Return true if there is exactly one use of this node. | |||
745 | bool hasOneUse() const { | |||
746 | return !use_empty() && std::next(use_begin()) == use_end(); | |||
747 | } | |||
748 | ||||
749 | /// Return the number of uses of this node. This method takes | |||
750 | /// time proportional to the number of uses. | |||
751 | size_t use_size() const { return std::distance(use_begin(), use_end()); } | |||
752 | ||||
753 | /// Return the unique node id. | |||
754 | int getNodeId() const { return NodeId; } | |||
755 | ||||
756 | /// Set unique node id. | |||
757 | void setNodeId(int Id) { NodeId = Id; } | |||
758 | ||||
759 | /// Return the node ordering. | |||
760 | unsigned getIROrder() const { return IROrder; } | |||
761 | ||||
762 | /// Set the node ordering. | |||
763 | void setIROrder(unsigned Order) { IROrder = Order; } | |||
764 | ||||
765 | /// Return the source location info. | |||
766 | const DebugLoc &getDebugLoc() const { return debugLoc; } | |||
767 | ||||
768 | /// Set source location info. Try to avoid this, putting | |||
769 | /// it in the constructor is preferable. | |||
770 | void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } | |||
771 | ||||
772 | /// This class provides iterator support for SDUse | |||
773 | /// operands that use a specific SDNode. | |||
774 | class use_iterator | |||
775 | : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> { | |||
776 | friend class SDNode; | |||
777 | ||||
778 | SDUse *Op = nullptr; | |||
779 | ||||
780 | explicit use_iterator(SDUse *op) : Op(op) {} | |||
781 | ||||
782 | public: | |||
783 | using reference = std::iterator<std::forward_iterator_tag, | |||
784 | SDUse, ptrdiff_t>::reference; | |||
785 | using pointer = std::iterator<std::forward_iterator_tag, | |||
786 | SDUse, ptrdiff_t>::pointer; | |||
787 | ||||
788 | use_iterator() = default; | |||
789 | use_iterator(const use_iterator &I) : Op(I.Op) {} | |||
790 | ||||
791 | bool operator==(const use_iterator &x) const { | |||
792 | return Op == x.Op; | |||
793 | } | |||
794 | bool operator!=(const use_iterator &x) const { | |||
795 | return !operator==(x); | |||
796 | } | |||
797 | ||||
798 | /// Return true if this iterator is at the end of uses list. | |||
799 | bool atEnd() const { return Op == nullptr; } | |||
800 | ||||
801 | // Iterator traversal: forward iteration only. | |||
802 | use_iterator &operator++() { // Preincrement | |||
803 | assert(Op && "Cannot increment end iterator!")((Op && "Cannot increment end iterator!") ? static_cast <void> (0) : __assert_fail ("Op && \"Cannot increment end iterator!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 803, __PRETTY_FUNCTION__)); | |||
804 | Op = Op->getNext(); | |||
805 | return *this; | |||
806 | } | |||
807 | ||||
808 | use_iterator operator++(int) { // Postincrement | |||
809 | use_iterator tmp = *this; ++*this; return tmp; | |||
810 | } | |||
811 | ||||
812 | /// Retrieve a pointer to the current user node. | |||
813 | SDNode *operator*() const { | |||
814 | assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast <void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 814, __PRETTY_FUNCTION__)); | |||
815 | return Op->getUser(); | |||
816 | } | |||
817 | ||||
818 | SDNode *operator->() const { return operator*(); } | |||
819 | ||||
820 | SDUse &getUse() const { return *Op; } | |||
821 | ||||
822 | /// Retrieve the operand # of this use in its user. | |||
823 | unsigned getOperandNo() const { | |||
824 | assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast <void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 824, __PRETTY_FUNCTION__)); | |||
825 | return (unsigned)(Op - Op->getUser()->OperandList); | |||
826 | } | |||
827 | }; | |||
828 | ||||
829 | /// Provide iteration support to walk over all uses of an SDNode. | |||
830 | use_iterator use_begin() const { | |||
831 | return use_iterator(UseList); | |||
832 | } | |||
833 | ||||
834 | static use_iterator use_end() { return use_iterator(nullptr); } | |||
835 | ||||
836 | inline iterator_range<use_iterator> uses() { | |||
837 | return make_range(use_begin(), use_end()); | |||
838 | } | |||
839 | inline iterator_range<use_iterator> uses() const { | |||
840 | return make_range(use_begin(), use_end()); | |||
841 | } | |||
842 | ||||
843 | /// Return true if there are exactly NUSES uses of the indicated value. | |||
844 | /// This method ignores uses of other values defined by this operation. | |||
845 | bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; | |||
846 | ||||
847 | /// Return true if there are any use of the indicated value. | |||
848 | /// This method ignores uses of other values defined by this operation. | |||
849 | bool hasAnyUseOfValue(unsigned Value) const; | |||
850 | ||||
851 | /// Return true if this node is the only use of N. | |||
852 | bool isOnlyUserOf(const SDNode *N) const; | |||
853 | ||||
854 | /// Return true if this node is an operand of N. | |||
855 | bool isOperandOf(const SDNode *N) const; | |||
856 | ||||
857 | /// Return true if this node is a predecessor of N. | |||
858 | /// NOTE: Implemented on top of hasPredecessor and every bit as | |||
859 | /// expensive. Use carefully. | |||
860 | bool isPredecessorOf(const SDNode *N) const { | |||
861 | return N->hasPredecessor(this); | |||
862 | } | |||
863 | ||||
864 | /// Return true if N is a predecessor of this node. | |||
865 | /// N is either an operand of this node, or can be reached by recursively | |||
866 | /// traversing up the operands. | |||
867 | /// NOTE: This is an expensive method. Use it carefully. | |||
868 | bool hasPredecessor(const SDNode *N) const; | |||
869 | ||||
870 | /// Returns true if N is a predecessor of any node in Worklist. This | |||
871 | /// helper keeps Visited and Worklist sets externally to allow unions | |||
872 | /// searches to be performed in parallel, caching of results across | |||
873 | /// queries and incremental addition to Worklist. Stops early if N is | |||
874 | /// found but will resume. Remember to clear Visited and Worklists | |||
875 | /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before | |||
876 | /// giving up. The TopologicalPrune flag signals that positive NodeIds are | |||
877 | /// topologically ordered (Operands have strictly smaller node id) and search | |||
878 | /// can be pruned leveraging this. | |||
879 | static bool hasPredecessorHelper(const SDNode *N, | |||
880 | SmallPtrSetImpl<const SDNode *> &Visited, | |||
881 | SmallVectorImpl<const SDNode *> &Worklist, | |||
882 | unsigned int MaxSteps = 0, | |||
883 | bool TopologicalPrune = false) { | |||
884 | SmallVector<const SDNode *, 8> DeferredNodes; | |||
885 | if (Visited.count(N)) | |||
886 | return true; | |||
887 | ||||
888 | // Node Id's are assigned in three places: As a topological | |||
889 | // ordering (> 0), during legalization (results in values set to | |||
890 | // 0), new nodes (set to -1). If N has a topolgical id then we | |||
891 | // know that all nodes with ids smaller than it cannot be | |||
892 | // successors and we need not check them. Filter out all node | |||
893 | // that can't be matches. We add them to the worklist before exit | |||
894 | // in case of multiple calls. Note that during selection the topological id | |||
895 | // may be violated if a node's predecessor is selected before it. We mark | |||
896 | // this at selection negating the id of unselected successors and | |||
897 | // restricting topological pruning to positive ids. | |||
898 | ||||
899 | int NId = N->getNodeId(); | |||
900 | // If we Invalidated the Id, reconstruct original NId. | |||
901 | if (NId < -1) | |||
902 | NId = -(NId + 1); | |||
903 | ||||
904 | bool Found = false; | |||
905 | while (!Worklist.empty()) { | |||
906 | const SDNode *M = Worklist.pop_back_val(); | |||
907 | int MId = M->getNodeId(); | |||
908 | if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) && | |||
909 | (MId > 0) && (MId < NId)) { | |||
910 | DeferredNodes.push_back(M); | |||
911 | continue; | |||
912 | } | |||
913 | for (const SDValue &OpV : M->op_values()) { | |||
914 | SDNode *Op = OpV.getNode(); | |||
915 | if (Visited.insert(Op).second) | |||
916 | Worklist.push_back(Op); | |||
917 | if (Op == N) | |||
918 | Found = true; | |||
919 | } | |||
920 | if (Found) | |||
921 | break; | |||
922 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
923 | break; | |||
924 | } | |||
925 | // Push deferred nodes back on worklist. | |||
926 | Worklist.append(DeferredNodes.begin(), DeferredNodes.end()); | |||
927 | // If we bailed early, conservatively return found. | |||
928 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
929 | return true; | |||
930 | return Found; | |||
931 | } | |||
932 | ||||
933 | /// Return true if all the users of N are contained in Nodes. | |||
934 | /// NOTE: Requires at least one match, but doesn't require them all. | |||
935 | static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N); | |||
936 | ||||
937 | /// Return the number of values used by this operation. | |||
938 | unsigned getNumOperands() const { return NumOperands; } | |||
939 | ||||
940 | /// Return the maximum number of operands that a SDNode can hold. | |||
941 | static constexpr size_t getMaxNumOperands() { | |||
942 | return std::numeric_limits<decltype(SDNode::NumOperands)>::max(); | |||
943 | } | |||
944 | ||||
945 | /// Helper method returns the integer value of a ConstantSDNode operand. | |||
946 | inline uint64_t getConstantOperandVal(unsigned Num) const; | |||
947 | ||||
948 | /// Helper method returns the APInt of a ConstantSDNode operand. | |||
949 | inline const APInt &getConstantOperandAPInt(unsigned Num) const; | |||
950 | ||||
951 | const SDValue &getOperand(unsigned Num) const { | |||
952 | assert(Num < NumOperands && "Invalid child # of SDNode!")((Num < NumOperands && "Invalid child # of SDNode!" ) ? static_cast<void> (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 952, __PRETTY_FUNCTION__)); | |||
953 | return OperandList[Num]; | |||
954 | } | |||
955 | ||||
956 | using op_iterator = SDUse *; | |||
957 | ||||
958 | op_iterator op_begin() const { return OperandList; } | |||
959 | op_iterator op_end() const { return OperandList+NumOperands; } | |||
960 | ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); } | |||
961 | ||||
962 | /// Iterator for directly iterating over the operand SDValue's. | |||
963 | struct value_op_iterator | |||
964 | : iterator_adaptor_base<value_op_iterator, op_iterator, | |||
965 | std::random_access_iterator_tag, SDValue, | |||
966 | ptrdiff_t, value_op_iterator *, | |||
967 | value_op_iterator *> { | |||
968 | explicit value_op_iterator(SDUse *U = nullptr) | |||
969 | : iterator_adaptor_base(U) {} | |||
970 | ||||
971 | const SDValue &operator*() const { return I->get(); } | |||
972 | }; | |||
973 | ||||
974 | iterator_range<value_op_iterator> op_values() const { | |||
975 | return make_range(value_op_iterator(op_begin()), | |||
976 | value_op_iterator(op_end())); | |||
977 | } | |||
978 | ||||
979 | SDVTList getVTList() const { | |||
980 | SDVTList X = { ValueList, NumValues }; | |||
981 | return X; | |||
982 | } | |||
983 | ||||
984 | /// If this node has a glue operand, return the node | |||
985 | /// to which the glue operand points. Otherwise return NULL. | |||
986 | SDNode *getGluedNode() const { | |||
987 | if (getNumOperands() != 0 && | |||
988 | getOperand(getNumOperands()-1).getValueType() == MVT::Glue) | |||
989 | return getOperand(getNumOperands()-1).getNode(); | |||
990 | return nullptr; | |||
991 | } | |||
992 | ||||
993 | /// If this node has a glue value with a user, return | |||
994 | /// the user (there is at most one). Otherwise return NULL. | |||
995 | SDNode *getGluedUser() const { | |||
996 | for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) | |||
997 | if (UI.getUse().get().getValueType() == MVT::Glue) | |||
998 | return *UI; | |||
999 | return nullptr; | |||
1000 | } | |||
1001 | ||||
1002 | const SDNodeFlags getFlags() const { return Flags; } | |||
1003 | void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; } | |||
1004 | bool isFast() { return Flags.isFast(); } | |||
1005 | ||||
1006 | /// Clear any flags in this node that aren't also set in Flags. | |||
1007 | /// If Flags is not in a defined state then this has no effect. | |||
1008 | void intersectFlagsWith(const SDNodeFlags Flags); | |||
1009 | ||||
1010 | /// Return the number of values defined/returned by this operator. | |||
1011 | unsigned getNumValues() const { return NumValues; } | |||
1012 | ||||
1013 | /// Return the type of a specified result. | |||
1014 | EVT getValueType(unsigned ResNo) const { | |||
1015 | assert(ResNo < NumValues && "Illegal result number!")((ResNo < NumValues && "Illegal result number!") ? static_cast<void> (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1015, __PRETTY_FUNCTION__)); | |||
1016 | return ValueList[ResNo]; | |||
1017 | } | |||
1018 | ||||
1019 | /// Return the type of a specified result as a simple type. | |||
1020 | MVT getSimpleValueType(unsigned ResNo) const { | |||
1021 | return getValueType(ResNo).getSimpleVT(); | |||
1022 | } | |||
1023 | ||||
1024 | /// Returns MVT::getSizeInBits(getValueType(ResNo)). | |||
1025 | unsigned getValueSizeInBits(unsigned ResNo) const { | |||
1026 | return getValueType(ResNo).getSizeInBits(); | |||
1027 | } | |||
1028 | ||||
1029 | using value_iterator = const EVT *; | |||
1030 | ||||
1031 | value_iterator value_begin() const { return ValueList; } | |||
1032 | value_iterator value_end() const { return ValueList+NumValues; } | |||
1033 | ||||
1034 | /// Return the opcode of this operation for printing. | |||
1035 | std::string getOperationName(const SelectionDAG *G = nullptr) const; | |||
1036 | static const char* getIndexedModeName(ISD::MemIndexedMode AM); | |||
1037 | void print_types(raw_ostream &OS, const SelectionDAG *G) const; | |||
1038 | void print_details(raw_ostream &OS, const SelectionDAG *G) const; | |||
1039 | void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
1040 | void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
1041 | ||||
1042 | /// Print a SelectionDAG node and all children down to | |||
1043 | /// the leaves. The given SelectionDAG allows target-specific nodes | |||
1044 | /// to be printed in human-readable form. Unlike printr, this will | |||
1045 | /// print the whole DAG, including children that appear multiple | |||
1046 | /// times. | |||
1047 | /// | |||
1048 | void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; | |||
1049 | ||||
1050 | /// Print a SelectionDAG node and children up to | |||
1051 | /// depth "depth." The given SelectionDAG allows target-specific | |||
1052 | /// nodes to be printed in human-readable form. Unlike printr, this | |||
1053 | /// will print children that appear multiple times wherever they are | |||
1054 | /// used. | |||
1055 | /// | |||
1056 | void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, | |||
1057 | unsigned depth = 100) const; | |||
1058 | ||||
1059 | /// Dump this node, for debugging. | |||
1060 | void dump() const; | |||
1061 | ||||
1062 | /// Dump (recursively) this node and its use-def subgraph. | |||
1063 | void dumpr() const; | |||
1064 | ||||
1065 | /// Dump this node, for debugging. | |||
1066 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
1067 | /// in human-readable form. | |||
1068 | void dump(const SelectionDAG *G) const; | |||
1069 | ||||
1070 | /// Dump (recursively) this node and its use-def subgraph. | |||
1071 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
1072 | /// in human-readable form. | |||
1073 | void dumpr(const SelectionDAG *G) const; | |||
1074 | ||||
1075 | /// printrFull to dbgs(). The given SelectionDAG allows | |||
1076 | /// target-specific nodes to be printed in human-readable form. | |||
1077 | /// Unlike dumpr, this will print the whole DAG, including children | |||
1078 | /// that appear multiple times. | |||
1079 | void dumprFull(const SelectionDAG *G = nullptr) const; | |||
1080 | ||||
1081 | /// printrWithDepth to dbgs(). The given | |||
1082 | /// SelectionDAG allows target-specific nodes to be printed in | |||
1083 | /// human-readable form. Unlike dumpr, this will print children | |||
1084 | /// that appear multiple times wherever they are used. | |||
1085 | /// | |||
1086 | void dumprWithDepth(const SelectionDAG *G = nullptr, | |||
1087 | unsigned depth = 100) const; | |||
1088 | ||||
1089 | /// Gather unique data for the node. | |||
1090 | void Profile(FoldingSetNodeID &ID) const; | |||
1091 | ||||
1092 | /// This method should only be used by the SDUse class. | |||
1093 | void addUse(SDUse &U) { U.addToList(&UseList); } | |||
1094 | ||||
1095 | protected: | |||
1096 | static SDVTList getSDVTList(EVT VT) { | |||
1097 | SDVTList Ret = { getValueTypeList(VT), 1 }; | |||
1098 | return Ret; | |||
1099 | } | |||
1100 | ||||
1101 | /// Create an SDNode. | |||
1102 | /// | |||
1103 | /// SDNodes are created without any operands, and never own the operand | |||
1104 | /// storage. To add operands, see SelectionDAG::createOperands. | |||
1105 | SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) | |||
1106 | : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs), | |||
1107 | IROrder(Order), debugLoc(std::move(dl)) { | |||
1108 | memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits)); | |||
1109 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor" ) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1109, __PRETTY_FUNCTION__)); | |||
1110 | assert(NumValues == VTs.NumVTs &&((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!" ) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1111, __PRETTY_FUNCTION__)) | |||
1111 | "NumValues wasn't wide enough for its operands!")((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!" ) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1111, __PRETTY_FUNCTION__)); | |||
1112 | } | |||
1113 | ||||
1114 | /// Release the operands and set this node to have zero operands. | |||
1115 | void DropOperands(); | |||
1116 | }; | |||
1117 | ||||
1118 | /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed | |||
1119 | /// into SDNode creation functions. | |||
1120 | /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted | |||
1121 | /// from the original Instruction, and IROrder is the ordinal position of | |||
1122 | /// the instruction. | |||
1123 | /// When an SDNode is created after the DAG is being built, both DebugLoc and | |||
1124 | /// the IROrder are propagated from the original SDNode. | |||
1125 | /// So SDLoc class provides two constructors besides the default one, one to | |||
1126 | /// be used by the DAGBuilder, the other to be used by others. | |||
1127 | class SDLoc { | |||
1128 | private: | |||
1129 | DebugLoc DL; | |||
1130 | int IROrder = 0; | |||
1131 | ||||
1132 | public: | |||
1133 | SDLoc() = default; | |||
1134 | SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {} | |||
1135 | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} | |||
1136 | SDLoc(const Instruction *I, int Order) : IROrder(Order) { | |||
1137 | assert(Order >= 0 && "bad IROrder")((Order >= 0 && "bad IROrder") ? static_cast<void > (0) : __assert_fail ("Order >= 0 && \"bad IROrder\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1137, __PRETTY_FUNCTION__)); | |||
1138 | if (I) | |||
1139 | DL = I->getDebugLoc(); | |||
1140 | } | |||
1141 | ||||
1142 | unsigned getIROrder() const { return IROrder; } | |||
1143 | const DebugLoc &getDebugLoc() const { return DL; } | |||
1144 | }; | |||
1145 | ||||
1146 | // Define inline functions from the SDValue class. | |||
1147 | ||||
1148 | inline SDValue::SDValue(SDNode *node, unsigned resno) | |||
1149 | : Node(node), ResNo(resno) { | |||
1150 | // Explicitly check for !ResNo to avoid use-after-free, because there are | |||
1151 | // callers that use SDValue(N, 0) with a deleted N to indicate successful | |||
1152 | // combines. | |||
1153 | assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(((!Node || !ResNo || ResNo < Node->getNumValues()) && "Invalid result number for the given node!") ? static_cast< void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1154, __PRETTY_FUNCTION__)) | |||
1154 | "Invalid result number for the given node!")(((!Node || !ResNo || ResNo < Node->getNumValues()) && "Invalid result number for the given node!") ? static_cast< void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1154, __PRETTY_FUNCTION__)); | |||
1155 | assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((ResNo < -2U && "Cannot use result numbers reserved for DenseMaps." ) ? static_cast<void> (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1155, __PRETTY_FUNCTION__)); | |||
1156 | } | |||
1157 | ||||
1158 | inline unsigned SDValue::getOpcode() const { | |||
1159 | return Node->getOpcode(); | |||
1160 | } | |||
1161 | ||||
1162 | inline EVT SDValue::getValueType() const { | |||
1163 | return Node->getValueType(ResNo); | |||
| ||||
1164 | } | |||
1165 | ||||
1166 | inline unsigned SDValue::getNumOperands() const { | |||
1167 | return Node->getNumOperands(); | |||
1168 | } | |||
1169 | ||||
1170 | inline const SDValue &SDValue::getOperand(unsigned i) const { | |||
1171 | return Node->getOperand(i); | |||
1172 | } | |||
1173 | ||||
1174 | inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { | |||
1175 | return Node->getConstantOperandVal(i); | |||
1176 | } | |||
1177 | ||||
1178 | inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { | |||
1179 | return Node->getConstantOperandAPInt(i); | |||
1180 | } | |||
1181 | ||||
1182 | inline bool SDValue::isTargetOpcode() const { | |||
1183 | return Node->isTargetOpcode(); | |||
1184 | } | |||
1185 | ||||
1186 | inline bool SDValue::isTargetMemoryOpcode() const { | |||
1187 | return Node->isTargetMemoryOpcode(); | |||
1188 | } | |||
1189 | ||||
1190 | inline bool SDValue::isMachineOpcode() const { | |||
1191 | return Node->isMachineOpcode(); | |||
1192 | } | |||
1193 | ||||
1194 | inline unsigned SDValue::getMachineOpcode() const { | |||
1195 | return Node->getMachineOpcode(); | |||
1196 | } | |||
1197 | ||||
1198 | inline bool SDValue::isUndef() const { | |||
1199 | return Node->isUndef(); | |||
1200 | } | |||
1201 | ||||
1202 | inline bool SDValue::use_empty() const { | |||
1203 | return !Node->hasAnyUseOfValue(ResNo); | |||
1204 | } | |||
1205 | ||||
1206 | inline bool SDValue::hasOneUse() const { | |||
1207 | return Node->hasNUsesOfValue(1, ResNo); | |||
1208 | } | |||
1209 | ||||
1210 | inline const DebugLoc &SDValue::getDebugLoc() const { | |||
1211 | return Node->getDebugLoc(); | |||
1212 | } | |||
1213 | ||||
1214 | inline void SDValue::dump() const { | |||
1215 | return Node->dump(); | |||
1216 | } | |||
1217 | ||||
1218 | inline void SDValue::dump(const SelectionDAG *G) const { | |||
1219 | return Node->dump(G); | |||
1220 | } | |||
1221 | ||||
1222 | inline void SDValue::dumpr() const { | |||
1223 | return Node->dumpr(); | |||
1224 | } | |||
1225 | ||||
1226 | inline void SDValue::dumpr(const SelectionDAG *G) const { | |||
1227 | return Node->dumpr(G); | |||
1228 | } | |||
1229 | ||||
1230 | // Define inline functions from the SDUse class. | |||
1231 | ||||
1232 | inline void SDUse::set(const SDValue &V) { | |||
1233 | if (Val.getNode()) removeFromList(); | |||
1234 | Val = V; | |||
1235 | if (V.getNode()) V.getNode()->addUse(*this); | |||
1236 | } | |||
1237 | ||||
1238 | inline void SDUse::setInitial(const SDValue &V) { | |||
1239 | Val = V; | |||
1240 | V.getNode()->addUse(*this); | |||
1241 | } | |||
1242 | ||||
1243 | inline void SDUse::setNode(SDNode *N) { | |||
1244 | if (Val.getNode()) removeFromList(); | |||
1245 | Val.setNode(N); | |||
1246 | if (N) N->addUse(*this); | |||
1247 | } | |||
1248 | ||||
1249 | /// This class is used to form a handle around another node that | |||
1250 | /// is persistent and is updated across invocations of replaceAllUsesWith on its | |||
1251 | /// operand. This node should be directly created by end-users and not added to | |||
1252 | /// the AllNodes list. | |||
1253 | class HandleSDNode : public SDNode { | |||
1254 | SDUse Op; | |||
1255 | ||||
1256 | public: | |||
1257 | explicit HandleSDNode(SDValue X) | |||
1258 | : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { | |||
1259 | // HandleSDNodes are never inserted into the DAG, so they won't be | |||
1260 | // auto-numbered. Use ID 65535 as a sentinel. | |||
1261 | PersistentId = 0xffff; | |||
1262 | ||||
1263 | // Manually set up the operand list. This node type is special in that it's | |||
1264 | // always stack allocated and SelectionDAG does not manage its operands. | |||
1265 | // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not | |||
1266 | // be so special. | |||
1267 | Op.setUser(this); | |||
1268 | Op.setInitial(X); | |||
1269 | NumOperands = 1; | |||
1270 | OperandList = &Op; | |||
1271 | } | |||
1272 | ~HandleSDNode(); | |||
1273 | ||||
1274 | const SDValue &getValue() const { return Op; } | |||
1275 | }; | |||
1276 | ||||
1277 | class AddrSpaceCastSDNode : public SDNode { | |||
1278 | private: | |||
1279 | unsigned SrcAddrSpace; | |||
1280 | unsigned DestAddrSpace; | |||
1281 | ||||
1282 | public: | |||
1283 | AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, | |||
1284 | unsigned SrcAS, unsigned DestAS); | |||
1285 | ||||
1286 | unsigned getSrcAddressSpace() const { return SrcAddrSpace; } | |||
1287 | unsigned getDestAddressSpace() const { return DestAddrSpace; } | |||
1288 | ||||
1289 | static bool classof(const SDNode *N) { | |||
1290 | return N->getOpcode() == ISD::ADDRSPACECAST; | |||
1291 | } | |||
1292 | }; | |||
1293 | ||||
1294 | /// This is an abstract virtual class for memory operations. | |||
1295 | class MemSDNode : public SDNode { | |||
1296 | private: | |||
1297 | // VT of in-memory value. | |||
1298 | EVT MemoryVT; | |||
1299 | ||||
1300 | protected: | |||
1301 | /// Memory reference information. | |||
1302 | MachineMemOperand *MMO; | |||
1303 | ||||
1304 | public: | |||
1305 | MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
1306 | EVT memvt, MachineMemOperand *MMO); | |||
1307 | ||||
1308 | bool readMem() const { return MMO->isLoad(); } | |||
1309 | bool writeMem() const { return MMO->isStore(); } | |||
1310 | ||||
1311 | /// Returns alignment and volatility of the memory access | |||
1312 | unsigned getOriginalAlignment() const { | |||
1313 | return MMO->getBaseAlignment(); | |||
1314 | } | |||
1315 | unsigned getAlignment() const { | |||
1316 | return MMO->getAlignment(); | |||
1317 | } | |||
1318 | ||||
1319 | /// Return the SubclassData value, without HasDebugValue. This contains an | |||
1320 | /// encoding of the volatile flag, as well as bits used by subclasses. This | |||
1321 | /// function should only be used to compute a FoldingSetNodeID value. | |||
1322 | /// The HasDebugValue bit is masked out because CSE map needs to match | |||
1323 | /// nodes with debug info with nodes without debug info. Same is about | |||
1324 | /// isDivergent bit. | |||
1325 | unsigned getRawSubclassData() const { | |||
1326 | uint16_t Data; | |||
1327 | union { | |||
1328 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
1329 | SDNodeBitfields SDNodeBits; | |||
1330 | }; | |||
1331 | memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits)); | |||
1332 | SDNodeBits.HasDebugValue = 0; | |||
1333 | SDNodeBits.IsDivergent = false; | |||
1334 | memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits)); | |||
1335 | return Data; | |||
1336 | } | |||
1337 | ||||
1338 | bool isVolatile() const { return MemSDNodeBits.IsVolatile; } | |||
1339 | bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } | |||
1340 | bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } | |||
1341 | bool isInvariant() const { return MemSDNodeBits.IsInvariant; } | |||
1342 | ||||
1343 | // Returns the offset from the location of the access. | |||
1344 | int64_t getSrcValueOffset() const { return MMO->getOffset(); } | |||
1345 | ||||
1346 | /// Returns the AA info that describes the dereference. | |||
1347 | AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } | |||
1348 | ||||
1349 | /// Returns the Ranges that describes the dereference. | |||
1350 | const MDNode *getRanges() const { return MMO->getRanges(); } | |||
1351 | ||||
1352 | /// Returns the synchronization scope ID for this memory operation. | |||
1353 | SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } | |||
1354 | ||||
1355 | /// Return the atomic ordering requirements for this memory operation. For | |||
1356 | /// cmpxchg atomic operations, return the atomic ordering requirements when | |||
1357 | /// store occurs. | |||
1358 | AtomicOrdering getOrdering() const { return MMO->getOrdering(); } | |||
1359 | ||||
1360 | /// Return true if the memory operation ordering is Unordered or higher. | |||
1361 | bool isAtomic() const { return MMO->isAtomic(); } | |||
1362 | ||||
1363 | /// Returns true if the memory operation doesn't imply any ordering | |||
1364 | /// constraints on surrounding memory operations beyond the normal memory | |||
1365 | /// aliasing rules. | |||
1366 | bool isUnordered() const { return MMO->isUnordered(); } | |||
1367 | ||||
1368 | /// Returns true if the memory operation is neither atomic or volatile. | |||
1369 | bool isSimple() const { return !isAtomic() && !isVolatile(); } | |||
1370 | ||||
1371 | /// Return the type of the in-memory value. | |||
1372 | EVT getMemoryVT() const { return MemoryVT; } | |||
1373 | ||||
1374 | /// Return a MachineMemOperand object describing the memory | |||
1375 | /// reference performed by operation. | |||
1376 | MachineMemOperand *getMemOperand() const { return MMO; } | |||
1377 | ||||
1378 | const MachinePointerInfo &getPointerInfo() const { | |||
1379 | return MMO->getPointerInfo(); | |||
1380 | } | |||
1381 | ||||
1382 | /// Return the address space for the associated pointer | |||
1383 | unsigned getAddressSpace() const { | |||
1384 | return getPointerInfo().getAddrSpace(); | |||
1385 | } | |||
1386 | ||||
1387 | /// Update this MemSDNode's MachineMemOperand information | |||
1388 | /// to reflect the alignment of NewMMO, if it has a greater alignment. | |||
1389 | /// This must only be used when the new alignment applies to all users of | |||
1390 | /// this MachineMemOperand. | |||
1391 | void refineAlignment(const MachineMemOperand *NewMMO) { | |||
1392 | MMO->refineAlignment(NewMMO); | |||
1393 | } | |||
1394 | ||||
1395 | const SDValue &getChain() const { return getOperand(0); } | |||
1396 | const SDValue &getBasePtr() const { | |||
1397 | return getOperand(getOpcode() == ISD::STORE ? 2 : 1); | |||
1398 | } | |||
1399 | ||||
1400 | // Methods to support isa and dyn_cast | |||
1401 | static bool classof(const SDNode *N) { | |||
1402 | // For some targets, we lower some target intrinsics to a MemIntrinsicNode | |||
1403 | // with either an intrinsic or a target opcode. | |||
1404 | return N->getOpcode() == ISD::LOAD || | |||
1405 | N->getOpcode() == ISD::STORE || | |||
1406 | N->getOpcode() == ISD::PREFETCH || | |||
1407 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP || | |||
1408 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || | |||
1409 | N->getOpcode() == ISD::ATOMIC_SWAP || | |||
1410 | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || | |||
1411 | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || | |||
1412 | N->getOpcode() == ISD::ATOMIC_LOAD_AND || | |||
1413 | N->getOpcode() == ISD::ATOMIC_LOAD_CLR || | |||
1414 | N->getOpcode() == ISD::ATOMIC_LOAD_OR || | |||
1415 | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || | |||
1416 | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || | |||
1417 | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || | |||
1418 | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || | |||
1419 | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || | |||
1420 | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || | |||
1421 | N->getOpcode() == ISD::ATOMIC_LOAD_FADD || | |||
1422 | N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || | |||
1423 | N->getOpcode() == ISD::ATOMIC_LOAD || | |||
1424 | N->getOpcode() == ISD::ATOMIC_STORE || | |||
1425 | N->getOpcode() == ISD::MLOAD || | |||
1426 | N->getOpcode() == ISD::MSTORE || | |||
1427 | N->getOpcode() == ISD::MGATHER || | |||
1428 | N->getOpcode() == ISD::MSCATTER || | |||
1429 | N->isMemIntrinsic() || | |||
1430 | N->isTargetMemoryOpcode(); | |||
1431 | } | |||
1432 | }; | |||
1433 | ||||
1434 | /// This is an SDNode representing atomic operations. | |||
1435 | class AtomicSDNode : public MemSDNode { | |||
1436 | public: | |||
1437 | AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, | |||
1438 | EVT MemVT, MachineMemOperand *MMO) | |||
1439 | : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { | |||
1440 | assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE ) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1441, __PRETTY_FUNCTION__)) | |||
1441 | MMO->isAtomic()) && "then why are we using an AtomicSDNode?")((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE ) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1441, __PRETTY_FUNCTION__)); | |||
1442 | } | |||
1443 | ||||
1444 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
1445 | const SDValue &getVal() const { return getOperand(2); } | |||
1446 | ||||
1447 | /// Returns true if this SDNode represents cmpxchg atomic operation, false | |||
1448 | /// otherwise. | |||
1449 | bool isCompareAndSwap() const { | |||
1450 | unsigned Op = getOpcode(); | |||
1451 | return Op == ISD::ATOMIC_CMP_SWAP || | |||
1452 | Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; | |||
1453 | } | |||
1454 | ||||
1455 | /// For cmpxchg atomic operations, return the atomic ordering requirements | |||
1456 | /// when store does not occur. | |||
1457 | AtomicOrdering getFailureOrdering() const { | |||
1458 | assert(isCompareAndSwap() && "Must be cmpxchg operation")((isCompareAndSwap() && "Must be cmpxchg operation") ? static_cast<void> (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1458, __PRETTY_FUNCTION__)); | |||
1459 | return MMO->getFailureOrdering(); | |||
1460 | } | |||
1461 | ||||
1462 | // Methods to support isa and dyn_cast | |||
1463 | static bool classof(const SDNode *N) { | |||
1464 | return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || | |||
1465 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || | |||
1466 | N->getOpcode() == ISD::ATOMIC_SWAP || | |||
1467 | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || | |||
1468 | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || | |||
1469 | N->getOpcode() == ISD::ATOMIC_LOAD_AND || | |||
1470 | N->getOpcode() == ISD::ATOMIC_LOAD_CLR || | |||
1471 | N->getOpcode() == ISD::ATOMIC_LOAD_OR || | |||
1472 | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || | |||
1473 | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || | |||
1474 | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || | |||
1475 | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || | |||
1476 | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || | |||
1477 | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || | |||
1478 | N->getOpcode() == ISD::ATOMIC_LOAD_FADD || | |||
1479 | N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || | |||
1480 | N->getOpcode() == ISD::ATOMIC_LOAD || | |||
1481 | N->getOpcode() == ISD::ATOMIC_STORE; | |||
1482 | } | |||
1483 | }; | |||
1484 | ||||
1485 | /// This SDNode is used for target intrinsics that touch | |||
1486 | /// memory and need an associated MachineMemOperand. Its opcode may be | |||
1487 | /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode | |||
1488 | /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. | |||
1489 | class MemIntrinsicSDNode : public MemSDNode { | |||
1490 | public: | |||
1491 | MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, | |||
1492 | SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) | |||
1493 | : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { | |||
1494 | SDNodeBits.IsMemIntrinsic = true; | |||
1495 | } | |||
1496 | ||||
1497 | // Methods to support isa and dyn_cast | |||
1498 | static bool classof(const SDNode *N) { | |||
1499 | // We lower some target intrinsics to their target opcode | |||
1500 | // early a node with a target opcode can be of this class | |||
1501 | return N->isMemIntrinsic() || | |||
1502 | N->getOpcode() == ISD::PREFETCH || | |||
1503 | N->isTargetMemoryOpcode(); | |||
1504 | } | |||
1505 | }; | |||
1506 | ||||
1507 | /// This SDNode is used to implement the code generator | |||
1508 | /// support for the llvm IR shufflevector instruction. It combines elements | |||
1509 | /// from two input vectors into a new input vector, with the selection and | |||
1510 | /// ordering of elements determined by an array of integers, referred to as | |||
1511 | /// the shuffle mask. For input vectors of width N, mask indices of 0..N-1 | |||
1512 | /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS. | |||
1513 | /// An index of -1 is treated as undef, such that the code generator may put | |||
1514 | /// any value in the corresponding element of the result. | |||
1515 | class ShuffleVectorSDNode : public SDNode { | |||
1516 | // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and | |||
1517 | // is freed when the SelectionDAG object is destroyed. | |||
1518 | const int *Mask; | |||
1519 | ||||
1520 | protected: | |||
1521 | friend class SelectionDAG; | |||
1522 | ||||
1523 | ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M) | |||
1524 | : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} | |||
1525 | ||||
1526 | public: | |||
1527 | ArrayRef<int> getMask() const { | |||
1528 | EVT VT = getValueType(0); | |||
1529 | return makeArrayRef(Mask, VT.getVectorNumElements()); | |||
1530 | } | |||
1531 | ||||
1532 | int getMaskElt(unsigned Idx) const { | |||
1533 | assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")((Idx < getValueType(0).getVectorNumElements() && "Idx out of range!" ) ? static_cast<void> (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1533, __PRETTY_FUNCTION__)); | |||
1534 | return Mask[Idx]; | |||
1535 | } | |||
1536 | ||||
1537 | bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } | |||
1538 | ||||
1539 | int getSplatIndex() const { | |||
1540 | assert(isSplat() && "Cannot get splat index for non-splat!")((isSplat() && "Cannot get splat index for non-splat!" ) ? static_cast<void> (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1540, __PRETTY_FUNCTION__)); | |||
1541 | EVT VT = getValueType(0); | |||
1542 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) | |||
1543 | if (Mask[i] >= 0) | |||
1544 | return Mask[i]; | |||
1545 | ||||
1546 | // We can choose any index value here and be correct because all elements | |||
1547 | // are undefined. Return 0 for better potential for callers to simplify. | |||
1548 | return 0; | |||
1549 | } | |||
1550 | ||||
1551 | static bool isSplatMask(const int *Mask, EVT VT); | |||
1552 | ||||
1553 | /// Change values in a shuffle permute mask assuming | |||
1554 | /// the two vector operands have swapped position. | |||
1555 | static void commuteMask(MutableArrayRef<int> Mask) { | |||
1556 | unsigned NumElems = Mask.size(); | |||
1557 | for (unsigned i = 0; i != NumElems; ++i) { | |||
1558 | int idx = Mask[i]; | |||
1559 | if (idx < 0) | |||
1560 | continue; | |||
1561 | else if (idx < (int)NumElems) | |||
1562 | Mask[i] = idx + NumElems; | |||
1563 | else | |||
1564 | Mask[i] = idx - NumElems; | |||
1565 | } | |||
1566 | } | |||
1567 | ||||
1568 | static bool classof(const SDNode *N) { | |||
1569 | return N->getOpcode() == ISD::VECTOR_SHUFFLE; | |||
1570 | } | |||
1571 | }; | |||
1572 | ||||
1573 | class ConstantSDNode : public SDNode { | |||
1574 | friend class SelectionDAG; | |||
1575 | ||||
1576 | const ConstantInt *Value; | |||
1577 | ||||
1578 | ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) | |||
1579 | : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(), | |||
1580 | getSDVTList(VT)), | |||
1581 | Value(val) { | |||
1582 | ConstantSDNodeBits.IsOpaque = isOpaque; | |||
1583 | } | |||
1584 | ||||
1585 | public: | |||
1586 | const ConstantInt *getConstantIntValue() const { return Value; } | |||
1587 | const APInt &getAPIntValue() const { return Value->getValue(); } | |||
1588 | uint64_t getZExtValue() const { return Value->getZExtValue(); } | |||
1589 | int64_t getSExtValue() const { return Value->getSExtValue(); } | |||
1590 | uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) { | |||
1591 | return Value->getLimitedValue(Limit); | |||
1592 | } | |||
1593 | ||||
1594 | bool isOne() const { return Value->isOne(); } | |||
1595 | bool isNullValue() const { return Value->isZero(); } | |||
1596 | bool isAllOnesValue() const { return Value->isMinusOne(); } | |||
1597 | ||||
1598 | bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } | |||
1599 | ||||
1600 | static bool classof(const SDNode *N) { | |||
1601 | return N->getOpcode() == ISD::Constant || | |||
1602 | N->getOpcode() == ISD::TargetConstant; | |||
1603 | } | |||
1604 | }; | |||
1605 | ||||
1606 | uint64_t SDNode::getConstantOperandVal(unsigned Num) const { | |||
1607 | return cast<ConstantSDNode>(getOperand(Num))->getZExtValue(); | |||
1608 | } | |||
1609 | ||||
1610 | const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { | |||
1611 | return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue(); | |||
1612 | } | |||
1613 | ||||
1614 | class ConstantFPSDNode : public SDNode { | |||
1615 | friend class SelectionDAG; | |||
1616 | ||||
1617 | const ConstantFP *Value; | |||
1618 | ||||
1619 | ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) | |||
1620 | : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, | |||
1621 | DebugLoc(), getSDVTList(VT)), | |||
1622 | Value(val) {} | |||
1623 | ||||
1624 | public: | |||
1625 | const APFloat& getValueAPF() const { return Value->getValueAPF(); } | |||
1626 | const ConstantFP *getConstantFPValue() const { return Value; } | |||
1627 | ||||
1628 | /// Return true if the value is positive or negative zero. | |||
1629 | bool isZero() const { return Value->isZero(); } | |||
1630 | ||||
1631 | /// Return true if the value is a NaN. | |||
1632 | bool isNaN() const { return Value->isNaN(); } | |||
1633 | ||||
1634 | /// Return true if the value is an infinity | |||
1635 | bool isInfinity() const { return Value->isInfinity(); } | |||
1636 | ||||
1637 | /// Return true if the value is negative. | |||
1638 | bool isNegative() const { return Value->isNegative(); } | |||
1639 | ||||
1640 | /// We don't rely on operator== working on double values, as | |||
1641 | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. | |||
1642 | /// As such, this method can be used to do an exact bit-for-bit comparison of | |||
1643 | /// two floating point values. | |||
1644 | ||||
1645 | /// We leave the version with the double argument here because it's just so | |||
1646 | /// convenient to write "2.0" and the like. Without this function we'd | |||
1647 | /// have to duplicate its logic everywhere it's called. | |||
1648 | bool isExactlyValue(double V) const { | |||
1649 | return Value->getValueAPF().isExactlyValue(V); | |||
1650 | } | |||
1651 | bool isExactlyValue(const APFloat& V) const; | |||
1652 | ||||
1653 | static bool isValueValidForType(EVT VT, const APFloat& Val); | |||
1654 | ||||
1655 | static bool classof(const SDNode *N) { | |||
1656 | return N->getOpcode() == ISD::ConstantFP || | |||
1657 | N->getOpcode() == ISD::TargetConstantFP; | |||
1658 | } | |||
1659 | }; | |||
1660 | ||||
1661 | /// Returns true if \p V is a constant integer zero. | |||
1662 | bool isNullConstant(SDValue V); | |||
1663 | ||||
1664 | /// Returns true if \p V is an FP constant with a value of positive zero. | |||
1665 | bool isNullFPConstant(SDValue V); | |||
1666 | ||||
1667 | /// Returns true if \p V is an integer constant with all bits set. | |||
1668 | bool isAllOnesConstant(SDValue V); | |||
1669 | ||||
1670 | /// Returns true if \p V is a constant integer one. | |||
1671 | bool isOneConstant(SDValue V); | |||
1672 | ||||
1673 | /// Return the non-bitcasted source operand of \p V if it exists. | |||
1674 | /// If \p V is not a bitcasted value, it is returned as-is. | |||
1675 | SDValue peekThroughBitcasts(SDValue V); | |||
1676 | ||||
1677 | /// Return the non-bitcasted and one-use source operand of \p V if it exists. | |||
1678 | /// If \p V is not a bitcasted one-use value, it is returned as-is. | |||
1679 | SDValue peekThroughOneUseBitcasts(SDValue V); | |||
1680 | ||||
1681 | /// Return the non-extracted vector source operand of \p V if it exists. | |||
1682 | /// If \p V is not an extracted subvector, it is returned as-is. | |||
1683 | SDValue peekThroughExtractSubvectors(SDValue V); | |||
1684 | ||||
1685 | /// Returns true if \p V is a bitwise not operation. Assumes that an all ones | |||
1686 | /// constant is canonicalized to be operand 1. | |||
1687 | bool isBitwiseNot(SDValue V, bool AllowUndefs = false); | |||
1688 | ||||
1689 | /// Returns the SDNode if it is a constant splat BuildVector or constant int. | |||
1690 | ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, | |||
1691 | bool AllowTruncation = false); | |||
1692 | ||||
1693 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
1694 | /// constant int. | |||
1695 | ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, | |||
1696 | bool AllowUndefs = false, | |||
1697 | bool AllowTruncation = false); | |||
1698 | ||||
1699 | /// Returns the SDNode if it is a constant splat BuildVector or constant float. | |||
1700 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); | |||
1701 | ||||
1702 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
1703 | /// constant float. | |||
1704 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, | |||
1705 | bool AllowUndefs = false); | |||
1706 | ||||
1707 | /// Return true if the value is a constant 0 integer or a splatted vector of | |||
1708 | /// a constant 0 integer (with no undefs by default). | |||
1709 | /// Build vector implicit truncation is not an issue for null values. | |||
1710 | bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); | |||
1711 | ||||
1712 | /// Return true if the value is a constant 1 integer or a splatted vector of a | |||
1713 | /// constant 1 integer (with no undefs). | |||
1714 | /// Does not permit build vector implicit truncation. | |||
1715 | bool isOneOrOneSplat(SDValue V); | |||
1716 | ||||
1717 | /// Return true if the value is a constant -1 integer or a splatted vector of a | |||
1718 | /// constant -1 integer (with no undefs). | |||
1719 | /// Does not permit build vector implicit truncation. | |||
1720 | bool isAllOnesOrAllOnesSplat(SDValue V); | |||
1721 | ||||
1722 | class GlobalAddressSDNode : public SDNode { | |||
1723 | friend class SelectionDAG; | |||
1724 | ||||
1725 | const GlobalValue *TheGlobal; | |||
1726 | int64_t Offset; | |||
1727 | unsigned TargetFlags; | |||
1728 | ||||
1729 | GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, | |||
1730 | const GlobalValue *GA, EVT VT, int64_t o, | |||
1731 | unsigned TF); | |||
1732 | ||||
1733 | public: | |||
1734 | const GlobalValue *getGlobal() const { return TheGlobal; } | |||
1735 | int64_t getOffset() const { return Offset; } | |||
1736 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1737 | // Return the address space this GlobalAddress belongs to. | |||
1738 | unsigned getAddressSpace() const; | |||
1739 | ||||
1740 | static bool classof(const SDNode *N) { | |||
1741 | return N->getOpcode() == ISD::GlobalAddress || | |||
1742 | N->getOpcode() == ISD::TargetGlobalAddress || | |||
1743 | N->getOpcode() == ISD::GlobalTLSAddress || | |||
1744 | N->getOpcode() == ISD::TargetGlobalTLSAddress; | |||
1745 | } | |||
1746 | }; | |||
1747 | ||||
1748 | class FrameIndexSDNode : public SDNode { | |||
1749 | friend class SelectionDAG; | |||
1750 | ||||
1751 | int FI; | |||
1752 | ||||
1753 | FrameIndexSDNode(int fi, EVT VT, bool isTarg) | |||
1754 | : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, | |||
1755 | 0, DebugLoc(), getSDVTList(VT)), FI(fi) { | |||
1756 | } | |||
1757 | ||||
1758 | public: | |||
1759 | int getIndex() const { return FI; } | |||
1760 | ||||
1761 | static bool classof(const SDNode *N) { | |||
1762 | return N->getOpcode() == ISD::FrameIndex || | |||
1763 | N->getOpcode() == ISD::TargetFrameIndex; | |||
1764 | } | |||
1765 | }; | |||
1766 | ||||
1767 | /// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate | |||
1768 | /// the offet and size that are started/ended in the underlying FrameIndex. | |||
1769 | class LifetimeSDNode : public SDNode { | |||
1770 | friend class SelectionDAG; | |||
1771 | int64_t Size; | |||
1772 | int64_t Offset; // -1 if offset is unknown. | |||
1773 | ||||
1774 | LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, | |||
1775 | SDVTList VTs, int64_t Size, int64_t Offset) | |||
1776 | : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} | |||
1777 | public: | |||
1778 | int64_t getFrameIndex() const { | |||
1779 | return cast<FrameIndexSDNode>(getOperand(1))->getIndex(); | |||
1780 | } | |||
1781 | ||||
1782 | bool hasOffset() const { return Offset >= 0; } | |||
1783 | int64_t getOffset() const { | |||
1784 | assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast< void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1784, __PRETTY_FUNCTION__)); | |||
1785 | return Offset; | |||
1786 | } | |||
1787 | int64_t getSize() const { | |||
1788 | assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast< void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1788, __PRETTY_FUNCTION__)); | |||
1789 | return Size; | |||
1790 | } | |||
1791 | ||||
1792 | // Methods to support isa and dyn_cast | |||
1793 | static bool classof(const SDNode *N) { | |||
1794 | return N->getOpcode() == ISD::LIFETIME_START || | |||
1795 | N->getOpcode() == ISD::LIFETIME_END; | |||
1796 | } | |||
1797 | }; | |||
1798 | ||||
1799 | class JumpTableSDNode : public SDNode { | |||
1800 | friend class SelectionDAG; | |||
1801 | ||||
1802 | int JTI; | |||
1803 | unsigned TargetFlags; | |||
1804 | ||||
1805 | JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF) | |||
1806 | : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, | |||
1807 | 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { | |||
1808 | } | |||
1809 | ||||
1810 | public: | |||
1811 | int getIndex() const { return JTI; } | |||
1812 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1813 | ||||
1814 | static bool classof(const SDNode *N) { | |||
1815 | return N->getOpcode() == ISD::JumpTable || | |||
1816 | N->getOpcode() == ISD::TargetJumpTable; | |||
1817 | } | |||
1818 | }; | |||
1819 | ||||
1820 | class ConstantPoolSDNode : public SDNode { | |||
1821 | friend class SelectionDAG; | |||
1822 | ||||
1823 | union { | |||
1824 | const Constant *ConstVal; | |||
1825 | MachineConstantPoolValue *MachineCPVal; | |||
1826 | } Val; | |||
1827 | int Offset; // It's a MachineConstantPoolValue if top bit is set. | |||
1828 | unsigned Alignment; // Minimum alignment requirement of CP (not log2 value). | |||
1829 | unsigned TargetFlags; | |||
1830 | ||||
1831 | ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, | |||
1832 | unsigned Align, unsigned TF) | |||
1833 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
1834 | DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), | |||
1835 | TargetFlags(TF) { | |||
1836 | assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast <void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1836, __PRETTY_FUNCTION__)); | |||
1837 | Val.ConstVal = c; | |||
1838 | } | |||
1839 | ||||
1840 | ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, | |||
1841 | EVT VT, int o, unsigned Align, unsigned TF) | |||
1842 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
1843 | DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), | |||
1844 | TargetFlags(TF) { | |||
1845 | assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast <void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1845, __PRETTY_FUNCTION__)); | |||
1846 | Val.MachineCPVal = v; | |||
1847 | Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1); | |||
1848 | } | |||
1849 | ||||
1850 | public: | |||
1851 | bool isMachineConstantPoolEntry() const { | |||
1852 | return Offset < 0; | |||
1853 | } | |||
1854 | ||||
1855 | const Constant *getConstVal() const { | |||
1856 | assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")((!isMachineConstantPoolEntry() && "Wrong constantpool type" ) ? static_cast<void> (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1856, __PRETTY_FUNCTION__)); | |||
1857 | return Val.ConstVal; | |||
1858 | } | |||
1859 | ||||
1860 | MachineConstantPoolValue *getMachineCPVal() const { | |||
1861 | assert(isMachineConstantPoolEntry() && "Wrong constantpool type")((isMachineConstantPoolEntry() && "Wrong constantpool type" ) ? static_cast<void> (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1861, __PRETTY_FUNCTION__)); | |||
1862 | return Val.MachineCPVal; | |||
1863 | } | |||
1864 | ||||
1865 | int getOffset() const { | |||
1866 | return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1)); | |||
1867 | } | |||
1868 | ||||
1869 | // Return the alignment of this constant pool object, which is either 0 (for | |||
1870 | // default alignment) or the desired value. | |||
1871 | unsigned getAlignment() const { return Alignment; } | |||
1872 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1873 | ||||
1874 | Type *getType() const; | |||
1875 | ||||
1876 | static bool classof(const SDNode *N) { | |||
1877 | return N->getOpcode() == ISD::ConstantPool || | |||
1878 | N->getOpcode() == ISD::TargetConstantPool; | |||
1879 | } | |||
1880 | }; | |||
1881 | ||||
1882 | /// Completely target-dependent object reference. | |||
1883 | class TargetIndexSDNode : public SDNode { | |||
1884 | friend class SelectionDAG; | |||
1885 | ||||
1886 | unsigned TargetFlags; | |||
1887 | int Index; | |||
1888 | int64_t Offset; | |||
1889 | ||||
1890 | public: | |||
1891 | TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF) | |||
1892 | : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), | |||
1893 | TargetFlags(TF), Index(Idx), Offset(Ofs) {} | |||
1894 | ||||
1895 | unsigned getTargetFlags() const { return TargetFlags; } | |||
1896 | int getIndex() const { return Index; } | |||
1897 | int64_t getOffset() const { return Offset; } | |||
1898 | ||||
1899 | static bool classof(const SDNode *N) { | |||
1900 | return N->getOpcode() == ISD::TargetIndex; | |||
1901 | } | |||
1902 | }; | |||
1903 | ||||
1904 | class BasicBlockSDNode : public SDNode { | |||
1905 | friend class SelectionDAG; | |||
1906 | ||||
1907 | MachineBasicBlock *MBB; | |||
1908 | ||||
1909 | /// Debug info is meaningful and potentially useful here, but we create | |||
1910 | /// blocks out of order when they're jumped to, which makes it a bit | |||
1911 | /// harder. Let's see if we need it first. | |||
1912 | explicit BasicBlockSDNode(MachineBasicBlock *mbb) | |||
1913 | : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) | |||
1914 | {} | |||
1915 | ||||
1916 | public: | |||
1917 | MachineBasicBlock *getBasicBlock() const { return MBB; } | |||
1918 | ||||
1919 | static bool classof(const SDNode *N) { | |||
1920 | return N->getOpcode() == ISD::BasicBlock; | |||
1921 | } | |||
1922 | }; | |||
1923 | ||||
1924 | /// A "pseudo-class" with methods for operating on BUILD_VECTORs. | |||
1925 | class BuildVectorSDNode : public SDNode { | |||
1926 | public: | |||
1927 | // These are constructed as SDNodes and then cast to BuildVectorSDNodes. | |||
1928 | explicit BuildVectorSDNode() = delete; | |||
1929 | ||||
1930 | /// Check if this is a constant splat, and if so, find the | |||
1931 | /// smallest element size that splats the vector. If MinSplatBits is | |||
1932 | /// nonzero, the element size must be at least that large. Note that the | |||
1933 | /// splat element may be the entire vector (i.e., a one element vector). | |||
1934 | /// Returns the splat element value in SplatValue. Any undefined bits in | |||
1935 | /// that value are zero, and the corresponding bits in the SplatUndef mask | |||
1936 | /// are set. The SplatBitSize value is set to the splat element size in | |||
1937 | /// bits. HasAnyUndefs is set to true if any bits in the vector are | |||
1938 | /// undefined. isBigEndian describes the endianness of the target. | |||
1939 | bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, | |||
1940 | unsigned &SplatBitSize, bool &HasAnyUndefs, | |||
1941 | unsigned MinSplatBits = 0, | |||
1942 | bool isBigEndian = false) const; | |||
1943 | ||||
1944 | /// Returns the demanded splatted value or a null value if this is not a | |||
1945 | /// splat. | |||
1946 | /// | |||
1947 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
1948 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1949 | /// the vector width and set the bits where elements are undef. | |||
1950 | SDValue getSplatValue(const APInt &DemandedElts, | |||
1951 | BitVector *UndefElements = nullptr) const; | |||
1952 | ||||
1953 | /// Returns the splatted value or a null value if this is not a splat. | |||
1954 | /// | |||
1955 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1956 | /// the vector width and set the bits where elements are undef. | |||
1957 | SDValue getSplatValue(BitVector *UndefElements = nullptr) const; | |||
1958 | ||||
1959 | /// Returns the demanded splatted constant or null if this is not a constant | |||
1960 | /// splat. | |||
1961 | /// | |||
1962 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
1963 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1964 | /// the vector width and set the bits where elements are undef. | |||
1965 | ConstantSDNode * | |||
1966 | getConstantSplatNode(const APInt &DemandedElts, | |||
1967 | BitVector *UndefElements = nullptr) const; | |||
1968 | ||||
1969 | /// Returns the splatted constant or null if this is not a constant | |||
1970 | /// splat. | |||
1971 | /// | |||
1972 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1973 | /// the vector width and set the bits where elements are undef. | |||
1974 | ConstantSDNode * | |||
1975 | getConstantSplatNode(BitVector *UndefElements = nullptr) const; | |||
1976 | ||||
1977 | /// Returns the demanded splatted constant FP or null if this is not a | |||
1978 | /// constant FP splat. | |||
1979 | /// | |||
1980 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
1981 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1982 | /// the vector width and set the bits where elements are undef. | |||
1983 | ConstantFPSDNode * | |||
1984 | getConstantFPSplatNode(const APInt &DemandedElts, | |||
1985 | BitVector *UndefElements = nullptr) const; | |||
1986 | ||||
1987 | /// Returns the splatted constant FP or null if this is not a constant | |||
1988 | /// FP splat. | |||
1989 | /// | |||
1990 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
1991 | /// the vector width and set the bits where elements are undef. | |||
1992 | ConstantFPSDNode * | |||
1993 | getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; | |||
1994 | ||||
1995 | /// If this is a constant FP splat and the splatted constant FP is an | |||
1996 | /// exact power or 2, return the log base 2 integer value. Otherwise, | |||
1997 | /// return -1. | |||
1998 | /// | |||
1999 | /// The BitWidth specifies the necessary bit precision. | |||
2000 | int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, | |||
2001 | uint32_t BitWidth) const; | |||
2002 | ||||
2003 | bool isConstant() const; | |||
2004 | ||||
2005 | static bool classof(const SDNode *N) { | |||
2006 | return N->getOpcode() == ISD::BUILD_VECTOR; | |||
2007 | } | |||
2008 | }; | |||
2009 | ||||
2010 | /// An SDNode that holds an arbitrary LLVM IR Value. This is | |||
2011 | /// used when the SelectionDAG needs to make a simple reference to something | |||
2012 | /// in the LLVM IR representation. | |||
2013 | /// | |||
2014 | class SrcValueSDNode : public SDNode { | |||
2015 | friend class SelectionDAG; | |||
2016 | ||||
2017 | const Value *V; | |||
2018 | ||||
2019 | /// Create a SrcValue for a general value. | |||
2020 | explicit SrcValueSDNode(const Value *v) | |||
2021 | : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} | |||
2022 | ||||
2023 | public: | |||
2024 | /// Return the contained Value. | |||
2025 | const Value *getValue() const { return V; } | |||
2026 | ||||
2027 | static bool classof(const SDNode *N) { | |||
2028 | return N->getOpcode() == ISD::SRCVALUE; | |||
2029 | } | |||
2030 | }; | |||
2031 | ||||
2032 | class MDNodeSDNode : public SDNode { | |||
2033 | friend class SelectionDAG; | |||
2034 | ||||
2035 | const MDNode *MD; | |||
2036 | ||||
2037 | explicit MDNodeSDNode(const MDNode *md) | |||
2038 | : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) | |||
2039 | {} | |||
2040 | ||||
2041 | public: | |||
2042 | const MDNode *getMD() const { return MD; } | |||
2043 | ||||
2044 | static bool classof(const SDNode *N) { | |||
2045 | return N->getOpcode() == ISD::MDNODE_SDNODE; | |||
2046 | } | |||
2047 | }; | |||
2048 | ||||
2049 | class RegisterSDNode : public SDNode { | |||
2050 |