Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1110, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name HexagonISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/build-llvm/lib/Target/Hexagon -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/build-llvm/lib/Target/Hexagon -I /build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon -I /build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/build-llvm/lib/Target/Hexagon -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-06-15-105110-21684-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp

/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp

1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/RuntimeLibcalls.h"
31#include "llvm/CodeGen/SelectionDAG.h"
32#include "llvm/CodeGen/TargetCallingConv.h"
33#include "llvm/CodeGen/ValueTypes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/CallingConv.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/Function.h"
39#include "llvm/IR/GlobalValue.h"
40#include "llvm/IR/InlineAsm.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/Intrinsics.h"
44#include "llvm/IR/IntrinsicsHexagon.h"
45#include "llvm/IR/IRBuilder.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/Type.h"
48#include "llvm/IR/Value.h"
49#include "llvm/MC/MCRegisterInfo.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/raw_ostream.h"
57#include "llvm/Target/TargetMachine.h"
58#include <algorithm>
59#include <cassert>
60#include <cstddef>
61#include <cstdint>
62#include <limits>
63#include <utility>
64
65using namespace llvm;
66
67#define DEBUG_TYPE"hexagon-lowering" "hexagon-lowering"
68
69static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
70 cl::init(true), cl::Hidden,
71 cl::desc("Control jump table emission on Hexagon target"));
72
73static cl::opt<bool> EnableHexSDNodeSched("enable-hexagon-sdnode-sched",
74 cl::Hidden, cl::ZeroOrMore, cl::init(false),
75 cl::desc("Enable Hexagon SDNode scheduling"));
76
77static cl::opt<bool> EnableFastMath("ffast-math",
78 cl::Hidden, cl::ZeroOrMore, cl::init(false),
79 cl::desc("Enable Fast Math processing"));
80
81static cl::opt<int> MinimumJumpTables("minimum-jump-tables",
82 cl::Hidden, cl::ZeroOrMore, cl::init(5),
83 cl::desc("Set minimum jump tables"));
84
85static cl::opt<int> MaxStoresPerMemcpyCL("max-store-memcpy",
86 cl::Hidden, cl::ZeroOrMore, cl::init(6),
87 cl::desc("Max #stores to inline memcpy"));
88
89static cl::opt<int> MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os",
90 cl::Hidden, cl::ZeroOrMore, cl::init(4),
91 cl::desc("Max #stores to inline memcpy"));
92
93static cl::opt<int> MaxStoresPerMemmoveCL("max-store-memmove",
94 cl::Hidden, cl::ZeroOrMore, cl::init(6),
95 cl::desc("Max #stores to inline memmove"));
96
97static cl::opt<int> MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os",
98 cl::Hidden, cl::ZeroOrMore, cl::init(4),
99 cl::desc("Max #stores to inline memmove"));
100
101static cl::opt<int> MaxStoresPerMemsetCL("max-store-memset",
102 cl::Hidden, cl::ZeroOrMore, cl::init(8),
103 cl::desc("Max #stores to inline memset"));
104
105static cl::opt<int> MaxStoresPerMemsetOptSizeCL("max-store-memset-Os",
106 cl::Hidden, cl::ZeroOrMore, cl::init(4),
107 cl::desc("Max #stores to inline memset"));
108
109static cl::opt<bool> AlignLoads("hexagon-align-loads",
110 cl::Hidden, cl::init(false),
111 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
112
113static cl::opt<bool>
114 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
115 cl::init(false),
116 cl::desc("Disable minimum alignment of 1 for "
117 "arguments passed by value on stack"));
118
119namespace {
120
121 class HexagonCCState : public CCState {
122 unsigned NumNamedVarArgParams = 0;
123
124 public:
125 HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF,
126 SmallVectorImpl<CCValAssign> &locs, LLVMContext &C,
127 unsigned NumNamedArgs)
128 : CCState(CC, IsVarArg, MF, locs, C),
129 NumNamedVarArgParams(NumNamedArgs) {}
130 unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
131 };
132
133} // end anonymous namespace
134
135
136// Implement calling convention for Hexagon.
137
138static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
139 CCValAssign::LocInfo &LocInfo,
140 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
141 static const MCPhysReg ArgRegs[] = {
142 Hexagon::R0, Hexagon::R1, Hexagon::R2,
143 Hexagon::R3, Hexagon::R4, Hexagon::R5
144 };
145 const unsigned NumArgRegs = array_lengthof(ArgRegs);
146 unsigned RegNum = State.getFirstUnallocated(ArgRegs);
147
148 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
149 if (RegNum != NumArgRegs && RegNum % 2 == 1)
150 State.AllocateReg(ArgRegs[RegNum]);
151
152 // Always return false here, as this function only makes sure that the first
153 // unallocated register has an even register number and does not actually
154 // allocate a register for the current argument.
155 return false;
156}
157
158#include "HexagonGenCallingConv.inc"
159
160
161SDValue
162HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
163 const {
164 return SDValue();
165}
166
167/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
168/// by "Src" to address "Dst" of size "Size". Alignment information is
169/// specified by the specific parameter attribute. The copy will be passed as
170/// a byval function parameter. Sometimes what we are copying is the end of a
171/// larger object, the part that does not fit in registers.
172static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
173 SDValue Chain, ISD::ArgFlagsTy Flags,
174 SelectionDAG &DAG, const SDLoc &dl) {
175 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
176 return DAG.getMemcpy(
177 Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
178 /*isVolatile=*/false, /*AlwaysInline=*/false,
179 /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
180}
181
182bool
183HexagonTargetLowering::CanLowerReturn(
184 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
185 const SmallVectorImpl<ISD::OutputArg> &Outs,
186 LLVMContext &Context) const {
187 SmallVector<CCValAssign, 16> RVLocs;
188 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
189
190 if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
191 return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX);
192 return CCInfo.CheckReturn(Outs, RetCC_Hexagon);
193}
194
195// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
196// passed by value, the function prototype is modified to return void and
197// the value is stored in memory pointed by a pointer passed by caller.
198SDValue
199HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
200 bool IsVarArg,
201 const SmallVectorImpl<ISD::OutputArg> &Outs,
202 const SmallVectorImpl<SDValue> &OutVals,
203 const SDLoc &dl, SelectionDAG &DAG) const {
204 // CCValAssign - represent the assignment of the return value to locations.
205 SmallVector<CCValAssign, 16> RVLocs;
206
207 // CCState - Info about the registers and stack slot.
208 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
209 *DAG.getContext());
210
211 // Analyze return values of ISD::RET
212 if (Subtarget.useHVXOps())
213 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX);
214 else
215 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon);
216
217 SDValue Flag;
218 SmallVector<SDValue, 4> RetOps(1, Chain);
219
220 // Copy the result values into the output registers.
221 for (unsigned i = 0; i != RVLocs.size(); ++i) {
222 CCValAssign &VA = RVLocs[i];
223 SDValue Val = OutVals[i];
224
225 switch (VA.getLocInfo()) {
226 default:
227 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
228 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 228)
;
229 case CCValAssign::Full:
230 break;
231 case CCValAssign::BCvt:
232 Val = DAG.getBitcast(VA.getLocVT(), Val);
233 break;
234 case CCValAssign::SExt:
235 Val = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Val);
236 break;
237 case CCValAssign::ZExt:
238 Val = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Val);
239 break;
240 case CCValAssign::AExt:
241 Val = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Val);
242 break;
243 }
244
245 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Val, Flag);
246
247 // Guarantee that all emitted copies are stuck together with flags.
248 Flag = Chain.getValue(1);
249 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
250 }
251
252 RetOps[0] = Chain; // Update chain.
253
254 // Add the flag if we have it.
255 if (Flag.getNode())
256 RetOps.push_back(Flag);
257
258 return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps);
259}
260
261bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
262 // If either no tail call or told not to tail call at all, don't.
263 return CI->isTailCall();
264}
265
266Register HexagonTargetLowering::getRegisterByName(
267 const char* RegName, LLT VT, const MachineFunction &) const {
268 // Just support r19, the linux kernel uses it.
269 Register Reg = StringSwitch<Register>(RegName)
270 .Case("r0", Hexagon::R0)
271 .Case("r1", Hexagon::R1)
272 .Case("r2", Hexagon::R2)
273 .Case("r3", Hexagon::R3)
274 .Case("r4", Hexagon::R4)
275 .Case("r5", Hexagon::R5)
276 .Case("r6", Hexagon::R6)
277 .Case("r7", Hexagon::R7)
278 .Case("r8", Hexagon::R8)
279 .Case("r9", Hexagon::R9)
280 .Case("r10", Hexagon::R10)
281 .Case("r11", Hexagon::R11)
282 .Case("r12", Hexagon::R12)
283 .Case("r13", Hexagon::R13)
284 .Case("r14", Hexagon::R14)
285 .Case("r15", Hexagon::R15)
286 .Case("r16", Hexagon::R16)
287 .Case("r17", Hexagon::R17)
288 .Case("r18", Hexagon::R18)
289 .Case("r19", Hexagon::R19)
290 .Case("r20", Hexagon::R20)
291 .Case("r21", Hexagon::R21)
292 .Case("r22", Hexagon::R22)
293 .Case("r23", Hexagon::R23)
294 .Case("r24", Hexagon::R24)
295 .Case("r25", Hexagon::R25)
296 .Case("r26", Hexagon::R26)
297 .Case("r27", Hexagon::R27)
298 .Case("r28", Hexagon::R28)
299 .Case("r29", Hexagon::R29)
300 .Case("r30", Hexagon::R30)
301 .Case("r31", Hexagon::R31)
302 .Case("r1:0", Hexagon::D0)
303 .Case("r3:2", Hexagon::D1)
304 .Case("r5:4", Hexagon::D2)
305 .Case("r7:6", Hexagon::D3)
306 .Case("r9:8", Hexagon::D4)
307 .Case("r11:10", Hexagon::D5)
308 .Case("r13:12", Hexagon::D6)
309 .Case("r15:14", Hexagon::D7)
310 .Case("r17:16", Hexagon::D8)
311 .Case("r19:18", Hexagon::D9)
312 .Case("r21:20", Hexagon::D10)
313 .Case("r23:22", Hexagon::D11)
314 .Case("r25:24", Hexagon::D12)
315 .Case("r27:26", Hexagon::D13)
316 .Case("r29:28", Hexagon::D14)
317 .Case("r31:30", Hexagon::D15)
318 .Case("sp", Hexagon::R29)
319 .Case("fp", Hexagon::R30)
320 .Case("lr", Hexagon::R31)
321 .Case("p0", Hexagon::P0)
322 .Case("p1", Hexagon::P1)
323 .Case("p2", Hexagon::P2)
324 .Case("p3", Hexagon::P3)
325 .Case("sa0", Hexagon::SA0)
326 .Case("lc0", Hexagon::LC0)
327 .Case("sa1", Hexagon::SA1)
328 .Case("lc1", Hexagon::LC1)
329 .Case("m0", Hexagon::M0)
330 .Case("m1", Hexagon::M1)
331 .Case("usr", Hexagon::USR)
332 .Case("ugp", Hexagon::UGP)
333 .Case("cs0", Hexagon::CS0)
334 .Case("cs1", Hexagon::CS1)
335 .Default(Register());
336 if (Reg)
337 return Reg;
338
339 report_fatal_error("Invalid register name global variable");
340}
341
342/// LowerCallResult - Lower the result values of an ISD::CALL into the
343/// appropriate copies out of appropriate physical registers. This assumes that
344/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
345/// being lowered. Returns a SDNode with the same number of values as the
346/// ISD::CALL.
347SDValue HexagonTargetLowering::LowerCallResult(
348 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
349 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
350 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
351 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
352 // Assign locations to each value returned by this call.
353 SmallVector<CCValAssign, 16> RVLocs;
354
355 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
356 *DAG.getContext());
357
358 if (Subtarget.useHVXOps())
359 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX);
360 else
361 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon);
362
363 // Copy all of the result registers out of their specified physreg.
364 for (unsigned i = 0; i != RVLocs.size(); ++i) {
365 SDValue RetVal;
366 if (RVLocs[i].getValVT() == MVT::i1) {
367 // Return values of type MVT::i1 require special handling. The reason
368 // is that MVT::i1 is associated with the PredRegs register class, but
369 // values of that type are still returned in R0. Generate an explicit
370 // copy into a predicate register from R0, and treat the value of the
371 // predicate register as the call result.
372 auto &MRI = DAG.getMachineFunction().getRegInfo();
373 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
374 MVT::i32, Glue);
375 // FR0 = (Value, Chain, Glue)
376 Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass);
377 SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR,
378 FR0.getValue(0), FR0.getValue(2));
379 // TPR = (Chain, Glue)
380 // Don't glue this CopyFromReg, because it copies from a virtual
381 // register. If it is glued to the call, InstrEmitter will add it
382 // as an implicit def to the call (EmitMachineNode).
383 RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1);
384 Glue = TPR.getValue(1);
385 Chain = TPR.getValue(0);
386 } else {
387 RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
388 RVLocs[i].getValVT(), Glue);
389 Glue = RetVal.getValue(2);
390 Chain = RetVal.getValue(1);
391 }
392 InVals.push_back(RetVal.getValue(0));
393 }
394
395 return Chain;
396}
397
398/// LowerCall - Functions arguments are copied from virtual regs to
399/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
400SDValue
401HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
402 SmallVectorImpl<SDValue> &InVals) const {
403 SelectionDAG &DAG = CLI.DAG;
404 SDLoc &dl = CLI.DL;
405 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
406 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
407 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
408 SDValue Chain = CLI.Chain;
409 SDValue Callee = CLI.Callee;
410 CallingConv::ID CallConv = CLI.CallConv;
411 bool IsVarArg = CLI.IsVarArg;
412 bool DoesNotReturn = CLI.DoesNotReturn;
413
414 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
415 MachineFunction &MF = DAG.getMachineFunction();
416 MachineFrameInfo &MFI = MF.getFrameInfo();
417 auto PtrVT = getPointerTy(MF.getDataLayout());
418
419 unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0;
420 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee))
421 Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32);
422
423 // Linux ABI treats var-arg calls the same way as regular ones.
424 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
425
426 // Analyze operands of the call, assigning locations to each operand.
427 SmallVector<CCValAssign, 16> ArgLocs;
428 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(),
429 NumParams);
430
431 if (Subtarget.useHVXOps())
432 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
433 else if (DisableArgsMinAlignment)
434 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
435 else
436 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
437
438 if (CLI.IsTailCall) {
439 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
440 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
441 IsVarArg, IsStructRet, StructAttrFlag, Outs,
442 OutVals, Ins, DAG);
443 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
444 CCValAssign &VA = ArgLocs[i];
445 if (VA.isMemLoc()) {
446 CLI.IsTailCall = false;
447 break;
448 }
449 }
450 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
: "Argument must be passed on stack. " "Not eligible for Tail Call\n"
); } } while (false)
451 : "Argument must be passed on stack. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
: "Argument must be passed on stack. " "Not eligible for Tail Call\n"
); } } while (false)
452 "Not eligible for Tail Call\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
: "Argument must be passed on stack. " "Not eligible for Tail Call\n"
); } } while (false)
;
453 }
454 // Get a count of how many bytes are to be pushed on the stack.
455 unsigned NumBytes = CCInfo.getNextStackOffset();
456 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
457 SmallVector<SDValue, 8> MemOpChains;
458
459 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
460 SDValue StackPtr =
461 DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT);
462
463 bool NeedsArgAlign = false;
464 Align LargestAlignSeen;
465 // Walk the register/memloc assignments, inserting copies/loads.
466 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
467 CCValAssign &VA = ArgLocs[i];
468 SDValue Arg = OutVals[i];
469 ISD::ArgFlagsTy Flags = Outs[i].Flags;
470 // Record if we need > 8 byte alignment on an argument.
471 bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT());
472 NeedsArgAlign |= ArgAlign;
473
474 // Promote the value if needed.
475 switch (VA.getLocInfo()) {
476 default:
477 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
478 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 478)
;
479 case CCValAssign::Full:
480 break;
481 case CCValAssign::BCvt:
482 Arg = DAG.getBitcast(VA.getLocVT(), Arg);
483 break;
484 case CCValAssign::SExt:
485 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
486 break;
487 case CCValAssign::ZExt:
488 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
489 break;
490 case CCValAssign::AExt:
491 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
492 break;
493 }
494
495 if (VA.isMemLoc()) {
496 unsigned LocMemOffset = VA.getLocMemOffset();
497 SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,
498 StackPtr.getValueType());
499 MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr);
500 if (ArgAlign)
501 LargestAlignSeen = std::max(
502 LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8));
503 if (Flags.isByVal()) {
504 // The argument is a struct passed by value. According to LLVM, "Arg"
505 // is a pointer.
506 MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain,
507 Flags, DAG, dl));
508 } else {
509 MachinePointerInfo LocPI = MachinePointerInfo::getStack(
510 DAG.getMachineFunction(), LocMemOffset);
511 SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI);
512 MemOpChains.push_back(S);
513 }
514 continue;
515 }
516
517 // Arguments that can be passed on register must be kept at RegsToPass
518 // vector.
519 if (VA.isRegLoc())
520 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
521 }
522
523 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
524 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << "Function needs byte stack align due to call args\n"
; } } while (false)
;
525 Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
526 LargestAlignSeen = std::max(LargestAlignSeen, VecAlign);
527 MFI.ensureMaxAlignment(LargestAlignSeen);
528 }
529 // Transform all store nodes into one single node because all store
530 // nodes are independent of each other.
531 if (!MemOpChains.empty())
532 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
533
534 SDValue Glue;
535 if (!CLI.IsTailCall) {
536 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
537 Glue = Chain.getValue(1);
538 }
539
540 // Build a sequence of copy-to-reg nodes chained together with token
541 // chain and flag operands which copy the outgoing args into registers.
542 // The Glue is necessary since all emitted instructions must be
543 // stuck together.
544 if (!CLI.IsTailCall) {
545 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
546 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
547 RegsToPass[i].second, Glue);
548 Glue = Chain.getValue(1);
549 }
550 } else {
551 // For tail calls lower the arguments to the 'real' stack slot.
552 //
553 // Force all the incoming stack arguments to be loaded from the stack
554 // before any new outgoing arguments are stored to the stack, because the
555 // outgoing stack slots may alias the incoming argument stack slots, and
556 // the alias isn't otherwise explicit. This is slightly more conservative
557 // than necessary, because it means that each store effectively depends
558 // on every argument instead of just those arguments it would clobber.
559 //
560 // Do not flag preceding copytoreg stuff together with the following stuff.
561 Glue = SDValue();
562 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
563 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
564 RegsToPass[i].second, Glue);
565 Glue = Chain.getValue(1);
566 }
567 Glue = SDValue();
568 }
569
570 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
571 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
572
573 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
574 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
575 // node so that legalize doesn't hack it.
576 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
577 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags);
578 } else if (ExternalSymbolSDNode *S =
579 dyn_cast<ExternalSymbolSDNode>(Callee)) {
580 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags);
581 }
582
583 // Returns a chain & a flag for retval copy to use.
584 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
585 SmallVector<SDValue, 8> Ops;
586 Ops.push_back(Chain);
587 Ops.push_back(Callee);
588
589 // Add argument registers to the end of the list so that they are
590 // known live into the call.
591 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
592 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
593 RegsToPass[i].second.getValueType()));
594 }
595
596 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
597 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 597, __extension__ __PRETTY_FUNCTION__))
;
598 Ops.push_back(DAG.getRegisterMask(Mask));
599
600 if (Glue.getNode())
601 Ops.push_back(Glue);
602
603 if (CLI.IsTailCall) {
604 MFI.setHasTailCall();
605 return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops);
606 }
607
608 // Set this here because we need to know this for "hasFP" in frame lowering.
609 // The target-independent code calls getFrameRegister before setting it, and
610 // getFrameRegister uses hasFP to determine whether the function has FP.
611 MFI.setHasCalls(true);
612
613 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
614 Chain = DAG.getNode(OpCode, dl, NodeTys, Ops);
615 Glue = Chain.getValue(1);
616
617 // Create the CALLSEQ_END node.
618 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
619 DAG.getIntPtrConstant(0, dl, true), Glue, dl);
620 Glue = Chain.getValue(1);
621
622 // Handle result values, copying them out of physregs into vregs that we
623 // return.
624 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
625 InVals, OutVals, Callee);
626}
627
628/// Returns true by value, base pointer and offset pointer and addressing
629/// mode by reference if this node can be combined with a load / store to
630/// form a post-indexed load / store.
631bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
632 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
633 SelectionDAG &DAG) const {
634 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N);
635 if (!LSN)
636 return false;
637 EVT VT = LSN->getMemoryVT();
638 if (!VT.isSimple())
639 return false;
640 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
641 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
642 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
643 VT == MVT::v4i16 || VT == MVT::v8i8 ||
644 Subtarget.isHVXVectorType(VT.getSimpleVT());
645 if (!IsLegalType)
646 return false;
647
648 if (Op->getOpcode() != ISD::ADD)
649 return false;
650 Base = Op->getOperand(0);
651 Offset = Op->getOperand(1);
652 if (!isa<ConstantSDNode>(Offset.getNode()))
653 return false;
654 AM = ISD::POST_INC;
655
656 int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
657 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V);
658}
659
660SDValue
661HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
662 MachineFunction &MF = DAG.getMachineFunction();
663 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
664 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
665 unsigned LR = HRI.getRARegister();
666
667 if ((Op.getOpcode() != ISD::INLINEASM &&
668 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
669 return Op;
670
671 unsigned NumOps = Op.getNumOperands();
672 if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue)
673 --NumOps; // Ignore the flag operand.
674
675 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
676 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue();
677 unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
678 ++i; // Skip the ID value.
679
680 switch (InlineAsm::getKind(Flags)) {
681 default:
682 llvm_unreachable("Bad flags!")::llvm::llvm_unreachable_internal("Bad flags!", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 682)
;
683 case InlineAsm::Kind_RegUse:
684 case InlineAsm::Kind_Imm:
685 case InlineAsm::Kind_Mem:
686 i += NumVals;
687 break;
688 case InlineAsm::Kind_Clobber:
689 case InlineAsm::Kind_RegDef:
690 case InlineAsm::Kind_RegDefEarlyClobber: {
691 for (; NumVals; --NumVals, ++i) {
692 unsigned Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
693 if (Reg != LR)
694 continue;
695 HMFI.setHasClobberLR(true);
696 return Op;
697 }
698 break;
699 }
700 }
701 }
702
703 return Op;
704}
705
706// Need to transform ISD::PREFETCH into something that doesn't inherit
707// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
708// SDNPMayStore.
709SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
710 SelectionDAG &DAG) const {
711 SDValue Chain = Op.getOperand(0);
712 SDValue Addr = Op.getOperand(1);
713 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
714 // if the "reg" is fed by an "add".
715 SDLoc DL(Op);
716 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
717 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
718}
719
720// Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode
721// is marked as having side-effects, while the register read on Hexagon does
722// not have any. TableGen refuses to accept the direct pattern from that node
723// to the A4_tfrcpp.
724SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
725 SelectionDAG &DAG) const {
726 SDValue Chain = Op.getOperand(0);
727 SDLoc dl(Op);
728 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
729 return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain);
730}
731
732SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
733 SelectionDAG &DAG) const {
734 SDValue Chain = Op.getOperand(0);
735 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
736 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
737 if (IntNo == Intrinsic::hexagon_prefetch) {
738 SDValue Addr = Op.getOperand(2);
739 SDLoc DL(Op);
740 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
741 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
742 }
743 return SDValue();
744}
745
746SDValue
747HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
748 SelectionDAG &DAG) const {
749 SDValue Chain = Op.getOperand(0);
750 SDValue Size = Op.getOperand(1);
751 SDValue Align = Op.getOperand(2);
752 SDLoc dl(Op);
753
754 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align);
755 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC")(static_cast <bool> (AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC"
) ? void (0) : __assert_fail ("AlignConst && \"Non-constant Align in LowerDYNAMIC_STACKALLOC\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 755, __extension__ __PRETTY_FUNCTION__))
;
756
757 unsigned A = AlignConst->getSExtValue();
758 auto &HFI = *Subtarget.getFrameLowering();
759 // "Zero" means natural stack alignment.
760 if (A == 0)
761 A = HFI.getStackAlign().value();
762
763 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
764 dbgs () << __func__ << " Align: " << A << " Size: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
765 Size.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
766 dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
767 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
;
768
769 SDValue AC = DAG.getConstant(A, dl, MVT::i32);
770 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
771 SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);
772
773 DAG.ReplaceAllUsesOfValueWith(Op, AA);
774 return AA;
775}
776
777SDValue HexagonTargetLowering::LowerFormalArguments(
778 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
779 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
780 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
781 MachineFunction &MF = DAG.getMachineFunction();
782 MachineFrameInfo &MFI = MF.getFrameInfo();
783 MachineRegisterInfo &MRI = MF.getRegInfo();
784
785 // Linux ABI treats var-arg calls the same way as regular ones.
786 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
787
788 // Assign locations to all of the incoming arguments.
789 SmallVector<CCValAssign, 16> ArgLocs;
790 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
791 *DAG.getContext(),
792 MF.getFunction().getFunctionType()->getNumParams());
793
794 if (Subtarget.useHVXOps())
795 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
796 else if (DisableArgsMinAlignment)
797 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);
798 else
799 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);
800
801 // For LLVM, in the case when returning a struct by value (>8byte),
802 // the first argument is a pointer that points to the location on caller's
803 // stack where the return value will be stored. For Hexagon, the location on
804 // caller's stack is passed only when the struct size is smaller than (and
805 // equal to) 8 bytes. If not, no address will be passed into callee and
806 // callee return the result direclty through R0/R1.
807 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
808 switch (RC.getID()) {
809 case Hexagon::IntRegsRegClassID:
810 return Reg - Hexagon::R0 + 1;
811 case Hexagon::DoubleRegsRegClassID:
812 return (Reg - Hexagon::D0 + 1) * 2;
813 case Hexagon::HvxVRRegClassID:
814 return Reg - Hexagon::V0 + 1;
815 case Hexagon::HvxWRRegClassID:
816 return (Reg - Hexagon::W0 + 1) * 2;
817 }
818 llvm_unreachable("Unexpected register class")::llvm::llvm_unreachable_internal("Unexpected register class"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 818)
;
819 };
820
821 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
822 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
823 HFL.FirstVarArgSavedReg = 0;
824 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
825
826 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
827 CCValAssign &VA = ArgLocs[i];
828 ISD::ArgFlagsTy Flags = Ins[i].Flags;
829 bool ByVal = Flags.isByVal();
830
831 // Arguments passed in registers:
832 // 1. 32- and 64-bit values and HVX vectors are passed directly,
833 // 2. Large structs are passed via an address, and the address is
834 // passed in a register.
835 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
836 llvm_unreachable("ByValSize must be bigger than 8 bytes")::llvm::llvm_unreachable_internal("ByValSize must be bigger than 8 bytes"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 836)
;
837
838 bool InReg = VA.isRegLoc() &&
839 (!ByVal || (ByVal && Flags.getByValSize() > 8));
840
841 if (InReg) {
842 MVT RegVT = VA.getLocVT();
843 if (VA.getLocInfo() == CCValAssign::BCvt)
844 RegVT = VA.getValVT();
845
846 const TargetRegisterClass *RC = getRegClassFor(RegVT);
847 Register VReg = MRI.createVirtualRegister(RC);
848 SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);
849
850 // Treat values of type MVT::i1 specially: they are passed in
851 // registers of type i32, but they need to remain as values of
852 // type i1 for consistency of the argument lowering.
853 if (VA.getValVT() == MVT::i1) {
854 assert(RegVT.getSizeInBits() <= 32)(static_cast <bool> (RegVT.getSizeInBits() <= 32) ? void
(0) : __assert_fail ("RegVT.getSizeInBits() <= 32", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 854, __extension__ __PRETTY_FUNCTION__))
;
855 SDValue T = DAG.getNode(ISD::AND, dl, RegVT,
856 Copy, DAG.getConstant(1, dl, RegVT));
857 Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT),
858 ISD::SETNE);
859 } else {
860#ifndef NDEBUG
861 unsigned RegSize = RegVT.getSizeInBits();
862 assert(RegSize == 32 || RegSize == 64 ||(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget
.isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 863, __extension__ __PRETTY_FUNCTION__))
863 Subtarget.isHVXVectorType(RegVT))(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget
.isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 863, __extension__ __PRETTY_FUNCTION__))
;
864#endif
865 }
866 InVals.push_back(Copy);
867 MRI.addLiveIn(VA.getLocReg(), VReg);
868 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
869 } else {
870 assert(VA.isMemLoc() && "Argument should be passed in memory")(static_cast <bool> (VA.isMemLoc() && "Argument should be passed in memory"
) ? void (0) : __assert_fail ("VA.isMemLoc() && \"Argument should be passed in memory\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 870, __extension__ __PRETTY_FUNCTION__))
;
871
872 // If it's a byval parameter, then we need to compute the
873 // "real" size, not the size of the pointer.
874 unsigned ObjSize = Flags.isByVal()
875 ? Flags.getByValSize()
876 : VA.getLocVT().getStoreSizeInBits() / 8;
877
878 // Create the frame index object for this incoming parameter.
879 int Offset = HEXAGON_LRFP_SIZE8 + VA.getLocMemOffset();
880 int FI = MFI.CreateFixedObject(ObjSize, Offset, true);
881 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
882
883 if (Flags.isByVal()) {
884 // If it's a pass-by-value aggregate, then do not dereference the stack
885 // location. Instead, we should generate a reference to the stack
886 // location.
887 InVals.push_back(FIN);
888 } else {
889 SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
890 MachinePointerInfo::getFixedStack(MF, FI, 0));
891 InVals.push_back(L);
892 }
893 }
894 }
895
896 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
897 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
898 MRI.addLiveIn(Hexagon::R0+i);
899 }
900
901 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
902 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
903 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
904
905 // Create Frame index for the start of register saved area.
906 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
907 bool RequiresPadding = (NumVarArgRegs & 1);
908 int RegSaveAreaSizePlusPadding = RequiresPadding
909 ? (NumVarArgRegs + 1) * 4
910 : NumVarArgRegs * 4;
911
912 if (RegSaveAreaSizePlusPadding > 0) {
913 // The offset to saved register area should be 8 byte aligned.
914 int RegAreaStart = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset();
915 if (!(RegAreaStart % 8))
916 RegAreaStart = (RegAreaStart + 7) & -8;
917
918 int RegSaveAreaFrameIndex =
919 MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true);
920 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
921
922 // This will point to the next argument passed via stack.
923 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
924 int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true);
925 HMFI.setVarArgsFrameIndex(FI);
926 } else {
927 // This will point to the next argument passed via stack, when
928 // there is no saved register area.
929 int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset();
930 int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true);
931 HMFI.setRegSavedAreaStartFrameIndex(FI);
932 HMFI.setVarArgsFrameIndex(FI);
933 }
934 }
935
936
937 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
938 // This will point to the next argument passed via stack.
939 int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset();
940 int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true);
941 HMFI.setVarArgsFrameIndex(FI);
942 }
943
944 return Chain;
945}
946
947SDValue
948HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
949 // VASTART stores the address of the VarArgsFrameIndex slot into the
950 // memory location argument.
951 MachineFunction &MF = DAG.getMachineFunction();
952 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
953 SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32);
954 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
955
956 if (!Subtarget.isEnvironmentMusl()) {
957 return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1),
958 MachinePointerInfo(SV));
959 }
960 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
961 auto &HFL = *Subtarget.getFrameLowering();
962 SDLoc DL(Op);
963 SmallVector<SDValue, 8> MemOps;
964
965 // Get frame index of va_list.
966 SDValue FIN = Op.getOperand(1);
967
968 // If first Vararg register is odd, add 4 bytes to start of
969 // saved register area to point to the first register location.
970 // This is because the saved register area has to be 8 byte aligned.
971 // Incase of an odd start register, there will be 4 bytes of padding in
972 // the beginning of saved register area. If all registers area used up,
973 // the following condition will handle it correctly.
974 SDValue SavedRegAreaStartFrameIndex =
975 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32);
976
977 auto PtrVT = getPointerTy(DAG.getDataLayout());
978
979 if (HFL.FirstVarArgSavedReg & 1)
980 SavedRegAreaStartFrameIndex =
981 DAG.getNode(ISD::ADD, DL, PtrVT,
982 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(),
983 MVT::i32),
984 DAG.getIntPtrConstant(4, DL));
985
986 // Store the saved register area start pointer.
987 SDValue Store =
988 DAG.getStore(Op.getOperand(0), DL,
989 SavedRegAreaStartFrameIndex,
990 FIN, MachinePointerInfo(SV));
991 MemOps.push_back(Store);
992
993 // Store saved register area end pointer.
994 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
995 FIN, DAG.getIntPtrConstant(4, DL));
996 Store = DAG.getStore(Op.getOperand(0), DL,
997 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
998 PtrVT),
999 FIN, MachinePointerInfo(SV, 4));
1000 MemOps.push_back(Store);
1001
1002 // Store overflow area pointer.
1003 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
1004 FIN, DAG.getIntPtrConstant(4, DL));
1005 Store = DAG.getStore(Op.getOperand(0), DL,
1006 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
1007 PtrVT),
1008 FIN, MachinePointerInfo(SV, 8));
1009 MemOps.push_back(Store);
1010
1011 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
1012}
1013
1014SDValue
1015HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1016 // Assert that the linux ABI is enabled for the current compilation.
1017 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled")(static_cast <bool> (Subtarget.isEnvironmentMusl() &&
"Linux ABI should be enabled") ? void (0) : __assert_fail ("Subtarget.isEnvironmentMusl() && \"Linux ABI should be enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1017, __extension__ __PRETTY_FUNCTION__))
;
1018 SDValue Chain = Op.getOperand(0);
1019 SDValue DestPtr = Op.getOperand(1);
1020 SDValue SrcPtr = Op.getOperand(2);
1021 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
1022 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
1023 SDLoc DL(Op);
1024 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1025 // we need to memcopy 12 bytes from va_list to another similar list.
1026 return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
1027 DAG.getIntPtrConstant(12, DL), Align(4),
1028 /*isVolatile*/ false, false, false,
1029 MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
1030}
1031
1032SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1033 const SDLoc &dl(Op);
1034 SDValue LHS = Op.getOperand(0);
1035 SDValue RHS = Op.getOperand(1);
1036 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1037 MVT ResTy = ty(Op);
1038 MVT OpTy = ty(LHS);
1039
1040 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
8
Taking false branch
1041 MVT ElemTy = OpTy.getVectorElementType();
1042 assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void (
0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1042, __extension__ __PRETTY_FUNCTION__))
;
1043 MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1044 OpTy.getVectorNumElements());
1045 return DAG.getSetCC(dl, ResTy,
1046 DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy),
1047 DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC);
1048 }
1049
1050 // Treat all other vector types as legal.
1051 if (ResTy.isVector())
9
Taking false branch
1052 return Op;
1053
1054 // Comparisons of short integers should use sign-extend, not zero-extend,
1055 // since we can represent small negative values in the compare instructions.
1056 // The LLVM default is to use zero-extend arbitrarily in these cases.
1057 auto isSExtFree = [this](SDValue N) {
1058 switch (N.getOpcode()) {
14
Calling 'SDValue::getOpcode'
1059 case ISD::TRUNCATE: {
1060 // A sign-extend of a truncate of a sign-extend is free.
1061 SDValue Op = N.getOperand(0);
1062 if (Op.getOpcode() != ISD::AssertSext)
1063 return false;
1064 EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT();
1065 unsigned ThisBW = ty(N).getSizeInBits();
1066 unsigned OrigBW = OrigTy.getSizeInBits();
1067 // The type that was sign-extended to get the AssertSext must be
1068 // narrower than the type of N (so that N has still the same value
1069 // as the original).
1070 return ThisBW >= OrigBW;
1071 }
1072 case ISD::LOAD:
1073 // We have sign-extended loads.
1074 return true;
1075 }
1076 return false;
1077 };
1078
1079 if (OpTy == MVT::i8 || OpTy == MVT::i16) {
10
Taking true branch
1080 ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS);
1081 bool IsNegative = C && C->getAPIntValue().isNegative();
11
Assuming 'C' is null
1082 if (IsNegative
11.1
'IsNegative' is false
11.1
'IsNegative' is false
|| isSExtFree(LHS) || isSExtFree(RHS))
12
The value of 'RHS' is assigned to 'N.Node'
13
Calling 'operator()'
1083 return DAG.getSetCC(dl, ResTy,
1084 DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32),
1085 DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC);
1086 }
1087
1088 return SDValue();
1089}
1090
1091SDValue
1092HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1093 SDValue PredOp = Op.getOperand(0);
1094 SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);
1095 MVT OpTy = ty(Op1);
1096 const SDLoc &dl(Op);
1097
1098 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1099 MVT ElemTy = OpTy.getVectorElementType();
1100 assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void (
0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1100, __extension__ __PRETTY_FUNCTION__))
;
1101 MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1102 OpTy.getVectorNumElements());
1103 // Generate (trunc (select (_, sext, sext))).
1104 return DAG.getSExtOrTrunc(
1105 DAG.getSelect(dl, WideTy, PredOp,
1106 DAG.getSExtOrTrunc(Op1, dl, WideTy),
1107 DAG.getSExtOrTrunc(Op2, dl, WideTy)),
1108 dl, OpTy);
1109 }
1110
1111 return SDValue();
1112}
1113
1114SDValue
1115HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1116 EVT ValTy = Op.getValueType();
1117 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op);
1118 Constant *CVal = nullptr;
1119 bool isVTi1Type = false;
1120 if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {
1121 if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1122 IRBuilder<> IRB(CV->getContext());
1123 SmallVector<Constant*, 128> NewConst;
1124 unsigned VecLen = CV->getNumOperands();
1125 assert(isPowerOf2_32(VecLen) &&(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize"
) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1126, __extension__ __PRETTY_FUNCTION__))
1126 "conversion only supported for pow2 VectorSize")(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize"
) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1126, __extension__ __PRETTY_FUNCTION__))
;
1127 for (unsigned i = 0; i < VecLen; ++i)
1128 NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue()));
1129
1130 CVal = ConstantVector::get(NewConst);
1131 isVTi1Type = true;
1132 }
1133 }
1134 Align Alignment = CPN->getAlign();
1135 bool IsPositionIndependent = isPositionIndependent();
1136 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1137
1138 unsigned Offset = 0;
1139 SDValue T;
1140 if (CPN->isMachineConstantPoolEntry())
1141 T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment,
1142 Offset, TF);
1143 else if (isVTi1Type)
1144 T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF);
1145 else
1146 T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset,
1147 TF);
1148
1149 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&(static_cast <bool> (cast<ConstantPoolSDNode>(T)->
getTargetFlags() == TF && "Inconsistent target flag encountered"
) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1150, __extension__ __PRETTY_FUNCTION__))
1150 "Inconsistent target flag encountered")(static_cast <bool> (cast<ConstantPoolSDNode>(T)->
getTargetFlags() == TF && "Inconsistent target flag encountered"
) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1150, __extension__ __PRETTY_FUNCTION__))
;
1151
1152 if (IsPositionIndependent)
1153 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T);
1154 return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T);
1155}
1156
1157SDValue
1158HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1159 EVT VT = Op.getValueType();
1160 int Idx = cast<JumpTableSDNode>(Op)->getIndex();
1161 if (isPositionIndependent()) {
1162 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
1163 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T);
1164 }
1165
1166 SDValue T = DAG.getTargetJumpTable(Idx, VT);
1167 return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T);
1168}
1169
1170SDValue
1171HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1172 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1173 MachineFunction &MF = DAG.getMachineFunction();
1174 MachineFrameInfo &MFI = MF.getFrameInfo();
1175 MFI.setReturnAddressIsTaken(true);
1176
1177 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1178 return SDValue();
1179
1180 EVT VT = Op.getValueType();
1181 SDLoc dl(Op);
1182 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1183 if (Depth) {
1184 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1185 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
1186 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
1187 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
1188 MachinePointerInfo());
1189 }
1190
1191 // Return LR, which contains the return address. Mark it an implicit live-in.
1192 unsigned Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32));
1193 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
1194}
1195
1196SDValue
1197HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1198 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1199 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1200 MFI.setFrameAddressIsTaken(true);
1201
1202 EVT VT = Op.getValueType();
1203 SDLoc dl(Op);
1204 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1205 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
1206 HRI.getFrameRegister(), VT);
1207 while (Depth--)
1208 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
1209 MachinePointerInfo());
1210 return FrameAddr;
1211}
1212
1213SDValue
1214HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1215 SDLoc dl(Op);
1216 return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0));
1217}
1218
1219SDValue
1220HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1221 SDLoc dl(Op);
1222 auto *GAN = cast<GlobalAddressSDNode>(Op);
1223 auto PtrVT = getPointerTy(DAG.getDataLayout());
1224 auto *GV = GAN->getGlobal();
1225 int64_t Offset = GAN->getOffset();
1226
1227 auto &HLOF = *HTM.getObjFileLowering();
1228 Reloc::Model RM = HTM.getRelocationModel();
1229
1230 if (RM == Reloc::Static) {
1231 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
1232 const GlobalObject *GO = GV->getBaseObject();
1233 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1234 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA);
1235 return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA);
1236 }
1237
1238 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
1239 if (UsePCRel) {
1240 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset,
1241 HexagonII::MO_PCREL);
1242 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA);
1243 }
1244
1245 // Use GOT index.
1246 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
1247 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT);
1248 SDValue Off = DAG.getConstant(Offset, dl, MVT::i32);
1249 return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off);
1250}
1251
1252// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1253SDValue
1254HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1255 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
1256 SDLoc dl(Op);
1257 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1258
1259 Reloc::Model RM = HTM.getRelocationModel();
1260 if (RM == Reloc::Static) {
1261 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT);
1262 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A);
1263 }
1264
1265 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL);
1266 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A);
1267}
1268
1269SDValue
1270HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1271 const {
1272 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1273 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME"_GLOBAL_OFFSET_TABLE_", PtrVT,
1274 HexagonII::MO_PCREL);
1275 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym);
1276}
1277
1278SDValue
1279HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1280 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1281 unsigned char OperandFlags) const {
1282 MachineFunction &MF = DAG.getMachineFunction();
1283 MachineFrameInfo &MFI = MF.getFrameInfo();
1284 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1285 SDLoc dl(GA);
1286 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
1287 GA->getValueType(0),
1288 GA->getOffset(),
1289 OperandFlags);
1290 // Create Operands for the call.The Operands should have the following:
1291 // 1. Chain SDValue
1292 // 2. Callee which in this case is the Global address value.
1293 // 3. Registers live into the call.In this case its R0, as we
1294 // have just one argument to be passed.
1295 // 4. Glue.
1296 // Note: The order is important.
1297
1298 const auto &HRI = *Subtarget.getRegisterInfo();
1299 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1300 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1300, __extension__ __PRETTY_FUNCTION__))
;
1301 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT),
1302 DAG.getRegisterMask(Mask), Glue };
1303 Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops);
1304
1305 // Inform MFI that function has calls.
1306 MFI.setAdjustsStack(true);
1307
1308 Glue = Chain.getValue(1);
1309 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
1310}
1311
1312//
1313// Lower using the intial executable model for TLS addresses
1314//
1315SDValue
1316HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1317 SelectionDAG &DAG) const {
1318 SDLoc dl(GA);
1319 int64_t Offset = GA->getOffset();
1320 auto PtrVT = getPointerTy(DAG.getDataLayout());
1321
1322 // Get the thread pointer.
1323 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1324
1325 bool IsPositionIndependent = isPositionIndependent();
1326 unsigned char TF =
1327 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1328
1329 // First generate the TLS symbol address
1330 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT,
1331 Offset, TF);
1332
1333 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1334
1335 if (IsPositionIndependent) {
1336 // Generate the GOT pointer in case of position independent code
1337 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG);
1338
1339 // Add the TLS Symbol address to GOT pointer.This gives
1340 // GOT relative relocation for the symbol.
1341 Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1342 }
1343
1344 // Load the offset value for TLS symbol.This offset is relative to
1345 // thread pointer.
1346 SDValue LoadOffset =
1347 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo());
1348
1349 // Address of the thread local variable is the add of thread
1350 // pointer and the offset of the variable.
1351 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset);
1352}
1353
1354//
1355// Lower using the local executable model for TLS addresses
1356//
1357SDValue
1358HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1359 SelectionDAG &DAG) const {
1360 SDLoc dl(GA);
1361 int64_t Offset = GA->getOffset();
1362 auto PtrVT = getPointerTy(DAG.getDataLayout());
1363
1364 // Get the thread pointer.
1365 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1366 // Generate the TLS symbol address
1367 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1368 HexagonII::MO_TPREL);
1369 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1370
1371 // Address of the thread local variable is the add of thread
1372 // pointer and the offset of the variable.
1373 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym);
1374}
1375
1376//
1377// Lower using the general dynamic model for TLS addresses
1378//
1379SDValue
1380HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1381 SelectionDAG &DAG) const {
1382 SDLoc dl(GA);
1383 int64_t Offset = GA->getOffset();
1384 auto PtrVT = getPointerTy(DAG.getDataLayout());
1385
1386 // First generate the TLS symbol address
1387 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1388 HexagonII::MO_GDGOT);
1389
1390 // Then, generate the GOT pointer
1391 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG);
1392
1393 // Add the TLS symbol and the GOT pointer
1394 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1395 SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1396
1397 // Copy over the argument to R0
1398 SDValue InFlag;
1399 Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag);
1400 InFlag = Chain.getValue(1);
1401
1402 unsigned Flags =
1403 static_cast<const HexagonSubtarget &>(DAG.getSubtarget()).useLongCalls()
1404 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1405 : HexagonII::MO_GDPLT;
1406
1407 return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT,
1408 Hexagon::R0, Flags);
1409}
1410
1411//
1412// Lower TLS addresses.
1413//
1414// For now for dynamic models, we only support the general dynamic model.
1415//
1416SDValue
1417HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1418 SelectionDAG &DAG) const {
1419 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
1420
1421 switch (HTM.getTLSModel(GA->getGlobal())) {
1422 case TLSModel::GeneralDynamic:
1423 case TLSModel::LocalDynamic:
1424 return LowerToTLSGeneralDynamicModel(GA, DAG);
1425 case TLSModel::InitialExec:
1426 return LowerToTLSInitialExecModel(GA, DAG);
1427 case TLSModel::LocalExec:
1428 return LowerToTLSLocalExecModel(GA, DAG);
1429 }
1430 llvm_unreachable("Bogus TLS model")::llvm::llvm_unreachable_internal("Bogus TLS model", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1430)
;
1431}
1432
1433//===----------------------------------------------------------------------===//
1434// TargetLowering Implementation
1435//===----------------------------------------------------------------------===//
1436
1437HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1438 const HexagonSubtarget &ST)
1439 : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)),
1440 Subtarget(ST) {
1441 auto &HRI = *Subtarget.getRegisterInfo();
1442
1443 setPrefLoopAlignment(Align(16));
1444 setMinFunctionAlignment(Align(4));
1445 setPrefFunctionAlignment(Align(16));
1446 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1447 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1448 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1449
1450 setMaxAtomicSizeInBitsSupported(64);
1451 setMinCmpXchgSizeInBits(32);
1452
1453 if (EnableHexSDNodeSched)
1454 setSchedulingPreference(Sched::VLIW);
1455 else
1456 setSchedulingPreference(Sched::Source);
1457
1458 // Limits for inline expansion of memcpy/memmove
1459 MaxStoresPerMemcpy = MaxStoresPerMemcpyCL;
1460 MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL;
1461 MaxStoresPerMemmove = MaxStoresPerMemmoveCL;
1462 MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL;
1463 MaxStoresPerMemset = MaxStoresPerMemsetCL;
1464 MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL;
1465
1466 //
1467 // Set up register classes.
1468 //
1469
1470 addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass);
1471 addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass); // bbbbaaaa
1472 addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass); // ddccbbaa
1473 addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass); // hgfedcba
1474 addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass);
1475 addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass);
1476 addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass);
1477 addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass);
1478 addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass);
1479 addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);
1480 addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);
1481
1482 addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass);
1483 addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass);
1484
1485 //
1486 // Handling of scalar operations.
1487 //
1488 // All operations default to "legal", except:
1489 // - indexed loads and stores (pre-/post-incremented),
1490 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1491 // ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1492 // FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP,
1493 // FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG,
1494 // which default to "expand" for at least one type.
1495
1496 // Misc operations.
1497 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
1498 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
1499 setOperationAction(ISD::TRAP, MVT::Other, Legal);
1500 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
1501 setOperationAction(ISD::JumpTable, MVT::i32, Custom);
1502 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
1503 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
1504 setOperationAction(ISD::INLINEASM, MVT::Other, Custom);
1505 setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom);
1506 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
1507 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
1508 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1509 setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
1510 setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom);
1511 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
1512 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
1513
1514 // Custom legalize GlobalAddress nodes into CONST32.
1515 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
1516 setOperationAction(ISD::GlobalAddress, MVT::i8, Custom);
1517 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
1518
1519 // Hexagon needs to optimize cases with negative constants.
1520 setOperationAction(ISD::SETCC, MVT::i8, Custom);
1521 setOperationAction(ISD::SETCC, MVT::i16, Custom);
1522 setOperationAction(ISD::SETCC, MVT::v4i8, Custom);
1523 setOperationAction(ISD::SETCC, MVT::v2i16, Custom);
1524
1525 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1526 setOperationAction(ISD::VASTART, MVT::Other, Custom);
1527 setOperationAction(ISD::VAEND, MVT::Other, Expand);
1528 setOperationAction(ISD::VAARG, MVT::Other, Expand);
1529 if (Subtarget.isEnvironmentMusl())
1530 setOperationAction(ISD::VACOPY, MVT::Other, Custom);
1531 else
1532 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
1533
1534 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
1535 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
1536 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
1537
1538 if (EmitJumpTables)
1539 setMinimumJumpTableEntries(MinimumJumpTables);
1540 else
1541 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1542 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
1543
1544 for (unsigned LegalIntOp :
1545 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1546 setOperationAction(LegalIntOp, MVT::i32, Legal);
1547 setOperationAction(LegalIntOp, MVT::i64, Legal);
1548 }
1549
1550 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1551 // but they only operate on i64.
1552 for (MVT VT : MVT::integer_valuetypes()) {
1553 setOperationAction(ISD::UADDO, VT, Custom);
1554 setOperationAction(ISD::USUBO, VT, Custom);
1555 setOperationAction(ISD::SADDO, VT, Expand);
1556 setOperationAction(ISD::SSUBO, VT, Expand);
1557 setOperationAction(ISD::ADDCARRY, VT, Expand);
1558 setOperationAction(ISD::SUBCARRY, VT, Expand);
1559 }
1560 setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
1561 setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
1562
1563 setOperationAction(ISD::CTLZ, MVT::i8, Promote);
1564 setOperationAction(ISD::CTLZ, MVT::i16, Promote);
1565 setOperationAction(ISD::CTTZ, MVT::i8, Promote);
1566 setOperationAction(ISD::CTTZ, MVT::i16, Promote);
1567
1568 // Popcount can count # of 1s in i64 but returns i32.
1569 setOperationAction(ISD::CTPOP, MVT::i8, Promote);
1570 setOperationAction(ISD::CTPOP, MVT::i16, Promote);
1571 setOperationAction(ISD::CTPOP, MVT::i32, Promote);
1572 setOperationAction(ISD::CTPOP, MVT::i64, Legal);
1573
1574 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
1575 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
1576 setOperationAction(ISD::BSWAP, MVT::i32, Legal);
1577 setOperationAction(ISD::BSWAP, MVT::i64, Legal);
1578
1579 setOperationAction(ISD::FSHL, MVT::i32, Legal);
1580 setOperationAction(ISD::FSHL, MVT::i64, Legal);
1581 setOperationAction(ISD::FSHR, MVT::i32, Legal);
1582 setOperationAction(ISD::FSHR, MVT::i64, Legal);
1583
1584 for (unsigned IntExpOp :
1585 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1586 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,
1587 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1588 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1589 for (MVT VT : MVT::integer_valuetypes())
1590 setOperationAction(IntExpOp, VT, Expand);
1591 }
1592
1593 for (unsigned FPExpOp :
1594 {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
1595 ISD::FPOW, ISD::FCOPYSIGN}) {
1596 for (MVT VT : MVT::fp_valuetypes())
1597 setOperationAction(FPExpOp, VT, Expand);
1598 }
1599
1600 // No extending loads from i32.
1601 for (MVT VT : MVT::integer_valuetypes()) {
1602 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
1603 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
1604 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);
1605 }
1606 // Turn FP truncstore into trunc + store.
1607 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
1608 // Turn FP extload into load/fpextend.
1609 for (MVT VT : MVT::fp_valuetypes())
1610 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1611
1612 // Expand BR_CC and SELECT_CC for all integer and fp types.
1613 for (MVT VT : MVT::integer_valuetypes()) {
1614 setOperationAction(ISD::BR_CC, VT, Expand);
1615 setOperationAction(ISD::SELECT_CC, VT, Expand);
1616 }
1617 for (MVT VT : MVT::fp_valuetypes()) {
1618 setOperationAction(ISD::BR_CC, VT, Expand);
1619 setOperationAction(ISD::SELECT_CC, VT, Expand);
1620 }
1621 setOperationAction(ISD::BR_CC, MVT::Other, Expand);
1622
1623 //
1624 // Handling of vector operations.
1625 //
1626
1627 // Set the action for vector operations to "expand", then override it with
1628 // either "custom" or "legal" for specific cases.
1629 static const unsigned VectExpOps[] = {
1630 // Integer arithmetic:
1631 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,
1632 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,
1633 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1634 // Logical/bit:
1635 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,
1636 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ,
1637 // Floating point arithmetic/math functions:
1638 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,
1639 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,
1640 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1641 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
1642 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
1643 ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS,
1644 // Misc:
1645 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
1646 // Vector:
1647 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,
1648 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,
1649 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1650 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,
1651 ISD::SPLAT_VECTOR,
1652 };
1653
1654 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1655 for (unsigned VectExpOp : VectExpOps)
1656 setOperationAction(VectExpOp, VT, Expand);
1657
1658 // Expand all extending loads and truncating stores:
1659 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1660 if (TargetVT == VT)
1661 continue;
1662 setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);
1663 setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand);
1664 setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand);
1665 setTruncStoreAction(VT, TargetVT, Expand);
1666 }
1667
1668 // Normalize all inputs to SELECT to be vectors of i32.
1669 if (VT.getVectorElementType() != MVT::i32) {
1670 MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);
1671 setOperationAction(ISD::SELECT, VT, Promote);
1672 AddPromotedToType(ISD::SELECT, VT, VT32);
1673 }
1674 setOperationAction(ISD::SRA, VT, Custom);
1675 setOperationAction(ISD::SHL, VT, Custom);
1676 setOperationAction(ISD::SRL, VT, Custom);
1677 }
1678
1679 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1680 // are legal.
1681 setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1682 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1683 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1684 setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1685 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1686 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1687
1688 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal);
1689 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal);
1690 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);
1691
1692 // Types natively supported:
1693 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1694 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1695 setOperationAction(ISD::BUILD_VECTOR, NativeVT, Custom);
1696 setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom);
1697 setOperationAction(ISD::INSERT_VECTOR_ELT, NativeVT, Custom);
1698 setOperationAction(ISD::EXTRACT_SUBVECTOR, NativeVT, Custom);
1699 setOperationAction(ISD::INSERT_SUBVECTOR, NativeVT, Custom);
1700 setOperationAction(ISD::CONCAT_VECTORS, NativeVT, Custom);
1701
1702 setOperationAction(ISD::ADD, NativeVT, Legal);
1703 setOperationAction(ISD::SUB, NativeVT, Legal);
1704 setOperationAction(ISD::MUL, NativeVT, Legal);
1705 setOperationAction(ISD::AND, NativeVT, Legal);
1706 setOperationAction(ISD::OR, NativeVT, Legal);
1707 setOperationAction(ISD::XOR, NativeVT, Legal);
1708
1709 if (NativeVT.getVectorElementType() != MVT::i1)
1710 setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal);
1711 }
1712
1713 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1714 setOperationAction(ISD::SMIN, VT, Legal);
1715 setOperationAction(ISD::SMAX, VT, Legal);
1716 setOperationAction(ISD::UMIN, VT, Legal);
1717 setOperationAction(ISD::UMAX, VT, Legal);
1718 }
1719
1720 // Custom lower unaligned loads.
1721 // Also, for both loads and stores, verify the alignment of the address
1722 // in case it is a compile-time constant. This is a usability feature to
1723 // provide a meaningful error message to users.
1724 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1725 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1726 setOperationAction(ISD::LOAD, VT, Custom);
1727 setOperationAction(ISD::STORE, VT, Custom);
1728 }
1729
1730 // Custom-lower load/stores of boolean vectors.
1731 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1732 setOperationAction(ISD::LOAD, VT, Custom);
1733 setOperationAction(ISD::STORE, VT, Custom);
1734 }
1735
1736 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1737 MVT::v2i32}) {
1738 setCondCodeAction(ISD::SETNE, VT, Expand);
1739 setCondCodeAction(ISD::SETLE, VT, Expand);
1740 setCondCodeAction(ISD::SETGE, VT, Expand);
1741 setCondCodeAction(ISD::SETLT, VT, Expand);
1742 setCondCodeAction(ISD::SETULE, VT, Expand);
1743 setCondCodeAction(ISD::SETUGE, VT, Expand);
1744 setCondCodeAction(ISD::SETULT, VT, Expand);
1745 }
1746
1747 // Custom-lower bitcasts from i8 to v8i1.
1748 setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1749 setOperationAction(ISD::SETCC, MVT::v2i16, Custom);
1750 setOperationAction(ISD::VSELECT, MVT::v4i8, Custom);
1751 setOperationAction(ISD::VSELECT, MVT::v2i16, Custom);
1752 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom);
1753 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
1754 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom);
1755
1756 // V5+.
1757 setOperationAction(ISD::FMA, MVT::f64, Expand);
1758 setOperationAction(ISD::FADD, MVT::f64, Expand);
1759 setOperationAction(ISD::FSUB, MVT::f64, Expand);
1760 setOperationAction(ISD::FMUL, MVT::f64, Expand);
1761
1762 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
1763 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
1764
1765 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);
1766 setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
1767 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
1768 setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote);
1769 setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
1770 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
1771 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);
1772 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
1773 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
1774 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote);
1775 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
1776 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
1777
1778 // Handling of indexed loads/stores: default is "expand".
1779 //
1780 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1781 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1782 setIndexedLoadAction(ISD::POST_INC, VT, Legal);
1783 setIndexedStoreAction(ISD::POST_INC, VT, Legal);
1784 }
1785
1786 // Subtarget-specific operation actions.
1787 //
1788 if (Subtarget.hasV60Ops()) {
1789 setOperationAction(ISD::ROTL, MVT::i32, Legal);
1790 setOperationAction(ISD::ROTL, MVT::i64, Legal);
1791 setOperationAction(ISD::ROTR, MVT::i32, Legal);
1792 setOperationAction(ISD::ROTR, MVT::i64, Legal);
1793 }
1794 if (Subtarget.hasV66Ops()) {
1795 setOperationAction(ISD::FADD, MVT::f64, Legal);
1796 setOperationAction(ISD::FSUB, MVT::f64, Legal);
1797 }
1798 if (Subtarget.hasV67Ops()) {
1799 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
1800 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
1801 setOperationAction(ISD::FMUL, MVT::f64, Legal);
1802 }
1803
1804 setTargetDAGCombine(ISD::VSELECT);
1805
1806 if (Subtarget.useHVXOps())
1807 initializeHVXLowering();
1808
1809 computeRegisterProperties(&HRI);
1810
1811 //
1812 // Library calls for unsupported operations
1813 //
1814 bool FastMath = EnableFastMath;
1815
1816 setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
1817 setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
1818 setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
1819 setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
1820 setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
1821 setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
1822 setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
1823 setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
1824
1825 setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf");
1826 setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf");
1827 setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti");
1828 setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti");
1829 setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti");
1830 setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti");
1831
1832 // This is the only fast library function for sqrtd.
1833 if (FastMath)
1834 setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
1835
1836 // Prefix is: nothing for "slow-math",
1837 // "fast2_" for V5+ fast-math double-precision
1838 // (actually, keep fast-math and fast-math2 separate for now)
1839 if (FastMath) {
1840 setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
1841 setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
1842 setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
1843 setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
1844 setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
1845 } else {
1846 setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
1847 setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
1848 setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
1849 setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
1850 setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
1851 }
1852
1853 if (FastMath)
1854 setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
1855 else
1856 setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
1857
1858 // These cause problems when the shift amount is non-constant.
1859 setLibcallName(RTLIB::SHL_I128, nullptr);
1860 setLibcallName(RTLIB::SRL_I128, nullptr);
1861 setLibcallName(RTLIB::SRA_I128, nullptr);
1862}
1863
1864const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
1865 switch ((HexagonISD::NodeType)Opcode) {
1866 case HexagonISD::ADDC: return "HexagonISD::ADDC";
1867 case HexagonISD::SUBC: return "HexagonISD::SUBC";
1868 case HexagonISD::ALLOCA: return "HexagonISD::ALLOCA";
1869 case HexagonISD::AT_GOT: return "HexagonISD::AT_GOT";
1870 case HexagonISD::AT_PCREL: return "HexagonISD::AT_PCREL";
1871 case HexagonISD::BARRIER: return "HexagonISD::BARRIER";
1872 case HexagonISD::CALL: return "HexagonISD::CALL";
1873 case HexagonISD::CALLnr: return "HexagonISD::CALLnr";
1874 case HexagonISD::CALLR: return "HexagonISD::CALLR";
1875 case HexagonISD::COMBINE: return "HexagonISD::COMBINE";
1876 case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP";
1877 case HexagonISD::CONST32: return "HexagonISD::CONST32";
1878 case HexagonISD::CP: return "HexagonISD::CP";
1879 case HexagonISD::DCFETCH: return "HexagonISD::DCFETCH";
1880 case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN";
1881 case HexagonISD::TSTBIT: return "HexagonISD::TSTBIT";
1882 case HexagonISD::EXTRACTU: return "HexagonISD::EXTRACTU";
1883 case HexagonISD::INSERT: return "HexagonISD::INSERT";
1884 case HexagonISD::JT: return "HexagonISD::JT";
1885 case HexagonISD::RET_FLAG: return "HexagonISD::RET_FLAG";
1886 case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN";
1887 case HexagonISD::VASL: return "HexagonISD::VASL";
1888 case HexagonISD::VASR: return "HexagonISD::VASR";
1889 case HexagonISD::VLSR: return "HexagonISD::VLSR";
1890 case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW";
1891 case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0";
1892 case HexagonISD::VROR: return "HexagonISD::VROR";
1893 case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE";
1894 case HexagonISD::PTRUE: return "HexagonISD::PTRUE";
1895 case HexagonISD::PFALSE: return "HexagonISD::PFALSE";
1896 case HexagonISD::D2P: return "HexagonISD::D2P";
1897 case HexagonISD::P2D: return "HexagonISD::P2D";
1898 case HexagonISD::V2Q: return "HexagonISD::V2Q";
1899 case HexagonISD::Q2V: return "HexagonISD::Q2V";
1900 case HexagonISD::QCAT: return "HexagonISD::QCAT";
1901 case HexagonISD::QTRUE: return "HexagonISD::QTRUE";
1902 case HexagonISD::QFALSE: return "HexagonISD::QFALSE";
1903 case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST";
1904 case HexagonISD::VALIGN: return "HexagonISD::VALIGN";
1905 case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR";
1906 case HexagonISD::VPACKL: return "HexagonISD::VPACKL";
1907 case HexagonISD::VUNPACK: return "HexagonISD::VUNPACK";
1908 case HexagonISD::VUNPACKU: return "HexagonISD::VUNPACKU";
1909 case HexagonISD::ISEL: return "HexagonISD::ISEL";
1910 case HexagonISD::OP_END: break;
1911 }
1912 return nullptr;
1913}
1914
1915void
1916HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl,
1917 unsigned NeedAlign) const {
1918 auto *CA = dyn_cast<ConstantSDNode>(Ptr);
1919 if (!CA)
1920 return;
1921 unsigned Addr = CA->getZExtValue();
1922 unsigned HaveAlign = Addr != 0 ? 1u << countTrailingZeros(Addr) : NeedAlign;
1923 if (HaveAlign < NeedAlign) {
1924 std::string ErrMsg;
1925 raw_string_ostream O(ErrMsg);
1926 O << "Misaligned constant address: " << format_hex(Addr, 10)
1927 << " has alignment " << HaveAlign
1928 << ", but the memory access requires " << NeedAlign;
1929 if (DebugLoc DL = dl.getDebugLoc())
1930 DL.print(O << ", at ");
1931 report_fatal_error(O.str());
1932 }
1933}
1934
1935// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1936// intrinsic.
1937static bool isBrevLdIntrinsic(const Value *Inst) {
1938 unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1939 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1940 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1941 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1942 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1943 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1944 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1945}
1946
1947// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
1948// instruction. So far we only handle bitcast, extract value and bit reverse
1949// load intrinsic instructions. Should we handle CGEP ?
1950static Value *getBrevLdObject(Value *V) {
1951 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
1952 Operator::getOpcode(V) == Instruction::BitCast)
1953 V = cast<Operator>(V)->getOperand(0);
1954 else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V))
1955 V = cast<Instruction>(V)->getOperand(0);
1956 return V;
1957}
1958
1959// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
1960// a back edge. If the back edge comes from the intrinsic itself, the incoming
1961// edge is returned.
1962static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
1963 const BasicBlock *Parent = PN->getParent();
1964 int Idx = -1;
1965 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
1966 BasicBlock *Blk = PN->getIncomingBlock(i);
1967 // Determine if the back edge is originated from intrinsic.
1968 if (Blk == Parent) {
1969 Value *BackEdgeVal = PN->getIncomingValue(i);
1970 Value *BaseVal;
1971 // Loop over till we return the same Value or we hit the IntrBaseVal.
1972 do {
1973 BaseVal = BackEdgeVal;
1974 BackEdgeVal = getBrevLdObject(BackEdgeVal);
1975 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
1976 // If the getBrevLdObject returns IntrBaseVal, we should return the
1977 // incoming edge.
1978 if (IntrBaseVal == BackEdgeVal)
1979 continue;
1980 Idx = i;
1981 break;
1982 } else // Set the node to incoming edge.
1983 Idx = i;
1984 }
1985 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI")(static_cast <bool> (Idx >= 0 && "Unexpected index to incoming argument in PHI"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Unexpected index to incoming argument in PHI\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1985, __extension__ __PRETTY_FUNCTION__))
;
1986 return PN->getIncomingValue(Idx);
1987}
1988
1989// Bit-reverse Load Intrinsic: Figure out the underlying object the base
1990// pointer points to, for the bit-reverse load intrinsic. Setting this to
1991// memoperand might help alias analysis to figure out the dependencies.
1992static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
1993 Value *IntrBaseVal = V;
1994 Value *BaseVal;
1995 // Loop over till we return the same Value, implies we either figure out
1996 // the object or we hit a PHI
1997 do {
1998 BaseVal = V;
1999 V = getBrevLdObject(V);
2000 } while (BaseVal != V);
2001
2002 // Identify the object from PHINode.
2003 if (const PHINode *PN = dyn_cast<PHINode>(V))
2004 return returnEdge(PN, IntrBaseVal);
2005 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2006 else
2007 return V;
2008}
2009
2010/// Given an intrinsic, checks if on the target the intrinsic will need to map
2011/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
2012/// true and store the intrinsic information into the IntrinsicInfo that was
2013/// passed to the function.
2014bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2015 const CallInst &I,
2016 MachineFunction &MF,
2017 unsigned Intrinsic) const {
2018 switch (Intrinsic) {
2019 case Intrinsic::hexagon_L2_loadrd_pbr:
2020 case Intrinsic::hexagon_L2_loadri_pbr:
2021 case Intrinsic::hexagon_L2_loadrh_pbr:
2022 case Intrinsic::hexagon_L2_loadruh_pbr:
2023 case Intrinsic::hexagon_L2_loadrb_pbr:
2024 case Intrinsic::hexagon_L2_loadrub_pbr: {
2025 Info.opc = ISD::INTRINSIC_W_CHAIN;
2026 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
2027 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2028 // The intrinsic function call is of the form { ElTy, i8* }
2029 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2030 // should be derived from ElTy.
2031 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0);
2032 Info.memVT = MVT::getVT(ElTy);
2033 llvm::Value *BasePtrVal = I.getOperand(0);
2034 Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal);
2035 // The offset value comes through Modifier register. For now, assume the
2036 // offset is 0.
2037 Info.offset = 0;
2038 Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont));
2039 Info.flags = MachineMemOperand::MOLoad;
2040 return true;
2041 }
2042 case Intrinsic::hexagon_V6_vgathermw:
2043 case Intrinsic::hexagon_V6_vgathermw_128B:
2044 case Intrinsic::hexagon_V6_vgathermh:
2045 case Intrinsic::hexagon_V6_vgathermh_128B:
2046 case Intrinsic::hexagon_V6_vgathermhw:
2047 case Intrinsic::hexagon_V6_vgathermhw_128B:
2048 case Intrinsic::hexagon_V6_vgathermwq:
2049 case Intrinsic::hexagon_V6_vgathermwq_128B:
2050 case Intrinsic::hexagon_V6_vgathermhq:
2051 case Intrinsic::hexagon_V6_vgathermhq_128B:
2052 case Intrinsic::hexagon_V6_vgathermhwq:
2053 case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2054 const Module &M = *I.getParent()->getParent()->getParent();
2055 Info.opc = ISD::INTRINSIC_W_CHAIN;
2056 Type *VecTy = I.getArgOperand(1)->getType();
2057 Info.memVT = MVT::getVT(VecTy);
2058 Info.ptrVal = I.getArgOperand(0);
2059 Info.offset = 0;
2060 Info.align =
2061 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2062 Info.flags = MachineMemOperand::MOLoad |
2063 MachineMemOperand::MOStore |
2064 MachineMemOperand::MOVolatile;
2065 return true;
2066 }
2067 default:
2068 break;
2069 }
2070 return false;
2071}
2072
2073bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2074 return X.getValueType().isScalarInteger(); // 'tstbit'
2075}
2076
2077bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2078 return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2));
2079}
2080
2081bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2082 if (!VT1.isSimple() || !VT2.isSimple())
2083 return false;
2084 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2085}
2086
2087bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2088 const MachineFunction &MF, EVT VT) const {
2089 return isOperationLegalOrCustom(ISD::FMA, VT);
2090}
2091
2092// Should we expand the build vector with shuffles?
2093bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2094 unsigned DefinedValues) const {
2095 return false;
2096}
2097
2098bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2099 EVT VT) const {
2100 return true;
2101}
2102
2103TargetLoweringBase::LegalizeTypeAction
2104HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2105 unsigned VecLen = VT.getVectorMinNumElements();
2106 MVT ElemTy = VT.getVectorElementType();
2107
2108 if (VecLen == 1 || VT.isScalableVector())
2109 return TargetLoweringBase::TypeScalarizeVector;
2110
2111 if (Subtarget.useHVXOps()) {
2112 unsigned Action = getPreferredHvxVectorAction(VT);
2113 if (Action != ~0u)
2114 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2115 }
2116
2117 // Always widen (remaining) vectors of i1.
2118 if (ElemTy == MVT::i1)
2119 return TargetLoweringBase::TypeWidenVector;
2120
2121 return TargetLoweringBase::TypeSplitVector;
2122}
2123
2124std::pair<SDValue, int>
2125HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2126 if (Addr.getOpcode() == ISD::ADD) {
2127 SDValue Op1 = Addr.getOperand(1);
2128 if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode()))
2129 return { Addr.getOperand(0), CN->getSExtValue() };
2130 }
2131 return { Addr, 0 };
2132}
2133
2134// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2135// to select data from, V3 is the permutation.
2136SDValue
2137HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2138 const {
2139 const auto *SVN = cast<ShuffleVectorSDNode>(Op);
2140 ArrayRef<int> AM = SVN->getMask();
2141 assert(AM.size() <= 8 && "Unexpected shuffle mask")(static_cast <bool> (AM.size() <= 8 && "Unexpected shuffle mask"
) ? void (0) : __assert_fail ("AM.size() <= 8 && \"Unexpected shuffle mask\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2141, __extension__ __PRETTY_FUNCTION__))
;
2142 unsigned VecLen = AM.size();
2143
2144 MVT VecTy = ty(Op);
2145 assert(!Subtarget.isHVXVectorType(VecTy, true) &&(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true
) && "HVX shuffles should be legal") ? void (0) : __assert_fail
("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2146, __extension__ __PRETTY_FUNCTION__))
2146 "HVX shuffles should be legal")(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true
) && "HVX shuffles should be legal") ? void (0) : __assert_fail
("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2146, __extension__ __PRETTY_FUNCTION__))
;
2147 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length")(static_cast <bool> (VecTy.getSizeInBits() <= 64 &&
"Unexpected vector length") ? void (0) : __assert_fail ("VecTy.getSizeInBits() <= 64 && \"Unexpected vector length\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2147, __extension__ __PRETTY_FUNCTION__))
;
2148
2149 SDValue Op0 = Op.getOperand(0);
2150 SDValue Op1 = Op.getOperand(1);
2151 const SDLoc &dl(Op);
2152
2153 // If the inputs are not the same as the output, bail. This is not an
2154 // error situation, but complicates the handling and the default expansion
2155 // (into BUILD_VECTOR) should be adequate.
2156 if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2157 return SDValue();
2158
2159 // Normalize the mask so that the first non-negative index comes from
2160 // the first operand.
2161 SmallVector<int,8> Mask(AM.begin(), AM.end());
2162 unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data();
2163 if (F == AM.size())
2164 return DAG.getUNDEF(VecTy);
2165 if (AM[F] >= int(VecLen)) {
2166 ShuffleVectorSDNode::commuteMask(Mask);
2167 std::swap(Op0, Op1);
2168 }
2169
2170 // Express the shuffle mask in terms of bytes.
2171 SmallVector<int,8> ByteMask;
2172 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2173 for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
2174 int M = Mask[i];
2175 if (M < 0) {
2176 for (unsigned j = 0; j != ElemBytes; ++j)
2177 ByteMask.push_back(-1);
2178 } else {
2179 for (unsigned j = 0; j != ElemBytes; ++j)
2180 ByteMask.push_back(M*ElemBytes + j);
2181 }
2182 }
2183 assert(ByteMask.size() <= 8)(static_cast <bool> (ByteMask.size() <= 8) ? void (0
) : __assert_fail ("ByteMask.size() <= 8", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2183, __extension__ __PRETTY_FUNCTION__))
;
2184
2185 // All non-undef (non-negative) indexes are well within [0..127], so they
2186 // fit in a single byte. Build two 64-bit words:
2187 // - MaskIdx where each byte is the corresponding index (for non-negative
2188 // indexes), and 0xFF for negative indexes, and
2189 // - MaskUnd that has 0xFF for each negative index.
2190 uint64_t MaskIdx = 0;
2191 uint64_t MaskUnd = 0;
2192 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2193 unsigned S = 8*i;
2194 uint64_t M = ByteMask[i] & 0xFF;
2195 if (M == 0xFF)
2196 MaskUnd |= M << S;
2197 MaskIdx |= M << S;
2198 }
2199
2200 if (ByteMask.size() == 4) {
2201 // Identity.
2202 if (MaskIdx == (0x03020100 | MaskUnd))
2203 return Op0;
2204 // Byte swap.
2205 if (MaskIdx == (0x00010203 | MaskUnd)) {
2206 SDValue T0 = DAG.getBitcast(MVT::i32, Op0);
2207 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0);
2208 return DAG.getBitcast(VecTy, T1);
2209 }
2210
2211 // Byte packs.
2212 SDValue Concat10 = DAG.getNode(HexagonISD::COMBINE, dl,
2213 typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0});
2214 if (MaskIdx == (0x06040200 | MaskUnd))
2215 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2216 if (MaskIdx == (0x07050301 | MaskUnd))
2217 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2218
2219 SDValue Concat01 = DAG.getNode(HexagonISD::COMBINE, dl,
2220 typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1});
2221 if (MaskIdx == (0x02000604 | MaskUnd))
2222 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2223 if (MaskIdx == (0x03010705 | MaskUnd))
2224 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2225 }
2226
2227 if (ByteMask.size() == 8) {
2228 // Identity.
2229 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2230 return Op0;
2231 // Byte swap.
2232 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2233 SDValue T0 = DAG.getBitcast(MVT::i64, Op0);
2234 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0);
2235 return DAG.getBitcast(VecTy, T1);
2236 }
2237
2238 // Halfword picks.
2239 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2240 return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2241 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2242 return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2243 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2244 return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2245 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2246 return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2247 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2248 VectorPair P = opSplit(Op0, dl, DAG);
2249 return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG);
2250 }
2251
2252 // Byte packs.
2253 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2254 return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2255 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2256 return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2257 }
2258
2259 return SDValue();
2260}
2261
2262// Create a Hexagon-specific node for shifting a vector by an integer.
2263SDValue
2264HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2265 const {
2266 unsigned NewOpc;
2267 switch (Op.getOpcode()) {
2268 case ISD::SHL:
2269 NewOpc = HexagonISD::VASL;
2270 break;
2271 case ISD::SRA:
2272 NewOpc = HexagonISD::VASR;
2273 break;
2274 case ISD::SRL:
2275 NewOpc = HexagonISD::VLSR;
2276 break;
2277 default:
2278 llvm_unreachable("Unexpected shift opcode")::llvm::llvm_unreachable_internal("Unexpected shift opcode", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2278)
;
2279 }
2280
2281 SDValue Op0 = Op.getOperand(0);
2282 SDValue Op1 = Op.getOperand(1);
2283 const SDLoc &dl(Op);
2284
2285 switch (Op1.getOpcode()) {
2286 case ISD::BUILD_VECTOR:
2287 if (SDValue S = cast<BuildVectorSDNode>(Op1)->getSplatValue())
2288 return DAG.getNode(NewOpc, dl, ty(Op), Op0, S);
2289 break;
2290 case ISD::SPLAT_VECTOR:
2291 return DAG.getNode(NewOpc, dl, ty(Op), Op0, Op1.getOperand(0));
2292 }
2293 return SDValue();
2294}
2295
2296SDValue
2297HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2298 return getVectorShiftByInt(Op, DAG);
2299}
2300
2301SDValue
2302HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2303 if (isa<ConstantSDNode>(Op.getOperand(1).getNode()))
2304 return Op;
2305 return SDValue();
2306}
2307
2308SDValue
2309HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2310 MVT ResTy = ty(Op);
2311 SDValue InpV = Op.getOperand(0);
2312 MVT InpTy = ty(InpV);
2313 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits())(static_cast <bool> (ResTy.getSizeInBits() == InpTy.getSizeInBits
()) ? void (0) : __assert_fail ("ResTy.getSizeInBits() == InpTy.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2313, __extension__ __PRETTY_FUNCTION__))
;
2314 const SDLoc &dl(Op);
2315
2316 // Handle conversion from i8 to v8i1.
2317 if (InpTy == MVT::i8) {
2318 if (ResTy == MVT::v8i1) {
2319 SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV);
2320 SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32);
2321 return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG);
2322 }
2323 return SDValue();
2324 }
2325
2326 return Op;
2327}
2328
2329bool
2330HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2331 MVT VecTy, SelectionDAG &DAG,
2332 MutableArrayRef<ConstantInt*> Consts) const {
2333 MVT ElemTy = VecTy.getVectorElementType();
2334 unsigned ElemWidth = ElemTy.getSizeInBits();
2335 IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth);
2336 bool AllConst = true;
2337
2338 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2339 SDValue V = Values[i];
2340 if (V.isUndef()) {
2341 Consts[i] = ConstantInt::get(IntTy, 0);
2342 continue;
2343 }
2344 // Make sure to always cast to IntTy.
2345 if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) {
2346 const ConstantInt *CI = CN->getConstantIntValue();
2347 Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue());
2348 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) {
2349 const ConstantFP *CF = CN->getConstantFPValue();
2350 APInt A = CF->getValueAPF().bitcastToAPInt();
2351 Consts[i] = ConstantInt::get(IntTy, A.getZExtValue());
2352 } else {
2353 AllConst = false;
2354 }
2355 }
2356 return AllConst;
2357}
2358
2359SDValue
2360HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2361 MVT VecTy, SelectionDAG &DAG) const {
2362 MVT ElemTy = VecTy.getVectorElementType();
2363 assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem
.size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2363, __extension__ __PRETTY_FUNCTION__))
;
2364
2365 SmallVector<ConstantInt*,4> Consts(Elem.size());
2366 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2367
2368 unsigned First, Num = Elem.size();
2369 for (First = 0; First != Num; ++First) {
2370 if (!isUndef(Elem[First]))
2371 break;
2372 }
2373 if (First == Num)
2374 return DAG.getUNDEF(VecTy);
2375
2376 if (AllConst &&
2377 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2378 return getZero(dl, VecTy, DAG);
2379
2380 if (ElemTy == MVT::i16) {
2381 assert(Elem.size() == 2)(static_cast <bool> (Elem.size() == 2) ? void (0) : __assert_fail
("Elem.size() == 2", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2381, __extension__ __PRETTY_FUNCTION__))
;
2382 if (AllConst) {
2383 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2384 Consts[1]->getZExtValue() << 16;
2385 return DAG.getBitcast(MVT::v2i16, DAG.getConstant(V, dl, MVT::i32));
2386 }
2387 SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32,
2388 {Elem[1], Elem[0]}, DAG);
2389 return DAG.getBitcast(MVT::v2i16, N);
2390 }
2391
2392 if (ElemTy == MVT::i8) {
2393 // First try generating a constant.
2394 if (AllConst) {
2395 int32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2396 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2397 (Consts[1]->getZExtValue() & 0xFF) << 16 |
2398 Consts[2]->getZExtValue() << 24;
2399 return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32));
2400 }
2401
2402 // Then try splat.
2403 bool IsSplat = true;
2404 for (unsigned i = First+1; i != Num; ++i) {
2405 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2406 continue;
2407 IsSplat = false;
2408 break;
2409 }
2410 if (IsSplat) {
2411 // Legalize the operand of SPLAT_VECTOR.
2412 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2413 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2414 }
2415
2416 // Generate
2417 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2418 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2419 assert(Elem.size() == 4)(static_cast <bool> (Elem.size() == 4) ? void (0) : __assert_fail
("Elem.size() == 4", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2419, __extension__ __PRETTY_FUNCTION__))
;
2420 SDValue Vs[4];
2421 for (unsigned i = 0; i != 4; ++i) {
2422 Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32);
2423 Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8);
2424 }
2425 SDValue S8 = DAG.getConstant(8, dl, MVT::i32);
2426 SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8});
2427 SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8});
2428 SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0});
2429 SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1});
2430
2431 SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG);
2432 return DAG.getBitcast(MVT::v4i8, R);
2433 }
2434
2435#ifndef NDEBUG
2436 dbgs() << "VecTy: " << EVT(VecTy).getEVTString() << '\n';
2437#endif
2438 llvm_unreachable("Unexpected vector element type")::llvm::llvm_unreachable_internal("Unexpected vector element type"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2438)
;
2439}
2440
2441SDValue
2442HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2443 MVT VecTy, SelectionDAG &DAG) const {
2444 MVT ElemTy = VecTy.getVectorElementType();
2445 assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem
.size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2445, __extension__ __PRETTY_FUNCTION__))
;
2446
2447 SmallVector<ConstantInt*,8> Consts(Elem.size());
2448 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2449
2450 unsigned First, Num = Elem.size();
2451 for (First = 0; First != Num; ++First) {
2452 if (!isUndef(Elem[First]))
2453 break;
2454 }
2455 if (First == Num)
2456 return DAG.getUNDEF(VecTy);
2457
2458 if (AllConst &&
2459 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2460 return getZero(dl, VecTy, DAG);
2461
2462 // First try splat if possible.
2463 if (ElemTy == MVT::i16) {
2464 bool IsSplat = true;
2465 for (unsigned i = First+1; i != Num; ++i) {
2466 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2467 continue;
2468 IsSplat = false;
2469 break;
2470 }
2471 if (IsSplat) {
2472 // Legalize the operand of SPLAT_VECTOR
2473 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2474 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2475 }
2476 }
2477
2478 // Then try constant.
2479 if (AllConst) {
2480 uint64_t Val = 0;
2481 unsigned W = ElemTy.getSizeInBits();
2482 uint64_t Mask = (ElemTy == MVT::i8) ? 0xFFull
2483 : (ElemTy == MVT::i16) ? 0xFFFFull : 0xFFFFFFFFull;
2484 for (unsigned i = 0; i != Num; ++i)
2485 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2486 SDValue V0 = DAG.getConstant(Val, dl, MVT::i64);
2487 return DAG.getBitcast(VecTy, V0);
2488 }
2489
2490 // Build two 32-bit vectors and concatenate.
2491 MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2);
2492 SDValue L = (ElemTy == MVT::i32)
2493 ? Elem[0]
2494 : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG);
2495 SDValue H = (ElemTy == MVT::i32)
2496 ? Elem[1]
2497 : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG);
2498 return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, {H, L});
2499}
2500
2501SDValue
2502HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2503 const SDLoc &dl, MVT ValTy, MVT ResTy,
2504 SelectionDAG &DAG) const {
2505 MVT VecTy = ty(VecV);
2506 assert(!ValTy.isVector() ||(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType
() == ValTy.getVectorElementType()) ? void (0) : __assert_fail
("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2507, __extension__ __PRETTY_FUNCTION__))
2507 VecTy.getVectorElementType() == ValTy.getVectorElementType())(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType
() == ValTy.getVectorElementType()) ? void (0) : __assert_fail
("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2507, __extension__ __PRETTY_FUNCTION__))
;
2508 unsigned VecWidth = VecTy.getSizeInBits();
2509 unsigned ValWidth = ValTy.getSizeInBits();
2510 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2511 assert((VecWidth % ElemWidth) == 0)(static_cast <bool> ((VecWidth % ElemWidth) == 0) ? void
(0) : __assert_fail ("(VecWidth % ElemWidth) == 0", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2511, __extension__ __PRETTY_FUNCTION__))
;
2512 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV);
2513
2514 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2515 // without any coprocessors).
2516 if (ElemWidth == 1) {
2517 assert(VecWidth == VecTy.getVectorNumElements() && "Sanity failure")(static_cast <bool> (VecWidth == VecTy.getVectorNumElements
() && "Sanity failure") ? void (0) : __assert_fail ("VecWidth == VecTy.getVectorNumElements() && \"Sanity failure\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2517, __extension__ __PRETTY_FUNCTION__))
;
2518 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2)(static_cast <bool> (VecWidth == 8 || VecWidth == 4 || VecWidth
== 2) ? void (0) : __assert_fail ("VecWidth == 8 || VecWidth == 4 || VecWidth == 2"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2518, __extension__ __PRETTY_FUNCTION__))
;
2519 // Check if this is an extract of the lowest bit.
2520 if (IdxN) {
2521 // Extracting the lowest bit is a no-op, but it changes the type,
2522 // so it must be kept as an operation to avoid errors related to
2523 // type mismatches.
2524 if (IdxN->isNullValue() && ValTy.getSizeInBits() == 1)
2525 return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV);
2526 }
2527
2528 // If the value extracted is a single bit, use tstbit.
2529 if (ValWidth == 1) {
2530 SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2531 SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32);
2532 SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0);
2533 return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0);
2534 }
2535
2536 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2537 // a predicate register. The elements of the vector are repeated
2538 // in the register (if necessary) so that the total number is 8.
2539 // The extracted subvector will need to be expanded in such a way.
2540 unsigned Scale = VecWidth / ValWidth;
2541
2542 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2543 // position 0.
2544 assert(ty(IdxV) == MVT::i32)(static_cast <bool> (ty(IdxV) == MVT::i32) ? void (0) :
__assert_fail ("ty(IdxV) == MVT::i32", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2544, __extension__ __PRETTY_FUNCTION__))
;
2545 unsigned VecRep = 8 / VecWidth;
2546 SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2547 DAG.getConstant(8*VecRep, dl, MVT::i32));
2548 SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2549 SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0);
2550 while (Scale > 1) {
2551 // The longest possible subvector is at most 32 bits, so it is always
2552 // contained in the low subregister.
2553 T1 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, T1);
2554 T1 = expandPredicate(T1, dl, DAG);
2555 Scale /= 2;
2556 }
2557
2558 return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1);
2559 }
2560
2561 assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ?
void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2561, __extension__ __PRETTY_FUNCTION__))
;
2562
2563 // Cast everything to scalar integer types.
2564 MVT ScalarTy = tyScalar(VecTy);
2565 VecV = DAG.getBitcast(ScalarTy, VecV);
2566
2567 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2568 SDValue ExtV;
2569
2570 if (IdxN) {
2571 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2572 if (VecWidth == 64 && ValWidth == 32) {
2573 assert(Off == 0 || Off == 32)(static_cast <bool> (Off == 0 || Off == 32) ? void (0) :
__assert_fail ("Off == 0 || Off == 32", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2573, __extension__ __PRETTY_FUNCTION__))
;
2574 unsigned SubIdx = Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi;
2575 ExtV = DAG.getTargetExtractSubreg(SubIdx, dl, MVT::i32, VecV);
2576 } else if (Off == 0 && (ValWidth % 8) == 0) {
2577 ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy));
2578 } else {
2579 SDValue OffV = DAG.getConstant(Off, dl, MVT::i32);
2580 // The return type of EXTRACTU must be the same as the type of the
2581 // input vector.
2582 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2583 {VecV, WidthV, OffV});
2584 }
2585 } else {
2586 if (ty(IdxV) != MVT::i32)
2587 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2588 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2589 DAG.getConstant(ElemWidth, dl, MVT::i32));
2590 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2591 {VecV, WidthV, OffV});
2592 }
2593
2594 // Cast ExtV to the requested result type.
2595 ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy));
2596 ExtV = DAG.getBitcast(ResTy, ExtV);
2597 return ExtV;
2598}
2599
2600SDValue
2601HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2602 const SDLoc &dl, MVT ValTy,
2603 SelectionDAG &DAG) const {
2604 MVT VecTy = ty(VecV);
2605 if (VecTy.getVectorElementType() == MVT::i1) {
2606 MVT ValTy = ty(ValV);
2607 assert(ValTy.getVectorElementType() == MVT::i1)(static_cast <bool> (ValTy.getVectorElementType() == MVT
::i1) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == MVT::i1"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2607, __extension__ __PRETTY_FUNCTION__))
;
2608 SDValue ValR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV);
2609 unsigned VecLen = VecTy.getVectorNumElements();
2610 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2611 assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail
("Scale > 1", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2611, __extension__ __PRETTY_FUNCTION__))
;
2612
2613 for (unsigned R = Scale; R > 1; R /= 2) {
2614 ValR = contractPredicate(ValR, dl, DAG);
2615 ValR = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2616 DAG.getUNDEF(MVT::i32), ValR);
2617 }
2618 // The longest possible subvector is at most 32 bits, so it is always
2619 // contained in the low subregister.
2620 ValR = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, ValR);
2621
2622 unsigned ValBytes = 64 / Scale;
2623 SDValue Width = DAG.getConstant(ValBytes*8, dl, MVT::i32);
2624 SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2625 DAG.getConstant(8, dl, MVT::i32));
2626 SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2627 SDValue Ins = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2628 {VecR, ValR, Width, Idx});
2629 return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins);
2630 }
2631
2632 unsigned VecWidth = VecTy.getSizeInBits();
2633 unsigned ValWidth = ValTy.getSizeInBits();
2634 assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ?
void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2634, __extension__ __PRETTY_FUNCTION__))
;
2635 assert((VecWidth % ValWidth) == 0)(static_cast <bool> ((VecWidth % ValWidth) == 0) ? void
(0) : __assert_fail ("(VecWidth % ValWidth) == 0", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2635, __extension__ __PRETTY_FUNCTION__))
;
2636
2637 // Cast everything to scalar integer types.
2638 MVT ScalarTy = MVT::getIntegerVT(VecWidth);
2639 // The actual type of ValV may be different than ValTy (which is related
2640 // to the vector type).
2641 unsigned VW = ty(ValV).getSizeInBits();
2642 ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV);
2643 VecV = DAG.getBitcast(ScalarTy, VecV);
2644 if (VW != VecWidth)
2645 ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy);
2646
2647 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2648 SDValue InsV;
2649
2650 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) {
2651 unsigned W = C->getZExtValue() * ValWidth;
2652 SDValue OffV = DAG.getConstant(W, dl, MVT::i32);
2653 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2654 {VecV, ValV, WidthV, OffV});
2655 } else {
2656 if (ty(IdxV) != MVT::i32)
2657 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2658 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV);
2659 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2660 {VecV, ValV, WidthV, OffV});
2661 }
2662
2663 return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV);
2664}
2665
2666SDValue
2667HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2668 SelectionDAG &DAG) const {
2669 assert(ty(Vec32).getSizeInBits() == 32)(static_cast <bool> (ty(Vec32).getSizeInBits() == 32) ?
void (0) : __assert_fail ("ty(Vec32).getSizeInBits() == 32",
"/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2669, __extension__ __PRETTY_FUNCTION__))
;
2670 if (isUndef(Vec32))
2671 return DAG.getUNDEF(MVT::i64);
2672 return getInstr(Hexagon::S2_vsxtbh, dl, MVT::i64, {Vec32}, DAG);
2673}
2674
2675SDValue
2676HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2677 SelectionDAG &DAG) const {
2678 assert(ty(Vec64).getSizeInBits() == 64)(static_cast <bool> (ty(Vec64).getSizeInBits() == 64) ?
void (0) : __assert_fail ("ty(Vec64).getSizeInBits() == 64",
"/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2678, __extension__ __PRETTY_FUNCTION__))
;
2679 if (isUndef(Vec64))
2680 return DAG.getUNDEF(MVT::i32);
2681 return getInstr(Hexagon::S2_vtrunehb, dl, MVT::i32, {Vec64}, DAG);
2682}
2683
2684SDValue
2685HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2686 const {
2687 if (Ty.isVector()) {
2688 assert(Ty.isInteger() && "Only integer vectors are supported here")(static_cast <bool> (Ty.isInteger() && "Only integer vectors are supported here"
) ? void (0) : __assert_fail ("Ty.isInteger() && \"Only integer vectors are supported here\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2688, __extension__ __PRETTY_FUNCTION__))
;
2689 unsigned W = Ty.getSizeInBits();
2690 if (W <= 64)
2691 return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W)));
2692 return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG));
2693 }
2694
2695 if (Ty.isInteger())
2696 return DAG.getConstant(0, dl, Ty);
2697 if (Ty.isFloatingPoint())
2698 return DAG.getConstantFP(0.0, dl, Ty);
2699 llvm_unreachable("Invalid type for zero")::llvm::llvm_unreachable_internal("Invalid type for zero", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2699)
;
2700}
2701
2702SDValue
2703HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2704 const {
2705 MVT ValTy = ty(Val);
2706 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType())(static_cast <bool> (ValTy.getVectorElementType() == ResTy
.getVectorElementType()) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == ResTy.getVectorElementType()"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2706, __extension__ __PRETTY_FUNCTION__))
;
2707
2708 unsigned ValLen = ValTy.getVectorNumElements();
2709 unsigned ResLen = ResTy.getVectorNumElements();
2710 if (ValLen == ResLen)
2711 return Val;
2712
2713 const SDLoc &dl(Val);
2714 assert(ValLen < ResLen)(static_cast <bool> (ValLen < ResLen) ? void (0) : __assert_fail
("ValLen < ResLen", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2714, __extension__ __PRETTY_FUNCTION__))
;
2715 assert(ResLen % ValLen == 0)(static_cast <bool> (ResLen % ValLen == 0) ? void (0) :
__assert_fail ("ResLen % ValLen == 0", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2715, __extension__ __PRETTY_FUNCTION__))
;
2716
2717 SmallVector<SDValue, 4> Concats = {Val};
2718 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2719 Concats.push_back(DAG.getUNDEF(ValTy));
2720
2721 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats);
2722}
2723
2724SDValue
2725HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2726 MVT VecTy = ty(Op);
2727 unsigned BW = VecTy.getSizeInBits();
2728 const SDLoc &dl(Op);
2729 SmallVector<SDValue,8> Ops;
2730 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2731 Ops.push_back(Op.getOperand(i));
2732
2733 if (BW == 32)
2734 return buildVector32(Ops, dl, VecTy, DAG);
2735 if (BW == 64)
2736 return buildVector64(Ops, dl, VecTy, DAG);
2737
2738 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2739 // Check if this is a special case or all-0 or all-1.
2740 bool All0 = true, All1 = true;
2741 for (SDValue P : Ops) {
2742 auto *CN = dyn_cast<ConstantSDNode>(P.getNode());
2743 if (CN == nullptr) {
2744 All0 = All1 = false;
2745 break;
2746 }
2747 uint32_t C = CN->getZExtValue();
2748 All0 &= (C == 0);
2749 All1 &= (C == 1);
2750 }
2751 if (All0)
2752 return DAG.getNode(HexagonISD::PFALSE, dl, VecTy);
2753 if (All1)
2754 return DAG.getNode(HexagonISD::PTRUE, dl, VecTy);
2755
2756 // For each i1 element in the resulting predicate register, put 1
2757 // shifted by the index of the element into a general-purpose register,
2758 // then or them together and transfer it back into a predicate register.
2759 SDValue Rs[8];
2760 SDValue Z = getZero(dl, MVT::i32, DAG);
2761 // Always produce 8 bits, repeat inputs if necessary.
2762 unsigned Rep = 8 / VecTy.getVectorNumElements();
2763 for (unsigned i = 0; i != 8; ++i) {
2764 SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32);
2765 Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z);
2766 }
2767 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) {
2768 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2769 Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]);
2770 }
2771 // Move the value directly to a predicate register.
2772 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2773 }
2774
2775 return SDValue();
2776}
2777
2778SDValue
2779HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2780 SelectionDAG &DAG) const {
2781 MVT VecTy = ty(Op);
2782 const SDLoc &dl(Op);
2783 if (VecTy.getSizeInBits() == 64) {
2784 assert(Op.getNumOperands() == 2)(static_cast <bool> (Op.getNumOperands() == 2) ? void (
0) : __assert_fail ("Op.getNumOperands() == 2", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2784, __extension__ __PRETTY_FUNCTION__))
;
2785 return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, Op.getOperand(1),
2786 Op.getOperand(0));
2787 }
2788
2789 MVT ElemTy = VecTy.getVectorElementType();
2790 if (ElemTy == MVT::i1) {
2791 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1)(static_cast <bool> (VecTy == MVT::v2i1 || VecTy == MVT
::v4i1 || VecTy == MVT::v8i1) ? void (0) : __assert_fail ("VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2791, __extension__ __PRETTY_FUNCTION__))
;
2792 MVT OpTy = ty(Op.getOperand(0));
2793 // Scale is how many times the operands need to be contracted to match
2794 // the representation in the target register.
2795 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2796 assert(Scale == Op.getNumOperands() && Scale > 1)(static_cast <bool> (Scale == Op.getNumOperands() &&
Scale > 1) ? void (0) : __assert_fail ("Scale == Op.getNumOperands() && Scale > 1"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2796, __extension__ __PRETTY_FUNCTION__))
;
2797
2798 // First, convert all bool vectors to integers, then generate pairwise
2799 // inserts to form values of doubled length. Up until there are only
2800 // two values left to concatenate, all of these values will fit in a
2801 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2802 SmallVector<SDValue,4> Words[2];
2803 unsigned IdxW = 0;
2804
2805 for (SDValue P : Op.getNode()->op_values()) {
2806 SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P);
2807 for (unsigned R = Scale; R > 1; R /= 2) {
2808 W = contractPredicate(W, dl, DAG);
2809 W = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2810 DAG.getUNDEF(MVT::i32), W);
2811 }
2812 W = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, W);
2813 Words[IdxW].push_back(W);
2814 }
2815
2816 while (Scale > 2) {
2817 SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32);
2818 Words[IdxW ^ 1].clear();
2819
2820 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
2821 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
2822 // Insert W1 into W0 right next to the significant bits of W0.
2823 SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2824 {W0, W1, WidthV, WidthV});
2825 Words[IdxW ^ 1].push_back(T);
2826 }
2827 IdxW ^= 1;
2828 Scale /= 2;
2829 }
2830
2831 // Another sanity check. At this point there should only be two words
2832 // left, and Scale should be 2.
2833 assert(Scale == 2 && Words[IdxW].size() == 2)(static_cast <bool> (Scale == 2 && Words[IdxW].
size() == 2) ? void (0) : __assert_fail ("Scale == 2 && Words[IdxW].size() == 2"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2833, __extension__ __PRETTY_FUNCTION__))
;
2834
2835 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2836 Words[IdxW][1], Words[IdxW][0]);
2837 return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW);
2838 }
2839
2840 return SDValue();
2841}
2842
2843SDValue
2844HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2845 SelectionDAG &DAG) const {
2846 SDValue Vec = Op.getOperand(0);
2847 MVT ElemTy = ty(Vec).getVectorElementType();
2848 return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG);
2849}
2850
2851SDValue
2852HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
2853 SelectionDAG &DAG) const {
2854 return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op),
2855 ty(Op), ty(Op), DAG);
2856}
2857
2858SDValue
2859HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2860 SelectionDAG &DAG) const {
2861 return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
2862 SDLoc(Op), ty(Op).getVectorElementType(), DAG);
2863}
2864
2865SDValue
2866HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
2867 SelectionDAG &DAG) const {
2868 SDValue ValV = Op.getOperand(1);
2869 return insertVector(Op.getOperand(0), ValV, Op.getOperand(2),
2870 SDLoc(Op), ty(ValV), DAG);
2871}
2872
2873bool
2874HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
2875 // Assuming the caller does not have either a signext or zeroext modifier, and
2876 // only one value is accepted, any reasonable truncation is allowed.
2877 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
2878 return false;
2879
2880 // FIXME: in principle up to 64-bit could be made safe, but it would be very
2881 // fragile at the moment: any support for multiple value returns would be
2882 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
2883 return Ty1->getPrimitiveSizeInBits() <= 32;
2884}
2885
2886SDValue
2887HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
2888 MVT Ty = ty(Op);
2889 const SDLoc &dl(Op);
2890 // Lower loads of scalar predicate vectors (v2i1, v4i1, v8i1) to loads of i1
2891 // followed by a TYPECAST.
2892 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
2893 bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1);
2894 if (DoCast) {
2895 SDValue NL = DAG.getLoad(
2896 LN->getAddressingMode(), LN->getExtensionType(), MVT::i1, dl,
2897 LN->getChain(), LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(),
2898 /*MemoryVT*/ MVT::i1, LN->getAlign(), LN->getMemOperand()->getFlags(),
2899 LN->getAAInfo(), LN->getRanges());
2900 LN = cast<LoadSDNode>(NL.getNode());
2901 }
2902
2903 unsigned ClaimAlign = LN->getAlignment();
2904 validateConstPtrAlignment(LN->getBasePtr(), dl, ClaimAlign);
2905 // Call LowerUnalignedLoad for all loads, it recognizes loads that
2906 // don't need extra aligning.
2907 SDValue LU = LowerUnalignedLoad(SDValue(LN, 0), DAG);
2908 if (DoCast) {
2909 SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, Ty, LU);
2910 SDValue Ch = cast<LoadSDNode>(LU.getNode())->getChain();
2911 return DAG.getMergeValues({TC, Ch}, dl);
2912 }
2913 return LU;
2914}
2915
2916SDValue
2917HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
2918 const SDLoc &dl(Op);
2919 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
2920 SDValue Val = SN->getValue();
2921 MVT Ty = ty(Val);
2922
2923 bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1);
2924 if (DoCast) {
2925 SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, Val);
2926 SDValue NS = DAG.getStore(SN->getChain(), dl, TC, SN->getBasePtr(),
2927 SN->getMemOperand());
2928 if (SN->isIndexed()) {
2929 NS = DAG.getIndexedStore(NS, dl, SN->getBasePtr(), SN->getOffset(),
2930 SN->getAddressingMode());
2931 }
2932 SN = cast<StoreSDNode>(NS.getNode());
2933 }
2934
2935 unsigned ClaimAlign = SN->getAlignment();
2936 SDValue Ptr = SN->getBasePtr();
2937 validateConstPtrAlignment(Ptr, dl, ClaimAlign);
2938
2939 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
2940 unsigned NeedAlign = Subtarget.getTypeAlignment(StoreTy);
2941 if (ClaimAlign < NeedAlign)
2942 return expandUnalignedStore(SN, DAG);
2943 return SDValue(SN, 0);
2944}
2945
2946SDValue
2947HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
2948 const {
2949 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
2950 MVT LoadTy = ty(Op);
2951 unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy);
2952 unsigned HaveAlign = LN->getAlignment();
2953 if (HaveAlign >= NeedAlign)
2954 return Op;
2955
2956 const SDLoc &dl(Op);
2957 const DataLayout &DL = DAG.getDataLayout();
2958 LLVMContext &Ctx = *DAG.getContext();
2959
2960 // If the load aligning is disabled or the load can be broken up into two
2961 // smaller legal loads, do the default (target-independent) expansion.
2962 bool DoDefault = false;
2963 // Handle it in the default way if this is an indexed load.
2964 if (!LN->isUnindexed())
2965 DoDefault = true;
2966
2967 if (!AlignLoads) {
2968 if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(),
2969 *LN->getMemOperand()))
2970 return Op;
2971 DoDefault = true;
2972 }
2973 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
2974 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
2975 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign)
2976 : MVT::getVectorVT(MVT::i8, HaveAlign);
2977 DoDefault =
2978 allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand());
2979 }
2980 if (DoDefault) {
2981 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
2982 return DAG.getMergeValues({P.first, P.second}, dl);
2983 }
2984
2985 // The code below generates two loads, both aligned as NeedAlign, and
2986 // with the distance of NeedAlign between them. For that to cover the
2987 // bits that need to be loaded (and without overlapping), the size of
2988 // the loads should be equal to NeedAlign. This is true for all loadable
2989 // types, but add an assertion in case something changes in the future.
2990 assert(LoadTy.getSizeInBits() == 8*NeedAlign)(static_cast <bool> (LoadTy.getSizeInBits() == 8*NeedAlign
) ? void (0) : __assert_fail ("LoadTy.getSizeInBits() == 8*NeedAlign"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))
;
2991
2992 unsigned LoadLen = NeedAlign;
2993 SDValue Base = LN->getBasePtr();
2994 SDValue Chain = LN->getChain();
2995 auto BO = getBaseAndOffset(Base);
2996 unsigned BaseOpc = BO.first.getOpcode();
2997 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
2998 return Op;
2999
3000 if (BO.second % LoadLen != 0) {
3001 BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first,
3002 DAG.getConstant(BO.second % LoadLen, dl, MVT::i32));
3003 BO.second -= BO.second % LoadLen;
3004 }
3005 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3006 ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first,
3007 DAG.getConstant(NeedAlign, dl, MVT::i32))
3008 : BO.first;
3009 SDValue Base0 =
3010 DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl);
3011 SDValue Base1 = DAG.getMemBasePlusOffset(
3012 BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl);
3013
3014 MachineMemOperand *WideMMO = nullptr;
3015 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3016 MachineFunction &MF = DAG.getMachineFunction();
3017 WideMMO = MF.getMachineMemOperand(
3018 MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
3019 MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
3020 MMO->getOrdering(), MMO->getFailureOrdering());
3021 }
3022
3023 SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
3024 SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO);
3025
3026 SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy,
3027 {Load1, Load0, BaseNoOff.getOperand(0)});
3028 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3029 Load0.getValue(1), Load1.getValue(1));
3030 SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl);
3031 return M;
3032}
3033
3034SDValue
3035HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3036 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
3037 auto *CY = dyn_cast<ConstantSDNode>(Y);
3038 if (!CY)
3039 return SDValue();
3040
3041 const SDLoc &dl(Op);
3042 SDVTList VTs = Op.getNode()->getVTList();
3043 assert(VTs.NumVTs == 2)(static_cast <bool> (VTs.NumVTs == 2) ? void (0) : __assert_fail
("VTs.NumVTs == 2", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3043, __extension__ __PRETTY_FUNCTION__))
;
3044 assert(VTs.VTs[1] == MVT::i1)(static_cast <bool> (VTs.VTs[1] == MVT::i1) ? void (0) :
__assert_fail ("VTs.VTs[1] == MVT::i1", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3044, __extension__ __PRETTY_FUNCTION__))
;
3045 unsigned Opc = Op.getOpcode();
3046
3047 if (CY) {
3048 uint32_t VY = CY->getZExtValue();
3049 assert(VY != 0 && "This should have been folded")(static_cast <bool> (VY != 0 && "This should have been folded"
) ? void (0) : __assert_fail ("VY != 0 && \"This should have been folded\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3049, __extension__ __PRETTY_FUNCTION__))
;
3050 // X +/- 1
3051 if (VY != 1)
3052 return SDValue();
3053
3054 if (Opc == ISD::UADDO) {
3055 SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y});
3056 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG),
3057 ISD::SETEQ);
3058 return DAG.getMergeValues({Op, Ov}, dl);
3059 }
3060 if (Opc == ISD::USUBO) {
3061 SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y});
3062 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op,
3063 DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ);
3064 return DAG.getMergeValues({Op, Ov}, dl);
3065 }
3066 }
3067
3068 return SDValue();
3069}
3070
3071SDValue
3072HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const {
3073 const SDLoc &dl(Op);
3074 unsigned Opc = Op.getOpcode();
3075 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);
3076
3077 if (Opc == ISD::ADDCARRY)
3078 return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(),
3079 { X, Y, C });
3080
3081 EVT CarryTy = C.getValueType();
3082 SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(),
3083 { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3084 SDValue Out[] = { SubC.getValue(0),
3085 DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) };
3086 return DAG.getMergeValues(Out, dl);
3087}
3088
3089SDValue
3090HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3091 SDValue Chain = Op.getOperand(0);
3092 SDValue Offset = Op.getOperand(1);
3093 SDValue Handler = Op.getOperand(2);
3094 SDLoc dl(Op);
3095 auto PtrVT = getPointerTy(DAG.getDataLayout());
3096
3097 // Mark function as containing a call to EH_RETURN.
3098 HexagonMachineFunctionInfo *FuncInfo =
3099 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3100 FuncInfo->setHasEHReturn();
3101
3102 unsigned OffsetReg = Hexagon::R28;
3103
3104 SDValue StoreAddr =
3105 DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT),
3106 DAG.getIntPtrConstant(4, dl));
3107 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
3108 Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
3109
3110 // Not needed we already use it as explict input to EH_RETURN.
3111 // MF.getRegInfo().addLiveOut(OffsetReg);
3112
3113 return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain);
3114}
3115
3116SDValue
3117HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3118 unsigned Opc = Op.getOpcode();
3119
3120 // Handle INLINEASM first.
3121 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
1
Assuming 'Opc' is not equal to INLINEASM
2
Assuming 'Opc' is not equal to INLINEASM_BR
3
Taking false branch
3122 return LowerINLINEASM(Op, DAG);
3123
3124 if (isHvxOperation(Op.getNode(), DAG)) {
4
Assuming the condition is false
5
Taking false branch
3125 // If HVX lowering returns nothing, try the default lowering.
3126 if (SDValue V = LowerHvxOperation(Op, DAG))
3127 return V;
3128 }
3129
3130 switch (Opc) {
6
Control jumps to 'case SETCC:' at line 3169
3131 default:
3132#ifndef NDEBUG
3133 Op.getNode()->dumpr(&DAG);
3134 if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END)
3135 errs() << "Error: check for a non-legal type in this operation\n";
3136#endif
3137 llvm_unreachable("Should not custom lower this!")::llvm::llvm_unreachable_internal("Should not custom lower this!"
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3137)
;
3138 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3139 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3140 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3141 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3142 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3143 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3144 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3145 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3146 case ISD::LOAD: return LowerLoad(Op, DAG);
3147 case ISD::STORE: return LowerStore(Op, DAG);
3148 case ISD::UADDO:
3149 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3150 case ISD::ADDCARRY:
3151 case ISD::SUBCARRY: return LowerAddSubCarry(Op, DAG);
3152 case ISD::SRA:
3153 case ISD::SHL:
3154 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3155 case ISD::ROTL: return LowerROTL(Op, DAG);
3156 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3157 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3158 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3159 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3160 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3161 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3162 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3163 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3164 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3165 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3166 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3167 case ISD::VASTART: return LowerVASTART(Op, DAG);
3168 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
3169 case ISD::SETCC: return LowerSETCC(Op, DAG);
7
Calling 'HexagonTargetLowering::LowerSETCC'
3170 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3171 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3172 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3173 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG);
3174 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG);
3175 break;
3176 }
3177
3178 return SDValue();
3179}
3180
3181void
3182HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3183 SmallVectorImpl<SDValue> &Results,
3184 SelectionDAG &DAG) const {
3185 if (isHvxOperation(N, DAG)) {
3186 LowerHvxOperationWrapper(N, Results, DAG);
3187 if (!Results.empty())
3188 return;
3189 }
3190
3191 // We are only custom-lowering stores to verify the alignment of the
3192 // address if it is a compile-time constant. Since a store can be modified
3193 // during type-legalization (the value being stored may need legalization),
3194 // return empty Results here to indicate that we don't really make any
3195 // changes in the custom lowering.
3196 if (N->getOpcode() != ISD::STORE)
3197 return TargetLowering::LowerOperationWrapper(N, Results, DAG);
3198}
3199
3200void
3201HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3202 SmallVectorImpl<SDValue> &Results,
3203 SelectionDAG &DAG) const {
3204 if (isHvxOperation(N, DAG)) {
3205 ReplaceHvxNodeResults(N, Results, DAG);
3206 if (!Results.empty())
3207 return;
3208 }
3209
3210 const SDLoc &dl(N);
3211 switch (N->getOpcode()) {
3212 case ISD::SRL:
3213 case ISD::SRA:
3214 case ISD::SHL:
3215 return;
3216 case ISD::BITCAST:
3217 // Handle a bitcast from v8i1 to i8.
3218 if (N->getValueType(0) == MVT::i8) {
3219 if (N->getOperand(0).getValueType() == MVT::v8i1) {
3220 SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32,
3221 N->getOperand(0), DAG);
3222 SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8);
3223 Results.push_back(T);
3224 }
3225 }
3226 break;
3227 }
3228}
3229
3230SDValue
3231HexagonTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
3232 const {
3233 if (isHvxOperation(N, DCI.DAG)) {
3234 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3235 return V;
3236 return SDValue();
3237 }
3238
3239 if (DCI.isBeforeLegalizeOps())
3240 return SDValue();
3241
3242 SDValue Op(N, 0);
3243 const SDLoc &dl(Op);
3244 unsigned Opc = Op.getOpcode();
3245
3246 if (Opc == HexagonISD::P2D) {
3247 SDValue P = Op.getOperand(0);
3248 switch (P.getOpcode()) {
3249 case HexagonISD::PTRUE:
3250 return DCI.DAG.getConstant(-1, dl, ty(Op));
3251 case HexagonISD::PFALSE:
3252 return getZero(dl, ty(Op), DCI.DAG);
3253 default:
3254 break;
3255 }
3256 } else if (Opc == ISD::VSELECT) {
3257 // This is pretty much duplicated in HexagonISelLoweringHVX...
3258 //
3259 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3260 SDValue Cond = Op.getOperand(0);
3261 if (Cond->getOpcode() == ISD::XOR) {
3262 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
3263 if (C1->getOpcode() == HexagonISD::PTRUE) {
3264 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
3265 Op.getOperand(2), Op.getOperand(1));
3266 return VSel;
3267 }
3268 }
3269 }
3270
3271 return SDValue();
3272}
3273
3274/// Returns relocation base for the given PIC jumptable.
3275SDValue
3276HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3277 SelectionDAG &DAG) const {
3278 int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3279 EVT VT = Table.getValueType();
3280 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
3281 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T);
3282}
3283
3284//===----------------------------------------------------------------------===//
3285// Inline Assembly Support
3286//===----------------------------------------------------------------------===//
3287
3288TargetLowering::ConstraintType
3289HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3290 if (Constraint.size() == 1) {
3291 switch (Constraint[0]) {
3292 case 'q':
3293 case 'v':
3294 if (Subtarget.useHVXOps())
3295 return C_RegisterClass;
3296 break;
3297 case 'a':
3298 return C_RegisterClass;
3299 default:
3300 break;
3301 }
3302 }
3303 return TargetLowering::getConstraintType(Constraint);
3304}
3305
3306std::pair<unsigned, const TargetRegisterClass*>
3307HexagonTargetLowering::getRegForInlineAsmConstraint(
3308 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3309
3310 if (Constraint.size() == 1) {
3311 switch (Constraint[0]) {
3312 case 'r': // R0-R31
3313 switch (VT.SimpleTy) {
3314 default:
3315 return {0u, nullptr};
3316 case MVT::i1:
3317 case MVT::i8:
3318 case MVT::i16:
3319 case MVT::i32:
3320 case MVT::f32:
3321 return {0u, &Hexagon::IntRegsRegClass};
3322 case MVT::i64:
3323 case MVT::f64:
3324 return {0u, &Hexagon::DoubleRegsRegClass};
3325 }
3326 break;
3327 case 'a': // M0-M1
3328 if (VT != MVT::i32)
3329 return {0u, nullptr};
3330 return {0u, &Hexagon::ModRegsRegClass};
3331 case 'q': // q0-q3
3332 switch (VT.getSizeInBits()) {
3333 default:
3334 return {0u, nullptr};
3335 case 64:
3336 case 128:
3337 return {0u, &Hexagon::HvxQRRegClass};
3338 }
3339 break;
3340 case 'v': // V0-V31
3341 switch (VT.getSizeInBits()) {
3342 default:
3343 return {0u, nullptr};
3344 case 512:
3345 return {0u, &Hexagon::HvxVRRegClass};
3346 case 1024:
3347 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3348 return {0u, &Hexagon::HvxVRRegClass};
3349 return {0u, &Hexagon::HvxWRRegClass};
3350 case 2048:
3351 return {0u, &Hexagon::HvxWRRegClass};
3352 }
3353 break;
3354 default:
3355 return {0u, nullptr};
3356 }
3357 }
3358
3359 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3360}
3361
3362/// isFPImmLegal - Returns true if the target can instruction select the
3363/// specified FP immediate natively. If false, the legalizer will
3364/// materialize the FP immediate as a load from a constant pool.
3365bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3366 bool ForCodeSize) const {
3367 return true;
3368}
3369
3370/// isLegalAddressingMode - Return true if the addressing mode represented by
3371/// AM is legal for this target, for a load/store of the specified type.
3372bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3373 const AddrMode &AM, Type *Ty,
3374 unsigned AS, Instruction *I) const {
3375 if (Ty->isSized()) {
3376 // When LSR detects uses of the same base address to access different
3377 // types (e.g. unions), it will assume a conservative type for these
3378 // uses:
3379 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3380 // The type Ty passed here would then be "void". Skip the alignment
3381 // checks, but do not return false right away, since that confuses
3382 // LSR into crashing.
3383 Align A = DL.getABITypeAlign(Ty);
3384 // The base offset must be a multiple of the alignment.
3385 if (!isAligned(A, AM.BaseOffs))
3386 return false;
3387 // The shifted offset must fit in 11 bits.
3388 if (!isInt<11>(AM.BaseOffs >> Log2(A)))
3389 return false;
3390 }
3391
3392 // No global is ever allowed as a base.
3393 if (AM.BaseGV)
3394 return false;
3395
3396 int Scale = AM.Scale;
3397 if (Scale < 0)
3398 Scale = -Scale;
3399 switch (Scale) {
3400 case 0: // No scale reg, "r+i", "r", or just "i".
3401 break;
3402 default: // No scaled addressing mode.
3403 return false;
3404 }
3405 return true;
3406}
3407
3408/// Return true if folding a constant offset with the given GlobalAddress is
3409/// legal. It is frequently not legal in PIC relocation models.
3410bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3411 const {
3412 return HTM.getRelocationModel() == Reloc::Static;
3413}
3414
3415/// isLegalICmpImmediate - Return true if the specified immediate is legal
3416/// icmp immediate, that is the target has icmp instructions which can compare
3417/// a register against the immediate without having to materialize the
3418/// immediate into a register.
3419bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3420 return Imm >= -512 && Imm <= 511;
3421}
3422
3423/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3424/// for tail call optimization. Targets which want to do tail call
3425/// optimization should implement this function.
3426bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3427 SDValue Callee,
3428 CallingConv::ID CalleeCC,
3429 bool IsVarArg,
3430 bool IsCalleeStructRet,
3431 bool IsCallerStructRet,
3432 const SmallVectorImpl<ISD::OutputArg> &Outs,
3433 const SmallVectorImpl<SDValue> &OutVals,
3434 const SmallVectorImpl<ISD::InputArg> &Ins,
3435 SelectionDAG& DAG) const {
3436 const Function &CallerF = DAG.getMachineFunction().getFunction();
3437 CallingConv::ID CallerCC = CallerF.getCallingConv();
3438 bool CCMatch = CallerCC == CalleeCC;
3439
3440 // ***************************************************************************
3441 // Look for obvious safe cases to perform tail call optimization that do not
3442 // require ABI changes.
3443 // ***************************************************************************
3444
3445 // If this is a tail call via a function pointer, then don't do it!
3446 if (!isa<GlobalAddressSDNode>(Callee) &&
3447 !isa<ExternalSymbolSDNode>(Callee)) {
3448 return false;
3449 }
3450
3451 // Do not optimize if the calling conventions do not match and the conventions
3452 // used are not C or Fast.
3453 if (!CCMatch) {
3454 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3455 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3456 // If R & E, then ok.
3457 if (!R || !E)
3458 return false;
3459 }
3460
3461 // Do not tail call optimize vararg calls.
3462 if (IsVarArg)
3463 return false;
3464
3465 // Also avoid tail call optimization if either caller or callee uses struct
3466 // return semantics.
3467 if (IsCalleeStructRet || IsCallerStructRet)
3468 return false;
3469
3470 // In addition to the cases above, we also disable Tail Call Optimization if
3471 // the calling convention code that at least one outgoing argument needs to
3472 // go on the stack. We cannot check that here because at this point that
3473 // information is not available.
3474 return true;
3475}
3476
3477/// Returns the target specific optimal type for load and store operations as
3478/// a result of memset, memcpy, and memmove lowering.
3479///
3480/// If DstAlign is zero that means it's safe to destination alignment can
3481/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3482/// a need to check it against alignment requirement, probably because the
3483/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3484/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3485/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3486/// does not need to be loaded. It returns EVT::Other if the type should be
3487/// determined using generic target-independent logic.
3488EVT HexagonTargetLowering::getOptimalMemOpType(
3489 const MemOp &Op, const AttributeList &FuncAttributes) const {
3490 if (Op.size() >= 8 && Op.isAligned(Align(8)))
3491 return MVT::i64;
3492 if (Op.size() >= 4 && Op.isAligned(Align(4)))
3493 return MVT::i32;
3494 if (Op.size() >= 2 && Op.isAligned(Align(2)))
3495 return MVT::i16;
3496 return MVT::Other;
3497}
3498
3499bool HexagonTargetLowering::allowsMemoryAccess(
3500 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3501 Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
3502 MVT SVT = VT.getSimpleVT();
3503 if (Subtarget.isHVXVectorType(SVT, true))
3504 return allowsHvxMemoryAccess(SVT, Flags, Fast);
3505 return TargetLoweringBase::allowsMemoryAccess(
3506 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3507}
3508
3509bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3510 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3511 bool *Fast) const {
3512 MVT SVT = VT.getSimpleVT();
3513 if (Subtarget.isHVXVectorType(SVT, true))
3514 return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
3515 if (Fast)
3516 *Fast = false;
3517 return false;
3518}
3519
3520std::pair<const TargetRegisterClass*, uint8_t>
3521HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3522 MVT VT) const {
3523 if (Subtarget.isHVXVectorType(VT, true)) {
3524 unsigned BitWidth = VT.getSizeInBits();
3525 unsigned VecWidth = Subtarget.getVectorLength() * 8;
3526
3527 if (VT.getVectorElementType() == MVT::i1)
3528 return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3529 if (BitWidth == VecWidth)
3530 return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3531 assert(BitWidth == 2 * VecWidth)(static_cast <bool> (BitWidth == 2 * VecWidth) ? void (
0) : __assert_fail ("BitWidth == 2 * VecWidth", "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3531, __extension__ __PRETTY_FUNCTION__))
;
3532 return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3533 }
3534
3535 return TargetLowering::findRepresentativeClass(TRI, VT);
3536}
3537
3538bool HexagonTargetLowering::shouldReduceLoadWidth(SDNode *Load,
3539 ISD::LoadExtType ExtTy, EVT NewVT) const {
3540 // TODO: This may be worth removing. Check regression tests for diffs.
3541 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT))
3542 return false;
3543
3544 auto *L = cast<LoadSDNode>(Load);
3545 std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3546 // Small-data object, do not shrink.
3547 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3548 return false;
3549 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) {
3550 auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine());
3551 const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3552 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM);
3553 }
3554 return true;
3555}
3556
3557Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
3558 Value *Addr,
3559 AtomicOrdering Ord) const {
3560 BasicBlock *BB = Builder.GetInsertBlock();
3561 Module *M = BB->getParent()->getParent();
3562 auto PT = cast<PointerType>(Addr->getType());
3563 Type *Ty = PT->getElementType();
3564 unsigned SZ = Ty->getPrimitiveSizeInBits();
3565 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported"
) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic loads supported\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3565, __extension__ __PRETTY_FUNCTION__))
;
3566 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3567 : Intrinsic::hexagon_L4_loadd_locked;
3568 Function *Fn = Intrinsic::getDeclaration(M, IntID);
3569
3570 PointerType *NewPtrTy
3571 = Builder.getIntNTy(SZ)->getPointerTo(PT->getAddressSpace());
3572 Addr = Builder.CreateBitCast(Addr, NewPtrTy);
3573
3574 Value *Call = Builder.CreateCall(Fn, Addr, "larx");
3575
3576 return Builder.CreateBitCast(Call, Ty);
3577}
3578
3579/// Perform a store-conditional operation to Addr. Return the status of the
3580/// store. This should be 0 if the store succeeded, non-zero otherwise.
3581Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
3582 Value *Val, Value *Addr,
3583 AtomicOrdering Ord) const {
3584 BasicBlock *BB = Builder.GetInsertBlock();
3585 Module *M = BB->getParent()->getParent();
3586 Type *Ty = Val->getType();
3587 unsigned SZ = Ty->getPrimitiveSizeInBits();
3588
3589 Type *CastTy = Builder.getIntNTy(SZ);
3590 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported"
) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic stores supported\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3590, __extension__ __PRETTY_FUNCTION__))
;
3591 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3592 : Intrinsic::hexagon_S4_stored_locked;
3593 Function *Fn = Intrinsic::getDeclaration(M, IntID);
3594
3595 unsigned AS = Addr->getType()->getPointerAddressSpace();
3596 Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS));
3597 Val = Builder.CreateBitCast(Val, CastTy);
3598
3599 Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx");
3600 Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");
3601 Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));
3602 return Ext;
3603}
3604
3605TargetLowering::AtomicExpansionKind
3606HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
3607 // Do not expand loads and stores that don't exceed 64 bits.
3608 return LI->getType()->getPrimitiveSizeInBits() > 64
3609 ? AtomicExpansionKind::LLOnly
3610 : AtomicExpansionKind::None;
3611}
3612
3613bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
3614 // Do not expand loads and stores that don't exceed 64 bits.
3615 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64;
3616}
3617
3618TargetLowering::AtomicExpansionKind
3619HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3620 AtomicCmpXchgInst *AI) const {
3621 return AtomicExpansionKind::LLSC;
3622}

/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class MaskedLoadStoreSDNode;
513 friend class MaskedGatherScatterSDNode;
514
515 uint16_t : NumMemSDNodeBits;
516
517 // This storage is shared between disparate class hierarchies to hold an
518 // enumeration specific to the class hierarchy in use.
519 // LSBaseSDNode => enum ISD::MemIndexedMode
520 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
521 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
522 uint16_t AddressingMode : 3;
523 };
524 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
525
526 class LoadSDNodeBitfields {
527 friend class LoadSDNode;
528 friend class MaskedLoadSDNode;
529 friend class MaskedGatherSDNode;
530
531 uint16_t : NumLSBaseSDNodeBits;
532
533 uint16_t ExtTy : 2; // enum ISD::LoadExtType
534 uint16_t IsExpanding : 1;
535 };
536
537 class StoreSDNodeBitfields {
538 friend class StoreSDNode;
539 friend class MaskedStoreSDNode;
540 friend class MaskedScatterSDNode;
541
542 uint16_t : NumLSBaseSDNodeBits;
543
544 uint16_t IsTruncating : 1;
545 uint16_t IsCompressing : 1;
546 };
547
548 union {
549 char RawSDNodeBits[sizeof(uint16_t)];
550 SDNodeBitfields SDNodeBits;
551 ConstantSDNodeBitfields ConstantSDNodeBits;
552 MemSDNodeBitfields MemSDNodeBits;
553 LSBaseSDNodeBitfields LSBaseSDNodeBits;
554 LoadSDNodeBitfields LoadSDNodeBits;
555 StoreSDNodeBitfields StoreSDNodeBits;
556 };
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK
560
561 // RawSDNodeBits must cover the entirety of the union. This means that all of
562 // the union's members must have size <= RawSDNodeBits. We write the RHS as
563 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
564 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
565 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
566 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
567 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
568 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
569 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
570
571private:
572 friend class SelectionDAG;
573 // TODO: unfriend HandleSDNode once we fix its operand handling.
574 friend class HandleSDNode;
575
576 /// Unique id per SDNode in the DAG.
577 int NodeId = -1;
578
579 /// The values that are used by this operation.
580 SDUse *OperandList = nullptr;
581
582 /// The types of the values this node defines. SDNode's may
583 /// define multiple values simultaneously.
584 const EVT *ValueList;
585
586 /// List of uses for this SDNode.
587 SDUse *UseList = nullptr;
588
589 /// The number of entries in the Operand/Value list.
590 unsigned short NumOperands = 0;
591 unsigned short NumValues;
592
593 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
594 // original LLVM instructions.
595 // This is used for turning off scheduling, because we'll forgo
596 // the normal scheduling algorithms and output the instructions according to
597 // this ordering.
598 unsigned IROrder;
599
600 /// Source line information.
601 DebugLoc debugLoc;
602
603 /// Return a pointer to the specified value type.
604 static const EVT *getValueTypeList(EVT VT);
605
606 SDNodeFlags Flags;
607
608public:
609 /// Unique and persistent id per SDNode in the DAG.
610 /// Used for debug printing.
611 uint16_t PersistentId;
612
613 //===--------------------------------------------------------------------===//
614 // Accessors
615 //
616
617 /// Return the SelectionDAG opcode value for this node. For
618 /// pre-isel nodes (those for which isMachineOpcode returns false), these
619 /// are the opcode values in the ISD and <target>ISD namespaces. For
620 /// post-isel opcodes, see getMachineOpcode.
621 unsigned getOpcode() const { return (unsigned short)NodeType; }
622
623 /// Test if this node has a target-specific opcode (in the
624 /// \<target\>ISD namespace).
625 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
626
627 /// Test if this node has a target-specific opcode that may raise
628 /// FP exceptions (in the \<target\>ISD namespace and greater than
629 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
630 /// opcode are currently automatically considered to possibly raise
631 /// FP exceptions as well.
632 bool isTargetStrictFPOpcode() const {
633 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
634 }
635
636 /// Test if this node has a target-specific
637 /// memory-referencing opcode (in the \<target\>ISD namespace and
638 /// greater than FIRST_TARGET_MEMORY_OPCODE).
639 bool isTargetMemoryOpcode() const {
640 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
641 }
642
643 /// Return true if the type of the node type undefined.
644 bool isUndef() const { return NodeType == ISD::UNDEF; }
645
646 /// Test if this node is a memory intrinsic (with valid pointer information).
647 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
648 /// non-memory intrinsics (with chains) that are not really instances of
649 /// MemSDNode. For such nodes, we need some extra state to determine the
650 /// proper classof relationship.
651 bool isMemIntrinsic() const {
652 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
653 NodeType == ISD::INTRINSIC_VOID) &&
654 SDNodeBits.IsMemIntrinsic;
655 }
656
657 /// Test if this node is a strict floating point pseudo-op.
658 bool isStrictFPOpcode() {
659 switch (NodeType) {
660 default:
661 return false;
662 case ISD::STRICT_FP16_TO_FP:
663 case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
665 case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
667 return true;
668 }
669 }
670
671 /// Test if this node has a post-isel opcode, directly
672 /// corresponding to a MachineInstr opcode.
673 bool isMachineOpcode() const { return NodeType < 0; }
674
675 /// This may only be called if isMachineOpcode returns
676 /// true. It returns the MachineInstr opcode value that the node's opcode
677 /// corresponds to.
678 unsigned getMachineOpcode() const {
679 assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 679, __extension__ __PRETTY_FUNCTION__))
;
680 return ~NodeType;
681 }
682
683 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
684 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
685
686 bool isDivergent() const { return SDNodeBits.IsDivergent; }
687
688 /// Return true if there are no uses of this node.
689 bool use_empty() const { return UseList == nullptr; }
690
691 /// Return true if there is exactly one use of this node.
692 bool hasOneUse() const { return hasSingleElement(uses()); }
693
694 /// Return the number of uses of this node. This method takes
695 /// time proportional to the number of uses.
696 size_t use_size() const { return std::distance(use_begin(), use_end()); }
697
698 /// Return the unique node id.
699 int getNodeId() const { return NodeId; }
700
701 /// Set unique node id.
702 void setNodeId(int Id) { NodeId = Id; }
703
704 /// Return the node ordering.
705 unsigned getIROrder() const { return IROrder; }
706
707 /// Set the node ordering.
708 void setIROrder(unsigned Order) { IROrder = Order; }
709
710 /// Return the source location info.
711 const DebugLoc &getDebugLoc() const { return debugLoc; }
712
713 /// Set source location info. Try to avoid this, putting
714 /// it in the constructor is preferable.
715 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
716
717 /// This class provides iterator support for SDUse
718 /// operands that use a specific SDNode.
719 class use_iterator {
720 friend class SDNode;
721
722 SDUse *Op = nullptr;
723
724 explicit use_iterator(SDUse *op) : Op(op) {}
725
726 public:
727 using iterator_category = std::forward_iterator_tag;
728 using value_type = SDUse;
729 using difference_type = std::ptrdiff_t;
730 using pointer = value_type *;
731 using reference = value_type &;
732
733 use_iterator() = default;
734 use_iterator(const use_iterator &I) : Op(I.Op) {}
735
736 bool operator==(const use_iterator &x) const {
737 return Op == x.Op;
738 }
739 bool operator!=(const use_iterator &x) const {
740 return !operator==(x);
741 }
742
743 /// Return true if this iterator is at the end of uses list.
744 bool atEnd() const { return Op == nullptr; }
745
746 // Iterator traversal: forward iteration only.
747 use_iterator &operator++() { // Preincrement
748 assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 748, __extension__ __PRETTY_FUNCTION__))
;
749 Op = Op->getNext();
750 return *this;
751 }
752
753 use_iterator operator++(int) { // Postincrement
754 use_iterator tmp = *this; ++*this; return tmp;
755 }
756
757 /// Retrieve a pointer to the current user node.
758 SDNode *operator*() const {
759 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 759, __extension__ __PRETTY_FUNCTION__))
;
760 return Op->getUser();
761 }
762
763 SDNode *operator->() const { return operator*(); }
764
765 SDUse &getUse() const { return *Op; }
766
767 /// Retrieve the operand # of this use in its user.
768 unsigned getOperandNo() const {
769 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 769, __extension__ __PRETTY_FUNCTION__))
;
770 return (unsigned)(Op - Op->getUser()->OperandList);
771 }
772 };
773
774 /// Provide iteration support to walk over all uses of an SDNode.
775 use_iterator use_begin() const {
776 return use_iterator(UseList);
777 }
778
779 static use_iterator use_end() { return use_iterator(nullptr); }
780
781 inline iterator_range<use_iterator> uses() {
782 return make_range(use_begin(), use_end());
783 }
784 inline iterator_range<use_iterator> uses() const {
785 return make_range(use_begin(), use_end());
786 }
787
788 /// Return true if there are exactly NUSES uses of the indicated value.
789 /// This method ignores uses of other values defined by this operation.
790 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
791
792 /// Return true if there are any use of the indicated value.
793 /// This method ignores uses of other values defined by this operation.
794 bool hasAnyUseOfValue(unsigned Value) const;
795
796 /// Return true if this node is the only use of N.
797 bool isOnlyUserOf(const SDNode *N) const;
798
799 /// Return true if this node is an operand of N.
800 bool isOperandOf(const SDNode *N) const;
801
802 /// Return true if this node is a predecessor of N.
803 /// NOTE: Implemented on top of hasPredecessor and every bit as
804 /// expensive. Use carefully.
805 bool isPredecessorOf(const SDNode *N) const {
806 return N->hasPredecessor(this);
807 }
808
809 /// Return true if N is a predecessor of this node.
810 /// N is either an operand of this node, or can be reached by recursively
811 /// traversing up the operands.
812 /// NOTE: This is an expensive method. Use it carefully.
813 bool hasPredecessor(const SDNode *N) const;
814
815 /// Returns true if N is a predecessor of any node in Worklist. This
816 /// helper keeps Visited and Worklist sets externally to allow unions
817 /// searches to be performed in parallel, caching of results across
818 /// queries and incremental addition to Worklist. Stops early if N is
819 /// found but will resume. Remember to clear Visited and Worklists
820 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
821 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
822 /// topologically ordered (Operands have strictly smaller node id) and search
823 /// can be pruned leveraging this.
824 static bool hasPredecessorHelper(const SDNode *N,
825 SmallPtrSetImpl<const SDNode *> &Visited,
826 SmallVectorImpl<const SDNode *> &Worklist,
827 unsigned int MaxSteps = 0,
828 bool TopologicalPrune = false) {
829 SmallVector<const SDNode *, 8> DeferredNodes;
830 if (Visited.count(N))
831 return true;
832
833 // Node Id's are assigned in three places: As a topological
834 // ordering (> 0), during legalization (results in values set to
835 // 0), new nodes (set to -1). If N has a topolgical id then we
836 // know that all nodes with ids smaller than it cannot be
837 // successors and we need not check them. Filter out all node
838 // that can't be matches. We add them to the worklist before exit
839 // in case of multiple calls. Note that during selection the topological id
840 // may be violated if a node's predecessor is selected before it. We mark
841 // this at selection negating the id of unselected successors and
842 // restricting topological pruning to positive ids.
843
844 int NId = N->getNodeId();
845 // If we Invalidated the Id, reconstruct original NId.
846 if (NId < -1)
847 NId = -(NId + 1);
848
849 bool Found = false;
850 while (!Worklist.empty()) {
851 const SDNode *M = Worklist.pop_back_val();
852 int MId = M->getNodeId();
853 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
854 (MId > 0) && (MId < NId)) {
855 DeferredNodes.push_back(M);
856 continue;
857 }
858 for (const SDValue &OpV : M->op_values()) {
859 SDNode *Op = OpV.getNode();
860 if (Visited.insert(Op).second)
861 Worklist.push_back(Op);
862 if (Op == N)
863 Found = true;
864 }
865 if (Found)
866 break;
867 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
868 break;
869 }
870 // Push deferred nodes back on worklist.
871 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
872 // If we bailed early, conservatively return found.
873 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
874 return true;
875 return Found;
876 }
877
878 /// Return true if all the users of N are contained in Nodes.
879 /// NOTE: Requires at least one match, but doesn't require them all.
880 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
881
882 /// Return the number of values used by this operation.
883 unsigned getNumOperands() const { return NumOperands; }
884
885 /// Return the maximum number of operands that a SDNode can hold.
886 static constexpr size_t getMaxNumOperands() {
887 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
888 }
889
890 /// Helper method returns the integer value of a ConstantSDNode operand.
891 inline uint64_t getConstantOperandVal(unsigned Num) const;
892
893 /// Helper method returns the APInt of a ConstantSDNode operand.
894 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
895
896 const SDValue &getOperand(unsigned Num) const {
897 assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 897, __extension__ __PRETTY_FUNCTION__))
;
898 return OperandList[Num];
899 }
900
901 using op_iterator = SDUse *;
902
903 op_iterator op_begin() const { return OperandList; }
904 op_iterator op_end() const { return OperandList+NumOperands; }
905 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
906
907 /// Iterator for directly iterating over the operand SDValue's.
908 struct value_op_iterator
909 : iterator_adaptor_base<value_op_iterator, op_iterator,
910 std::random_access_iterator_tag, SDValue,
911 ptrdiff_t, value_op_iterator *,
912 value_op_iterator *> {
913 explicit value_op_iterator(SDUse *U = nullptr)
914 : iterator_adaptor_base(U) {}
915
916 const SDValue &operator*() const { return I->get(); }
917 };
918
919 iterator_range<value_op_iterator> op_values() const {
920 return make_range(value_op_iterator(op_begin()),
921 value_op_iterator(op_end()));
922 }
923
924 SDVTList getVTList() const {
925 SDVTList X = { ValueList, NumValues };
926 return X;
927 }
928
929 /// If this node has a glue operand, return the node
930 /// to which the glue operand points. Otherwise return NULL.
931 SDNode *getGluedNode() const {
932 if (getNumOperands() != 0 &&
933 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
934 return getOperand(getNumOperands()-1).getNode();
935 return nullptr;
936 }
937
938 /// If this node has a glue value with a user, return
939 /// the user (there is at most one). Otherwise return NULL.
940 SDNode *getGluedUser() const {
941 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
942 if (UI.getUse().get().getValueType() == MVT::Glue)
943 return *UI;
944 return nullptr;
945 }
946
947 SDNodeFlags getFlags() const { return Flags; }
948 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
949
950 /// Clear any flags in this node that aren't also set in Flags.
951 /// If Flags is not in a defined state then this has no effect.
952 void intersectFlagsWith(const SDNodeFlags Flags);
953
954 /// Return the number of values defined/returned by this operator.
955 unsigned getNumValues() const { return NumValues; }
956
957 /// Return the type of a specified result.
958 EVT getValueType(unsigned ResNo) const {
959 assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 959, __extension__ __PRETTY_FUNCTION__))
;
960 return ValueList[ResNo];
961 }
962
963 /// Return the type of a specified result as a simple type.
964 MVT getSimpleValueType(unsigned ResNo) const {
965 return getValueType(ResNo).getSimpleVT();
966 }
967
968 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
969 ///
970 /// If the value type is a scalable vector type, the scalable property will
971 /// be set and the runtime size will be a positive integer multiple of the
972 /// base size.
973 TypeSize getValueSizeInBits(unsigned ResNo) const {
974 return getValueType(ResNo).getSizeInBits();
975 }
976
977 using value_iterator = const EVT *;
978
979 value_iterator value_begin() const { return ValueList; }
980 value_iterator value_end() const { return ValueList+NumValues; }
981 iterator_range<value_iterator> values() const {
982 return llvm::make_range(value_begin(), value_end());
983 }
984
985 /// Return the opcode of this operation for printing.
986 std::string getOperationName(const SelectionDAG *G = nullptr) const;
987 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
988 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
989 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
990 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
991 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
992
993 /// Print a SelectionDAG node and all children down to
994 /// the leaves. The given SelectionDAG allows target-specific nodes
995 /// to be printed in human-readable form. Unlike printr, this will
996 /// print the whole DAG, including children that appear multiple
997 /// times.
998 ///
999 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and children up to
1002 /// depth "depth." The given SelectionDAG allows target-specific
1003 /// nodes to be printed in human-readable form. Unlike printr, this
1004 /// will print children that appear multiple times wherever they are
1005 /// used.
1006 ///
1007 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1008 unsigned depth = 100) const;
1009
1010 /// Dump this node, for debugging.
1011 void dump() const;
1012
1013 /// Dump (recursively) this node and its use-def subgraph.
1014 void dumpr() const;
1015
1016 /// Dump this node, for debugging.
1017 /// The given SelectionDAG allows target-specific nodes to be printed
1018 /// in human-readable form.
1019 void dump(const SelectionDAG *G) const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 /// The given SelectionDAG allows target-specific nodes to be printed
1023 /// in human-readable form.
1024 void dumpr(const SelectionDAG *G) const;
1025
1026 /// printrFull to dbgs(). The given SelectionDAG allows
1027 /// target-specific nodes to be printed in human-readable form.
1028 /// Unlike dumpr, this will print the whole DAG, including children
1029 /// that appear multiple times.
1030 void dumprFull(const SelectionDAG *G = nullptr) const;
1031
1032 /// printrWithDepth to dbgs(). The given
1033 /// SelectionDAG allows target-specific nodes to be printed in
1034 /// human-readable form. Unlike dumpr, this will print children
1035 /// that appear multiple times wherever they are used.
1036 ///
1037 void dumprWithDepth(const SelectionDAG *G = nullptr,
1038 unsigned depth = 100) const;
1039
1040 /// Gather unique data for the node.
1041 void Profile(FoldingSetNodeID &ID) const;
1042
1043 /// This method should only be used by the SDUse class.
1044 void addUse(SDUse &U) { U.addToList(&UseList); }
1045
1046protected:
1047 static SDVTList getSDVTList(EVT VT) {
1048 SDVTList Ret = { getValueTypeList(VT), 1 };
1049 return Ret;
1050 }
1051
1052 /// Create an SDNode.
1053 ///
1054 /// SDNodes are created without any operands, and never own the operand
1055 /// storage. To add operands, see SelectionDAG::createOperands.
1056 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1057 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1058 IROrder(Order), debugLoc(std::move(dl)) {
1059 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1060 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
"Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1060, __extension__ __PRETTY_FUNCTION__))
;
1061 assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
1062 "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
;
1063 }
1064
1065 /// Release the operands and set this node to have zero operands.
1066 void DropOperands();
1067};
1068
1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
1080 DebugLoc DL;
1081 int IROrder = 0;
1082
1083public:
1084 SDLoc() = default;
1085 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1086 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1087 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1088 assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1088, __extension__ __PRETTY_FUNCTION__))
;
1089 if (I)
1090 DL = I->getDebugLoc();
1091 }
1092
1093 unsigned getIROrder() const { return IROrder; }
1094 const DebugLoc &getDebugLoc() const { return DL; }
1095};
1096
1097// Define inline functions from the SDValue class.
1098
1099inline SDValue::SDValue(SDNode *node, unsigned resno)
1100 : Node(node), ResNo(resno) {
1101 // Explicitly check for !ResNo to avoid use-after-free, because there are
1102 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1103 // combines.
1104 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
1105 "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
;
1106 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1106, __extension__ __PRETTY_FUNCTION__))
;
1107}
1108
1109inline unsigned SDValue::getOpcode() const {
1110 return Node->getOpcode();
15
Called C++ object pointer is null
1111}
1112
1113inline EVT SDValue::getValueType() const {
1114 return Node->getValueType(ResNo);
1115}
1116
1117inline unsigned SDValue::getNumOperands() const {
1118 return Node->getNumOperands();
1119}
1120
1121inline const SDValue &SDValue::getOperand(unsigned i) const {
1122 return Node->getOperand(i);
1123}
1124
1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1126 return Node->getConstantOperandVal(i);
1127}
1128
1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1130 return Node->getConstantOperandAPInt(i);
1131}
1132
1133inline bool SDValue::isTargetOpcode() const {
1134 return Node->isTargetOpcode();
1135}
1136
1137inline bool SDValue::isTargetMemoryOpcode() const {
1138 return Node->isTargetMemoryOpcode();
1139}
1140
1141inline bool SDValue::isMachineOpcode() const {
1142 return Node->isMachineOpcode();
1143}
1144
1145inline unsigned SDValue::getMachineOpcode() const {
1146 return Node->getMachineOpcode();
1147}
1148
1149inline bool SDValue::isUndef() const {
1150 return Node->isUndef();
1151}
1152
1153inline bool SDValue::use_empty() const {
1154 return !Node->hasAnyUseOfValue(ResNo);
1155}
1156
1157inline bool SDValue::hasOneUse() const {
1158 return Node->hasNUsesOfValue(1, ResNo);
1159}
1160
1161inline const DebugLoc &SDValue::getDebugLoc() const {
1162 return Node->getDebugLoc();
1163}
1164
1165inline void SDValue::dump() const {
1166 return Node->dump();
1167}
1168
1169inline void SDValue::dump(const SelectionDAG *G) const {
1170 return Node->dump(G);
1171}
1172
1173inline void SDValue::dumpr() const {
1174 return Node->dumpr();
1175}
1176
1177inline void SDValue::dumpr(const SelectionDAG *G) const {
1178 return Node->dumpr(G);
1179}
1180
1181// Define inline functions from the SDUse class.
1182
1183inline void SDUse::set(const SDValue &V) {
1184 if (Val.getNode()) removeFromList();
1185 Val = V;
1186 if (V.getNode()) V.getNode()->addUse(*this);
1187}
1188
1189inline void SDUse::setInitial(const SDValue &V) {
1190 Val = V;
1191 V.getNode()->addUse(*this);
1192}
1193
1194inline void SDUse::setNode(SDNode *N) {
1195 if (Val.getNode()) removeFromList();
1196 Val.setNode(N);
1197 if (N) N->addUse(*this);
1198}
1199
1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand. This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
1205 SDUse Op;
1206
1207public:
1208 explicit HandleSDNode(SDValue X)
1209 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1210 // HandleSDNodes are never inserted into the DAG, so they won't be
1211 // auto-numbered. Use ID 65535 as a sentinel.
1212 PersistentId = 0xffff;
1213
1214 // Manually set up the operand list. This node type is special in that it's
1215 // always stack allocated and SelectionDAG does not manage its operands.
1216 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1217 // be so special.
1218 Op.setUser(this);
1219 Op.setInitial(X);
1220 NumOperands = 1;
1221 OperandList = &Op;
1222 }
1223 ~HandleSDNode();
1224
1225 const SDValue &getValue() const { return Op; }
1226};
1227
1228class AddrSpaceCastSDNode : public SDNode {
1229private:
1230 unsigned SrcAddrSpace;
1231 unsigned DestAddrSpace;
1232
1233public:
1234 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1235 unsigned SrcAS, unsigned DestAS);
1236
1237 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1238 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1239
1240 static bool classof(const SDNode *N) {
1241 return N->getOpcode() == ISD::ADDRSPACECAST;
1242 }
1243};
1244
1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
1248 // VT of in-memory value.
1249 EVT MemoryVT;
1250
1251protected:
1252 /// Memory reference information.
1253 MachineMemOperand *MMO;
1254
1255public:
1256 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1257 EVT memvt, MachineMemOperand *MMO);
1258
1259 bool readMem() const { return MMO->isLoad(); }
1260 bool writeMem() const { return MMO->isStore(); }
1261
1262 /// Returns alignment and volatility of the memory access
1263 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1264 Align getAlign() const { return MMO->getAlign(); }
1265 // FIXME: Remove once transition to getAlign is over.
1266 unsigned getAlignment() const { return MMO->getAlign().value(); }
1267
1268 /// Return the SubclassData value, without HasDebugValue. This contains an
1269 /// encoding of the volatile flag, as well as bits used by subclasses. This
1270 /// function should only be used to compute a FoldingSetNodeID value.
1271 /// The HasDebugValue bit is masked out because CSE map needs to match
1272 /// nodes with debug info with nodes without debug info. Same is about
1273 /// isDivergent bit.
1274 unsigned getRawSubclassData() const {
1275 uint16_t Data;
1276 union {
1277 char RawSDNodeBits[sizeof(uint16_t)];
1278 SDNodeBitfields SDNodeBits;
1279 };
1280 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1281 SDNodeBits.HasDebugValue = 0;
1282 SDNodeBits.IsDivergent = false;
1283 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1284 return Data;
1285 }
1286
1287 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1288 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1289 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1290 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1291
1292 // Returns the offset from the location of the access.
1293 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1294
1295 /// Returns the AA info that describes the dereference.
1296 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1297
1298 /// Returns the Ranges that describes the dereference.
1299 const MDNode *getRanges() const { return MMO->getRanges(); }
1300
1301 /// Returns the synchronization scope ID for this memory operation.
1302 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1303
1304 /// Return the atomic ordering requirements for this memory operation. For
1305 /// cmpxchg atomic operations, return the atomic ordering requirements when
1306 /// store occurs.
1307 AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
1308
1309 /// Return a single atomic ordering that is at least as strong as both the
1310 /// success and failure orderings for an atomic operation. (For operations
1311 /// other than cmpxchg, this is equivalent to getOrdering().)
1312 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1313
1314 /// Return true if the memory operation ordering is Unordered or higher.
1315 bool isAtomic() const { return MMO->isAtomic(); }
1316
1317 /// Returns true if the memory operation doesn't imply any ordering
1318 /// constraints on surrounding memory operations beyond the normal memory
1319 /// aliasing rules.
1320 bool isUnordered() const { return MMO->isUnordered(); }
1321
1322 /// Returns true if the memory operation is neither atomic or volatile.
1323 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1324
1325 /// Return the type of the in-memory value.
1326 EVT getMemoryVT() const { return MemoryVT; }
1327
1328 /// Return a MachineMemOperand object describing the memory
1329 /// reference performed by operation.
1330 MachineMemOperand *getMemOperand() const { return MMO; }
1331
1332 const MachinePointerInfo &getPointerInfo() const {
1333 return MMO->getPointerInfo();
1334 }
1335
1336 /// Return the address space for the associated pointer
1337 unsigned getAddressSpace() const {
1338 return getPointerInfo().getAddrSpace();
1339 }
1340
1341 /// Update this MemSDNode's MachineMemOperand information
1342 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1343 /// This must only be used when the new alignment applies to all users of
1344 /// this MachineMemOperand.
1345 void refineAlignment(const MachineMemOperand *NewMMO) {
1346 MMO->refineAlignment(NewMMO);
1347 }
1348
1349 const SDValue &getChain() const { return getOperand(0); }
1350
1351 const SDValue &getBasePtr() const {
1352 switch (getOpcode()) {
1353 case ISD::STORE:
1354 case ISD::MSTORE:
1355 return getOperand(2);
1356 case ISD::MGATHER:
1357 case ISD::MSCATTER:
1358 return getOperand(3);
1359 default:
1360 return getOperand(1);
1361 }
1362 }
1363
1364 // Methods to support isa and dyn_cast
1365 static bool classof(const SDNode *N) {
1366 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1367 // with either an intrinsic or a target opcode.
1368 return N->getOpcode() == ISD::LOAD ||
1369 N->getOpcode() == ISD::STORE ||
1370 N->getOpcode() == ISD::PREFETCH ||
1371 N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1372 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1373 N->getOpcode() == ISD::ATOMIC_SWAP ||
1374 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1375 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1376 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1377 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1378 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1379 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1380 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1381 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1382 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1383 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1384 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1385 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1386 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1387 N->getOpcode() == ISD::ATOMIC_LOAD ||
1388 N->getOpcode() == ISD::ATOMIC_STORE ||
1389 N->getOpcode() == ISD::MLOAD ||
1390 N->getOpcode() == ISD::MSTORE ||
1391 N->getOpcode() == ISD::MGATHER ||
1392 N->getOpcode() == ISD::MSCATTER ||
1393 N->isMemIntrinsic() ||
1394 N->isTargetMemoryOpcode();
1395 }
1396};
1397
1398/// This is an SDNode representing atomic operations.
1399class AtomicSDNode : public MemSDNode {
1400public:
1401 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1402 EVT MemVT, MachineMemOperand *MMO)
1403 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1404 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1405, __extension__ __PRETTY_FUNCTION__))
1405 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1405, __extension__ __PRETTY_FUNCTION__))
;
1406 }
1407
1408 const SDValue &getBasePtr() const { return getOperand(1); }
1409 const SDValue &getVal() const { return getOperand(2); }
1410
1411 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1412 /// otherwise.
1413 bool isCompareAndSwap() const {
1414 unsigned Op = getOpcode();
1415 return Op == ISD::ATOMIC_CMP_SWAP ||
1416 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1417 }
1418
1419 /// For cmpxchg atomic operations, return the atomic ordering requirements
1420 /// when store does not occur.
1421 AtomicOrdering getFailureOrdering() const {
1422 assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1422, __extension__ __PRETTY_FUNCTION__))
;
1423 return MMO->getFailureOrdering();
1424 }
1425
1426 // Methods to support isa and dyn_cast
1427 static bool classof(const SDNode *N) {
1428 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1429 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1430 N->getOpcode() == ISD::ATOMIC_SWAP ||
1431 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1432 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1433 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1434 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1435 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1436 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1437 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1438 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1439 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1440 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1441 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1442 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1443 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1444 N->getOpcode() == ISD::ATOMIC_LOAD ||
1445 N->getOpcode() == ISD::ATOMIC_STORE;
1446 }
1447};
1448
1449/// This SDNode is used for target intrinsics that touch
1450/// memory and need an associated MachineMemOperand. Its opcode may be
1451/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1452/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1453class MemIntrinsicSDNode : public MemSDNode {
1454public:
1455 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1456 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1457 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1458 SDNodeBits.IsMemIntrinsic = true;
1459 }
1460
1461 // Methods to support isa and dyn_cast
1462 static bool classof(const SDNode *N) {
1463 // We lower some target intrinsics to their target opcode
1464 // early a node with a target opcode can be of this class
1465 return N->isMemIntrinsic() ||
1466 N->getOpcode() == ISD::PREFETCH ||
1467 N->isTargetMemoryOpcode();
1468 }
1469};
1470
1471/// This SDNode is used to implement the code generator
1472/// support for the llvm IR shufflevector instruction. It combines elements
1473/// from two input vectors into a new input vector, with the selection and
1474/// ordering of elements determined by an array of integers, referred to as
1475/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1476/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1477/// An index of -1 is treated as undef, such that the code generator may put
1478/// any value in the corresponding element of the result.
1479class ShuffleVectorSDNode : public SDNode {
1480 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1481 // is freed when the SelectionDAG object is destroyed.
1482 const int *Mask;
1483
1484protected:
1485 friend class SelectionDAG;
1486
1487 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1488 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1489
1490public:
1491 ArrayRef<int> getMask() const {
1492 EVT VT = getValueType(0);
1493 return makeArrayRef(Mask, VT.getVectorNumElements());
1494 }
1495
1496 int getMaskElt(unsigned Idx) const {
1497 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1497, __extension__ __PRETTY_FUNCTION__))
;
1498 return Mask[Idx];
1499 }
1500
1501 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1502
1503 int getSplatIndex() const {
1504 assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1504, __extension__ __PRETTY_FUNCTION__))
;
1505 EVT VT = getValueType(0);
1506 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1507 if (Mask[i] >= 0)
1508 return Mask[i];
1509
1510 // We can choose any index value here and be correct because all elements
1511 // are undefined. Return 0 for better potential for callers to simplify.
1512 return 0;
1513 }
1514
1515 static bool isSplatMask(const int *Mask, EVT VT);
1516
1517 /// Change values in a shuffle permute mask assuming
1518 /// the two vector operands have swapped position.
1519 static void commuteMask(MutableArrayRef<int> Mask) {
1520 unsigned NumElems = Mask.size();
1521 for (unsigned i = 0; i != NumElems; ++i) {
1522 int idx = Mask[i];
1523 if (idx < 0)
1524 continue;
1525 else if (idx < (int)NumElems)
1526 Mask[i] = idx + NumElems;
1527 else
1528 Mask[i] = idx - NumElems;
1529 }
1530 }
1531
1532 static bool classof(const SDNode *N) {
1533 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1534 }
1535};
1536
1537class ConstantSDNode : public SDNode {
1538 friend class SelectionDAG;
1539
1540 const ConstantInt *Value;
1541
1542 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1543 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1544 getSDVTList(VT)),
1545 Value(val) {
1546 ConstantSDNodeBits.IsOpaque = isOpaque;
1547 }
1548
1549public:
1550 const ConstantInt *getConstantIntValue() const { return Value; }
1551 const APInt &getAPIntValue() const { return Value->getValue(); }
1552 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1553 int64_t getSExtValue() const { return Value->getSExtValue(); }
1554 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1555 return Value->getLimitedValue(Limit);
1556 }
1557 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1558 Align getAlignValue() const { return Value->getAlignValue(); }
1559
1560 bool isOne() const { return Value->isOne(); }
1561 bool isNullValue() const { return Value->isZero(); }
1562 bool isAllOnesValue() const { return Value->isMinusOne(); }
1563
1564 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1565
1566 static bool classof(const SDNode *N) {
1567 return N->getOpcode() == ISD::Constant ||
1568 N->getOpcode() == ISD::TargetConstant;
1569 }
1570};
1571
1572uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1573 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1574}
1575
1576const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1577 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1578}
1579
1580class ConstantFPSDNode : public SDNode {
1581 friend class SelectionDAG;
1582
1583 const ConstantFP *Value;
1584
1585 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1586 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1587 DebugLoc(), getSDVTList(VT)),
1588 Value(val) {}
1589
1590public:
1591 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1592 const ConstantFP *getConstantFPValue() const { return Value; }
1593
1594 /// Return true if the value is positive or negative zero.
1595 bool isZero() const { return Value->isZero(); }
1596
1597 /// Return true if the value is a NaN.
1598 bool isNaN() const { return Value->isNaN(); }
1599
1600 /// Return true if the value is an infinity
1601 bool isInfinity() const { return Value->isInfinity(); }
1602
1603 /// Return true if the value is negative.
1604 bool isNegative() const { return Value->isNegative(); }
1605
1606 /// We don't rely on operator== working on double values, as
1607 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1608 /// As such, this method can be used to do an exact bit-for-bit comparison of
1609 /// two floating point values.
1610
1611 /// We leave the version with the double argument here because it's just so
1612 /// convenient to write "2.0" and the like. Without this function we'd
1613 /// have to duplicate its logic everywhere it's called.
1614 bool isExactlyValue(double V) const {
1615 return Value->getValueAPF().isExactlyValue(V);
1616 }
1617 bool isExactlyValue(const APFloat& V) const;
1618
1619 static bool isValueValidForType(EVT VT, const APFloat& Val);
1620
1621 static bool classof(const SDNode *N) {
1622 return N->getOpcode() == ISD::ConstantFP ||
1623 N->getOpcode() == ISD::TargetConstantFP;
1624 }
1625};
1626
1627/// Returns true if \p V is a constant integer zero.
1628bool isNullConstant(SDValue V);
1629
1630/// Returns true if \p V is an FP constant with a value of positive zero.
1631bool isNullFPConstant(SDValue V);
1632
1633/// Returns true if \p V is an integer constant with all bits set.
1634bool isAllOnesConstant(SDValue V);
1635
1636/// Returns true if \p V is a constant integer one.
1637bool isOneConstant(SDValue V);
1638
1639/// Return the non-bitcasted source operand of \p V if it exists.
1640/// If \p V is not a bitcasted value, it is returned as-is.
1641SDValue peekThroughBitcasts(SDValue V);
1642
1643/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1644/// If \p V is not a bitcasted one-use value, it is returned as-is.
1645SDValue peekThroughOneUseBitcasts(SDValue V);
1646
1647/// Return the non-extracted vector source operand of \p V if it exists.
1648/// If \p V is not an extracted subvector, it is returned as-is.
1649SDValue peekThroughExtractSubvectors(SDValue V);
1650
1651/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1652/// constant is canonicalized to be operand 1.
1653bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1654
1655/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1656ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1657 bool AllowTruncation = false);
1658
1659/// Returns the SDNode if it is a demanded constant splat BuildVector or
1660/// constant int.
1661ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1662 bool AllowUndefs = false,
1663 bool AllowTruncation = false);
1664
1665/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1666ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1667
1668/// Returns the SDNode if it is a demanded constant splat BuildVector or
1669/// constant float.
1670ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1671 bool AllowUndefs = false);
1672
1673/// Return true if the value is a constant 0 integer or a splatted vector of
1674/// a constant 0 integer (with no undefs by default).
1675/// Build vector implicit truncation is not an issue for null values.
1676bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1677
1678/// Return true if the value is a constant 1 integer or a splatted vector of a
1679/// constant 1 integer (with no undefs).
1680/// Does not permit build vector implicit truncation.
1681bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1682
1683/// Return true if the value is a constant -1 integer or a splatted vector of a
1684/// constant -1 integer (with no undefs).
1685/// Does not permit build vector implicit truncation.
1686bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1687
1688/// Return true if \p V is either a integer or FP constant.
1689inline bool isIntOrFPConstant(SDValue V) {
1690 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1691}
1692
1693class GlobalAddressSDNode : public SDNode {
1694 friend class SelectionDAG;
1695
1696 const GlobalValue *TheGlobal;
1697 int64_t Offset;
1698 unsigned TargetFlags;
1699
1700 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1701 const GlobalValue *GA, EVT VT, int64_t o,
1702 unsigned TF);
1703
1704public:
1705 const GlobalValue *getGlobal() const { return TheGlobal; }
1706 int64_t getOffset() const { return Offset; }
1707 unsigned getTargetFlags() const { return TargetFlags; }
1708 // Return the address space this GlobalAddress belongs to.
1709 unsigned getAddressSpace() const;
1710
1711 static bool classof(const SDNode *N) {
1712 return N->getOpcode() == ISD::GlobalAddress ||
1713 N->getOpcode() == ISD::TargetGlobalAddress ||
1714 N->getOpcode() == ISD::GlobalTLSAddress ||
1715 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1716 }
1717};
1718
1719class FrameIndexSDNode : public SDNode {
1720 friend class SelectionDAG;
1721
1722 int FI;
1723
1724 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1725 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1726 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1727 }
1728
1729public:
1730 int getIndex() const { return FI; }
1731
1732 static bool classof(const SDNode *N) {
1733 return N->getOpcode() == ISD::FrameIndex ||
1734 N->getOpcode() == ISD::TargetFrameIndex;
1735 }
1736};
1737
1738/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1739/// the offet and size that are started/ended in the underlying FrameIndex.
1740class LifetimeSDNode : public SDNode {
1741 friend class SelectionDAG;
1742 int64_t Size;
1743 int64_t Offset; // -1 if offset is unknown.
1744
1745 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1746 SDVTList VTs, int64_t Size, int64_t Offset)
1747 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1748public:
1749 int64_t getFrameIndex() const {
1750 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1751 }
1752
1753 bool hasOffset() const { return Offset >= 0; }
1754 int64_t getOffset() const {
1755 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1755, __extension__ __PRETTY_FUNCTION__))
;
1756 return Offset;
1757 }
1758 int64_t getSize() const {
1759 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1759, __extension__ __PRETTY_FUNCTION__))
;
1760 return Size;
1761 }
1762
1763 // Methods to support isa and dyn_cast
1764 static bool classof(const SDNode *N) {
1765 return N->getOpcode() == ISD::LIFETIME_START ||
1766 N->getOpcode() == ISD::LIFETIME_END;
1767 }
1768};
1769
1770/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1771/// the index of the basic block being probed. A pseudo probe serves as a place
1772/// holder and will be removed at the end of compilation. It does not have any
1773/// operand because we do not want the instruction selection to deal with any.
1774class PseudoProbeSDNode : public SDNode {
1775 friend class SelectionDAG;
1776 uint64_t Guid;
1777 uint64_t Index;
1778 uint32_t Attributes;
1779
1780 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1781 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1782 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1783 Attributes(Attr) {}
1784
1785public:
1786 uint64_t getGuid() const { return Guid; }
1787 uint64_t getIndex() const { return Index; }
1788 uint32_t getAttributes() const { return Attributes; }
1789
1790 // Methods to support isa and dyn_cast
1791 static bool classof(const SDNode *N) {
1792 return N->getOpcode() == ISD::PSEUDO_PROBE;
1793 }
1794};
1795
1796class JumpTableSDNode : public SDNode {
1797 friend class SelectionDAG;
1798
1799 int JTI;
1800 unsigned TargetFlags;
1801
1802 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1803 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1804 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1805 }
1806
1807public:
1808 int getIndex() const { return JTI; }
1809 unsigned getTargetFlags() const { return TargetFlags; }
1810
1811 static bool classof(const SDNode *N) {
1812 return N->getOpcode() == ISD::JumpTable ||
1813 N->getOpcode() == ISD::TargetJumpTable;
1814 }
1815};
1816
1817class ConstantPoolSDNode : public SDNode {
1818 friend class SelectionDAG;
1819
1820 union {
1821 const Constant *ConstVal;
1822 MachineConstantPoolValue *MachineCPVal;
1823 } Val;
1824 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1825 Align Alignment; // Minimum alignment requirement of CP.
1826 unsigned TargetFlags;
1827
1828 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1829 Align Alignment, unsigned TF)
1830 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1831 DebugLoc(), getSDVTList(VT)),
1832 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1833 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1833, __extension__ __PRETTY_FUNCTION__))
;
1834 Val.ConstVal = c;
1835 }
1836
1837 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1838 Align Alignment, unsigned TF)
1839 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1840 DebugLoc(), getSDVTList(VT)),
1841 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1842 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1842, __extension__ __PRETTY_FUNCTION__))
;
1843 Val.MachineCPVal = v;
1844 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1845 }
1846
1847public:
1848 bool isMachineConstantPoolEntry() const {
1849 return Offset < 0;
1850 }
1851
1852 const Constant *getConstVal() const {
1853 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1853, __extension__ __PRETTY_FUNCTION__))
;
1854 return Val.ConstVal;
1855 }
1856
1857 MachineConstantPoolValue *getMachineCPVal() const {
1858 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1858, __extension__ __PRETTY_FUNCTION__))
;
1859 return Val.MachineCPVal;
1860 }
1861
1862 int getOffset() const {
1863 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1864 }
1865
1866 // Return the alignment of this constant pool object, which is either 0 (for
1867 // default alignment) or the desired value.
1868 Align getAlign() const { return Alignment; }
1869 unsigned getTargetFlags() const { return TargetFlags; }
1870
1871 Type *getType() const;
1872
1873 static bool classof(const SDNode *N) {
1874 return N->getOpcode() == ISD::ConstantPool ||
1875 N->getOpcode() == ISD::TargetConstantPool;
1876 }
1877};
1878
1879/// Completely target-dependent object reference.
1880class TargetIndexSDNode : public SDNode {
1881 friend class SelectionDAG;
1882
1883 unsigned TargetFlags;
1884 int Index;
1885 int64_t Offset;
1886
1887public:
1888 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1889 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1890 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1891
1892 unsigned getTargetFlags() const { return TargetFlags; }
1893 int getIndex() const { return Index; }
1894 int64_t getOffset() const { return Offset; }
1895
1896 static bool classof(const SDNode *N) {
1897 return N->getOpcode() == ISD::TargetIndex;
1898 }
1899};
1900
1901class BasicBlockSDNode : public SDNode {
1902 friend class SelectionDAG;
1903
1904 MachineBasicBlock *MBB;
1905
1906 /// Debug info is meaningful and potentially useful here, but we create
1907 /// blocks out of order when they're jumped to, which makes it a bit
1908 /// harder. Let's see if we need it first.
1909 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1910 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1911 {}
1912
1913public:
1914 MachineBasicBlock *getBasicBlock() const { return MBB; }
1915
1916 static bool classof(const SDNode *N) {
1917 return N->getOpcode() == ISD::BasicBlock;
1918 }
1919};
1920
1921/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1922class BuildVectorSDNode : public SDNode {
1923public:
1924 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1925 explicit BuildVectorSDNode() = delete;
1926
1927 /// Check if this is a constant splat, and if so, find the
1928 /// smallest element size that splats the vector. If MinSplatBits is
1929 /// nonzero, the element size must be at least that large. Note that the
1930 /// splat element may be the entire vector (i.e., a one element vector).
1931 /// Returns the splat element value in SplatValue. Any undefined bits in
1932 /// that value are zero, and the corresponding bits in the SplatUndef mask
1933 /// are set. The SplatBitSize value is set to the splat element size in
1934 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1935 /// undefined. isBigEndian describes the endianness of the target.
1936 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1937 unsigned &SplatBitSize, bool &HasAnyUndefs,
1938 unsigned MinSplatBits = 0,
1939 bool isBigEndian = false) const;
1940
1941 /// Returns the demanded splatted value or a null value if this is not a
1942 /// splat.
1943 ///
1944 /// The DemandedElts mask indicates the elements that must be in the splat.
1945 /// If passed a non-null UndefElements bitvector, it will resize it to match
1946 /// the vector width and set the bits where elements are undef.
1947 SDValue getSplatValue(const APInt &DemandedElts,
1948 BitVector *UndefElements = nullptr) const;
1949
1950 /// Returns the splatted value or a null value if this is not a splat.
1951 ///
1952 /// If passed a non-null UndefElements bitvector, it will resize it to match
1953 /// the vector width and set the bits where elements are undef.
1954 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1955
1956 /// Find the shortest repeating sequence of values in the build vector.
1957 ///
1958 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1959 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1960 ///
1961 /// Currently this must be a power-of-2 build vector.
1962 /// The DemandedElts mask indicates the elements that must be present,
1963 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1964 /// non-null UndefElements bitvector, it will resize it to match the original
1965 /// vector width and set the bits where elements are undef. If result is
1966 /// false, Sequence will be empty.
1967 bool getRepeatedSequence(const APInt &DemandedElts,
1968 SmallVectorImpl<SDValue> &Sequence,
1969 BitVector *UndefElements = nullptr) const;
1970
1971 /// Find the shortest repeating sequence of values in the build vector.
1972 ///
1973 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1974 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1975 ///
1976 /// Currently this must be a power-of-2 build vector.
1977 /// If passed a non-null UndefElements bitvector, it will resize it to match
1978 /// the original vector width and set the bits where elements are undef.
1979 /// If result is false, Sequence will be empty.
1980 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
1981 BitVector *UndefElements = nullptr) const;
1982
1983 /// Returns the demanded splatted constant or null if this is not a constant
1984 /// splat.
1985 ///
1986 /// The DemandedElts mask indicates the elements that must be in the splat.
1987 /// If passed a non-null UndefElements bitvector, it will resize it to match
1988 /// the vector width and set the bits where elements are undef.
1989 ConstantSDNode *
1990 getConstantSplatNode(const APInt &DemandedElts,
1991 BitVector *UndefElements = nullptr) const;
1992
1993 /// Returns the splatted constant or null if this is not a constant
1994 /// splat.
1995 ///
1996 /// If passed a non-null UndefElements bitvector, it will resize it to match
1997 /// the vector width and set the bits where elements are undef.
1998 ConstantSDNode *
1999 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2000
2001 /// Returns the demanded splatted constant FP or null if this is not a
2002 /// constant FP splat.
2003 ///
2004 /// The DemandedElts mask indicates the elements that must be in the splat.
2005 /// If passed a non-null UndefElements bitvector, it will resize it to match
2006 /// the vector width and set the bits where elements are undef.
2007 ConstantFPSDNode *
2008 getConstantFPSplatNode(const APInt &DemandedElts,
2009 BitVector *UndefElements = nullptr) const;
2010
2011 /// Returns the splatted constant FP or null if this is not a constant
2012 /// FP splat.
2013 ///
2014 /// If passed a non-null UndefElements bitvector, it will resize it to match
2015 /// the vector width and set the bits where elements are undef.
2016 ConstantFPSDNode *
2017 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2018
2019 /// If this is a constant FP splat and the splatted constant FP is an
2020 /// exact power or 2, return the log base 2 integer value. Otherwise,
2021 /// return -1.
2022 ///
2023 /// The BitWidth specifies the necessary bit precision.
2024 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2025 uint32_t BitWidth) const;
2026
2027 bool isConstant() const;
2028
2029 static bool classof(const SDNode *N) {
2030 return N->getOpcode() == ISD::BUILD_VECTOR;
2031 }
2032};
2033
2034/// An SDNode that holds an arbitrary LLVM IR Value. This is
2035/// used when the SelectionDAG needs to make a simple reference to something
2036/// in the LLVM IR representation.
2037///
2038class SrcValueSDNode : public SDNode {
2039 friend class SelectionDAG;
2040
2041 const Value *V;
2042
2043 /// Create a SrcValue for a general value.
2044 explicit SrcValueSDNode(const Value *v)
2045 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2046
2047public:
2048 /// Return the contained Value.
2049 const Value *getValue() const { return V; }
2050
2051 static bool classof(const SDNode *N) {
2052 return N->getOpcode() == ISD::SRCVALUE;
2053 }
2054};
2055
2056class MDNodeSDNode : public SDNode {
2057 friend class SelectionDAG;
2058
2059 const MDNode *MD;
2060
2061 explicit MDNodeSDNode(const MDNode *md)
2062 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2063 {}
2064
2065public:
2066 const MDNode *getMD() const { return MD; }
2067
2068 static bool classof(const SDNode *N) {
2069 return N->getOpcode() == ISD::MDNODE_SDNODE;
2070 }
2071};
2072
2073class RegisterSDNode : public SDNode {
2074 friend class SelectionDAG;
2075
2076 Register Reg;
2077
2078 RegisterSDNode(Register reg, EVT VT)
2079 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2080
2081public:
2082 Register getReg() const { return Reg; }
2083
2084 static bool classof(const SDNode *N) {
2085 return N->getOpcode() == ISD::Register;
2086 }
2087};
2088
2089class RegisterMaskSDNode : public SDNode {
2090 friend class SelectionDAG;
2091
2092 // The memory for RegMask is not owned by the node.
2093 const uint32_t *RegMask;
2094
2095 RegisterMaskSDNode(const uint32_t *mask)
2096 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2097 RegMask(mask) {}
2098
2099public:
2100 const uint32_t *getRegMask() const { return RegMask; }
2101
2102 static bool classof(const SDNode *N) {
2103 return N->getOpcode() == ISD::RegisterMask;
2104 }
2105};
2106
2107class BlockAddressSDNode : public SDNode {
2108 friend class SelectionDAG;
2109
2110 const BlockAddress *BA;
2111 int64_t Offset;
2112 unsigned TargetFlags;
2113
2114 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2115 int64_t o, unsigned Flags)
2116 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2117 BA(ba), Offset(o), TargetFlags(Flags) {}
2118
2119public:
2120 const BlockAddress *getBlockAddress() const { return BA; }
2121 int64_t getOffset() const { return Offset; }
2122 unsigned getTargetFlags() const { return TargetFlags; }
2123
2124 static bool classof(const SDNode *N) {
2125 return N->getOpcode() == ISD::BlockAddress ||
2126 N->getOpcode() == ISD::TargetBlockAddress;
2127 }
2128};
2129
2130class LabelSDNode : public SDNode {
2131 friend class SelectionDAG;
2132
2133 MCSymbol *Label;
2134
2135 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2136 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2137 assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
"not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2137, __extension__ __PRETTY_FUNCTION__))
;
2138 }
2139
2140public:
2141 MCSymbol *getLabel() const { return Label; }
2142
2143 static bool classof(const SDNode *N) {
2144 return N->getOpcode() == ISD::EH_LABEL ||
2145 N->getOpcode() == ISD::ANNOTATION_LABEL;
2146 }
2147};
2148
2149class ExternalSymbolSDNode : public SDNode {
2150 friend class SelectionDAG;
2151
2152 const char *Symbol;
2153 unsigned TargetFlags;
2154
2155 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2156 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2157 DebugLoc(), getSDVTList(VT)),
2158 Symbol(Sym), TargetFlags(TF) {}
2159
2160public:
2161 const char *getSymbol() const { return Symbol; }
2162 unsigned getTargetFlags() const { return TargetFlags; }
2163
2164 static bool classof(const SDNode *N) {
2165 return N->getOpcode() == ISD::ExternalSymbol ||
2166 N->getOpcode() == ISD::TargetExternalSymbol;
2167 }
2168};
2169
2170class MCSymbolSDNode : public SDNode {
2171 friend class SelectionDAG;
2172
2173 MCSymbol *Symbol;
2174
2175 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2176 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2177
2178public:
2179 MCSymbol *getMCSymbol() const { return Symbol; }
2180
2181 static bool classof(const SDNode *N) {
2182 return N->getOpcode() == ISD::MCSymbol;
2183 }
2184};
2185
2186class CondCodeSDNode : public SDNode {
2187 friend class SelectionDAG;
2188
2189 ISD::CondCode Condition;
2190
2191 explicit CondCodeSDNode(ISD::CondCode Cond)
2192 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2193 Condition(Cond) {}
2194
2195public:
2196 ISD::CondCode get() const { return Condition; }
2197
2198 static bool classof(const SDNode *N) {
2199 return N->getOpcode() == ISD::CONDCODE;
2200 }
2201};
2202
2203/// This class is used to represent EVT's, which are used
2204/// to parameterize some operations.
2205class VTSDNode : public SDNode {
2206 friend class SelectionDAG;
2207
2208 EVT ValueType;
2209
2210 explicit VTSDNode(EVT VT)
2211 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2212 ValueType(VT) {}
2213
2214public:
2215 EVT getVT() const { return ValueType; }
2216
2217 static bool classof(const SDNode *N) {
2218 return N->getOpcode() == ISD::VALUETYPE;
2219 }
2220};
2221
2222/// Base class for LoadSDNode and StoreSDNode
2223class LSBaseSDNode : public MemSDNode {
2224public:
2225 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2226 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2227 MachineMemOperand *MMO)
2228 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2229 LSBaseSDNodeBits.AddressingMode = AM;
2230 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2230, __extension__ __PRETTY_FUNCTION__))
;
2231 }
2232
2233 const SDValue &getOffset() const {
2234 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2235 }
2236
2237 /// Return the addressing mode for this load or store:
2238 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2239 ISD::MemIndexedMode getAddressingMode() const {
2240 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2241 }
2242
2243 /// Return true if this is a pre/post inc/dec load/store.
2244 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2245
2246 /// Return true if this is NOT a pre/post inc/dec load/store.
2247 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2248
2249 static bool classof(const SDNode *N) {
2250 return N->getOpcode() == ISD::LOAD ||
2251 N->getOpcode() == ISD::STORE;
2252 }
2253};
2254
2255/// This class is used to represent ISD::LOAD nodes.
2256class LoadSDNode : public LSBaseSDNode {
2257 friend class SelectionDAG;
2258
2259 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2260 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2261 MachineMemOperand *MMO)
2262 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2263 LoadSDNodeBits.ExtTy = ETy;
2264 assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2264, __extension__ __PRETTY_FUNCTION__))
;
2265 assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-13~++20210615111110+88da6c1ead3f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2265, __extension__ __PRETTY_FUNCTION__))
;
2266 }
2267
2268public:
2269 /// Return whether this is a plain node,
2270 /// or one of the varieties of value-extending loads.
2271 ISD::LoadExtType getExtensionType() const {
2272 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2273 }
2274
2275 const SDValue &getBasePtr() const { return getOperand(1); }
2276 const SDValue &getOffset() const { return getOperand(2); }
2277
2278 static bool classof(const SDNode *N) {
2279 return N->getOpcode() == ISD::LOAD;
2280 }
2281};
2282
2283/// This class is used to represent ISD::STORE nodes.
2284class StoreSDNode : public LSBaseSDNode {
2285 friend class SelectionDAG;
2286
2287 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2288 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2289 MachineMemOperand *MMO)
2290 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2291 StoreSDNodeBits.IsTruncating = isTrunc;
2292 assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <