Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1110, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name HexagonISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/Hexagon -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/Hexagon -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/Hexagon -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-08-28-193554-24367-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp

1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/RuntimeLibcalls.h"
31#include "llvm/CodeGen/SelectionDAG.h"
32#include "llvm/CodeGen/TargetCallingConv.h"
33#include "llvm/CodeGen/ValueTypes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/CallingConv.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/DiagnosticInfo.h"
39#include "llvm/IR/DiagnosticPrinter.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/Module.h"
49#include "llvm/IR/Type.h"
50#include "llvm/IR/Value.h"
51#include "llvm/MC/MCRegisterInfo.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CodeGen.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/Debug.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/MathExtras.h"
58#include "llvm/Support/raw_ostream.h"
59#include "llvm/Target/TargetMachine.h"
60#include <algorithm>
61#include <cassert>
62#include <cstddef>
63#include <cstdint>
64#include <limits>
65#include <utility>
66
67using namespace llvm;
68
69#define DEBUG_TYPE"hexagon-lowering" "hexagon-lowering"
70
71static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
72 cl::init(true), cl::Hidden,
73 cl::desc("Control jump table emission on Hexagon target"));
74
75static cl::opt<bool> EnableHexSDNodeSched("enable-hexagon-sdnode-sched",
76 cl::Hidden, cl::ZeroOrMore, cl::init(false),
77 cl::desc("Enable Hexagon SDNode scheduling"));
78
79static cl::opt<bool> EnableFastMath("ffast-math",
80 cl::Hidden, cl::ZeroOrMore, cl::init(false),
81 cl::desc("Enable Fast Math processing"));
82
83static cl::opt<int> MinimumJumpTables("minimum-jump-tables",
84 cl::Hidden, cl::ZeroOrMore, cl::init(5),
85 cl::desc("Set minimum jump tables"));
86
87static cl::opt<int> MaxStoresPerMemcpyCL("max-store-memcpy",
88 cl::Hidden, cl::ZeroOrMore, cl::init(6),
89 cl::desc("Max #stores to inline memcpy"));
90
91static cl::opt<int> MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os",
92 cl::Hidden, cl::ZeroOrMore, cl::init(4),
93 cl::desc("Max #stores to inline memcpy"));
94
95static cl::opt<int> MaxStoresPerMemmoveCL("max-store-memmove",
96 cl::Hidden, cl::ZeroOrMore, cl::init(6),
97 cl::desc("Max #stores to inline memmove"));
98
99static cl::opt<int> MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os",
100 cl::Hidden, cl::ZeroOrMore, cl::init(4),
101 cl::desc("Max #stores to inline memmove"));
102
103static cl::opt<int> MaxStoresPerMemsetCL("max-store-memset",
104 cl::Hidden, cl::ZeroOrMore, cl::init(8),
105 cl::desc("Max #stores to inline memset"));
106
107static cl::opt<int> MaxStoresPerMemsetOptSizeCL("max-store-memset-Os",
108 cl::Hidden, cl::ZeroOrMore, cl::init(4),
109 cl::desc("Max #stores to inline memset"));
110
111static cl::opt<bool> AlignLoads("hexagon-align-loads",
112 cl::Hidden, cl::init(false),
113 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
114
115static cl::opt<bool>
116 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
117 cl::init(false),
118 cl::desc("Disable minimum alignment of 1 for "
119 "arguments passed by value on stack"));
120
121namespace {
122
123 class HexagonCCState : public CCState {
124 unsigned NumNamedVarArgParams = 0;
125
126 public:
127 HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF,
128 SmallVectorImpl<CCValAssign> &locs, LLVMContext &C,
129 unsigned NumNamedArgs)
130 : CCState(CC, IsVarArg, MF, locs, C),
131 NumNamedVarArgParams(NumNamedArgs) {}
132 unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
133 };
134
135} // end anonymous namespace
136
137
138// Implement calling convention for Hexagon.
139
140static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
141 CCValAssign::LocInfo &LocInfo,
142 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
143 static const MCPhysReg ArgRegs[] = {
144 Hexagon::R0, Hexagon::R1, Hexagon::R2,
145 Hexagon::R3, Hexagon::R4, Hexagon::R5
146 };
147 const unsigned NumArgRegs = array_lengthof(ArgRegs);
148 unsigned RegNum = State.getFirstUnallocated(ArgRegs);
149
150 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
151 if (RegNum != NumArgRegs && RegNum % 2 == 1)
152 State.AllocateReg(ArgRegs[RegNum]);
153
154 // Always return false here, as this function only makes sure that the first
155 // unallocated register has an even register number and does not actually
156 // allocate a register for the current argument.
157 return false;
158}
159
160#include "HexagonGenCallingConv.inc"
161
162
163SDValue
164HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
165 const {
166 return SDValue();
167}
168
169/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
170/// by "Src" to address "Dst" of size "Size". Alignment information is
171/// specified by the specific parameter attribute. The copy will be passed as
172/// a byval function parameter. Sometimes what we are copying is the end of a
173/// larger object, the part that does not fit in registers.
174static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
175 SDValue Chain, ISD::ArgFlagsTy Flags,
176 SelectionDAG &DAG, const SDLoc &dl) {
177 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
178 return DAG.getMemcpy(
179 Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
180 /*isVolatile=*/false, /*AlwaysInline=*/false,
181 /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
182}
183
184bool
185HexagonTargetLowering::CanLowerReturn(
186 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
187 const SmallVectorImpl<ISD::OutputArg> &Outs,
188 LLVMContext &Context) const {
189 SmallVector<CCValAssign, 16> RVLocs;
190 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
191
192 if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
193 return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX);
194 return CCInfo.CheckReturn(Outs, RetCC_Hexagon);
195}
196
197// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
198// passed by value, the function prototype is modified to return void and
199// the value is stored in memory pointed by a pointer passed by caller.
200SDValue
201HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
202 bool IsVarArg,
203 const SmallVectorImpl<ISD::OutputArg> &Outs,
204 const SmallVectorImpl<SDValue> &OutVals,
205 const SDLoc &dl, SelectionDAG &DAG) const {
206 // CCValAssign - represent the assignment of the return value to locations.
207 SmallVector<CCValAssign, 16> RVLocs;
208
209 // CCState - Info about the registers and stack slot.
210 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
211 *DAG.getContext());
212
213 // Analyze return values of ISD::RET
214 if (Subtarget.useHVXOps())
215 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX);
216 else
217 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon);
218
219 SDValue Flag;
220 SmallVector<SDValue, 4> RetOps(1, Chain);
221
222 // Copy the result values into the output registers.
223 for (unsigned i = 0; i != RVLocs.size(); ++i) {
224 CCValAssign &VA = RVLocs[i];
225 SDValue Val = OutVals[i];
226
227 switch (VA.getLocInfo()) {
228 default:
229 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
230 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 230)
;
231 case CCValAssign::Full:
232 break;
233 case CCValAssign::BCvt:
234 Val = DAG.getBitcast(VA.getLocVT(), Val);
235 break;
236 case CCValAssign::SExt:
237 Val = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Val);
238 break;
239 case CCValAssign::ZExt:
240 Val = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Val);
241 break;
242 case CCValAssign::AExt:
243 Val = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Val);
244 break;
245 }
246
247 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Val, Flag);
248
249 // Guarantee that all emitted copies are stuck together with flags.
250 Flag = Chain.getValue(1);
251 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
252 }
253
254 RetOps[0] = Chain; // Update chain.
255
256 // Add the flag if we have it.
257 if (Flag.getNode())
258 RetOps.push_back(Flag);
259
260 return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps);
261}
262
263bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
264 // If either no tail call or told not to tail call at all, don't.
265 return CI->isTailCall();
266}
267
268Register HexagonTargetLowering::getRegisterByName(
269 const char* RegName, LLT VT, const MachineFunction &) const {
270 // Just support r19, the linux kernel uses it.
271 Register Reg = StringSwitch<Register>(RegName)
272 .Case("r0", Hexagon::R0)
273 .Case("r1", Hexagon::R1)
274 .Case("r2", Hexagon::R2)
275 .Case("r3", Hexagon::R3)
276 .Case("r4", Hexagon::R4)
277 .Case("r5", Hexagon::R5)
278 .Case("r6", Hexagon::R6)
279 .Case("r7", Hexagon::R7)
280 .Case("r8", Hexagon::R8)
281 .Case("r9", Hexagon::R9)
282 .Case("r10", Hexagon::R10)
283 .Case("r11", Hexagon::R11)
284 .Case("r12", Hexagon::R12)
285 .Case("r13", Hexagon::R13)
286 .Case("r14", Hexagon::R14)
287 .Case("r15", Hexagon::R15)
288 .Case("r16", Hexagon::R16)
289 .Case("r17", Hexagon::R17)
290 .Case("r18", Hexagon::R18)
291 .Case("r19", Hexagon::R19)
292 .Case("r20", Hexagon::R20)
293 .Case("r21", Hexagon::R21)
294 .Case("r22", Hexagon::R22)
295 .Case("r23", Hexagon::R23)
296 .Case("r24", Hexagon::R24)
297 .Case("r25", Hexagon::R25)
298 .Case("r26", Hexagon::R26)
299 .Case("r27", Hexagon::R27)
300 .Case("r28", Hexagon::R28)
301 .Case("r29", Hexagon::R29)
302 .Case("r30", Hexagon::R30)
303 .Case("r31", Hexagon::R31)
304 .Case("r1:0", Hexagon::D0)
305 .Case("r3:2", Hexagon::D1)
306 .Case("r5:4", Hexagon::D2)
307 .Case("r7:6", Hexagon::D3)
308 .Case("r9:8", Hexagon::D4)
309 .Case("r11:10", Hexagon::D5)
310 .Case("r13:12", Hexagon::D6)
311 .Case("r15:14", Hexagon::D7)
312 .Case("r17:16", Hexagon::D8)
313 .Case("r19:18", Hexagon::D9)
314 .Case("r21:20", Hexagon::D10)
315 .Case("r23:22", Hexagon::D11)
316 .Case("r25:24", Hexagon::D12)
317 .Case("r27:26", Hexagon::D13)
318 .Case("r29:28", Hexagon::D14)
319 .Case("r31:30", Hexagon::D15)
320 .Case("sp", Hexagon::R29)
321 .Case("fp", Hexagon::R30)
322 .Case("lr", Hexagon::R31)
323 .Case("p0", Hexagon::P0)
324 .Case("p1", Hexagon::P1)
325 .Case("p2", Hexagon::P2)
326 .Case("p3", Hexagon::P3)
327 .Case("sa0", Hexagon::SA0)
328 .Case("lc0", Hexagon::LC0)
329 .Case("sa1", Hexagon::SA1)
330 .Case("lc1", Hexagon::LC1)
331 .Case("m0", Hexagon::M0)
332 .Case("m1", Hexagon::M1)
333 .Case("usr", Hexagon::USR)
334 .Case("ugp", Hexagon::UGP)
335 .Case("cs0", Hexagon::CS0)
336 .Case("cs1", Hexagon::CS1)
337 .Default(Register());
338 if (Reg)
339 return Reg;
340
341 report_fatal_error("Invalid register name global variable");
342}
343
344/// LowerCallResult - Lower the result values of an ISD::CALL into the
345/// appropriate copies out of appropriate physical registers. This assumes that
346/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
347/// being lowered. Returns a SDNode with the same number of values as the
348/// ISD::CALL.
349SDValue HexagonTargetLowering::LowerCallResult(
350 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
351 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
352 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
353 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
354 // Assign locations to each value returned by this call.
355 SmallVector<CCValAssign, 16> RVLocs;
356
357 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
358 *DAG.getContext());
359
360 if (Subtarget.useHVXOps())
361 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX);
362 else
363 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon);
364
365 // Copy all of the result registers out of their specified physreg.
366 for (unsigned i = 0; i != RVLocs.size(); ++i) {
367 SDValue RetVal;
368 if (RVLocs[i].getValVT() == MVT::i1) {
369 // Return values of type MVT::i1 require special handling. The reason
370 // is that MVT::i1 is associated with the PredRegs register class, but
371 // values of that type are still returned in R0. Generate an explicit
372 // copy into a predicate register from R0, and treat the value of the
373 // predicate register as the call result.
374 auto &MRI = DAG.getMachineFunction().getRegInfo();
375 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
376 MVT::i32, Glue);
377 // FR0 = (Value, Chain, Glue)
378 Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass);
379 SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR,
380 FR0.getValue(0), FR0.getValue(2));
381 // TPR = (Chain, Glue)
382 // Don't glue this CopyFromReg, because it copies from a virtual
383 // register. If it is glued to the call, InstrEmitter will add it
384 // as an implicit def to the call (EmitMachineNode).
385 RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1);
386 Glue = TPR.getValue(1);
387 Chain = TPR.getValue(0);
388 } else {
389 RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
390 RVLocs[i].getValVT(), Glue);
391 Glue = RetVal.getValue(2);
392 Chain = RetVal.getValue(1);
393 }
394 InVals.push_back(RetVal.getValue(0));
395 }
396
397 return Chain;
398}
399
400/// LowerCall - Functions arguments are copied from virtual regs to
401/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
402SDValue
403HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
404 SmallVectorImpl<SDValue> &InVals) const {
405 SelectionDAG &DAG = CLI.DAG;
406 SDLoc &dl = CLI.DL;
407 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
408 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
409 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
410 SDValue Chain = CLI.Chain;
411 SDValue Callee = CLI.Callee;
412 CallingConv::ID CallConv = CLI.CallConv;
413 bool IsVarArg = CLI.IsVarArg;
414 bool DoesNotReturn = CLI.DoesNotReturn;
415
416 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
417 MachineFunction &MF = DAG.getMachineFunction();
418 MachineFrameInfo &MFI = MF.getFrameInfo();
419 auto PtrVT = getPointerTy(MF.getDataLayout());
420
421 unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0;
422 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee))
423 Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32);
424
425 // Linux ABI treats var-arg calls the same way as regular ones.
426 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
427
428 // Analyze operands of the call, assigning locations to each operand.
429 SmallVector<CCValAssign, 16> ArgLocs;
430 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(),
431 NumParams);
432
433 if (Subtarget.useHVXOps())
434 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
435 else if (DisableArgsMinAlignment)
436 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
437 else
438 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
439
440 if (CLI.IsTailCall) {
441 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
442 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
443 IsVarArg, IsStructRet, StructAttrFlag, Outs,
444 OutVals, Ins, DAG);
445 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
446 CCValAssign &VA = ArgLocs[i];
447 if (VA.isMemLoc()) {
448 CLI.IsTailCall = false;
449 break;
450 }
451 }
452 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
: "Argument must be passed on stack. " "Not eligible for Tail Call\n"
); } } while (false)
453 : "Argument must be passed on stack. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
: "Argument must be passed on stack. " "Not eligible for Tail Call\n"
); } } while (false)
454 "Not eligible for Tail Call\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
: "Argument must be passed on stack. " "Not eligible for Tail Call\n"
); } } while (false)
;
455 }
456 // Get a count of how many bytes are to be pushed on the stack.
457 unsigned NumBytes = CCInfo.getNextStackOffset();
458 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
459 SmallVector<SDValue, 8> MemOpChains;
460
461 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
462 SDValue StackPtr =
463 DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT);
464
465 bool NeedsArgAlign = false;
466 Align LargestAlignSeen;
467 // Walk the register/memloc assignments, inserting copies/loads.
468 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
469 CCValAssign &VA = ArgLocs[i];
470 SDValue Arg = OutVals[i];
471 ISD::ArgFlagsTy Flags = Outs[i].Flags;
472 // Record if we need > 8 byte alignment on an argument.
473 bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT());
474 NeedsArgAlign |= ArgAlign;
475
476 // Promote the value if needed.
477 switch (VA.getLocInfo()) {
478 default:
479 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
480 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 480)
;
481 case CCValAssign::Full:
482 break;
483 case CCValAssign::BCvt:
484 Arg = DAG.getBitcast(VA.getLocVT(), Arg);
485 break;
486 case CCValAssign::SExt:
487 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
488 break;
489 case CCValAssign::ZExt:
490 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
491 break;
492 case CCValAssign::AExt:
493 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
494 break;
495 }
496
497 if (VA.isMemLoc()) {
498 unsigned LocMemOffset = VA.getLocMemOffset();
499 SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,
500 StackPtr.getValueType());
501 MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr);
502 if (ArgAlign)
503 LargestAlignSeen = std::max(
504 LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8));
505 if (Flags.isByVal()) {
506 // The argument is a struct passed by value. According to LLVM, "Arg"
507 // is a pointer.
508 MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain,
509 Flags, DAG, dl));
510 } else {
511 MachinePointerInfo LocPI = MachinePointerInfo::getStack(
512 DAG.getMachineFunction(), LocMemOffset);
513 SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI);
514 MemOpChains.push_back(S);
515 }
516 continue;
517 }
518
519 // Arguments that can be passed on register must be kept at RegsToPass
520 // vector.
521 if (VA.isRegLoc())
522 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
523 }
524
525 if (NeedsArgAlign && Subtarget.hasV60Ops()) {
526 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { dbgs() << "Function needs byte stack align due to call args\n"
; } } while (false)
;
527 Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
528 LargestAlignSeen = std::max(LargestAlignSeen, VecAlign);
529 MFI.ensureMaxAlignment(LargestAlignSeen);
530 }
531 // Transform all store nodes into one single node because all store
532 // nodes are independent of each other.
533 if (!MemOpChains.empty())
534 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
535
536 SDValue Glue;
537 if (!CLI.IsTailCall) {
538 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
539 Glue = Chain.getValue(1);
540 }
541
542 // Build a sequence of copy-to-reg nodes chained together with token
543 // chain and flag operands which copy the outgoing args into registers.
544 // The Glue is necessary since all emitted instructions must be
545 // stuck together.
546 if (!CLI.IsTailCall) {
547 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
548 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
549 RegsToPass[i].second, Glue);
550 Glue = Chain.getValue(1);
551 }
552 } else {
553 // For tail calls lower the arguments to the 'real' stack slot.
554 //
555 // Force all the incoming stack arguments to be loaded from the stack
556 // before any new outgoing arguments are stored to the stack, because the
557 // outgoing stack slots may alias the incoming argument stack slots, and
558 // the alias isn't otherwise explicit. This is slightly more conservative
559 // than necessary, because it means that each store effectively depends
560 // on every argument instead of just those arguments it would clobber.
561 //
562 // Do not flag preceding copytoreg stuff together with the following stuff.
563 Glue = SDValue();
564 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
565 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
566 RegsToPass[i].second, Glue);
567 Glue = Chain.getValue(1);
568 }
569 Glue = SDValue();
570 }
571
572 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
573 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
574
575 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
576 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
577 // node so that legalize doesn't hack it.
578 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
579 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags);
580 } else if (ExternalSymbolSDNode *S =
581 dyn_cast<ExternalSymbolSDNode>(Callee)) {
582 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags);
583 }
584
585 // Returns a chain & a flag for retval copy to use.
586 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
587 SmallVector<SDValue, 8> Ops;
588 Ops.push_back(Chain);
589 Ops.push_back(Callee);
590
591 // Add argument registers to the end of the list so that they are
592 // known live into the call.
593 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
594 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
595 RegsToPass[i].second.getValueType()));
596 }
597
598 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
599 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 599, __extension__ __PRETTY_FUNCTION__))
;
600 Ops.push_back(DAG.getRegisterMask(Mask));
601
602 if (Glue.getNode())
603 Ops.push_back(Glue);
604
605 if (CLI.IsTailCall) {
606 MFI.setHasTailCall();
607 return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops);
608 }
609
610 // Set this here because we need to know this for "hasFP" in frame lowering.
611 // The target-independent code calls getFrameRegister before setting it, and
612 // getFrameRegister uses hasFP to determine whether the function has FP.
613 MFI.setHasCalls(true);
614
615 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
616 Chain = DAG.getNode(OpCode, dl, NodeTys, Ops);
617 Glue = Chain.getValue(1);
618
619 // Create the CALLSEQ_END node.
620 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
621 DAG.getIntPtrConstant(0, dl, true), Glue, dl);
622 Glue = Chain.getValue(1);
623
624 // Handle result values, copying them out of physregs into vregs that we
625 // return.
626 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
627 InVals, OutVals, Callee);
628}
629
630/// Returns true by value, base pointer and offset pointer and addressing
631/// mode by reference if this node can be combined with a load / store to
632/// form a post-indexed load / store.
633bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
634 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
635 SelectionDAG &DAG) const {
636 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N);
637 if (!LSN)
638 return false;
639 EVT VT = LSN->getMemoryVT();
640 if (!VT.isSimple())
641 return false;
642 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
643 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
644 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
645 VT == MVT::v4i16 || VT == MVT::v8i8 ||
646 Subtarget.isHVXVectorType(VT.getSimpleVT());
647 if (!IsLegalType)
648 return false;
649
650 if (Op->getOpcode() != ISD::ADD)
651 return false;
652 Base = Op->getOperand(0);
653 Offset = Op->getOperand(1);
654 if (!isa<ConstantSDNode>(Offset.getNode()))
655 return false;
656 AM = ISD::POST_INC;
657
658 int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
659 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V);
660}
661
662SDValue
663HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
664 MachineFunction &MF = DAG.getMachineFunction();
665 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
666 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
667 unsigned LR = HRI.getRARegister();
668
669 if ((Op.getOpcode() != ISD::INLINEASM &&
670 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
671 return Op;
672
673 unsigned NumOps = Op.getNumOperands();
674 if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue)
675 --NumOps; // Ignore the flag operand.
676
677 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
678 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue();
679 unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
680 ++i; // Skip the ID value.
681
682 switch (InlineAsm::getKind(Flags)) {
683 default:
684 llvm_unreachable("Bad flags!")::llvm::llvm_unreachable_internal("Bad flags!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 684)
;
685 case InlineAsm::Kind_RegUse:
686 case InlineAsm::Kind_Imm:
687 case InlineAsm::Kind_Mem:
688 i += NumVals;
689 break;
690 case InlineAsm::Kind_Clobber:
691 case InlineAsm::Kind_RegDef:
692 case InlineAsm::Kind_RegDefEarlyClobber: {
693 for (; NumVals; --NumVals, ++i) {
694 unsigned Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
695 if (Reg != LR)
696 continue;
697 HMFI.setHasClobberLR(true);
698 return Op;
699 }
700 break;
701 }
702 }
703 }
704
705 return Op;
706}
707
708// Need to transform ISD::PREFETCH into something that doesn't inherit
709// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
710// SDNPMayStore.
711SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
712 SelectionDAG &DAG) const {
713 SDValue Chain = Op.getOperand(0);
714 SDValue Addr = Op.getOperand(1);
715 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,
716 // if the "reg" is fed by an "add".
717 SDLoc DL(Op);
718 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
719 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
720}
721
722// Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode
723// is marked as having side-effects, while the register read on Hexagon does
724// not have any. TableGen refuses to accept the direct pattern from that node
725// to the A4_tfrcpp.
726SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
727 SelectionDAG &DAG) const {
728 SDValue Chain = Op.getOperand(0);
729 SDLoc dl(Op);
730 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
731 return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain);
732}
733
734SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
735 SelectionDAG &DAG) const {
736 SDValue Chain = Op.getOperand(0);
737 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
738 // Lower the hexagon_prefetch builtin to DCFETCH, as above.
739 if (IntNo == Intrinsic::hexagon_prefetch) {
740 SDValue Addr = Op.getOperand(2);
741 SDLoc DL(Op);
742 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
743 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
744 }
745 return SDValue();
746}
747
748SDValue
749HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
750 SelectionDAG &DAG) const {
751 SDValue Chain = Op.getOperand(0);
752 SDValue Size = Op.getOperand(1);
753 SDValue Align = Op.getOperand(2);
754 SDLoc dl(Op);
755
756 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align);
757 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC")(static_cast <bool> (AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC"
) ? void (0) : __assert_fail ("AlignConst && \"Non-constant Align in LowerDYNAMIC_STACKALLOC\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 757, __extension__ __PRETTY_FUNCTION__))
;
758
759 unsigned A = AlignConst->getSExtValue();
760 auto &HFI = *Subtarget.getFrameLowering();
761 // "Zero" means natural stack alignment.
762 if (A == 0)
763 A = HFI.getStackAlign().value();
764
765 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
766 dbgs () << __func__ << " Align: " << A << " Size: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
767 Size.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
768 dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
769 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("hexagon-lowering")) { { dbgs () << __func__ << " Align: "
<< A << " Size: "; Size.getNode()->dump(&
DAG); dbgs() << "\n"; }; } } while (false)
;
770
771 SDValue AC = DAG.getConstant(A, dl, MVT::i32);
772 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
773 SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);
774
775 DAG.ReplaceAllUsesOfValueWith(Op, AA);
776 return AA;
777}
778
779SDValue HexagonTargetLowering::LowerFormalArguments(
780 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
781 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
782 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
783 MachineFunction &MF = DAG.getMachineFunction();
784 MachineFrameInfo &MFI = MF.getFrameInfo();
785 MachineRegisterInfo &MRI = MF.getRegInfo();
786
787 // Linux ABI treats var-arg calls the same way as regular ones.
788 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
789
790 // Assign locations to all of the incoming arguments.
791 SmallVector<CCValAssign, 16> ArgLocs;
792 HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
793 *DAG.getContext(),
794 MF.getFunction().getFunctionType()->getNumParams());
795
796 if (Subtarget.useHVXOps())
797 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
798 else if (DisableArgsMinAlignment)
799 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);
800 else
801 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);
802
803 // For LLVM, in the case when returning a struct by value (>8byte),
804 // the first argument is a pointer that points to the location on caller's
805 // stack where the return value will be stored. For Hexagon, the location on
806 // caller's stack is passed only when the struct size is smaller than (and
807 // equal to) 8 bytes. If not, no address will be passed into callee and
808 // callee return the result direclty through R0/R1.
809 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
810 switch (RC.getID()) {
811 case Hexagon::IntRegsRegClassID:
812 return Reg - Hexagon::R0 + 1;
813 case Hexagon::DoubleRegsRegClassID:
814 return (Reg - Hexagon::D0 + 1) * 2;
815 case Hexagon::HvxVRRegClassID:
816 return Reg - Hexagon::V0 + 1;
817 case Hexagon::HvxWRRegClassID:
818 return (Reg - Hexagon::W0 + 1) * 2;
819 }
820 llvm_unreachable("Unexpected register class")::llvm::llvm_unreachable_internal("Unexpected register class"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 820)
;
821 };
822
823 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
824 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
825 HFL.FirstVarArgSavedReg = 0;
826 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
827
828 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
829 CCValAssign &VA = ArgLocs[i];
830 ISD::ArgFlagsTy Flags = Ins[i].Flags;
831 bool ByVal = Flags.isByVal();
832
833 // Arguments passed in registers:
834 // 1. 32- and 64-bit values and HVX vectors are passed directly,
835 // 2. Large structs are passed via an address, and the address is
836 // passed in a register.
837 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
838 llvm_unreachable("ByValSize must be bigger than 8 bytes")::llvm::llvm_unreachable_internal("ByValSize must be bigger than 8 bytes"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 838)
;
839
840 bool InReg = VA.isRegLoc() &&
841 (!ByVal || (ByVal && Flags.getByValSize() > 8));
842
843 if (InReg) {
844 MVT RegVT = VA.getLocVT();
845 if (VA.getLocInfo() == CCValAssign::BCvt)
846 RegVT = VA.getValVT();
847
848 const TargetRegisterClass *RC = getRegClassFor(RegVT);
849 Register VReg = MRI.createVirtualRegister(RC);
850 SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);
851
852 // Treat values of type MVT::i1 specially: they are passed in
853 // registers of type i32, but they need to remain as values of
854 // type i1 for consistency of the argument lowering.
855 if (VA.getValVT() == MVT::i1) {
856 assert(RegVT.getSizeInBits() <= 32)(static_cast <bool> (RegVT.getSizeInBits() <= 32) ? void
(0) : __assert_fail ("RegVT.getSizeInBits() <= 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 856, __extension__ __PRETTY_FUNCTION__))
;
857 SDValue T = DAG.getNode(ISD::AND, dl, RegVT,
858 Copy, DAG.getConstant(1, dl, RegVT));
859 Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT),
860 ISD::SETNE);
861 } else {
862#ifndef NDEBUG
863 unsigned RegSize = RegVT.getSizeInBits();
864 assert(RegSize == 32 || RegSize == 64 ||(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget
.isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 865, __extension__ __PRETTY_FUNCTION__))
865 Subtarget.isHVXVectorType(RegVT))(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget
.isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 865, __extension__ __PRETTY_FUNCTION__))
;
866#endif
867 }
868 InVals.push_back(Copy);
869 MRI.addLiveIn(VA.getLocReg(), VReg);
870 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
871 } else {
872 assert(VA.isMemLoc() && "Argument should be passed in memory")(static_cast <bool> (VA.isMemLoc() && "Argument should be passed in memory"
) ? void (0) : __assert_fail ("VA.isMemLoc() && \"Argument should be passed in memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 872, __extension__ __PRETTY_FUNCTION__))
;
873
874 // If it's a byval parameter, then we need to compute the
875 // "real" size, not the size of the pointer.
876 unsigned ObjSize = Flags.isByVal()
877 ? Flags.getByValSize()
878 : VA.getLocVT().getStoreSizeInBits() / 8;
879
880 // Create the frame index object for this incoming parameter.
881 int Offset = HEXAGON_LRFP_SIZE8 + VA.getLocMemOffset();
882 int FI = MFI.CreateFixedObject(ObjSize, Offset, true);
883 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
884
885 if (Flags.isByVal()) {
886 // If it's a pass-by-value aggregate, then do not dereference the stack
887 // location. Instead, we should generate a reference to the stack
888 // location.
889 InVals.push_back(FIN);
890 } else {
891 SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
892 MachinePointerInfo::getFixedStack(MF, FI, 0));
893 InVals.push_back(L);
894 }
895 }
896 }
897
898 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
899 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
900 MRI.addLiveIn(Hexagon::R0+i);
901 }
902
903 if (IsVarArg && Subtarget.isEnvironmentMusl()) {
904 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
905 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
906
907 // Create Frame index for the start of register saved area.
908 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
909 bool RequiresPadding = (NumVarArgRegs & 1);
910 int RegSaveAreaSizePlusPadding = RequiresPadding
911 ? (NumVarArgRegs + 1) * 4
912 : NumVarArgRegs * 4;
913
914 if (RegSaveAreaSizePlusPadding > 0) {
915 // The offset to saved register area should be 8 byte aligned.
916 int RegAreaStart = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset();
917 if (!(RegAreaStart % 8))
918 RegAreaStart = (RegAreaStart + 7) & -8;
919
920 int RegSaveAreaFrameIndex =
921 MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true);
922 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
923
924 // This will point to the next argument passed via stack.
925 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
926 int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true);
927 HMFI.setVarArgsFrameIndex(FI);
928 } else {
929 // This will point to the next argument passed via stack, when
930 // there is no saved register area.
931 int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset();
932 int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true);
933 HMFI.setRegSavedAreaStartFrameIndex(FI);
934 HMFI.setVarArgsFrameIndex(FI);
935 }
936 }
937
938
939 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
940 // This will point to the next argument passed via stack.
941 int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset();
942 int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true);
943 HMFI.setVarArgsFrameIndex(FI);
944 }
945
946 return Chain;
947}
948
949SDValue
950HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
951 // VASTART stores the address of the VarArgsFrameIndex slot into the
952 // memory location argument.
953 MachineFunction &MF = DAG.getMachineFunction();
954 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
955 SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32);
956 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
957
958 if (!Subtarget.isEnvironmentMusl()) {
959 return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1),
960 MachinePointerInfo(SV));
961 }
962 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
963 auto &HFL = *Subtarget.getFrameLowering();
964 SDLoc DL(Op);
965 SmallVector<SDValue, 8> MemOps;
966
967 // Get frame index of va_list.
968 SDValue FIN = Op.getOperand(1);
969
970 // If first Vararg register is odd, add 4 bytes to start of
971 // saved register area to point to the first register location.
972 // This is because the saved register area has to be 8 byte aligned.
973 // Incase of an odd start register, there will be 4 bytes of padding in
974 // the beginning of saved register area. If all registers area used up,
975 // the following condition will handle it correctly.
976 SDValue SavedRegAreaStartFrameIndex =
977 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32);
978
979 auto PtrVT = getPointerTy(DAG.getDataLayout());
980
981 if (HFL.FirstVarArgSavedReg & 1)
982 SavedRegAreaStartFrameIndex =
983 DAG.getNode(ISD::ADD, DL, PtrVT,
984 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(),
985 MVT::i32),
986 DAG.getIntPtrConstant(4, DL));
987
988 // Store the saved register area start pointer.
989 SDValue Store =
990 DAG.getStore(Op.getOperand(0), DL,
991 SavedRegAreaStartFrameIndex,
992 FIN, MachinePointerInfo(SV));
993 MemOps.push_back(Store);
994
995 // Store saved register area end pointer.
996 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
997 FIN, DAG.getIntPtrConstant(4, DL));
998 Store = DAG.getStore(Op.getOperand(0), DL,
999 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
1000 PtrVT),
1001 FIN, MachinePointerInfo(SV, 4));
1002 MemOps.push_back(Store);
1003
1004 // Store overflow area pointer.
1005 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
1006 FIN, DAG.getIntPtrConstant(4, DL));
1007 Store = DAG.getStore(Op.getOperand(0), DL,
1008 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
1009 PtrVT),
1010 FIN, MachinePointerInfo(SV, 8));
1011 MemOps.push_back(Store);
1012
1013 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
1014}
1015
1016SDValue
1017HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1018 // Assert that the linux ABI is enabled for the current compilation.
1019 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled")(static_cast <bool> (Subtarget.isEnvironmentMusl() &&
"Linux ABI should be enabled") ? void (0) : __assert_fail ("Subtarget.isEnvironmentMusl() && \"Linux ABI should be enabled\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1019, __extension__ __PRETTY_FUNCTION__))
;
1020 SDValue Chain = Op.getOperand(0);
1021 SDValue DestPtr = Op.getOperand(1);
1022 SDValue SrcPtr = Op.getOperand(2);
1023 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
1024 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
1025 SDLoc DL(Op);
1026 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1027 // we need to memcopy 12 bytes from va_list to another similar list.
1028 return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
1029 DAG.getIntPtrConstant(12, DL), Align(4),
1030 /*isVolatile*/ false, false, false,
1031 MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
1032}
1033
1034SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1035 const SDLoc &dl(Op);
1036 SDValue LHS = Op.getOperand(0);
1037 SDValue RHS = Op.getOperand(1);
1038 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1039 MVT ResTy = ty(Op);
1040 MVT OpTy = ty(LHS);
1041
1042 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
8
Taking false branch
1043 MVT ElemTy = OpTy.getVectorElementType();
1044 assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void (
0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1044, __extension__ __PRETTY_FUNCTION__))
;
1045 MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1046 OpTy.getVectorNumElements());
1047 return DAG.getSetCC(dl, ResTy,
1048 DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy),
1049 DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC);
1050 }
1051
1052 // Treat all other vector types as legal.
1053 if (ResTy.isVector())
9
Taking false branch
1054 return Op;
1055
1056 // Comparisons of short integers should use sign-extend, not zero-extend,
1057 // since we can represent small negative values in the compare instructions.
1058 // The LLVM default is to use zero-extend arbitrarily in these cases.
1059 auto isSExtFree = [this](SDValue N) {
1060 switch (N.getOpcode()) {
14
Calling 'SDValue::getOpcode'
1061 case ISD::TRUNCATE: {
1062 // A sign-extend of a truncate of a sign-extend is free.
1063 SDValue Op = N.getOperand(0);
1064 if (Op.getOpcode() != ISD::AssertSext)
1065 return false;
1066 EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT();
1067 unsigned ThisBW = ty(N).getSizeInBits();
1068 unsigned OrigBW = OrigTy.getSizeInBits();
1069 // The type that was sign-extended to get the AssertSext must be
1070 // narrower than the type of N (so that N has still the same value
1071 // as the original).
1072 return ThisBW >= OrigBW;
1073 }
1074 case ISD::LOAD:
1075 // We have sign-extended loads.
1076 return true;
1077 }
1078 return false;
1079 };
1080
1081 if (OpTy == MVT::i8 || OpTy == MVT::i16) {
10
Taking true branch
1082 ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS);
1083 bool IsNegative = C && C->getAPIntValue().isNegative();
11
Assuming 'C' is null
1084 if (IsNegative
11.1
'IsNegative' is false
11.1
'IsNegative' is false
|| isSExtFree(LHS) || isSExtFree(RHS))
12
The value of 'RHS' is assigned to 'N.Node'
13
Calling 'operator()'
1085 return DAG.getSetCC(dl, ResTy,
1086 DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32),
1087 DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC);
1088 }
1089
1090 return SDValue();
1091}
1092
1093SDValue
1094HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1095 SDValue PredOp = Op.getOperand(0);
1096 SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);
1097 MVT OpTy = ty(Op1);
1098 const SDLoc &dl(Op);
1099
1100 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1101 MVT ElemTy = OpTy.getVectorElementType();
1102 assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void (
0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1102, __extension__ __PRETTY_FUNCTION__))
;
1103 MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1104 OpTy.getVectorNumElements());
1105 // Generate (trunc (select (_, sext, sext))).
1106 return DAG.getSExtOrTrunc(
1107 DAG.getSelect(dl, WideTy, PredOp,
1108 DAG.getSExtOrTrunc(Op1, dl, WideTy),
1109 DAG.getSExtOrTrunc(Op2, dl, WideTy)),
1110 dl, OpTy);
1111 }
1112
1113 return SDValue();
1114}
1115
1116SDValue
1117HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1118 EVT ValTy = Op.getValueType();
1119 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op);
1120 Constant *CVal = nullptr;
1121 bool isVTi1Type = false;
1122 if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {
1123 if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1124 IRBuilder<> IRB(CV->getContext());
1125 SmallVector<Constant*, 128> NewConst;
1126 unsigned VecLen = CV->getNumOperands();
1127 assert(isPowerOf2_32(VecLen) &&(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize"
) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1128, __extension__ __PRETTY_FUNCTION__))
1128 "conversion only supported for pow2 VectorSize")(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize"
) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1128, __extension__ __PRETTY_FUNCTION__))
;
1129 for (unsigned i = 0; i < VecLen; ++i)
1130 NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue()));
1131
1132 CVal = ConstantVector::get(NewConst);
1133 isVTi1Type = true;
1134 }
1135 }
1136 Align Alignment = CPN->getAlign();
1137 bool IsPositionIndependent = isPositionIndependent();
1138 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1139
1140 unsigned Offset = 0;
1141 SDValue T;
1142 if (CPN->isMachineConstantPoolEntry())
1143 T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment,
1144 Offset, TF);
1145 else if (isVTi1Type)
1146 T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF);
1147 else
1148 T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset,
1149 TF);
1150
1151 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&(static_cast <bool> (cast<ConstantPoolSDNode>(T)->
getTargetFlags() == TF && "Inconsistent target flag encountered"
) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1152, __extension__ __PRETTY_FUNCTION__))
1152 "Inconsistent target flag encountered")(static_cast <bool> (cast<ConstantPoolSDNode>(T)->
getTargetFlags() == TF && "Inconsistent target flag encountered"
) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1152, __extension__ __PRETTY_FUNCTION__))
;
1153
1154 if (IsPositionIndependent)
1155 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T);
1156 return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T);
1157}
1158
1159SDValue
1160HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1161 EVT VT = Op.getValueType();
1162 int Idx = cast<JumpTableSDNode>(Op)->getIndex();
1163 if (isPositionIndependent()) {
1164 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
1165 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T);
1166 }
1167
1168 SDValue T = DAG.getTargetJumpTable(Idx, VT);
1169 return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T);
1170}
1171
1172SDValue
1173HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1174 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1175 MachineFunction &MF = DAG.getMachineFunction();
1176 MachineFrameInfo &MFI = MF.getFrameInfo();
1177 MFI.setReturnAddressIsTaken(true);
1178
1179 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1180 return SDValue();
1181
1182 EVT VT = Op.getValueType();
1183 SDLoc dl(Op);
1184 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1185 if (Depth) {
1186 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1187 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
1188 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
1189 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
1190 MachinePointerInfo());
1191 }
1192
1193 // Return LR, which contains the return address. Mark it an implicit live-in.
1194 unsigned Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32));
1195 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
1196}
1197
1198SDValue
1199HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1200 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1201 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1202 MFI.setFrameAddressIsTaken(true);
1203
1204 EVT VT = Op.getValueType();
1205 SDLoc dl(Op);
1206 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1207 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
1208 HRI.getFrameRegister(), VT);
1209 while (Depth--)
1210 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
1211 MachinePointerInfo());
1212 return FrameAddr;
1213}
1214
1215SDValue
1216HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1217 SDLoc dl(Op);
1218 return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0));
1219}
1220
1221SDValue
1222HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1223 SDLoc dl(Op);
1224 auto *GAN = cast<GlobalAddressSDNode>(Op);
1225 auto PtrVT = getPointerTy(DAG.getDataLayout());
1226 auto *GV = GAN->getGlobal();
1227 int64_t Offset = GAN->getOffset();
1228
1229 auto &HLOF = *HTM.getObjFileLowering();
1230 Reloc::Model RM = HTM.getRelocationModel();
1231
1232 if (RM == Reloc::Static) {
1233 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
1234 const GlobalObject *GO = GV->getBaseObject();
1235 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1236 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA);
1237 return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA);
1238 }
1239
1240 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
1241 if (UsePCRel) {
1242 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset,
1243 HexagonII::MO_PCREL);
1244 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA);
1245 }
1246
1247 // Use GOT index.
1248 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
1249 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT);
1250 SDValue Off = DAG.getConstant(Offset, dl, MVT::i32);
1251 return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off);
1252}
1253
1254// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1255SDValue
1256HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1257 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
1258 SDLoc dl(Op);
1259 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1260
1261 Reloc::Model RM = HTM.getRelocationModel();
1262 if (RM == Reloc::Static) {
1263 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT);
1264 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A);
1265 }
1266
1267 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL);
1268 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A);
1269}
1270
1271SDValue
1272HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1273 const {
1274 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1275 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME"_GLOBAL_OFFSET_TABLE_", PtrVT,
1276 HexagonII::MO_PCREL);
1277 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym);
1278}
1279
1280SDValue
1281HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1282 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1283 unsigned char OperandFlags) const {
1284 MachineFunction &MF = DAG.getMachineFunction();
1285 MachineFrameInfo &MFI = MF.getFrameInfo();
1286 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1287 SDLoc dl(GA);
1288 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
1289 GA->getValueType(0),
1290 GA->getOffset(),
1291 OperandFlags);
1292 // Create Operands for the call.The Operands should have the following:
1293 // 1. Chain SDValue
1294 // 2. Callee which in this case is the Global address value.
1295 // 3. Registers live into the call.In this case its R0, as we
1296 // have just one argument to be passed.
1297 // 4. Glue.
1298 // Note: The order is important.
1299
1300 const auto &HRI = *Subtarget.getRegisterInfo();
1301 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1302 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1302, __extension__ __PRETTY_FUNCTION__))
;
1303 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT),
1304 DAG.getRegisterMask(Mask), Glue };
1305 Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops);
1306
1307 // Inform MFI that function has calls.
1308 MFI.setAdjustsStack(true);
1309
1310 Glue = Chain.getValue(1);
1311 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
1312}
1313
1314//
1315// Lower using the intial executable model for TLS addresses
1316//
1317SDValue
1318HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1319 SelectionDAG &DAG) const {
1320 SDLoc dl(GA);
1321 int64_t Offset = GA->getOffset();
1322 auto PtrVT = getPointerTy(DAG.getDataLayout());
1323
1324 // Get the thread pointer.
1325 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1326
1327 bool IsPositionIndependent = isPositionIndependent();
1328 unsigned char TF =
1329 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1330
1331 // First generate the TLS symbol address
1332 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT,
1333 Offset, TF);
1334
1335 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1336
1337 if (IsPositionIndependent) {
1338 // Generate the GOT pointer in case of position independent code
1339 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG);
1340
1341 // Add the TLS Symbol address to GOT pointer.This gives
1342 // GOT relative relocation for the symbol.
1343 Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1344 }
1345
1346 // Load the offset value for TLS symbol.This offset is relative to
1347 // thread pointer.
1348 SDValue LoadOffset =
1349 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo());
1350
1351 // Address of the thread local variable is the add of thread
1352 // pointer and the offset of the variable.
1353 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset);
1354}
1355
1356//
1357// Lower using the local executable model for TLS addresses
1358//
1359SDValue
1360HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1361 SelectionDAG &DAG) const {
1362 SDLoc dl(GA);
1363 int64_t Offset = GA->getOffset();
1364 auto PtrVT = getPointerTy(DAG.getDataLayout());
1365
1366 // Get the thread pointer.
1367 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1368 // Generate the TLS symbol address
1369 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1370 HexagonII::MO_TPREL);
1371 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1372
1373 // Address of the thread local variable is the add of thread
1374 // pointer and the offset of the variable.
1375 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym);
1376}
1377
1378//
1379// Lower using the general dynamic model for TLS addresses
1380//
1381SDValue
1382HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1383 SelectionDAG &DAG) const {
1384 SDLoc dl(GA);
1385 int64_t Offset = GA->getOffset();
1386 auto PtrVT = getPointerTy(DAG.getDataLayout());
1387
1388 // First generate the TLS symbol address
1389 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1390 HexagonII::MO_GDGOT);
1391
1392 // Then, generate the GOT pointer
1393 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG);
1394
1395 // Add the TLS symbol and the GOT pointer
1396 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1397 SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1398
1399 // Copy over the argument to R0
1400 SDValue InFlag;
1401 Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag);
1402 InFlag = Chain.getValue(1);
1403
1404 unsigned Flags =
1405 static_cast<const HexagonSubtarget &>(DAG.getSubtarget()).useLongCalls()
1406 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1407 : HexagonII::MO_GDPLT;
1408
1409 return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT,
1410 Hexagon::R0, Flags);
1411}
1412
1413//
1414// Lower TLS addresses.
1415//
1416// For now for dynamic models, we only support the general dynamic model.
1417//
1418SDValue
1419HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1420 SelectionDAG &DAG) const {
1421 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
1422
1423 switch (HTM.getTLSModel(GA->getGlobal())) {
1424 case TLSModel::GeneralDynamic:
1425 case TLSModel::LocalDynamic:
1426 return LowerToTLSGeneralDynamicModel(GA, DAG);
1427 case TLSModel::InitialExec:
1428 return LowerToTLSInitialExecModel(GA, DAG);
1429 case TLSModel::LocalExec:
1430 return LowerToTLSLocalExecModel(GA, DAG);
1431 }
1432 llvm_unreachable("Bogus TLS model")::llvm::llvm_unreachable_internal("Bogus TLS model", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1432)
;
1433}
1434
1435//===----------------------------------------------------------------------===//
1436// TargetLowering Implementation
1437//===----------------------------------------------------------------------===//
1438
1439HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1440 const HexagonSubtarget &ST)
1441 : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)),
1442 Subtarget(ST) {
1443 auto &HRI = *Subtarget.getRegisterInfo();
1444
1445 setPrefLoopAlignment(Align(16));
1446 setMinFunctionAlignment(Align(4));
1447 setPrefFunctionAlignment(Align(16));
1448 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1449 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1450 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1451
1452 setMaxAtomicSizeInBitsSupported(64);
1453 setMinCmpXchgSizeInBits(32);
1454
1455 if (EnableHexSDNodeSched)
1456 setSchedulingPreference(Sched::VLIW);
1457 else
1458 setSchedulingPreference(Sched::Source);
1459
1460 // Limits for inline expansion of memcpy/memmove
1461 MaxStoresPerMemcpy = MaxStoresPerMemcpyCL;
1462 MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL;
1463 MaxStoresPerMemmove = MaxStoresPerMemmoveCL;
1464 MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL;
1465 MaxStoresPerMemset = MaxStoresPerMemsetCL;
1466 MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL;
1467
1468 //
1469 // Set up register classes.
1470 //
1471
1472 addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass);
1473 addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass); // bbbbaaaa
1474 addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass); // ddccbbaa
1475 addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass); // hgfedcba
1476 addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass);
1477 addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass);
1478 addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass);
1479 addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass);
1480 addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass);
1481 addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);
1482 addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);
1483
1484 addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass);
1485 addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass);
1486
1487 //
1488 // Handling of scalar operations.
1489 //
1490 // All operations default to "legal", except:
1491 // - indexed loads and stores (pre-/post-incremented),
1492 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1493 // ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1494 // FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP,
1495 // FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG,
1496 // which default to "expand" for at least one type.
1497
1498 // Misc operations.
1499 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
1500 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
1501 setOperationAction(ISD::TRAP, MVT::Other, Legal);
1502 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
1503 setOperationAction(ISD::JumpTable, MVT::i32, Custom);
1504 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
1505 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
1506 setOperationAction(ISD::INLINEASM, MVT::Other, Custom);
1507 setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom);
1508 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
1509 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
1510 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1511 setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
1512 setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom);
1513 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
1514 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
1515
1516 // Custom legalize GlobalAddress nodes into CONST32.
1517 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
1518 setOperationAction(ISD::GlobalAddress, MVT::i8, Custom);
1519 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
1520
1521 // Hexagon needs to optimize cases with negative constants.
1522 setOperationAction(ISD::SETCC, MVT::i8, Custom);
1523 setOperationAction(ISD::SETCC, MVT::i16, Custom);
1524 setOperationAction(ISD::SETCC, MVT::v4i8, Custom);
1525 setOperationAction(ISD::SETCC, MVT::v2i16, Custom);
1526
1527 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1528 setOperationAction(ISD::VASTART, MVT::Other, Custom);
1529 setOperationAction(ISD::VAEND, MVT::Other, Expand);
1530 setOperationAction(ISD::VAARG, MVT::Other, Expand);
1531 if (Subtarget.isEnvironmentMusl())
1532 setOperationAction(ISD::VACOPY, MVT::Other, Custom);
1533 else
1534 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
1535
1536 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
1537 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
1538 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
1539
1540 if (EmitJumpTables)
1541 setMinimumJumpTableEntries(MinimumJumpTables);
1542 else
1543 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1544 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
1545
1546 for (unsigned LegalIntOp :
1547 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1548 setOperationAction(LegalIntOp, MVT::i32, Legal);
1549 setOperationAction(LegalIntOp, MVT::i64, Legal);
1550 }
1551
1552 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1553 // but they only operate on i64.
1554 for (MVT VT : MVT::integer_valuetypes()) {
1555 setOperationAction(ISD::UADDO, VT, Custom);
1556 setOperationAction(ISD::USUBO, VT, Custom);
1557 setOperationAction(ISD::SADDO, VT, Expand);
1558 setOperationAction(ISD::SSUBO, VT, Expand);
1559 setOperationAction(ISD::ADDCARRY, VT, Expand);
1560 setOperationAction(ISD::SUBCARRY, VT, Expand);
1561 }
1562 setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
1563 setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
1564
1565 setOperationAction(ISD::CTLZ, MVT::i8, Promote);
1566 setOperationAction(ISD::CTLZ, MVT::i16, Promote);
1567 setOperationAction(ISD::CTTZ, MVT::i8, Promote);
1568 setOperationAction(ISD::CTTZ, MVT::i16, Promote);
1569
1570 // Popcount can count # of 1s in i64 but returns i32.
1571 setOperationAction(ISD::CTPOP, MVT::i8, Promote);
1572 setOperationAction(ISD::CTPOP, MVT::i16, Promote);
1573 setOperationAction(ISD::CTPOP, MVT::i32, Promote);
1574 setOperationAction(ISD::CTPOP, MVT::i64, Legal);
1575
1576 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
1577 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
1578 setOperationAction(ISD::BSWAP, MVT::i32, Legal);
1579 setOperationAction(ISD::BSWAP, MVT::i64, Legal);
1580
1581 setOperationAction(ISD::FSHL, MVT::i32, Legal);
1582 setOperationAction(ISD::FSHL, MVT::i64, Legal);
1583 setOperationAction(ISD::FSHR, MVT::i32, Legal);
1584 setOperationAction(ISD::FSHR, MVT::i64, Legal);
1585
1586 for (unsigned IntExpOp :
1587 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1588 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,
1589 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1590 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1591 for (MVT VT : MVT::integer_valuetypes())
1592 setOperationAction(IntExpOp, VT, Expand);
1593 }
1594
1595 for (unsigned FPExpOp :
1596 {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
1597 ISD::FPOW, ISD::FCOPYSIGN}) {
1598 for (MVT VT : MVT::fp_valuetypes())
1599 setOperationAction(FPExpOp, VT, Expand);
1600 }
1601
1602 // No extending loads from i32.
1603 for (MVT VT : MVT::integer_valuetypes()) {
1604 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
1605 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
1606 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);
1607 }
1608 // Turn FP truncstore into trunc + store.
1609 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
1610 // Turn FP extload into load/fpextend.
1611 for (MVT VT : MVT::fp_valuetypes())
1612 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1613
1614 // Expand BR_CC and SELECT_CC for all integer and fp types.
1615 for (MVT VT : MVT::integer_valuetypes()) {
1616 setOperationAction(ISD::BR_CC, VT, Expand);
1617 setOperationAction(ISD::SELECT_CC, VT, Expand);
1618 }
1619 for (MVT VT : MVT::fp_valuetypes()) {
1620 setOperationAction(ISD::BR_CC, VT, Expand);
1621 setOperationAction(ISD::SELECT_CC, VT, Expand);
1622 }
1623 setOperationAction(ISD::BR_CC, MVT::Other, Expand);
1624
1625 //
1626 // Handling of vector operations.
1627 //
1628
1629 // Set the action for vector operations to "expand", then override it with
1630 // either "custom" or "legal" for specific cases.
1631 static const unsigned VectExpOps[] = {
1632 // Integer arithmetic:
1633 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,
1634 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,
1635 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1636 // Logical/bit:
1637 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,
1638 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ,
1639 // Floating point arithmetic/math functions:
1640 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,
1641 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,
1642 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1643 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
1644 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
1645 ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS,
1646 // Misc:
1647 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
1648 // Vector:
1649 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,
1650 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,
1651 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1652 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,
1653 ISD::SPLAT_VECTOR,
1654 };
1655
1656 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1657 for (unsigned VectExpOp : VectExpOps)
1658 setOperationAction(VectExpOp, VT, Expand);
1659
1660 // Expand all extending loads and truncating stores:
1661 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1662 if (TargetVT == VT)
1663 continue;
1664 setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);
1665 setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand);
1666 setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand);
1667 setTruncStoreAction(VT, TargetVT, Expand);
1668 }
1669
1670 // Normalize all inputs to SELECT to be vectors of i32.
1671 if (VT.getVectorElementType() != MVT::i32) {
1672 MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);
1673 setOperationAction(ISD::SELECT, VT, Promote);
1674 AddPromotedToType(ISD::SELECT, VT, VT32);
1675 }
1676 setOperationAction(ISD::SRA, VT, Custom);
1677 setOperationAction(ISD::SHL, VT, Custom);
1678 setOperationAction(ISD::SRL, VT, Custom);
1679 }
1680
1681 // Extending loads from (native) vectors of i8 into (native) vectors of i16
1682 // are legal.
1683 setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1684 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1685 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1686 setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1687 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1688 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1689
1690 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal);
1691 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal);
1692 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);
1693
1694 // Types natively supported:
1695 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1696 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1697 setOperationAction(ISD::BUILD_VECTOR, NativeVT, Custom);
1698 setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom);
1699 setOperationAction(ISD::INSERT_VECTOR_ELT, NativeVT, Custom);
1700 setOperationAction(ISD::EXTRACT_SUBVECTOR, NativeVT, Custom);
1701 setOperationAction(ISD::INSERT_SUBVECTOR, NativeVT, Custom);
1702 setOperationAction(ISD::CONCAT_VECTORS, NativeVT, Custom);
1703
1704 setOperationAction(ISD::ADD, NativeVT, Legal);
1705 setOperationAction(ISD::SUB, NativeVT, Legal);
1706 setOperationAction(ISD::MUL, NativeVT, Legal);
1707 setOperationAction(ISD::AND, NativeVT, Legal);
1708 setOperationAction(ISD::OR, NativeVT, Legal);
1709 setOperationAction(ISD::XOR, NativeVT, Legal);
1710
1711 if (NativeVT.getVectorElementType() != MVT::i1)
1712 setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal);
1713 }
1714
1715 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1716 setOperationAction(ISD::SMIN, VT, Legal);
1717 setOperationAction(ISD::SMAX, VT, Legal);
1718 setOperationAction(ISD::UMIN, VT, Legal);
1719 setOperationAction(ISD::UMAX, VT, Legal);
1720 }
1721
1722 // Custom lower unaligned loads.
1723 // Also, for both loads and stores, verify the alignment of the address
1724 // in case it is a compile-time constant. This is a usability feature to
1725 // provide a meaningful error message to users.
1726 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1727 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1728 setOperationAction(ISD::LOAD, VT, Custom);
1729 setOperationAction(ISD::STORE, VT, Custom);
1730 }
1731
1732 // Custom-lower load/stores of boolean vectors.
1733 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1734 setOperationAction(ISD::LOAD, VT, Custom);
1735 setOperationAction(ISD::STORE, VT, Custom);
1736 }
1737
1738 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1739 MVT::v2i32}) {
1740 setCondCodeAction(ISD::SETNE, VT, Expand);
1741 setCondCodeAction(ISD::SETLE, VT, Expand);
1742 setCondCodeAction(ISD::SETGE, VT, Expand);
1743 setCondCodeAction(ISD::SETLT, VT, Expand);
1744 setCondCodeAction(ISD::SETULE, VT, Expand);
1745 setCondCodeAction(ISD::SETUGE, VT, Expand);
1746 setCondCodeAction(ISD::SETULT, VT, Expand);
1747 }
1748
1749 // Custom-lower bitcasts from i8 to v8i1.
1750 setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1751 setOperationAction(ISD::SETCC, MVT::v2i16, Custom);
1752 setOperationAction(ISD::VSELECT, MVT::v4i8, Custom);
1753 setOperationAction(ISD::VSELECT, MVT::v2i16, Custom);
1754 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom);
1755 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
1756 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom);
1757
1758 // V5+.
1759 setOperationAction(ISD::FMA, MVT::f64, Expand);
1760 setOperationAction(ISD::FADD, MVT::f64, Expand);
1761 setOperationAction(ISD::FSUB, MVT::f64, Expand);
1762 setOperationAction(ISD::FMUL, MVT::f64, Expand);
1763
1764 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
1765 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
1766
1767 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);
1768 setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
1769 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
1770 setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote);
1771 setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
1772 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
1773 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);
1774 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
1775 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
1776 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote);
1777 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
1778 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
1779
1780 // Handling of indexed loads/stores: default is "expand".
1781 //
1782 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1783 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1784 setIndexedLoadAction(ISD::POST_INC, VT, Legal);
1785 setIndexedStoreAction(ISD::POST_INC, VT, Legal);
1786 }
1787
1788 // Subtarget-specific operation actions.
1789 //
1790 if (Subtarget.hasV60Ops()) {
1791 setOperationAction(ISD::ROTL, MVT::i32, Legal);
1792 setOperationAction(ISD::ROTL, MVT::i64, Legal);
1793 setOperationAction(ISD::ROTR, MVT::i32, Legal);
1794 setOperationAction(ISD::ROTR, MVT::i64, Legal);
1795 }
1796 if (Subtarget.hasV66Ops()) {
1797 setOperationAction(ISD::FADD, MVT::f64, Legal);
1798 setOperationAction(ISD::FSUB, MVT::f64, Legal);
1799 }
1800 if (Subtarget.hasV67Ops()) {
1801 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
1802 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
1803 setOperationAction(ISD::FMUL, MVT::f64, Legal);
1804 }
1805
1806 setTargetDAGCombine(ISD::VSELECT);
1807
1808 if (Subtarget.useHVXOps())
1809 initializeHVXLowering();
1810
1811 computeRegisterProperties(&HRI);
1812
1813 //
1814 // Library calls for unsupported operations
1815 //
1816 bool FastMath = EnableFastMath;
1817
1818 setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
1819 setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
1820 setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
1821 setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
1822 setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
1823 setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
1824 setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
1825 setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
1826
1827 setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf");
1828 setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf");
1829 setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti");
1830 setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti");
1831 setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti");
1832 setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti");
1833
1834 // This is the only fast library function for sqrtd.
1835 if (FastMath)
1836 setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
1837
1838 // Prefix is: nothing for "slow-math",
1839 // "fast2_" for V5+ fast-math double-precision
1840 // (actually, keep fast-math and fast-math2 separate for now)
1841 if (FastMath) {
1842 setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
1843 setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
1844 setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
1845 setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
1846 setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
1847 } else {
1848 setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
1849 setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
1850 setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
1851 setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
1852 setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
1853 }
1854
1855 if (FastMath)
1856 setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
1857 else
1858 setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
1859
1860 // These cause problems when the shift amount is non-constant.
1861 setLibcallName(RTLIB::SHL_I128, nullptr);
1862 setLibcallName(RTLIB::SRL_I128, nullptr);
1863 setLibcallName(RTLIB::SRA_I128, nullptr);
1864}
1865
1866const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
1867 switch ((HexagonISD::NodeType)Opcode) {
1868 case HexagonISD::ADDC: return "HexagonISD::ADDC";
1869 case HexagonISD::SUBC: return "HexagonISD::SUBC";
1870 case HexagonISD::ALLOCA: return "HexagonISD::ALLOCA";
1871 case HexagonISD::AT_GOT: return "HexagonISD::AT_GOT";
1872 case HexagonISD::AT_PCREL: return "HexagonISD::AT_PCREL";
1873 case HexagonISD::BARRIER: return "HexagonISD::BARRIER";
1874 case HexagonISD::CALL: return "HexagonISD::CALL";
1875 case HexagonISD::CALLnr: return "HexagonISD::CALLnr";
1876 case HexagonISD::CALLR: return "HexagonISD::CALLR";
1877 case HexagonISD::COMBINE: return "HexagonISD::COMBINE";
1878 case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP";
1879 case HexagonISD::CONST32: return "HexagonISD::CONST32";
1880 case HexagonISD::CP: return "HexagonISD::CP";
1881 case HexagonISD::DCFETCH: return "HexagonISD::DCFETCH";
1882 case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN";
1883 case HexagonISD::TSTBIT: return "HexagonISD::TSTBIT";
1884 case HexagonISD::EXTRACTU: return "HexagonISD::EXTRACTU";
1885 case HexagonISD::INSERT: return "HexagonISD::INSERT";
1886 case HexagonISD::JT: return "HexagonISD::JT";
1887 case HexagonISD::RET_FLAG: return "HexagonISD::RET_FLAG";
1888 case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN";
1889 case HexagonISD::VASL: return "HexagonISD::VASL";
1890 case HexagonISD::VASR: return "HexagonISD::VASR";
1891 case HexagonISD::VLSR: return "HexagonISD::VLSR";
1892 case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW";
1893 case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0";
1894 case HexagonISD::VROR: return "HexagonISD::VROR";
1895 case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE";
1896 case HexagonISD::PTRUE: return "HexagonISD::PTRUE";
1897 case HexagonISD::PFALSE: return "HexagonISD::PFALSE";
1898 case HexagonISD::D2P: return "HexagonISD::D2P";
1899 case HexagonISD::P2D: return "HexagonISD::P2D";
1900 case HexagonISD::V2Q: return "HexagonISD::V2Q";
1901 case HexagonISD::Q2V: return "HexagonISD::Q2V";
1902 case HexagonISD::QCAT: return "HexagonISD::QCAT";
1903 case HexagonISD::QTRUE: return "HexagonISD::QTRUE";
1904 case HexagonISD::QFALSE: return "HexagonISD::QFALSE";
1905 case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST";
1906 case HexagonISD::VALIGN: return "HexagonISD::VALIGN";
1907 case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR";
1908 case HexagonISD::VPACKL: return "HexagonISD::VPACKL";
1909 case HexagonISD::VUNPACK: return "HexagonISD::VUNPACK";
1910 case HexagonISD::VUNPACKU: return "HexagonISD::VUNPACKU";
1911 case HexagonISD::ISEL: return "HexagonISD::ISEL";
1912 case HexagonISD::OP_END: break;
1913 }
1914 return nullptr;
1915}
1916
1917bool
1918HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1919 const SDLoc &dl, SelectionDAG &DAG) const {
1920 auto *CA = dyn_cast<ConstantSDNode>(Ptr);
1921 if (!CA)
1922 return true;
1923 unsigned Addr = CA->getZExtValue();
1924 Align HaveAlign =
1925 Addr != 0 ? Align(1ull << countTrailingZeros(Addr)) : NeedAlign;
1926 if (HaveAlign >= NeedAlign)
1927 return true;
1928
1929 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
1930
1931 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
1932 DiagnosticInfoMisalignedTrap(StringRef M)
1933 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
1934 void print(DiagnosticPrinter &DP) const override {
1935 DP << Msg;
1936 }
1937 static bool classof(const DiagnosticInfo *DI) {
1938 return DI->getKind() == DK_MisalignedTrap;
1939 }
1940 StringRef Msg;
1941 };
1942
1943 std::string ErrMsg;
1944 raw_string_ostream O(ErrMsg);
1945 O << "Misaligned constant address: " << format_hex(Addr, 10)
1946 << " has alignment " << HaveAlign.value()
1947 << ", but the memory access requires " << NeedAlign.value();
1948 if (DebugLoc DL = dl.getDebugLoc())
1949 DL.print(O << ", at ");
1950 O << ". The instruction has been replaced with a trap.";
1951
1952 DAG.getContext()->diagnose(DiagnosticInfoMisalignedTrap(O.str()));
1953 return false;
1954}
1955
1956SDValue
1957HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
1958 const {
1959 const SDLoc &dl(Op);
1960 auto *LS = cast<LSBaseSDNode>(Op.getNode());
1961 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address")(static_cast <bool> (!LS->isIndexed() && "Not expecting indexed ops on constant address"
) ? void (0) : __assert_fail ("!LS->isIndexed() && \"Not expecting indexed ops on constant address\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 1961, __extension__ __PRETTY_FUNCTION__))
;
1962
1963 SDValue Chain = LS->getChain();
1964 SDValue Trap = DAG.getNode(ISD::TRAP, dl, MVT::Other, Chain);
1965 if (LS->getOpcode() == ISD::LOAD)
1966 return DAG.getMergeValues({DAG.getUNDEF(ty(Op)), Trap}, dl);
1967 return Trap;
1968}
1969
1970// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1971// intrinsic.
1972static bool isBrevLdIntrinsic(const Value *Inst) {
1973 unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1974 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1975 ID == Intrinsic::hexagon_L2_loadri_pbr ||
1976 ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1977 ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1978 ID == Intrinsic::hexagon_L2_loadrb_pbr ||
1979 ID == Intrinsic::hexagon_L2_loadrub_pbr);
1980}
1981
1982// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
1983// instruction. So far we only handle bitcast, extract value and bit reverse
1984// load intrinsic instructions. Should we handle CGEP ?
1985static Value *getBrevLdObject(Value *V) {
1986 if (Operator::getOpcode(V) == Instruction::ExtractValue ||
1987 Operator::getOpcode(V) == Instruction::BitCast)
1988 V = cast<Operator>(V)->getOperand(0);
1989 else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V))
1990 V = cast<Instruction>(V)->getOperand(0);
1991 return V;
1992}
1993
1994// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
1995// a back edge. If the back edge comes from the intrinsic itself, the incoming
1996// edge is returned.
1997static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
1998 const BasicBlock *Parent = PN->getParent();
1999 int Idx = -1;
2000 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
2001 BasicBlock *Blk = PN->getIncomingBlock(i);
2002 // Determine if the back edge is originated from intrinsic.
2003 if (Blk == Parent) {
2004 Value *BackEdgeVal = PN->getIncomingValue(i);
2005 Value *BaseVal;
2006 // Loop over till we return the same Value or we hit the IntrBaseVal.
2007 do {
2008 BaseVal = BackEdgeVal;
2009 BackEdgeVal = getBrevLdObject(BackEdgeVal);
2010 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
2011 // If the getBrevLdObject returns IntrBaseVal, we should return the
2012 // incoming edge.
2013 if (IntrBaseVal == BackEdgeVal)
2014 continue;
2015 Idx = i;
2016 break;
2017 } else // Set the node to incoming edge.
2018 Idx = i;
2019 }
2020 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI")(static_cast <bool> (Idx >= 0 && "Unexpected index to incoming argument in PHI"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Unexpected index to incoming argument in PHI\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2020, __extension__ __PRETTY_FUNCTION__))
;
2021 return PN->getIncomingValue(Idx);
2022}
2023
2024// Bit-reverse Load Intrinsic: Figure out the underlying object the base
2025// pointer points to, for the bit-reverse load intrinsic. Setting this to
2026// memoperand might help alias analysis to figure out the dependencies.
2027static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
2028 Value *IntrBaseVal = V;
2029 Value *BaseVal;
2030 // Loop over till we return the same Value, implies we either figure out
2031 // the object or we hit a PHI
2032 do {
2033 BaseVal = V;
2034 V = getBrevLdObject(V);
2035 } while (BaseVal != V);
2036
2037 // Identify the object from PHINode.
2038 if (const PHINode *PN = dyn_cast<PHINode>(V))
2039 return returnEdge(PN, IntrBaseVal);
2040 // For non PHI nodes, the object is the last value returned by getBrevLdObject
2041 else
2042 return V;
2043}
2044
2045/// Given an intrinsic, checks if on the target the intrinsic will need to map
2046/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
2047/// true and store the intrinsic information into the IntrinsicInfo that was
2048/// passed to the function.
2049bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2050 const CallInst &I,
2051 MachineFunction &MF,
2052 unsigned Intrinsic) const {
2053 switch (Intrinsic) {
2054 case Intrinsic::hexagon_L2_loadrd_pbr:
2055 case Intrinsic::hexagon_L2_loadri_pbr:
2056 case Intrinsic::hexagon_L2_loadrh_pbr:
2057 case Intrinsic::hexagon_L2_loadruh_pbr:
2058 case Intrinsic::hexagon_L2_loadrb_pbr:
2059 case Intrinsic::hexagon_L2_loadrub_pbr: {
2060 Info.opc = ISD::INTRINSIC_W_CHAIN;
2061 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
2062 auto &Cont = I.getCalledFunction()->getParent()->getContext();
2063 // The intrinsic function call is of the form { ElTy, i8* }
2064 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2065 // should be derived from ElTy.
2066 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0);
2067 Info.memVT = MVT::getVT(ElTy);
2068 llvm::Value *BasePtrVal = I.getOperand(0);
2069 Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal);
2070 // The offset value comes through Modifier register. For now, assume the
2071 // offset is 0.
2072 Info.offset = 0;
2073 Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont));
2074 Info.flags = MachineMemOperand::MOLoad;
2075 return true;
2076 }
2077 case Intrinsic::hexagon_V6_vgathermw:
2078 case Intrinsic::hexagon_V6_vgathermw_128B:
2079 case Intrinsic::hexagon_V6_vgathermh:
2080 case Intrinsic::hexagon_V6_vgathermh_128B:
2081 case Intrinsic::hexagon_V6_vgathermhw:
2082 case Intrinsic::hexagon_V6_vgathermhw_128B:
2083 case Intrinsic::hexagon_V6_vgathermwq:
2084 case Intrinsic::hexagon_V6_vgathermwq_128B:
2085 case Intrinsic::hexagon_V6_vgathermhq:
2086 case Intrinsic::hexagon_V6_vgathermhq_128B:
2087 case Intrinsic::hexagon_V6_vgathermhwq:
2088 case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2089 const Module &M = *I.getParent()->getParent()->getParent();
2090 Info.opc = ISD::INTRINSIC_W_CHAIN;
2091 Type *VecTy = I.getArgOperand(1)->getType();
2092 Info.memVT = MVT::getVT(VecTy);
2093 Info.ptrVal = I.getArgOperand(0);
2094 Info.offset = 0;
2095 Info.align =
2096 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2097 Info.flags = MachineMemOperand::MOLoad |
2098 MachineMemOperand::MOStore |
2099 MachineMemOperand::MOVolatile;
2100 return true;
2101 }
2102 default:
2103 break;
2104 }
2105 return false;
2106}
2107
2108bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2109 return X.getValueType().isScalarInteger(); // 'tstbit'
2110}
2111
2112bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2113 return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2));
2114}
2115
2116bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2117 if (!VT1.isSimple() || !VT2.isSimple())
2118 return false;
2119 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2120}
2121
2122bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2123 const MachineFunction &MF, EVT VT) const {
2124 return isOperationLegalOrCustom(ISD::FMA, VT);
2125}
2126
2127// Should we expand the build vector with shuffles?
2128bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2129 unsigned DefinedValues) const {
2130 return false;
2131}
2132
2133bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2134 EVT VT) const {
2135 return true;
2136}
2137
2138TargetLoweringBase::LegalizeTypeAction
2139HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2140 unsigned VecLen = VT.getVectorMinNumElements();
2141 MVT ElemTy = VT.getVectorElementType();
2142
2143 if (VecLen == 1 || VT.isScalableVector())
2144 return TargetLoweringBase::TypeScalarizeVector;
2145
2146 if (Subtarget.useHVXOps()) {
2147 unsigned Action = getPreferredHvxVectorAction(VT);
2148 if (Action != ~0u)
2149 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2150 }
2151
2152 // Always widen (remaining) vectors of i1.
2153 if (ElemTy == MVT::i1)
2154 return TargetLoweringBase::TypeWidenVector;
2155
2156 return TargetLoweringBase::TypeSplitVector;
2157}
2158
2159std::pair<SDValue, int>
2160HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2161 if (Addr.getOpcode() == ISD::ADD) {
2162 SDValue Op1 = Addr.getOperand(1);
2163 if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode()))
2164 return { Addr.getOperand(0), CN->getSExtValue() };
2165 }
2166 return { Addr, 0 };
2167}
2168
2169// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors
2170// to select data from, V3 is the permutation.
2171SDValue
2172HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2173 const {
2174 const auto *SVN = cast<ShuffleVectorSDNode>(Op);
2175 ArrayRef<int> AM = SVN->getMask();
2176 assert(AM.size() <= 8 && "Unexpected shuffle mask")(static_cast <bool> (AM.size() <= 8 && "Unexpected shuffle mask"
) ? void (0) : __assert_fail ("AM.size() <= 8 && \"Unexpected shuffle mask\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2176, __extension__ __PRETTY_FUNCTION__))
;
2177 unsigned VecLen = AM.size();
2178
2179 MVT VecTy = ty(Op);
2180 assert(!Subtarget.isHVXVectorType(VecTy, true) &&(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true
) && "HVX shuffles should be legal") ? void (0) : __assert_fail
("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2181, __extension__ __PRETTY_FUNCTION__))
2181 "HVX shuffles should be legal")(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true
) && "HVX shuffles should be legal") ? void (0) : __assert_fail
("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2181, __extension__ __PRETTY_FUNCTION__))
;
2182 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length")(static_cast <bool> (VecTy.getSizeInBits() <= 64 &&
"Unexpected vector length") ? void (0) : __assert_fail ("VecTy.getSizeInBits() <= 64 && \"Unexpected vector length\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2182, __extension__ __PRETTY_FUNCTION__))
;
2183
2184 SDValue Op0 = Op.getOperand(0);
2185 SDValue Op1 = Op.getOperand(1);
2186 const SDLoc &dl(Op);
2187
2188 // If the inputs are not the same as the output, bail. This is not an
2189 // error situation, but complicates the handling and the default expansion
2190 // (into BUILD_VECTOR) should be adequate.
2191 if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2192 return SDValue();
2193
2194 // Normalize the mask so that the first non-negative index comes from
2195 // the first operand.
2196 SmallVector<int,8> Mask(AM.begin(), AM.end());
2197 unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data();
2198 if (F == AM.size())
2199 return DAG.getUNDEF(VecTy);
2200 if (AM[F] >= int(VecLen)) {
2201 ShuffleVectorSDNode::commuteMask(Mask);
2202 std::swap(Op0, Op1);
2203 }
2204
2205 // Express the shuffle mask in terms of bytes.
2206 SmallVector<int,8> ByteMask;
2207 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2208 for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
2209 int M = Mask[i];
2210 if (M < 0) {
2211 for (unsigned j = 0; j != ElemBytes; ++j)
2212 ByteMask.push_back(-1);
2213 } else {
2214 for (unsigned j = 0; j != ElemBytes; ++j)
2215 ByteMask.push_back(M*ElemBytes + j);
2216 }
2217 }
2218 assert(ByteMask.size() <= 8)(static_cast <bool> (ByteMask.size() <= 8) ? void (0
) : __assert_fail ("ByteMask.size() <= 8", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2218, __extension__ __PRETTY_FUNCTION__))
;
2219
2220 // All non-undef (non-negative) indexes are well within [0..127], so they
2221 // fit in a single byte. Build two 64-bit words:
2222 // - MaskIdx where each byte is the corresponding index (for non-negative
2223 // indexes), and 0xFF for negative indexes, and
2224 // - MaskUnd that has 0xFF for each negative index.
2225 uint64_t MaskIdx = 0;
2226 uint64_t MaskUnd = 0;
2227 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2228 unsigned S = 8*i;
2229 uint64_t M = ByteMask[i] & 0xFF;
2230 if (M == 0xFF)
2231 MaskUnd |= M << S;
2232 MaskIdx |= M << S;
2233 }
2234
2235 if (ByteMask.size() == 4) {
2236 // Identity.
2237 if (MaskIdx == (0x03020100 | MaskUnd))
2238 return Op0;
2239 // Byte swap.
2240 if (MaskIdx == (0x00010203 | MaskUnd)) {
2241 SDValue T0 = DAG.getBitcast(MVT::i32, Op0);
2242 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0);
2243 return DAG.getBitcast(VecTy, T1);
2244 }
2245
2246 // Byte packs.
2247 SDValue Concat10 = DAG.getNode(HexagonISD::COMBINE, dl,
2248 typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0});
2249 if (MaskIdx == (0x06040200 | MaskUnd))
2250 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2251 if (MaskIdx == (0x07050301 | MaskUnd))
2252 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2253
2254 SDValue Concat01 = DAG.getNode(HexagonISD::COMBINE, dl,
2255 typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1});
2256 if (MaskIdx == (0x02000604 | MaskUnd))
2257 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2258 if (MaskIdx == (0x03010705 | MaskUnd))
2259 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2260 }
2261
2262 if (ByteMask.size() == 8) {
2263 // Identity.
2264 if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2265 return Op0;
2266 // Byte swap.
2267 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2268 SDValue T0 = DAG.getBitcast(MVT::i64, Op0);
2269 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0);
2270 return DAG.getBitcast(VecTy, T1);
2271 }
2272
2273 // Halfword picks.
2274 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2275 return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2276 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2277 return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2278 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2279 return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2280 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2281 return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2282 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2283 VectorPair P = opSplit(Op0, dl, DAG);
2284 return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG);
2285 }
2286
2287 // Byte packs.
2288 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2289 return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2290 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2291 return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2292 }
2293
2294 return SDValue();
2295}
2296
2297// Create a Hexagon-specific node for shifting a vector by an integer.
2298SDValue
2299HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2300 const {
2301 unsigned NewOpc;
2302 switch (Op.getOpcode()) {
2303 case ISD::SHL:
2304 NewOpc = HexagonISD::VASL;
2305 break;
2306 case ISD::SRA:
2307 NewOpc = HexagonISD::VASR;
2308 break;
2309 case ISD::SRL:
2310 NewOpc = HexagonISD::VLSR;
2311 break;
2312 default:
2313 llvm_unreachable("Unexpected shift opcode")::llvm::llvm_unreachable_internal("Unexpected shift opcode", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2313)
;
2314 }
2315
2316 SDValue Op0 = Op.getOperand(0);
2317 SDValue Op1 = Op.getOperand(1);
2318 const SDLoc &dl(Op);
2319
2320 switch (Op1.getOpcode()) {
2321 case ISD::BUILD_VECTOR:
2322 if (SDValue S = cast<BuildVectorSDNode>(Op1)->getSplatValue())
2323 return DAG.getNode(NewOpc, dl, ty(Op), Op0, S);
2324 break;
2325 case ISD::SPLAT_VECTOR:
2326 return DAG.getNode(NewOpc, dl, ty(Op), Op0, Op1.getOperand(0));
2327 }
2328 return SDValue();
2329}
2330
2331SDValue
2332HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2333 return getVectorShiftByInt(Op, DAG);
2334}
2335
2336SDValue
2337HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2338 if (isa<ConstantSDNode>(Op.getOperand(1).getNode()))
2339 return Op;
2340 return SDValue();
2341}
2342
2343SDValue
2344HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2345 MVT ResTy = ty(Op);
2346 SDValue InpV = Op.getOperand(0);
2347 MVT InpTy = ty(InpV);
2348 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits())(static_cast <bool> (ResTy.getSizeInBits() == InpTy.getSizeInBits
()) ? void (0) : __assert_fail ("ResTy.getSizeInBits() == InpTy.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2348, __extension__ __PRETTY_FUNCTION__))
;
2349 const SDLoc &dl(Op);
2350
2351 // Handle conversion from i8 to v8i1.
2352 if (InpTy == MVT::i8) {
2353 if (ResTy == MVT::v8i1) {
2354 SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV);
2355 SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32);
2356 return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG);
2357 }
2358 return SDValue();
2359 }
2360
2361 return Op;
2362}
2363
2364bool
2365HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2366 MVT VecTy, SelectionDAG &DAG,
2367 MutableArrayRef<ConstantInt*> Consts) const {
2368 MVT ElemTy = VecTy.getVectorElementType();
2369 unsigned ElemWidth = ElemTy.getSizeInBits();
2370 IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth);
2371 bool AllConst = true;
2372
2373 for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2374 SDValue V = Values[i];
2375 if (V.isUndef()) {
2376 Consts[i] = ConstantInt::get(IntTy, 0);
2377 continue;
2378 }
2379 // Make sure to always cast to IntTy.
2380 if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) {
2381 const ConstantInt *CI = CN->getConstantIntValue();
2382 Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue());
2383 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) {
2384 const ConstantFP *CF = CN->getConstantFPValue();
2385 APInt A = CF->getValueAPF().bitcastToAPInt();
2386 Consts[i] = ConstantInt::get(IntTy, A.getZExtValue());
2387 } else {
2388 AllConst = false;
2389 }
2390 }
2391 return AllConst;
2392}
2393
2394SDValue
2395HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2396 MVT VecTy, SelectionDAG &DAG) const {
2397 MVT ElemTy = VecTy.getVectorElementType();
2398 assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem
.size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2398, __extension__ __PRETTY_FUNCTION__))
;
2399
2400 SmallVector<ConstantInt*,4> Consts(Elem.size());
2401 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2402
2403 unsigned First, Num = Elem.size();
2404 for (First = 0; First != Num; ++First) {
2405 if (!isUndef(Elem[First]))
2406 break;
2407 }
2408 if (First == Num)
2409 return DAG.getUNDEF(VecTy);
2410
2411 if (AllConst &&
2412 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2413 return getZero(dl, VecTy, DAG);
2414
2415 if (ElemTy == MVT::i16) {
2416 assert(Elem.size() == 2)(static_cast <bool> (Elem.size() == 2) ? void (0) : __assert_fail
("Elem.size() == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2416, __extension__ __PRETTY_FUNCTION__))
;
2417 if (AllConst) {
2418 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2419 Consts[1]->getZExtValue() << 16;
2420 return DAG.getBitcast(MVT::v2i16, DAG.getConstant(V, dl, MVT::i32));
2421 }
2422 SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32,
2423 {Elem[1], Elem[0]}, DAG);
2424 return DAG.getBitcast(MVT::v2i16, N);
2425 }
2426
2427 if (ElemTy == MVT::i8) {
2428 // First try generating a constant.
2429 if (AllConst) {
2430 int32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2431 (Consts[1]->getZExtValue() & 0xFF) << 8 |
2432 (Consts[1]->getZExtValue() & 0xFF) << 16 |
2433 Consts[2]->getZExtValue() << 24;
2434 return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32));
2435 }
2436
2437 // Then try splat.
2438 bool IsSplat = true;
2439 for (unsigned i = First+1; i != Num; ++i) {
2440 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2441 continue;
2442 IsSplat = false;
2443 break;
2444 }
2445 if (IsSplat) {
2446 // Legalize the operand of SPLAT_VECTOR.
2447 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2448 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2449 }
2450
2451 // Generate
2452 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2453 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2454 assert(Elem.size() == 4)(static_cast <bool> (Elem.size() == 4) ? void (0) : __assert_fail
("Elem.size() == 4", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2454, __extension__ __PRETTY_FUNCTION__))
;
2455 SDValue Vs[4];
2456 for (unsigned i = 0; i != 4; ++i) {
2457 Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32);
2458 Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8);
2459 }
2460 SDValue S8 = DAG.getConstant(8, dl, MVT::i32);
2461 SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8});
2462 SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8});
2463 SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0});
2464 SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1});
2465
2466 SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG);
2467 return DAG.getBitcast(MVT::v4i8, R);
2468 }
2469
2470#ifndef NDEBUG
2471 dbgs() << "VecTy: " << EVT(VecTy).getEVTString() << '\n';
2472#endif
2473 llvm_unreachable("Unexpected vector element type")::llvm::llvm_unreachable_internal("Unexpected vector element type"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2473)
;
2474}
2475
2476SDValue
2477HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2478 MVT VecTy, SelectionDAG &DAG) const {
2479 MVT ElemTy = VecTy.getVectorElementType();
2480 assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem
.size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2480, __extension__ __PRETTY_FUNCTION__))
;
2481
2482 SmallVector<ConstantInt*,8> Consts(Elem.size());
2483 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2484
2485 unsigned First, Num = Elem.size();
2486 for (First = 0; First != Num; ++First) {
2487 if (!isUndef(Elem[First]))
2488 break;
2489 }
2490 if (First == Num)
2491 return DAG.getUNDEF(VecTy);
2492
2493 if (AllConst &&
2494 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2495 return getZero(dl, VecTy, DAG);
2496
2497 // First try splat if possible.
2498 if (ElemTy == MVT::i16) {
2499 bool IsSplat = true;
2500 for (unsigned i = First+1; i != Num; ++i) {
2501 if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2502 continue;
2503 IsSplat = false;
2504 break;
2505 }
2506 if (IsSplat) {
2507 // Legalize the operand of SPLAT_VECTOR
2508 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2509 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2510 }
2511 }
2512
2513 // Then try constant.
2514 if (AllConst) {
2515 uint64_t Val = 0;
2516 unsigned W = ElemTy.getSizeInBits();
2517 uint64_t Mask = (ElemTy == MVT::i8) ? 0xFFull
2518 : (ElemTy == MVT::i16) ? 0xFFFFull : 0xFFFFFFFFull;
2519 for (unsigned i = 0; i != Num; ++i)
2520 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2521 SDValue V0 = DAG.getConstant(Val, dl, MVT::i64);
2522 return DAG.getBitcast(VecTy, V0);
2523 }
2524
2525 // Build two 32-bit vectors and concatenate.
2526 MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2);
2527 SDValue L = (ElemTy == MVT::i32)
2528 ? Elem[0]
2529 : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG);
2530 SDValue H = (ElemTy == MVT::i32)
2531 ? Elem[1]
2532 : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG);
2533 return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, {H, L});
2534}
2535
2536SDValue
2537HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2538 const SDLoc &dl, MVT ValTy, MVT ResTy,
2539 SelectionDAG &DAG) const {
2540 MVT VecTy = ty(VecV);
2541 assert(!ValTy.isVector() ||(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType
() == ValTy.getVectorElementType()) ? void (0) : __assert_fail
("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2542, __extension__ __PRETTY_FUNCTION__))
2542 VecTy.getVectorElementType() == ValTy.getVectorElementType())(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType
() == ValTy.getVectorElementType()) ? void (0) : __assert_fail
("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2542, __extension__ __PRETTY_FUNCTION__))
;
2543 unsigned VecWidth = VecTy.getSizeInBits();
2544 unsigned ValWidth = ValTy.getSizeInBits();
2545 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2546 assert((VecWidth % ElemWidth) == 0)(static_cast <bool> ((VecWidth % ElemWidth) == 0) ? void
(0) : __assert_fail ("(VecWidth % ElemWidth) == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2546, __extension__ __PRETTY_FUNCTION__))
;
2547 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV);
2548
2549 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2550 // without any coprocessors).
2551 if (ElemWidth == 1) {
2552 assert(VecWidth == VecTy.getVectorNumElements() && "Sanity failure")(static_cast <bool> (VecWidth == VecTy.getVectorNumElements
() && "Sanity failure") ? void (0) : __assert_fail ("VecWidth == VecTy.getVectorNumElements() && \"Sanity failure\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2552, __extension__ __PRETTY_FUNCTION__))
;
2553 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2)(static_cast <bool> (VecWidth == 8 || VecWidth == 4 || VecWidth
== 2) ? void (0) : __assert_fail ("VecWidth == 8 || VecWidth == 4 || VecWidth == 2"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2553, __extension__ __PRETTY_FUNCTION__))
;
2554 // Check if this is an extract of the lowest bit.
2555 if (IdxN) {
2556 // Extracting the lowest bit is a no-op, but it changes the type,
2557 // so it must be kept as an operation to avoid errors related to
2558 // type mismatches.
2559 if (IdxN->isNullValue() && ValTy.getSizeInBits() == 1)
2560 return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV);
2561 }
2562
2563 // If the value extracted is a single bit, use tstbit.
2564 if (ValWidth == 1) {
2565 SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2566 SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32);
2567 SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0);
2568 return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0);
2569 }
2570
2571 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2572 // a predicate register. The elements of the vector are repeated
2573 // in the register (if necessary) so that the total number is 8.
2574 // The extracted subvector will need to be expanded in such a way.
2575 unsigned Scale = VecWidth / ValWidth;
2576
2577 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2578 // position 0.
2579 assert(ty(IdxV) == MVT::i32)(static_cast <bool> (ty(IdxV) == MVT::i32) ? void (0) :
__assert_fail ("ty(IdxV) == MVT::i32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2579, __extension__ __PRETTY_FUNCTION__))
;
2580 unsigned VecRep = 8 / VecWidth;
2581 SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2582 DAG.getConstant(8*VecRep, dl, MVT::i32));
2583 SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2584 SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0);
2585 while (Scale > 1) {
2586 // The longest possible subvector is at most 32 bits, so it is always
2587 // contained in the low subregister.
2588 T1 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, T1);
2589 T1 = expandPredicate(T1, dl, DAG);
2590 Scale /= 2;
2591 }
2592
2593 return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1);
2594 }
2595
2596 assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ?
void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2596, __extension__ __PRETTY_FUNCTION__))
;
2597
2598 // Cast everything to scalar integer types.
2599 MVT ScalarTy = tyScalar(VecTy);
2600 VecV = DAG.getBitcast(ScalarTy, VecV);
2601
2602 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2603 SDValue ExtV;
2604
2605 if (IdxN) {
2606 unsigned Off = IdxN->getZExtValue() * ElemWidth;
2607 if (VecWidth == 64 && ValWidth == 32) {
2608 assert(Off == 0 || Off == 32)(static_cast <bool> (Off == 0 || Off == 32) ? void (0) :
__assert_fail ("Off == 0 || Off == 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2608, __extension__ __PRETTY_FUNCTION__))
;
2609 unsigned SubIdx = Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi;
2610 ExtV = DAG.getTargetExtractSubreg(SubIdx, dl, MVT::i32, VecV);
2611 } else if (Off == 0 && (ValWidth % 8) == 0) {
2612 ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy));
2613 } else {
2614 SDValue OffV = DAG.getConstant(Off, dl, MVT::i32);
2615 // The return type of EXTRACTU must be the same as the type of the
2616 // input vector.
2617 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2618 {VecV, WidthV, OffV});
2619 }
2620 } else {
2621 if (ty(IdxV) != MVT::i32)
2622 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2623 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2624 DAG.getConstant(ElemWidth, dl, MVT::i32));
2625 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2626 {VecV, WidthV, OffV});
2627 }
2628
2629 // Cast ExtV to the requested result type.
2630 ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy));
2631 ExtV = DAG.getBitcast(ResTy, ExtV);
2632 return ExtV;
2633}
2634
2635SDValue
2636HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2637 const SDLoc &dl, MVT ValTy,
2638 SelectionDAG &DAG) const {
2639 MVT VecTy = ty(VecV);
2640 if (VecTy.getVectorElementType() == MVT::i1) {
2641 MVT ValTy = ty(ValV);
2642 assert(ValTy.getVectorElementType() == MVT::i1)(static_cast <bool> (ValTy.getVectorElementType() == MVT
::i1) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == MVT::i1"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2642, __extension__ __PRETTY_FUNCTION__))
;
2643 SDValue ValR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV);
2644 unsigned VecLen = VecTy.getVectorNumElements();
2645 unsigned Scale = VecLen / ValTy.getVectorNumElements();
2646 assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail
("Scale > 1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2646, __extension__ __PRETTY_FUNCTION__))
;
2647
2648 for (unsigned R = Scale; R > 1; R /= 2) {
2649 ValR = contractPredicate(ValR, dl, DAG);
2650 ValR = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2651 DAG.getUNDEF(MVT::i32), ValR);
2652 }
2653 // The longest possible subvector is at most 32 bits, so it is always
2654 // contained in the low subregister.
2655 ValR = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, ValR);
2656
2657 unsigned ValBytes = 64 / Scale;
2658 SDValue Width = DAG.getConstant(ValBytes*8, dl, MVT::i32);
2659 SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2660 DAG.getConstant(8, dl, MVT::i32));
2661 SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2662 SDValue Ins = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2663 {VecR, ValR, Width, Idx});
2664 return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins);
2665 }
2666
2667 unsigned VecWidth = VecTy.getSizeInBits();
2668 unsigned ValWidth = ValTy.getSizeInBits();
2669 assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ?
void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2669, __extension__ __PRETTY_FUNCTION__))
;
2670 assert((VecWidth % ValWidth) == 0)(static_cast <bool> ((VecWidth % ValWidth) == 0) ? void
(0) : __assert_fail ("(VecWidth % ValWidth) == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2670, __extension__ __PRETTY_FUNCTION__))
;
2671
2672 // Cast everything to scalar integer types.
2673 MVT ScalarTy = MVT::getIntegerVT(VecWidth);
2674 // The actual type of ValV may be different than ValTy (which is related
2675 // to the vector type).
2676 unsigned VW = ty(ValV).getSizeInBits();
2677 ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV);
2678 VecV = DAG.getBitcast(ScalarTy, VecV);
2679 if (VW != VecWidth)
2680 ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy);
2681
2682 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2683 SDValue InsV;
2684
2685 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) {
2686 unsigned W = C->getZExtValue() * ValWidth;
2687 SDValue OffV = DAG.getConstant(W, dl, MVT::i32);
2688 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2689 {VecV, ValV, WidthV, OffV});
2690 } else {
2691 if (ty(IdxV) != MVT::i32)
2692 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2693 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV);
2694 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2695 {VecV, ValV, WidthV, OffV});
2696 }
2697
2698 return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV);
2699}
2700
2701SDValue
2702HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2703 SelectionDAG &DAG) const {
2704 assert(ty(Vec32).getSizeInBits() == 32)(static_cast <bool> (ty(Vec32).getSizeInBits() == 32) ?
void (0) : __assert_fail ("ty(Vec32).getSizeInBits() == 32",
"/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2704, __extension__ __PRETTY_FUNCTION__))
;
2705 if (isUndef(Vec32))
2706 return DAG.getUNDEF(MVT::i64);
2707 return getInstr(Hexagon::S2_vsxtbh, dl, MVT::i64, {Vec32}, DAG);
2708}
2709
2710SDValue
2711HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2712 SelectionDAG &DAG) const {
2713 assert(ty(Vec64).getSizeInBits() == 64)(static_cast <bool> (ty(Vec64).getSizeInBits() == 64) ?
void (0) : __assert_fail ("ty(Vec64).getSizeInBits() == 64",
"/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2713, __extension__ __PRETTY_FUNCTION__))
;
2714 if (isUndef(Vec64))
2715 return DAG.getUNDEF(MVT::i32);
2716 return getInstr(Hexagon::S2_vtrunehb, dl, MVT::i32, {Vec64}, DAG);
2717}
2718
2719SDValue
2720HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2721 const {
2722 if (Ty.isVector()) {
2723 assert(Ty.isInteger() && "Only integer vectors are supported here")(static_cast <bool> (Ty.isInteger() && "Only integer vectors are supported here"
) ? void (0) : __assert_fail ("Ty.isInteger() && \"Only integer vectors are supported here\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2723, __extension__ __PRETTY_FUNCTION__))
;
2724 unsigned W = Ty.getSizeInBits();
2725 if (W <= 64)
2726 return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W)));
2727 return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG));
2728 }
2729
2730 if (Ty.isInteger())
2731 return DAG.getConstant(0, dl, Ty);
2732 if (Ty.isFloatingPoint())
2733 return DAG.getConstantFP(0.0, dl, Ty);
2734 llvm_unreachable("Invalid type for zero")::llvm::llvm_unreachable_internal("Invalid type for zero", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2734)
;
2735}
2736
2737SDValue
2738HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2739 const {
2740 MVT ValTy = ty(Val);
2741 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType())(static_cast <bool> (ValTy.getVectorElementType() == ResTy
.getVectorElementType()) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == ResTy.getVectorElementType()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2741, __extension__ __PRETTY_FUNCTION__))
;
2742
2743 unsigned ValLen = ValTy.getVectorNumElements();
2744 unsigned ResLen = ResTy.getVectorNumElements();
2745 if (ValLen == ResLen)
2746 return Val;
2747
2748 const SDLoc &dl(Val);
2749 assert(ValLen < ResLen)(static_cast <bool> (ValLen < ResLen) ? void (0) : __assert_fail
("ValLen < ResLen", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2749, __extension__ __PRETTY_FUNCTION__))
;
2750 assert(ResLen % ValLen == 0)(static_cast <bool> (ResLen % ValLen == 0) ? void (0) :
__assert_fail ("ResLen % ValLen == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2750, __extension__ __PRETTY_FUNCTION__))
;
2751
2752 SmallVector<SDValue, 4> Concats = {Val};
2753 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2754 Concats.push_back(DAG.getUNDEF(ValTy));
2755
2756 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats);
2757}
2758
2759SDValue
2760HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2761 MVT VecTy = ty(Op);
2762 unsigned BW = VecTy.getSizeInBits();
2763 const SDLoc &dl(Op);
2764 SmallVector<SDValue,8> Ops;
2765 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2766 Ops.push_back(Op.getOperand(i));
2767
2768 if (BW == 32)
2769 return buildVector32(Ops, dl, VecTy, DAG);
2770 if (BW == 64)
2771 return buildVector64(Ops, dl, VecTy, DAG);
2772
2773 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2774 // Check if this is a special case or all-0 or all-1.
2775 bool All0 = true, All1 = true;
2776 for (SDValue P : Ops) {
2777 auto *CN = dyn_cast<ConstantSDNode>(P.getNode());
2778 if (CN == nullptr) {
2779 All0 = All1 = false;
2780 break;
2781 }
2782 uint32_t C = CN->getZExtValue();
2783 All0 &= (C == 0);
2784 All1 &= (C == 1);
2785 }
2786 if (All0)
2787 return DAG.getNode(HexagonISD::PFALSE, dl, VecTy);
2788 if (All1)
2789 return DAG.getNode(HexagonISD::PTRUE, dl, VecTy);
2790
2791 // For each i1 element in the resulting predicate register, put 1
2792 // shifted by the index of the element into a general-purpose register,
2793 // then or them together and transfer it back into a predicate register.
2794 SDValue Rs[8];
2795 SDValue Z = getZero(dl, MVT::i32, DAG);
2796 // Always produce 8 bits, repeat inputs if necessary.
2797 unsigned Rep = 8 / VecTy.getVectorNumElements();
2798 for (unsigned i = 0; i != 8; ++i) {
2799 SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32);
2800 Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z);
2801 }
2802 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) {
2803 for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2804 Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]);
2805 }
2806 // Move the value directly to a predicate register.
2807 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2808 }
2809
2810 return SDValue();
2811}
2812
2813SDValue
2814HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2815 SelectionDAG &DAG) const {
2816 MVT VecTy = ty(Op);
2817 const SDLoc &dl(Op);
2818 if (VecTy.getSizeInBits() == 64) {
2819 assert(Op.getNumOperands() == 2)(static_cast <bool> (Op.getNumOperands() == 2) ? void (
0) : __assert_fail ("Op.getNumOperands() == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2819, __extension__ __PRETTY_FUNCTION__))
;
2820 return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, Op.getOperand(1),
2821 Op.getOperand(0));
2822 }
2823
2824 MVT ElemTy = VecTy.getVectorElementType();
2825 if (ElemTy == MVT::i1) {
2826 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1)(static_cast <bool> (VecTy == MVT::v2i1 || VecTy == MVT
::v4i1 || VecTy == MVT::v8i1) ? void (0) : __assert_fail ("VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2826, __extension__ __PRETTY_FUNCTION__))
;
2827 MVT OpTy = ty(Op.getOperand(0));
2828 // Scale is how many times the operands need to be contracted to match
2829 // the representation in the target register.
2830 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2831 assert(Scale == Op.getNumOperands() && Scale > 1)(static_cast <bool> (Scale == Op.getNumOperands() &&
Scale > 1) ? void (0) : __assert_fail ("Scale == Op.getNumOperands() && Scale > 1"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2831, __extension__ __PRETTY_FUNCTION__))
;
2832
2833 // First, convert all bool vectors to integers, then generate pairwise
2834 // inserts to form values of doubled length. Up until there are only
2835 // two values left to concatenate, all of these values will fit in a
2836 // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2837 SmallVector<SDValue,4> Words[2];
2838 unsigned IdxW = 0;
2839
2840 for (SDValue P : Op.getNode()->op_values()) {
2841 SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P);
2842 for (unsigned R = Scale; R > 1; R /= 2) {
2843 W = contractPredicate(W, dl, DAG);
2844 W = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2845 DAG.getUNDEF(MVT::i32), W);
2846 }
2847 W = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, W);
2848 Words[IdxW].push_back(W);
2849 }
2850
2851 while (Scale > 2) {
2852 SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32);
2853 Words[IdxW ^ 1].clear();
2854
2855 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
2856 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
2857 // Insert W1 into W0 right next to the significant bits of W0.
2858 SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
2859 {W0, W1, WidthV, WidthV});
2860 Words[IdxW ^ 1].push_back(T);
2861 }
2862 IdxW ^= 1;
2863 Scale /= 2;
2864 }
2865
2866 // Another sanity check. At this point there should only be two words
2867 // left, and Scale should be 2.
2868 assert(Scale == 2 && Words[IdxW].size() == 2)(static_cast <bool> (Scale == 2 && Words[IdxW].
size() == 2) ? void (0) : __assert_fail ("Scale == 2 && Words[IdxW].size() == 2"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 2868, __extension__ __PRETTY_FUNCTION__))
;
2869
2870 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64,
2871 Words[IdxW][1], Words[IdxW][0]);
2872 return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW);
2873 }
2874
2875 return SDValue();
2876}
2877
2878SDValue
2879HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2880 SelectionDAG &DAG) const {
2881 SDValue Vec = Op.getOperand(0);
2882 MVT ElemTy = ty(Vec).getVectorElementType();
2883 return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG);
2884}
2885
2886SDValue
2887HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
2888 SelectionDAG &DAG) const {
2889 return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op),
2890 ty(Op), ty(Op), DAG);
2891}
2892
2893SDValue
2894HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2895 SelectionDAG &DAG) const {
2896 return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
2897 SDLoc(Op), ty(Op).getVectorElementType(), DAG);
2898}
2899
2900SDValue
2901HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
2902 SelectionDAG &DAG) const {
2903 SDValue ValV = Op.getOperand(1);
2904 return insertVector(Op.getOperand(0), ValV, Op.getOperand(2),
2905 SDLoc(Op), ty(ValV), DAG);
2906}
2907
2908bool
2909HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
2910 // Assuming the caller does not have either a signext or zeroext modifier, and
2911 // only one value is accepted, any reasonable truncation is allowed.
2912 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
2913 return false;
2914
2915 // FIXME: in principle up to 64-bit could be made safe, but it would be very
2916 // fragile at the moment: any support for multiple value returns would be
2917 // liable to disallow tail calls involving i64 -> iN truncation in many cases.
2918 return Ty1->getPrimitiveSizeInBits() <= 32;
2919}
2920
2921SDValue
2922HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
2923 MVT Ty = ty(Op);
2924 const SDLoc &dl(Op);
2925 // Lower loads of scalar predicate vectors (v2i1, v4i1, v8i1) to loads of i1
2926 // followed by a TYPECAST.
2927 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
2928 bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1);
2929 if (DoCast) {
2930 SDValue NL = DAG.getLoad(
2931 LN->getAddressingMode(), LN->getExtensionType(), MVT::i1, dl,
2932 LN->getChain(), LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(),
2933 /*MemoryVT*/ MVT::i1, LN->getAlign(), LN->getMemOperand()->getFlags(),
2934 LN->getAAInfo(), LN->getRanges());
2935 LN = cast<LoadSDNode>(NL.getNode());
2936 }
2937
2938 Align ClaimAlign = LN->getAlign();
2939 if (!validateConstPtrAlignment(LN->getBasePtr(), ClaimAlign, dl, DAG))
2940 return replaceMemWithUndef(Op, DAG);
2941
2942 // Call LowerUnalignedLoad for all loads, it recognizes loads that
2943 // don't need extra aligning.
2944 SDValue LU = LowerUnalignedLoad(SDValue(LN, 0), DAG);
2945 if (DoCast) {
2946 SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, Ty, LU);
2947 SDValue Ch = cast<LoadSDNode>(LU.getNode())->getChain();
2948 return DAG.getMergeValues({TC, Ch}, dl);
2949 }
2950 return LU;
2951}
2952
2953SDValue
2954HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
2955 const SDLoc &dl(Op);
2956 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
2957 SDValue Val = SN->getValue();
2958 MVT Ty = ty(Val);
2959
2960 bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1);
2961 if (DoCast) {
2962 SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, Val);
2963 SDValue NS = DAG.getStore(SN->getChain(), dl, TC, SN->getBasePtr(),
2964 SN->getMemOperand());
2965 if (SN->isIndexed()) {
2966 NS = DAG.getIndexedStore(NS, dl, SN->getBasePtr(), SN->getOffset(),
2967 SN->getAddressingMode());
2968 }
2969 SN = cast<StoreSDNode>(NS.getNode());
2970 }
2971
2972 Align ClaimAlign = SN->getAlign();
2973 if (!validateConstPtrAlignment(SN->getBasePtr(), ClaimAlign, dl, DAG))
2974 return replaceMemWithUndef(Op, DAG);
2975
2976 MVT StoreTy = SN->getMemoryVT().getSimpleVT();
2977 Align NeedAlign = Subtarget.getTypeAlignment(StoreTy);
2978 if (ClaimAlign < NeedAlign)
2979 return expandUnalignedStore(SN, DAG);
2980 return SDValue(SN, 0);
2981}
2982
2983SDValue
2984HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
2985 const {
2986 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
2987 MVT LoadTy = ty(Op);
2988 unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy).value();
2989 unsigned HaveAlign = LN->getAlign().value();
2990 if (HaveAlign >= NeedAlign)
2991 return Op;
2992
2993 const SDLoc &dl(Op);
2994 const DataLayout &DL = DAG.getDataLayout();
2995 LLVMContext &Ctx = *DAG.getContext();
2996
2997 // If the load aligning is disabled or the load can be broken up into two
2998 // smaller legal loads, do the default (target-independent) expansion.
2999 bool DoDefault = false;
3000 // Handle it in the default way if this is an indexed load.
3001 if (!LN->isUnindexed())
3002 DoDefault = true;
3003
3004 if (!AlignLoads) {
3005 if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(),
3006 *LN->getMemOperand()))
3007 return Op;
3008 DoDefault = true;
3009 }
3010 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3011 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3012 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign)
3013 : MVT::getVectorVT(MVT::i8, HaveAlign);
3014 DoDefault =
3015 allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand());
3016 }
3017 if (DoDefault) {
3018 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
3019 return DAG.getMergeValues({P.first, P.second}, dl);
3020 }
3021
3022 // The code below generates two loads, both aligned as NeedAlign, and
3023 // with the distance of NeedAlign between them. For that to cover the
3024 // bits that need to be loaded (and without overlapping), the size of
3025 // the loads should be equal to NeedAlign. This is true for all loadable
3026 // types, but add an assertion in case something changes in the future.
3027 assert(LoadTy.getSizeInBits() == 8*NeedAlign)(static_cast <bool> (LoadTy.getSizeInBits() == 8*NeedAlign
) ? void (0) : __assert_fail ("LoadTy.getSizeInBits() == 8*NeedAlign"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3027, __extension__ __PRETTY_FUNCTION__))
;
3028
3029 unsigned LoadLen = NeedAlign;
3030 SDValue Base = LN->getBasePtr();
3031 SDValue Chain = LN->getChain();
3032 auto BO = getBaseAndOffset(Base);
3033 unsigned BaseOpc = BO.first.getOpcode();
3034 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3035 return Op;
3036
3037 if (BO.second % LoadLen != 0) {
3038 BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first,
3039 DAG.getConstant(BO.second % LoadLen, dl, MVT::i32));
3040 BO.second -= BO.second % LoadLen;
3041 }
3042 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3043 ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first,
3044 DAG.getConstant(NeedAlign, dl, MVT::i32))
3045 : BO.first;
3046 SDValue Base0 =
3047 DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl);
3048 SDValue Base1 = DAG.getMemBasePlusOffset(
3049 BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl);
3050
3051 MachineMemOperand *WideMMO = nullptr;
3052 if (MachineMemOperand *MMO = LN->getMemOperand()) {
3053 MachineFunction &MF = DAG.getMachineFunction();
3054 WideMMO = MF.getMachineMemOperand(
3055 MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
3056 MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
3057 MMO->getSuccessOrdering(), MMO->getFailureOrdering());
3058 }
3059
3060 SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
3061 SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO);
3062
3063 SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy,
3064 {Load1, Load0, BaseNoOff.getOperand(0)});
3065 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3066 Load0.getValue(1), Load1.getValue(1));
3067 SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl);
3068 return M;
3069}
3070
3071SDValue
3072HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3073 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
3074 auto *CY = dyn_cast<ConstantSDNode>(Y);
3075 if (!CY)
3076 return SDValue();
3077
3078 const SDLoc &dl(Op);
3079 SDVTList VTs = Op.getNode()->getVTList();
3080 assert(VTs.NumVTs == 2)(static_cast <bool> (VTs.NumVTs == 2) ? void (0) : __assert_fail
("VTs.NumVTs == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
;
3081 assert(VTs.VTs[1] == MVT::i1)(static_cast <bool> (VTs.VTs[1] == MVT::i1) ? void (0) :
__assert_fail ("VTs.VTs[1] == MVT::i1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))
;
3082 unsigned Opc = Op.getOpcode();
3083
3084 if (CY) {
3085 uint32_t VY = CY->getZExtValue();
3086 assert(VY != 0 && "This should have been folded")(static_cast <bool> (VY != 0 && "This should have been folded"
) ? void (0) : __assert_fail ("VY != 0 && \"This should have been folded\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3086, __extension__ __PRETTY_FUNCTION__))
;
3087 // X +/- 1
3088 if (VY != 1)
3089 return SDValue();
3090
3091 if (Opc == ISD::UADDO) {
3092 SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y});
3093 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG),
3094 ISD::SETEQ);
3095 return DAG.getMergeValues({Op, Ov}, dl);
3096 }
3097 if (Opc == ISD::USUBO) {
3098 SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y});
3099 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op,
3100 DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ);
3101 return DAG.getMergeValues({Op, Ov}, dl);
3102 }
3103 }
3104
3105 return SDValue();
3106}
3107
3108SDValue
3109HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const {
3110 const SDLoc &dl(Op);
3111 unsigned Opc = Op.getOpcode();
3112 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);
3113
3114 if (Opc == ISD::ADDCARRY)
3115 return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(),
3116 { X, Y, C });
3117
3118 EVT CarryTy = C.getValueType();
3119 SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(),
3120 { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3121 SDValue Out[] = { SubC.getValue(0),
3122 DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) };
3123 return DAG.getMergeValues(Out, dl);
3124}
3125
3126SDValue
3127HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3128 SDValue Chain = Op.getOperand(0);
3129 SDValue Offset = Op.getOperand(1);
3130 SDValue Handler = Op.getOperand(2);
3131 SDLoc dl(Op);
3132 auto PtrVT = getPointerTy(DAG.getDataLayout());
3133
3134 // Mark function as containing a call to EH_RETURN.
3135 HexagonMachineFunctionInfo *FuncInfo =
3136 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3137 FuncInfo->setHasEHReturn();
3138
3139 unsigned OffsetReg = Hexagon::R28;
3140
3141 SDValue StoreAddr =
3142 DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT),
3143 DAG.getIntPtrConstant(4, dl));
3144 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
3145 Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
3146
3147 // Not needed we already use it as explict input to EH_RETURN.
3148 // MF.getRegInfo().addLiveOut(OffsetReg);
3149
3150 return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain);
3151}
3152
3153SDValue
3154HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3155 unsigned Opc = Op.getOpcode();
3156
3157 // Handle INLINEASM first.
3158 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
1
Assuming 'Opc' is not equal to INLINEASM
2
Assuming 'Opc' is not equal to INLINEASM_BR
3
Taking false branch
3159 return LowerINLINEASM(Op, DAG);
3160
3161 if (isHvxOperation(Op.getNode(), DAG)) {
4
Assuming the condition is false
5
Taking false branch
3162 // If HVX lowering returns nothing, try the default lowering.
3163 if (SDValue V = LowerHvxOperation(Op, DAG))
3164 return V;
3165 }
3166
3167 switch (Opc) {
6
Control jumps to 'case SETCC:' at line 3206
3168 default:
3169#ifndef NDEBUG
3170 Op.getNode()->dumpr(&DAG);
3171 if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END)
3172 errs() << "Error: check for a non-legal type in this operation\n";
3173#endif
3174 llvm_unreachable("Should not custom lower this!")::llvm::llvm_unreachable_internal("Should not custom lower this!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3174)
;
3175 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
3176 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);
3177 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
3178 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
3179 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3180 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
3181 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
3182 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
3183 case ISD::LOAD: return LowerLoad(Op, DAG);
3184 case ISD::STORE: return LowerStore(Op, DAG);
3185 case ISD::UADDO:
3186 case ISD::USUBO: return LowerUAddSubO(Op, DAG);
3187 case ISD::ADDCARRY:
3188 case ISD::SUBCARRY: return LowerAddSubCarry(Op, DAG);
3189 case ISD::SRA:
3190 case ISD::SHL:
3191 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);
3192 case ISD::ROTL: return LowerROTL(Op, DAG);
3193 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
3194 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
3195 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
3196 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
3197 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
3198 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
3199 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
3200 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);
3201 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
3202 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3203 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
3204 case ISD::VASTART: return LowerVASTART(Op, DAG);
3205 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
3206 case ISD::SETCC: return LowerSETCC(Op, DAG);
7
Calling 'HexagonTargetLowering::LowerSETCC'
3207 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
3208 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3209 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
3210 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG);
3211 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG);
3212 break;
3213 }
3214
3215 return SDValue();
3216}
3217
3218void
3219HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3220 SmallVectorImpl<SDValue> &Results,
3221 SelectionDAG &DAG) const {
3222 if (isHvxOperation(N, DAG)) {
3223 LowerHvxOperationWrapper(N, Results, DAG);
3224 if (!Results.empty())
3225 return;
3226 }
3227
3228 // We are only custom-lowering stores to verify the alignment of the
3229 // address if it is a compile-time constant. Since a store can be modified
3230 // during type-legalization (the value being stored may need legalization),
3231 // return empty Results here to indicate that we don't really make any
3232 // changes in the custom lowering.
3233 if (N->getOpcode() != ISD::STORE)
3234 return TargetLowering::LowerOperationWrapper(N, Results, DAG);
3235}
3236
3237void
3238HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3239 SmallVectorImpl<SDValue> &Results,
3240 SelectionDAG &DAG) const {
3241 if (isHvxOperation(N, DAG)) {
3242 ReplaceHvxNodeResults(N, Results, DAG);
3243 if (!Results.empty())
3244 return;
3245 }
3246
3247 const SDLoc &dl(N);
3248 switch (N->getOpcode()) {
3249 case ISD::SRL:
3250 case ISD::SRA:
3251 case ISD::SHL:
3252 return;
3253 case ISD::BITCAST:
3254 // Handle a bitcast from v8i1 to i8.
3255 if (N->getValueType(0) == MVT::i8) {
3256 if (N->getOperand(0).getValueType() == MVT::v8i1) {
3257 SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32,
3258 N->getOperand(0), DAG);
3259 SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8);
3260 Results.push_back(T);
3261 }
3262 }
3263 break;
3264 }
3265}
3266
3267SDValue
3268HexagonTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
3269 const {
3270 if (isHvxOperation(N, DCI.DAG)) {
3271 if (SDValue V = PerformHvxDAGCombine(N, DCI))
3272 return V;
3273 return SDValue();
3274 }
3275
3276 if (DCI.isBeforeLegalizeOps())
3277 return SDValue();
3278
3279 SDValue Op(N, 0);
3280 const SDLoc &dl(Op);
3281 unsigned Opc = Op.getOpcode();
3282
3283 if (Opc == HexagonISD::P2D) {
3284 SDValue P = Op.getOperand(0);
3285 switch (P.getOpcode()) {
3286 case HexagonISD::PTRUE:
3287 return DCI.DAG.getConstant(-1, dl, ty(Op));
3288 case HexagonISD::PFALSE:
3289 return getZero(dl, ty(Op), DCI.DAG);
3290 default:
3291 break;
3292 }
3293 } else if (Opc == ISD::VSELECT) {
3294 // This is pretty much duplicated in HexagonISelLoweringHVX...
3295 //
3296 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3297 SDValue Cond = Op.getOperand(0);
3298 if (Cond->getOpcode() == ISD::XOR) {
3299 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
3300 if (C1->getOpcode() == HexagonISD::PTRUE) {
3301 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
3302 Op.getOperand(2), Op.getOperand(1));
3303 return VSel;
3304 }
3305 }
3306 }
3307
3308 return SDValue();
3309}
3310
3311/// Returns relocation base for the given PIC jumptable.
3312SDValue
3313HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3314 SelectionDAG &DAG) const {
3315 int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3316 EVT VT = Table.getValueType();
3317 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
3318 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T);
3319}
3320
3321//===----------------------------------------------------------------------===//
3322// Inline Assembly Support
3323//===----------------------------------------------------------------------===//
3324
3325TargetLowering::ConstraintType
3326HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3327 if (Constraint.size() == 1) {
3328 switch (Constraint[0]) {
3329 case 'q':
3330 case 'v':
3331 if (Subtarget.useHVXOps())
3332 return C_RegisterClass;
3333 break;
3334 case 'a':
3335 return C_RegisterClass;
3336 default:
3337 break;
3338 }
3339 }
3340 return TargetLowering::getConstraintType(Constraint);
3341}
3342
3343std::pair<unsigned, const TargetRegisterClass*>
3344HexagonTargetLowering::getRegForInlineAsmConstraint(
3345 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3346
3347 if (Constraint.size() == 1) {
3348 switch (Constraint[0]) {
3349 case 'r': // R0-R31
3350 switch (VT.SimpleTy) {
3351 default:
3352 return {0u, nullptr};
3353 case MVT::i1:
3354 case MVT::i8:
3355 case MVT::i16:
3356 case MVT::i32:
3357 case MVT::f32:
3358 return {0u, &Hexagon::IntRegsRegClass};
3359 case MVT::i64:
3360 case MVT::f64:
3361 return {0u, &Hexagon::DoubleRegsRegClass};
3362 }
3363 break;
3364 case 'a': // M0-M1
3365 if (VT != MVT::i32)
3366 return {0u, nullptr};
3367 return {0u, &Hexagon::ModRegsRegClass};
3368 case 'q': // q0-q3
3369 switch (VT.getSizeInBits()) {
3370 default:
3371 return {0u, nullptr};
3372 case 64:
3373 case 128:
3374 return {0u, &Hexagon::HvxQRRegClass};
3375 }
3376 break;
3377 case 'v': // V0-V31
3378 switch (VT.getSizeInBits()) {
3379 default:
3380 return {0u, nullptr};
3381 case 512:
3382 return {0u, &Hexagon::HvxVRRegClass};
3383 case 1024:
3384 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3385 return {0u, &Hexagon::HvxVRRegClass};
3386 return {0u, &Hexagon::HvxWRRegClass};
3387 case 2048:
3388 return {0u, &Hexagon::HvxWRRegClass};
3389 }
3390 break;
3391 default:
3392 return {0u, nullptr};
3393 }
3394 }
3395
3396 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3397}
3398
3399/// isFPImmLegal - Returns true if the target can instruction select the
3400/// specified FP immediate natively. If false, the legalizer will
3401/// materialize the FP immediate as a load from a constant pool.
3402bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3403 bool ForCodeSize) const {
3404 return true;
3405}
3406
3407/// isLegalAddressingMode - Return true if the addressing mode represented by
3408/// AM is legal for this target, for a load/store of the specified type.
3409bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3410 const AddrMode &AM, Type *Ty,
3411 unsigned AS, Instruction *I) const {
3412 if (Ty->isSized()) {
3413 // When LSR detects uses of the same base address to access different
3414 // types (e.g. unions), it will assume a conservative type for these
3415 // uses:
3416 // LSR Use: Kind=Address of void in addrspace(4294967295), ...
3417 // The type Ty passed here would then be "void". Skip the alignment
3418 // checks, but do not return false right away, since that confuses
3419 // LSR into crashing.
3420 Align A = DL.getABITypeAlign(Ty);
3421 // The base offset must be a multiple of the alignment.
3422 if (!isAligned(A, AM.BaseOffs))
3423 return false;
3424 // The shifted offset must fit in 11 bits.
3425 if (!isInt<11>(AM.BaseOffs >> Log2(A)))
3426 return false;
3427 }
3428
3429 // No global is ever allowed as a base.
3430 if (AM.BaseGV)
3431 return false;
3432
3433 int Scale = AM.Scale;
3434 if (Scale < 0)
3435 Scale = -Scale;
3436 switch (Scale) {
3437 case 0: // No scale reg, "r+i", "r", or just "i".
3438 break;
3439 default: // No scaled addressing mode.
3440 return false;
3441 }
3442 return true;
3443}
3444
3445/// Return true if folding a constant offset with the given GlobalAddress is
3446/// legal. It is frequently not legal in PIC relocation models.
3447bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3448 const {
3449 return HTM.getRelocationModel() == Reloc::Static;
3450}
3451
3452/// isLegalICmpImmediate - Return true if the specified immediate is legal
3453/// icmp immediate, that is the target has icmp instructions which can compare
3454/// a register against the immediate without having to materialize the
3455/// immediate into a register.
3456bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3457 return Imm >= -512 && Imm <= 511;
3458}
3459
3460/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3461/// for tail call optimization. Targets which want to do tail call
3462/// optimization should implement this function.
3463bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3464 SDValue Callee,
3465 CallingConv::ID CalleeCC,
3466 bool IsVarArg,
3467 bool IsCalleeStructRet,
3468 bool IsCallerStructRet,
3469 const SmallVectorImpl<ISD::OutputArg> &Outs,
3470 const SmallVectorImpl<SDValue> &OutVals,
3471 const SmallVectorImpl<ISD::InputArg> &Ins,
3472 SelectionDAG& DAG) const {
3473 const Function &CallerF = DAG.getMachineFunction().getFunction();
3474 CallingConv::ID CallerCC = CallerF.getCallingConv();
3475 bool CCMatch = CallerCC == CalleeCC;
3476
3477 // ***************************************************************************
3478 // Look for obvious safe cases to perform tail call optimization that do not
3479 // require ABI changes.
3480 // ***************************************************************************
3481
3482 // If this is a tail call via a function pointer, then don't do it!
3483 if (!isa<GlobalAddressSDNode>(Callee) &&
3484 !isa<ExternalSymbolSDNode>(Callee)) {
3485 return false;
3486 }
3487
3488 // Do not optimize if the calling conventions do not match and the conventions
3489 // used are not C or Fast.
3490 if (!CCMatch) {
3491 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3492 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3493 // If R & E, then ok.
3494 if (!R || !E)
3495 return false;
3496 }
3497
3498 // Do not tail call optimize vararg calls.
3499 if (IsVarArg)
3500 return false;
3501
3502 // Also avoid tail call optimization if either caller or callee uses struct
3503 // return semantics.
3504 if (IsCalleeStructRet || IsCallerStructRet)
3505 return false;
3506
3507 // In addition to the cases above, we also disable Tail Call Optimization if
3508 // the calling convention code that at least one outgoing argument needs to
3509 // go on the stack. We cannot check that here because at this point that
3510 // information is not available.
3511 return true;
3512}
3513
3514/// Returns the target specific optimal type for load and store operations as
3515/// a result of memset, memcpy, and memmove lowering.
3516///
3517/// If DstAlign is zero that means it's safe to destination alignment can
3518/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3519/// a need to check it against alignment requirement, probably because the
3520/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3521/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3522/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3523/// does not need to be loaded. It returns EVT::Other if the type should be
3524/// determined using generic target-independent logic.
3525EVT HexagonTargetLowering::getOptimalMemOpType(
3526 const MemOp &Op, const AttributeList &FuncAttributes) const {
3527 if (Op.size() >= 8 && Op.isAligned(Align(8)))
3528 return MVT::i64;
3529 if (Op.size() >= 4 && Op.isAligned(Align(4)))
3530 return MVT::i32;
3531 if (Op.size() >= 2 && Op.isAligned(Align(2)))
3532 return MVT::i16;
3533 return MVT::Other;
3534}
3535
3536bool HexagonTargetLowering::allowsMemoryAccess(
3537 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3538 Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
3539 MVT SVT = VT.getSimpleVT();
3540 if (Subtarget.isHVXVectorType(SVT, true))
3541 return allowsHvxMemoryAccess(SVT, Flags, Fast);
3542 return TargetLoweringBase::allowsMemoryAccess(
3543 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3544}
3545
3546bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3547 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3548 bool *Fast) const {
3549 MVT SVT = VT.getSimpleVT();
3550 if (Subtarget.isHVXVectorType(SVT, true))
3551 return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
3552 if (Fast)
3553 *Fast = false;
3554 return false;
3555}
3556
3557std::pair<const TargetRegisterClass*, uint8_t>
3558HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3559 MVT VT) const {
3560 if (Subtarget.isHVXVectorType(VT, true)) {
3561 unsigned BitWidth = VT.getSizeInBits();
3562 unsigned VecWidth = Subtarget.getVectorLength() * 8;
3563
3564 if (VT.getVectorElementType() == MVT::i1)
3565 return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3566 if (BitWidth == VecWidth)
3567 return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3568 assert(BitWidth == 2 * VecWidth)(static_cast <bool> (BitWidth == 2 * VecWidth) ? void (
0) : __assert_fail ("BitWidth == 2 * VecWidth", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3568, __extension__ __PRETTY_FUNCTION__))
;
3569 return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3570 }
3571
3572 return TargetLowering::findRepresentativeClass(TRI, VT);
3573}
3574
3575bool HexagonTargetLowering::shouldReduceLoadWidth(SDNode *Load,
3576 ISD::LoadExtType ExtTy, EVT NewVT) const {
3577 // TODO: This may be worth removing. Check regression tests for diffs.
3578 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT))
3579 return false;
3580
3581 auto *L = cast<LoadSDNode>(Load);
3582 std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3583 // Small-data object, do not shrink.
3584 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3585 return false;
3586 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) {
3587 auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine());
3588 const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3589 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM);
3590 }
3591 return true;
3592}
3593
3594Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
3595 Type *ValueTy, Value *Addr,
3596 AtomicOrdering Ord) const {
3597 BasicBlock *BB = Builder.GetInsertBlock();
3598 Module *M = BB->getParent()->getParent();
3599 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
3600 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported"
) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic loads supported\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3600, __extension__ __PRETTY_FUNCTION__))
;
3601 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3602 : Intrinsic::hexagon_L4_loadd_locked;
3603 Function *Fn = Intrinsic::getDeclaration(M, IntID);
3604
3605 auto PtrTy = cast<PointerType>(Addr->getType());
3606 PointerType *NewPtrTy =
3607 Builder.getIntNTy(SZ)->getPointerTo(PtrTy->getAddressSpace());
3608 Addr = Builder.CreateBitCast(Addr, NewPtrTy);
3609
3610 Value *Call = Builder.CreateCall(Fn, Addr, "larx");
3611
3612 return Builder.CreateBitCast(Call, ValueTy);
3613}
3614
3615/// Perform a store-conditional operation to Addr. Return the status of the
3616/// store. This should be 0 if the store succeeded, non-zero otherwise.
3617Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
3618 Value *Val, Value *Addr,
3619 AtomicOrdering Ord) const {
3620 BasicBlock *BB = Builder.GetInsertBlock();
3621 Module *M = BB->getParent()->getParent();
3622 Type *Ty = Val->getType();
3623 unsigned SZ = Ty->getPrimitiveSizeInBits();
3624
3625 Type *CastTy = Builder.getIntNTy(SZ);
3626 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported"
) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic stores supported\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp"
, 3626, __extension__ __PRETTY_FUNCTION__))
;
3627 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3628 : Intrinsic::hexagon_S4_stored_locked;
3629 Function *Fn = Intrinsic::getDeclaration(M, IntID);
3630
3631 unsigned AS = Addr->getType()->getPointerAddressSpace();
3632 Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS));
3633 Val = Builder.CreateBitCast(Val, CastTy);
3634
3635 Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx");
3636 Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");
3637 Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));
3638 return Ext;
3639}
3640
3641TargetLowering::AtomicExpansionKind
3642HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
3643 // Do not expand loads and stores that don't exceed 64 bits.
3644 return LI->getType()->getPrimitiveSizeInBits() > 64
3645 ? AtomicExpansionKind::LLOnly
3646 : AtomicExpansionKind::None;
3647}
3648
3649bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
3650 // Do not expand loads and stores that don't exceed 64 bits.
3651 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64;
3652}
3653
3654TargetLowering::AtomicExpansionKind
3655HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3656 AtomicCmpXchgInst *AI) const {
3657 return AtomicExpansionKind::LLSC;
3658}

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class MaskedLoadStoreSDNode;
513 friend class MaskedGatherScatterSDNode;
514
515 uint16_t : NumMemSDNodeBits;
516
517 // This storage is shared between disparate class hierarchies to hold an
518 // enumeration specific to the class hierarchy in use.
519 // LSBaseSDNode => enum ISD::MemIndexedMode
520 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
521 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
522 uint16_t AddressingMode : 3;
523 };
524 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
525
526 class LoadSDNodeBitfields {
527 friend class LoadSDNode;
528 friend class MaskedLoadSDNode;
529 friend class MaskedGatherSDNode;
530
531 uint16_t : NumLSBaseSDNodeBits;
532
533 uint16_t ExtTy : 2; // enum ISD::LoadExtType
534 uint16_t IsExpanding : 1;
535 };
536
537 class StoreSDNodeBitfields {
538 friend class StoreSDNode;
539 friend class MaskedStoreSDNode;
540 friend class MaskedScatterSDNode;
541
542 uint16_t : NumLSBaseSDNodeBits;
543
544 uint16_t IsTruncating : 1;
545 uint16_t IsCompressing : 1;
546 };
547
548 union {
549 char RawSDNodeBits[sizeof(uint16_t)];
550 SDNodeBitfields SDNodeBits;
551 ConstantSDNodeBitfields ConstantSDNodeBits;
552 MemSDNodeBitfields MemSDNodeBits;
553 LSBaseSDNodeBitfields LSBaseSDNodeBits;
554 LoadSDNodeBitfields LoadSDNodeBits;
555 StoreSDNodeBitfields StoreSDNodeBits;
556 };
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK
560
561 // RawSDNodeBits must cover the entirety of the union. This means that all of
562 // the union's members must have size <= RawSDNodeBits. We write the RHS as
563 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
564 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
565 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
566 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
567 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
568 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
569 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
570
571private:
572 friend class SelectionDAG;
573 // TODO: unfriend HandleSDNode once we fix its operand handling.
574 friend class HandleSDNode;
575
576 /// Unique id per SDNode in the DAG.
577 int NodeId = -1;
578
579 /// The values that are used by this operation.
580 SDUse *OperandList = nullptr;
581
582 /// The types of the values this node defines. SDNode's may
583 /// define multiple values simultaneously.
584 const EVT *ValueList;
585
586 /// List of uses for this SDNode.
587 SDUse *UseList = nullptr;
588
589 /// The number of entries in the Operand/Value list.
590 unsigned short NumOperands = 0;
591 unsigned short NumValues;
592
593 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
594 // original LLVM instructions.
595 // This is used for turning off scheduling, because we'll forgo
596 // the normal scheduling algorithms and output the instructions according to
597 // this ordering.
598 unsigned IROrder;
599
600 /// Source line information.
601 DebugLoc debugLoc;
602
603 /// Return a pointer to the specified value type.
604 static const EVT *getValueTypeList(EVT VT);
605
606 SDNodeFlags Flags;
607
608public:
609 /// Unique and persistent id per SDNode in the DAG.
610 /// Used for debug printing.
611 uint16_t PersistentId;
612
613 //===--------------------------------------------------------------------===//
614 // Accessors
615 //
616
617 /// Return the SelectionDAG opcode value for this node. For
618 /// pre-isel nodes (those for which isMachineOpcode returns false), these
619 /// are the opcode values in the ISD and <target>ISD namespaces. For
620 /// post-isel opcodes, see getMachineOpcode.
621 unsigned getOpcode() const { return (unsigned short)NodeType; }
622
623 /// Test if this node has a target-specific opcode (in the
624 /// \<target\>ISD namespace).
625 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
626
627 /// Test if this node has a target-specific opcode that may raise
628 /// FP exceptions (in the \<target\>ISD namespace and greater than
629 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
630 /// opcode are currently automatically considered to possibly raise
631 /// FP exceptions as well.
632 bool isTargetStrictFPOpcode() const {
633 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
634 }
635
636 /// Test if this node has a target-specific
637 /// memory-referencing opcode (in the \<target\>ISD namespace and
638 /// greater than FIRST_TARGET_MEMORY_OPCODE).
639 bool isTargetMemoryOpcode() const {
640 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
641 }
642
643 /// Return true if the type of the node type undefined.
644 bool isUndef() const { return NodeType == ISD::UNDEF; }
645
646 /// Test if this node is a memory intrinsic (with valid pointer information).
647 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
648 /// non-memory intrinsics (with chains) that are not really instances of
649 /// MemSDNode. For such nodes, we need some extra state to determine the
650 /// proper classof relationship.
651 bool isMemIntrinsic() const {
652 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
653 NodeType == ISD::INTRINSIC_VOID) &&
654 SDNodeBits.IsMemIntrinsic;
655 }
656
657 /// Test if this node is a strict floating point pseudo-op.
658 bool isStrictFPOpcode() {
659 switch (NodeType) {
660 default:
661 return false;
662 case ISD::STRICT_FP16_TO_FP:
663 case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
665 case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
667 return true;
668 }
669 }
670
671 /// Test if this node has a post-isel opcode, directly
672 /// corresponding to a MachineInstr opcode.
673 bool isMachineOpcode() const { return NodeType < 0; }
674
675 /// This may only be called if isMachineOpcode returns
676 /// true. It returns the MachineInstr opcode value that the node's opcode
677 /// corresponds to.
678 unsigned getMachineOpcode() const {
679 assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 679, __extension__ __PRETTY_FUNCTION__))
;
680 return ~NodeType;
681 }
682
683 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
684 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
685
686 bool isDivergent() const { return SDNodeBits.IsDivergent; }
687
688 /// Return true if there are no uses of this node.
689 bool use_empty() const { return UseList == nullptr; }
690
691 /// Return true if there is exactly one use of this node.
692 bool hasOneUse() const { return hasSingleElement(uses()); }
693
694 /// Return the number of uses of this node. This method takes
695 /// time proportional to the number of uses.
696 size_t use_size() const { return std::distance(use_begin(), use_end()); }
697
698 /// Return the unique node id.
699 int getNodeId() const { return NodeId; }
700
701 /// Set unique node id.
702 void setNodeId(int Id) { NodeId = Id; }
703
704 /// Return the node ordering.
705 unsigned getIROrder() const { return IROrder; }
706
707 /// Set the node ordering.
708 void setIROrder(unsigned Order) { IROrder = Order; }
709
710 /// Return the source location info.
711 const DebugLoc &getDebugLoc() const { return debugLoc; }
712
713 /// Set source location info. Try to avoid this, putting
714 /// it in the constructor is preferable.
715 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
716
717 /// This class provides iterator support for SDUse
718 /// operands that use a specific SDNode.
719 class use_iterator {
720 friend class SDNode;
721
722 SDUse *Op = nullptr;
723
724 explicit use_iterator(SDUse *op) : Op(op) {}
725
726 public:
727 using iterator_category = std::forward_iterator_tag;
728 using value_type = SDUse;
729 using difference_type = std::ptrdiff_t;
730 using pointer = value_type *;
731 using reference = value_type &;
732
733 use_iterator() = default;
734 use_iterator(const use_iterator &I) : Op(I.Op) {}
735
736 bool operator==(const use_iterator &x) const {
737 return Op == x.Op;
738 }
739 bool operator!=(const use_iterator &x) const {
740 return !operator==(x);
741 }
742
743 /// Return true if this iterator is at the end of uses list.
744 bool atEnd() const { return Op == nullptr; }
745
746 // Iterator traversal: forward iteration only.
747 use_iterator &operator++() { // Preincrement
748 assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 748, __extension__ __PRETTY_FUNCTION__))
;
749 Op = Op->getNext();
750 return *this;
751 }
752
753 use_iterator operator++(int) { // Postincrement
754 use_iterator tmp = *this; ++*this; return tmp;
755 }
756
757 /// Retrieve a pointer to the current user node.
758 SDNode *operator*() const {
759 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 759, __extension__ __PRETTY_FUNCTION__))
;
760 return Op->getUser();
761 }
762
763 SDNode *operator->() const { return operator*(); }
764
765 SDUse &getUse() const { return *Op; }
766
767 /// Retrieve the operand # of this use in its user.
768 unsigned getOperandNo() const {
769 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 769, __extension__ __PRETTY_FUNCTION__))
;
770 return (unsigned)(Op - Op->getUser()->OperandList);
771 }
772 };
773
774 /// Provide iteration support to walk over all uses of an SDNode.
775 use_iterator use_begin() const {
776 return use_iterator(UseList);
777 }
778
779 static use_iterator use_end() { return use_iterator(nullptr); }
780
781 inline iterator_range<use_iterator> uses() {
782 return make_range(use_begin(), use_end());
783 }
784 inline iterator_range<use_iterator> uses() const {
785 return make_range(use_begin(), use_end());
786 }
787
788 /// Return true if there are exactly NUSES uses of the indicated value.
789 /// This method ignores uses of other values defined by this operation.
790 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
791
792 /// Return true if there are any use of the indicated value.
793 /// This method ignores uses of other values defined by this operation.
794 bool hasAnyUseOfValue(unsigned Value) const;
795
796 /// Return true if this node is the only use of N.
797 bool isOnlyUserOf(const SDNode *N) const;
798
799 /// Return true if this node is an operand of N.
800 bool isOperandOf(const SDNode *N) const;
801
802 /// Return true if this node is a predecessor of N.
803 /// NOTE: Implemented on top of hasPredecessor and every bit as
804 /// expensive. Use carefully.
805 bool isPredecessorOf(const SDNode *N) const {
806 return N->hasPredecessor(this);
807 }
808
809 /// Return true if N is a predecessor of this node.
810 /// N is either an operand of this node, or can be reached by recursively
811 /// traversing up the operands.
812 /// NOTE: This is an expensive method. Use it carefully.
813 bool hasPredecessor(const SDNode *N) const;
814
815 /// Returns true if N is a predecessor of any node in Worklist. This
816 /// helper keeps Visited and Worklist sets externally to allow unions
817 /// searches to be performed in parallel, caching of results across
818 /// queries and incremental addition to Worklist. Stops early if N is
819 /// found but will resume. Remember to clear Visited and Worklists
820 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
821 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
822 /// topologically ordered (Operands have strictly smaller node id) and search
823 /// can be pruned leveraging this.
824 static bool hasPredecessorHelper(const SDNode *N,
825 SmallPtrSetImpl<const SDNode *> &Visited,
826 SmallVectorImpl<const SDNode *> &Worklist,
827 unsigned int MaxSteps = 0,
828 bool TopologicalPrune = false) {
829 SmallVector<const SDNode *, 8> DeferredNodes;
830 if (Visited.count(N))
831 return true;
832
833 // Node Id's are assigned in three places: As a topological
834 // ordering (> 0), during legalization (results in values set to
835 // 0), new nodes (set to -1). If N has a topolgical id then we
836 // know that all nodes with ids smaller than it cannot be
837 // successors and we need not check them. Filter out all node
838 // that can't be matches. We add them to the worklist before exit
839 // in case of multiple calls. Note that during selection the topological id
840 // may be violated if a node's predecessor is selected before it. We mark
841 // this at selection negating the id of unselected successors and
842 // restricting topological pruning to positive ids.
843
844 int NId = N->getNodeId();
845 // If we Invalidated the Id, reconstruct original NId.
846 if (NId < -1)
847 NId = -(NId + 1);
848
849 bool Found = false;
850 while (!Worklist.empty()) {
851 const SDNode *M = Worklist.pop_back_val();
852 int MId = M->getNodeId();
853 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
854 (MId > 0) && (MId < NId)) {
855 DeferredNodes.push_back(M);
856 continue;
857 }
858 for (const SDValue &OpV : M->op_values()) {
859 SDNode *Op = OpV.getNode();
860 if (Visited.insert(Op).second)
861 Worklist.push_back(Op);
862 if (Op == N)
863 Found = true;
864 }
865 if (Found)
866 break;
867 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
868 break;
869 }
870 // Push deferred nodes back on worklist.
871 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
872 // If we bailed early, conservatively return found.
873 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
874 return true;
875 return Found;
876 }
877
878 /// Return true if all the users of N are contained in Nodes.
879 /// NOTE: Requires at least one match, but doesn't require them all.
880 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
881
882 /// Return the number of values used by this operation.
883 unsigned getNumOperands() const { return NumOperands; }
884
885 /// Return the maximum number of operands that a SDNode can hold.
886 static constexpr size_t getMaxNumOperands() {
887 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
888 }
889
890 /// Helper method returns the integer value of a ConstantSDNode operand.
891 inline uint64_t getConstantOperandVal(unsigned Num) const;
892
893 /// Helper method returns the APInt of a ConstantSDNode operand.
894 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
895
896 const SDValue &getOperand(unsigned Num) const {
897 assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 897, __extension__ __PRETTY_FUNCTION__))
;
898 return OperandList[Num];
899 }
900
901 using op_iterator = SDUse *;
902
903 op_iterator op_begin() const { return OperandList; }
904 op_iterator op_end() const { return OperandList+NumOperands; }
905 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
906
907 /// Iterator for directly iterating over the operand SDValue's.
908 struct value_op_iterator
909 : iterator_adaptor_base<value_op_iterator, op_iterator,
910 std::random_access_iterator_tag, SDValue,
911 ptrdiff_t, value_op_iterator *,
912 value_op_iterator *> {
913 explicit value_op_iterator(SDUse *U = nullptr)
914 : iterator_adaptor_base(U) {}
915
916 const SDValue &operator*() const { return I->get(); }
917 };
918
919 iterator_range<value_op_iterator> op_values() const {
920 return make_range(value_op_iterator(op_begin()),
921 value_op_iterator(op_end()));
922 }
923
924 SDVTList getVTList() const {
925 SDVTList X = { ValueList, NumValues };
926 return X;
927 }
928
929 /// If this node has a glue operand, return the node
930 /// to which the glue operand points. Otherwise return NULL.
931 SDNode *getGluedNode() const {
932 if (getNumOperands() != 0 &&
933 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
934 return getOperand(getNumOperands()-1).getNode();
935 return nullptr;
936 }
937
938 /// If this node has a glue value with a user, return
939 /// the user (there is at most one). Otherwise return NULL.
940 SDNode *getGluedUser() const {
941 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
942 if (UI.getUse().get().getValueType() == MVT::Glue)
943 return *UI;
944 return nullptr;
945 }
946
947 SDNodeFlags getFlags() const { return Flags; }
948 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
949
950 /// Clear any flags in this node that aren't also set in Flags.
951 /// If Flags is not in a defined state then this has no effect.
952 void intersectFlagsWith(const SDNodeFlags Flags);
953
954 /// Return the number of values defined/returned by this operator.
955 unsigned getNumValues() const { return NumValues; }
956
957 /// Return the type of a specified result.
958 EVT getValueType(unsigned ResNo) const {
959 assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 959, __extension__ __PRETTY_FUNCTION__))
;
960 return ValueList[ResNo];
961 }
962
963 /// Return the type of a specified result as a simple type.
964 MVT getSimpleValueType(unsigned ResNo) const {
965 return getValueType(ResNo).getSimpleVT();
966 }
967
968 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
969 ///
970 /// If the value type is a scalable vector type, the scalable property will
971 /// be set and the runtime size will be a positive integer multiple of the
972 /// base size.
973 TypeSize getValueSizeInBits(unsigned ResNo) const {
974 return getValueType(ResNo).getSizeInBits();
975 }
976
977 using value_iterator = const EVT *;
978
979 value_iterator value_begin() const { return ValueList; }
980 value_iterator value_end() const { return ValueList+NumValues; }
981 iterator_range<value_iterator> values() const {
982 return llvm::make_range(value_begin(), value_end());
983 }
984
985 /// Return the opcode of this operation for printing.
986 std::string getOperationName(const SelectionDAG *G = nullptr) const;
987 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
988 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
989 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
990 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
991 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
992
993 /// Print a SelectionDAG node and all children down to
994 /// the leaves. The given SelectionDAG allows target-specific nodes
995 /// to be printed in human-readable form. Unlike printr, this will
996 /// print the whole DAG, including children that appear multiple
997 /// times.
998 ///
999 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and children up to
1002 /// depth "depth." The given SelectionDAG allows target-specific
1003 /// nodes to be printed in human-readable form. Unlike printr, this
1004 /// will print children that appear multiple times wherever they are
1005 /// used.
1006 ///
1007 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1008 unsigned depth = 100) const;
1009
1010 /// Dump this node, for debugging.
1011 void dump() const;
1012
1013 /// Dump (recursively) this node and its use-def subgraph.
1014 void dumpr() const;
1015
1016 /// Dump this node, for debugging.
1017 /// The given SelectionDAG allows target-specific nodes to be printed
1018 /// in human-readable form.
1019 void dump(const SelectionDAG *G) const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 /// The given SelectionDAG allows target-specific nodes to be printed
1023 /// in human-readable form.
1024 void dumpr(const SelectionDAG *G) const;
1025
1026 /// printrFull to dbgs(). The given SelectionDAG allows
1027 /// target-specific nodes to be printed in human-readable form.
1028 /// Unlike dumpr, this will print the whole DAG, including children
1029 /// that appear multiple times.
1030 void dumprFull(const SelectionDAG *G = nullptr) const;
1031
1032 /// printrWithDepth to dbgs(). The given
1033 /// SelectionDAG allows target-specific nodes to be printed in
1034 /// human-readable form. Unlike dumpr, this will print children
1035 /// that appear multiple times wherever they are used.
1036 ///
1037 void dumprWithDepth(const SelectionDAG *G = nullptr,
1038 unsigned depth = 100) const;
1039
1040 /// Gather unique data for the node.
1041 void Profile(FoldingSetNodeID &ID) const;
1042
1043 /// This method should only be used by the SDUse class.
1044 void addUse(SDUse &U) { U.addToList(&UseList); }
1045
1046protected:
1047 static SDVTList getSDVTList(EVT VT) {
1048 SDVTList Ret = { getValueTypeList(VT), 1 };
1049 return Ret;
1050 }
1051
1052 /// Create an SDNode.
1053 ///
1054 /// SDNodes are created without any operands, and never own the operand
1055 /// storage. To add operands, see SelectionDAG::createOperands.
1056 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1057 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1058 IROrder(Order), debugLoc(std::move(dl)) {
1059 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1060 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
"Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1060, __extension__ __PRETTY_FUNCTION__))
;
1061 assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
1062 "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
;
1063 }
1064
1065 /// Release the operands and set this node to have zero operands.
1066 void DropOperands();
1067};
1068
1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
1080 DebugLoc DL;
1081 int IROrder = 0;
1082
1083public:
1084 SDLoc() = default;
1085 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1086 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1087 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1088 assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1088, __extension__ __PRETTY_FUNCTION__))
;
1089 if (I)
1090 DL = I->getDebugLoc();
1091 }
1092
1093 unsigned getIROrder() const { return IROrder; }
1094 const DebugLoc &getDebugLoc() const { return DL; }
1095};
1096
1097// Define inline functions from the SDValue class.
1098
1099inline SDValue::SDValue(SDNode *node, unsigned resno)
1100 : Node(node), ResNo(resno) {
1101 // Explicitly check for !ResNo to avoid use-after-free, because there are
1102 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1103 // combines.
1104 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
1105 "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
;
1106 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1106, __extension__ __PRETTY_FUNCTION__))
;
1107}
1108
1109inline unsigned SDValue::getOpcode() const {
1110 return Node->getOpcode();
15
Called C++ object pointer is null
1111}
1112
1113inline EVT SDValue::getValueType() const {
1114 return Node->getValueType(ResNo);
1115}
1116
1117inline unsigned SDValue::getNumOperands() const {
1118 return Node->getNumOperands();
1119}
1120
1121inline const SDValue &SDValue::getOperand(unsigned i) const {
1122 return Node->getOperand(i);
1123}
1124
1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1126 return Node->getConstantOperandVal(i);
1127}
1128
1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1130 return Node->getConstantOperandAPInt(i);
1131}
1132
1133inline bool SDValue::isTargetOpcode() const {
1134 return Node->isTargetOpcode();
1135}
1136
1137inline bool SDValue::isTargetMemoryOpcode() const {
1138 return Node->isTargetMemoryOpcode();
1139}
1140
1141inline bool SDValue::isMachineOpcode() const {
1142 return Node->isMachineOpcode();
1143}
1144
1145inline unsigned SDValue::getMachineOpcode() const {
1146 return Node->getMachineOpcode();
1147}
1148
1149inline bool SDValue::isUndef() const {
1150 return Node->isUndef();
1151}
1152
1153inline bool SDValue::use_empty() const {
1154 return !Node->hasAnyUseOfValue(ResNo);
1155}
1156
1157inline bool SDValue::hasOneUse() const {
1158 return Node->hasNUsesOfValue(1, ResNo);
1159}
1160
1161inline const DebugLoc &SDValue::getDebugLoc() const {
1162 return Node->getDebugLoc();
1163}
1164
1165inline void SDValue::dump() const {
1166 return Node->dump();
1167}
1168
1169inline void SDValue::dump(const SelectionDAG *G) const {
1170 return Node->dump(G);
1171}
1172
1173inline void SDValue::dumpr() const {
1174 return Node->dumpr();
1175}
1176
1177inline void SDValue::dumpr(const SelectionDAG *G) const {
1178 return Node->dumpr(G);
1179}
1180
1181// Define inline functions from the SDUse class.
1182
1183inline void SDUse::set(const SDValue &V) {
1184 if (Val.getNode()) removeFromList();
1185 Val = V;
1186 if (V.getNode()) V.getNode()->addUse(*this);
1187}
1188
1189inline void SDUse::setInitial(const SDValue &V) {
1190 Val = V;
1191 V.getNode()->addUse(*this);
1192}
1193
1194inline void SDUse::setNode(SDNode *N) {
1195 if (Val.getNode()) removeFromList();
1196 Val.setNode(N);
1197 if (N) N->addUse(*this);
1198}
1199
1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand. This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
1205 SDUse Op;
1206
1207public:
1208 explicit HandleSDNode(SDValue X)
1209 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1210 // HandleSDNodes are never inserted into the DAG, so they won't be
1211 // auto-numbered. Use ID 65535 as a sentinel.
1212 PersistentId = 0xffff;
1213
1214 // Manually set up the operand list. This node type is special in that it's
1215 // always stack allocated and SelectionDAG does not manage its operands.
1216 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1217 // be so special.
1218 Op.setUser(this);
1219 Op.setInitial(X);
1220 NumOperands = 1;
1221 OperandList = &Op;
1222 }
1223 ~HandleSDNode();
1224
1225 const SDValue &getValue() const { return Op; }
1226};
1227
1228class AddrSpaceCastSDNode : public SDNode {
1229private:
1230 unsigned SrcAddrSpace;
1231 unsigned DestAddrSpace;
1232
1233public:
1234 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1235 unsigned SrcAS, unsigned DestAS);
1236
1237 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1238 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1239
1240 static bool classof(const SDNode *N) {
1241 return N->getOpcode() == ISD::ADDRSPACECAST;
1242 }
1243};
1244
1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
1248 // VT of in-memory value.
1249 EVT MemoryVT;
1250
1251protected:
1252 /// Memory reference information.
1253 MachineMemOperand *MMO;
1254
1255public:
1256 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1257 EVT memvt, MachineMemOperand *MMO);
1258
1259 bool readMem() const { return MMO->isLoad(); }
1260 bool writeMem() const { return MMO->isStore(); }
1261
1262 /// Returns alignment and volatility of the memory access
1263 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1264 Align getAlign() const { return MMO->getAlign(); }
1265 // FIXME: Remove once transition to getAlign is over.
1266 unsigned getAlignment() const { return MMO->getAlign().value(); }
1267
1268 /// Return the SubclassData value, without HasDebugValue. This contains an
1269 /// encoding of the volatile flag, as well as bits used by subclasses. This
1270 /// function should only be used to compute a FoldingSetNodeID value.
1271 /// The HasDebugValue bit is masked out because CSE map needs to match
1272 /// nodes with debug info with nodes without debug info. Same is about
1273 /// isDivergent bit.
1274 unsigned getRawSubclassData() const {
1275 uint16_t Data;
1276 union {
1277 char RawSDNodeBits[sizeof(uint16_t)];
1278 SDNodeBitfields SDNodeBits;
1279 };
1280 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1281 SDNodeBits.HasDebugValue = 0;
1282 SDNodeBits.IsDivergent = false;
1283 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1284 return Data;
1285 }
1286
1287 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1288 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1289 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1290 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1291
1292 // Returns the offset from the location of the access.
1293 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1294
1295 /// Returns the AA info that describes the dereference.
1296 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1297
1298 /// Returns the Ranges that describes the dereference.
1299 const MDNode *getRanges() const { return MMO->getRanges(); }
1300
1301 /// Returns the synchronization scope ID for this memory operation.
1302 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1303
1304 /// Return the atomic ordering requirements for this memory operation. For
1305 /// cmpxchg atomic operations, return the atomic ordering requirements when
1306 /// store occurs.
1307 AtomicOrdering getSuccessOrdering() const {
1308 return MMO->getSuccessOrdering();
1309 }
1310
1311 /// Return a single atomic ordering that is at least as strong as both the
1312 /// success and failure orderings for an atomic operation. (For operations
1313 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1314 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1315
1316 /// Return true if the memory operation ordering is Unordered or higher.
1317 bool isAtomic() const { return MMO->isAtomic(); }
1318
1319 /// Returns true if the memory operation doesn't imply any ordering
1320 /// constraints on surrounding memory operations beyond the normal memory
1321 /// aliasing rules.
1322 bool isUnordered() const { return MMO->isUnordered(); }
1323
1324 /// Returns true if the memory operation is neither atomic or volatile.
1325 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1326
1327 /// Return the type of the in-memory value.
1328 EVT getMemoryVT() const { return MemoryVT; }
1329
1330 /// Return a MachineMemOperand object describing the memory
1331 /// reference performed by operation.
1332 MachineMemOperand *getMemOperand() const { return MMO; }
1333
1334 const MachinePointerInfo &getPointerInfo() const {
1335 return MMO->getPointerInfo();
1336 }
1337
1338 /// Return the address space for the associated pointer
1339 unsigned getAddressSpace() const {
1340 return getPointerInfo().getAddrSpace();
1341 }
1342
1343 /// Update this MemSDNode's MachineMemOperand information
1344 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1345 /// This must only be used when the new alignment applies to all users of
1346 /// this MachineMemOperand.
1347 void refineAlignment(const MachineMemOperand *NewMMO) {
1348 MMO->refineAlignment(NewMMO);
1349 }
1350
1351 const SDValue &getChain() const { return getOperand(0); }
1352
1353 const SDValue &getBasePtr() const {
1354 switch (getOpcode()) {
1355 case ISD::STORE:
1356 case ISD::MSTORE:
1357 return getOperand(2);
1358 case ISD::MGATHER:
1359 case ISD::MSCATTER:
1360 return getOperand(3);
1361 default:
1362 return getOperand(1);
1363 }
1364 }
1365
1366 // Methods to support isa and dyn_cast
1367 static bool classof(const SDNode *N) {
1368 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1369 // with either an intrinsic or a target opcode.
1370 switch (N->getOpcode()) {
1371 case ISD::LOAD:
1372 case ISD::STORE:
1373 case ISD::PREFETCH:
1374 case ISD::ATOMIC_CMP_SWAP:
1375 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1376 case ISD::ATOMIC_SWAP:
1377 case ISD::ATOMIC_LOAD_ADD:
1378 case ISD::ATOMIC_LOAD_SUB:
1379 case ISD::ATOMIC_LOAD_AND:
1380 case ISD::ATOMIC_LOAD_CLR:
1381 case ISD::ATOMIC_LOAD_OR:
1382 case ISD::ATOMIC_LOAD_XOR:
1383 case ISD::ATOMIC_LOAD_NAND:
1384 case ISD::ATOMIC_LOAD_MIN:
1385 case ISD::ATOMIC_LOAD_MAX:
1386 case ISD::ATOMIC_LOAD_UMIN:
1387 case ISD::ATOMIC_LOAD_UMAX:
1388 case ISD::ATOMIC_LOAD_FADD:
1389 case ISD::ATOMIC_LOAD_FSUB:
1390 case ISD::ATOMIC_LOAD:
1391 case ISD::ATOMIC_STORE:
1392 case ISD::MLOAD:
1393 case ISD::MSTORE:
1394 case ISD::MGATHER:
1395 case ISD::MSCATTER:
1396 return true;
1397 default:
1398 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1399 }
1400 }
1401};
1402
1403/// This is an SDNode representing atomic operations.
1404class AtomicSDNode : public MemSDNode {
1405public:
1406 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1407 EVT MemVT, MachineMemOperand *MMO)
1408 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1409 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__))
1410 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__))
;
1411 }
1412
1413 const SDValue &getBasePtr() const { return getOperand(1); }
1414 const SDValue &getVal() const { return getOperand(2); }
1415
1416 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1417 /// otherwise.
1418 bool isCompareAndSwap() const {
1419 unsigned Op = getOpcode();
1420 return Op == ISD::ATOMIC_CMP_SWAP ||
1421 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1422 }
1423
1424 /// For cmpxchg atomic operations, return the atomic ordering requirements
1425 /// when store does not occur.
1426 AtomicOrdering getFailureOrdering() const {
1427 assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1427, __extension__ __PRETTY_FUNCTION__))
;
1428 return MMO->getFailureOrdering();
1429 }
1430
1431 // Methods to support isa and dyn_cast
1432 static bool classof(const SDNode *N) {
1433 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1434 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1435 N->getOpcode() == ISD::ATOMIC_SWAP ||
1436 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1437 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1438 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1439 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1440 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1441 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1442 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1443 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1444 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1445 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1446 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1447 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1448 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1449 N->getOpcode() == ISD::ATOMIC_LOAD ||
1450 N->getOpcode() == ISD::ATOMIC_STORE;
1451 }
1452};
1453
1454/// This SDNode is used for target intrinsics that touch
1455/// memory and need an associated MachineMemOperand. Its opcode may be
1456/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1457/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1458class MemIntrinsicSDNode : public MemSDNode {
1459public:
1460 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1461 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1462 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1463 SDNodeBits.IsMemIntrinsic = true;
1464 }
1465
1466 // Methods to support isa and dyn_cast
1467 static bool classof(const SDNode *N) {
1468 // We lower some target intrinsics to their target opcode
1469 // early a node with a target opcode can be of this class
1470 return N->isMemIntrinsic() ||
1471 N->getOpcode() == ISD::PREFETCH ||
1472 N->isTargetMemoryOpcode();
1473 }
1474};
1475
1476/// This SDNode is used to implement the code generator
1477/// support for the llvm IR shufflevector instruction. It combines elements
1478/// from two input vectors into a new input vector, with the selection and
1479/// ordering of elements determined by an array of integers, referred to as
1480/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1481/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1482/// An index of -1 is treated as undef, such that the code generator may put
1483/// any value in the corresponding element of the result.
1484class ShuffleVectorSDNode : public SDNode {
1485 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1486 // is freed when the SelectionDAG object is destroyed.
1487 const int *Mask;
1488
1489protected:
1490 friend class SelectionDAG;
1491
1492 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1493 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1494
1495public:
1496 ArrayRef<int> getMask() const {
1497 EVT VT = getValueType(0);
1498 return makeArrayRef(Mask, VT.getVectorNumElements());
1499 }
1500
1501 int getMaskElt(unsigned Idx) const {
1502 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1502, __extension__ __PRETTY_FUNCTION__))
;
1503 return Mask[Idx];
1504 }
1505
1506 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1507
1508 int getSplatIndex() const {
1509 assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1509, __extension__ __PRETTY_FUNCTION__))
;
1510 EVT VT = getValueType(0);
1511 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1512 if (Mask[i] >= 0)
1513 return Mask[i];
1514
1515 // We can choose any index value here and be correct because all elements
1516 // are undefined. Return 0 for better potential for callers to simplify.
1517 return 0;
1518 }
1519
1520 static bool isSplatMask(const int *Mask, EVT VT);
1521
1522 /// Change values in a shuffle permute mask assuming
1523 /// the two vector operands have swapped position.
1524 static void commuteMask(MutableArrayRef<int> Mask) {
1525 unsigned NumElems = Mask.size();
1526 for (unsigned i = 0; i != NumElems; ++i) {
1527 int idx = Mask[i];
1528 if (idx < 0)
1529 continue;
1530 else if (idx < (int)NumElems)
1531 Mask[i] = idx + NumElems;
1532 else
1533 Mask[i] = idx - NumElems;
1534 }
1535 }
1536
1537 static bool classof(const SDNode *N) {
1538 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1539 }
1540};
1541
1542class ConstantSDNode : public SDNode {
1543 friend class SelectionDAG;
1544
1545 const ConstantInt *Value;
1546
1547 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1548 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1549 getSDVTList(VT)),
1550 Value(val) {
1551 ConstantSDNodeBits.IsOpaque = isOpaque;
1552 }
1553
1554public:
1555 const ConstantInt *getConstantIntValue() const { return Value; }
1556 const APInt &getAPIntValue() const { return Value->getValue(); }
1557 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1558 int64_t getSExtValue() const { return Value->getSExtValue(); }
1559 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1560 return Value->getLimitedValue(Limit);
1561 }
1562 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1563 Align getAlignValue() const { return Value->getAlignValue(); }
1564
1565 bool isOne() const { return Value->isOne(); }
1566 bool isNullValue() const { return Value->isZero(); }
1567 bool isAllOnesValue() const { return Value->isMinusOne(); }
1568 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1569 bool isMinSignedValue() const { return Value->isMinValue(true); }
1570
1571 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1572
1573 static bool classof(const SDNode *N) {
1574 return N->getOpcode() == ISD::Constant ||
1575 N->getOpcode() == ISD::TargetConstant;
1576 }
1577};
1578
1579uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1580 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1581}
1582
1583const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1584 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1585}
1586
1587class ConstantFPSDNode : public SDNode {
1588 friend class SelectionDAG;
1589
1590 const ConstantFP *Value;
1591
1592 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1593 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1594 DebugLoc(), getSDVTList(VT)),
1595 Value(val) {}
1596
1597public:
1598 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1599 const ConstantFP *getConstantFPValue() const { return Value; }
1600
1601 /// Return true if the value is positive or negative zero.
1602 bool isZero() const { return Value->isZero(); }
1603
1604 /// Return true if the value is a NaN.
1605 bool isNaN() const { return Value->isNaN(); }
1606
1607 /// Return true if the value is an infinity
1608 bool isInfinity() const { return Value->isInfinity(); }
1609
1610 /// Return true if the value is negative.
1611 bool isNegative() const { return Value->isNegative(); }
1612
1613 /// We don't rely on operator== working on double values, as
1614 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1615 /// As such, this method can be used to do an exact bit-for-bit comparison of
1616 /// two floating point values.
1617
1618 /// We leave the version with the double argument here because it's just so
1619 /// convenient to write "2.0" and the like. Without this function we'd
1620 /// have to duplicate its logic everywhere it's called.
1621 bool isExactlyValue(double V) const {
1622 return Value->getValueAPF().isExactlyValue(V);
1623 }
1624 bool isExactlyValue(const APFloat& V) const;
1625
1626 static bool isValueValidForType(EVT VT, const APFloat& Val);
1627
1628 static bool classof(const SDNode *N) {
1629 return N->getOpcode() == ISD::ConstantFP ||
1630 N->getOpcode() == ISD::TargetConstantFP;
1631 }
1632};
1633
1634/// Returns true if \p V is a constant integer zero.
1635bool isNullConstant(SDValue V);
1636
1637/// Returns true if \p V is an FP constant with a value of positive zero.
1638bool isNullFPConstant(SDValue V);
1639
1640/// Returns true if \p V is an integer constant with all bits set.
1641bool isAllOnesConstant(SDValue V);
1642
1643/// Returns true if \p V is a constant integer one.
1644bool isOneConstant(SDValue V);
1645
1646/// Return the non-bitcasted source operand of \p V if it exists.
1647/// If \p V is not a bitcasted value, it is returned as-is.
1648SDValue peekThroughBitcasts(SDValue V);
1649
1650/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1651/// If \p V is not a bitcasted one-use value, it is returned as-is.
1652SDValue peekThroughOneUseBitcasts(SDValue V);
1653
1654/// Return the non-extracted vector source operand of \p V if it exists.
1655/// If \p V is not an extracted subvector, it is returned as-is.
1656SDValue peekThroughExtractSubvectors(SDValue V);
1657
1658/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1659/// constant is canonicalized to be operand 1.
1660bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1661
1662/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1663ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1664 bool AllowTruncation = false);
1665
1666/// Returns the SDNode if it is a demanded constant splat BuildVector or
1667/// constant int.
1668ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1669 bool AllowUndefs = false,
1670 bool AllowTruncation = false);
1671
1672/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1673ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1674
1675/// Returns the SDNode if it is a demanded constant splat BuildVector or
1676/// constant float.
1677ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1678 bool AllowUndefs = false);
1679
1680/// Return true if the value is a constant 0 integer or a splatted vector of
1681/// a constant 0 integer (with no undefs by default).
1682/// Build vector implicit truncation is not an issue for null values.
1683bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1684
1685/// Return true if the value is a constant 1 integer or a splatted vector of a
1686/// constant 1 integer (with no undefs).
1687/// Does not permit build vector implicit truncation.
1688bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1689
1690/// Return true if the value is a constant -1 integer or a splatted vector of a
1691/// constant -1 integer (with no undefs).
1692/// Does not permit build vector implicit truncation.
1693bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1694
1695/// Return true if \p V is either a integer or FP constant.
1696inline bool isIntOrFPConstant(SDValue V) {
1697 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1698}
1699
1700class GlobalAddressSDNode : public SDNode {
1701 friend class SelectionDAG;
1702
1703 const GlobalValue *TheGlobal;
1704 int64_t Offset;
1705 unsigned TargetFlags;
1706
1707 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1708 const GlobalValue *GA, EVT VT, int64_t o,
1709 unsigned TF);
1710
1711public:
1712 const GlobalValue *getGlobal() const { return TheGlobal; }
1713 int64_t getOffset() const { return Offset; }
1714 unsigned getTargetFlags() const { return TargetFlags; }
1715 // Return the address space this GlobalAddress belongs to.
1716 unsigned getAddressSpace() const;
1717
1718 static bool classof(const SDNode *N) {
1719 return N->getOpcode() == ISD::GlobalAddress ||
1720 N->getOpcode() == ISD::TargetGlobalAddress ||
1721 N->getOpcode() == ISD::GlobalTLSAddress ||
1722 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1723 }
1724};
1725
1726class FrameIndexSDNode : public SDNode {
1727 friend class SelectionDAG;
1728
1729 int FI;
1730
1731 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1732 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1733 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1734 }
1735
1736public:
1737 int getIndex() const { return FI; }
1738
1739 static bool classof(const SDNode *N) {
1740 return N->getOpcode() == ISD::FrameIndex ||
1741 N->getOpcode() == ISD::TargetFrameIndex;
1742 }
1743};
1744
1745/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1746/// the offet and size that are started/ended in the underlying FrameIndex.
1747class LifetimeSDNode : public SDNode {
1748 friend class SelectionDAG;
1749 int64_t Size;
1750 int64_t Offset; // -1 if offset is unknown.
1751
1752 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1753 SDVTList VTs, int64_t Size, int64_t Offset)
1754 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1755public:
1756 int64_t getFrameIndex() const {
1757 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1758 }
1759
1760 bool hasOffset() const { return Offset >= 0; }
1761 int64_t getOffset() const {
1762 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1762, __extension__ __PRETTY_FUNCTION__))
;
1763 return Offset;
1764 }
1765 int64_t getSize() const {
1766 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1766, __extension__ __PRETTY_FUNCTION__))
;
1767 return Size;
1768 }
1769
1770 // Methods to support isa and dyn_cast
1771 static bool classof(const SDNode *N) {
1772 return N->getOpcode() == ISD::LIFETIME_START ||
1773 N->getOpcode() == ISD::LIFETIME_END;
1774 }
1775};
1776
1777/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1778/// the index of the basic block being probed. A pseudo probe serves as a place
1779/// holder and will be removed at the end of compilation. It does not have any
1780/// operand because we do not want the instruction selection to deal with any.
1781class PseudoProbeSDNode : public SDNode {
1782 friend class SelectionDAG;
1783 uint64_t Guid;
1784 uint64_t Index;
1785 uint32_t Attributes;
1786
1787 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1788 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1789 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1790 Attributes(Attr) {}
1791
1792public:
1793 uint64_t getGuid() const { return Guid; }
1794 uint64_t getIndex() const { return Index; }
1795 uint32_t getAttributes() const { return Attributes; }
1796
1797 // Methods to support isa and dyn_cast
1798 static bool classof(const SDNode *N) {
1799 return N->getOpcode() == ISD::PSEUDO_PROBE;
1800 }
1801};
1802
1803class JumpTableSDNode : public SDNode {
1804 friend class SelectionDAG;
1805
1806 int JTI;
1807 unsigned TargetFlags;
1808
1809 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1810 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1811 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1812 }
1813
1814public:
1815 int getIndex() const { return JTI; }
1816 unsigned getTargetFlags() const { return TargetFlags; }
1817
1818 static bool classof(const SDNode *N) {
1819 return N->getOpcode() == ISD::JumpTable ||
1820 N->getOpcode() == ISD::TargetJumpTable;
1821 }
1822};
1823
1824class ConstantPoolSDNode : public SDNode {
1825 friend class SelectionDAG;
1826
1827 union {
1828 const Constant *ConstVal;
1829 MachineConstantPoolValue *MachineCPVal;
1830 } Val;
1831 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1832 Align Alignment; // Minimum alignment requirement of CP.
1833 unsigned TargetFlags;
1834
1835 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1836 Align Alignment, unsigned TF)
1837 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1838 DebugLoc(), getSDVTList(VT)),
1839 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1840 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1840, __extension__ __PRETTY_FUNCTION__))
;
1841 Val.ConstVal = c;
1842 }
1843
1844 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1845 Align Alignment, unsigned TF)
1846 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1847 DebugLoc(), getSDVTList(VT)),
1848 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1849 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1849, __extension__ __PRETTY_FUNCTION__))
;
1850 Val.MachineCPVal = v;
1851 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1852 }
1853
1854public:
1855 bool isMachineConstantPoolEntry() const {
1856 return Offset < 0;
1857 }
1858
1859 const Constant *getConstVal() const {
1860 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1860, __extension__ __PRETTY_FUNCTION__))
;
1861 return Val.ConstVal;
1862 }
1863
1864 MachineConstantPoolValue *getMachineCPVal() const {
1865 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1865, __extension__ __PRETTY_FUNCTION__))
;
1866 return Val.MachineCPVal;
1867 }
1868
1869 int getOffset() const {
1870 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1871 }
1872
1873 // Return the alignment of this constant pool object, which is either 0 (for
1874 // default alignment) or the desired value.
1875 Align getAlign() const { return Alignment; }
1876 unsigned getTargetFlags() const { return TargetFlags; }
1877
1878 Type *getType() const;
1879
1880 static bool classof(const SDNode *N) {
1881 return N->getOpcode() == ISD::ConstantPool ||
1882 N->getOpcode() == ISD::TargetConstantPool;
1883 }
1884};
1885
1886/// Completely target-dependent object reference.
1887class TargetIndexSDNode : public SDNode {
1888 friend class SelectionDAG;
1889
1890 unsigned TargetFlags;
1891 int Index;
1892 int64_t Offset;
1893
1894public:
1895 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1896 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1897 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1898
1899 unsigned getTargetFlags() const { return TargetFlags; }
1900 int getIndex() const { return Index; }
1901 int64_t getOffset() const { return Offset; }
1902
1903 static bool classof(const SDNode *N) {
1904 return N->getOpcode() == ISD::TargetIndex;
1905 }
1906};
1907
1908class BasicBlockSDNode : public SDNode {
1909 friend class SelectionDAG;
1910
1911 MachineBasicBlock *MBB;
1912
1913 /// Debug info is meaningful and potentially useful here, but we create
1914 /// blocks out of order when they're jumped to, which makes it a bit
1915 /// harder. Let's see if we need it first.
1916 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1917 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1918 {}
1919
1920public:
1921 MachineBasicBlock *getBasicBlock() const { return MBB; }
1922
1923 static bool classof(const SDNode *N) {
1924 return N->getOpcode() == ISD::BasicBlock;
1925 }
1926};
1927
1928/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1929class BuildVectorSDNode : public SDNode {
1930public:
1931 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1932 explicit BuildVectorSDNode() = delete;
1933
1934 /// Check if this is a constant splat, and if so, find the
1935 /// smallest element size that splats the vector. If MinSplatBits is
1936 /// nonzero, the element size must be at least that large. Note that the
1937 /// splat element may be the entire vector (i.e., a one element vector).
1938 /// Returns the splat element value in SplatValue. Any undefined bits in
1939 /// that value are zero, and the corresponding bits in the SplatUndef mask
1940 /// are set. The SplatBitSize value is set to the splat element size in
1941 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1942 /// undefined. isBigEndian describes the endianness of the target.
1943 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1944 unsigned &SplatBitSize, bool &HasAnyUndefs,
1945 unsigned MinSplatBits = 0,
1946 bool isBigEndian = false) const;
1947
1948 /// Returns the demanded splatted value or a null value if this is not a
1949 /// splat.
1950 ///
1951 /// The DemandedElts mask indicates the elements that must be in the splat.
1952 /// If passed a non-null UndefElements bitvector, it will resize it to match
1953 /// the vector width and set the bits where elements are undef.
1954 SDValue getSplatValue(const APInt &DemandedElts,
1955 BitVector *UndefElements = nullptr) const;
1956
1957 /// Returns the splatted value or a null value if this is not a splat.
1958 ///
1959 /// If passed a non-null UndefElements bitvector, it will resize it to match
1960 /// the vector width and set the bits where elements are undef.
1961 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1962
1963 /// Find the shortest repeating sequence of values in the build vector.
1964 ///
1965 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1966 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1967 ///
1968 /// Currently this must be a power-of-2 build vector.
1969 /// The DemandedElts mask indicates the elements that must be present,
1970 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1971 /// non-null UndefElements bitvector, it will resize it to match the original
1972 /// vector width and set the bits where elements are undef. If result is
1973 /// false, Sequence will be empty.
1974 bool getRepeatedSequence(const APInt &DemandedElts,
1975 SmallVectorImpl<SDValue> &Sequence,
1976 BitVector *UndefElements = nullptr) const;
1977
1978 /// Find the shortest repeating sequence of values in the build vector.
1979 ///
1980 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1981 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1982 ///
1983 /// Currently this must be a power-of-2 build vector.
1984 /// If passed a non-null UndefElements bitvector, it will resize it to match
1985 /// the original vector width and set the bits where elements are undef.
1986 /// If result is false, Sequence will be empty.
1987 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
1988 BitVector *UndefElements = nullptr) const;
1989
1990 /// Returns the demanded splatted constant or null if this is not a constant
1991 /// splat.
1992 ///
1993 /// The DemandedElts mask indicates the elements that must be in the splat.
1994 /// If passed a non-null UndefElements bitvector, it will resize it to match
1995 /// the vector width and set the bits where elements are undef.
1996 ConstantSDNode *
1997 getConstantSplatNode(const APInt &DemandedElts,
1998 BitVector *UndefElements = nullptr) const;
1999
2000 /// Returns the splatted constant or null if this is not a constant
2001 /// splat.
2002 ///
2003 /// If passed a non-null UndefElements bitvector, it will resize it to match
2004 /// the vector width and set the bits where elements are undef.
2005 ConstantSDNode *
2006 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2007
2008 /// Returns the demanded splatted constant FP or null if this is not a
2009 /// constant FP splat.
2010 ///
2011 /// The DemandedElts mask indicates the elements that must be in the splat.
2012 /// If passed a non-null UndefElements bitvector, it will resize it to match
2013 /// the vector width and set the bits where elements are undef.
2014 ConstantFPSDNode *
2015 getConstantFPSplatNode(const APInt &DemandedElts,
2016 BitVector *UndefElements = nullptr) const;
2017
2018 /// Returns the splatted constant FP or null if this is not a constant
2019 /// FP splat.
2020 ///
2021 /// If passed a non-null UndefElements bitvector, it will resize it to match
2022 /// the vector width and set the bits where elements are undef.
2023 ConstantFPSDNode *
2024 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2025
2026 /// If this is a constant FP splat and the splatted constant FP is an
2027 /// exact power or 2, return the log base 2 integer value. Otherwise,
2028 /// return -1.
2029 ///
2030 /// The BitWidth specifies the necessary bit precision.
2031 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2032 uint32_t BitWidth) const;
2033
2034 bool isConstant() const;
2035
2036 static bool classof(const SDNode *N) {
2037 return N->getOpcode() == ISD::BUILD_VECTOR;
2038 }
2039};
2040
2041/// An SDNode that holds an arbitrary LLVM IR Value. This is
2042/// used when the SelectionDAG needs to make a simple reference to something
2043/// in the LLVM IR representation.
2044///
2045class SrcValueSDNode : public SDNode {
2046 friend class SelectionDAG;
2047
2048 const Value *V;
2049
2050 /// Create a SrcValue for a general value.
2051 explicit SrcValueSDNode(const Value *v)
2052 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2053
2054public:
2055 /// Return the contained Value.
2056 const Value *getValue() const { return V; }
2057
2058 static bool classof(const SDNode *N) {
2059 return N->getOpcode() == ISD::SRCVALUE;
2060 }
2061};
2062
2063class MDNodeSDNode : public SDNode {
2064 friend class SelectionDAG;
2065
2066 const MDNode *MD;
2067
2068 explicit MDNodeSDNode(const MDNode *md)
2069 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2070 {}
2071
2072public:
2073 const MDNode *getMD() const { return MD; }
2074
2075 static bool classof(const SDNode *N) {
2076 return N->getOpcode() == ISD::MDNODE_SDNODE;
2077 }
2078};
2079
2080class RegisterSDNode : public SDNode {
2081 friend class SelectionDAG;
2082
2083 Register Reg;
2084
2085 RegisterSDNode(Register reg, EVT VT)
2086 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2087
2088public:
2089 Register getReg() const { return Reg; }
2090
2091 static bool classof(const SDNode *N) {
2092 return N->getOpcode() == ISD::Register;
2093 }
2094};
2095
2096class RegisterMaskSDNode : public SDNode {
2097 friend class SelectionDAG;
2098
2099 // The memory for RegMask is not owned by the node.
2100 const uint32_t *RegMask;
2101
2102 RegisterMaskSDNode(const uint32_t *mask)
2103 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2104 RegMask(mask) {}
2105
2106public:
2107 const uint32_t *getRegMask() const { return RegMask; }
2108
2109 static bool classof(const SDNode *N) {
2110 return N->getOpcode() == ISD::RegisterMask;
2111 }
2112};
2113
2114class BlockAddressSDNode : public SDNode {
2115 friend class SelectionDAG;
2116
2117 const BlockAddress *BA;
2118 int64_t Offset;
2119 unsigned TargetFlags;
2120
2121 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2122 int64_t o, unsigned Flags)
2123 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2124 BA(ba), Offset(o), TargetFlags(Flags) {}
2125
2126public:
2127 const BlockAddress *getBlockAddress() const { return BA; }
2128 int64_t getOffset() const { return Offset; }
2129 unsigned getTargetFlags() const { return TargetFlags; }
2130
2131 static bool classof(const SDNode *N) {
2132 return N->getOpcode() == ISD::BlockAddress ||
2133 N->getOpcode() == ISD::TargetBlockAddress;
2134 }
2135};
2136
2137class LabelSDNode : public SDNode {
2138 friend class SelectionDAG;
2139
2140 MCSymbol *Label;
2141
2142 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2143 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2144 assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
"not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2144, __extension__ __PRETTY_FUNCTION__))
;
2145 }
2146
2147public:
2148 MCSymbol *getLabel() const { return Label; }
2149
2150 static bool classof(const SDNode *N) {
2151 return N->getOpcode() == ISD::EH_LABEL ||
2152 N->getOpcode() == ISD::ANNOTATION_LABEL;
2153 }
2154};
2155
2156class ExternalSymbolSDNode : public SDNode {
2157 friend class SelectionDAG;
2158
2159 const char *Symbol;
2160 unsigned TargetFlags;
2161
2162 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)