| File: | llvm/include/llvm/CodeGen/SelectionDAGNodes.h |
| Warning: | line 1110, column 10 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===// | ||||
| 2 | // | ||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
| 6 | // | ||||
| 7 | //===----------------------------------------------------------------------===// | ||||
| 8 | // | ||||
| 9 | // This file implements the interfaces that Hexagon uses to lower LLVM code | ||||
| 10 | // into a selection DAG. | ||||
| 11 | // | ||||
| 12 | //===----------------------------------------------------------------------===// | ||||
| 13 | |||||
| 14 | #include "HexagonISelLowering.h" | ||||
| 15 | #include "Hexagon.h" | ||||
| 16 | #include "HexagonMachineFunctionInfo.h" | ||||
| 17 | #include "HexagonRegisterInfo.h" | ||||
| 18 | #include "HexagonSubtarget.h" | ||||
| 19 | #include "HexagonTargetMachine.h" | ||||
| 20 | #include "HexagonTargetObjectFile.h" | ||||
| 21 | #include "llvm/ADT/APInt.h" | ||||
| 22 | #include "llvm/ADT/ArrayRef.h" | ||||
| 23 | #include "llvm/ADT/SmallVector.h" | ||||
| 24 | #include "llvm/ADT/StringSwitch.h" | ||||
| 25 | #include "llvm/CodeGen/CallingConvLower.h" | ||||
| 26 | #include "llvm/CodeGen/MachineFrameInfo.h" | ||||
| 27 | #include "llvm/CodeGen/MachineFunction.h" | ||||
| 28 | #include "llvm/CodeGen/MachineMemOperand.h" | ||||
| 29 | #include "llvm/CodeGen/MachineRegisterInfo.h" | ||||
| 30 | #include "llvm/CodeGen/RuntimeLibcalls.h" | ||||
| 31 | #include "llvm/CodeGen/SelectionDAG.h" | ||||
| 32 | #include "llvm/CodeGen/TargetCallingConv.h" | ||||
| 33 | #include "llvm/CodeGen/ValueTypes.h" | ||||
| 34 | #include "llvm/IR/BasicBlock.h" | ||||
| 35 | #include "llvm/IR/CallingConv.h" | ||||
| 36 | #include "llvm/IR/DataLayout.h" | ||||
| 37 | #include "llvm/IR/DerivedTypes.h" | ||||
| 38 | #include "llvm/IR/DiagnosticInfo.h" | ||||
| 39 | #include "llvm/IR/DiagnosticPrinter.h" | ||||
| 40 | #include "llvm/IR/Function.h" | ||||
| 41 | #include "llvm/IR/GlobalValue.h" | ||||
| 42 | #include "llvm/IR/InlineAsm.h" | ||||
| 43 | #include "llvm/IR/Instructions.h" | ||||
| 44 | #include "llvm/IR/IntrinsicInst.h" | ||||
| 45 | #include "llvm/IR/Intrinsics.h" | ||||
| 46 | #include "llvm/IR/IntrinsicsHexagon.h" | ||||
| 47 | #include "llvm/IR/IRBuilder.h" | ||||
| 48 | #include "llvm/IR/Module.h" | ||||
| 49 | #include "llvm/IR/Type.h" | ||||
| 50 | #include "llvm/IR/Value.h" | ||||
| 51 | #include "llvm/MC/MCRegisterInfo.h" | ||||
| 52 | #include "llvm/Support/Casting.h" | ||||
| 53 | #include "llvm/Support/CodeGen.h" | ||||
| 54 | #include "llvm/Support/CommandLine.h" | ||||
| 55 | #include "llvm/Support/Debug.h" | ||||
| 56 | #include "llvm/Support/ErrorHandling.h" | ||||
| 57 | #include "llvm/Support/MathExtras.h" | ||||
| 58 | #include "llvm/Support/raw_ostream.h" | ||||
| 59 | #include "llvm/Target/TargetMachine.h" | ||||
| 60 | #include <algorithm> | ||||
| 61 | #include <cassert> | ||||
| 62 | #include <cstddef> | ||||
| 63 | #include <cstdint> | ||||
| 64 | #include <limits> | ||||
| 65 | #include <utility> | ||||
| 66 | |||||
| 67 | using namespace llvm; | ||||
| 68 | |||||
| 69 | #define DEBUG_TYPE"hexagon-lowering" "hexagon-lowering" | ||||
| 70 | |||||
| 71 | static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables", | ||||
| 72 | cl::init(true), cl::Hidden, | ||||
| 73 | cl::desc("Control jump table emission on Hexagon target")); | ||||
| 74 | |||||
| 75 | static cl::opt<bool> EnableHexSDNodeSched("enable-hexagon-sdnode-sched", | ||||
| 76 | cl::Hidden, cl::ZeroOrMore, cl::init(false), | ||||
| 77 | cl::desc("Enable Hexagon SDNode scheduling")); | ||||
| 78 | |||||
| 79 | static cl::opt<bool> EnableFastMath("ffast-math", | ||||
| 80 | cl::Hidden, cl::ZeroOrMore, cl::init(false), | ||||
| 81 | cl::desc("Enable Fast Math processing")); | ||||
| 82 | |||||
| 83 | static cl::opt<int> MinimumJumpTables("minimum-jump-tables", | ||||
| 84 | cl::Hidden, cl::ZeroOrMore, cl::init(5), | ||||
| 85 | cl::desc("Set minimum jump tables")); | ||||
| 86 | |||||
| 87 | static cl::opt<int> MaxStoresPerMemcpyCL("max-store-memcpy", | ||||
| 88 | cl::Hidden, cl::ZeroOrMore, cl::init(6), | ||||
| 89 | cl::desc("Max #stores to inline memcpy")); | ||||
| 90 | |||||
| 91 | static cl::opt<int> MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os", | ||||
| 92 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | ||||
| 93 | cl::desc("Max #stores to inline memcpy")); | ||||
| 94 | |||||
| 95 | static cl::opt<int> MaxStoresPerMemmoveCL("max-store-memmove", | ||||
| 96 | cl::Hidden, cl::ZeroOrMore, cl::init(6), | ||||
| 97 | cl::desc("Max #stores to inline memmove")); | ||||
| 98 | |||||
| 99 | static cl::opt<int> MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os", | ||||
| 100 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | ||||
| 101 | cl::desc("Max #stores to inline memmove")); | ||||
| 102 | |||||
| 103 | static cl::opt<int> MaxStoresPerMemsetCL("max-store-memset", | ||||
| 104 | cl::Hidden, cl::ZeroOrMore, cl::init(8), | ||||
| 105 | cl::desc("Max #stores to inline memset")); | ||||
| 106 | |||||
| 107 | static cl::opt<int> MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", | ||||
| 108 | cl::Hidden, cl::ZeroOrMore, cl::init(4), | ||||
| 109 | cl::desc("Max #stores to inline memset")); | ||||
| 110 | |||||
| 111 | static cl::opt<bool> AlignLoads("hexagon-align-loads", | ||||
| 112 | cl::Hidden, cl::init(false), | ||||
| 113 | cl::desc("Rewrite unaligned loads as a pair of aligned loads")); | ||||
| 114 | |||||
| 115 | static cl::opt<bool> | ||||
| 116 | DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden, | ||||
| 117 | cl::init(false), | ||||
| 118 | cl::desc("Disable minimum alignment of 1 for " | ||||
| 119 | "arguments passed by value on stack")); | ||||
| 120 | |||||
| 121 | namespace { | ||||
| 122 | |||||
| 123 | class HexagonCCState : public CCState { | ||||
| 124 | unsigned NumNamedVarArgParams = 0; | ||||
| 125 | |||||
| 126 | public: | ||||
| 127 | HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF, | ||||
| 128 | SmallVectorImpl<CCValAssign> &locs, LLVMContext &C, | ||||
| 129 | unsigned NumNamedArgs) | ||||
| 130 | : CCState(CC, IsVarArg, MF, locs, C), | ||||
| 131 | NumNamedVarArgParams(NumNamedArgs) {} | ||||
| 132 | unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; } | ||||
| 133 | }; | ||||
| 134 | |||||
| 135 | } // end anonymous namespace | ||||
| 136 | |||||
| 137 | |||||
| 138 | // Implement calling convention for Hexagon. | ||||
| 139 | |||||
| 140 | static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT, | ||||
| 141 | CCValAssign::LocInfo &LocInfo, | ||||
| 142 | ISD::ArgFlagsTy &ArgFlags, CCState &State) { | ||||
| 143 | static const MCPhysReg ArgRegs[] = { | ||||
| 144 | Hexagon::R0, Hexagon::R1, Hexagon::R2, | ||||
| 145 | Hexagon::R3, Hexagon::R4, Hexagon::R5 | ||||
| 146 | }; | ||||
| 147 | const unsigned NumArgRegs = array_lengthof(ArgRegs); | ||||
| 148 | unsigned RegNum = State.getFirstUnallocated(ArgRegs); | ||||
| 149 | |||||
| 150 | // RegNum is an index into ArgRegs: skip a register if RegNum is odd. | ||||
| 151 | if (RegNum != NumArgRegs && RegNum % 2 == 1) | ||||
| 152 | State.AllocateReg(ArgRegs[RegNum]); | ||||
| 153 | |||||
| 154 | // Always return false here, as this function only makes sure that the first | ||||
| 155 | // unallocated register has an even register number and does not actually | ||||
| 156 | // allocate a register for the current argument. | ||||
| 157 | return false; | ||||
| 158 | } | ||||
| 159 | |||||
| 160 | #include "HexagonGenCallingConv.inc" | ||||
| 161 | |||||
| 162 | |||||
| 163 | SDValue | ||||
| 164 | HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) | ||||
| 165 | const { | ||||
| 166 | return SDValue(); | ||||
| 167 | } | ||||
| 168 | |||||
| 169 | /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified | ||||
| 170 | /// by "Src" to address "Dst" of size "Size". Alignment information is | ||||
| 171 | /// specified by the specific parameter attribute. The copy will be passed as | ||||
| 172 | /// a byval function parameter. Sometimes what we are copying is the end of a | ||||
| 173 | /// larger object, the part that does not fit in registers. | ||||
| 174 | static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, | ||||
| 175 | SDValue Chain, ISD::ArgFlagsTy Flags, | ||||
| 176 | SelectionDAG &DAG, const SDLoc &dl) { | ||||
| 177 | SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); | ||||
| 178 | return DAG.getMemcpy( | ||||
| 179 | Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(), | ||||
| 180 | /*isVolatile=*/false, /*AlwaysInline=*/false, | ||||
| 181 | /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo()); | ||||
| 182 | } | ||||
| 183 | |||||
| 184 | bool | ||||
| 185 | HexagonTargetLowering::CanLowerReturn( | ||||
| 186 | CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, | ||||
| 187 | const SmallVectorImpl<ISD::OutputArg> &Outs, | ||||
| 188 | LLVMContext &Context) const { | ||||
| 189 | SmallVector<CCValAssign, 16> RVLocs; | ||||
| 190 | CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); | ||||
| 191 | |||||
| 192 | if (MF.getSubtarget<HexagonSubtarget>().useHVXOps()) | ||||
| 193 | return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX); | ||||
| 194 | return CCInfo.CheckReturn(Outs, RetCC_Hexagon); | ||||
| 195 | } | ||||
| 196 | |||||
| 197 | // LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is | ||||
| 198 | // passed by value, the function prototype is modified to return void and | ||||
| 199 | // the value is stored in memory pointed by a pointer passed by caller. | ||||
| 200 | SDValue | ||||
| 201 | HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, | ||||
| 202 | bool IsVarArg, | ||||
| 203 | const SmallVectorImpl<ISD::OutputArg> &Outs, | ||||
| 204 | const SmallVectorImpl<SDValue> &OutVals, | ||||
| 205 | const SDLoc &dl, SelectionDAG &DAG) const { | ||||
| 206 | // CCValAssign - represent the assignment of the return value to locations. | ||||
| 207 | SmallVector<CCValAssign, 16> RVLocs; | ||||
| 208 | |||||
| 209 | // CCState - Info about the registers and stack slot. | ||||
| 210 | CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, | ||||
| 211 | *DAG.getContext()); | ||||
| 212 | |||||
| 213 | // Analyze return values of ISD::RET | ||||
| 214 | if (Subtarget.useHVXOps()) | ||||
| 215 | CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX); | ||||
| 216 | else | ||||
| 217 | CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon); | ||||
| 218 | |||||
| 219 | SDValue Flag; | ||||
| 220 | SmallVector<SDValue, 4> RetOps(1, Chain); | ||||
| 221 | |||||
| 222 | // Copy the result values into the output registers. | ||||
| 223 | for (unsigned i = 0; i != RVLocs.size(); ++i) { | ||||
| 224 | CCValAssign &VA = RVLocs[i]; | ||||
| 225 | SDValue Val = OutVals[i]; | ||||
| 226 | |||||
| 227 | switch (VA.getLocInfo()) { | ||||
| 228 | default: | ||||
| 229 | // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt. | ||||
| 230 | llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 230); | ||||
| 231 | case CCValAssign::Full: | ||||
| 232 | break; | ||||
| 233 | case CCValAssign::BCvt: | ||||
| 234 | Val = DAG.getBitcast(VA.getLocVT(), Val); | ||||
| 235 | break; | ||||
| 236 | case CCValAssign::SExt: | ||||
| 237 | Val = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Val); | ||||
| 238 | break; | ||||
| 239 | case CCValAssign::ZExt: | ||||
| 240 | Val = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Val); | ||||
| 241 | break; | ||||
| 242 | case CCValAssign::AExt: | ||||
| 243 | Val = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Val); | ||||
| 244 | break; | ||||
| 245 | } | ||||
| 246 | |||||
| 247 | Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Val, Flag); | ||||
| 248 | |||||
| 249 | // Guarantee that all emitted copies are stuck together with flags. | ||||
| 250 | Flag = Chain.getValue(1); | ||||
| 251 | RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); | ||||
| 252 | } | ||||
| 253 | |||||
| 254 | RetOps[0] = Chain; // Update chain. | ||||
| 255 | |||||
| 256 | // Add the flag if we have it. | ||||
| 257 | if (Flag.getNode()) | ||||
| 258 | RetOps.push_back(Flag); | ||||
| 259 | |||||
| 260 | return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps); | ||||
| 261 | } | ||||
| 262 | |||||
| 263 | bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { | ||||
| 264 | // If either no tail call or told not to tail call at all, don't. | ||||
| 265 | return CI->isTailCall(); | ||||
| 266 | } | ||||
| 267 | |||||
| 268 | Register HexagonTargetLowering::getRegisterByName( | ||||
| 269 | const char* RegName, LLT VT, const MachineFunction &) const { | ||||
| 270 | // Just support r19, the linux kernel uses it. | ||||
| 271 | Register Reg = StringSwitch<Register>(RegName) | ||||
| 272 | .Case("r0", Hexagon::R0) | ||||
| 273 | .Case("r1", Hexagon::R1) | ||||
| 274 | .Case("r2", Hexagon::R2) | ||||
| 275 | .Case("r3", Hexagon::R3) | ||||
| 276 | .Case("r4", Hexagon::R4) | ||||
| 277 | .Case("r5", Hexagon::R5) | ||||
| 278 | .Case("r6", Hexagon::R6) | ||||
| 279 | .Case("r7", Hexagon::R7) | ||||
| 280 | .Case("r8", Hexagon::R8) | ||||
| 281 | .Case("r9", Hexagon::R9) | ||||
| 282 | .Case("r10", Hexagon::R10) | ||||
| 283 | .Case("r11", Hexagon::R11) | ||||
| 284 | .Case("r12", Hexagon::R12) | ||||
| 285 | .Case("r13", Hexagon::R13) | ||||
| 286 | .Case("r14", Hexagon::R14) | ||||
| 287 | .Case("r15", Hexagon::R15) | ||||
| 288 | .Case("r16", Hexagon::R16) | ||||
| 289 | .Case("r17", Hexagon::R17) | ||||
| 290 | .Case("r18", Hexagon::R18) | ||||
| 291 | .Case("r19", Hexagon::R19) | ||||
| 292 | .Case("r20", Hexagon::R20) | ||||
| 293 | .Case("r21", Hexagon::R21) | ||||
| 294 | .Case("r22", Hexagon::R22) | ||||
| 295 | .Case("r23", Hexagon::R23) | ||||
| 296 | .Case("r24", Hexagon::R24) | ||||
| 297 | .Case("r25", Hexagon::R25) | ||||
| 298 | .Case("r26", Hexagon::R26) | ||||
| 299 | .Case("r27", Hexagon::R27) | ||||
| 300 | .Case("r28", Hexagon::R28) | ||||
| 301 | .Case("r29", Hexagon::R29) | ||||
| 302 | .Case("r30", Hexagon::R30) | ||||
| 303 | .Case("r31", Hexagon::R31) | ||||
| 304 | .Case("r1:0", Hexagon::D0) | ||||
| 305 | .Case("r3:2", Hexagon::D1) | ||||
| 306 | .Case("r5:4", Hexagon::D2) | ||||
| 307 | .Case("r7:6", Hexagon::D3) | ||||
| 308 | .Case("r9:8", Hexagon::D4) | ||||
| 309 | .Case("r11:10", Hexagon::D5) | ||||
| 310 | .Case("r13:12", Hexagon::D6) | ||||
| 311 | .Case("r15:14", Hexagon::D7) | ||||
| 312 | .Case("r17:16", Hexagon::D8) | ||||
| 313 | .Case("r19:18", Hexagon::D9) | ||||
| 314 | .Case("r21:20", Hexagon::D10) | ||||
| 315 | .Case("r23:22", Hexagon::D11) | ||||
| 316 | .Case("r25:24", Hexagon::D12) | ||||
| 317 | .Case("r27:26", Hexagon::D13) | ||||
| 318 | .Case("r29:28", Hexagon::D14) | ||||
| 319 | .Case("r31:30", Hexagon::D15) | ||||
| 320 | .Case("sp", Hexagon::R29) | ||||
| 321 | .Case("fp", Hexagon::R30) | ||||
| 322 | .Case("lr", Hexagon::R31) | ||||
| 323 | .Case("p0", Hexagon::P0) | ||||
| 324 | .Case("p1", Hexagon::P1) | ||||
| 325 | .Case("p2", Hexagon::P2) | ||||
| 326 | .Case("p3", Hexagon::P3) | ||||
| 327 | .Case("sa0", Hexagon::SA0) | ||||
| 328 | .Case("lc0", Hexagon::LC0) | ||||
| 329 | .Case("sa1", Hexagon::SA1) | ||||
| 330 | .Case("lc1", Hexagon::LC1) | ||||
| 331 | .Case("m0", Hexagon::M0) | ||||
| 332 | .Case("m1", Hexagon::M1) | ||||
| 333 | .Case("usr", Hexagon::USR) | ||||
| 334 | .Case("ugp", Hexagon::UGP) | ||||
| 335 | .Case("cs0", Hexagon::CS0) | ||||
| 336 | .Case("cs1", Hexagon::CS1) | ||||
| 337 | .Default(Register()); | ||||
| 338 | if (Reg) | ||||
| 339 | return Reg; | ||||
| 340 | |||||
| 341 | report_fatal_error("Invalid register name global variable"); | ||||
| 342 | } | ||||
| 343 | |||||
| 344 | /// LowerCallResult - Lower the result values of an ISD::CALL into the | ||||
| 345 | /// appropriate copies out of appropriate physical registers. This assumes that | ||||
| 346 | /// Chain/Glue are the input chain/glue to use, and that TheCall is the call | ||||
| 347 | /// being lowered. Returns a SDNode with the same number of values as the | ||||
| 348 | /// ISD::CALL. | ||||
| 349 | SDValue HexagonTargetLowering::LowerCallResult( | ||||
| 350 | SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg, | ||||
| 351 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, | ||||
| 352 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, | ||||
| 353 | const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const { | ||||
| 354 | // Assign locations to each value returned by this call. | ||||
| 355 | SmallVector<CCValAssign, 16> RVLocs; | ||||
| 356 | |||||
| 357 | CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, | ||||
| 358 | *DAG.getContext()); | ||||
| 359 | |||||
| 360 | if (Subtarget.useHVXOps()) | ||||
| 361 | CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX); | ||||
| 362 | else | ||||
| 363 | CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon); | ||||
| 364 | |||||
| 365 | // Copy all of the result registers out of their specified physreg. | ||||
| 366 | for (unsigned i = 0; i != RVLocs.size(); ++i) { | ||||
| 367 | SDValue RetVal; | ||||
| 368 | if (RVLocs[i].getValVT() == MVT::i1) { | ||||
| 369 | // Return values of type MVT::i1 require special handling. The reason | ||||
| 370 | // is that MVT::i1 is associated with the PredRegs register class, but | ||||
| 371 | // values of that type are still returned in R0. Generate an explicit | ||||
| 372 | // copy into a predicate register from R0, and treat the value of the | ||||
| 373 | // predicate register as the call result. | ||||
| 374 | auto &MRI = DAG.getMachineFunction().getRegInfo(); | ||||
| 375 | SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(), | ||||
| 376 | MVT::i32, Glue); | ||||
| 377 | // FR0 = (Value, Chain, Glue) | ||||
| 378 | Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass); | ||||
| 379 | SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR, | ||||
| 380 | FR0.getValue(0), FR0.getValue(2)); | ||||
| 381 | // TPR = (Chain, Glue) | ||||
| 382 | // Don't glue this CopyFromReg, because it copies from a virtual | ||||
| 383 | // register. If it is glued to the call, InstrEmitter will add it | ||||
| 384 | // as an implicit def to the call (EmitMachineNode). | ||||
| 385 | RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1); | ||||
| 386 | Glue = TPR.getValue(1); | ||||
| 387 | Chain = TPR.getValue(0); | ||||
| 388 | } else { | ||||
| 389 | RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(), | ||||
| 390 | RVLocs[i].getValVT(), Glue); | ||||
| 391 | Glue = RetVal.getValue(2); | ||||
| 392 | Chain = RetVal.getValue(1); | ||||
| 393 | } | ||||
| 394 | InVals.push_back(RetVal.getValue(0)); | ||||
| 395 | } | ||||
| 396 | |||||
| 397 | return Chain; | ||||
| 398 | } | ||||
| 399 | |||||
| 400 | /// LowerCall - Functions arguments are copied from virtual regs to | ||||
| 401 | /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted. | ||||
| 402 | SDValue | ||||
| 403 | HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, | ||||
| 404 | SmallVectorImpl<SDValue> &InVals) const { | ||||
| 405 | SelectionDAG &DAG = CLI.DAG; | ||||
| 406 | SDLoc &dl = CLI.DL; | ||||
| 407 | SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; | ||||
| 408 | SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; | ||||
| 409 | SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; | ||||
| 410 | SDValue Chain = CLI.Chain; | ||||
| 411 | SDValue Callee = CLI.Callee; | ||||
| 412 | CallingConv::ID CallConv = CLI.CallConv; | ||||
| 413 | bool IsVarArg = CLI.IsVarArg; | ||||
| 414 | bool DoesNotReturn = CLI.DoesNotReturn; | ||||
| 415 | |||||
| 416 | bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet(); | ||||
| 417 | MachineFunction &MF = DAG.getMachineFunction(); | ||||
| 418 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||
| 419 | auto PtrVT = getPointerTy(MF.getDataLayout()); | ||||
| 420 | |||||
| 421 | unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0; | ||||
| 422 | if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee)) | ||||
| 423 | Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32); | ||||
| 424 | |||||
| 425 | // Linux ABI treats var-arg calls the same way as regular ones. | ||||
| 426 | bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg; | ||||
| 427 | |||||
| 428 | // Analyze operands of the call, assigning locations to each operand. | ||||
| 429 | SmallVector<CCValAssign, 16> ArgLocs; | ||||
| 430 | HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(), | ||||
| 431 | NumParams); | ||||
| 432 | |||||
| 433 | if (Subtarget.useHVXOps()) | ||||
| 434 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX); | ||||
| 435 | else if (DisableArgsMinAlignment) | ||||
| 436 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy); | ||||
| 437 | else | ||||
| 438 | CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon); | ||||
| 439 | |||||
| 440 | if (CLI.IsTailCall) { | ||||
| 441 | bool StructAttrFlag = MF.getFunction().hasStructRetAttr(); | ||||
| 442 | CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, | ||||
| 443 | IsVarArg, IsStructRet, StructAttrFlag, Outs, | ||||
| 444 | OutVals, Ins, DAG); | ||||
| 445 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | ||||
| 446 | CCValAssign &VA = ArgLocs[i]; | ||||
| 447 | if (VA.isMemLoc()) { | ||||
| 448 | CLI.IsTailCall = false; | ||||
| 449 | break; | ||||
| 450 | } | ||||
| 451 | } | ||||
| 452 | LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false) | ||||
| 453 | : "Argument must be passed on stack. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false) | ||||
| 454 | "Not eligible for Tail Call\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n" : "Argument must be passed on stack. " "Not eligible for Tail Call\n" ); } } while (false); | ||||
| 455 | } | ||||
| 456 | // Get a count of how many bytes are to be pushed on the stack. | ||||
| 457 | unsigned NumBytes = CCInfo.getNextStackOffset(); | ||||
| 458 | SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass; | ||||
| 459 | SmallVector<SDValue, 8> MemOpChains; | ||||
| 460 | |||||
| 461 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||
| 462 | SDValue StackPtr = | ||||
| 463 | DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT); | ||||
| 464 | |||||
| 465 | bool NeedsArgAlign = false; | ||||
| 466 | Align LargestAlignSeen; | ||||
| 467 | // Walk the register/memloc assignments, inserting copies/loads. | ||||
| 468 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | ||||
| 469 | CCValAssign &VA = ArgLocs[i]; | ||||
| 470 | SDValue Arg = OutVals[i]; | ||||
| 471 | ISD::ArgFlagsTy Flags = Outs[i].Flags; | ||||
| 472 | // Record if we need > 8 byte alignment on an argument. | ||||
| 473 | bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT()); | ||||
| 474 | NeedsArgAlign |= ArgAlign; | ||||
| 475 | |||||
| 476 | // Promote the value if needed. | ||||
| 477 | switch (VA.getLocInfo()) { | ||||
| 478 | default: | ||||
| 479 | // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt. | ||||
| 480 | llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 480); | ||||
| 481 | case CCValAssign::Full: | ||||
| 482 | break; | ||||
| 483 | case CCValAssign::BCvt: | ||||
| 484 | Arg = DAG.getBitcast(VA.getLocVT(), Arg); | ||||
| 485 | break; | ||||
| 486 | case CCValAssign::SExt: | ||||
| 487 | Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); | ||||
| 488 | break; | ||||
| 489 | case CCValAssign::ZExt: | ||||
| 490 | Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); | ||||
| 491 | break; | ||||
| 492 | case CCValAssign::AExt: | ||||
| 493 | Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); | ||||
| 494 | break; | ||||
| 495 | } | ||||
| 496 | |||||
| 497 | if (VA.isMemLoc()) { | ||||
| 498 | unsigned LocMemOffset = VA.getLocMemOffset(); | ||||
| 499 | SDValue MemAddr = DAG.getConstant(LocMemOffset, dl, | ||||
| 500 | StackPtr.getValueType()); | ||||
| 501 | MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr); | ||||
| 502 | if (ArgAlign) | ||||
| 503 | LargestAlignSeen = std::max( | ||||
| 504 | LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8)); | ||||
| 505 | if (Flags.isByVal()) { | ||||
| 506 | // The argument is a struct passed by value. According to LLVM, "Arg" | ||||
| 507 | // is a pointer. | ||||
| 508 | MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain, | ||||
| 509 | Flags, DAG, dl)); | ||||
| 510 | } else { | ||||
| 511 | MachinePointerInfo LocPI = MachinePointerInfo::getStack( | ||||
| 512 | DAG.getMachineFunction(), LocMemOffset); | ||||
| 513 | SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI); | ||||
| 514 | MemOpChains.push_back(S); | ||||
| 515 | } | ||||
| 516 | continue; | ||||
| 517 | } | ||||
| 518 | |||||
| 519 | // Arguments that can be passed on register must be kept at RegsToPass | ||||
| 520 | // vector. | ||||
| 521 | if (VA.isRegLoc()) | ||||
| 522 | RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); | ||||
| 523 | } | ||||
| 524 | |||||
| 525 | if (NeedsArgAlign && Subtarget.hasV60Ops()) { | ||||
| 526 | LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { dbgs() << "Function needs byte stack align due to call args\n" ; } } while (false); | ||||
| 527 | Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass); | ||||
| 528 | LargestAlignSeen = std::max(LargestAlignSeen, VecAlign); | ||||
| 529 | MFI.ensureMaxAlignment(LargestAlignSeen); | ||||
| 530 | } | ||||
| 531 | // Transform all store nodes into one single node because all store | ||||
| 532 | // nodes are independent of each other. | ||||
| 533 | if (!MemOpChains.empty()) | ||||
| 534 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); | ||||
| 535 | |||||
| 536 | SDValue Glue; | ||||
| 537 | if (!CLI.IsTailCall) { | ||||
| 538 | Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); | ||||
| 539 | Glue = Chain.getValue(1); | ||||
| 540 | } | ||||
| 541 | |||||
| 542 | // Build a sequence of copy-to-reg nodes chained together with token | ||||
| 543 | // chain and flag operands which copy the outgoing args into registers. | ||||
| 544 | // The Glue is necessary since all emitted instructions must be | ||||
| 545 | // stuck together. | ||||
| 546 | if (!CLI.IsTailCall) { | ||||
| 547 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | ||||
| 548 | Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, | ||||
| 549 | RegsToPass[i].second, Glue); | ||||
| 550 | Glue = Chain.getValue(1); | ||||
| 551 | } | ||||
| 552 | } else { | ||||
| 553 | // For tail calls lower the arguments to the 'real' stack slot. | ||||
| 554 | // | ||||
| 555 | // Force all the incoming stack arguments to be loaded from the stack | ||||
| 556 | // before any new outgoing arguments are stored to the stack, because the | ||||
| 557 | // outgoing stack slots may alias the incoming argument stack slots, and | ||||
| 558 | // the alias isn't otherwise explicit. This is slightly more conservative | ||||
| 559 | // than necessary, because it means that each store effectively depends | ||||
| 560 | // on every argument instead of just those arguments it would clobber. | ||||
| 561 | // | ||||
| 562 | // Do not flag preceding copytoreg stuff together with the following stuff. | ||||
| 563 | Glue = SDValue(); | ||||
| 564 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | ||||
| 565 | Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, | ||||
| 566 | RegsToPass[i].second, Glue); | ||||
| 567 | Glue = Chain.getValue(1); | ||||
| 568 | } | ||||
| 569 | Glue = SDValue(); | ||||
| 570 | } | ||||
| 571 | |||||
| 572 | bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls(); | ||||
| 573 | unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0; | ||||
| 574 | |||||
| 575 | // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every | ||||
| 576 | // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol | ||||
| 577 | // node so that legalize doesn't hack it. | ||||
| 578 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { | ||||
| 579 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags); | ||||
| 580 | } else if (ExternalSymbolSDNode *S = | ||||
| 581 | dyn_cast<ExternalSymbolSDNode>(Callee)) { | ||||
| 582 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags); | ||||
| 583 | } | ||||
| 584 | |||||
| 585 | // Returns a chain & a flag for retval copy to use. | ||||
| 586 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||
| 587 | SmallVector<SDValue, 8> Ops; | ||||
| 588 | Ops.push_back(Chain); | ||||
| 589 | Ops.push_back(Callee); | ||||
| 590 | |||||
| 591 | // Add argument registers to the end of the list so that they are | ||||
| 592 | // known live into the call. | ||||
| 593 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { | ||||
| 594 | Ops.push_back(DAG.getRegister(RegsToPass[i].first, | ||||
| 595 | RegsToPass[i].second.getValueType())); | ||||
| 596 | } | ||||
| 597 | |||||
| 598 | const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv); | ||||
| 599 | assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention" ) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 599, __extension__ __PRETTY_FUNCTION__)); | ||||
| 600 | Ops.push_back(DAG.getRegisterMask(Mask)); | ||||
| 601 | |||||
| 602 | if (Glue.getNode()) | ||||
| 603 | Ops.push_back(Glue); | ||||
| 604 | |||||
| 605 | if (CLI.IsTailCall) { | ||||
| 606 | MFI.setHasTailCall(); | ||||
| 607 | return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops); | ||||
| 608 | } | ||||
| 609 | |||||
| 610 | // Set this here because we need to know this for "hasFP" in frame lowering. | ||||
| 611 | // The target-independent code calls getFrameRegister before setting it, and | ||||
| 612 | // getFrameRegister uses hasFP to determine whether the function has FP. | ||||
| 613 | MFI.setHasCalls(true); | ||||
| 614 | |||||
| 615 | unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL; | ||||
| 616 | Chain = DAG.getNode(OpCode, dl, NodeTys, Ops); | ||||
| 617 | Glue = Chain.getValue(1); | ||||
| 618 | |||||
| 619 | // Create the CALLSEQ_END node. | ||||
| 620 | Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), | ||||
| 621 | DAG.getIntPtrConstant(0, dl, true), Glue, dl); | ||||
| 622 | Glue = Chain.getValue(1); | ||||
| 623 | |||||
| 624 | // Handle result values, copying them out of physregs into vregs that we | ||||
| 625 | // return. | ||||
| 626 | return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG, | ||||
| 627 | InVals, OutVals, Callee); | ||||
| 628 | } | ||||
| 629 | |||||
| 630 | /// Returns true by value, base pointer and offset pointer and addressing | ||||
| 631 | /// mode by reference if this node can be combined with a load / store to | ||||
| 632 | /// form a post-indexed load / store. | ||||
| 633 | bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op, | ||||
| 634 | SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, | ||||
| 635 | SelectionDAG &DAG) const { | ||||
| 636 | LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N); | ||||
| 637 | if (!LSN) | ||||
| 638 | return false; | ||||
| 639 | EVT VT = LSN->getMemoryVT(); | ||||
| 640 | if (!VT.isSimple()) | ||||
| 641 | return false; | ||||
| 642 | bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || | ||||
| 643 | VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 || | ||||
| 644 | VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 || | ||||
| 645 | VT == MVT::v4i16 || VT == MVT::v8i8 || | ||||
| 646 | Subtarget.isHVXVectorType(VT.getSimpleVT()); | ||||
| 647 | if (!IsLegalType) | ||||
| 648 | return false; | ||||
| 649 | |||||
| 650 | if (Op->getOpcode() != ISD::ADD) | ||||
| 651 | return false; | ||||
| 652 | Base = Op->getOperand(0); | ||||
| 653 | Offset = Op->getOperand(1); | ||||
| 654 | if (!isa<ConstantSDNode>(Offset.getNode())) | ||||
| 655 | return false; | ||||
| 656 | AM = ISD::POST_INC; | ||||
| 657 | |||||
| 658 | int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue(); | ||||
| 659 | return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V); | ||||
| 660 | } | ||||
| 661 | |||||
| 662 | SDValue | ||||
| 663 | HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const { | ||||
| 664 | MachineFunction &MF = DAG.getMachineFunction(); | ||||
| 665 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); | ||||
| 666 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||
| 667 | unsigned LR = HRI.getRARegister(); | ||||
| 668 | |||||
| 669 | if ((Op.getOpcode() != ISD::INLINEASM && | ||||
| 670 | Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR()) | ||||
| 671 | return Op; | ||||
| 672 | |||||
| 673 | unsigned NumOps = Op.getNumOperands(); | ||||
| 674 | if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue) | ||||
| 675 | --NumOps; // Ignore the flag operand. | ||||
| 676 | |||||
| 677 | for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { | ||||
| 678 | unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue(); | ||||
| 679 | unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); | ||||
| 680 | ++i; // Skip the ID value. | ||||
| 681 | |||||
| 682 | switch (InlineAsm::getKind(Flags)) { | ||||
| 683 | default: | ||||
| 684 | llvm_unreachable("Bad flags!")::llvm::llvm_unreachable_internal("Bad flags!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 684); | ||||
| 685 | case InlineAsm::Kind_RegUse: | ||||
| 686 | case InlineAsm::Kind_Imm: | ||||
| 687 | case InlineAsm::Kind_Mem: | ||||
| 688 | i += NumVals; | ||||
| 689 | break; | ||||
| 690 | case InlineAsm::Kind_Clobber: | ||||
| 691 | case InlineAsm::Kind_RegDef: | ||||
| 692 | case InlineAsm::Kind_RegDefEarlyClobber: { | ||||
| 693 | for (; NumVals; --NumVals, ++i) { | ||||
| 694 | unsigned Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg(); | ||||
| 695 | if (Reg != LR) | ||||
| 696 | continue; | ||||
| 697 | HMFI.setHasClobberLR(true); | ||||
| 698 | return Op; | ||||
| 699 | } | ||||
| 700 | break; | ||||
| 701 | } | ||||
| 702 | } | ||||
| 703 | } | ||||
| 704 | |||||
| 705 | return Op; | ||||
| 706 | } | ||||
| 707 | |||||
| 708 | // Need to transform ISD::PREFETCH into something that doesn't inherit | ||||
| 709 | // all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and | ||||
| 710 | // SDNPMayStore. | ||||
| 711 | SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op, | ||||
| 712 | SelectionDAG &DAG) const { | ||||
| 713 | SDValue Chain = Op.getOperand(0); | ||||
| 714 | SDValue Addr = Op.getOperand(1); | ||||
| 715 | // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in, | ||||
| 716 | // if the "reg" is fed by an "add". | ||||
| 717 | SDLoc DL(Op); | ||||
| 718 | SDValue Zero = DAG.getConstant(0, DL, MVT::i32); | ||||
| 719 | return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero); | ||||
| 720 | } | ||||
| 721 | |||||
| 722 | // Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode | ||||
| 723 | // is marked as having side-effects, while the register read on Hexagon does | ||||
| 724 | // not have any. TableGen refuses to accept the direct pattern from that node | ||||
| 725 | // to the A4_tfrcpp. | ||||
| 726 | SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op, | ||||
| 727 | SelectionDAG &DAG) const { | ||||
| 728 | SDValue Chain = Op.getOperand(0); | ||||
| 729 | SDLoc dl(Op); | ||||
| 730 | SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other); | ||||
| 731 | return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain); | ||||
| 732 | } | ||||
| 733 | |||||
| 734 | SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op, | ||||
| 735 | SelectionDAG &DAG) const { | ||||
| 736 | SDValue Chain = Op.getOperand(0); | ||||
| 737 | unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); | ||||
| 738 | // Lower the hexagon_prefetch builtin to DCFETCH, as above. | ||||
| 739 | if (IntNo == Intrinsic::hexagon_prefetch) { | ||||
| 740 | SDValue Addr = Op.getOperand(2); | ||||
| 741 | SDLoc DL(Op); | ||||
| 742 | SDValue Zero = DAG.getConstant(0, DL, MVT::i32); | ||||
| 743 | return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero); | ||||
| 744 | } | ||||
| 745 | return SDValue(); | ||||
| 746 | } | ||||
| 747 | |||||
| 748 | SDValue | ||||
| 749 | HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, | ||||
| 750 | SelectionDAG &DAG) const { | ||||
| 751 | SDValue Chain = Op.getOperand(0); | ||||
| 752 | SDValue Size = Op.getOperand(1); | ||||
| 753 | SDValue Align = Op.getOperand(2); | ||||
| 754 | SDLoc dl(Op); | ||||
| 755 | |||||
| 756 | ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align); | ||||
| 757 | assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC")(static_cast <bool> (AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC" ) ? void (0) : __assert_fail ("AlignConst && \"Non-constant Align in LowerDYNAMIC_STACKALLOC\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 757, __extension__ __PRETTY_FUNCTION__)); | ||||
| 758 | |||||
| 759 | unsigned A = AlignConst->getSExtValue(); | ||||
| 760 | auto &HFI = *Subtarget.getFrameLowering(); | ||||
| 761 | // "Zero" means natural stack alignment. | ||||
| 762 | if (A == 0) | ||||
| 763 | A = HFI.getStackAlign().value(); | ||||
| 764 | |||||
| 765 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||
| 766 | dbgs () << __func__ << " Align: " << A << " Size: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||
| 767 | Size.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||
| 768 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false) | ||||
| 769 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("hexagon-lowering")) { { dbgs () << __func__ << " Align: " << A << " Size: "; Size.getNode()->dump(& DAG); dbgs() << "\n"; }; } } while (false); | ||||
| 770 | |||||
| 771 | SDValue AC = DAG.getConstant(A, dl, MVT::i32); | ||||
| 772 | SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); | ||||
| 773 | SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC); | ||||
| 774 | |||||
| 775 | DAG.ReplaceAllUsesOfValueWith(Op, AA); | ||||
| 776 | return AA; | ||||
| 777 | } | ||||
| 778 | |||||
| 779 | SDValue HexagonTargetLowering::LowerFormalArguments( | ||||
| 780 | SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, | ||||
| 781 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, | ||||
| 782 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { | ||||
| 783 | MachineFunction &MF = DAG.getMachineFunction(); | ||||
| 784 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||
| 785 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||
| 786 | |||||
| 787 | // Linux ABI treats var-arg calls the same way as regular ones. | ||||
| 788 | bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg; | ||||
| 789 | |||||
| 790 | // Assign locations to all of the incoming arguments. | ||||
| 791 | SmallVector<CCValAssign, 16> ArgLocs; | ||||
| 792 | HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, | ||||
| 793 | *DAG.getContext(), | ||||
| 794 | MF.getFunction().getFunctionType()->getNumParams()); | ||||
| 795 | |||||
| 796 | if (Subtarget.useHVXOps()) | ||||
| 797 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX); | ||||
| 798 | else if (DisableArgsMinAlignment) | ||||
| 799 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy); | ||||
| 800 | else | ||||
| 801 | CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon); | ||||
| 802 | |||||
| 803 | // For LLVM, in the case when returning a struct by value (>8byte), | ||||
| 804 | // the first argument is a pointer that points to the location on caller's | ||||
| 805 | // stack where the return value will be stored. For Hexagon, the location on | ||||
| 806 | // caller's stack is passed only when the struct size is smaller than (and | ||||
| 807 | // equal to) 8 bytes. If not, no address will be passed into callee and | ||||
| 808 | // callee return the result direclty through R0/R1. | ||||
| 809 | auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) { | ||||
| 810 | switch (RC.getID()) { | ||||
| 811 | case Hexagon::IntRegsRegClassID: | ||||
| 812 | return Reg - Hexagon::R0 + 1; | ||||
| 813 | case Hexagon::DoubleRegsRegClassID: | ||||
| 814 | return (Reg - Hexagon::D0 + 1) * 2; | ||||
| 815 | case Hexagon::HvxVRRegClassID: | ||||
| 816 | return Reg - Hexagon::V0 + 1; | ||||
| 817 | case Hexagon::HvxWRRegClassID: | ||||
| 818 | return (Reg - Hexagon::W0 + 1) * 2; | ||||
| 819 | } | ||||
| 820 | llvm_unreachable("Unexpected register class")::llvm::llvm_unreachable_internal("Unexpected register class" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 820); | ||||
| 821 | }; | ||||
| 822 | |||||
| 823 | auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering()); | ||||
| 824 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); | ||||
| 825 | HFL.FirstVarArgSavedReg = 0; | ||||
| 826 | HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects())); | ||||
| 827 | |||||
| 828 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { | ||||
| 829 | CCValAssign &VA = ArgLocs[i]; | ||||
| 830 | ISD::ArgFlagsTy Flags = Ins[i].Flags; | ||||
| 831 | bool ByVal = Flags.isByVal(); | ||||
| 832 | |||||
| 833 | // Arguments passed in registers: | ||||
| 834 | // 1. 32- and 64-bit values and HVX vectors are passed directly, | ||||
| 835 | // 2. Large structs are passed via an address, and the address is | ||||
| 836 | // passed in a register. | ||||
| 837 | if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8) | ||||
| 838 | llvm_unreachable("ByValSize must be bigger than 8 bytes")::llvm::llvm_unreachable_internal("ByValSize must be bigger than 8 bytes" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 838); | ||||
| 839 | |||||
| 840 | bool InReg = VA.isRegLoc() && | ||||
| 841 | (!ByVal || (ByVal && Flags.getByValSize() > 8)); | ||||
| 842 | |||||
| 843 | if (InReg) { | ||||
| 844 | MVT RegVT = VA.getLocVT(); | ||||
| 845 | if (VA.getLocInfo() == CCValAssign::BCvt) | ||||
| 846 | RegVT = VA.getValVT(); | ||||
| 847 | |||||
| 848 | const TargetRegisterClass *RC = getRegClassFor(RegVT); | ||||
| 849 | Register VReg = MRI.createVirtualRegister(RC); | ||||
| 850 | SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT); | ||||
| 851 | |||||
| 852 | // Treat values of type MVT::i1 specially: they are passed in | ||||
| 853 | // registers of type i32, but they need to remain as values of | ||||
| 854 | // type i1 for consistency of the argument lowering. | ||||
| 855 | if (VA.getValVT() == MVT::i1) { | ||||
| 856 | assert(RegVT.getSizeInBits() <= 32)(static_cast <bool> (RegVT.getSizeInBits() <= 32) ? void (0) : __assert_fail ("RegVT.getSizeInBits() <= 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 856, __extension__ __PRETTY_FUNCTION__)); | ||||
| 857 | SDValue T = DAG.getNode(ISD::AND, dl, RegVT, | ||||
| 858 | Copy, DAG.getConstant(1, dl, RegVT)); | ||||
| 859 | Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT), | ||||
| 860 | ISD::SETNE); | ||||
| 861 | } else { | ||||
| 862 | #ifndef NDEBUG | ||||
| 863 | unsigned RegSize = RegVT.getSizeInBits(); | ||||
| 864 | assert(RegSize == 32 || RegSize == 64 ||(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget .isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 865, __extension__ __PRETTY_FUNCTION__)) | ||||
| 865 | Subtarget.isHVXVectorType(RegVT))(static_cast <bool> (RegSize == 32 || RegSize == 64 || Subtarget .isHVXVectorType(RegVT)) ? void (0) : __assert_fail ("RegSize == 32 || RegSize == 64 || Subtarget.isHVXVectorType(RegVT)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 865, __extension__ __PRETTY_FUNCTION__)); | ||||
| 866 | #endif | ||||
| 867 | } | ||||
| 868 | InVals.push_back(Copy); | ||||
| 869 | MRI.addLiveIn(VA.getLocReg(), VReg); | ||||
| 870 | HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg()); | ||||
| 871 | } else { | ||||
| 872 | assert(VA.isMemLoc() && "Argument should be passed in memory")(static_cast <bool> (VA.isMemLoc() && "Argument should be passed in memory" ) ? void (0) : __assert_fail ("VA.isMemLoc() && \"Argument should be passed in memory\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 872, __extension__ __PRETTY_FUNCTION__)); | ||||
| 873 | |||||
| 874 | // If it's a byval parameter, then we need to compute the | ||||
| 875 | // "real" size, not the size of the pointer. | ||||
| 876 | unsigned ObjSize = Flags.isByVal() | ||||
| 877 | ? Flags.getByValSize() | ||||
| 878 | : VA.getLocVT().getStoreSizeInBits() / 8; | ||||
| 879 | |||||
| 880 | // Create the frame index object for this incoming parameter. | ||||
| 881 | int Offset = HEXAGON_LRFP_SIZE8 + VA.getLocMemOffset(); | ||||
| 882 | int FI = MFI.CreateFixedObject(ObjSize, Offset, true); | ||||
| 883 | SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); | ||||
| 884 | |||||
| 885 | if (Flags.isByVal()) { | ||||
| 886 | // If it's a pass-by-value aggregate, then do not dereference the stack | ||||
| 887 | // location. Instead, we should generate a reference to the stack | ||||
| 888 | // location. | ||||
| 889 | InVals.push_back(FIN); | ||||
| 890 | } else { | ||||
| 891 | SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN, | ||||
| 892 | MachinePointerInfo::getFixedStack(MF, FI, 0)); | ||||
| 893 | InVals.push_back(L); | ||||
| 894 | } | ||||
| 895 | } | ||||
| 896 | } | ||||
| 897 | |||||
| 898 | if (IsVarArg && Subtarget.isEnvironmentMusl()) { | ||||
| 899 | for (int i = HFL.FirstVarArgSavedReg; i < 6; i++) | ||||
| 900 | MRI.addLiveIn(Hexagon::R0+i); | ||||
| 901 | } | ||||
| 902 | |||||
| 903 | if (IsVarArg && Subtarget.isEnvironmentMusl()) { | ||||
| 904 | HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1); | ||||
| 905 | HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects())); | ||||
| 906 | |||||
| 907 | // Create Frame index for the start of register saved area. | ||||
| 908 | int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg; | ||||
| 909 | bool RequiresPadding = (NumVarArgRegs & 1); | ||||
| 910 | int RegSaveAreaSizePlusPadding = RequiresPadding | ||||
| 911 | ? (NumVarArgRegs + 1) * 4 | ||||
| 912 | : NumVarArgRegs * 4; | ||||
| 913 | |||||
| 914 | if (RegSaveAreaSizePlusPadding > 0) { | ||||
| 915 | // The offset to saved register area should be 8 byte aligned. | ||||
| 916 | int RegAreaStart = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset(); | ||||
| 917 | if (!(RegAreaStart % 8)) | ||||
| 918 | RegAreaStart = (RegAreaStart + 7) & -8; | ||||
| 919 | |||||
| 920 | int RegSaveAreaFrameIndex = | ||||
| 921 | MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true); | ||||
| 922 | HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex); | ||||
| 923 | |||||
| 924 | // This will point to the next argument passed via stack. | ||||
| 925 | int Offset = RegAreaStart + RegSaveAreaSizePlusPadding; | ||||
| 926 | int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true); | ||||
| 927 | HMFI.setVarArgsFrameIndex(FI); | ||||
| 928 | } else { | ||||
| 929 | // This will point to the next argument passed via stack, when | ||||
| 930 | // there is no saved register area. | ||||
| 931 | int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset(); | ||||
| 932 | int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true); | ||||
| 933 | HMFI.setRegSavedAreaStartFrameIndex(FI); | ||||
| 934 | HMFI.setVarArgsFrameIndex(FI); | ||||
| 935 | } | ||||
| 936 | } | ||||
| 937 | |||||
| 938 | |||||
| 939 | if (IsVarArg && !Subtarget.isEnvironmentMusl()) { | ||||
| 940 | // This will point to the next argument passed via stack. | ||||
| 941 | int Offset = HEXAGON_LRFP_SIZE8 + CCInfo.getNextStackOffset(); | ||||
| 942 | int FI = MFI.CreateFixedObject(Hexagon_PointerSize(4), Offset, true); | ||||
| 943 | HMFI.setVarArgsFrameIndex(FI); | ||||
| 944 | } | ||||
| 945 | |||||
| 946 | return Chain; | ||||
| 947 | } | ||||
| 948 | |||||
| 949 | SDValue | ||||
| 950 | HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { | ||||
| 951 | // VASTART stores the address of the VarArgsFrameIndex slot into the | ||||
| 952 | // memory location argument. | ||||
| 953 | MachineFunction &MF = DAG.getMachineFunction(); | ||||
| 954 | HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>(); | ||||
| 955 | SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32); | ||||
| 956 | const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); | ||||
| 957 | |||||
| 958 | if (!Subtarget.isEnvironmentMusl()) { | ||||
| 959 | return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1), | ||||
| 960 | MachinePointerInfo(SV)); | ||||
| 961 | } | ||||
| 962 | auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>(); | ||||
| 963 | auto &HFL = *Subtarget.getFrameLowering(); | ||||
| 964 | SDLoc DL(Op); | ||||
| 965 | SmallVector<SDValue, 8> MemOps; | ||||
| 966 | |||||
| 967 | // Get frame index of va_list. | ||||
| 968 | SDValue FIN = Op.getOperand(1); | ||||
| 969 | |||||
| 970 | // If first Vararg register is odd, add 4 bytes to start of | ||||
| 971 | // saved register area to point to the first register location. | ||||
| 972 | // This is because the saved register area has to be 8 byte aligned. | ||||
| 973 | // Incase of an odd start register, there will be 4 bytes of padding in | ||||
| 974 | // the beginning of saved register area. If all registers area used up, | ||||
| 975 | // the following condition will handle it correctly. | ||||
| 976 | SDValue SavedRegAreaStartFrameIndex = | ||||
| 977 | DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32); | ||||
| 978 | |||||
| 979 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 980 | |||||
| 981 | if (HFL.FirstVarArgSavedReg & 1) | ||||
| 982 | SavedRegAreaStartFrameIndex = | ||||
| 983 | DAG.getNode(ISD::ADD, DL, PtrVT, | ||||
| 984 | DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), | ||||
| 985 | MVT::i32), | ||||
| 986 | DAG.getIntPtrConstant(4, DL)); | ||||
| 987 | |||||
| 988 | // Store the saved register area start pointer. | ||||
| 989 | SDValue Store = | ||||
| 990 | DAG.getStore(Op.getOperand(0), DL, | ||||
| 991 | SavedRegAreaStartFrameIndex, | ||||
| 992 | FIN, MachinePointerInfo(SV)); | ||||
| 993 | MemOps.push_back(Store); | ||||
| 994 | |||||
| 995 | // Store saved register area end pointer. | ||||
| 996 | FIN = DAG.getNode(ISD::ADD, DL, PtrVT, | ||||
| 997 | FIN, DAG.getIntPtrConstant(4, DL)); | ||||
| 998 | Store = DAG.getStore(Op.getOperand(0), DL, | ||||
| 999 | DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(), | ||||
| 1000 | PtrVT), | ||||
| 1001 | FIN, MachinePointerInfo(SV, 4)); | ||||
| 1002 | MemOps.push_back(Store); | ||||
| 1003 | |||||
| 1004 | // Store overflow area pointer. | ||||
| 1005 | FIN = DAG.getNode(ISD::ADD, DL, PtrVT, | ||||
| 1006 | FIN, DAG.getIntPtrConstant(4, DL)); | ||||
| 1007 | Store = DAG.getStore(Op.getOperand(0), DL, | ||||
| 1008 | DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(), | ||||
| 1009 | PtrVT), | ||||
| 1010 | FIN, MachinePointerInfo(SV, 8)); | ||||
| 1011 | MemOps.push_back(Store); | ||||
| 1012 | |||||
| 1013 | return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); | ||||
| 1014 | } | ||||
| 1015 | |||||
| 1016 | SDValue | ||||
| 1017 | HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1018 | // Assert that the linux ABI is enabled for the current compilation. | ||||
| 1019 | assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled")(static_cast <bool> (Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled") ? void (0) : __assert_fail ("Subtarget.isEnvironmentMusl() && \"Linux ABI should be enabled\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1019, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1020 | SDValue Chain = Op.getOperand(0); | ||||
| 1021 | SDValue DestPtr = Op.getOperand(1); | ||||
| 1022 | SDValue SrcPtr = Op.getOperand(2); | ||||
| 1023 | const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); | ||||
| 1024 | const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); | ||||
| 1025 | SDLoc DL(Op); | ||||
| 1026 | // Size of the va_list is 12 bytes as it has 3 pointers. Therefore, | ||||
| 1027 | // we need to memcopy 12 bytes from va_list to another similar list. | ||||
| 1028 | return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr, | ||||
| 1029 | DAG.getIntPtrConstant(12, DL), Align(4), | ||||
| 1030 | /*isVolatile*/ false, false, false, | ||||
| 1031 | MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV)); | ||||
| 1032 | } | ||||
| 1033 | |||||
| 1034 | SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1035 | const SDLoc &dl(Op); | ||||
| 1036 | SDValue LHS = Op.getOperand(0); | ||||
| 1037 | SDValue RHS = Op.getOperand(1); | ||||
| 1038 | ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); | ||||
| 1039 | MVT ResTy = ty(Op); | ||||
| 1040 | MVT OpTy = ty(LHS); | ||||
| 1041 | |||||
| 1042 | if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) { | ||||
| 1043 | MVT ElemTy = OpTy.getVectorElementType(); | ||||
| 1044 | assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void ( 0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1044, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1045 | MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()), | ||||
| 1046 | OpTy.getVectorNumElements()); | ||||
| 1047 | return DAG.getSetCC(dl, ResTy, | ||||
| 1048 | DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy), | ||||
| 1049 | DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC); | ||||
| 1050 | } | ||||
| 1051 | |||||
| 1052 | // Treat all other vector types as legal. | ||||
| 1053 | if (ResTy.isVector()) | ||||
| 1054 | return Op; | ||||
| 1055 | |||||
| 1056 | // Comparisons of short integers should use sign-extend, not zero-extend, | ||||
| 1057 | // since we can represent small negative values in the compare instructions. | ||||
| 1058 | // The LLVM default is to use zero-extend arbitrarily in these cases. | ||||
| 1059 | auto isSExtFree = [this](SDValue N) { | ||||
| 1060 | switch (N.getOpcode()) { | ||||
| 1061 | case ISD::TRUNCATE: { | ||||
| 1062 | // A sign-extend of a truncate of a sign-extend is free. | ||||
| 1063 | SDValue Op = N.getOperand(0); | ||||
| 1064 | if (Op.getOpcode() != ISD::AssertSext) | ||||
| 1065 | return false; | ||||
| 1066 | EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT(); | ||||
| 1067 | unsigned ThisBW = ty(N).getSizeInBits(); | ||||
| 1068 | unsigned OrigBW = OrigTy.getSizeInBits(); | ||||
| 1069 | // The type that was sign-extended to get the AssertSext must be | ||||
| 1070 | // narrower than the type of N (so that N has still the same value | ||||
| 1071 | // as the original). | ||||
| 1072 | return ThisBW >= OrigBW; | ||||
| 1073 | } | ||||
| 1074 | case ISD::LOAD: | ||||
| 1075 | // We have sign-extended loads. | ||||
| 1076 | return true; | ||||
| 1077 | } | ||||
| 1078 | return false; | ||||
| 1079 | }; | ||||
| 1080 | |||||
| 1081 | if (OpTy == MVT::i8 || OpTy == MVT::i16) { | ||||
| 1082 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS); | ||||
| 1083 | bool IsNegative = C && C->getAPIntValue().isNegative(); | ||||
| 1084 | if (IsNegative
| ||||
| 1085 | return DAG.getSetCC(dl, ResTy, | ||||
| 1086 | DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32), | ||||
| 1087 | DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC); | ||||
| 1088 | } | ||||
| 1089 | |||||
| 1090 | return SDValue(); | ||||
| 1091 | } | ||||
| 1092 | |||||
| 1093 | SDValue | ||||
| 1094 | HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1095 | SDValue PredOp = Op.getOperand(0); | ||||
| 1096 | SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2); | ||||
| 1097 | MVT OpTy = ty(Op1); | ||||
| 1098 | const SDLoc &dl(Op); | ||||
| 1099 | |||||
| 1100 | if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) { | ||||
| 1101 | MVT ElemTy = OpTy.getVectorElementType(); | ||||
| 1102 | assert(ElemTy.isScalarInteger())(static_cast <bool> (ElemTy.isScalarInteger()) ? void ( 0) : __assert_fail ("ElemTy.isScalarInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1102, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1103 | MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()), | ||||
| 1104 | OpTy.getVectorNumElements()); | ||||
| 1105 | // Generate (trunc (select (_, sext, sext))). | ||||
| 1106 | return DAG.getSExtOrTrunc( | ||||
| 1107 | DAG.getSelect(dl, WideTy, PredOp, | ||||
| 1108 | DAG.getSExtOrTrunc(Op1, dl, WideTy), | ||||
| 1109 | DAG.getSExtOrTrunc(Op2, dl, WideTy)), | ||||
| 1110 | dl, OpTy); | ||||
| 1111 | } | ||||
| 1112 | |||||
| 1113 | return SDValue(); | ||||
| 1114 | } | ||||
| 1115 | |||||
| 1116 | SDValue | ||||
| 1117 | HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1118 | EVT ValTy = Op.getValueType(); | ||||
| 1119 | ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op); | ||||
| 1120 | Constant *CVal = nullptr; | ||||
| 1121 | bool isVTi1Type = false; | ||||
| 1122 | if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) { | ||||
| 1123 | if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) { | ||||
| 1124 | IRBuilder<> IRB(CV->getContext()); | ||||
| 1125 | SmallVector<Constant*, 128> NewConst; | ||||
| 1126 | unsigned VecLen = CV->getNumOperands(); | ||||
| 1127 | assert(isPowerOf2_32(VecLen) &&(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize" ) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1128, __extension__ __PRETTY_FUNCTION__)) | ||||
| 1128 | "conversion only supported for pow2 VectorSize")(static_cast <bool> (isPowerOf2_32(VecLen) && "conversion only supported for pow2 VectorSize" ) ? void (0) : __assert_fail ("isPowerOf2_32(VecLen) && \"conversion only supported for pow2 VectorSize\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1128, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1129 | for (unsigned i = 0; i < VecLen; ++i) | ||||
| 1130 | NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue())); | ||||
| 1131 | |||||
| 1132 | CVal = ConstantVector::get(NewConst); | ||||
| 1133 | isVTi1Type = true; | ||||
| 1134 | } | ||||
| 1135 | } | ||||
| 1136 | Align Alignment = CPN->getAlign(); | ||||
| 1137 | bool IsPositionIndependent = isPositionIndependent(); | ||||
| 1138 | unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0; | ||||
| 1139 | |||||
| 1140 | unsigned Offset = 0; | ||||
| 1141 | SDValue T; | ||||
| 1142 | if (CPN->isMachineConstantPoolEntry()) | ||||
| 1143 | T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment, | ||||
| 1144 | Offset, TF); | ||||
| 1145 | else if (isVTi1Type) | ||||
| 1146 | T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF); | ||||
| 1147 | else | ||||
| 1148 | T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset, | ||||
| 1149 | TF); | ||||
| 1150 | |||||
| 1151 | assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&(static_cast <bool> (cast<ConstantPoolSDNode>(T)-> getTargetFlags() == TF && "Inconsistent target flag encountered" ) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1152, __extension__ __PRETTY_FUNCTION__)) | ||||
| 1152 | "Inconsistent target flag encountered")(static_cast <bool> (cast<ConstantPoolSDNode>(T)-> getTargetFlags() == TF && "Inconsistent target flag encountered" ) ? void (0) : __assert_fail ("cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF && \"Inconsistent target flag encountered\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1152, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1153 | |||||
| 1154 | if (IsPositionIndependent) | ||||
| 1155 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T); | ||||
| 1156 | return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T); | ||||
| 1157 | } | ||||
| 1158 | |||||
| 1159 | SDValue | ||||
| 1160 | HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1161 | EVT VT = Op.getValueType(); | ||||
| 1162 | int Idx = cast<JumpTableSDNode>(Op)->getIndex(); | ||||
| 1163 | if (isPositionIndependent()) { | ||||
| 1164 | SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL); | ||||
| 1165 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T); | ||||
| 1166 | } | ||||
| 1167 | |||||
| 1168 | SDValue T = DAG.getTargetJumpTable(Idx, VT); | ||||
| 1169 | return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T); | ||||
| 1170 | } | ||||
| 1171 | |||||
| 1172 | SDValue | ||||
| 1173 | HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1174 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||
| 1175 | MachineFunction &MF = DAG.getMachineFunction(); | ||||
| 1176 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||
| 1177 | MFI.setReturnAddressIsTaken(true); | ||||
| 1178 | |||||
| 1179 | if (verifyReturnAddressArgumentIsConstant(Op, DAG)) | ||||
| 1180 | return SDValue(); | ||||
| 1181 | |||||
| 1182 | EVT VT = Op.getValueType(); | ||||
| 1183 | SDLoc dl(Op); | ||||
| 1184 | unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); | ||||
| 1185 | if (Depth) { | ||||
| 1186 | SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); | ||||
| 1187 | SDValue Offset = DAG.getConstant(4, dl, MVT::i32); | ||||
| 1188 | return DAG.getLoad(VT, dl, DAG.getEntryNode(), | ||||
| 1189 | DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), | ||||
| 1190 | MachinePointerInfo()); | ||||
| 1191 | } | ||||
| 1192 | |||||
| 1193 | // Return LR, which contains the return address. Mark it an implicit live-in. | ||||
| 1194 | unsigned Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32)); | ||||
| 1195 | return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); | ||||
| 1196 | } | ||||
| 1197 | |||||
| 1198 | SDValue | ||||
| 1199 | HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1200 | const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo(); | ||||
| 1201 | MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); | ||||
| 1202 | MFI.setFrameAddressIsTaken(true); | ||||
| 1203 | |||||
| 1204 | EVT VT = Op.getValueType(); | ||||
| 1205 | SDLoc dl(Op); | ||||
| 1206 | unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); | ||||
| 1207 | SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, | ||||
| 1208 | HRI.getFrameRegister(), VT); | ||||
| 1209 | while (Depth--) | ||||
| 1210 | FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, | ||||
| 1211 | MachinePointerInfo()); | ||||
| 1212 | return FrameAddr; | ||||
| 1213 | } | ||||
| 1214 | |||||
| 1215 | SDValue | ||||
| 1216 | HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const { | ||||
| 1217 | SDLoc dl(Op); | ||||
| 1218 | return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0)); | ||||
| 1219 | } | ||||
| 1220 | |||||
| 1221 | SDValue | ||||
| 1222 | HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1223 | SDLoc dl(Op); | ||||
| 1224 | auto *GAN = cast<GlobalAddressSDNode>(Op); | ||||
| 1225 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 1226 | auto *GV = GAN->getGlobal(); | ||||
| 1227 | int64_t Offset = GAN->getOffset(); | ||||
| 1228 | |||||
| 1229 | auto &HLOF = *HTM.getObjFileLowering(); | ||||
| 1230 | Reloc::Model RM = HTM.getRelocationModel(); | ||||
| 1231 | |||||
| 1232 | if (RM == Reloc::Static) { | ||||
| 1233 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset); | ||||
| 1234 | const GlobalObject *GO = GV->getBaseObject(); | ||||
| 1235 | if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM)) | ||||
| 1236 | return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA); | ||||
| 1237 | return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA); | ||||
| 1238 | } | ||||
| 1239 | |||||
| 1240 | bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV); | ||||
| 1241 | if (UsePCRel) { | ||||
| 1242 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset, | ||||
| 1243 | HexagonII::MO_PCREL); | ||||
| 1244 | return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA); | ||||
| 1245 | } | ||||
| 1246 | |||||
| 1247 | // Use GOT index. | ||||
| 1248 | SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT); | ||||
| 1249 | SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT); | ||||
| 1250 | SDValue Off = DAG.getConstant(Offset, dl, MVT::i32); | ||||
| 1251 | return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off); | ||||
| 1252 | } | ||||
| 1253 | |||||
| 1254 | // Specifies that for loads and stores VT can be promoted to PromotedLdStVT. | ||||
| 1255 | SDValue | ||||
| 1256 | HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { | ||||
| 1257 | const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); | ||||
| 1258 | SDLoc dl(Op); | ||||
| 1259 | EVT PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 1260 | |||||
| 1261 | Reloc::Model RM = HTM.getRelocationModel(); | ||||
| 1262 | if (RM == Reloc::Static) { | ||||
| 1263 | SDValue A = DAG.getTargetBlockAddress(BA, PtrVT); | ||||
| 1264 | return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A); | ||||
| 1265 | } | ||||
| 1266 | |||||
| 1267 | SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL); | ||||
| 1268 | return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A); | ||||
| 1269 | } | ||||
| 1270 | |||||
| 1271 | SDValue | ||||
| 1272 | HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG) | ||||
| 1273 | const { | ||||
| 1274 | EVT PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 1275 | SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME"_GLOBAL_OFFSET_TABLE_", PtrVT, | ||||
| 1276 | HexagonII::MO_PCREL); | ||||
| 1277 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym); | ||||
| 1278 | } | ||||
| 1279 | |||||
| 1280 | SDValue | ||||
| 1281 | HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain, | ||||
| 1282 | GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg, | ||||
| 1283 | unsigned char OperandFlags) const { | ||||
| 1284 | MachineFunction &MF = DAG.getMachineFunction(); | ||||
| 1285 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||
| 1286 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||
| 1287 | SDLoc dl(GA); | ||||
| 1288 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, | ||||
| 1289 | GA->getValueType(0), | ||||
| 1290 | GA->getOffset(), | ||||
| 1291 | OperandFlags); | ||||
| 1292 | // Create Operands for the call.The Operands should have the following: | ||||
| 1293 | // 1. Chain SDValue | ||||
| 1294 | // 2. Callee which in this case is the Global address value. | ||||
| 1295 | // 3. Registers live into the call.In this case its R0, as we | ||||
| 1296 | // have just one argument to be passed. | ||||
| 1297 | // 4. Glue. | ||||
| 1298 | // Note: The order is important. | ||||
| 1299 | |||||
| 1300 | const auto &HRI = *Subtarget.getRegisterInfo(); | ||||
| 1301 | const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C); | ||||
| 1302 | assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention" ) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1302, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1303 | SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT), | ||||
| 1304 | DAG.getRegisterMask(Mask), Glue }; | ||||
| 1305 | Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops); | ||||
| 1306 | |||||
| 1307 | // Inform MFI that function has calls. | ||||
| 1308 | MFI.setAdjustsStack(true); | ||||
| 1309 | |||||
| 1310 | Glue = Chain.getValue(1); | ||||
| 1311 | return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue); | ||||
| 1312 | } | ||||
| 1313 | |||||
| 1314 | // | ||||
| 1315 | // Lower using the intial executable model for TLS addresses | ||||
| 1316 | // | ||||
| 1317 | SDValue | ||||
| 1318 | HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA, | ||||
| 1319 | SelectionDAG &DAG) const { | ||||
| 1320 | SDLoc dl(GA); | ||||
| 1321 | int64_t Offset = GA->getOffset(); | ||||
| 1322 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 1323 | |||||
| 1324 | // Get the thread pointer. | ||||
| 1325 | SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT); | ||||
| 1326 | |||||
| 1327 | bool IsPositionIndependent = isPositionIndependent(); | ||||
| 1328 | unsigned char TF = | ||||
| 1329 | IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE; | ||||
| 1330 | |||||
| 1331 | // First generate the TLS symbol address | ||||
| 1332 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, | ||||
| 1333 | Offset, TF); | ||||
| 1334 | |||||
| 1335 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | ||||
| 1336 | |||||
| 1337 | if (IsPositionIndependent) { | ||||
| 1338 | // Generate the GOT pointer in case of position independent code | ||||
| 1339 | SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG); | ||||
| 1340 | |||||
| 1341 | // Add the TLS Symbol address to GOT pointer.This gives | ||||
| 1342 | // GOT relative relocation for the symbol. | ||||
| 1343 | Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym); | ||||
| 1344 | } | ||||
| 1345 | |||||
| 1346 | // Load the offset value for TLS symbol.This offset is relative to | ||||
| 1347 | // thread pointer. | ||||
| 1348 | SDValue LoadOffset = | ||||
| 1349 | DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo()); | ||||
| 1350 | |||||
| 1351 | // Address of the thread local variable is the add of thread | ||||
| 1352 | // pointer and the offset of the variable. | ||||
| 1353 | return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset); | ||||
| 1354 | } | ||||
| 1355 | |||||
| 1356 | // | ||||
| 1357 | // Lower using the local executable model for TLS addresses | ||||
| 1358 | // | ||||
| 1359 | SDValue | ||||
| 1360 | HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA, | ||||
| 1361 | SelectionDAG &DAG) const { | ||||
| 1362 | SDLoc dl(GA); | ||||
| 1363 | int64_t Offset = GA->getOffset(); | ||||
| 1364 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 1365 | |||||
| 1366 | // Get the thread pointer. | ||||
| 1367 | SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT); | ||||
| 1368 | // Generate the TLS symbol address | ||||
| 1369 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset, | ||||
| 1370 | HexagonII::MO_TPREL); | ||||
| 1371 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | ||||
| 1372 | |||||
| 1373 | // Address of the thread local variable is the add of thread | ||||
| 1374 | // pointer and the offset of the variable. | ||||
| 1375 | return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym); | ||||
| 1376 | } | ||||
| 1377 | |||||
| 1378 | // | ||||
| 1379 | // Lower using the general dynamic model for TLS addresses | ||||
| 1380 | // | ||||
| 1381 | SDValue | ||||
| 1382 | HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA, | ||||
| 1383 | SelectionDAG &DAG) const { | ||||
| 1384 | SDLoc dl(GA); | ||||
| 1385 | int64_t Offset = GA->getOffset(); | ||||
| 1386 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 1387 | |||||
| 1388 | // First generate the TLS symbol address | ||||
| 1389 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset, | ||||
| 1390 | HexagonII::MO_GDGOT); | ||||
| 1391 | |||||
| 1392 | // Then, generate the GOT pointer | ||||
| 1393 | SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG); | ||||
| 1394 | |||||
| 1395 | // Add the TLS symbol and the GOT pointer | ||||
| 1396 | SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA); | ||||
| 1397 | SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym); | ||||
| 1398 | |||||
| 1399 | // Copy over the argument to R0 | ||||
| 1400 | SDValue InFlag; | ||||
| 1401 | Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag); | ||||
| 1402 | InFlag = Chain.getValue(1); | ||||
| 1403 | |||||
| 1404 | unsigned Flags = | ||||
| 1405 | static_cast<const HexagonSubtarget &>(DAG.getSubtarget()).useLongCalls() | ||||
| 1406 | ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended | ||||
| 1407 | : HexagonII::MO_GDPLT; | ||||
| 1408 | |||||
| 1409 | return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT, | ||||
| 1410 | Hexagon::R0, Flags); | ||||
| 1411 | } | ||||
| 1412 | |||||
| 1413 | // | ||||
| 1414 | // Lower TLS addresses. | ||||
| 1415 | // | ||||
| 1416 | // For now for dynamic models, we only support the general dynamic model. | ||||
| 1417 | // | ||||
| 1418 | SDValue | ||||
| 1419 | HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op, | ||||
| 1420 | SelectionDAG &DAG) const { | ||||
| 1421 | GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); | ||||
| 1422 | |||||
| 1423 | switch (HTM.getTLSModel(GA->getGlobal())) { | ||||
| 1424 | case TLSModel::GeneralDynamic: | ||||
| 1425 | case TLSModel::LocalDynamic: | ||||
| 1426 | return LowerToTLSGeneralDynamicModel(GA, DAG); | ||||
| 1427 | case TLSModel::InitialExec: | ||||
| 1428 | return LowerToTLSInitialExecModel(GA, DAG); | ||||
| 1429 | case TLSModel::LocalExec: | ||||
| 1430 | return LowerToTLSLocalExecModel(GA, DAG); | ||||
| 1431 | } | ||||
| 1432 | llvm_unreachable("Bogus TLS model")::llvm::llvm_unreachable_internal("Bogus TLS model", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1432); | ||||
| 1433 | } | ||||
| 1434 | |||||
| 1435 | //===----------------------------------------------------------------------===// | ||||
| 1436 | // TargetLowering Implementation | ||||
| 1437 | //===----------------------------------------------------------------------===// | ||||
| 1438 | |||||
| 1439 | HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, | ||||
| 1440 | const HexagonSubtarget &ST) | ||||
| 1441 | : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)), | ||||
| 1442 | Subtarget(ST) { | ||||
| 1443 | auto &HRI = *Subtarget.getRegisterInfo(); | ||||
| 1444 | |||||
| 1445 | setPrefLoopAlignment(Align(16)); | ||||
| 1446 | setMinFunctionAlignment(Align(4)); | ||||
| 1447 | setPrefFunctionAlignment(Align(16)); | ||||
| 1448 | setStackPointerRegisterToSaveRestore(HRI.getStackRegister()); | ||||
| 1449 | setBooleanContents(TargetLoweringBase::UndefinedBooleanContent); | ||||
| 1450 | setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent); | ||||
| 1451 | |||||
| 1452 | setMaxAtomicSizeInBitsSupported(64); | ||||
| 1453 | setMinCmpXchgSizeInBits(32); | ||||
| 1454 | |||||
| 1455 | if (EnableHexSDNodeSched) | ||||
| 1456 | setSchedulingPreference(Sched::VLIW); | ||||
| 1457 | else | ||||
| 1458 | setSchedulingPreference(Sched::Source); | ||||
| 1459 | |||||
| 1460 | // Limits for inline expansion of memcpy/memmove | ||||
| 1461 | MaxStoresPerMemcpy = MaxStoresPerMemcpyCL; | ||||
| 1462 | MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL; | ||||
| 1463 | MaxStoresPerMemmove = MaxStoresPerMemmoveCL; | ||||
| 1464 | MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL; | ||||
| 1465 | MaxStoresPerMemset = MaxStoresPerMemsetCL; | ||||
| 1466 | MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL; | ||||
| 1467 | |||||
| 1468 | // | ||||
| 1469 | // Set up register classes. | ||||
| 1470 | // | ||||
| 1471 | |||||
| 1472 | addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass); | ||||
| 1473 | addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass); // bbbbaaaa | ||||
| 1474 | addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass); // ddccbbaa | ||||
| 1475 | addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass); // hgfedcba | ||||
| 1476 | addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass); | ||||
| 1477 | addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass); | ||||
| 1478 | addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass); | ||||
| 1479 | addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass); | ||||
| 1480 | addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass); | ||||
| 1481 | addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass); | ||||
| 1482 | addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass); | ||||
| 1483 | |||||
| 1484 | addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass); | ||||
| 1485 | addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass); | ||||
| 1486 | |||||
| 1487 | // | ||||
| 1488 | // Handling of scalar operations. | ||||
| 1489 | // | ||||
| 1490 | // All operations default to "legal", except: | ||||
| 1491 | // - indexed loads and stores (pre-/post-incremented), | ||||
| 1492 | // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS, | ||||
| 1493 | // ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN, | ||||
| 1494 | // FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP, | ||||
| 1495 | // FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG, | ||||
| 1496 | // which default to "expand" for at least one type. | ||||
| 1497 | |||||
| 1498 | // Misc operations. | ||||
| 1499 | setOperationAction(ISD::ConstantFP, MVT::f32, Legal); | ||||
| 1500 | setOperationAction(ISD::ConstantFP, MVT::f64, Legal); | ||||
| 1501 | setOperationAction(ISD::TRAP, MVT::Other, Legal); | ||||
| 1502 | setOperationAction(ISD::ConstantPool, MVT::i32, Custom); | ||||
| 1503 | setOperationAction(ISD::JumpTable, MVT::i32, Custom); | ||||
| 1504 | setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); | ||||
| 1505 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); | ||||
| 1506 | setOperationAction(ISD::INLINEASM, MVT::Other, Custom); | ||||
| 1507 | setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom); | ||||
| 1508 | setOperationAction(ISD::PREFETCH, MVT::Other, Custom); | ||||
| 1509 | setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); | ||||
| 1510 | setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); | ||||
| 1511 | setOperationAction(ISD::EH_RETURN, MVT::Other, Custom); | ||||
| 1512 | setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom); | ||||
| 1513 | setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); | ||||
| 1514 | setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); | ||||
| 1515 | |||||
| 1516 | // Custom legalize GlobalAddress nodes into CONST32. | ||||
| 1517 | setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); | ||||
| 1518 | setOperationAction(ISD::GlobalAddress, MVT::i8, Custom); | ||||
| 1519 | setOperationAction(ISD::BlockAddress, MVT::i32, Custom); | ||||
| 1520 | |||||
| 1521 | // Hexagon needs to optimize cases with negative constants. | ||||
| 1522 | setOperationAction(ISD::SETCC, MVT::i8, Custom); | ||||
| 1523 | setOperationAction(ISD::SETCC, MVT::i16, Custom); | ||||
| 1524 | setOperationAction(ISD::SETCC, MVT::v4i8, Custom); | ||||
| 1525 | setOperationAction(ISD::SETCC, MVT::v2i16, Custom); | ||||
| 1526 | |||||
| 1527 | // VASTART needs to be custom lowered to use the VarArgsFrameIndex. | ||||
| 1528 | setOperationAction(ISD::VASTART, MVT::Other, Custom); | ||||
| 1529 | setOperationAction(ISD::VAEND, MVT::Other, Expand); | ||||
| 1530 | setOperationAction(ISD::VAARG, MVT::Other, Expand); | ||||
| 1531 | if (Subtarget.isEnvironmentMusl()) | ||||
| 1532 | setOperationAction(ISD::VACOPY, MVT::Other, Custom); | ||||
| 1533 | else | ||||
| 1534 | setOperationAction(ISD::VACOPY, MVT::Other, Expand); | ||||
| 1535 | |||||
| 1536 | setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); | ||||
| 1537 | setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); | ||||
| 1538 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); | ||||
| 1539 | |||||
| 1540 | if (EmitJumpTables) | ||||
| 1541 | setMinimumJumpTableEntries(MinimumJumpTables); | ||||
| 1542 | else | ||||
| 1543 | setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max()); | ||||
| 1544 | setOperationAction(ISD::BR_JT, MVT::Other, Expand); | ||||
| 1545 | |||||
| 1546 | for (unsigned LegalIntOp : | ||||
| 1547 | {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) { | ||||
| 1548 | setOperationAction(LegalIntOp, MVT::i32, Legal); | ||||
| 1549 | setOperationAction(LegalIntOp, MVT::i64, Legal); | ||||
| 1550 | } | ||||
| 1551 | |||||
| 1552 | // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit, | ||||
| 1553 | // but they only operate on i64. | ||||
| 1554 | for (MVT VT : MVT::integer_valuetypes()) { | ||||
| 1555 | setOperationAction(ISD::UADDO, VT, Custom); | ||||
| 1556 | setOperationAction(ISD::USUBO, VT, Custom); | ||||
| 1557 | setOperationAction(ISD::SADDO, VT, Expand); | ||||
| 1558 | setOperationAction(ISD::SSUBO, VT, Expand); | ||||
| 1559 | setOperationAction(ISD::ADDCARRY, VT, Expand); | ||||
| 1560 | setOperationAction(ISD::SUBCARRY, VT, Expand); | ||||
| 1561 | } | ||||
| 1562 | setOperationAction(ISD::ADDCARRY, MVT::i64, Custom); | ||||
| 1563 | setOperationAction(ISD::SUBCARRY, MVT::i64, Custom); | ||||
| 1564 | |||||
| 1565 | setOperationAction(ISD::CTLZ, MVT::i8, Promote); | ||||
| 1566 | setOperationAction(ISD::CTLZ, MVT::i16, Promote); | ||||
| 1567 | setOperationAction(ISD::CTTZ, MVT::i8, Promote); | ||||
| 1568 | setOperationAction(ISD::CTTZ, MVT::i16, Promote); | ||||
| 1569 | |||||
| 1570 | // Popcount can count # of 1s in i64 but returns i32. | ||||
| 1571 | setOperationAction(ISD::CTPOP, MVT::i8, Promote); | ||||
| 1572 | setOperationAction(ISD::CTPOP, MVT::i16, Promote); | ||||
| 1573 | setOperationAction(ISD::CTPOP, MVT::i32, Promote); | ||||
| 1574 | setOperationAction(ISD::CTPOP, MVT::i64, Legal); | ||||
| 1575 | |||||
| 1576 | setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); | ||||
| 1577 | setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); | ||||
| 1578 | setOperationAction(ISD::BSWAP, MVT::i32, Legal); | ||||
| 1579 | setOperationAction(ISD::BSWAP, MVT::i64, Legal); | ||||
| 1580 | |||||
| 1581 | setOperationAction(ISD::FSHL, MVT::i32, Legal); | ||||
| 1582 | setOperationAction(ISD::FSHL, MVT::i64, Legal); | ||||
| 1583 | setOperationAction(ISD::FSHR, MVT::i32, Legal); | ||||
| 1584 | setOperationAction(ISD::FSHR, MVT::i64, Legal); | ||||
| 1585 | |||||
| 1586 | for (unsigned IntExpOp : | ||||
| 1587 | {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, | ||||
| 1588 | ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR, | ||||
| 1589 | ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS, | ||||
| 1590 | ISD::SMUL_LOHI, ISD::UMUL_LOHI}) { | ||||
| 1591 | for (MVT VT : MVT::integer_valuetypes()) | ||||
| 1592 | setOperationAction(IntExpOp, VT, Expand); | ||||
| 1593 | } | ||||
| 1594 | |||||
| 1595 | for (unsigned FPExpOp : | ||||
| 1596 | {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS, | ||||
| 1597 | ISD::FPOW, ISD::FCOPYSIGN}) { | ||||
| 1598 | for (MVT VT : MVT::fp_valuetypes()) | ||||
| 1599 | setOperationAction(FPExpOp, VT, Expand); | ||||
| 1600 | } | ||||
| 1601 | |||||
| 1602 | // No extending loads from i32. | ||||
| 1603 | for (MVT VT : MVT::integer_valuetypes()) { | ||||
| 1604 | setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand); | ||||
| 1605 | setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); | ||||
| 1606 | setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand); | ||||
| 1607 | } | ||||
| 1608 | // Turn FP truncstore into trunc + store. | ||||
| 1609 | setTruncStoreAction(MVT::f64, MVT::f32, Expand); | ||||
| 1610 | // Turn FP extload into load/fpextend. | ||||
| 1611 | for (MVT VT : MVT::fp_valuetypes()) | ||||
| 1612 | setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); | ||||
| 1613 | |||||
| 1614 | // Expand BR_CC and SELECT_CC for all integer and fp types. | ||||
| 1615 | for (MVT VT : MVT::integer_valuetypes()) { | ||||
| 1616 | setOperationAction(ISD::BR_CC, VT, Expand); | ||||
| 1617 | setOperationAction(ISD::SELECT_CC, VT, Expand); | ||||
| 1618 | } | ||||
| 1619 | for (MVT VT : MVT::fp_valuetypes()) { | ||||
| 1620 | setOperationAction(ISD::BR_CC, VT, Expand); | ||||
| 1621 | setOperationAction(ISD::SELECT_CC, VT, Expand); | ||||
| 1622 | } | ||||
| 1623 | setOperationAction(ISD::BR_CC, MVT::Other, Expand); | ||||
| 1624 | |||||
| 1625 | // | ||||
| 1626 | // Handling of vector operations. | ||||
| 1627 | // | ||||
| 1628 | |||||
| 1629 | // Set the action for vector operations to "expand", then override it with | ||||
| 1630 | // either "custom" or "legal" for specific cases. | ||||
| 1631 | static const unsigned VectExpOps[] = { | ||||
| 1632 | // Integer arithmetic: | ||||
| 1633 | ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV, | ||||
| 1634 | ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO, | ||||
| 1635 | ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI, | ||||
| 1636 | // Logical/bit: | ||||
| 1637 | ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR, | ||||
| 1638 | ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, | ||||
| 1639 | // Floating point arithmetic/math functions: | ||||
| 1640 | ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV, | ||||
| 1641 | ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN, | ||||
| 1642 | ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2, | ||||
| 1643 | ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC, | ||||
| 1644 | ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR, | ||||
| 1645 | ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS, | ||||
| 1646 | // Misc: | ||||
| 1647 | ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool, | ||||
| 1648 | // Vector: | ||||
| 1649 | ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR, | ||||
| 1650 | ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT, | ||||
| 1651 | ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR, | ||||
| 1652 | ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE, | ||||
| 1653 | ISD::SPLAT_VECTOR, | ||||
| 1654 | }; | ||||
| 1655 | |||||
| 1656 | for (MVT VT : MVT::fixedlen_vector_valuetypes()) { | ||||
| 1657 | for (unsigned VectExpOp : VectExpOps) | ||||
| 1658 | setOperationAction(VectExpOp, VT, Expand); | ||||
| 1659 | |||||
| 1660 | // Expand all extending loads and truncating stores: | ||||
| 1661 | for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) { | ||||
| 1662 | if (TargetVT == VT) | ||||
| 1663 | continue; | ||||
| 1664 | setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand); | ||||
| 1665 | setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand); | ||||
| 1666 | setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand); | ||||
| 1667 | setTruncStoreAction(VT, TargetVT, Expand); | ||||
| 1668 | } | ||||
| 1669 | |||||
| 1670 | // Normalize all inputs to SELECT to be vectors of i32. | ||||
| 1671 | if (VT.getVectorElementType() != MVT::i32) { | ||||
| 1672 | MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32); | ||||
| 1673 | setOperationAction(ISD::SELECT, VT, Promote); | ||||
| 1674 | AddPromotedToType(ISD::SELECT, VT, VT32); | ||||
| 1675 | } | ||||
| 1676 | setOperationAction(ISD::SRA, VT, Custom); | ||||
| 1677 | setOperationAction(ISD::SHL, VT, Custom); | ||||
| 1678 | setOperationAction(ISD::SRL, VT, Custom); | ||||
| 1679 | } | ||||
| 1680 | |||||
| 1681 | // Extending loads from (native) vectors of i8 into (native) vectors of i16 | ||||
| 1682 | // are legal. | ||||
| 1683 | setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | ||||
| 1684 | setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | ||||
| 1685 | setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal); | ||||
| 1686 | setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | ||||
| 1687 | setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | ||||
| 1688 | setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal); | ||||
| 1689 | |||||
| 1690 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal); | ||||
| 1691 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal); | ||||
| 1692 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); | ||||
| 1693 | |||||
| 1694 | // Types natively supported: | ||||
| 1695 | for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8, | ||||
| 1696 | MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) { | ||||
| 1697 | setOperationAction(ISD::BUILD_VECTOR, NativeVT, Custom); | ||||
| 1698 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom); | ||||
| 1699 | setOperationAction(ISD::INSERT_VECTOR_ELT, NativeVT, Custom); | ||||
| 1700 | setOperationAction(ISD::EXTRACT_SUBVECTOR, NativeVT, Custom); | ||||
| 1701 | setOperationAction(ISD::INSERT_SUBVECTOR, NativeVT, Custom); | ||||
| 1702 | setOperationAction(ISD::CONCAT_VECTORS, NativeVT, Custom); | ||||
| 1703 | |||||
| 1704 | setOperationAction(ISD::ADD, NativeVT, Legal); | ||||
| 1705 | setOperationAction(ISD::SUB, NativeVT, Legal); | ||||
| 1706 | setOperationAction(ISD::MUL, NativeVT, Legal); | ||||
| 1707 | setOperationAction(ISD::AND, NativeVT, Legal); | ||||
| 1708 | setOperationAction(ISD::OR, NativeVT, Legal); | ||||
| 1709 | setOperationAction(ISD::XOR, NativeVT, Legal); | ||||
| 1710 | |||||
| 1711 | if (NativeVT.getVectorElementType() != MVT::i1) | ||||
| 1712 | setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal); | ||||
| 1713 | } | ||||
| 1714 | |||||
| 1715 | for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) { | ||||
| 1716 | setOperationAction(ISD::SMIN, VT, Legal); | ||||
| 1717 | setOperationAction(ISD::SMAX, VT, Legal); | ||||
| 1718 | setOperationAction(ISD::UMIN, VT, Legal); | ||||
| 1719 | setOperationAction(ISD::UMAX, VT, Legal); | ||||
| 1720 | } | ||||
| 1721 | |||||
| 1722 | // Custom lower unaligned loads. | ||||
| 1723 | // Also, for both loads and stores, verify the alignment of the address | ||||
| 1724 | // in case it is a compile-time constant. This is a usability feature to | ||||
| 1725 | // provide a meaningful error message to users. | ||||
| 1726 | for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8, | ||||
| 1727 | MVT::v2i16, MVT::v4i16, MVT::v2i32}) { | ||||
| 1728 | setOperationAction(ISD::LOAD, VT, Custom); | ||||
| 1729 | setOperationAction(ISD::STORE, VT, Custom); | ||||
| 1730 | } | ||||
| 1731 | |||||
| 1732 | // Custom-lower load/stores of boolean vectors. | ||||
| 1733 | for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) { | ||||
| 1734 | setOperationAction(ISD::LOAD, VT, Custom); | ||||
| 1735 | setOperationAction(ISD::STORE, VT, Custom); | ||||
| 1736 | } | ||||
| 1737 | |||||
| 1738 | for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16, | ||||
| 1739 | MVT::v2i32}) { | ||||
| 1740 | setCondCodeAction(ISD::SETNE, VT, Expand); | ||||
| 1741 | setCondCodeAction(ISD::SETLE, VT, Expand); | ||||
| 1742 | setCondCodeAction(ISD::SETGE, VT, Expand); | ||||
| 1743 | setCondCodeAction(ISD::SETLT, VT, Expand); | ||||
| 1744 | setCondCodeAction(ISD::SETULE, VT, Expand); | ||||
| 1745 | setCondCodeAction(ISD::SETUGE, VT, Expand); | ||||
| 1746 | setCondCodeAction(ISD::SETULT, VT, Expand); | ||||
| 1747 | } | ||||
| 1748 | |||||
| 1749 | // Custom-lower bitcasts from i8 to v8i1. | ||||
| 1750 | setOperationAction(ISD::BITCAST, MVT::i8, Custom); | ||||
| 1751 | setOperationAction(ISD::SETCC, MVT::v2i16, Custom); | ||||
| 1752 | setOperationAction(ISD::VSELECT, MVT::v4i8, Custom); | ||||
| 1753 | setOperationAction(ISD::VSELECT, MVT::v2i16, Custom); | ||||
| 1754 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom); | ||||
| 1755 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); | ||||
| 1756 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); | ||||
| 1757 | |||||
| 1758 | // V5+. | ||||
| 1759 | setOperationAction(ISD::FMA, MVT::f64, Expand); | ||||
| 1760 | setOperationAction(ISD::FADD, MVT::f64, Expand); | ||||
| 1761 | setOperationAction(ISD::FSUB, MVT::f64, Expand); | ||||
| 1762 | setOperationAction(ISD::FMUL, MVT::f64, Expand); | ||||
| 1763 | |||||
| 1764 | setOperationAction(ISD::FMINNUM, MVT::f32, Legal); | ||||
| 1765 | setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); | ||||
| 1766 | |||||
| 1767 | setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote); | ||||
| 1768 | setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote); | ||||
| 1769 | setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote); | ||||
| 1770 | setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote); | ||||
| 1771 | setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote); | ||||
| 1772 | setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote); | ||||
| 1773 | setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); | ||||
| 1774 | setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote); | ||||
| 1775 | setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote); | ||||
| 1776 | setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); | ||||
| 1777 | setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote); | ||||
| 1778 | setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote); | ||||
| 1779 | |||||
| 1780 | // Handling of indexed loads/stores: default is "expand". | ||||
| 1781 | // | ||||
| 1782 | for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64, | ||||
| 1783 | MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) { | ||||
| 1784 | setIndexedLoadAction(ISD::POST_INC, VT, Legal); | ||||
| 1785 | setIndexedStoreAction(ISD::POST_INC, VT, Legal); | ||||
| 1786 | } | ||||
| 1787 | |||||
| 1788 | // Subtarget-specific operation actions. | ||||
| 1789 | // | ||||
| 1790 | if (Subtarget.hasV60Ops()) { | ||||
| 1791 | setOperationAction(ISD::ROTL, MVT::i32, Legal); | ||||
| 1792 | setOperationAction(ISD::ROTL, MVT::i64, Legal); | ||||
| 1793 | setOperationAction(ISD::ROTR, MVT::i32, Legal); | ||||
| 1794 | setOperationAction(ISD::ROTR, MVT::i64, Legal); | ||||
| 1795 | } | ||||
| 1796 | if (Subtarget.hasV66Ops()) { | ||||
| 1797 | setOperationAction(ISD::FADD, MVT::f64, Legal); | ||||
| 1798 | setOperationAction(ISD::FSUB, MVT::f64, Legal); | ||||
| 1799 | } | ||||
| 1800 | if (Subtarget.hasV67Ops()) { | ||||
| 1801 | setOperationAction(ISD::FMINNUM, MVT::f64, Legal); | ||||
| 1802 | setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); | ||||
| 1803 | setOperationAction(ISD::FMUL, MVT::f64, Legal); | ||||
| 1804 | } | ||||
| 1805 | |||||
| 1806 | setTargetDAGCombine(ISD::VSELECT); | ||||
| 1807 | |||||
| 1808 | if (Subtarget.useHVXOps()) | ||||
| 1809 | initializeHVXLowering(); | ||||
| 1810 | |||||
| 1811 | computeRegisterProperties(&HRI); | ||||
| 1812 | |||||
| 1813 | // | ||||
| 1814 | // Library calls for unsupported operations | ||||
| 1815 | // | ||||
| 1816 | bool FastMath = EnableFastMath; | ||||
| 1817 | |||||
| 1818 | setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3"); | ||||
| 1819 | setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3"); | ||||
| 1820 | setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3"); | ||||
| 1821 | setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3"); | ||||
| 1822 | setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3"); | ||||
| 1823 | setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3"); | ||||
| 1824 | setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3"); | ||||
| 1825 | setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3"); | ||||
| 1826 | |||||
| 1827 | setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf"); | ||||
| 1828 | setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf"); | ||||
| 1829 | setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti"); | ||||
| 1830 | setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti"); | ||||
| 1831 | setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti"); | ||||
| 1832 | setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti"); | ||||
| 1833 | |||||
| 1834 | // This is the only fast library function for sqrtd. | ||||
| 1835 | if (FastMath) | ||||
| 1836 | setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2"); | ||||
| 1837 | |||||
| 1838 | // Prefix is: nothing for "slow-math", | ||||
| 1839 | // "fast2_" for V5+ fast-math double-precision | ||||
| 1840 | // (actually, keep fast-math and fast-math2 separate for now) | ||||
| 1841 | if (FastMath) { | ||||
| 1842 | setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3"); | ||||
| 1843 | setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3"); | ||||
| 1844 | setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3"); | ||||
| 1845 | setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3"); | ||||
| 1846 | setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3"); | ||||
| 1847 | } else { | ||||
| 1848 | setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3"); | ||||
| 1849 | setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3"); | ||||
| 1850 | setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3"); | ||||
| 1851 | setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3"); | ||||
| 1852 | setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3"); | ||||
| 1853 | } | ||||
| 1854 | |||||
| 1855 | if (FastMath) | ||||
| 1856 | setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf"); | ||||
| 1857 | else | ||||
| 1858 | setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf"); | ||||
| 1859 | |||||
| 1860 | // These cause problems when the shift amount is non-constant. | ||||
| 1861 | setLibcallName(RTLIB::SHL_I128, nullptr); | ||||
| 1862 | setLibcallName(RTLIB::SRL_I128, nullptr); | ||||
| 1863 | setLibcallName(RTLIB::SRA_I128, nullptr); | ||||
| 1864 | } | ||||
| 1865 | |||||
| 1866 | const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const { | ||||
| 1867 | switch ((HexagonISD::NodeType)Opcode) { | ||||
| 1868 | case HexagonISD::ADDC: return "HexagonISD::ADDC"; | ||||
| 1869 | case HexagonISD::SUBC: return "HexagonISD::SUBC"; | ||||
| 1870 | case HexagonISD::ALLOCA: return "HexagonISD::ALLOCA"; | ||||
| 1871 | case HexagonISD::AT_GOT: return "HexagonISD::AT_GOT"; | ||||
| 1872 | case HexagonISD::AT_PCREL: return "HexagonISD::AT_PCREL"; | ||||
| 1873 | case HexagonISD::BARRIER: return "HexagonISD::BARRIER"; | ||||
| 1874 | case HexagonISD::CALL: return "HexagonISD::CALL"; | ||||
| 1875 | case HexagonISD::CALLnr: return "HexagonISD::CALLnr"; | ||||
| 1876 | case HexagonISD::CALLR: return "HexagonISD::CALLR"; | ||||
| 1877 | case HexagonISD::COMBINE: return "HexagonISD::COMBINE"; | ||||
| 1878 | case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP"; | ||||
| 1879 | case HexagonISD::CONST32: return "HexagonISD::CONST32"; | ||||
| 1880 | case HexagonISD::CP: return "HexagonISD::CP"; | ||||
| 1881 | case HexagonISD::DCFETCH: return "HexagonISD::DCFETCH"; | ||||
| 1882 | case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN"; | ||||
| 1883 | case HexagonISD::TSTBIT: return "HexagonISD::TSTBIT"; | ||||
| 1884 | case HexagonISD::EXTRACTU: return "HexagonISD::EXTRACTU"; | ||||
| 1885 | case HexagonISD::INSERT: return "HexagonISD::INSERT"; | ||||
| 1886 | case HexagonISD::JT: return "HexagonISD::JT"; | ||||
| 1887 | case HexagonISD::RET_FLAG: return "HexagonISD::RET_FLAG"; | ||||
| 1888 | case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN"; | ||||
| 1889 | case HexagonISD::VASL: return "HexagonISD::VASL"; | ||||
| 1890 | case HexagonISD::VASR: return "HexagonISD::VASR"; | ||||
| 1891 | case HexagonISD::VLSR: return "HexagonISD::VLSR"; | ||||
| 1892 | case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW"; | ||||
| 1893 | case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0"; | ||||
| 1894 | case HexagonISD::VROR: return "HexagonISD::VROR"; | ||||
| 1895 | case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE"; | ||||
| 1896 | case HexagonISD::PTRUE: return "HexagonISD::PTRUE"; | ||||
| 1897 | case HexagonISD::PFALSE: return "HexagonISD::PFALSE"; | ||||
| 1898 | case HexagonISD::D2P: return "HexagonISD::D2P"; | ||||
| 1899 | case HexagonISD::P2D: return "HexagonISD::P2D"; | ||||
| 1900 | case HexagonISD::V2Q: return "HexagonISD::V2Q"; | ||||
| 1901 | case HexagonISD::Q2V: return "HexagonISD::Q2V"; | ||||
| 1902 | case HexagonISD::QCAT: return "HexagonISD::QCAT"; | ||||
| 1903 | case HexagonISD::QTRUE: return "HexagonISD::QTRUE"; | ||||
| 1904 | case HexagonISD::QFALSE: return "HexagonISD::QFALSE"; | ||||
| 1905 | case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST"; | ||||
| 1906 | case HexagonISD::VALIGN: return "HexagonISD::VALIGN"; | ||||
| 1907 | case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR"; | ||||
| 1908 | case HexagonISD::VPACKL: return "HexagonISD::VPACKL"; | ||||
| 1909 | case HexagonISD::VUNPACK: return "HexagonISD::VUNPACK"; | ||||
| 1910 | case HexagonISD::VUNPACKU: return "HexagonISD::VUNPACKU"; | ||||
| 1911 | case HexagonISD::ISEL: return "HexagonISD::ISEL"; | ||||
| 1912 | case HexagonISD::OP_END: break; | ||||
| 1913 | } | ||||
| 1914 | return nullptr; | ||||
| 1915 | } | ||||
| 1916 | |||||
| 1917 | bool | ||||
| 1918 | HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign, | ||||
| 1919 | const SDLoc &dl, SelectionDAG &DAG) const { | ||||
| 1920 | auto *CA = dyn_cast<ConstantSDNode>(Ptr); | ||||
| 1921 | if (!CA) | ||||
| 1922 | return true; | ||||
| 1923 | unsigned Addr = CA->getZExtValue(); | ||||
| 1924 | Align HaveAlign = | ||||
| 1925 | Addr != 0 ? Align(1ull << countTrailingZeros(Addr)) : NeedAlign; | ||||
| 1926 | if (HaveAlign >= NeedAlign) | ||||
| 1927 | return true; | ||||
| 1928 | |||||
| 1929 | static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind(); | ||||
| 1930 | |||||
| 1931 | struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo { | ||||
| 1932 | DiagnosticInfoMisalignedTrap(StringRef M) | ||||
| 1933 | : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {} | ||||
| 1934 | void print(DiagnosticPrinter &DP) const override { | ||||
| 1935 | DP << Msg; | ||||
| 1936 | } | ||||
| 1937 | static bool classof(const DiagnosticInfo *DI) { | ||||
| 1938 | return DI->getKind() == DK_MisalignedTrap; | ||||
| 1939 | } | ||||
| 1940 | StringRef Msg; | ||||
| 1941 | }; | ||||
| 1942 | |||||
| 1943 | std::string ErrMsg; | ||||
| 1944 | raw_string_ostream O(ErrMsg); | ||||
| 1945 | O << "Misaligned constant address: " << format_hex(Addr, 10) | ||||
| 1946 | << " has alignment " << HaveAlign.value() | ||||
| 1947 | << ", but the memory access requires " << NeedAlign.value(); | ||||
| 1948 | if (DebugLoc DL = dl.getDebugLoc()) | ||||
| 1949 | DL.print(O << ", at "); | ||||
| 1950 | O << ". The instruction has been replaced with a trap."; | ||||
| 1951 | |||||
| 1952 | DAG.getContext()->diagnose(DiagnosticInfoMisalignedTrap(O.str())); | ||||
| 1953 | return false; | ||||
| 1954 | } | ||||
| 1955 | |||||
| 1956 | SDValue | ||||
| 1957 | HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG) | ||||
| 1958 | const { | ||||
| 1959 | const SDLoc &dl(Op); | ||||
| 1960 | auto *LS = cast<LSBaseSDNode>(Op.getNode()); | ||||
| 1961 | assert(!LS->isIndexed() && "Not expecting indexed ops on constant address")(static_cast <bool> (!LS->isIndexed() && "Not expecting indexed ops on constant address" ) ? void (0) : __assert_fail ("!LS->isIndexed() && \"Not expecting indexed ops on constant address\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 1961, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1962 | |||||
| 1963 | SDValue Chain = LS->getChain(); | ||||
| 1964 | SDValue Trap = DAG.getNode(ISD::TRAP, dl, MVT::Other, Chain); | ||||
| 1965 | if (LS->getOpcode() == ISD::LOAD) | ||||
| 1966 | return DAG.getMergeValues({DAG.getUNDEF(ty(Op)), Trap}, dl); | ||||
| 1967 | return Trap; | ||||
| 1968 | } | ||||
| 1969 | |||||
| 1970 | // Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load | ||||
| 1971 | // intrinsic. | ||||
| 1972 | static bool isBrevLdIntrinsic(const Value *Inst) { | ||||
| 1973 | unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID(); | ||||
| 1974 | return (ID == Intrinsic::hexagon_L2_loadrd_pbr || | ||||
| 1975 | ID == Intrinsic::hexagon_L2_loadri_pbr || | ||||
| 1976 | ID == Intrinsic::hexagon_L2_loadrh_pbr || | ||||
| 1977 | ID == Intrinsic::hexagon_L2_loadruh_pbr || | ||||
| 1978 | ID == Intrinsic::hexagon_L2_loadrb_pbr || | ||||
| 1979 | ID == Intrinsic::hexagon_L2_loadrub_pbr); | ||||
| 1980 | } | ||||
| 1981 | |||||
| 1982 | // Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous | ||||
| 1983 | // instruction. So far we only handle bitcast, extract value and bit reverse | ||||
| 1984 | // load intrinsic instructions. Should we handle CGEP ? | ||||
| 1985 | static Value *getBrevLdObject(Value *V) { | ||||
| 1986 | if (Operator::getOpcode(V) == Instruction::ExtractValue || | ||||
| 1987 | Operator::getOpcode(V) == Instruction::BitCast) | ||||
| 1988 | V = cast<Operator>(V)->getOperand(0); | ||||
| 1989 | else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V)) | ||||
| 1990 | V = cast<Instruction>(V)->getOperand(0); | ||||
| 1991 | return V; | ||||
| 1992 | } | ||||
| 1993 | |||||
| 1994 | // Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or | ||||
| 1995 | // a back edge. If the back edge comes from the intrinsic itself, the incoming | ||||
| 1996 | // edge is returned. | ||||
| 1997 | static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) { | ||||
| 1998 | const BasicBlock *Parent = PN->getParent(); | ||||
| 1999 | int Idx = -1; | ||||
| 2000 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) { | ||||
| 2001 | BasicBlock *Blk = PN->getIncomingBlock(i); | ||||
| 2002 | // Determine if the back edge is originated from intrinsic. | ||||
| 2003 | if (Blk == Parent) { | ||||
| 2004 | Value *BackEdgeVal = PN->getIncomingValue(i); | ||||
| 2005 | Value *BaseVal; | ||||
| 2006 | // Loop over till we return the same Value or we hit the IntrBaseVal. | ||||
| 2007 | do { | ||||
| 2008 | BaseVal = BackEdgeVal; | ||||
| 2009 | BackEdgeVal = getBrevLdObject(BackEdgeVal); | ||||
| 2010 | } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal)); | ||||
| 2011 | // If the getBrevLdObject returns IntrBaseVal, we should return the | ||||
| 2012 | // incoming edge. | ||||
| 2013 | if (IntrBaseVal == BackEdgeVal) | ||||
| 2014 | continue; | ||||
| 2015 | Idx = i; | ||||
| 2016 | break; | ||||
| 2017 | } else // Set the node to incoming edge. | ||||
| 2018 | Idx = i; | ||||
| 2019 | } | ||||
| 2020 | assert(Idx >= 0 && "Unexpected index to incoming argument in PHI")(static_cast <bool> (Idx >= 0 && "Unexpected index to incoming argument in PHI" ) ? void (0) : __assert_fail ("Idx >= 0 && \"Unexpected index to incoming argument in PHI\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2020, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2021 | return PN->getIncomingValue(Idx); | ||||
| 2022 | } | ||||
| 2023 | |||||
| 2024 | // Bit-reverse Load Intrinsic: Figure out the underlying object the base | ||||
| 2025 | // pointer points to, for the bit-reverse load intrinsic. Setting this to | ||||
| 2026 | // memoperand might help alias analysis to figure out the dependencies. | ||||
| 2027 | static Value *getUnderLyingObjectForBrevLdIntr(Value *V) { | ||||
| 2028 | Value *IntrBaseVal = V; | ||||
| 2029 | Value *BaseVal; | ||||
| 2030 | // Loop over till we return the same Value, implies we either figure out | ||||
| 2031 | // the object or we hit a PHI | ||||
| 2032 | do { | ||||
| 2033 | BaseVal = V; | ||||
| 2034 | V = getBrevLdObject(V); | ||||
| 2035 | } while (BaseVal != V); | ||||
| 2036 | |||||
| 2037 | // Identify the object from PHINode. | ||||
| 2038 | if (const PHINode *PN = dyn_cast<PHINode>(V)) | ||||
| 2039 | return returnEdge(PN, IntrBaseVal); | ||||
| 2040 | // For non PHI nodes, the object is the last value returned by getBrevLdObject | ||||
| 2041 | else | ||||
| 2042 | return V; | ||||
| 2043 | } | ||||
| 2044 | |||||
| 2045 | /// Given an intrinsic, checks if on the target the intrinsic will need to map | ||||
| 2046 | /// to a MemIntrinsicNode (touches memory). If this is the case, it returns | ||||
| 2047 | /// true and store the intrinsic information into the IntrinsicInfo that was | ||||
| 2048 | /// passed to the function. | ||||
| 2049 | bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, | ||||
| 2050 | const CallInst &I, | ||||
| 2051 | MachineFunction &MF, | ||||
| 2052 | unsigned Intrinsic) const { | ||||
| 2053 | switch (Intrinsic) { | ||||
| 2054 | case Intrinsic::hexagon_L2_loadrd_pbr: | ||||
| 2055 | case Intrinsic::hexagon_L2_loadri_pbr: | ||||
| 2056 | case Intrinsic::hexagon_L2_loadrh_pbr: | ||||
| 2057 | case Intrinsic::hexagon_L2_loadruh_pbr: | ||||
| 2058 | case Intrinsic::hexagon_L2_loadrb_pbr: | ||||
| 2059 | case Intrinsic::hexagon_L2_loadrub_pbr: { | ||||
| 2060 | Info.opc = ISD::INTRINSIC_W_CHAIN; | ||||
| 2061 | auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); | ||||
| 2062 | auto &Cont = I.getCalledFunction()->getParent()->getContext(); | ||||
| 2063 | // The intrinsic function call is of the form { ElTy, i8* } | ||||
| 2064 | // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type | ||||
| 2065 | // should be derived from ElTy. | ||||
| 2066 | Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0); | ||||
| 2067 | Info.memVT = MVT::getVT(ElTy); | ||||
| 2068 | llvm::Value *BasePtrVal = I.getOperand(0); | ||||
| 2069 | Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal); | ||||
| 2070 | // The offset value comes through Modifier register. For now, assume the | ||||
| 2071 | // offset is 0. | ||||
| 2072 | Info.offset = 0; | ||||
| 2073 | Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont)); | ||||
| 2074 | Info.flags = MachineMemOperand::MOLoad; | ||||
| 2075 | return true; | ||||
| 2076 | } | ||||
| 2077 | case Intrinsic::hexagon_V6_vgathermw: | ||||
| 2078 | case Intrinsic::hexagon_V6_vgathermw_128B: | ||||
| 2079 | case Intrinsic::hexagon_V6_vgathermh: | ||||
| 2080 | case Intrinsic::hexagon_V6_vgathermh_128B: | ||||
| 2081 | case Intrinsic::hexagon_V6_vgathermhw: | ||||
| 2082 | case Intrinsic::hexagon_V6_vgathermhw_128B: | ||||
| 2083 | case Intrinsic::hexagon_V6_vgathermwq: | ||||
| 2084 | case Intrinsic::hexagon_V6_vgathermwq_128B: | ||||
| 2085 | case Intrinsic::hexagon_V6_vgathermhq: | ||||
| 2086 | case Intrinsic::hexagon_V6_vgathermhq_128B: | ||||
| 2087 | case Intrinsic::hexagon_V6_vgathermhwq: | ||||
| 2088 | case Intrinsic::hexagon_V6_vgathermhwq_128B: { | ||||
| 2089 | const Module &M = *I.getParent()->getParent()->getParent(); | ||||
| 2090 | Info.opc = ISD::INTRINSIC_W_CHAIN; | ||||
| 2091 | Type *VecTy = I.getArgOperand(1)->getType(); | ||||
| 2092 | Info.memVT = MVT::getVT(VecTy); | ||||
| 2093 | Info.ptrVal = I.getArgOperand(0); | ||||
| 2094 | Info.offset = 0; | ||||
| 2095 | Info.align = | ||||
| 2096 | MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8); | ||||
| 2097 | Info.flags = MachineMemOperand::MOLoad | | ||||
| 2098 | MachineMemOperand::MOStore | | ||||
| 2099 | MachineMemOperand::MOVolatile; | ||||
| 2100 | return true; | ||||
| 2101 | } | ||||
| 2102 | default: | ||||
| 2103 | break; | ||||
| 2104 | } | ||||
| 2105 | return false; | ||||
| 2106 | } | ||||
| 2107 | |||||
| 2108 | bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const { | ||||
| 2109 | return X.getValueType().isScalarInteger(); // 'tstbit' | ||||
| 2110 | } | ||||
| 2111 | |||||
| 2112 | bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { | ||||
| 2113 | return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2)); | ||||
| 2114 | } | ||||
| 2115 | |||||
| 2116 | bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { | ||||
| 2117 | if (!VT1.isSimple() || !VT2.isSimple()) | ||||
| 2118 | return false; | ||||
| 2119 | return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32; | ||||
| 2120 | } | ||||
| 2121 | |||||
| 2122 | bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd( | ||||
| 2123 | const MachineFunction &MF, EVT VT) const { | ||||
| 2124 | return isOperationLegalOrCustom(ISD::FMA, VT); | ||||
| 2125 | } | ||||
| 2126 | |||||
| 2127 | // Should we expand the build vector with shuffles? | ||||
| 2128 | bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT, | ||||
| 2129 | unsigned DefinedValues) const { | ||||
| 2130 | return false; | ||||
| 2131 | } | ||||
| 2132 | |||||
| 2133 | bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, | ||||
| 2134 | EVT VT) const { | ||||
| 2135 | return true; | ||||
| 2136 | } | ||||
| 2137 | |||||
| 2138 | TargetLoweringBase::LegalizeTypeAction | ||||
| 2139 | HexagonTargetLowering::getPreferredVectorAction(MVT VT) const { | ||||
| 2140 | unsigned VecLen = VT.getVectorMinNumElements(); | ||||
| 2141 | MVT ElemTy = VT.getVectorElementType(); | ||||
| 2142 | |||||
| 2143 | if (VecLen == 1 || VT.isScalableVector()) | ||||
| 2144 | return TargetLoweringBase::TypeScalarizeVector; | ||||
| 2145 | |||||
| 2146 | if (Subtarget.useHVXOps()) { | ||||
| 2147 | unsigned Action = getPreferredHvxVectorAction(VT); | ||||
| 2148 | if (Action != ~0u) | ||||
| 2149 | return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action); | ||||
| 2150 | } | ||||
| 2151 | |||||
| 2152 | // Always widen (remaining) vectors of i1. | ||||
| 2153 | if (ElemTy == MVT::i1) | ||||
| 2154 | return TargetLoweringBase::TypeWidenVector; | ||||
| 2155 | |||||
| 2156 | return TargetLoweringBase::TypeSplitVector; | ||||
| 2157 | } | ||||
| 2158 | |||||
| 2159 | std::pair<SDValue, int> | ||||
| 2160 | HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const { | ||||
| 2161 | if (Addr.getOpcode() == ISD::ADD) { | ||||
| 2162 | SDValue Op1 = Addr.getOperand(1); | ||||
| 2163 | if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode())) | ||||
| 2164 | return { Addr.getOperand(0), CN->getSExtValue() }; | ||||
| 2165 | } | ||||
| 2166 | return { Addr, 0 }; | ||||
| 2167 | } | ||||
| 2168 | |||||
| 2169 | // Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors | ||||
| 2170 | // to select data from, V3 is the permutation. | ||||
| 2171 | SDValue | ||||
| 2172 | HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) | ||||
| 2173 | const { | ||||
| 2174 | const auto *SVN = cast<ShuffleVectorSDNode>(Op); | ||||
| 2175 | ArrayRef<int> AM = SVN->getMask(); | ||||
| 2176 | assert(AM.size() <= 8 && "Unexpected shuffle mask")(static_cast <bool> (AM.size() <= 8 && "Unexpected shuffle mask" ) ? void (0) : __assert_fail ("AM.size() <= 8 && \"Unexpected shuffle mask\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2176, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2177 | unsigned VecLen = AM.size(); | ||||
| 2178 | |||||
| 2179 | MVT VecTy = ty(Op); | ||||
| 2180 | assert(!Subtarget.isHVXVectorType(VecTy, true) &&(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true ) && "HVX shuffles should be legal") ? void (0) : __assert_fail ("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2181, __extension__ __PRETTY_FUNCTION__)) | ||||
| 2181 | "HVX shuffles should be legal")(static_cast <bool> (!Subtarget.isHVXVectorType(VecTy, true ) && "HVX shuffles should be legal") ? void (0) : __assert_fail ("!Subtarget.isHVXVectorType(VecTy, true) && \"HVX shuffles should be legal\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2181, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2182 | assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length")(static_cast <bool> (VecTy.getSizeInBits() <= 64 && "Unexpected vector length") ? void (0) : __assert_fail ("VecTy.getSizeInBits() <= 64 && \"Unexpected vector length\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2182, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2183 | |||||
| 2184 | SDValue Op0 = Op.getOperand(0); | ||||
| 2185 | SDValue Op1 = Op.getOperand(1); | ||||
| 2186 | const SDLoc &dl(Op); | ||||
| 2187 | |||||
| 2188 | // If the inputs are not the same as the output, bail. This is not an | ||||
| 2189 | // error situation, but complicates the handling and the default expansion | ||||
| 2190 | // (into BUILD_VECTOR) should be adequate. | ||||
| 2191 | if (ty(Op0) != VecTy || ty(Op1) != VecTy) | ||||
| 2192 | return SDValue(); | ||||
| 2193 | |||||
| 2194 | // Normalize the mask so that the first non-negative index comes from | ||||
| 2195 | // the first operand. | ||||
| 2196 | SmallVector<int,8> Mask(AM.begin(), AM.end()); | ||||
| 2197 | unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data(); | ||||
| 2198 | if (F == AM.size()) | ||||
| 2199 | return DAG.getUNDEF(VecTy); | ||||
| 2200 | if (AM[F] >= int(VecLen)) { | ||||
| 2201 | ShuffleVectorSDNode::commuteMask(Mask); | ||||
| 2202 | std::swap(Op0, Op1); | ||||
| 2203 | } | ||||
| 2204 | |||||
| 2205 | // Express the shuffle mask in terms of bytes. | ||||
| 2206 | SmallVector<int,8> ByteMask; | ||||
| 2207 | unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8; | ||||
| 2208 | for (unsigned i = 0, e = Mask.size(); i != e; ++i) { | ||||
| 2209 | int M = Mask[i]; | ||||
| 2210 | if (M < 0) { | ||||
| 2211 | for (unsigned j = 0; j != ElemBytes; ++j) | ||||
| 2212 | ByteMask.push_back(-1); | ||||
| 2213 | } else { | ||||
| 2214 | for (unsigned j = 0; j != ElemBytes; ++j) | ||||
| 2215 | ByteMask.push_back(M*ElemBytes + j); | ||||
| 2216 | } | ||||
| 2217 | } | ||||
| 2218 | assert(ByteMask.size() <= 8)(static_cast <bool> (ByteMask.size() <= 8) ? void (0 ) : __assert_fail ("ByteMask.size() <= 8", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2218, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2219 | |||||
| 2220 | // All non-undef (non-negative) indexes are well within [0..127], so they | ||||
| 2221 | // fit in a single byte. Build two 64-bit words: | ||||
| 2222 | // - MaskIdx where each byte is the corresponding index (for non-negative | ||||
| 2223 | // indexes), and 0xFF for negative indexes, and | ||||
| 2224 | // - MaskUnd that has 0xFF for each negative index. | ||||
| 2225 | uint64_t MaskIdx = 0; | ||||
| 2226 | uint64_t MaskUnd = 0; | ||||
| 2227 | for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) { | ||||
| 2228 | unsigned S = 8*i; | ||||
| 2229 | uint64_t M = ByteMask[i] & 0xFF; | ||||
| 2230 | if (M == 0xFF) | ||||
| 2231 | MaskUnd |= M << S; | ||||
| 2232 | MaskIdx |= M << S; | ||||
| 2233 | } | ||||
| 2234 | |||||
| 2235 | if (ByteMask.size() == 4) { | ||||
| 2236 | // Identity. | ||||
| 2237 | if (MaskIdx == (0x03020100 | MaskUnd)) | ||||
| 2238 | return Op0; | ||||
| 2239 | // Byte swap. | ||||
| 2240 | if (MaskIdx == (0x00010203 | MaskUnd)) { | ||||
| 2241 | SDValue T0 = DAG.getBitcast(MVT::i32, Op0); | ||||
| 2242 | SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0); | ||||
| 2243 | return DAG.getBitcast(VecTy, T1); | ||||
| 2244 | } | ||||
| 2245 | |||||
| 2246 | // Byte packs. | ||||
| 2247 | SDValue Concat10 = DAG.getNode(HexagonISD::COMBINE, dl, | ||||
| 2248 | typeJoin({ty(Op1), ty(Op0)}), {Op1, Op0}); | ||||
| 2249 | if (MaskIdx == (0x06040200 | MaskUnd)) | ||||
| 2250 | return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG); | ||||
| 2251 | if (MaskIdx == (0x07050301 | MaskUnd)) | ||||
| 2252 | return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG); | ||||
| 2253 | |||||
| 2254 | SDValue Concat01 = DAG.getNode(HexagonISD::COMBINE, dl, | ||||
| 2255 | typeJoin({ty(Op0), ty(Op1)}), {Op0, Op1}); | ||||
| 2256 | if (MaskIdx == (0x02000604 | MaskUnd)) | ||||
| 2257 | return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG); | ||||
| 2258 | if (MaskIdx == (0x03010705 | MaskUnd)) | ||||
| 2259 | return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG); | ||||
| 2260 | } | ||||
| 2261 | |||||
| 2262 | if (ByteMask.size() == 8) { | ||||
| 2263 | // Identity. | ||||
| 2264 | if (MaskIdx == (0x0706050403020100ull | MaskUnd)) | ||||
| 2265 | return Op0; | ||||
| 2266 | // Byte swap. | ||||
| 2267 | if (MaskIdx == (0x0001020304050607ull | MaskUnd)) { | ||||
| 2268 | SDValue T0 = DAG.getBitcast(MVT::i64, Op0); | ||||
| 2269 | SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0); | ||||
| 2270 | return DAG.getBitcast(VecTy, T1); | ||||
| 2271 | } | ||||
| 2272 | |||||
| 2273 | // Halfword picks. | ||||
| 2274 | if (MaskIdx == (0x0d0c050409080100ull | MaskUnd)) | ||||
| 2275 | return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG); | ||||
| 2276 | if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd)) | ||||
| 2277 | return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG); | ||||
| 2278 | if (MaskIdx == (0x0d0c090805040100ull | MaskUnd)) | ||||
| 2279 | return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG); | ||||
| 2280 | if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd)) | ||||
| 2281 | return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG); | ||||
| 2282 | if (MaskIdx == (0x0706030205040100ull | MaskUnd)) { | ||||
| 2283 | VectorPair P = opSplit(Op0, dl, DAG); | ||||
| 2284 | return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG); | ||||
| 2285 | } | ||||
| 2286 | |||||
| 2287 | // Byte packs. | ||||
| 2288 | if (MaskIdx == (0x0e060c040a020800ull | MaskUnd)) | ||||
| 2289 | return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG); | ||||
| 2290 | if (MaskIdx == (0x0f070d050b030901ull | MaskUnd)) | ||||
| 2291 | return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG); | ||||
| 2292 | } | ||||
| 2293 | |||||
| 2294 | return SDValue(); | ||||
| 2295 | } | ||||
| 2296 | |||||
| 2297 | // Create a Hexagon-specific node for shifting a vector by an integer. | ||||
| 2298 | SDValue | ||||
| 2299 | HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) | ||||
| 2300 | const { | ||||
| 2301 | unsigned NewOpc; | ||||
| 2302 | switch (Op.getOpcode()) { | ||||
| 2303 | case ISD::SHL: | ||||
| 2304 | NewOpc = HexagonISD::VASL; | ||||
| 2305 | break; | ||||
| 2306 | case ISD::SRA: | ||||
| 2307 | NewOpc = HexagonISD::VASR; | ||||
| 2308 | break; | ||||
| 2309 | case ISD::SRL: | ||||
| 2310 | NewOpc = HexagonISD::VLSR; | ||||
| 2311 | break; | ||||
| 2312 | default: | ||||
| 2313 | llvm_unreachable("Unexpected shift opcode")::llvm::llvm_unreachable_internal("Unexpected shift opcode", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2313); | ||||
| 2314 | } | ||||
| 2315 | |||||
| 2316 | SDValue Op0 = Op.getOperand(0); | ||||
| 2317 | SDValue Op1 = Op.getOperand(1); | ||||
| 2318 | const SDLoc &dl(Op); | ||||
| 2319 | |||||
| 2320 | switch (Op1.getOpcode()) { | ||||
| 2321 | case ISD::BUILD_VECTOR: | ||||
| 2322 | if (SDValue S = cast<BuildVectorSDNode>(Op1)->getSplatValue()) | ||||
| 2323 | return DAG.getNode(NewOpc, dl, ty(Op), Op0, S); | ||||
| 2324 | break; | ||||
| 2325 | case ISD::SPLAT_VECTOR: | ||||
| 2326 | return DAG.getNode(NewOpc, dl, ty(Op), Op0, Op1.getOperand(0)); | ||||
| 2327 | } | ||||
| 2328 | return SDValue(); | ||||
| 2329 | } | ||||
| 2330 | |||||
| 2331 | SDValue | ||||
| 2332 | HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { | ||||
| 2333 | return getVectorShiftByInt(Op, DAG); | ||||
| 2334 | } | ||||
| 2335 | |||||
| 2336 | SDValue | ||||
| 2337 | HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const { | ||||
| 2338 | if (isa<ConstantSDNode>(Op.getOperand(1).getNode())) | ||||
| 2339 | return Op; | ||||
| 2340 | return SDValue(); | ||||
| 2341 | } | ||||
| 2342 | |||||
| 2343 | SDValue | ||||
| 2344 | HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { | ||||
| 2345 | MVT ResTy = ty(Op); | ||||
| 2346 | SDValue InpV = Op.getOperand(0); | ||||
| 2347 | MVT InpTy = ty(InpV); | ||||
| 2348 | assert(ResTy.getSizeInBits() == InpTy.getSizeInBits())(static_cast <bool> (ResTy.getSizeInBits() == InpTy.getSizeInBits ()) ? void (0) : __assert_fail ("ResTy.getSizeInBits() == InpTy.getSizeInBits()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2348, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2349 | const SDLoc &dl(Op); | ||||
| 2350 | |||||
| 2351 | // Handle conversion from i8 to v8i1. | ||||
| 2352 | if (InpTy == MVT::i8) { | ||||
| 2353 | if (ResTy == MVT::v8i1) { | ||||
| 2354 | SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV); | ||||
| 2355 | SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32); | ||||
| 2356 | return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG); | ||||
| 2357 | } | ||||
| 2358 | return SDValue(); | ||||
| 2359 | } | ||||
| 2360 | |||||
| 2361 | return Op; | ||||
| 2362 | } | ||||
| 2363 | |||||
| 2364 | bool | ||||
| 2365 | HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values, | ||||
| 2366 | MVT VecTy, SelectionDAG &DAG, | ||||
| 2367 | MutableArrayRef<ConstantInt*> Consts) const { | ||||
| 2368 | MVT ElemTy = VecTy.getVectorElementType(); | ||||
| 2369 | unsigned ElemWidth = ElemTy.getSizeInBits(); | ||||
| 2370 | IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth); | ||||
| 2371 | bool AllConst = true; | ||||
| 2372 | |||||
| 2373 | for (unsigned i = 0, e = Values.size(); i != e; ++i) { | ||||
| 2374 | SDValue V = Values[i]; | ||||
| 2375 | if (V.isUndef()) { | ||||
| 2376 | Consts[i] = ConstantInt::get(IntTy, 0); | ||||
| 2377 | continue; | ||||
| 2378 | } | ||||
| 2379 | // Make sure to always cast to IntTy. | ||||
| 2380 | if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) { | ||||
| 2381 | const ConstantInt *CI = CN->getConstantIntValue(); | ||||
| 2382 | Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue()); | ||||
| 2383 | } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) { | ||||
| 2384 | const ConstantFP *CF = CN->getConstantFPValue(); | ||||
| 2385 | APInt A = CF->getValueAPF().bitcastToAPInt(); | ||||
| 2386 | Consts[i] = ConstantInt::get(IntTy, A.getZExtValue()); | ||||
| 2387 | } else { | ||||
| 2388 | AllConst = false; | ||||
| 2389 | } | ||||
| 2390 | } | ||||
| 2391 | return AllConst; | ||||
| 2392 | } | ||||
| 2393 | |||||
| 2394 | SDValue | ||||
| 2395 | HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl, | ||||
| 2396 | MVT VecTy, SelectionDAG &DAG) const { | ||||
| 2397 | MVT ElemTy = VecTy.getVectorElementType(); | ||||
| 2398 | assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem .size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2398, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2399 | |||||
| 2400 | SmallVector<ConstantInt*,4> Consts(Elem.size()); | ||||
| 2401 | bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); | ||||
| 2402 | |||||
| 2403 | unsigned First, Num = Elem.size(); | ||||
| 2404 | for (First = 0; First != Num; ++First) { | ||||
| 2405 | if (!isUndef(Elem[First])) | ||||
| 2406 | break; | ||||
| 2407 | } | ||||
| 2408 | if (First == Num) | ||||
| 2409 | return DAG.getUNDEF(VecTy); | ||||
| 2410 | |||||
| 2411 | if (AllConst && | ||||
| 2412 | llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); })) | ||||
| 2413 | return getZero(dl, VecTy, DAG); | ||||
| 2414 | |||||
| 2415 | if (ElemTy == MVT::i16) { | ||||
| 2416 | assert(Elem.size() == 2)(static_cast <bool> (Elem.size() == 2) ? void (0) : __assert_fail ("Elem.size() == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2416, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2417 | if (AllConst) { | ||||
| 2418 | uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) | | ||||
| 2419 | Consts[1]->getZExtValue() << 16; | ||||
| 2420 | return DAG.getBitcast(MVT::v2i16, DAG.getConstant(V, dl, MVT::i32)); | ||||
| 2421 | } | ||||
| 2422 | SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, | ||||
| 2423 | {Elem[1], Elem[0]}, DAG); | ||||
| 2424 | return DAG.getBitcast(MVT::v2i16, N); | ||||
| 2425 | } | ||||
| 2426 | |||||
| 2427 | if (ElemTy == MVT::i8) { | ||||
| 2428 | // First try generating a constant. | ||||
| 2429 | if (AllConst) { | ||||
| 2430 | int32_t V = (Consts[0]->getZExtValue() & 0xFF) | | ||||
| 2431 | (Consts[1]->getZExtValue() & 0xFF) << 8 | | ||||
| 2432 | (Consts[1]->getZExtValue() & 0xFF) << 16 | | ||||
| 2433 | Consts[2]->getZExtValue() << 24; | ||||
| 2434 | return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32)); | ||||
| 2435 | } | ||||
| 2436 | |||||
| 2437 | // Then try splat. | ||||
| 2438 | bool IsSplat = true; | ||||
| 2439 | for (unsigned i = First+1; i != Num; ++i) { | ||||
| 2440 | if (Elem[i] == Elem[First] || isUndef(Elem[i])) | ||||
| 2441 | continue; | ||||
| 2442 | IsSplat = false; | ||||
| 2443 | break; | ||||
| 2444 | } | ||||
| 2445 | if (IsSplat) { | ||||
| 2446 | // Legalize the operand of SPLAT_VECTOR. | ||||
| 2447 | SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); | ||||
| 2448 | return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext); | ||||
| 2449 | } | ||||
| 2450 | |||||
| 2451 | // Generate | ||||
| 2452 | // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) | | ||||
| 2453 | // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16 | ||||
| 2454 | assert(Elem.size() == 4)(static_cast <bool> (Elem.size() == 4) ? void (0) : __assert_fail ("Elem.size() == 4", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2454, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2455 | SDValue Vs[4]; | ||||
| 2456 | for (unsigned i = 0; i != 4; ++i) { | ||||
| 2457 | Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32); | ||||
| 2458 | Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8); | ||||
| 2459 | } | ||||
| 2460 | SDValue S8 = DAG.getConstant(8, dl, MVT::i32); | ||||
| 2461 | SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8}); | ||||
| 2462 | SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8}); | ||||
| 2463 | SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0}); | ||||
| 2464 | SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1}); | ||||
| 2465 | |||||
| 2466 | SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG); | ||||
| 2467 | return DAG.getBitcast(MVT::v4i8, R); | ||||
| 2468 | } | ||||
| 2469 | |||||
| 2470 | #ifndef NDEBUG | ||||
| 2471 | dbgs() << "VecTy: " << EVT(VecTy).getEVTString() << '\n'; | ||||
| 2472 | #endif | ||||
| 2473 | llvm_unreachable("Unexpected vector element type")::llvm::llvm_unreachable_internal("Unexpected vector element type" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2473); | ||||
| 2474 | } | ||||
| 2475 | |||||
| 2476 | SDValue | ||||
| 2477 | HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl, | ||||
| 2478 | MVT VecTy, SelectionDAG &DAG) const { | ||||
| 2479 | MVT ElemTy = VecTy.getVectorElementType(); | ||||
| 2480 | assert(VecTy.getVectorNumElements() == Elem.size())(static_cast <bool> (VecTy.getVectorNumElements() == Elem .size()) ? void (0) : __assert_fail ("VecTy.getVectorNumElements() == Elem.size()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2480, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2481 | |||||
| 2482 | SmallVector<ConstantInt*,8> Consts(Elem.size()); | ||||
| 2483 | bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts); | ||||
| 2484 | |||||
| 2485 | unsigned First, Num = Elem.size(); | ||||
| 2486 | for (First = 0; First != Num; ++First) { | ||||
| 2487 | if (!isUndef(Elem[First])) | ||||
| 2488 | break; | ||||
| 2489 | } | ||||
| 2490 | if (First == Num) | ||||
| 2491 | return DAG.getUNDEF(VecTy); | ||||
| 2492 | |||||
| 2493 | if (AllConst && | ||||
| 2494 | llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); })) | ||||
| 2495 | return getZero(dl, VecTy, DAG); | ||||
| 2496 | |||||
| 2497 | // First try splat if possible. | ||||
| 2498 | if (ElemTy == MVT::i16) { | ||||
| 2499 | bool IsSplat = true; | ||||
| 2500 | for (unsigned i = First+1; i != Num; ++i) { | ||||
| 2501 | if (Elem[i] == Elem[First] || isUndef(Elem[i])) | ||||
| 2502 | continue; | ||||
| 2503 | IsSplat = false; | ||||
| 2504 | break; | ||||
| 2505 | } | ||||
| 2506 | if (IsSplat) { | ||||
| 2507 | // Legalize the operand of SPLAT_VECTOR | ||||
| 2508 | SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32); | ||||
| 2509 | return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext); | ||||
| 2510 | } | ||||
| 2511 | } | ||||
| 2512 | |||||
| 2513 | // Then try constant. | ||||
| 2514 | if (AllConst) { | ||||
| 2515 | uint64_t Val = 0; | ||||
| 2516 | unsigned W = ElemTy.getSizeInBits(); | ||||
| 2517 | uint64_t Mask = (ElemTy == MVT::i8) ? 0xFFull | ||||
| 2518 | : (ElemTy == MVT::i16) ? 0xFFFFull : 0xFFFFFFFFull; | ||||
| 2519 | for (unsigned i = 0; i != Num; ++i) | ||||
| 2520 | Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask); | ||||
| 2521 | SDValue V0 = DAG.getConstant(Val, dl, MVT::i64); | ||||
| 2522 | return DAG.getBitcast(VecTy, V0); | ||||
| 2523 | } | ||||
| 2524 | |||||
| 2525 | // Build two 32-bit vectors and concatenate. | ||||
| 2526 | MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2); | ||||
| 2527 | SDValue L = (ElemTy == MVT::i32) | ||||
| 2528 | ? Elem[0] | ||||
| 2529 | : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG); | ||||
| 2530 | SDValue H = (ElemTy == MVT::i32) | ||||
| 2531 | ? Elem[1] | ||||
| 2532 | : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG); | ||||
| 2533 | return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, {H, L}); | ||||
| 2534 | } | ||||
| 2535 | |||||
| 2536 | SDValue | ||||
| 2537 | HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV, | ||||
| 2538 | const SDLoc &dl, MVT ValTy, MVT ResTy, | ||||
| 2539 | SelectionDAG &DAG) const { | ||||
| 2540 | MVT VecTy = ty(VecV); | ||||
| 2541 | assert(!ValTy.isVector() ||(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType () == ValTy.getVectorElementType()) ? void (0) : __assert_fail ("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2542, __extension__ __PRETTY_FUNCTION__)) | ||||
| 2542 | VecTy.getVectorElementType() == ValTy.getVectorElementType())(static_cast <bool> (!ValTy.isVector() || VecTy.getVectorElementType () == ValTy.getVectorElementType()) ? void (0) : __assert_fail ("!ValTy.isVector() || VecTy.getVectorElementType() == ValTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2542, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2543 | unsigned VecWidth = VecTy.getSizeInBits(); | ||||
| 2544 | unsigned ValWidth = ValTy.getSizeInBits(); | ||||
| 2545 | unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits(); | ||||
| 2546 | assert((VecWidth % ElemWidth) == 0)(static_cast <bool> ((VecWidth % ElemWidth) == 0) ? void (0) : __assert_fail ("(VecWidth % ElemWidth) == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2546, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2547 | auto *IdxN = dyn_cast<ConstantSDNode>(IdxV); | ||||
| 2548 | |||||
| 2549 | // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon | ||||
| 2550 | // without any coprocessors). | ||||
| 2551 | if (ElemWidth == 1) { | ||||
| 2552 | assert(VecWidth == VecTy.getVectorNumElements() && "Sanity failure")(static_cast <bool> (VecWidth == VecTy.getVectorNumElements () && "Sanity failure") ? void (0) : __assert_fail ("VecWidth == VecTy.getVectorNumElements() && \"Sanity failure\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2552, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2553 | assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2)(static_cast <bool> (VecWidth == 8 || VecWidth == 4 || VecWidth == 2) ? void (0) : __assert_fail ("VecWidth == 8 || VecWidth == 4 || VecWidth == 2" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2553, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2554 | // Check if this is an extract of the lowest bit. | ||||
| 2555 | if (IdxN) { | ||||
| 2556 | // Extracting the lowest bit is a no-op, but it changes the type, | ||||
| 2557 | // so it must be kept as an operation to avoid errors related to | ||||
| 2558 | // type mismatches. | ||||
| 2559 | if (IdxN->isNullValue() && ValTy.getSizeInBits() == 1) | ||||
| 2560 | return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV); | ||||
| 2561 | } | ||||
| 2562 | |||||
| 2563 | // If the value extracted is a single bit, use tstbit. | ||||
| 2564 | if (ValWidth == 1) { | ||||
| 2565 | SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG); | ||||
| 2566 | SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32); | ||||
| 2567 | SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0); | ||||
| 2568 | return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0); | ||||
| 2569 | } | ||||
| 2570 | |||||
| 2571 | // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in | ||||
| 2572 | // a predicate register. The elements of the vector are repeated | ||||
| 2573 | // in the register (if necessary) so that the total number is 8. | ||||
| 2574 | // The extracted subvector will need to be expanded in such a way. | ||||
| 2575 | unsigned Scale = VecWidth / ValWidth; | ||||
| 2576 | |||||
| 2577 | // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to | ||||
| 2578 | // position 0. | ||||
| 2579 | assert(ty(IdxV) == MVT::i32)(static_cast <bool> (ty(IdxV) == MVT::i32) ? void (0) : __assert_fail ("ty(IdxV) == MVT::i32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2579, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2580 | unsigned VecRep = 8 / VecWidth; | ||||
| 2581 | SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | ||||
| 2582 | DAG.getConstant(8*VecRep, dl, MVT::i32)); | ||||
| 2583 | SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV); | ||||
| 2584 | SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0); | ||||
| 2585 | while (Scale > 1) { | ||||
| 2586 | // The longest possible subvector is at most 32 bits, so it is always | ||||
| 2587 | // contained in the low subregister. | ||||
| 2588 | T1 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, T1); | ||||
| 2589 | T1 = expandPredicate(T1, dl, DAG); | ||||
| 2590 | Scale /= 2; | ||||
| 2591 | } | ||||
| 2592 | |||||
| 2593 | return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1); | ||||
| 2594 | } | ||||
| 2595 | |||||
| 2596 | assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ? void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2596, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2597 | |||||
| 2598 | // Cast everything to scalar integer types. | ||||
| 2599 | MVT ScalarTy = tyScalar(VecTy); | ||||
| 2600 | VecV = DAG.getBitcast(ScalarTy, VecV); | ||||
| 2601 | |||||
| 2602 | SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32); | ||||
| 2603 | SDValue ExtV; | ||||
| 2604 | |||||
| 2605 | if (IdxN) { | ||||
| 2606 | unsigned Off = IdxN->getZExtValue() * ElemWidth; | ||||
| 2607 | if (VecWidth == 64 && ValWidth == 32) { | ||||
| 2608 | assert(Off == 0 || Off == 32)(static_cast <bool> (Off == 0 || Off == 32) ? void (0) : __assert_fail ("Off == 0 || Off == 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2608, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2609 | unsigned SubIdx = Off == 0 ? Hexagon::isub_lo : Hexagon::isub_hi; | ||||
| 2610 | ExtV = DAG.getTargetExtractSubreg(SubIdx, dl, MVT::i32, VecV); | ||||
| 2611 | } else if (Off == 0 && (ValWidth % 8) == 0) { | ||||
| 2612 | ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy)); | ||||
| 2613 | } else { | ||||
| 2614 | SDValue OffV = DAG.getConstant(Off, dl, MVT::i32); | ||||
| 2615 | // The return type of EXTRACTU must be the same as the type of the | ||||
| 2616 | // input vector. | ||||
| 2617 | ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy, | ||||
| 2618 | {VecV, WidthV, OffV}); | ||||
| 2619 | } | ||||
| 2620 | } else { | ||||
| 2621 | if (ty(IdxV) != MVT::i32) | ||||
| 2622 | IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32); | ||||
| 2623 | SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | ||||
| 2624 | DAG.getConstant(ElemWidth, dl, MVT::i32)); | ||||
| 2625 | ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy, | ||||
| 2626 | {VecV, WidthV, OffV}); | ||||
| 2627 | } | ||||
| 2628 | |||||
| 2629 | // Cast ExtV to the requested result type. | ||||
| 2630 | ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy)); | ||||
| 2631 | ExtV = DAG.getBitcast(ResTy, ExtV); | ||||
| 2632 | return ExtV; | ||||
| 2633 | } | ||||
| 2634 | |||||
| 2635 | SDValue | ||||
| 2636 | HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV, | ||||
| 2637 | const SDLoc &dl, MVT ValTy, | ||||
| 2638 | SelectionDAG &DAG) const { | ||||
| 2639 | MVT VecTy = ty(VecV); | ||||
| 2640 | if (VecTy.getVectorElementType() == MVT::i1) { | ||||
| 2641 | MVT ValTy = ty(ValV); | ||||
| 2642 | assert(ValTy.getVectorElementType() == MVT::i1)(static_cast <bool> (ValTy.getVectorElementType() == MVT ::i1) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == MVT::i1" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2642, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2643 | SDValue ValR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV); | ||||
| 2644 | unsigned VecLen = VecTy.getVectorNumElements(); | ||||
| 2645 | unsigned Scale = VecLen / ValTy.getVectorNumElements(); | ||||
| 2646 | assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail ("Scale > 1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2646, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2647 | |||||
| 2648 | for (unsigned R = Scale; R > 1; R /= 2) { | ||||
| 2649 | ValR = contractPredicate(ValR, dl, DAG); | ||||
| 2650 | ValR = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | ||||
| 2651 | DAG.getUNDEF(MVT::i32), ValR); | ||||
| 2652 | } | ||||
| 2653 | // The longest possible subvector is at most 32 bits, so it is always | ||||
| 2654 | // contained in the low subregister. | ||||
| 2655 | ValR = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, ValR); | ||||
| 2656 | |||||
| 2657 | unsigned ValBytes = 64 / Scale; | ||||
| 2658 | SDValue Width = DAG.getConstant(ValBytes*8, dl, MVT::i32); | ||||
| 2659 | SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, | ||||
| 2660 | DAG.getConstant(8, dl, MVT::i32)); | ||||
| 2661 | SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV); | ||||
| 2662 | SDValue Ins = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, | ||||
| 2663 | {VecR, ValR, Width, Idx}); | ||||
| 2664 | return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins); | ||||
| 2665 | } | ||||
| 2666 | |||||
| 2667 | unsigned VecWidth = VecTy.getSizeInBits(); | ||||
| 2668 | unsigned ValWidth = ValTy.getSizeInBits(); | ||||
| 2669 | assert(VecWidth == 32 || VecWidth == 64)(static_cast <bool> (VecWidth == 32 || VecWidth == 64) ? void (0) : __assert_fail ("VecWidth == 32 || VecWidth == 64" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2669, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2670 | assert((VecWidth % ValWidth) == 0)(static_cast <bool> ((VecWidth % ValWidth) == 0) ? void (0) : __assert_fail ("(VecWidth % ValWidth) == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2670, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2671 | |||||
| 2672 | // Cast everything to scalar integer types. | ||||
| 2673 | MVT ScalarTy = MVT::getIntegerVT(VecWidth); | ||||
| 2674 | // The actual type of ValV may be different than ValTy (which is related | ||||
| 2675 | // to the vector type). | ||||
| 2676 | unsigned VW = ty(ValV).getSizeInBits(); | ||||
| 2677 | ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV); | ||||
| 2678 | VecV = DAG.getBitcast(ScalarTy, VecV); | ||||
| 2679 | if (VW != VecWidth) | ||||
| 2680 | ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy); | ||||
| 2681 | |||||
| 2682 | SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32); | ||||
| 2683 | SDValue InsV; | ||||
| 2684 | |||||
| 2685 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) { | ||||
| 2686 | unsigned W = C->getZExtValue() * ValWidth; | ||||
| 2687 | SDValue OffV = DAG.getConstant(W, dl, MVT::i32); | ||||
| 2688 | InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy, | ||||
| 2689 | {VecV, ValV, WidthV, OffV}); | ||||
| 2690 | } else { | ||||
| 2691 | if (ty(IdxV) != MVT::i32) | ||||
| 2692 | IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32); | ||||
| 2693 | SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV); | ||||
| 2694 | InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy, | ||||
| 2695 | {VecV, ValV, WidthV, OffV}); | ||||
| 2696 | } | ||||
| 2697 | |||||
| 2698 | return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV); | ||||
| 2699 | } | ||||
| 2700 | |||||
| 2701 | SDValue | ||||
| 2702 | HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl, | ||||
| 2703 | SelectionDAG &DAG) const { | ||||
| 2704 | assert(ty(Vec32).getSizeInBits() == 32)(static_cast <bool> (ty(Vec32).getSizeInBits() == 32) ? void (0) : __assert_fail ("ty(Vec32).getSizeInBits() == 32", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2704, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2705 | if (isUndef(Vec32)) | ||||
| 2706 | return DAG.getUNDEF(MVT::i64); | ||||
| 2707 | return getInstr(Hexagon::S2_vsxtbh, dl, MVT::i64, {Vec32}, DAG); | ||||
| 2708 | } | ||||
| 2709 | |||||
| 2710 | SDValue | ||||
| 2711 | HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl, | ||||
| 2712 | SelectionDAG &DAG) const { | ||||
| 2713 | assert(ty(Vec64).getSizeInBits() == 64)(static_cast <bool> (ty(Vec64).getSizeInBits() == 64) ? void (0) : __assert_fail ("ty(Vec64).getSizeInBits() == 64", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2713, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2714 | if (isUndef(Vec64)) | ||||
| 2715 | return DAG.getUNDEF(MVT::i32); | ||||
| 2716 | return getInstr(Hexagon::S2_vtrunehb, dl, MVT::i32, {Vec64}, DAG); | ||||
| 2717 | } | ||||
| 2718 | |||||
| 2719 | SDValue | ||||
| 2720 | HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) | ||||
| 2721 | const { | ||||
| 2722 | if (Ty.isVector()) { | ||||
| 2723 | assert(Ty.isInteger() && "Only integer vectors are supported here")(static_cast <bool> (Ty.isInteger() && "Only integer vectors are supported here" ) ? void (0) : __assert_fail ("Ty.isInteger() && \"Only integer vectors are supported here\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2723, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2724 | unsigned W = Ty.getSizeInBits(); | ||||
| 2725 | if (W <= 64) | ||||
| 2726 | return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W))); | ||||
| 2727 | return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG)); | ||||
| 2728 | } | ||||
| 2729 | |||||
| 2730 | if (Ty.isInteger()) | ||||
| 2731 | return DAG.getConstant(0, dl, Ty); | ||||
| 2732 | if (Ty.isFloatingPoint()) | ||||
| 2733 | return DAG.getConstantFP(0.0, dl, Ty); | ||||
| 2734 | llvm_unreachable("Invalid type for zero")::llvm::llvm_unreachable_internal("Invalid type for zero", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2734); | ||||
| 2735 | } | ||||
| 2736 | |||||
| 2737 | SDValue | ||||
| 2738 | HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG) | ||||
| 2739 | const { | ||||
| 2740 | MVT ValTy = ty(Val); | ||||
| 2741 | assert(ValTy.getVectorElementType() == ResTy.getVectorElementType())(static_cast <bool> (ValTy.getVectorElementType() == ResTy .getVectorElementType()) ? void (0) : __assert_fail ("ValTy.getVectorElementType() == ResTy.getVectorElementType()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2741, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2742 | |||||
| 2743 | unsigned ValLen = ValTy.getVectorNumElements(); | ||||
| 2744 | unsigned ResLen = ResTy.getVectorNumElements(); | ||||
| 2745 | if (ValLen == ResLen) | ||||
| 2746 | return Val; | ||||
| 2747 | |||||
| 2748 | const SDLoc &dl(Val); | ||||
| 2749 | assert(ValLen < ResLen)(static_cast <bool> (ValLen < ResLen) ? void (0) : __assert_fail ("ValLen < ResLen", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2749, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2750 | assert(ResLen % ValLen == 0)(static_cast <bool> (ResLen % ValLen == 0) ? void (0) : __assert_fail ("ResLen % ValLen == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2750, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2751 | |||||
| 2752 | SmallVector<SDValue, 4> Concats = {Val}; | ||||
| 2753 | for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i) | ||||
| 2754 | Concats.push_back(DAG.getUNDEF(ValTy)); | ||||
| 2755 | |||||
| 2756 | return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats); | ||||
| 2757 | } | ||||
| 2758 | |||||
| 2759 | SDValue | ||||
| 2760 | HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { | ||||
| 2761 | MVT VecTy = ty(Op); | ||||
| 2762 | unsigned BW = VecTy.getSizeInBits(); | ||||
| 2763 | const SDLoc &dl(Op); | ||||
| 2764 | SmallVector<SDValue,8> Ops; | ||||
| 2765 | for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) | ||||
| 2766 | Ops.push_back(Op.getOperand(i)); | ||||
| 2767 | |||||
| 2768 | if (BW == 32) | ||||
| 2769 | return buildVector32(Ops, dl, VecTy, DAG); | ||||
| 2770 | if (BW == 64) | ||||
| 2771 | return buildVector64(Ops, dl, VecTy, DAG); | ||||
| 2772 | |||||
| 2773 | if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) { | ||||
| 2774 | // Check if this is a special case or all-0 or all-1. | ||||
| 2775 | bool All0 = true, All1 = true; | ||||
| 2776 | for (SDValue P : Ops) { | ||||
| 2777 | auto *CN = dyn_cast<ConstantSDNode>(P.getNode()); | ||||
| 2778 | if (CN == nullptr) { | ||||
| 2779 | All0 = All1 = false; | ||||
| 2780 | break; | ||||
| 2781 | } | ||||
| 2782 | uint32_t C = CN->getZExtValue(); | ||||
| 2783 | All0 &= (C == 0); | ||||
| 2784 | All1 &= (C == 1); | ||||
| 2785 | } | ||||
| 2786 | if (All0) | ||||
| 2787 | return DAG.getNode(HexagonISD::PFALSE, dl, VecTy); | ||||
| 2788 | if (All1) | ||||
| 2789 | return DAG.getNode(HexagonISD::PTRUE, dl, VecTy); | ||||
| 2790 | |||||
| 2791 | // For each i1 element in the resulting predicate register, put 1 | ||||
| 2792 | // shifted by the index of the element into a general-purpose register, | ||||
| 2793 | // then or them together and transfer it back into a predicate register. | ||||
| 2794 | SDValue Rs[8]; | ||||
| 2795 | SDValue Z = getZero(dl, MVT::i32, DAG); | ||||
| 2796 | // Always produce 8 bits, repeat inputs if necessary. | ||||
| 2797 | unsigned Rep = 8 / VecTy.getVectorNumElements(); | ||||
| 2798 | for (unsigned i = 0; i != 8; ++i) { | ||||
| 2799 | SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32); | ||||
| 2800 | Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z); | ||||
| 2801 | } | ||||
| 2802 | for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) { | ||||
| 2803 | for (unsigned i = 0, e = A.size()/2; i != e; ++i) | ||||
| 2804 | Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]); | ||||
| 2805 | } | ||||
| 2806 | // Move the value directly to a predicate register. | ||||
| 2807 | return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG); | ||||
| 2808 | } | ||||
| 2809 | |||||
| 2810 | return SDValue(); | ||||
| 2811 | } | ||||
| 2812 | |||||
| 2813 | SDValue | ||||
| 2814 | HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op, | ||||
| 2815 | SelectionDAG &DAG) const { | ||||
| 2816 | MVT VecTy = ty(Op); | ||||
| 2817 | const SDLoc &dl(Op); | ||||
| 2818 | if (VecTy.getSizeInBits() == 64) { | ||||
| 2819 | assert(Op.getNumOperands() == 2)(static_cast <bool> (Op.getNumOperands() == 2) ? void ( 0) : __assert_fail ("Op.getNumOperands() == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2819, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2820 | return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, Op.getOperand(1), | ||||
| 2821 | Op.getOperand(0)); | ||||
| 2822 | } | ||||
| 2823 | |||||
| 2824 | MVT ElemTy = VecTy.getVectorElementType(); | ||||
| 2825 | if (ElemTy == MVT::i1) { | ||||
| 2826 | assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1)(static_cast <bool> (VecTy == MVT::v2i1 || VecTy == MVT ::v4i1 || VecTy == MVT::v8i1) ? void (0) : __assert_fail ("VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2826, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2827 | MVT OpTy = ty(Op.getOperand(0)); | ||||
| 2828 | // Scale is how many times the operands need to be contracted to match | ||||
| 2829 | // the representation in the target register. | ||||
| 2830 | unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements(); | ||||
| 2831 | assert(Scale == Op.getNumOperands() && Scale > 1)(static_cast <bool> (Scale == Op.getNumOperands() && Scale > 1) ? void (0) : __assert_fail ("Scale == Op.getNumOperands() && Scale > 1" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2831, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2832 | |||||
| 2833 | // First, convert all bool vectors to integers, then generate pairwise | ||||
| 2834 | // inserts to form values of doubled length. Up until there are only | ||||
| 2835 | // two values left to concatenate, all of these values will fit in a | ||||
| 2836 | // 32-bit integer, so keep them as i32 to use 32-bit inserts. | ||||
| 2837 | SmallVector<SDValue,4> Words[2]; | ||||
| 2838 | unsigned IdxW = 0; | ||||
| 2839 | |||||
| 2840 | for (SDValue P : Op.getNode()->op_values()) { | ||||
| 2841 | SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P); | ||||
| 2842 | for (unsigned R = Scale; R > 1; R /= 2) { | ||||
| 2843 | W = contractPredicate(W, dl, DAG); | ||||
| 2844 | W = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | ||||
| 2845 | DAG.getUNDEF(MVT::i32), W); | ||||
| 2846 | } | ||||
| 2847 | W = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, W); | ||||
| 2848 | Words[IdxW].push_back(W); | ||||
| 2849 | } | ||||
| 2850 | |||||
| 2851 | while (Scale > 2) { | ||||
| 2852 | SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32); | ||||
| 2853 | Words[IdxW ^ 1].clear(); | ||||
| 2854 | |||||
| 2855 | for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) { | ||||
| 2856 | SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1]; | ||||
| 2857 | // Insert W1 into W0 right next to the significant bits of W0. | ||||
| 2858 | SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, | ||||
| 2859 | {W0, W1, WidthV, WidthV}); | ||||
| 2860 | Words[IdxW ^ 1].push_back(T); | ||||
| 2861 | } | ||||
| 2862 | IdxW ^= 1; | ||||
| 2863 | Scale /= 2; | ||||
| 2864 | } | ||||
| 2865 | |||||
| 2866 | // Another sanity check. At this point there should only be two words | ||||
| 2867 | // left, and Scale should be 2. | ||||
| 2868 | assert(Scale == 2 && Words[IdxW].size() == 2)(static_cast <bool> (Scale == 2 && Words[IdxW]. size() == 2) ? void (0) : __assert_fail ("Scale == 2 && Words[IdxW].size() == 2" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 2868, __extension__ __PRETTY_FUNCTION__)); | ||||
| 2869 | |||||
| 2870 | SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, | ||||
| 2871 | Words[IdxW][1], Words[IdxW][0]); | ||||
| 2872 | return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW); | ||||
| 2873 | } | ||||
| 2874 | |||||
| 2875 | return SDValue(); | ||||
| 2876 | } | ||||
| 2877 | |||||
| 2878 | SDValue | ||||
| 2879 | HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, | ||||
| 2880 | SelectionDAG &DAG) const { | ||||
| 2881 | SDValue Vec = Op.getOperand(0); | ||||
| 2882 | MVT ElemTy = ty(Vec).getVectorElementType(); | ||||
| 2883 | return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG); | ||||
| 2884 | } | ||||
| 2885 | |||||
| 2886 | SDValue | ||||
| 2887 | HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, | ||||
| 2888 | SelectionDAG &DAG) const { | ||||
| 2889 | return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op), | ||||
| 2890 | ty(Op), ty(Op), DAG); | ||||
| 2891 | } | ||||
| 2892 | |||||
| 2893 | SDValue | ||||
| 2894 | HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, | ||||
| 2895 | SelectionDAG &DAG) const { | ||||
| 2896 | return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2), | ||||
| 2897 | SDLoc(Op), ty(Op).getVectorElementType(), DAG); | ||||
| 2898 | } | ||||
| 2899 | |||||
| 2900 | SDValue | ||||
| 2901 | HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, | ||||
| 2902 | SelectionDAG &DAG) const { | ||||
| 2903 | SDValue ValV = Op.getOperand(1); | ||||
| 2904 | return insertVector(Op.getOperand(0), ValV, Op.getOperand(2), | ||||
| 2905 | SDLoc(Op), ty(ValV), DAG); | ||||
| 2906 | } | ||||
| 2907 | |||||
| 2908 | bool | ||||
| 2909 | HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const { | ||||
| 2910 | // Assuming the caller does not have either a signext or zeroext modifier, and | ||||
| 2911 | // only one value is accepted, any reasonable truncation is allowed. | ||||
| 2912 | if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) | ||||
| 2913 | return false; | ||||
| 2914 | |||||
| 2915 | // FIXME: in principle up to 64-bit could be made safe, but it would be very | ||||
| 2916 | // fragile at the moment: any support for multiple value returns would be | ||||
| 2917 | // liable to disallow tail calls involving i64 -> iN truncation in many cases. | ||||
| 2918 | return Ty1->getPrimitiveSizeInBits() <= 32; | ||||
| 2919 | } | ||||
| 2920 | |||||
| 2921 | SDValue | ||||
| 2922 | HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const { | ||||
| 2923 | MVT Ty = ty(Op); | ||||
| 2924 | const SDLoc &dl(Op); | ||||
| 2925 | // Lower loads of scalar predicate vectors (v2i1, v4i1, v8i1) to loads of i1 | ||||
| 2926 | // followed by a TYPECAST. | ||||
| 2927 | LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); | ||||
| 2928 | bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1); | ||||
| 2929 | if (DoCast) { | ||||
| 2930 | SDValue NL = DAG.getLoad( | ||||
| 2931 | LN->getAddressingMode(), LN->getExtensionType(), MVT::i1, dl, | ||||
| 2932 | LN->getChain(), LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(), | ||||
| 2933 | /*MemoryVT*/ MVT::i1, LN->getAlign(), LN->getMemOperand()->getFlags(), | ||||
| 2934 | LN->getAAInfo(), LN->getRanges()); | ||||
| 2935 | LN = cast<LoadSDNode>(NL.getNode()); | ||||
| 2936 | } | ||||
| 2937 | |||||
| 2938 | Align ClaimAlign = LN->getAlign(); | ||||
| 2939 | if (!validateConstPtrAlignment(LN->getBasePtr(), ClaimAlign, dl, DAG)) | ||||
| 2940 | return replaceMemWithUndef(Op, DAG); | ||||
| 2941 | |||||
| 2942 | // Call LowerUnalignedLoad for all loads, it recognizes loads that | ||||
| 2943 | // don't need extra aligning. | ||||
| 2944 | SDValue LU = LowerUnalignedLoad(SDValue(LN, 0), DAG); | ||||
| 2945 | if (DoCast) { | ||||
| 2946 | SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, Ty, LU); | ||||
| 2947 | SDValue Ch = cast<LoadSDNode>(LU.getNode())->getChain(); | ||||
| 2948 | return DAG.getMergeValues({TC, Ch}, dl); | ||||
| 2949 | } | ||||
| 2950 | return LU; | ||||
| 2951 | } | ||||
| 2952 | |||||
| 2953 | SDValue | ||||
| 2954 | HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const { | ||||
| 2955 | const SDLoc &dl(Op); | ||||
| 2956 | StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); | ||||
| 2957 | SDValue Val = SN->getValue(); | ||||
| 2958 | MVT Ty = ty(Val); | ||||
| 2959 | |||||
| 2960 | bool DoCast = (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1); | ||||
| 2961 | if (DoCast) { | ||||
| 2962 | SDValue TC = DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, Val); | ||||
| 2963 | SDValue NS = DAG.getStore(SN->getChain(), dl, TC, SN->getBasePtr(), | ||||
| 2964 | SN->getMemOperand()); | ||||
| 2965 | if (SN->isIndexed()) { | ||||
| 2966 | NS = DAG.getIndexedStore(NS, dl, SN->getBasePtr(), SN->getOffset(), | ||||
| 2967 | SN->getAddressingMode()); | ||||
| 2968 | } | ||||
| 2969 | SN = cast<StoreSDNode>(NS.getNode()); | ||||
| 2970 | } | ||||
| 2971 | |||||
| 2972 | Align ClaimAlign = SN->getAlign(); | ||||
| 2973 | if (!validateConstPtrAlignment(SN->getBasePtr(), ClaimAlign, dl, DAG)) | ||||
| 2974 | return replaceMemWithUndef(Op, DAG); | ||||
| 2975 | |||||
| 2976 | MVT StoreTy = SN->getMemoryVT().getSimpleVT(); | ||||
| 2977 | Align NeedAlign = Subtarget.getTypeAlignment(StoreTy); | ||||
| 2978 | if (ClaimAlign < NeedAlign) | ||||
| 2979 | return expandUnalignedStore(SN, DAG); | ||||
| 2980 | return SDValue(SN, 0); | ||||
| 2981 | } | ||||
| 2982 | |||||
| 2983 | SDValue | ||||
| 2984 | HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) | ||||
| 2985 | const { | ||||
| 2986 | LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); | ||||
| 2987 | MVT LoadTy = ty(Op); | ||||
| 2988 | unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy).value(); | ||||
| 2989 | unsigned HaveAlign = LN->getAlign().value(); | ||||
| 2990 | if (HaveAlign >= NeedAlign) | ||||
| 2991 | return Op; | ||||
| 2992 | |||||
| 2993 | const SDLoc &dl(Op); | ||||
| 2994 | const DataLayout &DL = DAG.getDataLayout(); | ||||
| 2995 | LLVMContext &Ctx = *DAG.getContext(); | ||||
| 2996 | |||||
| 2997 | // If the load aligning is disabled or the load can be broken up into two | ||||
| 2998 | // smaller legal loads, do the default (target-independent) expansion. | ||||
| 2999 | bool DoDefault = false; | ||||
| 3000 | // Handle it in the default way if this is an indexed load. | ||||
| 3001 | if (!LN->isUnindexed()) | ||||
| 3002 | DoDefault = true; | ||||
| 3003 | |||||
| 3004 | if (!AlignLoads) { | ||||
| 3005 | if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(), | ||||
| 3006 | *LN->getMemOperand())) | ||||
| 3007 | return Op; | ||||
| 3008 | DoDefault = true; | ||||
| 3009 | } | ||||
| 3010 | if (!DoDefault && (2 * HaveAlign) == NeedAlign) { | ||||
| 3011 | // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)". | ||||
| 3012 | MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign) | ||||
| 3013 | : MVT::getVectorVT(MVT::i8, HaveAlign); | ||||
| 3014 | DoDefault = | ||||
| 3015 | allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand()); | ||||
| 3016 | } | ||||
| 3017 | if (DoDefault) { | ||||
| 3018 | std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG); | ||||
| 3019 | return DAG.getMergeValues({P.first, P.second}, dl); | ||||
| 3020 | } | ||||
| 3021 | |||||
| 3022 | // The code below generates two loads, both aligned as NeedAlign, and | ||||
| 3023 | // with the distance of NeedAlign between them. For that to cover the | ||||
| 3024 | // bits that need to be loaded (and without overlapping), the size of | ||||
| 3025 | // the loads should be equal to NeedAlign. This is true for all loadable | ||||
| 3026 | // types, but add an assertion in case something changes in the future. | ||||
| 3027 | assert(LoadTy.getSizeInBits() == 8*NeedAlign)(static_cast <bool> (LoadTy.getSizeInBits() == 8*NeedAlign ) ? void (0) : __assert_fail ("LoadTy.getSizeInBits() == 8*NeedAlign" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3027, __extension__ __PRETTY_FUNCTION__)); | ||||
| 3028 | |||||
| 3029 | unsigned LoadLen = NeedAlign; | ||||
| 3030 | SDValue Base = LN->getBasePtr(); | ||||
| 3031 | SDValue Chain = LN->getChain(); | ||||
| 3032 | auto BO = getBaseAndOffset(Base); | ||||
| 3033 | unsigned BaseOpc = BO.first.getOpcode(); | ||||
| 3034 | if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0) | ||||
| 3035 | return Op; | ||||
| 3036 | |||||
| 3037 | if (BO.second % LoadLen != 0) { | ||||
| 3038 | BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first, | ||||
| 3039 | DAG.getConstant(BO.second % LoadLen, dl, MVT::i32)); | ||||
| 3040 | BO.second -= BO.second % LoadLen; | ||||
| 3041 | } | ||||
| 3042 | SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR) | ||||
| 3043 | ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first, | ||||
| 3044 | DAG.getConstant(NeedAlign, dl, MVT::i32)) | ||||
| 3045 | : BO.first; | ||||
| 3046 | SDValue Base0 = | ||||
| 3047 | DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl); | ||||
| 3048 | SDValue Base1 = DAG.getMemBasePlusOffset( | ||||
| 3049 | BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl); | ||||
| 3050 | |||||
| 3051 | MachineMemOperand *WideMMO = nullptr; | ||||
| 3052 | if (MachineMemOperand *MMO = LN->getMemOperand()) { | ||||
| 3053 | MachineFunction &MF = DAG.getMachineFunction(); | ||||
| 3054 | WideMMO = MF.getMachineMemOperand( | ||||
| 3055 | MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen), | ||||
| 3056 | MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(), | ||||
| 3057 | MMO->getSuccessOrdering(), MMO->getFailureOrdering()); | ||||
| 3058 | } | ||||
| 3059 | |||||
| 3060 | SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO); | ||||
| 3061 | SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO); | ||||
| 3062 | |||||
| 3063 | SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy, | ||||
| 3064 | {Load1, Load0, BaseNoOff.getOperand(0)}); | ||||
| 3065 | SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||
| 3066 | Load0.getValue(1), Load1.getValue(1)); | ||||
| 3067 | SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl); | ||||
| 3068 | return M; | ||||
| 3069 | } | ||||
| 3070 | |||||
| 3071 | SDValue | ||||
| 3072 | HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const { | ||||
| 3073 | SDValue X = Op.getOperand(0), Y = Op.getOperand(1); | ||||
| 3074 | auto *CY = dyn_cast<ConstantSDNode>(Y); | ||||
| 3075 | if (!CY) | ||||
| 3076 | return SDValue(); | ||||
| 3077 | |||||
| 3078 | const SDLoc &dl(Op); | ||||
| 3079 | SDVTList VTs = Op.getNode()->getVTList(); | ||||
| 3080 | assert(VTs.NumVTs == 2)(static_cast <bool> (VTs.NumVTs == 2) ? void (0) : __assert_fail ("VTs.NumVTs == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3080, __extension__ __PRETTY_FUNCTION__)); | ||||
| 3081 | assert(VTs.VTs[1] == MVT::i1)(static_cast <bool> (VTs.VTs[1] == MVT::i1) ? void (0) : __assert_fail ("VTs.VTs[1] == MVT::i1", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3081, __extension__ __PRETTY_FUNCTION__)); | ||||
| 3082 | unsigned Opc = Op.getOpcode(); | ||||
| 3083 | |||||
| 3084 | if (CY) { | ||||
| 3085 | uint32_t VY = CY->getZExtValue(); | ||||
| 3086 | assert(VY != 0 && "This should have been folded")(static_cast <bool> (VY != 0 && "This should have been folded" ) ? void (0) : __assert_fail ("VY != 0 && \"This should have been folded\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3086, __extension__ __PRETTY_FUNCTION__)); | ||||
| 3087 | // X +/- 1 | ||||
| 3088 | if (VY != 1) | ||||
| 3089 | return SDValue(); | ||||
| 3090 | |||||
| 3091 | if (Opc == ISD::UADDO) { | ||||
| 3092 | SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y}); | ||||
| 3093 | SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG), | ||||
| 3094 | ISD::SETEQ); | ||||
| 3095 | return DAG.getMergeValues({Op, Ov}, dl); | ||||
| 3096 | } | ||||
| 3097 | if (Opc == ISD::USUBO) { | ||||
| 3098 | SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y}); | ||||
| 3099 | SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, | ||||
| 3100 | DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ); | ||||
| 3101 | return DAG.getMergeValues({Op, Ov}, dl); | ||||
| 3102 | } | ||||
| 3103 | } | ||||
| 3104 | |||||
| 3105 | return SDValue(); | ||||
| 3106 | } | ||||
| 3107 | |||||
| 3108 | SDValue | ||||
| 3109 | HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const { | ||||
| 3110 | const SDLoc &dl(Op); | ||||
| 3111 | unsigned Opc = Op.getOpcode(); | ||||
| 3112 | SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2); | ||||
| 3113 | |||||
| 3114 | if (Opc == ISD::ADDCARRY) | ||||
| 3115 | return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(), | ||||
| 3116 | { X, Y, C }); | ||||
| 3117 | |||||
| 3118 | EVT CarryTy = C.getValueType(); | ||||
| 3119 | SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(), | ||||
| 3120 | { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) }); | ||||
| 3121 | SDValue Out[] = { SubC.getValue(0), | ||||
| 3122 | DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) }; | ||||
| 3123 | return DAG.getMergeValues(Out, dl); | ||||
| 3124 | } | ||||
| 3125 | |||||
| 3126 | SDValue | ||||
| 3127 | HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const { | ||||
| 3128 | SDValue Chain = Op.getOperand(0); | ||||
| 3129 | SDValue Offset = Op.getOperand(1); | ||||
| 3130 | SDValue Handler = Op.getOperand(2); | ||||
| 3131 | SDLoc dl(Op); | ||||
| 3132 | auto PtrVT = getPointerTy(DAG.getDataLayout()); | ||||
| 3133 | |||||
| 3134 | // Mark function as containing a call to EH_RETURN. | ||||
| 3135 | HexagonMachineFunctionInfo *FuncInfo = | ||||
| 3136 | DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>(); | ||||
| 3137 | FuncInfo->setHasEHReturn(); | ||||
| 3138 | |||||
| 3139 | unsigned OffsetReg = Hexagon::R28; | ||||
| 3140 | |||||
| 3141 | SDValue StoreAddr = | ||||
| 3142 | DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT), | ||||
| 3143 | DAG.getIntPtrConstant(4, dl)); | ||||
| 3144 | Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo()); | ||||
| 3145 | Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset); | ||||
| 3146 | |||||
| 3147 | // Not needed we already use it as explict input to EH_RETURN. | ||||
| 3148 | // MF.getRegInfo().addLiveOut(OffsetReg); | ||||
| 3149 | |||||
| 3150 | return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain); | ||||
| 3151 | } | ||||
| 3152 | |||||
| 3153 | SDValue | ||||
| 3154 | HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { | ||||
| 3155 | unsigned Opc = Op.getOpcode(); | ||||
| 3156 | |||||
| 3157 | // Handle INLINEASM first. | ||||
| 3158 | if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR) | ||||
| |||||
| 3159 | return LowerINLINEASM(Op, DAG); | ||||
| 3160 | |||||
| 3161 | if (isHvxOperation(Op.getNode(), DAG)) { | ||||
| 3162 | // If HVX lowering returns nothing, try the default lowering. | ||||
| 3163 | if (SDValue V = LowerHvxOperation(Op, DAG)) | ||||
| 3164 | return V; | ||||
| 3165 | } | ||||
| 3166 | |||||
| 3167 | switch (Opc) { | ||||
| 3168 | default: | ||||
| 3169 | #ifndef NDEBUG | ||||
| 3170 | Op.getNode()->dumpr(&DAG); | ||||
| 3171 | if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END) | ||||
| 3172 | errs() << "Error: check for a non-legal type in this operation\n"; | ||||
| 3173 | #endif | ||||
| 3174 | llvm_unreachable("Should not custom lower this!")::llvm::llvm_unreachable_internal("Should not custom lower this!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3174); | ||||
| 3175 | case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); | ||||
| 3176 | case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG); | ||||
| 3177 | case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); | ||||
| 3178 | case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); | ||||
| 3179 | case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); | ||||
| 3180 | case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); | ||||
| 3181 | case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); | ||||
| 3182 | case ISD::BITCAST: return LowerBITCAST(Op, DAG); | ||||
| 3183 | case ISD::LOAD: return LowerLoad(Op, DAG); | ||||
| 3184 | case ISD::STORE: return LowerStore(Op, DAG); | ||||
| 3185 | case ISD::UADDO: | ||||
| 3186 | case ISD::USUBO: return LowerUAddSubO(Op, DAG); | ||||
| 3187 | case ISD::ADDCARRY: | ||||
| 3188 | case ISD::SUBCARRY: return LowerAddSubCarry(Op, DAG); | ||||
| 3189 | case ISD::SRA: | ||||
| 3190 | case ISD::SHL: | ||||
| 3191 | case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG); | ||||
| 3192 | case ISD::ROTL: return LowerROTL(Op, DAG); | ||||
| 3193 | case ISD::ConstantPool: return LowerConstantPool(Op, DAG); | ||||
| 3194 | case ISD::JumpTable: return LowerJumpTable(Op, DAG); | ||||
| 3195 | case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); | ||||
| 3196 | case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); | ||||
| 3197 | case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); | ||||
| 3198 | case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); | ||||
| 3199 | case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG); | ||||
| 3200 | case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG); | ||||
| 3201 | case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); | ||||
| 3202 | case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG); | ||||
| 3203 | case ISD::VACOPY: return LowerVACOPY(Op, DAG); | ||||
| 3204 | case ISD::VASTART: return LowerVASTART(Op, DAG); | ||||
| 3205 | case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); | ||||
| 3206 | case ISD::SETCC: return LowerSETCC(Op, DAG); | ||||
| 3207 | case ISD::VSELECT: return LowerVSELECT(Op, DAG); | ||||
| 3208 | case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); | ||||
| 3209 | case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG); | ||||
| 3210 | case ISD::PREFETCH: return LowerPREFETCH(Op, DAG); | ||||
| 3211 | case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); | ||||
| 3212 | break; | ||||
| 3213 | } | ||||
| 3214 | |||||
| 3215 | return SDValue(); | ||||
| 3216 | } | ||||
| 3217 | |||||
| 3218 | void | ||||
| 3219 | HexagonTargetLowering::LowerOperationWrapper(SDNode *N, | ||||
| 3220 | SmallVectorImpl<SDValue> &Results, | ||||
| 3221 | SelectionDAG &DAG) const { | ||||
| 3222 | if (isHvxOperation(N, DAG)) { | ||||
| 3223 | LowerHvxOperationWrapper(N, Results, DAG); | ||||
| 3224 | if (!Results.empty()) | ||||
| 3225 | return; | ||||
| 3226 | } | ||||
| 3227 | |||||
| 3228 | // We are only custom-lowering stores to verify the alignment of the | ||||
| 3229 | // address if it is a compile-time constant. Since a store can be modified | ||||
| 3230 | // during type-legalization (the value being stored may need legalization), | ||||
| 3231 | // return empty Results here to indicate that we don't really make any | ||||
| 3232 | // changes in the custom lowering. | ||||
| 3233 | if (N->getOpcode() != ISD::STORE) | ||||
| 3234 | return TargetLowering::LowerOperationWrapper(N, Results, DAG); | ||||
| 3235 | } | ||||
| 3236 | |||||
| 3237 | void | ||||
| 3238 | HexagonTargetLowering::ReplaceNodeResults(SDNode *N, | ||||
| 3239 | SmallVectorImpl<SDValue> &Results, | ||||
| 3240 | SelectionDAG &DAG) const { | ||||
| 3241 | if (isHvxOperation(N, DAG)) { | ||||
| 3242 | ReplaceHvxNodeResults(N, Results, DAG); | ||||
| 3243 | if (!Results.empty()) | ||||
| 3244 | return; | ||||
| 3245 | } | ||||
| 3246 | |||||
| 3247 | const SDLoc &dl(N); | ||||
| 3248 | switch (N->getOpcode()) { | ||||
| 3249 | case ISD::SRL: | ||||
| 3250 | case ISD::SRA: | ||||
| 3251 | case ISD::SHL: | ||||
| 3252 | return; | ||||
| 3253 | case ISD::BITCAST: | ||||
| 3254 | // Handle a bitcast from v8i1 to i8. | ||||
| 3255 | if (N->getValueType(0) == MVT::i8) { | ||||
| 3256 | if (N->getOperand(0).getValueType() == MVT::v8i1) { | ||||
| 3257 | SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, | ||||
| 3258 | N->getOperand(0), DAG); | ||||
| 3259 | SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8); | ||||
| 3260 | Results.push_back(T); | ||||
| 3261 | } | ||||
| 3262 | } | ||||
| 3263 | break; | ||||
| 3264 | } | ||||
| 3265 | } | ||||
| 3266 | |||||
| 3267 | SDValue | ||||
| 3268 | HexagonTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) | ||||
| 3269 | const { | ||||
| 3270 | if (isHvxOperation(N, DCI.DAG)) { | ||||
| 3271 | if (SDValue V = PerformHvxDAGCombine(N, DCI)) | ||||
| 3272 | return V; | ||||
| 3273 | return SDValue(); | ||||
| 3274 | } | ||||
| 3275 | |||||
| 3276 | if (DCI.isBeforeLegalizeOps()) | ||||
| 3277 | return SDValue(); | ||||
| 3278 | |||||
| 3279 | SDValue Op(N, 0); | ||||
| 3280 | const SDLoc &dl(Op); | ||||
| 3281 | unsigned Opc = Op.getOpcode(); | ||||
| 3282 | |||||
| 3283 | if (Opc == HexagonISD::P2D) { | ||||
| 3284 | SDValue P = Op.getOperand(0); | ||||
| 3285 | switch (P.getOpcode()) { | ||||
| 3286 | case HexagonISD::PTRUE: | ||||
| 3287 | return DCI.DAG.getConstant(-1, dl, ty(Op)); | ||||
| 3288 | case HexagonISD::PFALSE: | ||||
| 3289 | return getZero(dl, ty(Op), DCI.DAG); | ||||
| 3290 | default: | ||||
| 3291 | break; | ||||
| 3292 | } | ||||
| 3293 | } else if (Opc == ISD::VSELECT) { | ||||
| 3294 | // This is pretty much duplicated in HexagonISelLoweringHVX... | ||||
| 3295 | // | ||||
| 3296 | // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0) | ||||
| 3297 | SDValue Cond = Op.getOperand(0); | ||||
| 3298 | if (Cond->getOpcode() == ISD::XOR) { | ||||
| 3299 | SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1); | ||||
| 3300 | if (C1->getOpcode() == HexagonISD::PTRUE) { | ||||
| 3301 | SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, | ||||
| 3302 | Op.getOperand(2), Op.getOperand(1)); | ||||
| 3303 | return VSel; | ||||
| 3304 | } | ||||
| 3305 | } | ||||
| 3306 | } | ||||
| 3307 | |||||
| 3308 | return SDValue(); | ||||
| 3309 | } | ||||
| 3310 | |||||
| 3311 | /// Returns relocation base for the given PIC jumptable. | ||||
| 3312 | SDValue | ||||
| 3313 | HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table, | ||||
| 3314 | SelectionDAG &DAG) const { | ||||
| 3315 | int Idx = cast<JumpTableSDNode>(Table)->getIndex(); | ||||
| 3316 | EVT VT = Table.getValueType(); | ||||
| 3317 | SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL); | ||||
| 3318 | return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T); | ||||
| 3319 | } | ||||
| 3320 | |||||
| 3321 | //===----------------------------------------------------------------------===// | ||||
| 3322 | // Inline Assembly Support | ||||
| 3323 | //===----------------------------------------------------------------------===// | ||||
| 3324 | |||||
| 3325 | TargetLowering::ConstraintType | ||||
| 3326 | HexagonTargetLowering::getConstraintType(StringRef Constraint) const { | ||||
| 3327 | if (Constraint.size() == 1) { | ||||
| 3328 | switch (Constraint[0]) { | ||||
| 3329 | case 'q': | ||||
| 3330 | case 'v': | ||||
| 3331 | if (Subtarget.useHVXOps()) | ||||
| 3332 | return C_RegisterClass; | ||||
| 3333 | break; | ||||
| 3334 | case 'a': | ||||
| 3335 | return C_RegisterClass; | ||||
| 3336 | default: | ||||
| 3337 | break; | ||||
| 3338 | } | ||||
| 3339 | } | ||||
| 3340 | return TargetLowering::getConstraintType(Constraint); | ||||
| 3341 | } | ||||
| 3342 | |||||
| 3343 | std::pair<unsigned, const TargetRegisterClass*> | ||||
| 3344 | HexagonTargetLowering::getRegForInlineAsmConstraint( | ||||
| 3345 | const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { | ||||
| 3346 | |||||
| 3347 | if (Constraint.size() == 1) { | ||||
| 3348 | switch (Constraint[0]) { | ||||
| 3349 | case 'r': // R0-R31 | ||||
| 3350 | switch (VT.SimpleTy) { | ||||
| 3351 | default: | ||||
| 3352 | return {0u, nullptr}; | ||||
| 3353 | case MVT::i1: | ||||
| 3354 | case MVT::i8: | ||||
| 3355 | case MVT::i16: | ||||
| 3356 | case MVT::i32: | ||||
| 3357 | case MVT::f32: | ||||
| 3358 | return {0u, &Hexagon::IntRegsRegClass}; | ||||
| 3359 | case MVT::i64: | ||||
| 3360 | case MVT::f64: | ||||
| 3361 | return {0u, &Hexagon::DoubleRegsRegClass}; | ||||
| 3362 | } | ||||
| 3363 | break; | ||||
| 3364 | case 'a': // M0-M1 | ||||
| 3365 | if (VT != MVT::i32) | ||||
| 3366 | return {0u, nullptr}; | ||||
| 3367 | return {0u, &Hexagon::ModRegsRegClass}; | ||||
| 3368 | case 'q': // q0-q3 | ||||
| 3369 | switch (VT.getSizeInBits()) { | ||||
| 3370 | default: | ||||
| 3371 | return {0u, nullptr}; | ||||
| 3372 | case 64: | ||||
| 3373 | case 128: | ||||
| 3374 | return {0u, &Hexagon::HvxQRRegClass}; | ||||
| 3375 | } | ||||
| 3376 | break; | ||||
| 3377 | case 'v': // V0-V31 | ||||
| 3378 | switch (VT.getSizeInBits()) { | ||||
| 3379 | default: | ||||
| 3380 | return {0u, nullptr}; | ||||
| 3381 | case 512: | ||||
| 3382 | return {0u, &Hexagon::HvxVRRegClass}; | ||||
| 3383 | case 1024: | ||||
| 3384 | if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps()) | ||||
| 3385 | return {0u, &Hexagon::HvxVRRegClass}; | ||||
| 3386 | return {0u, &Hexagon::HvxWRRegClass}; | ||||
| 3387 | case 2048: | ||||
| 3388 | return {0u, &Hexagon::HvxWRRegClass}; | ||||
| 3389 | } | ||||
| 3390 | break; | ||||
| 3391 | default: | ||||
| 3392 | return {0u, nullptr}; | ||||
| 3393 | } | ||||
| 3394 | } | ||||
| 3395 | |||||
| 3396 | return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); | ||||
| 3397 | } | ||||
| 3398 | |||||
| 3399 | /// isFPImmLegal - Returns true if the target can instruction select the | ||||
| 3400 | /// specified FP immediate natively. If false, the legalizer will | ||||
| 3401 | /// materialize the FP immediate as a load from a constant pool. | ||||
| 3402 | bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, | ||||
| 3403 | bool ForCodeSize) const { | ||||
| 3404 | return true; | ||||
| 3405 | } | ||||
| 3406 | |||||
| 3407 | /// isLegalAddressingMode - Return true if the addressing mode represented by | ||||
| 3408 | /// AM is legal for this target, for a load/store of the specified type. | ||||
| 3409 | bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL, | ||||
| 3410 | const AddrMode &AM, Type *Ty, | ||||
| 3411 | unsigned AS, Instruction *I) const { | ||||
| 3412 | if (Ty->isSized()) { | ||||
| 3413 | // When LSR detects uses of the same base address to access different | ||||
| 3414 | // types (e.g. unions), it will assume a conservative type for these | ||||
| 3415 | // uses: | ||||
| 3416 | // LSR Use: Kind=Address of void in addrspace(4294967295), ... | ||||
| 3417 | // The type Ty passed here would then be "void". Skip the alignment | ||||
| 3418 | // checks, but do not return false right away, since that confuses | ||||
| 3419 | // LSR into crashing. | ||||
| 3420 | Align A = DL.getABITypeAlign(Ty); | ||||
| 3421 | // The base offset must be a multiple of the alignment. | ||||
| 3422 | if (!isAligned(A, AM.BaseOffs)) | ||||
| 3423 | return false; | ||||
| 3424 | // The shifted offset must fit in 11 bits. | ||||
| 3425 | if (!isInt<11>(AM.BaseOffs >> Log2(A))) | ||||
| 3426 | return false; | ||||
| 3427 | } | ||||
| 3428 | |||||
| 3429 | // No global is ever allowed as a base. | ||||
| 3430 | if (AM.BaseGV) | ||||
| 3431 | return false; | ||||
| 3432 | |||||
| 3433 | int Scale = AM.Scale; | ||||
| 3434 | if (Scale < 0) | ||||
| 3435 | Scale = -Scale; | ||||
| 3436 | switch (Scale) { | ||||
| 3437 | case 0: // No scale reg, "r+i", "r", or just "i". | ||||
| 3438 | break; | ||||
| 3439 | default: // No scaled addressing mode. | ||||
| 3440 | return false; | ||||
| 3441 | } | ||||
| 3442 | return true; | ||||
| 3443 | } | ||||
| 3444 | |||||
| 3445 | /// Return true if folding a constant offset with the given GlobalAddress is | ||||
| 3446 | /// legal. It is frequently not legal in PIC relocation models. | ||||
| 3447 | bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) | ||||
| 3448 | const { | ||||
| 3449 | return HTM.getRelocationModel() == Reloc::Static; | ||||
| 3450 | } | ||||
| 3451 | |||||
| 3452 | /// isLegalICmpImmediate - Return true if the specified immediate is legal | ||||
| 3453 | /// icmp immediate, that is the target has icmp instructions which can compare | ||||
| 3454 | /// a register against the immediate without having to materialize the | ||||
| 3455 | /// immediate into a register. | ||||
| 3456 | bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const { | ||||
| 3457 | return Imm >= -512 && Imm <= 511; | ||||
| 3458 | } | ||||
| 3459 | |||||
| 3460 | /// IsEligibleForTailCallOptimization - Check whether the call is eligible | ||||
| 3461 | /// for tail call optimization. Targets which want to do tail call | ||||
| 3462 | /// optimization should implement this function. | ||||
| 3463 | bool HexagonTargetLowering::IsEligibleForTailCallOptimization( | ||||
| 3464 | SDValue Callee, | ||||
| 3465 | CallingConv::ID CalleeCC, | ||||
| 3466 | bool IsVarArg, | ||||
| 3467 | bool IsCalleeStructRet, | ||||
| 3468 | bool IsCallerStructRet, | ||||
| 3469 | const SmallVectorImpl<ISD::OutputArg> &Outs, | ||||
| 3470 | const SmallVectorImpl<SDValue> &OutVals, | ||||
| 3471 | const SmallVectorImpl<ISD::InputArg> &Ins, | ||||
| 3472 | SelectionDAG& DAG) const { | ||||
| 3473 | const Function &CallerF = DAG.getMachineFunction().getFunction(); | ||||
| 3474 | CallingConv::ID CallerCC = CallerF.getCallingConv(); | ||||
| 3475 | bool CCMatch = CallerCC == CalleeCC; | ||||
| 3476 | |||||
| 3477 | // *************************************************************************** | ||||
| 3478 | // Look for obvious safe cases to perform tail call optimization that do not | ||||
| 3479 | // require ABI changes. | ||||
| 3480 | // *************************************************************************** | ||||
| 3481 | |||||
| 3482 | // If this is a tail call via a function pointer, then don't do it! | ||||
| 3483 | if (!isa<GlobalAddressSDNode>(Callee) && | ||||
| 3484 | !isa<ExternalSymbolSDNode>(Callee)) { | ||||
| 3485 | return false; | ||||
| 3486 | } | ||||
| 3487 | |||||
| 3488 | // Do not optimize if the calling conventions do not match and the conventions | ||||
| 3489 | // used are not C or Fast. | ||||
| 3490 | if (!CCMatch) { | ||||
| 3491 | bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast); | ||||
| 3492 | bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast); | ||||
| 3493 | // If R & E, then ok. | ||||
| 3494 | if (!R || !E) | ||||
| 3495 | return false; | ||||
| 3496 | } | ||||
| 3497 | |||||
| 3498 | // Do not tail call optimize vararg calls. | ||||
| 3499 | if (IsVarArg) | ||||
| 3500 | return false; | ||||
| 3501 | |||||
| 3502 | // Also avoid tail call optimization if either caller or callee uses struct | ||||
| 3503 | // return semantics. | ||||
| 3504 | if (IsCalleeStructRet || IsCallerStructRet) | ||||
| 3505 | return false; | ||||
| 3506 | |||||
| 3507 | // In addition to the cases above, we also disable Tail Call Optimization if | ||||
| 3508 | // the calling convention code that at least one outgoing argument needs to | ||||
| 3509 | // go on the stack. We cannot check that here because at this point that | ||||
| 3510 | // information is not available. | ||||
| 3511 | return true; | ||||
| 3512 | } | ||||
| 3513 | |||||
| 3514 | /// Returns the target specific optimal type for load and store operations as | ||||
| 3515 | /// a result of memset, memcpy, and memmove lowering. | ||||
| 3516 | /// | ||||
| 3517 | /// If DstAlign is zero that means it's safe to destination alignment can | ||||
| 3518 | /// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't | ||||
| 3519 | /// a need to check it against alignment requirement, probably because the | ||||
| 3520 | /// source does not need to be loaded. If 'IsMemset' is true, that means it's | ||||
| 3521 | /// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of | ||||
| 3522 | /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it | ||||
| 3523 | /// does not need to be loaded. It returns EVT::Other if the type should be | ||||
| 3524 | /// determined using generic target-independent logic. | ||||
| 3525 | EVT HexagonTargetLowering::getOptimalMemOpType( | ||||
| 3526 | const MemOp &Op, const AttributeList &FuncAttributes) const { | ||||
| 3527 | if (Op.size() >= 8 && Op.isAligned(Align(8))) | ||||
| 3528 | return MVT::i64; | ||||
| 3529 | if (Op.size() >= 4 && Op.isAligned(Align(4))) | ||||
| 3530 | return MVT::i32; | ||||
| 3531 | if (Op.size() >= 2 && Op.isAligned(Align(2))) | ||||
| 3532 | return MVT::i16; | ||||
| 3533 | return MVT::Other; | ||||
| 3534 | } | ||||
| 3535 | |||||
| 3536 | bool HexagonTargetLowering::allowsMemoryAccess( | ||||
| 3537 | LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, | ||||
| 3538 | Align Alignment, MachineMemOperand::Flags Flags, bool *Fast) const { | ||||
| 3539 | MVT SVT = VT.getSimpleVT(); | ||||
| 3540 | if (Subtarget.isHVXVectorType(SVT, true)) | ||||
| 3541 | return allowsHvxMemoryAccess(SVT, Flags, Fast); | ||||
| 3542 | return TargetLoweringBase::allowsMemoryAccess( | ||||
| 3543 | Context, DL, VT, AddrSpace, Alignment, Flags, Fast); | ||||
| 3544 | } | ||||
| 3545 | |||||
| 3546 | bool HexagonTargetLowering::allowsMisalignedMemoryAccesses( | ||||
| 3547 | EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, | ||||
| 3548 | bool *Fast) const { | ||||
| 3549 | MVT SVT = VT.getSimpleVT(); | ||||
| 3550 | if (Subtarget.isHVXVectorType(SVT, true)) | ||||
| 3551 | return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast); | ||||
| 3552 | if (Fast) | ||||
| 3553 | *Fast = false; | ||||
| 3554 | return false; | ||||
| 3555 | } | ||||
| 3556 | |||||
| 3557 | std::pair<const TargetRegisterClass*, uint8_t> | ||||
| 3558 | HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI, | ||||
| 3559 | MVT VT) const { | ||||
| 3560 | if (Subtarget.isHVXVectorType(VT, true)) { | ||||
| 3561 | unsigned BitWidth = VT.getSizeInBits(); | ||||
| 3562 | unsigned VecWidth = Subtarget.getVectorLength() * 8; | ||||
| 3563 | |||||
| 3564 | if (VT.getVectorElementType() == MVT::i1) | ||||
| 3565 | return std::make_pair(&Hexagon::HvxQRRegClass, 1); | ||||
| 3566 | if (BitWidth == VecWidth) | ||||
| 3567 | return std::make_pair(&Hexagon::HvxVRRegClass, 1); | ||||
| 3568 | assert(BitWidth == 2 * VecWidth)(static_cast <bool> (BitWidth == 2 * VecWidth) ? void ( 0) : __assert_fail ("BitWidth == 2 * VecWidth", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3568, __extension__ __PRETTY_FUNCTION__)); | ||||
| 3569 | return std::make_pair(&Hexagon::HvxWRRegClass, 1); | ||||
| 3570 | } | ||||
| 3571 | |||||
| 3572 | return TargetLowering::findRepresentativeClass(TRI, VT); | ||||
| 3573 | } | ||||
| 3574 | |||||
| 3575 | bool HexagonTargetLowering::shouldReduceLoadWidth(SDNode *Load, | ||||
| 3576 | ISD::LoadExtType ExtTy, EVT NewVT) const { | ||||
| 3577 | // TODO: This may be worth removing. Check regression tests for diffs. | ||||
| 3578 | if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT)) | ||||
| 3579 | return false; | ||||
| 3580 | |||||
| 3581 | auto *L = cast<LoadSDNode>(Load); | ||||
| 3582 | std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr()); | ||||
| 3583 | // Small-data object, do not shrink. | ||||
| 3584 | if (BO.first.getOpcode() == HexagonISD::CONST32_GP) | ||||
| 3585 | return false; | ||||
| 3586 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) { | ||||
| 3587 | auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine()); | ||||
| 3588 | const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal()); | ||||
| 3589 | return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM); | ||||
| 3590 | } | ||||
| 3591 | return true; | ||||
| 3592 | } | ||||
| 3593 | |||||
| 3594 | Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder, | ||||
| 3595 | Type *ValueTy, Value *Addr, | ||||
| 3596 | AtomicOrdering Ord) const { | ||||
| 3597 | BasicBlock *BB = Builder.GetInsertBlock(); | ||||
| 3598 | Module *M = BB->getParent()->getParent(); | ||||
| 3599 | unsigned SZ = ValueTy->getPrimitiveSizeInBits(); | ||||
| 3600 | assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported" ) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic loads supported\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3600, __extension__ __PRETTY_FUNCTION__)); | ||||
| 3601 | Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked | ||||
| 3602 | : Intrinsic::hexagon_L4_loadd_locked; | ||||
| 3603 | Function *Fn = Intrinsic::getDeclaration(M, IntID); | ||||
| 3604 | |||||
| 3605 | auto PtrTy = cast<PointerType>(Addr->getType()); | ||||
| 3606 | PointerType *NewPtrTy = | ||||
| 3607 | Builder.getIntNTy(SZ)->getPointerTo(PtrTy->getAddressSpace()); | ||||
| 3608 | Addr = Builder.CreateBitCast(Addr, NewPtrTy); | ||||
| 3609 | |||||
| 3610 | Value *Call = Builder.CreateCall(Fn, Addr, "larx"); | ||||
| 3611 | |||||
| 3612 | return Builder.CreateBitCast(Call, ValueTy); | ||||
| 3613 | } | ||||
| 3614 | |||||
| 3615 | /// Perform a store-conditional operation to Addr. Return the status of the | ||||
| 3616 | /// store. This should be 0 if the store succeeded, non-zero otherwise. | ||||
| 3617 | Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder, | ||||
| 3618 | Value *Val, Value *Addr, | ||||
| 3619 | AtomicOrdering Ord) const { | ||||
| 3620 | BasicBlock *BB = Builder.GetInsertBlock(); | ||||
| 3621 | Module *M = BB->getParent()->getParent(); | ||||
| 3622 | Type *Ty = Val->getType(); | ||||
| 3623 | unsigned SZ = Ty->getPrimitiveSizeInBits(); | ||||
| 3624 | |||||
| 3625 | Type *CastTy = Builder.getIntNTy(SZ); | ||||
| 3626 | assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported")(static_cast <bool> ((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported" ) ? void (0) : __assert_fail ("(SZ == 32 || SZ == 64) && \"Only 32/64-bit atomic stores supported\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp" , 3626, __extension__ __PRETTY_FUNCTION__)); | ||||
| 3627 | Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked | ||||
| 3628 | : Intrinsic::hexagon_S4_stored_locked; | ||||
| 3629 | Function *Fn = Intrinsic::getDeclaration(M, IntID); | ||||
| 3630 | |||||
| 3631 | unsigned AS = Addr->getType()->getPointerAddressSpace(); | ||||
| 3632 | Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS)); | ||||
| 3633 | Val = Builder.CreateBitCast(Val, CastTy); | ||||
| 3634 | |||||
| 3635 | Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx"); | ||||
| 3636 | Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), ""); | ||||
| 3637 | Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext())); | ||||
| 3638 | return Ext; | ||||
| 3639 | } | ||||
| 3640 | |||||
| 3641 | TargetLowering::AtomicExpansionKind | ||||
| 3642 | HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { | ||||
| 3643 | // Do not expand loads and stores that don't exceed 64 bits. | ||||
| 3644 | return LI->getType()->getPrimitiveSizeInBits() > 64 | ||||
| 3645 | ? AtomicExpansionKind::LLOnly | ||||
| 3646 | : AtomicExpansionKind::None; | ||||
| 3647 | } | ||||
| 3648 | |||||
| 3649 | bool HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const { | ||||
| 3650 | // Do not expand loads and stores that don't exceed 64 bits. | ||||
| 3651 | return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64; | ||||
| 3652 | } | ||||
| 3653 | |||||
| 3654 | TargetLowering::AtomicExpansionKind | ||||
| 3655 | HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR( | ||||
| 3656 | AtomicCmpXchgInst *AI) const { | ||||
| 3657 | return AtomicExpansionKind::LLSC; | ||||
| 3658 | } |
| 1 | //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // This file declares the SDNode class and derived classes, which are used to | |||
| 10 | // represent the nodes and operations present in a SelectionDAG. These nodes | |||
| 11 | // and operations are machine code level operations, with some similarities to | |||
| 12 | // the GCC RTL representation. | |||
| 13 | // | |||
| 14 | // Clients should include the SelectionDAG.h file instead of this file directly. | |||
| 15 | // | |||
| 16 | //===----------------------------------------------------------------------===// | |||
| 17 | ||||
| 18 | #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
| 19 | #define LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
| 20 | ||||
| 21 | #include "llvm/ADT/APFloat.h" | |||
| 22 | #include "llvm/ADT/ArrayRef.h" | |||
| 23 | #include "llvm/ADT/BitVector.h" | |||
| 24 | #include "llvm/ADT/FoldingSet.h" | |||
| 25 | #include "llvm/ADT/GraphTraits.h" | |||
| 26 | #include "llvm/ADT/SmallPtrSet.h" | |||
| 27 | #include "llvm/ADT/SmallVector.h" | |||
| 28 | #include "llvm/ADT/ilist_node.h" | |||
| 29 | #include "llvm/ADT/iterator.h" | |||
| 30 | #include "llvm/ADT/iterator_range.h" | |||
| 31 | #include "llvm/CodeGen/ISDOpcodes.h" | |||
| 32 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
| 33 | #include "llvm/CodeGen/Register.h" | |||
| 34 | #include "llvm/CodeGen/ValueTypes.h" | |||
| 35 | #include "llvm/IR/Constants.h" | |||
| 36 | #include "llvm/IR/DebugLoc.h" | |||
| 37 | #include "llvm/IR/Instruction.h" | |||
| 38 | #include "llvm/IR/Instructions.h" | |||
| 39 | #include "llvm/IR/Metadata.h" | |||
| 40 | #include "llvm/IR/Operator.h" | |||
| 41 | #include "llvm/Support/AlignOf.h" | |||
| 42 | #include "llvm/Support/AtomicOrdering.h" | |||
| 43 | #include "llvm/Support/Casting.h" | |||
| 44 | #include "llvm/Support/ErrorHandling.h" | |||
| 45 | #include "llvm/Support/MachineValueType.h" | |||
| 46 | #include "llvm/Support/TypeSize.h" | |||
| 47 | #include <algorithm> | |||
| 48 | #include <cassert> | |||
| 49 | #include <climits> | |||
| 50 | #include <cstddef> | |||
| 51 | #include <cstdint> | |||
| 52 | #include <cstring> | |||
| 53 | #include <iterator> | |||
| 54 | #include <string> | |||
| 55 | #include <tuple> | |||
| 56 | ||||
| 57 | namespace llvm { | |||
| 58 | ||||
| 59 | class APInt; | |||
| 60 | class Constant; | |||
| 61 | template <typename T> struct DenseMapInfo; | |||
| 62 | class GlobalValue; | |||
| 63 | class MachineBasicBlock; | |||
| 64 | class MachineConstantPoolValue; | |||
| 65 | class MCSymbol; | |||
| 66 | class raw_ostream; | |||
| 67 | class SDNode; | |||
| 68 | class SelectionDAG; | |||
| 69 | class Type; | |||
| 70 | class Value; | |||
| 71 | ||||
| 72 | void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, | |||
| 73 | bool force = false); | |||
| 74 | ||||
| 75 | /// This represents a list of ValueType's that has been intern'd by | |||
| 76 | /// a SelectionDAG. Instances of this simple value class are returned by | |||
| 77 | /// SelectionDAG::getVTList(...). | |||
| 78 | /// | |||
| 79 | struct SDVTList { | |||
| 80 | const EVT *VTs; | |||
| 81 | unsigned int NumVTs; | |||
| 82 | }; | |||
| 83 | ||||
| 84 | namespace ISD { | |||
| 85 | ||||
| 86 | /// Node predicates | |||
| 87 | ||||
| 88 | /// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the | |||
| 89 | /// same constant or undefined, return true and return the constant value in | |||
| 90 | /// \p SplatValue. | |||
| 91 | bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); | |||
| 92 | ||||
| 93 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
| 94 | /// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to | |||
| 95 | /// true, it only checks BUILD_VECTOR. | |||
| 96 | bool isConstantSplatVectorAllOnes(const SDNode *N, | |||
| 97 | bool BuildVectorOnly = false); | |||
| 98 | ||||
| 99 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
| 100 | /// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it | |||
| 101 | /// only checks BUILD_VECTOR. | |||
| 102 | bool isConstantSplatVectorAllZeros(const SDNode *N, | |||
| 103 | bool BuildVectorOnly = false); | |||
| 104 | ||||
| 105 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
| 106 | /// elements are ~0 or undef. | |||
| 107 | bool isBuildVectorAllOnes(const SDNode *N); | |||
| 108 | ||||
| 109 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
| 110 | /// elements are 0 or undef. | |||
| 111 | bool isBuildVectorAllZeros(const SDNode *N); | |||
| 112 | ||||
| 113 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
| 114 | /// ConstantSDNode or undef. | |||
| 115 | bool isBuildVectorOfConstantSDNodes(const SDNode *N); | |||
| 116 | ||||
| 117 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
| 118 | /// ConstantFPSDNode or undef. | |||
| 119 | bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); | |||
| 120 | ||||
| 121 | /// Return true if the node has at least one operand and all operands of the | |||
| 122 | /// specified node are ISD::UNDEF. | |||
| 123 | bool allOperandsUndef(const SDNode *N); | |||
| 124 | ||||
| 125 | } // end namespace ISD | |||
| 126 | ||||
| 127 | //===----------------------------------------------------------------------===// | |||
| 128 | /// Unlike LLVM values, Selection DAG nodes may return multiple | |||
| 129 | /// values as the result of a computation. Many nodes return multiple values, | |||
| 130 | /// from loads (which define a token and a return value) to ADDC (which returns | |||
| 131 | /// a result and a carry value), to calls (which may return an arbitrary number | |||
| 132 | /// of values). | |||
| 133 | /// | |||
| 134 | /// As such, each use of a SelectionDAG computation must indicate the node that | |||
| 135 | /// computes it as well as which return value to use from that node. This pair | |||
| 136 | /// of information is represented with the SDValue value type. | |||
| 137 | /// | |||
| 138 | class SDValue { | |||
| 139 | friend struct DenseMapInfo<SDValue>; | |||
| 140 | ||||
| 141 | SDNode *Node = nullptr; // The node defining the value we are using. | |||
| 142 | unsigned ResNo = 0; // Which return value of the node we are using. | |||
| 143 | ||||
| 144 | public: | |||
| 145 | SDValue() = default; | |||
| 146 | SDValue(SDNode *node, unsigned resno); | |||
| 147 | ||||
| 148 | /// get the index which selects a specific result in the SDNode | |||
| 149 | unsigned getResNo() const { return ResNo; } | |||
| 150 | ||||
| 151 | /// get the SDNode which holds the desired result | |||
| 152 | SDNode *getNode() const { return Node; } | |||
| 153 | ||||
| 154 | /// set the SDNode | |||
| 155 | void setNode(SDNode *N) { Node = N; } | |||
| 156 | ||||
| 157 | inline SDNode *operator->() const { return Node; } | |||
| 158 | ||||
| 159 | bool operator==(const SDValue &O) const { | |||
| 160 | return Node == O.Node && ResNo == O.ResNo; | |||
| 161 | } | |||
| 162 | bool operator!=(const SDValue &O) const { | |||
| 163 | return !operator==(O); | |||
| 164 | } | |||
| 165 | bool operator<(const SDValue &O) const { | |||
| 166 | return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); | |||
| 167 | } | |||
| 168 | explicit operator bool() const { | |||
| 169 | return Node != nullptr; | |||
| 170 | } | |||
| 171 | ||||
| 172 | SDValue getValue(unsigned R) const { | |||
| 173 | return SDValue(Node, R); | |||
| 174 | } | |||
| 175 | ||||
| 176 | /// Return true if this node is an operand of N. | |||
| 177 | bool isOperandOf(const SDNode *N) const; | |||
| 178 | ||||
| 179 | /// Return the ValueType of the referenced return value. | |||
| 180 | inline EVT getValueType() const; | |||
| 181 | ||||
| 182 | /// Return the simple ValueType of the referenced return value. | |||
| 183 | MVT getSimpleValueType() const { | |||
| 184 | return getValueType().getSimpleVT(); | |||
| 185 | } | |||
| 186 | ||||
| 187 | /// Returns the size of the value in bits. | |||
| 188 | /// | |||
| 189 | /// If the value type is a scalable vector type, the scalable property will | |||
| 190 | /// be set and the runtime size will be a positive integer multiple of the | |||
| 191 | /// base size. | |||
| 192 | TypeSize getValueSizeInBits() const { | |||
| 193 | return getValueType().getSizeInBits(); | |||
| 194 | } | |||
| 195 | ||||
| 196 | uint64_t getScalarValueSizeInBits() const { | |||
| 197 | return getValueType().getScalarType().getFixedSizeInBits(); | |||
| 198 | } | |||
| 199 | ||||
| 200 | // Forwarding methods - These forward to the corresponding methods in SDNode. | |||
| 201 | inline unsigned getOpcode() const; | |||
| 202 | inline unsigned getNumOperands() const; | |||
| 203 | inline const SDValue &getOperand(unsigned i) const; | |||
| 204 | inline uint64_t getConstantOperandVal(unsigned i) const; | |||
| 205 | inline const APInt &getConstantOperandAPInt(unsigned i) const; | |||
| 206 | inline bool isTargetMemoryOpcode() const; | |||
| 207 | inline bool isTargetOpcode() const; | |||
| 208 | inline bool isMachineOpcode() const; | |||
| 209 | inline bool isUndef() const; | |||
| 210 | inline unsigned getMachineOpcode() const; | |||
| 211 | inline const DebugLoc &getDebugLoc() const; | |||
| 212 | inline void dump() const; | |||
| 213 | inline void dump(const SelectionDAG *G) const; | |||
| 214 | inline void dumpr() const; | |||
| 215 | inline void dumpr(const SelectionDAG *G) const; | |||
| 216 | ||||
| 217 | /// Return true if this operand (which must be a chain) reaches the | |||
| 218 | /// specified operand without crossing any side-effecting instructions. | |||
| 219 | /// In practice, this looks through token factors and non-volatile loads. | |||
| 220 | /// In order to remain efficient, this only | |||
| 221 | /// looks a couple of nodes in, it does not do an exhaustive search. | |||
| 222 | bool reachesChainWithoutSideEffects(SDValue Dest, | |||
| 223 | unsigned Depth = 2) const; | |||
| 224 | ||||
| 225 | /// Return true if there are no nodes using value ResNo of Node. | |||
| 226 | inline bool use_empty() const; | |||
| 227 | ||||
| 228 | /// Return true if there is exactly one node using value ResNo of Node. | |||
| 229 | inline bool hasOneUse() const; | |||
| 230 | }; | |||
| 231 | ||||
| 232 | template<> struct DenseMapInfo<SDValue> { | |||
| 233 | static inline SDValue getEmptyKey() { | |||
| 234 | SDValue V; | |||
| 235 | V.ResNo = -1U; | |||
| 236 | return V; | |||
| 237 | } | |||
| 238 | ||||
| 239 | static inline SDValue getTombstoneKey() { | |||
| 240 | SDValue V; | |||
| 241 | V.ResNo = -2U; | |||
| 242 | return V; | |||
| 243 | } | |||
| 244 | ||||
| 245 | static unsigned getHashValue(const SDValue &Val) { | |||
| 246 | return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ | |||
| 247 | (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo(); | |||
| 248 | } | |||
| 249 | ||||
| 250 | static bool isEqual(const SDValue &LHS, const SDValue &RHS) { | |||
| 251 | return LHS == RHS; | |||
| 252 | } | |||
| 253 | }; | |||
| 254 | ||||
| 255 | /// Allow casting operators to work directly on | |||
| 256 | /// SDValues as if they were SDNode*'s. | |||
| 257 | template<> struct simplify_type<SDValue> { | |||
| 258 | using SimpleType = SDNode *; | |||
| 259 | ||||
| 260 | static SimpleType getSimplifiedValue(SDValue &Val) { | |||
| 261 | return Val.getNode(); | |||
| 262 | } | |||
| 263 | }; | |||
| 264 | template<> struct simplify_type<const SDValue> { | |||
| 265 | using SimpleType = /*const*/ SDNode *; | |||
| 266 | ||||
| 267 | static SimpleType getSimplifiedValue(const SDValue &Val) { | |||
| 268 | return Val.getNode(); | |||
| 269 | } | |||
| 270 | }; | |||
| 271 | ||||
| 272 | /// Represents a use of a SDNode. This class holds an SDValue, | |||
| 273 | /// which records the SDNode being used and the result number, a | |||
| 274 | /// pointer to the SDNode using the value, and Next and Prev pointers, | |||
| 275 | /// which link together all the uses of an SDNode. | |||
| 276 | /// | |||
| 277 | class SDUse { | |||
| 278 | /// Val - The value being used. | |||
| 279 | SDValue Val; | |||
| 280 | /// User - The user of this value. | |||
| 281 | SDNode *User = nullptr; | |||
| 282 | /// Prev, Next - Pointers to the uses list of the SDNode referred by | |||
| 283 | /// this operand. | |||
| 284 | SDUse **Prev = nullptr; | |||
| 285 | SDUse *Next = nullptr; | |||
| 286 | ||||
| 287 | public: | |||
| 288 | SDUse() = default; | |||
| 289 | SDUse(const SDUse &U) = delete; | |||
| 290 | SDUse &operator=(const SDUse &) = delete; | |||
| 291 | ||||
| 292 | /// Normally SDUse will just implicitly convert to an SDValue that it holds. | |||
| 293 | operator const SDValue&() const { return Val; } | |||
| 294 | ||||
| 295 | /// If implicit conversion to SDValue doesn't work, the get() method returns | |||
| 296 | /// the SDValue. | |||
| 297 | const SDValue &get() const { return Val; } | |||
| 298 | ||||
| 299 | /// This returns the SDNode that contains this Use. | |||
| 300 | SDNode *getUser() { return User; } | |||
| 301 | ||||
| 302 | /// Get the next SDUse in the use list. | |||
| 303 | SDUse *getNext() const { return Next; } | |||
| 304 | ||||
| 305 | /// Convenience function for get().getNode(). | |||
| 306 | SDNode *getNode() const { return Val.getNode(); } | |||
| 307 | /// Convenience function for get().getResNo(). | |||
| 308 | unsigned getResNo() const { return Val.getResNo(); } | |||
| 309 | /// Convenience function for get().getValueType(). | |||
| 310 | EVT getValueType() const { return Val.getValueType(); } | |||
| 311 | ||||
| 312 | /// Convenience function for get().operator== | |||
| 313 | bool operator==(const SDValue &V) const { | |||
| 314 | return Val == V; | |||
| 315 | } | |||
| 316 | ||||
| 317 | /// Convenience function for get().operator!= | |||
| 318 | bool operator!=(const SDValue &V) const { | |||
| 319 | return Val != V; | |||
| 320 | } | |||
| 321 | ||||
| 322 | /// Convenience function for get().operator< | |||
| 323 | bool operator<(const SDValue &V) const { | |||
| 324 | return Val < V; | |||
| 325 | } | |||
| 326 | ||||
| 327 | private: | |||
| 328 | friend class SelectionDAG; | |||
| 329 | friend class SDNode; | |||
| 330 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
| 331 | friend class HandleSDNode; | |||
| 332 | ||||
| 333 | void setUser(SDNode *p) { User = p; } | |||
| 334 | ||||
| 335 | /// Remove this use from its existing use list, assign it the | |||
| 336 | /// given value, and add it to the new value's node's use list. | |||
| 337 | inline void set(const SDValue &V); | |||
| 338 | /// Like set, but only supports initializing a newly-allocated | |||
| 339 | /// SDUse with a non-null value. | |||
| 340 | inline void setInitial(const SDValue &V); | |||
| 341 | /// Like set, but only sets the Node portion of the value, | |||
| 342 | /// leaving the ResNo portion unmodified. | |||
| 343 | inline void setNode(SDNode *N); | |||
| 344 | ||||
| 345 | void addToList(SDUse **List) { | |||
| 346 | Next = *List; | |||
| 347 | if (Next) Next->Prev = &Next; | |||
| 348 | Prev = List; | |||
| 349 | *List = this; | |||
| 350 | } | |||
| 351 | ||||
| 352 | void removeFromList() { | |||
| 353 | *Prev = Next; | |||
| 354 | if (Next) Next->Prev = Prev; | |||
| 355 | } | |||
| 356 | }; | |||
| 357 | ||||
| 358 | /// simplify_type specializations - Allow casting operators to work directly on | |||
| 359 | /// SDValues as if they were SDNode*'s. | |||
| 360 | template<> struct simplify_type<SDUse> { | |||
| 361 | using SimpleType = SDNode *; | |||
| 362 | ||||
| 363 | static SimpleType getSimplifiedValue(SDUse &Val) { | |||
| 364 | return Val.getNode(); | |||
| 365 | } | |||
| 366 | }; | |||
| 367 | ||||
| 368 | /// These are IR-level optimization flags that may be propagated to SDNodes. | |||
| 369 | /// TODO: This data structure should be shared by the IR optimizer and the | |||
| 370 | /// the backend. | |||
| 371 | struct SDNodeFlags { | |||
| 372 | private: | |||
| 373 | bool NoUnsignedWrap : 1; | |||
| 374 | bool NoSignedWrap : 1; | |||
| 375 | bool Exact : 1; | |||
| 376 | bool NoNaNs : 1; | |||
| 377 | bool NoInfs : 1; | |||
| 378 | bool NoSignedZeros : 1; | |||
| 379 | bool AllowReciprocal : 1; | |||
| 380 | bool AllowContract : 1; | |||
| 381 | bool ApproximateFuncs : 1; | |||
| 382 | bool AllowReassociation : 1; | |||
| 383 | ||||
| 384 | // We assume instructions do not raise floating-point exceptions by default, | |||
| 385 | // and only those marked explicitly may do so. We could choose to represent | |||
| 386 | // this via a positive "FPExcept" flags like on the MI level, but having a | |||
| 387 | // negative "NoFPExcept" flag here (that defaults to true) makes the flag | |||
| 388 | // intersection logic more straightforward. | |||
| 389 | bool NoFPExcept : 1; | |||
| 390 | ||||
| 391 | public: | |||
| 392 | /// Default constructor turns off all optimization flags. | |||
| 393 | SDNodeFlags() | |||
| 394 | : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false), | |||
| 395 | NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), | |||
| 396 | AllowContract(false), ApproximateFuncs(false), | |||
| 397 | AllowReassociation(false), NoFPExcept(false) {} | |||
| 398 | ||||
| 399 | /// Propagate the fast-math-flags from an IR FPMathOperator. | |||
| 400 | void copyFMF(const FPMathOperator &FPMO) { | |||
| 401 | setNoNaNs(FPMO.hasNoNaNs()); | |||
| 402 | setNoInfs(FPMO.hasNoInfs()); | |||
| 403 | setNoSignedZeros(FPMO.hasNoSignedZeros()); | |||
| 404 | setAllowReciprocal(FPMO.hasAllowReciprocal()); | |||
| 405 | setAllowContract(FPMO.hasAllowContract()); | |||
| 406 | setApproximateFuncs(FPMO.hasApproxFunc()); | |||
| 407 | setAllowReassociation(FPMO.hasAllowReassoc()); | |||
| 408 | } | |||
| 409 | ||||
| 410 | // These are mutators for each flag. | |||
| 411 | void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } | |||
| 412 | void setNoSignedWrap(bool b) { NoSignedWrap = b; } | |||
| 413 | void setExact(bool b) { Exact = b; } | |||
| 414 | void setNoNaNs(bool b) { NoNaNs = b; } | |||
| 415 | void setNoInfs(bool b) { NoInfs = b; } | |||
| 416 | void setNoSignedZeros(bool b) { NoSignedZeros = b; } | |||
| 417 | void setAllowReciprocal(bool b) { AllowReciprocal = b; } | |||
| 418 | void setAllowContract(bool b) { AllowContract = b; } | |||
| 419 | void setApproximateFuncs(bool b) { ApproximateFuncs = b; } | |||
| 420 | void setAllowReassociation(bool b) { AllowReassociation = b; } | |||
| 421 | void setNoFPExcept(bool b) { NoFPExcept = b; } | |||
| 422 | ||||
| 423 | // These are accessors for each flag. | |||
| 424 | bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } | |||
| 425 | bool hasNoSignedWrap() const { return NoSignedWrap; } | |||
| 426 | bool hasExact() const { return Exact; } | |||
| 427 | bool hasNoNaNs() const { return NoNaNs; } | |||
| 428 | bool hasNoInfs() const { return NoInfs; } | |||
| 429 | bool hasNoSignedZeros() const { return NoSignedZeros; } | |||
| 430 | bool hasAllowReciprocal() const { return AllowReciprocal; } | |||
| 431 | bool hasAllowContract() const { return AllowContract; } | |||
| 432 | bool hasApproximateFuncs() const { return ApproximateFuncs; } | |||
| 433 | bool hasAllowReassociation() const { return AllowReassociation; } | |||
| 434 | bool hasNoFPExcept() const { return NoFPExcept; } | |||
| 435 | ||||
| 436 | /// Clear any flags in this flag set that aren't also set in Flags. All | |||
| 437 | /// flags will be cleared if Flags are undefined. | |||
| 438 | void intersectWith(const SDNodeFlags Flags) { | |||
| 439 | NoUnsignedWrap &= Flags.NoUnsignedWrap; | |||
| 440 | NoSignedWrap &= Flags.NoSignedWrap; | |||
| 441 | Exact &= Flags.Exact; | |||
| 442 | NoNaNs &= Flags.NoNaNs; | |||
| 443 | NoInfs &= Flags.NoInfs; | |||
| 444 | NoSignedZeros &= Flags.NoSignedZeros; | |||
| 445 | AllowReciprocal &= Flags.AllowReciprocal; | |||
| 446 | AllowContract &= Flags.AllowContract; | |||
| 447 | ApproximateFuncs &= Flags.ApproximateFuncs; | |||
| 448 | AllowReassociation &= Flags.AllowReassociation; | |||
| 449 | NoFPExcept &= Flags.NoFPExcept; | |||
| 450 | } | |||
| 451 | }; | |||
| 452 | ||||
| 453 | /// Represents one node in the SelectionDAG. | |||
| 454 | /// | |||
| 455 | class SDNode : public FoldingSetNode, public ilist_node<SDNode> { | |||
| 456 | private: | |||
| 457 | /// The operation that this node performs. | |||
| 458 | int16_t NodeType; | |||
| 459 | ||||
| 460 | protected: | |||
| 461 | // We define a set of mini-helper classes to help us interpret the bits in our | |||
| 462 | // SubclassData. These are designed to fit within a uint16_t so they pack | |||
| 463 | // with NodeType. | |||
| 464 | ||||
| 465 | #if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1)) | |||
| 466 | // Except for GCC; by default, AIX compilers store bit-fields in 4-byte words | |||
| 467 | // and give the `pack` pragma push semantics. | |||
| 468 | #define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2) | |||
| 469 | #define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop) | |||
| 470 | #else | |||
| 471 | #define BEGIN_TWO_BYTE_PACK() | |||
| 472 | #define END_TWO_BYTE_PACK() | |||
| 473 | #endif | |||
| 474 | ||||
| 475 | BEGIN_TWO_BYTE_PACK() | |||
| 476 | class SDNodeBitfields { | |||
| 477 | friend class SDNode; | |||
| 478 | friend class MemIntrinsicSDNode; | |||
| 479 | friend class MemSDNode; | |||
| 480 | friend class SelectionDAG; | |||
| 481 | ||||
| 482 | uint16_t HasDebugValue : 1; | |||
| 483 | uint16_t IsMemIntrinsic : 1; | |||
| 484 | uint16_t IsDivergent : 1; | |||
| 485 | }; | |||
| 486 | enum { NumSDNodeBits = 3 }; | |||
| 487 | ||||
| 488 | class ConstantSDNodeBitfields { | |||
| 489 | friend class ConstantSDNode; | |||
| 490 | ||||
| 491 | uint16_t : NumSDNodeBits; | |||
| 492 | ||||
| 493 | uint16_t IsOpaque : 1; | |||
| 494 | }; | |||
| 495 | ||||
| 496 | class MemSDNodeBitfields { | |||
| 497 | friend class MemSDNode; | |||
| 498 | friend class MemIntrinsicSDNode; | |||
| 499 | friend class AtomicSDNode; | |||
| 500 | ||||
| 501 | uint16_t : NumSDNodeBits; | |||
| 502 | ||||
| 503 | uint16_t IsVolatile : 1; | |||
| 504 | uint16_t IsNonTemporal : 1; | |||
| 505 | uint16_t IsDereferenceable : 1; | |||
| 506 | uint16_t IsInvariant : 1; | |||
| 507 | }; | |||
| 508 | enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; | |||
| 509 | ||||
| 510 | class LSBaseSDNodeBitfields { | |||
| 511 | friend class LSBaseSDNode; | |||
| 512 | friend class MaskedLoadStoreSDNode; | |||
| 513 | friend class MaskedGatherScatterSDNode; | |||
| 514 | ||||
| 515 | uint16_t : NumMemSDNodeBits; | |||
| 516 | ||||
| 517 | // This storage is shared between disparate class hierarchies to hold an | |||
| 518 | // enumeration specific to the class hierarchy in use. | |||
| 519 | // LSBaseSDNode => enum ISD::MemIndexedMode | |||
| 520 | // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode | |||
| 521 | // MaskedGatherScatterSDNode => enum ISD::MemIndexType | |||
| 522 | uint16_t AddressingMode : 3; | |||
| 523 | }; | |||
| 524 | enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; | |||
| 525 | ||||
| 526 | class LoadSDNodeBitfields { | |||
| 527 | friend class LoadSDNode; | |||
| 528 | friend class MaskedLoadSDNode; | |||
| 529 | friend class MaskedGatherSDNode; | |||
| 530 | ||||
| 531 | uint16_t : NumLSBaseSDNodeBits; | |||
| 532 | ||||
| 533 | uint16_t ExtTy : 2; // enum ISD::LoadExtType | |||
| 534 | uint16_t IsExpanding : 1; | |||
| 535 | }; | |||
| 536 | ||||
| 537 | class StoreSDNodeBitfields { | |||
| 538 | friend class StoreSDNode; | |||
| 539 | friend class MaskedStoreSDNode; | |||
| 540 | friend class MaskedScatterSDNode; | |||
| 541 | ||||
| 542 | uint16_t : NumLSBaseSDNodeBits; | |||
| 543 | ||||
| 544 | uint16_t IsTruncating : 1; | |||
| 545 | uint16_t IsCompressing : 1; | |||
| 546 | }; | |||
| 547 | ||||
| 548 | union { | |||
| 549 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
| 550 | SDNodeBitfields SDNodeBits; | |||
| 551 | ConstantSDNodeBitfields ConstantSDNodeBits; | |||
| 552 | MemSDNodeBitfields MemSDNodeBits; | |||
| 553 | LSBaseSDNodeBitfields LSBaseSDNodeBits; | |||
| 554 | LoadSDNodeBitfields LoadSDNodeBits; | |||
| 555 | StoreSDNodeBitfields StoreSDNodeBits; | |||
| 556 | }; | |||
| 557 | END_TWO_BYTE_PACK() | |||
| 558 | #undef BEGIN_TWO_BYTE_PACK | |||
| 559 | #undef END_TWO_BYTE_PACK | |||
| 560 | ||||
| 561 | // RawSDNodeBits must cover the entirety of the union. This means that all of | |||
| 562 | // the union's members must have size <= RawSDNodeBits. We write the RHS as | |||
| 563 | // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter. | |||
| 564 | static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide"); | |||
| 565 | static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide"); | |||
| 566 | static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide"); | |||
| 567 | static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide"); | |||
| 568 | static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide"); | |||
| 569 | static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide"); | |||
| 570 | ||||
| 571 | private: | |||
| 572 | friend class SelectionDAG; | |||
| 573 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
| 574 | friend class HandleSDNode; | |||
| 575 | ||||
| 576 | /// Unique id per SDNode in the DAG. | |||
| 577 | int NodeId = -1; | |||
| 578 | ||||
| 579 | /// The values that are used by this operation. | |||
| 580 | SDUse *OperandList = nullptr; | |||
| 581 | ||||
| 582 | /// The types of the values this node defines. SDNode's may | |||
| 583 | /// define multiple values simultaneously. | |||
| 584 | const EVT *ValueList; | |||
| 585 | ||||
| 586 | /// List of uses for this SDNode. | |||
| 587 | SDUse *UseList = nullptr; | |||
| 588 | ||||
| 589 | /// The number of entries in the Operand/Value list. | |||
| 590 | unsigned short NumOperands = 0; | |||
| 591 | unsigned short NumValues; | |||
| 592 | ||||
| 593 | // The ordering of the SDNodes. It roughly corresponds to the ordering of the | |||
| 594 | // original LLVM instructions. | |||
| 595 | // This is used for turning off scheduling, because we'll forgo | |||
| 596 | // the normal scheduling algorithms and output the instructions according to | |||
| 597 | // this ordering. | |||
| 598 | unsigned IROrder; | |||
| 599 | ||||
| 600 | /// Source line information. | |||
| 601 | DebugLoc debugLoc; | |||
| 602 | ||||
| 603 | /// Return a pointer to the specified value type. | |||
| 604 | static const EVT *getValueTypeList(EVT VT); | |||
| 605 | ||||
| 606 | SDNodeFlags Flags; | |||
| 607 | ||||
| 608 | public: | |||
| 609 | /// Unique and persistent id per SDNode in the DAG. | |||
| 610 | /// Used for debug printing. | |||
| 611 | uint16_t PersistentId; | |||
| 612 | ||||
| 613 | //===--------------------------------------------------------------------===// | |||
| 614 | // Accessors | |||
| 615 | // | |||
| 616 | ||||
| 617 | /// Return the SelectionDAG opcode value for this node. For | |||
| 618 | /// pre-isel nodes (those for which isMachineOpcode returns false), these | |||
| 619 | /// are the opcode values in the ISD and <target>ISD namespaces. For | |||
| 620 | /// post-isel opcodes, see getMachineOpcode. | |||
| 621 | unsigned getOpcode() const { return (unsigned short)NodeType; } | |||
| 622 | ||||
| 623 | /// Test if this node has a target-specific opcode (in the | |||
| 624 | /// \<target\>ISD namespace). | |||
| 625 | bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } | |||
| 626 | ||||
| 627 | /// Test if this node has a target-specific opcode that may raise | |||
| 628 | /// FP exceptions (in the \<target\>ISD namespace and greater than | |||
| 629 | /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory | |||
| 630 | /// opcode are currently automatically considered to possibly raise | |||
| 631 | /// FP exceptions as well. | |||
| 632 | bool isTargetStrictFPOpcode() const { | |||
| 633 | return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE; | |||
| 634 | } | |||
| 635 | ||||
| 636 | /// Test if this node has a target-specific | |||
| 637 | /// memory-referencing opcode (in the \<target\>ISD namespace and | |||
| 638 | /// greater than FIRST_TARGET_MEMORY_OPCODE). | |||
| 639 | bool isTargetMemoryOpcode() const { | |||
| 640 | return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; | |||
| 641 | } | |||
| 642 | ||||
| 643 | /// Return true if the type of the node type undefined. | |||
| 644 | bool isUndef() const { return NodeType == ISD::UNDEF; } | |||
| 645 | ||||
| 646 | /// Test if this node is a memory intrinsic (with valid pointer information). | |||
| 647 | /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for | |||
| 648 | /// non-memory intrinsics (with chains) that are not really instances of | |||
| 649 | /// MemSDNode. For such nodes, we need some extra state to determine the | |||
| 650 | /// proper classof relationship. | |||
| 651 | bool isMemIntrinsic() const { | |||
| 652 | return (NodeType == ISD::INTRINSIC_W_CHAIN || | |||
| 653 | NodeType == ISD::INTRINSIC_VOID) && | |||
| 654 | SDNodeBits.IsMemIntrinsic; | |||
| 655 | } | |||
| 656 | ||||
| 657 | /// Test if this node is a strict floating point pseudo-op. | |||
| 658 | bool isStrictFPOpcode() { | |||
| 659 | switch (NodeType) { | |||
| 660 | default: | |||
| 661 | return false; | |||
| 662 | case ISD::STRICT_FP16_TO_FP: | |||
| 663 | case ISD::STRICT_FP_TO_FP16: | |||
| 664 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | |||
| 665 | case ISD::STRICT_##DAGN: | |||
| 666 | #include "llvm/IR/ConstrainedOps.def" | |||
| 667 | return true; | |||
| 668 | } | |||
| 669 | } | |||
| 670 | ||||
| 671 | /// Test if this node has a post-isel opcode, directly | |||
| 672 | /// corresponding to a MachineInstr opcode. | |||
| 673 | bool isMachineOpcode() const { return NodeType < 0; } | |||
| 674 | ||||
| 675 | /// This may only be called if isMachineOpcode returns | |||
| 676 | /// true. It returns the MachineInstr opcode value that the node's opcode | |||
| 677 | /// corresponds to. | |||
| 678 | unsigned getMachineOpcode() const { | |||
| 679 | assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!" ) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 679, __extension__ __PRETTY_FUNCTION__)); | |||
| 680 | return ~NodeType; | |||
| 681 | } | |||
| 682 | ||||
| 683 | bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } | |||
| 684 | void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } | |||
| 685 | ||||
| 686 | bool isDivergent() const { return SDNodeBits.IsDivergent; } | |||
| 687 | ||||
| 688 | /// Return true if there are no uses of this node. | |||
| 689 | bool use_empty() const { return UseList == nullptr; } | |||
| 690 | ||||
| 691 | /// Return true if there is exactly one use of this node. | |||
| 692 | bool hasOneUse() const { return hasSingleElement(uses()); } | |||
| 693 | ||||
| 694 | /// Return the number of uses of this node. This method takes | |||
| 695 | /// time proportional to the number of uses. | |||
| 696 | size_t use_size() const { return std::distance(use_begin(), use_end()); } | |||
| 697 | ||||
| 698 | /// Return the unique node id. | |||
| 699 | int getNodeId() const { return NodeId; } | |||
| 700 | ||||
| 701 | /// Set unique node id. | |||
| 702 | void setNodeId(int Id) { NodeId = Id; } | |||
| 703 | ||||
| 704 | /// Return the node ordering. | |||
| 705 | unsigned getIROrder() const { return IROrder; } | |||
| 706 | ||||
| 707 | /// Set the node ordering. | |||
| 708 | void setIROrder(unsigned Order) { IROrder = Order; } | |||
| 709 | ||||
| 710 | /// Return the source location info. | |||
| 711 | const DebugLoc &getDebugLoc() const { return debugLoc; } | |||
| 712 | ||||
| 713 | /// Set source location info. Try to avoid this, putting | |||
| 714 | /// it in the constructor is preferable. | |||
| 715 | void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } | |||
| 716 | ||||
| 717 | /// This class provides iterator support for SDUse | |||
| 718 | /// operands that use a specific SDNode. | |||
| 719 | class use_iterator { | |||
| 720 | friend class SDNode; | |||
| 721 | ||||
| 722 | SDUse *Op = nullptr; | |||
| 723 | ||||
| 724 | explicit use_iterator(SDUse *op) : Op(op) {} | |||
| 725 | ||||
| 726 | public: | |||
| 727 | using iterator_category = std::forward_iterator_tag; | |||
| 728 | using value_type = SDUse; | |||
| 729 | using difference_type = std::ptrdiff_t; | |||
| 730 | using pointer = value_type *; | |||
| 731 | using reference = value_type &; | |||
| 732 | ||||
| 733 | use_iterator() = default; | |||
| 734 | use_iterator(const use_iterator &I) : Op(I.Op) {} | |||
| 735 | ||||
| 736 | bool operator==(const use_iterator &x) const { | |||
| 737 | return Op == x.Op; | |||
| 738 | } | |||
| 739 | bool operator!=(const use_iterator &x) const { | |||
| 740 | return !operator==(x); | |||
| 741 | } | |||
| 742 | ||||
| 743 | /// Return true if this iterator is at the end of uses list. | |||
| 744 | bool atEnd() const { return Op == nullptr; } | |||
| 745 | ||||
| 746 | // Iterator traversal: forward iteration only. | |||
| 747 | use_iterator &operator++() { // Preincrement | |||
| 748 | assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 748, __extension__ __PRETTY_FUNCTION__)); | |||
| 749 | Op = Op->getNext(); | |||
| 750 | return *this; | |||
| 751 | } | |||
| 752 | ||||
| 753 | use_iterator operator++(int) { // Postincrement | |||
| 754 | use_iterator tmp = *this; ++*this; return tmp; | |||
| 755 | } | |||
| 756 | ||||
| 757 | /// Retrieve a pointer to the current user node. | |||
| 758 | SDNode *operator*() const { | |||
| 759 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 759, __extension__ __PRETTY_FUNCTION__)); | |||
| 760 | return Op->getUser(); | |||
| 761 | } | |||
| 762 | ||||
| 763 | SDNode *operator->() const { return operator*(); } | |||
| 764 | ||||
| 765 | SDUse &getUse() const { return *Op; } | |||
| 766 | ||||
| 767 | /// Retrieve the operand # of this use in its user. | |||
| 768 | unsigned getOperandNo() const { | |||
| 769 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 769, __extension__ __PRETTY_FUNCTION__)); | |||
| 770 | return (unsigned)(Op - Op->getUser()->OperandList); | |||
| 771 | } | |||
| 772 | }; | |||
| 773 | ||||
| 774 | /// Provide iteration support to walk over all uses of an SDNode. | |||
| 775 | use_iterator use_begin() const { | |||
| 776 | return use_iterator(UseList); | |||
| 777 | } | |||
| 778 | ||||
| 779 | static use_iterator use_end() { return use_iterator(nullptr); } | |||
| 780 | ||||
| 781 | inline iterator_range<use_iterator> uses() { | |||
| 782 | return make_range(use_begin(), use_end()); | |||
| 783 | } | |||
| 784 | inline iterator_range<use_iterator> uses() const { | |||
| 785 | return make_range(use_begin(), use_end()); | |||
| 786 | } | |||
| 787 | ||||
| 788 | /// Return true if there are exactly NUSES uses of the indicated value. | |||
| 789 | /// This method ignores uses of other values defined by this operation. | |||
| 790 | bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; | |||
| 791 | ||||
| 792 | /// Return true if there are any use of the indicated value. | |||
| 793 | /// This method ignores uses of other values defined by this operation. | |||
| 794 | bool hasAnyUseOfValue(unsigned Value) const; | |||
| 795 | ||||
| 796 | /// Return true if this node is the only use of N. | |||
| 797 | bool isOnlyUserOf(const SDNode *N) const; | |||
| 798 | ||||
| 799 | /// Return true if this node is an operand of N. | |||
| 800 | bool isOperandOf(const SDNode *N) const; | |||
| 801 | ||||
| 802 | /// Return true if this node is a predecessor of N. | |||
| 803 | /// NOTE: Implemented on top of hasPredecessor and every bit as | |||
| 804 | /// expensive. Use carefully. | |||
| 805 | bool isPredecessorOf(const SDNode *N) const { | |||
| 806 | return N->hasPredecessor(this); | |||
| 807 | } | |||
| 808 | ||||
| 809 | /// Return true if N is a predecessor of this node. | |||
| 810 | /// N is either an operand of this node, or can be reached by recursively | |||
| 811 | /// traversing up the operands. | |||
| 812 | /// NOTE: This is an expensive method. Use it carefully. | |||
| 813 | bool hasPredecessor(const SDNode *N) const; | |||
| 814 | ||||
| 815 | /// Returns true if N is a predecessor of any node in Worklist. This | |||
| 816 | /// helper keeps Visited and Worklist sets externally to allow unions | |||
| 817 | /// searches to be performed in parallel, caching of results across | |||
| 818 | /// queries and incremental addition to Worklist. Stops early if N is | |||
| 819 | /// found but will resume. Remember to clear Visited and Worklists | |||
| 820 | /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before | |||
| 821 | /// giving up. The TopologicalPrune flag signals that positive NodeIds are | |||
| 822 | /// topologically ordered (Operands have strictly smaller node id) and search | |||
| 823 | /// can be pruned leveraging this. | |||
| 824 | static bool hasPredecessorHelper(const SDNode *N, | |||
| 825 | SmallPtrSetImpl<const SDNode *> &Visited, | |||
| 826 | SmallVectorImpl<const SDNode *> &Worklist, | |||
| 827 | unsigned int MaxSteps = 0, | |||
| 828 | bool TopologicalPrune = false) { | |||
| 829 | SmallVector<const SDNode *, 8> DeferredNodes; | |||
| 830 | if (Visited.count(N)) | |||
| 831 | return true; | |||
| 832 | ||||
| 833 | // Node Id's are assigned in three places: As a topological | |||
| 834 | // ordering (> 0), during legalization (results in values set to | |||
| 835 | // 0), new nodes (set to -1). If N has a topolgical id then we | |||
| 836 | // know that all nodes with ids smaller than it cannot be | |||
| 837 | // successors and we need not check them. Filter out all node | |||
| 838 | // that can't be matches. We add them to the worklist before exit | |||
| 839 | // in case of multiple calls. Note that during selection the topological id | |||
| 840 | // may be violated if a node's predecessor is selected before it. We mark | |||
| 841 | // this at selection negating the id of unselected successors and | |||
| 842 | // restricting topological pruning to positive ids. | |||
| 843 | ||||
| 844 | int NId = N->getNodeId(); | |||
| 845 | // If we Invalidated the Id, reconstruct original NId. | |||
| 846 | if (NId < -1) | |||
| 847 | NId = -(NId + 1); | |||
| 848 | ||||
| 849 | bool Found = false; | |||
| 850 | while (!Worklist.empty()) { | |||
| 851 | const SDNode *M = Worklist.pop_back_val(); | |||
| 852 | int MId = M->getNodeId(); | |||
| 853 | if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) && | |||
| 854 | (MId > 0) && (MId < NId)) { | |||
| 855 | DeferredNodes.push_back(M); | |||
| 856 | continue; | |||
| 857 | } | |||
| 858 | for (const SDValue &OpV : M->op_values()) { | |||
| 859 | SDNode *Op = OpV.getNode(); | |||
| 860 | if (Visited.insert(Op).second) | |||
| 861 | Worklist.push_back(Op); | |||
| 862 | if (Op == N) | |||
| 863 | Found = true; | |||
| 864 | } | |||
| 865 | if (Found) | |||
| 866 | break; | |||
| 867 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
| 868 | break; | |||
| 869 | } | |||
| 870 | // Push deferred nodes back on worklist. | |||
| 871 | Worklist.append(DeferredNodes.begin(), DeferredNodes.end()); | |||
| 872 | // If we bailed early, conservatively return found. | |||
| 873 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
| 874 | return true; | |||
| 875 | return Found; | |||
| 876 | } | |||
| 877 | ||||
| 878 | /// Return true if all the users of N are contained in Nodes. | |||
| 879 | /// NOTE: Requires at least one match, but doesn't require them all. | |||
| 880 | static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N); | |||
| 881 | ||||
| 882 | /// Return the number of values used by this operation. | |||
| 883 | unsigned getNumOperands() const { return NumOperands; } | |||
| 884 | ||||
| 885 | /// Return the maximum number of operands that a SDNode can hold. | |||
| 886 | static constexpr size_t getMaxNumOperands() { | |||
| 887 | return std::numeric_limits<decltype(SDNode::NumOperands)>::max(); | |||
| 888 | } | |||
| 889 | ||||
| 890 | /// Helper method returns the integer value of a ConstantSDNode operand. | |||
| 891 | inline uint64_t getConstantOperandVal(unsigned Num) const; | |||
| 892 | ||||
| 893 | /// Helper method returns the APInt of a ConstantSDNode operand. | |||
| 894 | inline const APInt &getConstantOperandAPInt(unsigned Num) const; | |||
| 895 | ||||
| 896 | const SDValue &getOperand(unsigned Num) const { | |||
| 897 | assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!" ) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 897, __extension__ __PRETTY_FUNCTION__)); | |||
| 898 | return OperandList[Num]; | |||
| 899 | } | |||
| 900 | ||||
| 901 | using op_iterator = SDUse *; | |||
| 902 | ||||
| 903 | op_iterator op_begin() const { return OperandList; } | |||
| 904 | op_iterator op_end() const { return OperandList+NumOperands; } | |||
| 905 | ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); } | |||
| 906 | ||||
| 907 | /// Iterator for directly iterating over the operand SDValue's. | |||
| 908 | struct value_op_iterator | |||
| 909 | : iterator_adaptor_base<value_op_iterator, op_iterator, | |||
| 910 | std::random_access_iterator_tag, SDValue, | |||
| 911 | ptrdiff_t, value_op_iterator *, | |||
| 912 | value_op_iterator *> { | |||
| 913 | explicit value_op_iterator(SDUse *U = nullptr) | |||
| 914 | : iterator_adaptor_base(U) {} | |||
| 915 | ||||
| 916 | const SDValue &operator*() const { return I->get(); } | |||
| 917 | }; | |||
| 918 | ||||
| 919 | iterator_range<value_op_iterator> op_values() const { | |||
| 920 | return make_range(value_op_iterator(op_begin()), | |||
| 921 | value_op_iterator(op_end())); | |||
| 922 | } | |||
| 923 | ||||
| 924 | SDVTList getVTList() const { | |||
| 925 | SDVTList X = { ValueList, NumValues }; | |||
| 926 | return X; | |||
| 927 | } | |||
| 928 | ||||
| 929 | /// If this node has a glue operand, return the node | |||
| 930 | /// to which the glue operand points. Otherwise return NULL. | |||
| 931 | SDNode *getGluedNode() const { | |||
| 932 | if (getNumOperands() != 0 && | |||
| 933 | getOperand(getNumOperands()-1).getValueType() == MVT::Glue) | |||
| 934 | return getOperand(getNumOperands()-1).getNode(); | |||
| 935 | return nullptr; | |||
| 936 | } | |||
| 937 | ||||
| 938 | /// If this node has a glue value with a user, return | |||
| 939 | /// the user (there is at most one). Otherwise return NULL. | |||
| 940 | SDNode *getGluedUser() const { | |||
| 941 | for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) | |||
| 942 | if (UI.getUse().get().getValueType() == MVT::Glue) | |||
| 943 | return *UI; | |||
| 944 | return nullptr; | |||
| 945 | } | |||
| 946 | ||||
| 947 | SDNodeFlags getFlags() const { return Flags; } | |||
| 948 | void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; } | |||
| 949 | ||||
| 950 | /// Clear any flags in this node that aren't also set in Flags. | |||
| 951 | /// If Flags is not in a defined state then this has no effect. | |||
| 952 | void intersectFlagsWith(const SDNodeFlags Flags); | |||
| 953 | ||||
| 954 | /// Return the number of values defined/returned by this operator. | |||
| 955 | unsigned getNumValues() const { return NumValues; } | |||
| 956 | ||||
| 957 | /// Return the type of a specified result. | |||
| 958 | EVT getValueType(unsigned ResNo) const { | |||
| 959 | assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!" ) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 959, __extension__ __PRETTY_FUNCTION__)); | |||
| 960 | return ValueList[ResNo]; | |||
| 961 | } | |||
| 962 | ||||
| 963 | /// Return the type of a specified result as a simple type. | |||
| 964 | MVT getSimpleValueType(unsigned ResNo) const { | |||
| 965 | return getValueType(ResNo).getSimpleVT(); | |||
| 966 | } | |||
| 967 | ||||
| 968 | /// Returns MVT::getSizeInBits(getValueType(ResNo)). | |||
| 969 | /// | |||
| 970 | /// If the value type is a scalable vector type, the scalable property will | |||
| 971 | /// be set and the runtime size will be a positive integer multiple of the | |||
| 972 | /// base size. | |||
| 973 | TypeSize getValueSizeInBits(unsigned ResNo) const { | |||
| 974 | return getValueType(ResNo).getSizeInBits(); | |||
| 975 | } | |||
| 976 | ||||
| 977 | using value_iterator = const EVT *; | |||
| 978 | ||||
| 979 | value_iterator value_begin() const { return ValueList; } | |||
| 980 | value_iterator value_end() const { return ValueList+NumValues; } | |||
| 981 | iterator_range<value_iterator> values() const { | |||
| 982 | return llvm::make_range(value_begin(), value_end()); | |||
| 983 | } | |||
| 984 | ||||
| 985 | /// Return the opcode of this operation for printing. | |||
| 986 | std::string getOperationName(const SelectionDAG *G = nullptr) const; | |||
| 987 | static const char* getIndexedModeName(ISD::MemIndexedMode AM); | |||
| 988 | void print_types(raw_ostream &OS, const SelectionDAG *G) const; | |||
| 989 | void print_details(raw_ostream &OS, const SelectionDAG *G) const; | |||
| 990 | void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
| 991 | void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
| 992 | ||||
| 993 | /// Print a SelectionDAG node and all children down to | |||
| 994 | /// the leaves. The given SelectionDAG allows target-specific nodes | |||
| 995 | /// to be printed in human-readable form. Unlike printr, this will | |||
| 996 | /// print the whole DAG, including children that appear multiple | |||
| 997 | /// times. | |||
| 998 | /// | |||
| 999 | void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; | |||
| 1000 | ||||
| 1001 | /// Print a SelectionDAG node and children up to | |||
| 1002 | /// depth "depth." The given SelectionDAG allows target-specific | |||
| 1003 | /// nodes to be printed in human-readable form. Unlike printr, this | |||
| 1004 | /// will print children that appear multiple times wherever they are | |||
| 1005 | /// used. | |||
| 1006 | /// | |||
| 1007 | void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, | |||
| 1008 | unsigned depth = 100) const; | |||
| 1009 | ||||
| 1010 | /// Dump this node, for debugging. | |||
| 1011 | void dump() const; | |||
| 1012 | ||||
| 1013 | /// Dump (recursively) this node and its use-def subgraph. | |||
| 1014 | void dumpr() const; | |||
| 1015 | ||||
| 1016 | /// Dump this node, for debugging. | |||
| 1017 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
| 1018 | /// in human-readable form. | |||
| 1019 | void dump(const SelectionDAG *G) const; | |||
| 1020 | ||||
| 1021 | /// Dump (recursively) this node and its use-def subgraph. | |||
| 1022 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
| 1023 | /// in human-readable form. | |||
| 1024 | void dumpr(const SelectionDAG *G) const; | |||
| 1025 | ||||
| 1026 | /// printrFull to dbgs(). The given SelectionDAG allows | |||
| 1027 | /// target-specific nodes to be printed in human-readable form. | |||
| 1028 | /// Unlike dumpr, this will print the whole DAG, including children | |||
| 1029 | /// that appear multiple times. | |||
| 1030 | void dumprFull(const SelectionDAG *G = nullptr) const; | |||
| 1031 | ||||
| 1032 | /// printrWithDepth to dbgs(). The given | |||
| 1033 | /// SelectionDAG allows target-specific nodes to be printed in | |||
| 1034 | /// human-readable form. Unlike dumpr, this will print children | |||
| 1035 | /// that appear multiple times wherever they are used. | |||
| 1036 | /// | |||
| 1037 | void dumprWithDepth(const SelectionDAG *G = nullptr, | |||
| 1038 | unsigned depth = 100) const; | |||
| 1039 | ||||
| 1040 | /// Gather unique data for the node. | |||
| 1041 | void Profile(FoldingSetNodeID &ID) const; | |||
| 1042 | ||||
| 1043 | /// This method should only be used by the SDUse class. | |||
| 1044 | void addUse(SDUse &U) { U.addToList(&UseList); } | |||
| 1045 | ||||
| 1046 | protected: | |||
| 1047 | static SDVTList getSDVTList(EVT VT) { | |||
| 1048 | SDVTList Ret = { getValueTypeList(VT), 1 }; | |||
| 1049 | return Ret; | |||
| 1050 | } | |||
| 1051 | ||||
| 1052 | /// Create an SDNode. | |||
| 1053 | /// | |||
| 1054 | /// SDNodes are created without any operands, and never own the operand | |||
| 1055 | /// storage. To add operands, see SelectionDAG::createOperands. | |||
| 1056 | SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) | |||
| 1057 | : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs), | |||
| 1058 | IROrder(Order), debugLoc(std::move(dl)) { | |||
| 1059 | memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits)); | |||
| 1060 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() && "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1060, __extension__ __PRETTY_FUNCTION__)); | |||
| 1061 | assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1062, __extension__ __PRETTY_FUNCTION__)) | |||
| 1062 | "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1062, __extension__ __PRETTY_FUNCTION__)); | |||
| 1063 | } | |||
| 1064 | ||||
| 1065 | /// Release the operands and set this node to have zero operands. | |||
| 1066 | void DropOperands(); | |||
| 1067 | }; | |||
| 1068 | ||||
| 1069 | /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed | |||
| 1070 | /// into SDNode creation functions. | |||
| 1071 | /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted | |||
| 1072 | /// from the original Instruction, and IROrder is the ordinal position of | |||
| 1073 | /// the instruction. | |||
| 1074 | /// When an SDNode is created after the DAG is being built, both DebugLoc and | |||
| 1075 | /// the IROrder are propagated from the original SDNode. | |||
| 1076 | /// So SDLoc class provides two constructors besides the default one, one to | |||
| 1077 | /// be used by the DAGBuilder, the other to be used by others. | |||
| 1078 | class SDLoc { | |||
| 1079 | private: | |||
| 1080 | DebugLoc DL; | |||
| 1081 | int IROrder = 0; | |||
| 1082 | ||||
| 1083 | public: | |||
| 1084 | SDLoc() = default; | |||
| 1085 | SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {} | |||
| 1086 | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} | |||
| 1087 | SDLoc(const Instruction *I, int Order) : IROrder(Order) { | |||
| 1088 | assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder" ) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1088, __extension__ __PRETTY_FUNCTION__)); | |||
| 1089 | if (I) | |||
| 1090 | DL = I->getDebugLoc(); | |||
| 1091 | } | |||
| 1092 | ||||
| 1093 | unsigned getIROrder() const { return IROrder; } | |||
| 1094 | const DebugLoc &getDebugLoc() const { return DL; } | |||
| 1095 | }; | |||
| 1096 | ||||
| 1097 | // Define inline functions from the SDValue class. | |||
| 1098 | ||||
| 1099 | inline SDValue::SDValue(SDNode *node, unsigned resno) | |||
| 1100 | : Node(node), ResNo(resno) { | |||
| 1101 | // Explicitly check for !ResNo to avoid use-after-free, because there are | |||
| 1102 | // callers that use SDValue(N, 0) with a deleted N to indicate successful | |||
| 1103 | // combines. | |||
| 1104 | assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1105, __extension__ __PRETTY_FUNCTION__)) | |||
| 1105 | "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1105, __extension__ __PRETTY_FUNCTION__)); | |||
| 1106 | assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps." ) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1106, __extension__ __PRETTY_FUNCTION__)); | |||
| 1107 | } | |||
| 1108 | ||||
| 1109 | inline unsigned SDValue::getOpcode() const { | |||
| 1110 | return Node->getOpcode(); | |||
| ||||
| 1111 | } | |||
| 1112 | ||||
| 1113 | inline EVT SDValue::getValueType() const { | |||
| 1114 | return Node->getValueType(ResNo); | |||
| 1115 | } | |||
| 1116 | ||||
| 1117 | inline unsigned SDValue::getNumOperands() const { | |||
| 1118 | return Node->getNumOperands(); | |||
| 1119 | } | |||
| 1120 | ||||
| 1121 | inline const SDValue &SDValue::getOperand(unsigned i) const { | |||
| 1122 | return Node->getOperand(i); | |||
| 1123 | } | |||
| 1124 | ||||
| 1125 | inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { | |||
| 1126 | return Node->getConstantOperandVal(i); | |||
| 1127 | } | |||
| 1128 | ||||
| 1129 | inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { | |||
| 1130 | return Node->getConstantOperandAPInt(i); | |||
| 1131 | } | |||
| 1132 | ||||
| 1133 | inline bool SDValue::isTargetOpcode() const { | |||
| 1134 | return Node->isTargetOpcode(); | |||
| 1135 | } | |||
| 1136 | ||||
| 1137 | inline bool SDValue::isTargetMemoryOpcode() const { | |||
| 1138 | return Node->isTargetMemoryOpcode(); | |||
| 1139 | } | |||
| 1140 | ||||
| 1141 | inline bool SDValue::isMachineOpcode() const { | |||
| 1142 | return Node->isMachineOpcode(); | |||
| 1143 | } | |||
| 1144 | ||||
| 1145 | inline unsigned SDValue::getMachineOpcode() const { | |||
| 1146 | return Node->getMachineOpcode(); | |||
| 1147 | } | |||
| 1148 | ||||
| 1149 | inline bool SDValue::isUndef() const { | |||
| 1150 | return Node->isUndef(); | |||
| 1151 | } | |||
| 1152 | ||||
| 1153 | inline bool SDValue::use_empty() const { | |||
| 1154 | return !Node->hasAnyUseOfValue(ResNo); | |||
| 1155 | } | |||
| 1156 | ||||
| 1157 | inline bool SDValue::hasOneUse() const { | |||
| 1158 | return Node->hasNUsesOfValue(1, ResNo); | |||
| 1159 | } | |||
| 1160 | ||||
| 1161 | inline const DebugLoc &SDValue::getDebugLoc() const { | |||
| 1162 | return Node->getDebugLoc(); | |||
| 1163 | } | |||
| 1164 | ||||
| 1165 | inline void SDValue::dump() const { | |||
| 1166 | return Node->dump(); | |||
| 1167 | } | |||
| 1168 | ||||
| 1169 | inline void SDValue::dump(const SelectionDAG *G) const { | |||
| 1170 | return Node->dump(G); | |||
| 1171 | } | |||
| 1172 | ||||
| 1173 | inline void SDValue::dumpr() const { | |||
| 1174 | return Node->dumpr(); | |||
| 1175 | } | |||
| 1176 | ||||
| 1177 | inline void SDValue::dumpr(const SelectionDAG *G) const { | |||
| 1178 | return Node->dumpr(G); | |||
| 1179 | } | |||
| 1180 | ||||
| 1181 | // Define inline functions from the SDUse class. | |||
| 1182 | ||||
| 1183 | inline void SDUse::set(const SDValue &V) { | |||
| 1184 | if (Val.getNode()) removeFromList(); | |||
| 1185 | Val = V; | |||
| 1186 | if (V.getNode()) V.getNode()->addUse(*this); | |||
| 1187 | } | |||
| 1188 | ||||
| 1189 | inline void SDUse::setInitial(const SDValue &V) { | |||
| 1190 | Val = V; | |||
| 1191 | V.getNode()->addUse(*this); | |||
| 1192 | } | |||
| 1193 | ||||
| 1194 | inline void SDUse::setNode(SDNode *N) { | |||
| 1195 | if (Val.getNode()) removeFromList(); | |||
| 1196 | Val.setNode(N); | |||
| 1197 | if (N) N->addUse(*this); | |||
| 1198 | } | |||
| 1199 | ||||
| 1200 | /// This class is used to form a handle around another node that | |||
| 1201 | /// is persistent and is updated across invocations of replaceAllUsesWith on its | |||
| 1202 | /// operand. This node should be directly created by end-users and not added to | |||
| 1203 | /// the AllNodes list. | |||
| 1204 | class HandleSDNode : public SDNode { | |||
| 1205 | SDUse Op; | |||
| 1206 | ||||
| 1207 | public: | |||
| 1208 | explicit HandleSDNode(SDValue X) | |||
| 1209 | : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { | |||
| 1210 | // HandleSDNodes are never inserted into the DAG, so they won't be | |||
| 1211 | // auto-numbered. Use ID 65535 as a sentinel. | |||
| 1212 | PersistentId = 0xffff; | |||
| 1213 | ||||
| 1214 | // Manually set up the operand list. This node type is special in that it's | |||
| 1215 | // always stack allocated and SelectionDAG does not manage its operands. | |||
| 1216 | // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not | |||
| 1217 | // be so special. | |||
| 1218 | Op.setUser(this); | |||
| 1219 | Op.setInitial(X); | |||
| 1220 | NumOperands = 1; | |||
| 1221 | OperandList = &Op; | |||
| 1222 | } | |||
| 1223 | ~HandleSDNode(); | |||
| 1224 | ||||
| 1225 | const SDValue &getValue() const { return Op; } | |||
| 1226 | }; | |||
| 1227 | ||||
| 1228 | class AddrSpaceCastSDNode : public SDNode { | |||
| 1229 | private: | |||
| 1230 | unsigned SrcAddrSpace; | |||
| 1231 | unsigned DestAddrSpace; | |||
| 1232 | ||||
| 1233 | public: | |||
| 1234 | AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, | |||
| 1235 | unsigned SrcAS, unsigned DestAS); | |||
| 1236 | ||||
| 1237 | unsigned getSrcAddressSpace() const { return SrcAddrSpace; } | |||
| 1238 | unsigned getDestAddressSpace() const { return DestAddrSpace; } | |||
| 1239 | ||||
| 1240 | static bool classof(const SDNode *N) { | |||
| 1241 | return N->getOpcode() == ISD::ADDRSPACECAST; | |||
| 1242 | } | |||
| 1243 | }; | |||
| 1244 | ||||
| 1245 | /// This is an abstract virtual class for memory operations. | |||
| 1246 | class MemSDNode : public SDNode { | |||
| 1247 | private: | |||
| 1248 | // VT of in-memory value. | |||
| 1249 | EVT MemoryVT; | |||
| 1250 | ||||
| 1251 | protected: | |||
| 1252 | /// Memory reference information. | |||
| 1253 | MachineMemOperand *MMO; | |||
| 1254 | ||||
| 1255 | public: | |||
| 1256 | MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 1257 | EVT memvt, MachineMemOperand *MMO); | |||
| 1258 | ||||
| 1259 | bool readMem() const { return MMO->isLoad(); } | |||
| 1260 | bool writeMem() const { return MMO->isStore(); } | |||
| 1261 | ||||
| 1262 | /// Returns alignment and volatility of the memory access | |||
| 1263 | Align getOriginalAlign() const { return MMO->getBaseAlign(); } | |||
| 1264 | Align getAlign() const { return MMO->getAlign(); } | |||
| 1265 | // FIXME: Remove once transition to getAlign is over. | |||
| 1266 | unsigned getAlignment() const { return MMO->getAlign().value(); } | |||
| 1267 | ||||
| 1268 | /// Return the SubclassData value, without HasDebugValue. This contains an | |||
| 1269 | /// encoding of the volatile flag, as well as bits used by subclasses. This | |||
| 1270 | /// function should only be used to compute a FoldingSetNodeID value. | |||
| 1271 | /// The HasDebugValue bit is masked out because CSE map needs to match | |||
| 1272 | /// nodes with debug info with nodes without debug info. Same is about | |||
| 1273 | /// isDivergent bit. | |||
| 1274 | unsigned getRawSubclassData() const { | |||
| 1275 | uint16_t Data; | |||
| 1276 | union { | |||
| 1277 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
| 1278 | SDNodeBitfields SDNodeBits; | |||
| 1279 | }; | |||
| 1280 | memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits)); | |||
| 1281 | SDNodeBits.HasDebugValue = 0; | |||
| 1282 | SDNodeBits.IsDivergent = false; | |||
| 1283 | memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits)); | |||
| 1284 | return Data; | |||
| 1285 | } | |||
| 1286 | ||||
| 1287 | bool isVolatile() const { return MemSDNodeBits.IsVolatile; } | |||
| 1288 | bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } | |||
| 1289 | bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } | |||
| 1290 | bool isInvariant() const { return MemSDNodeBits.IsInvariant; } | |||
| 1291 | ||||
| 1292 | // Returns the offset from the location of the access. | |||
| 1293 | int64_t getSrcValueOffset() const { return MMO->getOffset(); } | |||
| 1294 | ||||
| 1295 | /// Returns the AA info that describes the dereference. | |||
| 1296 | AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } | |||
| 1297 | ||||
| 1298 | /// Returns the Ranges that describes the dereference. | |||
| 1299 | const MDNode *getRanges() const { return MMO->getRanges(); } | |||
| 1300 | ||||
| 1301 | /// Returns the synchronization scope ID for this memory operation. | |||
| 1302 | SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } | |||
| 1303 | ||||
| 1304 | /// Return the atomic ordering requirements for this memory operation. For | |||
| 1305 | /// cmpxchg atomic operations, return the atomic ordering requirements when | |||
| 1306 | /// store occurs. | |||
| 1307 | AtomicOrdering getSuccessOrdering() const { | |||
| 1308 | return MMO->getSuccessOrdering(); | |||
| 1309 | } | |||
| 1310 | ||||
| 1311 | /// Return a single atomic ordering that is at least as strong as both the | |||
| 1312 | /// success and failure orderings for an atomic operation. (For operations | |||
| 1313 | /// other than cmpxchg, this is equivalent to getSuccessOrdering().) | |||
| 1314 | AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); } | |||
| 1315 | ||||
| 1316 | /// Return true if the memory operation ordering is Unordered or higher. | |||
| 1317 | bool isAtomic() const { return MMO->isAtomic(); } | |||
| 1318 | ||||
| 1319 | /// Returns true if the memory operation doesn't imply any ordering | |||
| 1320 | /// constraints on surrounding memory operations beyond the normal memory | |||
| 1321 | /// aliasing rules. | |||
| 1322 | bool isUnordered() const { return MMO->isUnordered(); } | |||
| 1323 | ||||
| 1324 | /// Returns true if the memory operation is neither atomic or volatile. | |||
| 1325 | bool isSimple() const { return !isAtomic() && !isVolatile(); } | |||
| 1326 | ||||
| 1327 | /// Return the type of the in-memory value. | |||
| 1328 | EVT getMemoryVT() const { return MemoryVT; } | |||
| 1329 | ||||
| 1330 | /// Return a MachineMemOperand object describing the memory | |||
| 1331 | /// reference performed by operation. | |||
| 1332 | MachineMemOperand *getMemOperand() const { return MMO; } | |||
| 1333 | ||||
| 1334 | const MachinePointerInfo &getPointerInfo() const { | |||
| 1335 | return MMO->getPointerInfo(); | |||
| 1336 | } | |||
| 1337 | ||||
| 1338 | /// Return the address space for the associated pointer | |||
| 1339 | unsigned getAddressSpace() const { | |||
| 1340 | return getPointerInfo().getAddrSpace(); | |||
| 1341 | } | |||
| 1342 | ||||
| 1343 | /// Update this MemSDNode's MachineMemOperand information | |||
| 1344 | /// to reflect the alignment of NewMMO, if it has a greater alignment. | |||
| 1345 | /// This must only be used when the new alignment applies to all users of | |||
| 1346 | /// this MachineMemOperand. | |||
| 1347 | void refineAlignment(const MachineMemOperand *NewMMO) { | |||
| 1348 | MMO->refineAlignment(NewMMO); | |||
| 1349 | } | |||
| 1350 | ||||
| 1351 | const SDValue &getChain() const { return getOperand(0); } | |||
| 1352 | ||||
| 1353 | const SDValue &getBasePtr() const { | |||
| 1354 | switch (getOpcode()) { | |||
| 1355 | case ISD::STORE: | |||
| 1356 | case ISD::MSTORE: | |||
| 1357 | return getOperand(2); | |||
| 1358 | case ISD::MGATHER: | |||
| 1359 | case ISD::MSCATTER: | |||
| 1360 | return getOperand(3); | |||
| 1361 | default: | |||
| 1362 | return getOperand(1); | |||
| 1363 | } | |||
| 1364 | } | |||
| 1365 | ||||
| 1366 | // Methods to support isa and dyn_cast | |||
| 1367 | static bool classof(const SDNode *N) { | |||
| 1368 | // For some targets, we lower some target intrinsics to a MemIntrinsicNode | |||
| 1369 | // with either an intrinsic or a target opcode. | |||
| 1370 | switch (N->getOpcode()) { | |||
| 1371 | case ISD::LOAD: | |||
| 1372 | case ISD::STORE: | |||
| 1373 | case ISD::PREFETCH: | |||
| 1374 | case ISD::ATOMIC_CMP_SWAP: | |||
| 1375 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: | |||
| 1376 | case ISD::ATOMIC_SWAP: | |||
| 1377 | case ISD::ATOMIC_LOAD_ADD: | |||
| 1378 | case ISD::ATOMIC_LOAD_SUB: | |||
| 1379 | case ISD::ATOMIC_LOAD_AND: | |||
| 1380 | case ISD::ATOMIC_LOAD_CLR: | |||
| 1381 | case ISD::ATOMIC_LOAD_OR: | |||
| 1382 | case ISD::ATOMIC_LOAD_XOR: | |||
| 1383 | case ISD::ATOMIC_LOAD_NAND: | |||
| 1384 | case ISD::ATOMIC_LOAD_MIN: | |||
| 1385 | case ISD::ATOMIC_LOAD_MAX: | |||
| 1386 | case ISD::ATOMIC_LOAD_UMIN: | |||
| 1387 | case ISD::ATOMIC_LOAD_UMAX: | |||
| 1388 | case ISD::ATOMIC_LOAD_FADD: | |||
| 1389 | case ISD::ATOMIC_LOAD_FSUB: | |||
| 1390 | case ISD::ATOMIC_LOAD: | |||
| 1391 | case ISD::ATOMIC_STORE: | |||
| 1392 | case ISD::MLOAD: | |||
| 1393 | case ISD::MSTORE: | |||
| 1394 | case ISD::MGATHER: | |||
| 1395 | case ISD::MSCATTER: | |||
| 1396 | return true; | |||
| 1397 | default: | |||
| 1398 | return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); | |||
| 1399 | } | |||
| 1400 | } | |||
| 1401 | }; | |||
| 1402 | ||||
| 1403 | /// This is an SDNode representing atomic operations. | |||
| 1404 | class AtomicSDNode : public MemSDNode { | |||
| 1405 | public: | |||
| 1406 | AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, | |||
| 1407 | EVT MemVT, MachineMemOperand *MMO) | |||
| 1408 | : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { | |||
| 1409 | assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1410, __extension__ __PRETTY_FUNCTION__)) | |||
| 1410 | MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1410, __extension__ __PRETTY_FUNCTION__)); | |||
| 1411 | } | |||
| 1412 | ||||
| 1413 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 1414 | const SDValue &getVal() const { return getOperand(2); } | |||
| 1415 | ||||
| 1416 | /// Returns true if this SDNode represents cmpxchg atomic operation, false | |||
| 1417 | /// otherwise. | |||
| 1418 | bool isCompareAndSwap() const { | |||
| 1419 | unsigned Op = getOpcode(); | |||
| 1420 | return Op == ISD::ATOMIC_CMP_SWAP || | |||
| 1421 | Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; | |||
| 1422 | } | |||
| 1423 | ||||
| 1424 | /// For cmpxchg atomic operations, return the atomic ordering requirements | |||
| 1425 | /// when store does not occur. | |||
| 1426 | AtomicOrdering getFailureOrdering() const { | |||
| 1427 | assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation" ) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1427, __extension__ __PRETTY_FUNCTION__)); | |||
| 1428 | return MMO->getFailureOrdering(); | |||
| 1429 | } | |||
| 1430 | ||||
| 1431 | // Methods to support isa and dyn_cast | |||
| 1432 | static bool classof(const SDNode *N) { | |||
| 1433 | return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || | |||
| 1434 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || | |||
| 1435 | N->getOpcode() == ISD::ATOMIC_SWAP || | |||
| 1436 | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || | |||
| 1437 | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || | |||
| 1438 | N->getOpcode() == ISD::ATOMIC_LOAD_AND || | |||
| 1439 | N->getOpcode() == ISD::ATOMIC_LOAD_CLR || | |||
| 1440 | N->getOpcode() == ISD::ATOMIC_LOAD_OR || | |||
| 1441 | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || | |||
| 1442 | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || | |||
| 1443 | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || | |||
| 1444 | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || | |||
| 1445 | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || | |||
| 1446 | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || | |||
| 1447 | N->getOpcode() == ISD::ATOMIC_LOAD_FADD || | |||
| 1448 | N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || | |||
| 1449 | N->getOpcode() == ISD::ATOMIC_LOAD || | |||
| 1450 | N->getOpcode() == ISD::ATOMIC_STORE; | |||
| 1451 | } | |||
| 1452 | }; | |||
| 1453 | ||||
| 1454 | /// This SDNode is used for target intrinsics that touch | |||
| 1455 | /// memory and need an associated MachineMemOperand. Its opcode may be | |||
| 1456 | /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode | |||
| 1457 | /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. | |||
| 1458 | class MemIntrinsicSDNode : public MemSDNode { | |||
| 1459 | public: | |||
| 1460 | MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, | |||
| 1461 | SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) | |||
| 1462 | : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { | |||
| 1463 | SDNodeBits.IsMemIntrinsic = true; | |||
| 1464 | } | |||
| 1465 | ||||
| 1466 | // Methods to support isa and dyn_cast | |||
| 1467 | static bool classof(const SDNode *N) { | |||
| 1468 | // We lower some target intrinsics to their target opcode | |||
| 1469 | // early a node with a target opcode can be of this class | |||
| 1470 | return N->isMemIntrinsic() || | |||
| 1471 | N->getOpcode() == ISD::PREFETCH || | |||
| 1472 | N->isTargetMemoryOpcode(); | |||
| 1473 | } | |||
| 1474 | }; | |||
| 1475 | ||||
| 1476 | /// This SDNode is used to implement the code generator | |||
| 1477 | /// support for the llvm IR shufflevector instruction. It combines elements | |||
| 1478 | /// from two input vectors into a new input vector, with the selection and | |||
| 1479 | /// ordering of elements determined by an array of integers, referred to as | |||
| 1480 | /// the shuffle mask. For input vectors of width N, mask indices of 0..N-1 | |||
| 1481 | /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS. | |||
| 1482 | /// An index of -1 is treated as undef, such that the code generator may put | |||
| 1483 | /// any value in the corresponding element of the result. | |||
| 1484 | class ShuffleVectorSDNode : public SDNode { | |||
| 1485 | // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and | |||
| 1486 | // is freed when the SelectionDAG object is destroyed. | |||
| 1487 | const int *Mask; | |||
| 1488 | ||||
| 1489 | protected: | |||
| 1490 | friend class SelectionDAG; | |||
| 1491 | ||||
| 1492 | ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M) | |||
| 1493 | : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} | |||
| 1494 | ||||
| 1495 | public: | |||
| 1496 | ArrayRef<int> getMask() const { | |||
| 1497 | EVT VT = getValueType(0); | |||
| 1498 | return makeArrayRef(Mask, VT.getVectorNumElements()); | |||
| 1499 | } | |||
| 1500 | ||||
| 1501 | int getMaskElt(unsigned Idx) const { | |||
| 1502 | assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements () && "Idx out of range!") ? void (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1502, __extension__ __PRETTY_FUNCTION__)); | |||
| 1503 | return Mask[Idx]; | |||
| 1504 | } | |||
| 1505 | ||||
| 1506 | bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } | |||
| 1507 | ||||
| 1508 | int getSplatIndex() const { | |||
| 1509 | assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!" ) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1509, __extension__ __PRETTY_FUNCTION__)); | |||
| 1510 | EVT VT = getValueType(0); | |||
| 1511 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) | |||
| 1512 | if (Mask[i] >= 0) | |||
| 1513 | return Mask[i]; | |||
| 1514 | ||||
| 1515 | // We can choose any index value here and be correct because all elements | |||
| 1516 | // are undefined. Return 0 for better potential for callers to simplify. | |||
| 1517 | return 0; | |||
| 1518 | } | |||
| 1519 | ||||
| 1520 | static bool isSplatMask(const int *Mask, EVT VT); | |||
| 1521 | ||||
| 1522 | /// Change values in a shuffle permute mask assuming | |||
| 1523 | /// the two vector operands have swapped position. | |||
| 1524 | static void commuteMask(MutableArrayRef<int> Mask) { | |||
| 1525 | unsigned NumElems = Mask.size(); | |||
| 1526 | for (unsigned i = 0; i != NumElems; ++i) { | |||
| 1527 | int idx = Mask[i]; | |||
| 1528 | if (idx < 0) | |||
| 1529 | continue; | |||
| 1530 | else if (idx < (int)NumElems) | |||
| 1531 | Mask[i] = idx + NumElems; | |||
| 1532 | else | |||
| 1533 | Mask[i] = idx - NumElems; | |||
| 1534 | } | |||
| 1535 | } | |||
| 1536 | ||||
| 1537 | static bool classof(const SDNode *N) { | |||
| 1538 | return N->getOpcode() == ISD::VECTOR_SHUFFLE; | |||
| 1539 | } | |||
| 1540 | }; | |||
| 1541 | ||||
| 1542 | class ConstantSDNode : public SDNode { | |||
| 1543 | friend class SelectionDAG; | |||
| 1544 | ||||
| 1545 | const ConstantInt *Value; | |||
| 1546 | ||||
| 1547 | ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) | |||
| 1548 | : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(), | |||
| 1549 | getSDVTList(VT)), | |||
| 1550 | Value(val) { | |||
| 1551 | ConstantSDNodeBits.IsOpaque = isOpaque; | |||
| 1552 | } | |||
| 1553 | ||||
| 1554 | public: | |||
| 1555 | const ConstantInt *getConstantIntValue() const { return Value; } | |||
| 1556 | const APInt &getAPIntValue() const { return Value->getValue(); } | |||
| 1557 | uint64_t getZExtValue() const { return Value->getZExtValue(); } | |||
| 1558 | int64_t getSExtValue() const { return Value->getSExtValue(); } | |||
| 1559 | uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) { | |||
| 1560 | return Value->getLimitedValue(Limit); | |||
| 1561 | } | |||
| 1562 | MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); } | |||
| 1563 | Align getAlignValue() const { return Value->getAlignValue(); } | |||
| 1564 | ||||
| 1565 | bool isOne() const { return Value->isOne(); } | |||
| 1566 | bool isNullValue() const { return Value->isZero(); } | |||
| 1567 | bool isAllOnesValue() const { return Value->isMinusOne(); } | |||
| 1568 | bool isMaxSignedValue() const { return Value->isMaxValue(true); } | |||
| 1569 | bool isMinSignedValue() const { return Value->isMinValue(true); } | |||
| 1570 | ||||
| 1571 | bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } | |||
| 1572 | ||||
| 1573 | static bool classof(const SDNode *N) { | |||
| 1574 | return N->getOpcode() == ISD::Constant || | |||
| 1575 | N->getOpcode() == ISD::TargetConstant; | |||
| 1576 | } | |||
| 1577 | }; | |||
| 1578 | ||||
| 1579 | uint64_t SDNode::getConstantOperandVal(unsigned Num) const { | |||
| 1580 | return cast<ConstantSDNode>(getOperand(Num))->getZExtValue(); | |||
| 1581 | } | |||
| 1582 | ||||
| 1583 | const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { | |||
| 1584 | return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue(); | |||
| 1585 | } | |||
| 1586 | ||||
| 1587 | class ConstantFPSDNode : public SDNode { | |||
| 1588 | friend class SelectionDAG; | |||
| 1589 | ||||
| 1590 | const ConstantFP *Value; | |||
| 1591 | ||||
| 1592 | ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) | |||
| 1593 | : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, | |||
| 1594 | DebugLoc(), getSDVTList(VT)), | |||
| 1595 | Value(val) {} | |||
| 1596 | ||||
| 1597 | public: | |||
| 1598 | const APFloat& getValueAPF() const { return Value->getValueAPF(); } | |||
| 1599 | const ConstantFP *getConstantFPValue() const { return Value; } | |||
| 1600 | ||||
| 1601 | /// Return true if the value is positive or negative zero. | |||
| 1602 | bool isZero() const { return Value->isZero(); } | |||
| 1603 | ||||
| 1604 | /// Return true if the value is a NaN. | |||
| 1605 | bool isNaN() const { return Value->isNaN(); } | |||
| 1606 | ||||
| 1607 | /// Return true if the value is an infinity | |||
| 1608 | bool isInfinity() const { return Value->isInfinity(); } | |||
| 1609 | ||||
| 1610 | /// Return true if the value is negative. | |||
| 1611 | bool isNegative() const { return Value->isNegative(); } | |||
| 1612 | ||||
| 1613 | /// We don't rely on operator== working on double values, as | |||
| 1614 | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. | |||
| 1615 | /// As such, this method can be used to do an exact bit-for-bit comparison of | |||
| 1616 | /// two floating point values. | |||
| 1617 | ||||
| 1618 | /// We leave the version with the double argument here because it's just so | |||
| 1619 | /// convenient to write "2.0" and the like. Without this function we'd | |||
| 1620 | /// have to duplicate its logic everywhere it's called. | |||
| 1621 | bool isExactlyValue(double V) const { | |||
| 1622 | return Value->getValueAPF().isExactlyValue(V); | |||
| 1623 | } | |||
| 1624 | bool isExactlyValue(const APFloat& V) const; | |||
| 1625 | ||||
| 1626 | static bool isValueValidForType(EVT VT, const APFloat& Val); | |||
| 1627 | ||||
| 1628 | static bool classof(const SDNode *N) { | |||
| 1629 | return N->getOpcode() == ISD::ConstantFP || | |||
| 1630 | N->getOpcode() == ISD::TargetConstantFP; | |||
| 1631 | } | |||
| 1632 | }; | |||
| 1633 | ||||
| 1634 | /// Returns true if \p V is a constant integer zero. | |||
| 1635 | bool isNullConstant(SDValue V); | |||
| 1636 | ||||
| 1637 | /// Returns true if \p V is an FP constant with a value of positive zero. | |||
| 1638 | bool isNullFPConstant(SDValue V); | |||
| 1639 | ||||
| 1640 | /// Returns true if \p V is an integer constant with all bits set. | |||
| 1641 | bool isAllOnesConstant(SDValue V); | |||
| 1642 | ||||
| 1643 | /// Returns true if \p V is a constant integer one. | |||
| 1644 | bool isOneConstant(SDValue V); | |||
| 1645 | ||||
| 1646 | /// Return the non-bitcasted source operand of \p V if it exists. | |||
| 1647 | /// If \p V is not a bitcasted value, it is returned as-is. | |||
| 1648 | SDValue peekThroughBitcasts(SDValue V); | |||
| 1649 | ||||
| 1650 | /// Return the non-bitcasted and one-use source operand of \p V if it exists. | |||
| 1651 | /// If \p V is not a bitcasted one-use value, it is returned as-is. | |||
| 1652 | SDValue peekThroughOneUseBitcasts(SDValue V); | |||
| 1653 | ||||
| 1654 | /// Return the non-extracted vector source operand of \p V if it exists. | |||
| 1655 | /// If \p V is not an extracted subvector, it is returned as-is. | |||
| 1656 | SDValue peekThroughExtractSubvectors(SDValue V); | |||
| 1657 | ||||
| 1658 | /// Returns true if \p V is a bitwise not operation. Assumes that an all ones | |||
| 1659 | /// constant is canonicalized to be operand 1. | |||
| 1660 | bool isBitwiseNot(SDValue V, bool AllowUndefs = false); | |||
| 1661 | ||||
| 1662 | /// Returns the SDNode if it is a constant splat BuildVector or constant int. | |||
| 1663 | ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, | |||
| 1664 | bool AllowTruncation = false); | |||
| 1665 | ||||
| 1666 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
| 1667 | /// constant int. | |||
| 1668 | ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, | |||
| 1669 | bool AllowUndefs = false, | |||
| 1670 | bool AllowTruncation = false); | |||
| 1671 | ||||
| 1672 | /// Returns the SDNode if it is a constant splat BuildVector or constant float. | |||
| 1673 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); | |||
| 1674 | ||||
| 1675 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
| 1676 | /// constant float. | |||
| 1677 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, | |||
| 1678 | bool AllowUndefs = false); | |||
| 1679 | ||||
| 1680 | /// Return true if the value is a constant 0 integer or a splatted vector of | |||
| 1681 | /// a constant 0 integer (with no undefs by default). | |||
| 1682 | /// Build vector implicit truncation is not an issue for null values. | |||
| 1683 | bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); | |||
| 1684 | ||||
| 1685 | /// Return true if the value is a constant 1 integer or a splatted vector of a | |||
| 1686 | /// constant 1 integer (with no undefs). | |||
| 1687 | /// Does not permit build vector implicit truncation. | |||
| 1688 | bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false); | |||
| 1689 | ||||
| 1690 | /// Return true if the value is a constant -1 integer or a splatted vector of a | |||
| 1691 | /// constant -1 integer (with no undefs). | |||
| 1692 | /// Does not permit build vector implicit truncation. | |||
| 1693 | bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false); | |||
| 1694 | ||||
| 1695 | /// Return true if \p V is either a integer or FP constant. | |||
| 1696 | inline bool isIntOrFPConstant(SDValue V) { | |||
| 1697 | return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V); | |||
| 1698 | } | |||
| 1699 | ||||
| 1700 | class GlobalAddressSDNode : public SDNode { | |||
| 1701 | friend class SelectionDAG; | |||
| 1702 | ||||
| 1703 | const GlobalValue *TheGlobal; | |||
| 1704 | int64_t Offset; | |||
| 1705 | unsigned TargetFlags; | |||
| 1706 | ||||
| 1707 | GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, | |||
| 1708 | const GlobalValue *GA, EVT VT, int64_t o, | |||
| 1709 | unsigned TF); | |||
| 1710 | ||||
| 1711 | public: | |||
| 1712 | const GlobalValue *getGlobal() const { return TheGlobal; } | |||
| 1713 | int64_t getOffset() const { return Offset; } | |||
| 1714 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1715 | // Return the address space this GlobalAddress belongs to. | |||
| 1716 | unsigned getAddressSpace() const; | |||
| 1717 | ||||
| 1718 | static bool classof(const SDNode *N) { | |||
| 1719 | return N->getOpcode() == ISD::GlobalAddress || | |||
| 1720 | N->getOpcode() == ISD::TargetGlobalAddress || | |||
| 1721 | N->getOpcode() == ISD::GlobalTLSAddress || | |||
| 1722 | N->getOpcode() == ISD::TargetGlobalTLSAddress; | |||
| 1723 | } | |||
| 1724 | }; | |||
| 1725 | ||||
| 1726 | class FrameIndexSDNode : public SDNode { | |||
| 1727 | friend class SelectionDAG; | |||
| 1728 | ||||
| 1729 | int FI; | |||
| 1730 | ||||
| 1731 | FrameIndexSDNode(int fi, EVT VT, bool isTarg) | |||
| 1732 | : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, | |||
| 1733 | 0, DebugLoc(), getSDVTList(VT)), FI(fi) { | |||
| 1734 | } | |||
| 1735 | ||||
| 1736 | public: | |||
| 1737 | int getIndex() const { return FI; } | |||
| 1738 | ||||
| 1739 | static bool classof(const SDNode *N) { | |||
| 1740 | return N->getOpcode() == ISD::FrameIndex || | |||
| 1741 | N->getOpcode() == ISD::TargetFrameIndex; | |||
| 1742 | } | |||
| 1743 | }; | |||
| 1744 | ||||
| 1745 | /// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate | |||
| 1746 | /// the offet and size that are started/ended in the underlying FrameIndex. | |||
| 1747 | class LifetimeSDNode : public SDNode { | |||
| 1748 | friend class SelectionDAG; | |||
| 1749 | int64_t Size; | |||
| 1750 | int64_t Offset; // -1 if offset is unknown. | |||
| 1751 | ||||
| 1752 | LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, | |||
| 1753 | SDVTList VTs, int64_t Size, int64_t Offset) | |||
| 1754 | : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} | |||
| 1755 | public: | |||
| 1756 | int64_t getFrameIndex() const { | |||
| 1757 | return cast<FrameIndexSDNode>(getOperand(1))->getIndex(); | |||
| 1758 | } | |||
| 1759 | ||||
| 1760 | bool hasOffset() const { return Offset >= 0; } | |||
| 1761 | int64_t getOffset() const { | |||
| 1762 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1762, __extension__ __PRETTY_FUNCTION__)); | |||
| 1763 | return Offset; | |||
| 1764 | } | |||
| 1765 | int64_t getSize() const { | |||
| 1766 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1766, __extension__ __PRETTY_FUNCTION__)); | |||
| 1767 | return Size; | |||
| 1768 | } | |||
| 1769 | ||||
| 1770 | // Methods to support isa and dyn_cast | |||
| 1771 | static bool classof(const SDNode *N) { | |||
| 1772 | return N->getOpcode() == ISD::LIFETIME_START || | |||
| 1773 | N->getOpcode() == ISD::LIFETIME_END; | |||
| 1774 | } | |||
| 1775 | }; | |||
| 1776 | ||||
| 1777 | /// This SDNode is used for PSEUDO_PROBE values, which are the function guid and | |||
| 1778 | /// the index of the basic block being probed. A pseudo probe serves as a place | |||
| 1779 | /// holder and will be removed at the end of compilation. It does not have any | |||
| 1780 | /// operand because we do not want the instruction selection to deal with any. | |||
| 1781 | class PseudoProbeSDNode : public SDNode { | |||
| 1782 | friend class SelectionDAG; | |||
| 1783 | uint64_t Guid; | |||
| 1784 | uint64_t Index; | |||
| 1785 | uint32_t Attributes; | |||
| 1786 | ||||
| 1787 | PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl, | |||
| 1788 | SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr) | |||
| 1789 | : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index), | |||
| 1790 | Attributes(Attr) {} | |||
| 1791 | ||||
| 1792 | public: | |||
| 1793 | uint64_t getGuid() const { return Guid; } | |||
| 1794 | uint64_t getIndex() const { return Index; } | |||
| 1795 | uint32_t getAttributes() const { return Attributes; } | |||
| 1796 | ||||
| 1797 | // Methods to support isa and dyn_cast | |||
| 1798 | static bool classof(const SDNode *N) { | |||
| 1799 | return N->getOpcode() == ISD::PSEUDO_PROBE; | |||
| 1800 | } | |||
| 1801 | }; | |||
| 1802 | ||||
| 1803 | class JumpTableSDNode : public SDNode { | |||
| 1804 | friend class SelectionDAG; | |||
| 1805 | ||||
| 1806 | int JTI; | |||
| 1807 | unsigned TargetFlags; | |||
| 1808 | ||||
| 1809 | JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF) | |||
| 1810 | : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, | |||
| 1811 | 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { | |||
| 1812 | } | |||
| 1813 | ||||
| 1814 | public: | |||
| 1815 | int getIndex() const { return JTI; } | |||
| 1816 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1817 | ||||
| 1818 | static bool classof(const SDNode *N) { | |||
| 1819 | return N->getOpcode() == ISD::JumpTable || | |||
| 1820 | N->getOpcode() == ISD::TargetJumpTable; | |||
| 1821 | } | |||
| 1822 | }; | |||
| 1823 | ||||
| 1824 | class ConstantPoolSDNode : public SDNode { | |||
| 1825 | friend class SelectionDAG; | |||
| 1826 | ||||
| 1827 | union { | |||
| 1828 | const Constant *ConstVal; | |||
| 1829 | MachineConstantPoolValue *MachineCPVal; | |||
| 1830 | } Val; | |||
| 1831 | int Offset; // It's a MachineConstantPoolValue if top bit is set. | |||
| 1832 | Align Alignment; // Minimum alignment requirement of CP. | |||
| 1833 | unsigned TargetFlags; | |||
| 1834 | ||||
| 1835 | ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, | |||
| 1836 | Align Alignment, unsigned TF) | |||
| 1837 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
| 1838 | DebugLoc(), getSDVTList(VT)), | |||
| 1839 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
| 1840 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1840, __extension__ __PRETTY_FUNCTION__)); | |||
| 1841 | Val.ConstVal = c; | |||
| 1842 | } | |||
| 1843 | ||||
| 1844 | ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o, | |||
| 1845 | Align Alignment, unsigned TF) | |||
| 1846 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
| 1847 | DebugLoc(), getSDVTList(VT)), | |||
| 1848 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
| 1849 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1849, __extension__ __PRETTY_FUNCTION__)); | |||
| 1850 | Val.MachineCPVal = v; | |||
| 1851 | Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1); | |||
| 1852 | } | |||
| 1853 | ||||
| 1854 | public: | |||
| 1855 | bool isMachineConstantPoolEntry() const { | |||
| 1856 | return Offset < 0; | |||
| 1857 | } | |||
| 1858 | ||||
| 1859 | const Constant *getConstVal() const { | |||
| 1860 | assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1860, __extension__ __PRETTY_FUNCTION__)); | |||
| 1861 | return Val.ConstVal; | |||
| 1862 | } | |||
| 1863 | ||||
| 1864 | MachineConstantPoolValue *getMachineCPVal() const { | |||
| 1865 | assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 1865, __extension__ __PRETTY_FUNCTION__)); | |||
| 1866 | return Val.MachineCPVal; | |||
| 1867 | } | |||
| 1868 | ||||
| 1869 | int getOffset() const { | |||
| 1870 | return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1)); | |||
| 1871 | } | |||
| 1872 | ||||
| 1873 | // Return the alignment of this constant pool object, which is either 0 (for | |||
| 1874 | // default alignment) or the desired value. | |||
| 1875 | Align getAlign() const { return Alignment; } | |||
| 1876 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1877 | ||||
| 1878 | Type *getType() const; | |||
| 1879 | ||||
| 1880 | static bool classof(const SDNode *N) { | |||
| 1881 | return N->getOpcode() == ISD::ConstantPool || | |||
| 1882 | N->getOpcode() == ISD::TargetConstantPool; | |||
| 1883 | } | |||
| 1884 | }; | |||
| 1885 | ||||
| 1886 | /// Completely target-dependent object reference. | |||
| 1887 | class TargetIndexSDNode : public SDNode { | |||
| 1888 | friend class SelectionDAG; | |||
| 1889 | ||||
| 1890 | unsigned TargetFlags; | |||
| 1891 | int Index; | |||
| 1892 | int64_t Offset; | |||
| 1893 | ||||
| 1894 | public: | |||
| 1895 | TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF) | |||
| 1896 | : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), | |||
| 1897 | TargetFlags(TF), Index(Idx), Offset(Ofs) {} | |||
| 1898 | ||||
| 1899 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1900 | int getIndex() const { return Index; } | |||
| 1901 | int64_t getOffset() const { return Offset; } | |||
| 1902 | ||||
| 1903 | static bool classof(const SDNode *N) { | |||
| 1904 | return N->getOpcode() == ISD::TargetIndex; | |||
| 1905 | } | |||
| 1906 | }; | |||
| 1907 | ||||
| 1908 | class BasicBlockSDNode : public SDNode { | |||
| 1909 | friend class SelectionDAG; | |||
| 1910 | ||||
| 1911 | MachineBasicBlock *MBB; | |||
| 1912 | ||||
| 1913 | /// Debug info is meaningful and potentially useful here, but we create | |||
| 1914 | /// blocks out of order when they're jumped to, which makes it a bit | |||
| 1915 | /// harder. Let's see if we need it first. | |||
| 1916 | explicit BasicBlockSDNode(MachineBasicBlock *mbb) | |||
| 1917 | : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) | |||
| 1918 | {} | |||
| 1919 | ||||
| 1920 | public: | |||
| 1921 | MachineBasicBlock *getBasicBlock() const { return MBB; } | |||
| 1922 | ||||
| 1923 | static bool classof(const SDNode *N) { | |||
| 1924 | return N->getOpcode() == ISD::BasicBlock; | |||
| 1925 | } | |||
| 1926 | }; | |||
| 1927 | ||||
| 1928 | /// A "pseudo-class" with methods for operating on BUILD_VECTORs. | |||
| 1929 | class BuildVectorSDNode : public SDNode { | |||
| 1930 | public: | |||
| 1931 | // These are constructed as SDNodes and then cast to BuildVectorSDNodes. | |||
| 1932 | explicit BuildVectorSDNode() = delete; | |||
| 1933 | ||||
| 1934 | /// Check if this is a constant splat, and if so, find the | |||
| 1935 | /// smallest element size that splats the vector. If MinSplatBits is | |||
| 1936 | /// nonzero, the element size must be at least that large. Note that the | |||
| 1937 | /// splat element may be the entire vector (i.e., a one element vector). | |||
| 1938 | /// Returns the splat element value in SplatValue. Any undefined bits in | |||
| 1939 | /// that value are zero, and the corresponding bits in the SplatUndef mask | |||
| 1940 | /// are set. The SplatBitSize value is set to the splat element size in | |||
| 1941 | /// bits. HasAnyUndefs is set to true if any bits in the vector are | |||
| 1942 | /// undefined. isBigEndian describes the endianness of the target. | |||
| 1943 | bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, | |||
| 1944 | unsigned &SplatBitSize, bool &HasAnyUndefs, | |||
| 1945 | unsigned MinSplatBits = 0, | |||
| 1946 | bool isBigEndian = false) const; | |||
| 1947 | ||||
| 1948 | /// Returns the demanded splatted value or a null value if this is not a | |||
| 1949 | /// splat. | |||
| 1950 | /// | |||
| 1951 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
| 1952 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 1953 | /// the vector width and set the bits where elements are undef. | |||
| 1954 | SDValue getSplatValue(const APInt &DemandedElts, | |||
| 1955 | BitVector *UndefElements = nullptr) const; | |||
| 1956 | ||||
| 1957 | /// Returns the splatted value or a null value if this is not a splat. | |||
| 1958 | /// | |||
| 1959 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 1960 | /// the vector width and set the bits where elements are undef. | |||
| 1961 | SDValue getSplatValue(BitVector *UndefElements = nullptr) const; | |||
| 1962 | ||||
| 1963 | /// Find the shortest repeating sequence of values in the build vector. | |||
| 1964 | /// | |||
| 1965 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
| 1966 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
| 1967 | /// | |||
| 1968 | /// Currently this must be a power-of-2 build vector. | |||
| 1969 | /// The DemandedElts mask indicates the elements that must be present, | |||
| 1970 | /// undemanded elements in Sequence may be null (SDValue()). If passed a | |||
| 1971 | /// non-null UndefElements bitvector, it will resize it to match the original | |||
| 1972 | /// vector width and set the bits where elements are undef. If result is | |||
| 1973 | /// false, Sequence will be empty. | |||
| 1974 | bool getRepeatedSequence(const APInt &DemandedElts, | |||
| 1975 | SmallVectorImpl<SDValue> &Sequence, | |||
| 1976 | BitVector *UndefElements = nullptr) const; | |||
| 1977 | ||||
| 1978 | /// Find the shortest repeating sequence of values in the build vector. | |||
| 1979 | /// | |||
| 1980 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
| 1981 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
| 1982 | /// | |||
| 1983 | /// Currently this must be a power-of-2 build vector. | |||
| 1984 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 1985 | /// the original vector width and set the bits where elements are undef. | |||
| 1986 | /// If result is false, Sequence will be empty. | |||
| 1987 | bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, | |||
| 1988 | BitVector *UndefElements = nullptr) const; | |||
| 1989 | ||||
| 1990 | /// Returns the demanded splatted constant or null if this is not a constant | |||
| 1991 | /// splat. | |||
| 1992 | /// | |||
| 1993 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
| 1994 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 1995 | /// the vector width and set the bits where elements are undef. | |||
| 1996 | ConstantSDNode * | |||
| 1997 | getConstantSplatNode(const APInt &DemandedElts, | |||
| 1998 | BitVector *UndefElements = nullptr) const; | |||
| 1999 | ||||
| 2000 | /// Returns the splatted constant or null if this is not a constant | |||
| 2001 | /// splat. | |||
| 2002 | /// | |||
| 2003 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2004 | /// the vector width and set the bits where elements are undef. | |||
| 2005 | ConstantSDNode * | |||
| 2006 | getConstantSplatNode(BitVector *UndefElements = nullptr) const; | |||
| 2007 | ||||
| 2008 | /// Returns the demanded splatted constant FP or null if this is not a | |||
| 2009 | /// constant FP splat. | |||
| 2010 | /// | |||
| 2011 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
| 2012 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2013 | /// the vector width and set the bits where elements are undef. | |||
| 2014 | ConstantFPSDNode * | |||
| 2015 | getConstantFPSplatNode(const APInt &DemandedElts, | |||
| 2016 | BitVector *UndefElements = nullptr) const; | |||
| 2017 | ||||
| 2018 | /// Returns the splatted constant FP or null if this is not a constant | |||
| 2019 | /// FP splat. | |||
| 2020 | /// | |||
| 2021 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2022 | /// the vector width and set the bits where elements are undef. | |||
| 2023 | ConstantFPSDNode * | |||
| 2024 | getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; | |||
| 2025 | ||||
| 2026 | /// If this is a constant FP splat and the splatted constant FP is an | |||
| 2027 | /// exact power or 2, return the log base 2 integer value. Otherwise, | |||
| 2028 | /// return -1. | |||
| 2029 | /// | |||
| 2030 | /// The BitWidth specifies the necessary bit precision. | |||
| 2031 | int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, | |||
| 2032 | uint32_t BitWidth) const; | |||
| 2033 | ||||
| 2034 | bool isConstant() const; | |||
| 2035 | ||||
| 2036 | static bool classof(const SDNode *N) { | |||
| 2037 | return N->getOpcode() == ISD::BUILD_VECTOR; | |||
| 2038 | } | |||
| 2039 | }; | |||
| 2040 | ||||
| 2041 | /// An SDNode that holds an arbitrary LLVM IR Value. This is | |||
| 2042 | /// used when the SelectionDAG needs to make a simple reference to something | |||
| 2043 | /// in the LLVM IR representation. | |||
| 2044 | /// | |||
| 2045 | class SrcValueSDNode : public SDNode { | |||
| 2046 | friend class SelectionDAG; | |||
| 2047 | ||||
| 2048 | const Value *V; | |||
| 2049 | ||||
| 2050 | /// Create a SrcValue for a general value. | |||
| 2051 | explicit SrcValueSDNode(const Value *v) | |||
| 2052 | : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} | |||
| 2053 | ||||
| 2054 | public: | |||
| 2055 | /// Return the contained Value. | |||
| 2056 | const Value *getValue() const { return V; } | |||
| 2057 | ||||
| 2058 | static bool classof(const SDNode *N) { | |||
| 2059 | return N->getOpcode() == ISD::SRCVALUE; | |||
| 2060 | } | |||
| 2061 | }; | |||
| 2062 | ||||
| 2063 | class MDNodeSDNode : public SDNode { | |||
| 2064 | friend class SelectionDAG; | |||
| 2065 | ||||
| 2066 | const MDNode *MD; | |||
| 2067 | ||||
| 2068 | explicit MDNodeSDNode(const MDNode *md) | |||
| 2069 | : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) | |||
| 2070 | {} | |||
| 2071 | ||||
| 2072 | public: | |||
| 2073 | const MDNode *getMD() const { return MD; } | |||
| 2074 | ||||
| 2075 | static bool classof(const SDNode *N) { | |||
| 2076 | return N->getOpcode() == ISD::MDNODE_SDNODE; | |||
| 2077 | } | |||
| 2078 | }; | |||
| 2079 | ||||
| 2080 | class RegisterSDNode : public SDNode { | |||
| 2081 | friend class SelectionDAG; | |||
| 2082 | ||||
| 2083 | Register Reg; | |||
| 2084 | ||||
| 2085 | RegisterSDNode(Register reg, EVT VT) | |||
| 2086 | : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {} | |||
| 2087 | ||||
| 2088 | public: | |||
| 2089 | Register getReg() const { return Reg; } | |||
| 2090 | ||||
| 2091 | static bool classof(const SDNode *N) { | |||
| 2092 | return N->getOpcode() == ISD::Register; | |||
| 2093 | } | |||
| 2094 | }; | |||
| 2095 | ||||
| 2096 | class RegisterMaskSDNode : public SDNode { | |||
| 2097 | friend class SelectionDAG; | |||
| 2098 | ||||
| 2099 | // The memory for RegMask is not owned by the node. | |||
| 2100 | const uint32_t *RegMask; | |||
| 2101 | ||||
| 2102 | RegisterMaskSDNode(const uint32_t *mask) | |||
| 2103 | : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)), | |||
| 2104 | RegMask(mask) {} | |||
| 2105 | ||||
| 2106 | public: | |||
| 2107 | const uint32_t *getRegMask() const { return RegMask; } | |||
| 2108 | ||||
| 2109 | static bool classof(const SDNode *N) { | |||
| 2110 | return N->getOpcode() == ISD::RegisterMask; | |||
| 2111 | } | |||
| 2112 | }; | |||
| 2113 | ||||
| 2114 | class BlockAddressSDNode : public SDNode { | |||
| 2115 | friend class SelectionDAG; | |||
| 2116 | ||||
| 2117 | const BlockAddress *BA; | |||
| 2118 | int64_t Offset; | |||
| 2119 | unsigned TargetFlags; | |||
| 2120 | ||||
| 2121 | BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, | |||
| 2122 | int64_t o, unsigned Flags) | |||
| 2123 | : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), | |||
| 2124 | BA(ba), Offset(o), TargetFlags(Flags) {} | |||
| 2125 | ||||
| 2126 | public: | |||
| 2127 | const BlockAddress *getBlockAddress() const { return BA; } | |||
| 2128 | int64_t getOffset() const { return Offset; } | |||
| 2129 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 2130 | ||||
| 2131 | static bool classof(const SDNode *N) { | |||
| 2132 | return N->getOpcode() == ISD::BlockAddress || | |||
| 2133 | N->getOpcode() == ISD::TargetBlockAddress; | |||
| 2134 | } | |||
| 2135 | }; | |||
| 2136 | ||||
| 2137 | class LabelSDNode : public SDNode { | |||
| 2138 | friend class SelectionDAG; | |||
| 2139 | ||||
| 2140 | MCSymbol *Label; | |||
| 2141 | ||||
| 2142 | LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L) | |||
| 2143 | : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) { | |||
| 2144 | assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) && "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2144, __extension__ __PRETTY_FUNCTION__)); | |||
| 2145 | } | |||
| 2146 | ||||
| 2147 | public: | |||
| 2148 | MCSymbol *getLabel() const { return Label; } | |||
| 2149 | ||||
| 2150 | static bool classof(const SDNode *N) { | |||
| 2151 | return N->getOpcode() == ISD::EH_LABEL || | |||
| 2152 | N->getOpcode() == ISD::ANNOTATION_LABEL; | |||
| 2153 | } | |||
| 2154 | }; | |||
| 2155 | ||||
| 2156 | class ExternalSymbolSDNode : public SDNode { | |||
| 2157 | friend class SelectionDAG; | |||
| 2158 | ||||
| 2159 | const char *Symbol; | |||
| 2160 | unsigned TargetFlags; | |||
| 2161 | ||||
| 2162 | ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT) | |||
| 2163 | : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0, | |||
| 2164 | DebugLoc(), getSDVTList(VT)), | |||
| 2165 | Symbol(Sym), TargetFlags(TF) {} | |||
| 2166 | ||||
| 2167 | public: | |||
| 2168 | const char *getSymbol() const { return Symbol; } | |||
| 2169 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 2170 | ||||
| 2171 | static bool classof(const SDNode *N) { | |||
| 2172 | return N->getOpcode() == ISD::ExternalSymbol || | |||
| 2173 | N->getOpcode() == ISD::TargetExternalSymbol; | |||
| 2174 | } | |||
| 2175 | }; | |||
| 2176 | ||||
| 2177 | class MCSymbolSDNode : public SDNode { | |||
| 2178 | friend class SelectionDAG; | |||
| 2179 | ||||
| 2180 | MCSymbol *Symbol; | |||
| 2181 | ||||
| 2182 | MCSymbolSDNode(MCSymbol *Symbol, EVT VT) | |||
| 2183 | : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {} | |||
| 2184 | ||||
| 2185 | public: | |||
| 2186 | MCSymbol *getMCSymbol() const { return Symbol; } | |||
| 2187 | ||||
| 2188 | static bool classof(const SDNode *N) { | |||
| 2189 | return N->getOpcode() == ISD::MCSymbol; | |||
| 2190 | } | |||
| 2191 | }; | |||
| 2192 | ||||
| 2193 | class CondCodeSDNode : public SDNode { | |||
| 2194 | friend class SelectionDAG; | |||
| 2195 | ||||
| 2196 | ISD::CondCode Condition; | |||
| 2197 | ||||
| 2198 | explicit CondCodeSDNode(ISD::CondCode Cond) | |||
| 2199 | : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
| 2200 | Condition(Cond) {} | |||
| 2201 | ||||
| 2202 | public: | |||
| 2203 | ISD::CondCode get() const { return Condition; } | |||
| 2204 | ||||
| 2205 | static bool classof(const SDNode *N) { | |||
| 2206 | return N->getOpcode() == ISD::CONDCODE; | |||
| 2207 | } | |||
| 2208 | }; | |||
| 2209 | ||||
| 2210 | /// This class is used to represent EVT's, which are used | |||
| 2211 | /// to parameterize some operations. | |||
| 2212 | class VTSDNode : public SDNode { | |||
| 2213 | friend class SelectionDAG; | |||
| 2214 | ||||
| 2215 | EVT ValueType; | |||
| 2216 | ||||
| 2217 | explicit VTSDNode(EVT VT) | |||
| 2218 | : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
| 2219 | ValueType(VT) {} | |||
| 2220 | ||||
| 2221 | public: | |||
| 2222 | EVT getVT() const { return ValueType; } | |||
| 2223 | ||||
| 2224 | static bool classof(const SDNode *N) { | |||
| 2225 | return N->getOpcode() == ISD::VALUETYPE; | |||
| 2226 | } | |||
| 2227 | }; | |||
| 2228 | ||||
| 2229 | /// Base class for LoadSDNode and StoreSDNode | |||
| 2230 | class LSBaseSDNode : public MemSDNode { | |||
| 2231 | public: | |||
| 2232 | LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, | |||
| 2233 | SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, | |||
| 2234 | MachineMemOperand *MMO) | |||
| 2235 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
| 2236 | LSBaseSDNodeBits.AddressingMode = AM; | |||
| 2237 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2237, __extension__ __PRETTY_FUNCTION__)); | |||
| 2238 | } | |||
| 2239 | ||||
| 2240 | const SDValue &getOffset() const { | |||
| 2241 | return getOperand(getOpcode() == ISD::LOAD ? 2 : 3); | |||
| 2242 | } | |||
| 2243 | ||||
| 2244 | /// Return the addressing mode for this load or store: | |||
| 2245 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
| 2246 | ISD::MemIndexedMode getAddressingMode() const { | |||
| 2247 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
| 2248 | } | |||
| 2249 | ||||
| 2250 | /// Return true if this is a pre/post inc/dec load/store. | |||
| 2251 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
| 2252 | ||||
| 2253 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
| 2254 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
| 2255 | ||||
| 2256 | static bool classof(const SDNode *N) { | |||
| 2257 | return N->getOpcode() == ISD::LOAD || | |||
| 2258 | N->getOpcode() == ISD::STORE; | |||
| 2259 | } | |||
| 2260 | }; | |||
| 2261 | ||||
| 2262 | /// This class is used to represent ISD::LOAD nodes. | |||
| 2263 | class LoadSDNode : public LSBaseSDNode { | |||
| 2264 | friend class SelectionDAG; | |||
| 2265 | ||||
| 2266 | LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2267 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, | |||
| 2268 | MachineMemOperand *MMO) | |||
| 2269 | : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2270 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2271 | assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!" ) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2271, __extension__ __PRETTY_FUNCTION__)); | |||
| 2272 | assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!" ) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2272, __extension__ __PRETTY_FUNCTION__)); | |||
| 2273 | } | |||
| 2274 | ||||
| 2275 | public: | |||
| 2276 | /// Return whether this is a plain node, | |||
| 2277 | /// or one of the varieties of value-extending loads. | |||
| 2278 | ISD::LoadExtType getExtensionType() const { | |||
| 2279 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
| 2280 | } | |||
| 2281 | ||||
| 2282 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 2283 | const SDValue &getOffset() const { return getOperand(2); } | |||
| 2284 | ||||
| 2285 | static bool classof(const SDNode *N) { | |||
| 2286 | return N->getOpcode() == ISD::LOAD; | |||
| 2287 | } | |||
| 2288 | }; | |||
| 2289 | ||||
| 2290 | /// This class is used to represent ISD::STORE nodes. | |||
| 2291 | class StoreSDNode : public LSBaseSDNode { | |||
| 2292 | friend class SelectionDAG; | |||
| 2293 | ||||
| 2294 | StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2295 | ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, | |||
| 2296 | MachineMemOperand *MMO) | |||
| 2297 | : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2298 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
| 2299 | assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!" ) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2299, __extension__ __PRETTY_FUNCTION__)); | |||
| 2300 | assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!" ) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2300, __extension__ __PRETTY_FUNCTION__)); | |||
| 2301 | } | |||
| 2302 | ||||
| 2303 | public: | |||
| 2304 | /// Return true if the op does a truncation before store. | |||
| 2305 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2306 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2307 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2308 | void setTruncatingStore(bool Truncating) { | |||
| 2309 | StoreSDNodeBits.IsTruncating = Truncating; | |||
| 2310 | } | |||
| 2311 | ||||
| 2312 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2313 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
| 2314 | const SDValue &getOffset() const { return getOperand(3); } | |||
| 2315 | ||||
| 2316 | static bool classof(const SDNode *N) { | |||
| 2317 | return N->getOpcode() == ISD::STORE; | |||
| 2318 | } | |||
| 2319 | }; | |||
| 2320 | ||||
| 2321 | /// This base class is used to represent MLOAD and MSTORE nodes | |||
| 2322 | class MaskedLoadStoreSDNode : public MemSDNode { | |||
| 2323 | public: | |||
| 2324 | friend class SelectionDAG; | |||
| 2325 | ||||
| 2326 | MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
| 2327 | const DebugLoc &dl, SDVTList VTs, | |||
| 2328 | ISD::MemIndexedMode AM, EVT MemVT, | |||
| 2329 | MachineMemOperand *MMO) | |||
| 2330 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
| 2331 | LSBaseSDNodeBits.AddressingMode = AM; | |||
| 2332 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2332, __extension__ __PRETTY_FUNCTION__)); | |||
| 2333 | } | |||
| 2334 | ||||
| 2335 | // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru) | |||
| 2336 | // MaskedStoreSDNode (Chain, data, ptr, offset, mask) | |||
| 2337 | // Mask is a vector of i1 elements | |||
| 2338 | const SDValue &getOffset() const { | |||
| 2339 | return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3); | |||
| 2340 | } | |||
| 2341 | const SDValue &getMask() const { | |||
| 2342 | return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4); | |||
| 2343 | } | |||
| 2344 | ||||
| 2345 | /// Return the addressing mode for this load or store: | |||
| 2346 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
| 2347 | ISD::MemIndexedMode getAddressingMode() const { | |||
| 2348 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
| 2349 | } | |||
| 2350 | ||||
| 2351 | /// Return true if this is a pre/post inc/dec load/store. | |||
| 2352 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
| 2353 | ||||
| 2354 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
| 2355 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
| 2356 | ||||
| 2357 | static bool classof(const SDNode *N) { | |||
| 2358 | return N->getOpcode() == ISD::MLOAD || | |||
| 2359 | N->getOpcode() == ISD::MSTORE; | |||
| 2360 | } | |||
| 2361 | }; | |||
| 2362 | ||||
| 2363 | /// This class is used to represent an MLOAD node | |||
| 2364 | class MaskedLoadSDNode : public MaskedLoadStoreSDNode { | |||
| 2365 | public: | |||
| 2366 | friend class SelectionDAG; | |||
| 2367 | ||||
| 2368 | MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2369 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, | |||
| 2370 | bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) | |||
| 2371 | : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2372 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2373 | LoadSDNodeBits.IsExpanding = IsExpanding; | |||
| 2374 | } | |||
| 2375 | ||||
| 2376 | ISD::LoadExtType getExtensionType() const { | |||
| 2377 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
| 2378 | } | |||
| 2379 | ||||
| 2380 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 2381 | const SDValue &getOffset() const { return getOperand(2); } | |||
| 2382 | const SDValue &getMask() const { return getOperand(3); } | |||
| 2383 | const SDValue &getPassThru() const { return getOperand(4); } | |||
| 2384 | ||||
| 2385 | static bool classof(const SDNode *N) { | |||
| 2386 | return N->getOpcode() == ISD::MLOAD; | |||
| 2387 | } | |||
| 2388 | ||||
| 2389 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } | |||
| 2390 | }; | |||
| 2391 | ||||
| 2392 | /// This class is used to represent an MSTORE node | |||
| 2393 | class MaskedStoreSDNode : public MaskedLoadStoreSDNode { | |||
| 2394 | public: | |||
| 2395 | friend class SelectionDAG; | |||
| 2396 | ||||
| 2397 | MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2398 | ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, | |||
| 2399 | EVT MemVT, MachineMemOperand *MMO) | |||
| 2400 | : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2401 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
| 2402 | StoreSDNodeBits.IsCompressing = isCompressing; | |||
| 2403 | } | |||
| 2404 | ||||
| 2405 | /// Return true if the op does a truncation before store. | |||
| 2406 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2407 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2408 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2409 | ||||
| 2410 | /// Returns true if the op does a compression to the vector before storing. | |||
| 2411 | /// The node contiguously stores the active elements (integers or floats) | |||
| 2412 | /// in src (those with their respective bit set in writemask k) to unaligned | |||
| 2413 | /// memory at base_addr. | |||
| 2414 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } | |||
| 2415 | ||||
| 2416 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2417 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
| 2418 | const SDValue &getOffset() const { return getOperand(3); } | |||
| 2419 | const SDValue &getMask() const { return getOperand(4); } | |||
| 2420 | ||||
| 2421 | static bool classof(const SDNode *N) { | |||
| 2422 | return N->getOpcode() == ISD::MSTORE; | |||
| 2423 | } | |||
| 2424 | }; | |||
| 2425 | ||||
| 2426 | /// This is a base class used to represent | |||
| 2427 | /// MGATHER and MSCATTER nodes | |||
| 2428 | /// | |||
| 2429 | class MaskedGatherScatterSDNode : public MemSDNode { | |||
| 2430 | public: | |||
| 2431 | friend class SelectionDAG; | |||
| 2432 | ||||
| 2433 | MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
| 2434 | const DebugLoc &dl, SDVTList VTs, EVT MemVT, | |||
| 2435 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) | |||
| 2436 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
| 2437 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
| 2438 | assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType && "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2438, __extension__ __PRETTY_FUNCTION__)); | |||
| 2439 | } | |||
| 2440 | ||||
| 2441 | /// How is Index applied to BasePtr when computing addresses. | |||
| 2442 | ISD::MemIndexType getIndexType() const { | |||
| 2443 | return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); | |||
| 2444 | } | |||
| 2445 | void setIndexType(ISD::MemIndexType IndexType) { | |||
| 2446 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
| 2447 | } | |||
| 2448 | bool isIndexScaled() const { | |||
| 2449 | return (getIndexType() == ISD::SIGNED_SCALED) || | |||
| 2450 | (getIndexType() == ISD::UNSIGNED_SCALED); | |||
| 2451 | } | |||
| 2452 | bool isIndexSigned() const { | |||
| 2453 | return (getIndexType() == ISD::SIGNED_SCALED) || | |||
| 2454 | (getIndexType() == ISD::SIGNED_UNSCALED); | |||
| 2455 | } | |||
| 2456 | ||||
| 2457 | // In the both nodes address is Op1, mask is Op2: | |||
| 2458 | // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale) | |||
| 2459 | // MaskedScatterSDNode (Chain, value, mask, base, index, scale) | |||
| 2460 | // Mask is a vector of i1 elements | |||
| 2461 | const SDValue &getBasePtr() const { return getOperand(3); } | |||
| 2462 | const SDValue &getIndex() const { return getOperand(4); } | |||
| 2463 | const SDValue &getMask() const { return getOperand(2); } | |||
| 2464 | const SDValue &getScale() const { return getOperand(5); } | |||
| 2465 | ||||
| 2466 | static bool classof(const SDNode *N) { | |||
| 2467 | return N->getOpcode() == ISD::MGATHER || | |||
| 2468 | N->getOpcode() == ISD::MSCATTER; | |||
| 2469 | } | |||
| 2470 | }; | |||
| 2471 | ||||
| 2472 | /// This class is used to represent an MGATHER node | |||
| 2473 | /// | |||
| 2474 | class MaskedGatherSDNode : public MaskedGatherScatterSDNode { | |||
| 2475 | public: | |||
| 2476 | friend class SelectionDAG; | |||
| 2477 | ||||
| 2478 | MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2479 | EVT MemVT, MachineMemOperand *MMO, | |||
| 2480 | ISD::MemIndexType IndexType, ISD::LoadExtType ETy) | |||
| 2481 | : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO, | |||
| 2482 | IndexType) { | |||
| 2483 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2484 | } | |||
| 2485 | ||||
| 2486 | const SDValue &getPassThru() const { return getOperand(1); } | |||
| 2487 | ||||
| 2488 | ISD::LoadExtType getExtensionType() const { | |||
| 2489 | return ISD::LoadExtType(LoadSDNodeBits.ExtTy); | |||
| 2490 | } | |||
| 2491 | ||||
| 2492 | static bool classof(const SDNode *N) { | |||
| 2493 | return N->getOpcode() == ISD::MGATHER; | |||
| 2494 | } | |||
| 2495 | }; | |||
| 2496 | ||||
| 2497 | /// This class is used to represent an MSCATTER node | |||
| 2498 | /// | |||
| 2499 | class MaskedScatterSDNode : public MaskedGatherScatterSDNode { | |||
| 2500 | public: | |||
| 2501 | friend class SelectionDAG; | |||
| 2502 | ||||
| 2503 | MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2504 | EVT MemVT, MachineMemOperand *MMO, | |||
| 2505 | ISD::MemIndexType IndexType, bool IsTrunc) | |||
| 2506 | : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO, | |||
| 2507 | IndexType) { | |||
| 2508 | StoreSDNodeBits.IsTruncating = IsTrunc; | |||
| 2509 | } | |||
| 2510 | ||||
| 2511 | /// Return true if the op does a truncation before store. | |||
| 2512 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2513 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2514 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2515 | ||||
| 2516 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2517 | ||||
| 2518 | static bool classof(const SDNode *N) { | |||
| 2519 | return N->getOpcode() == ISD::MSCATTER; | |||
| 2520 | } | |||
| 2521 | }; | |||
| 2522 | ||||
| 2523 | /// An SDNode that represents everything that will be needed | |||
| 2524 | /// to construct a MachineInstr. These nodes are created during the | |||
| 2525 | /// instruction selection proper phase. | |||
| 2526 | /// | |||
| 2527 | /// Note that the only supported way to set the `memoperands` is by calling the | |||
| 2528 | /// `SelectionDAG::setNodeMemRefs` function as the memory management happens | |||
| 2529 | /// inside the DAG rather than in the node. | |||
| 2530 | class MachineSDNode : public SDNode { | |||
| 2531 | private: | |||
| 2532 | friend class SelectionDAG; | |||
| 2533 | ||||
| 2534 | MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) | |||
| 2535 | : SDNode(Opc, Order, DL, VTs) {} | |||
| 2536 | ||||
| 2537 | // We use a pointer union between a single `MachineMemOperand` pointer and | |||
| 2538 | // a pointer to an array of `MachineMemOperand` pointers. This is null when | |||
| 2539 | // the number of these is zero, the single pointer variant used when the | |||
| 2540 | // number is one, and the array is used for larger numbers. | |||
| 2541 | // | |||
| 2542 | // The array is allocated via the `SelectionDAG`'s allocator and so will | |||
| 2543 | // always live until the DAG is cleaned up and doesn't require ownership here. | |||
| 2544 | // | |||
| 2545 | // We can't use something simpler like `TinyPtrVector` here because `SDNode` | |||
| 2546 | // subclasses aren't managed in a conforming C++ manner. See the comments on | |||
| 2547 | // `SelectionDAG::MorphNodeTo` which details what all goes on, but the | |||
| 2548 | // constraint here is that these don't manage memory with their constructor or | |||
| 2549 | // destructor and can be initialized to a good state even if they start off | |||
| 2550 | // uninitialized. | |||
| 2551 | PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {}; | |||
| 2552 | ||||
| 2553 | // Note that this could be folded into the above `MemRefs` member if doing so | |||
| 2554 | // is advantageous at some point. We don't need to store this in most cases. | |||
| 2555 | // However, at the moment this doesn't appear to make the allocation any | |||
| 2556 | // smaller and makes the code somewhat simpler to read. | |||
| 2557 | int NumMemRefs = 0; | |||
| 2558 | ||||
| 2559 | public: | |||
| 2560 | using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator; | |||
| 2561 | ||||
| 2562 | ArrayRef<MachineMemOperand *> memoperands() const { | |||
| 2563 | // Special case the common cases. | |||
| 2564 | if (NumMemRefs == 0) | |||
| 2565 | return {}; | |||
| 2566 | if (NumMemRefs == 1) | |||
| 2567 | return makeArrayRef(MemRefs.getAddrOfPtr1(), 1); | |||
| 2568 | ||||
| 2569 | // Otherwise we have an actual array. | |||
| 2570 | return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs); | |||
| 2571 | } | |||
| 2572 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } | |||
| 2573 | mmo_iterator memoperands_end() const { return memoperands().end(); } | |||
| 2574 | bool memoperands_empty() const { return memoperands().empty(); } | |||
| 2575 | ||||
| 2576 | /// Clear out the memory reference descriptor list. | |||
| 2577 | void clearMemRefs() { | |||
| 2578 | MemRefs = nullptr; | |||
| 2579 | NumMemRefs = 0; | |||
| 2580 | } | |||
| 2581 | ||||
| 2582 | static bool classof(const SDNode *N) { | |||
| 2583 | return N->isMachineOpcode(); | |||
| 2584 | } | |||
| 2585 | }; | |||
| 2586 | ||||
| 2587 | /// An SDNode that records if a register contains a value that is guaranteed to | |||
| 2588 | /// be aligned accordingly. | |||
| 2589 | class AssertAlignSDNode : public SDNode { | |||
| 2590 | Align Alignment; | |||
| 2591 | ||||
| 2592 | public: | |||
| 2593 | AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A) | |||
| 2594 | : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {} | |||
| 2595 | ||||
| 2596 | Align getAlign() const { return Alignment; } | |||
| 2597 | ||||
| 2598 | static bool classof(const SDNode *N) { | |||
| 2599 | return N->getOpcode() == ISD::AssertAlign; | |||
| 2600 | } | |||
| 2601 | }; | |||
| 2602 | ||||
| 2603 | class SDNodeIterator { | |||
| 2604 | const SDNode *Node; | |||
| 2605 | unsigned Operand; | |||
| 2606 | ||||
| 2607 | SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {} | |||
| 2608 | ||||
| 2609 | public: | |||
| 2610 | using iterator_category = std::forward_iterator_tag; | |||
| 2611 | using value_type = SDNode; | |||
| 2612 | using difference_type = std::ptrdiff_t; | |||
| 2613 | using pointer = value_type *; | |||
| 2614 | using reference = value_type &; | |||
| 2615 | ||||
| 2616 | bool operator==(const SDNodeIterator& x) const { | |||
| 2617 | return Operand == x.Operand; | |||
| 2618 | } | |||
| 2619 | bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } | |||
| 2620 | ||||
| 2621 | pointer operator*() const { | |||
| 2622 | return Node->getOperand(Operand).getNode(); | |||
| 2623 | } | |||
| 2624 | pointer operator->() const { return operator*(); } | |||
| 2625 | ||||
| 2626 | SDNodeIterator& operator++() { // Preincrement | |||
| 2627 | ++Operand; | |||
| 2628 | return *this; | |||
| 2629 | } | |||
| 2630 | SDNodeIterator operator++(int) { // Postincrement | |||
| 2631 | SDNodeIterator tmp = *this; ++*this; return tmp; | |||
| 2632 | } | |||
| 2633 | size_t operator-(SDNodeIterator Other) const { | |||
| 2634 | assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2635, __extension__ __PRETTY_FUNCTION__)) | |||
| 2635 | "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2635, __extension__ __PRETTY_FUNCTION__)); | |||
| 2636 | return Operand - Other.Operand; | |||
| 2637 | } | |||
| 2638 | ||||
| 2639 | static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); } | |||
| 2640 | static SDNodeIterator end (const SDNode *N) { | |||
| 2641 | return SDNodeIterator(N, N->getNumOperands()); | |||
| 2642 | } | |||
| 2643 | ||||
| 2644 | unsigned getOperand() const { return Operand; } | |||
| 2645 | const SDNode *getNode() const { return Node; } | |||
| 2646 | }; | |||
| 2647 | ||||
| 2648 | template <> struct GraphTraits<SDNode*> { | |||
| 2649 | using NodeRef = SDNode *; | |||
| 2650 | using ChildIteratorType = SDNodeIterator; | |||
| 2651 | ||||
| 2652 | static NodeRef getEntryNode(SDNode *N) { return N; } | |||
| 2653 | ||||
| 2654 | static ChildIteratorType child_begin(NodeRef N) { | |||
| 2655 | return SDNodeIterator::begin(N); | |||
| 2656 | } | |||
| 2657 | ||||
| 2658 | static ChildIteratorType child_end(NodeRef N) { | |||
| 2659 | return SDNodeIterator::end(N); | |||
| 2660 | } | |||
| 2661 | }; | |||
| 2662 | ||||
| 2663 | /// A representation of the largest SDNode, for use in sizeof(). | |||
| 2664 | /// | |||
| 2665 | /// This needs to be a union because the largest node differs on 32 bit systems | |||
| 2666 | /// with 4 and 8 byte pointer alignment, respectively. | |||
| 2667 | using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, | |||
| 2668 | BlockAddressSDNode, | |||
| 2669 | GlobalAddressSDNode, | |||
| 2670 | PseudoProbeSDNode>; | |||
| 2671 | ||||
| 2672 | /// The SDNode class with the greatest alignment requirement. | |||
| 2673 | using MostAlignedSDNode = GlobalAddressSDNode; | |||
| 2674 | ||||
| 2675 | namespace ISD { | |||
| 2676 | ||||
| 2677 | /// Returns true if the specified node is a non-extending and unindexed load. | |||
| 2678 | inline bool isNormalLoad(const SDNode *N) { | |||
| 2679 | const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N); | |||
| 2680 | return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && | |||
| 2681 | Ld->getAddressingMode() == ISD::UNINDEXED; | |||
| 2682 | } | |||
| 2683 | ||||
| 2684 | /// Returns true if the specified node is a non-extending load. | |||
| 2685 | inline bool isNON_EXTLoad(const SDNode *N) { | |||
| 2686 | return isa<LoadSDNode>(N) && | |||
| 2687 | cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; | |||
| 2688 | } | |||
| 2689 | ||||
| 2690 | /// Returns true if the specified node is a EXTLOAD. | |||
| 2691 | inline bool isEXTLoad(const SDNode *N) { | |||
| 2692 | return isa<LoadSDNode>(N) && | |||
| 2693 | cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; | |||
| 2694 | } | |||
| 2695 | ||||
| 2696 | /// Returns true if the specified node is a SEXTLOAD. | |||
| 2697 | inline bool isSEXTLoad(const SDNode *N) { | |||
| 2698 | return isa<LoadSDNode>(N) && | |||
| 2699 | cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; | |||
| 2700 | } | |||
| 2701 | ||||
| 2702 | /// Returns true if the specified node is a ZEXTLOAD. | |||
| 2703 | inline bool isZEXTLoad(const SDNode *N) { | |||
| 2704 | return isa<LoadSDNode>(N) && | |||
| 2705 | cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; | |||
| 2706 | } | |||
| 2707 | ||||
| 2708 | /// Returns true if the specified node is an unindexed load. | |||
| 2709 | inline bool isUNINDEXEDLoad(const SDNode *N) { | |||
| 2710 | return isa<LoadSDNode>(N) && | |||
| 2711 | cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
| 2712 | } | |||
| 2713 | ||||
| 2714 | /// Returns true if the specified node is a non-truncating | |||
| 2715 | /// and unindexed store. | |||
| 2716 | inline bool isNormalStore(const SDNode *N) { | |||
| 2717 | const StoreSDNode *St = dyn_cast<StoreSDNode>(N); | |||
| 2718 | return St && !St->isTruncatingStore() && | |||
| 2719 | St->getAddressingMode() == ISD::UNINDEXED; | |||
| 2720 | } | |||
| 2721 | ||||
| 2722 | /// Returns true if the specified node is an unindexed store. | |||
| 2723 | inline bool isUNINDEXEDStore(const SDNode *N) { | |||
| 2724 | return isa<StoreSDNode>(N) && | |||
| 2725 | cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
| 2726 | } | |||
| 2727 | ||||
| 2728 | /// Attempt to match a unary predicate against a scalar/splat constant or | |||
| 2729 | /// every element of a constant BUILD_VECTOR. | |||
| 2730 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
| 2731 | bool matchUnaryPredicate(SDValue Op, | |||
| 2732 | std::function<bool(ConstantSDNode *)> Match, | |||
| 2733 | bool AllowUndefs = false); | |||
| 2734 | ||||
| 2735 | /// Attempt to match a binary predicate against a pair of scalar/splat | |||
| 2736 | /// constants or every element of a pair of constant BUILD_VECTORs. | |||
| 2737 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
| 2738 | /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match. | |||
| 2739 | bool matchBinaryPredicate( | |||
| 2740 | SDValue LHS, SDValue RHS, | |||
| 2741 | std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, | |||
| 2742 | bool AllowUndefs = false, bool AllowTypeMismatch = false); | |||
| 2743 | ||||
| 2744 | /// Returns true if the specified value is the overflow result from one | |||
| 2745 | /// of the overflow intrinsic nodes. | |||
| 2746 | inline bool isOverflowIntrOpRes(SDValue Op) { | |||
| 2747 | unsigned Opc = Op.getOpcode(); | |||
| 2748 | return (Op.getResNo() == 1 && | |||
| 2749 | (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || | |||
| 2750 | Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)); | |||
| 2751 | } | |||
| 2752 | ||||
| 2753 | } // end namespace ISD | |||
| 2754 | ||||
| 2755 | } // end namespace llvm | |||
| 2756 | ||||
| 2757 | #endif // LLVM_CODEGEN_SELECTIONDAGNODES_H |