| File: | build/source/llvm/include/llvm/CodeGen/SelectionDAGNodes.h |
| Warning: | line 1138, column 10 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // This implements routines for translating from LLVM IR into SelectionDAG IR. | ||||||||
| 10 | // | ||||||||
| 11 | //===----------------------------------------------------------------------===// | ||||||||
| 12 | |||||||||
| 13 | #include "SelectionDAGBuilder.h" | ||||||||
| 14 | #include "SDNodeDbgValue.h" | ||||||||
| 15 | #include "llvm/ADT/APFloat.h" | ||||||||
| 16 | #include "llvm/ADT/APInt.h" | ||||||||
| 17 | #include "llvm/ADT/BitVector.h" | ||||||||
| 18 | #include "llvm/ADT/STLExtras.h" | ||||||||
| 19 | #include "llvm/ADT/SmallPtrSet.h" | ||||||||
| 20 | #include "llvm/ADT/SmallSet.h" | ||||||||
| 21 | #include "llvm/ADT/StringRef.h" | ||||||||
| 22 | #include "llvm/ADT/Twine.h" | ||||||||
| 23 | #include "llvm/Analysis/AliasAnalysis.h" | ||||||||
| 24 | #include "llvm/Analysis/BranchProbabilityInfo.h" | ||||||||
| 25 | #include "llvm/Analysis/ConstantFolding.h" | ||||||||
| 26 | #include "llvm/Analysis/Loads.h" | ||||||||
| 27 | #include "llvm/Analysis/MemoryLocation.h" | ||||||||
| 28 | #include "llvm/Analysis/TargetLibraryInfo.h" | ||||||||
| 29 | #include "llvm/Analysis/ValueTracking.h" | ||||||||
| 30 | #include "llvm/Analysis/VectorUtils.h" | ||||||||
| 31 | #include "llvm/CodeGen/Analysis.h" | ||||||||
| 32 | #include "llvm/CodeGen/AssignmentTrackingAnalysis.h" | ||||||||
| 33 | #include "llvm/CodeGen/CodeGenCommonISel.h" | ||||||||
| 34 | #include "llvm/CodeGen/FunctionLoweringInfo.h" | ||||||||
| 35 | #include "llvm/CodeGen/GCMetadata.h" | ||||||||
| 36 | #include "llvm/CodeGen/MachineBasicBlock.h" | ||||||||
| 37 | #include "llvm/CodeGen/MachineFrameInfo.h" | ||||||||
| 38 | #include "llvm/CodeGen/MachineFunction.h" | ||||||||
| 39 | #include "llvm/CodeGen/MachineInstrBuilder.h" | ||||||||
| 40 | #include "llvm/CodeGen/MachineInstrBundleIterator.h" | ||||||||
| 41 | #include "llvm/CodeGen/MachineMemOperand.h" | ||||||||
| 42 | #include "llvm/CodeGen/MachineModuleInfo.h" | ||||||||
| 43 | #include "llvm/CodeGen/MachineOperand.h" | ||||||||
| 44 | #include "llvm/CodeGen/MachineRegisterInfo.h" | ||||||||
| 45 | #include "llvm/CodeGen/RuntimeLibcalls.h" | ||||||||
| 46 | #include "llvm/CodeGen/SelectionDAG.h" | ||||||||
| 47 | #include "llvm/CodeGen/SelectionDAGTargetInfo.h" | ||||||||
| 48 | #include "llvm/CodeGen/StackMaps.h" | ||||||||
| 49 | #include "llvm/CodeGen/SwiftErrorValueTracking.h" | ||||||||
| 50 | #include "llvm/CodeGen/TargetFrameLowering.h" | ||||||||
| 51 | #include "llvm/CodeGen/TargetInstrInfo.h" | ||||||||
| 52 | #include "llvm/CodeGen/TargetOpcodes.h" | ||||||||
| 53 | #include "llvm/CodeGen/TargetRegisterInfo.h" | ||||||||
| 54 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | ||||||||
| 55 | #include "llvm/CodeGen/WinEHFuncInfo.h" | ||||||||
| 56 | #include "llvm/IR/Argument.h" | ||||||||
| 57 | #include "llvm/IR/Attributes.h" | ||||||||
| 58 | #include "llvm/IR/BasicBlock.h" | ||||||||
| 59 | #include "llvm/IR/CFG.h" | ||||||||
| 60 | #include "llvm/IR/CallingConv.h" | ||||||||
| 61 | #include "llvm/IR/Constant.h" | ||||||||
| 62 | #include "llvm/IR/ConstantRange.h" | ||||||||
| 63 | #include "llvm/IR/Constants.h" | ||||||||
| 64 | #include "llvm/IR/DataLayout.h" | ||||||||
| 65 | #include "llvm/IR/DebugInfo.h" | ||||||||
| 66 | #include "llvm/IR/DebugInfoMetadata.h" | ||||||||
| 67 | #include "llvm/IR/DerivedTypes.h" | ||||||||
| 68 | #include "llvm/IR/DiagnosticInfo.h" | ||||||||
| 69 | #include "llvm/IR/EHPersonalities.h" | ||||||||
| 70 | #include "llvm/IR/Function.h" | ||||||||
| 71 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||||||
| 72 | #include "llvm/IR/InlineAsm.h" | ||||||||
| 73 | #include "llvm/IR/InstrTypes.h" | ||||||||
| 74 | #include "llvm/IR/Instructions.h" | ||||||||
| 75 | #include "llvm/IR/IntrinsicInst.h" | ||||||||
| 76 | #include "llvm/IR/Intrinsics.h" | ||||||||
| 77 | #include "llvm/IR/IntrinsicsAArch64.h" | ||||||||
| 78 | #include "llvm/IR/IntrinsicsWebAssembly.h" | ||||||||
| 79 | #include "llvm/IR/LLVMContext.h" | ||||||||
| 80 | #include "llvm/IR/Metadata.h" | ||||||||
| 81 | #include "llvm/IR/Module.h" | ||||||||
| 82 | #include "llvm/IR/Operator.h" | ||||||||
| 83 | #include "llvm/IR/PatternMatch.h" | ||||||||
| 84 | #include "llvm/IR/Statepoint.h" | ||||||||
| 85 | #include "llvm/IR/Type.h" | ||||||||
| 86 | #include "llvm/IR/User.h" | ||||||||
| 87 | #include "llvm/IR/Value.h" | ||||||||
| 88 | #include "llvm/MC/MCContext.h" | ||||||||
| 89 | #include "llvm/Support/AtomicOrdering.h" | ||||||||
| 90 | #include "llvm/Support/Casting.h" | ||||||||
| 91 | #include "llvm/Support/CommandLine.h" | ||||||||
| 92 | #include "llvm/Support/Compiler.h" | ||||||||
| 93 | #include "llvm/Support/Debug.h" | ||||||||
| 94 | #include "llvm/Support/MathExtras.h" | ||||||||
| 95 | #include "llvm/Support/raw_ostream.h" | ||||||||
| 96 | #include "llvm/Target/TargetIntrinsicInfo.h" | ||||||||
| 97 | #include "llvm/Target/TargetMachine.h" | ||||||||
| 98 | #include "llvm/Target/TargetOptions.h" | ||||||||
| 99 | #include "llvm/TargetParser/Triple.h" | ||||||||
| 100 | #include "llvm/Transforms/Utils/Local.h" | ||||||||
| 101 | #include <cstddef> | ||||||||
| 102 | #include <iterator> | ||||||||
| 103 | #include <limits> | ||||||||
| 104 | #include <optional> | ||||||||
| 105 | #include <tuple> | ||||||||
| 106 | |||||||||
| 107 | using namespace llvm; | ||||||||
| 108 | using namespace PatternMatch; | ||||||||
| 109 | using namespace SwitchCG; | ||||||||
| 110 | |||||||||
| 111 | #define DEBUG_TYPE"isel" "isel" | ||||||||
| 112 | |||||||||
| 113 | /// LimitFloatPrecision - Generate low-precision inline sequences for | ||||||||
| 114 | /// some float libcalls (6, 8 or 12 bits). | ||||||||
| 115 | static unsigned LimitFloatPrecision; | ||||||||
| 116 | |||||||||
| 117 | static cl::opt<bool> | ||||||||
| 118 | InsertAssertAlign("insert-assert-align", cl::init(true), | ||||||||
| 119 | cl::desc("Insert the experimental `assertalign` node."), | ||||||||
| 120 | cl::ReallyHidden); | ||||||||
| 121 | |||||||||
| 122 | static cl::opt<unsigned, true> | ||||||||
| 123 | LimitFPPrecision("limit-float-precision", | ||||||||
| 124 | cl::desc("Generate low-precision inline sequences " | ||||||||
| 125 | "for some float libcalls"), | ||||||||
| 126 | cl::location(LimitFloatPrecision), cl::Hidden, | ||||||||
| 127 | cl::init(0)); | ||||||||
| 128 | |||||||||
| 129 | static cl::opt<unsigned> SwitchPeelThreshold( | ||||||||
| 130 | "switch-peel-threshold", cl::Hidden, cl::init(66), | ||||||||
| 131 | cl::desc("Set the case probability threshold for peeling the case from a " | ||||||||
| 132 | "switch statement. A value greater than 100 will void this " | ||||||||
| 133 | "optimization")); | ||||||||
| 134 | |||||||||
| 135 | // Limit the width of DAG chains. This is important in general to prevent | ||||||||
| 136 | // DAG-based analysis from blowing up. For example, alias analysis and | ||||||||
| 137 | // load clustering may not complete in reasonable time. It is difficult to | ||||||||
| 138 | // recognize and avoid this situation within each individual analysis, and | ||||||||
| 139 | // future analyses are likely to have the same behavior. Limiting DAG width is | ||||||||
| 140 | // the safe approach and will be especially important with global DAGs. | ||||||||
| 141 | // | ||||||||
| 142 | // MaxParallelChains default is arbitrarily high to avoid affecting | ||||||||
| 143 | // optimization, but could be lowered to improve compile time. Any ld-ld-st-st | ||||||||
| 144 | // sequence over this should have been converted to llvm.memcpy by the | ||||||||
| 145 | // frontend. It is easy to induce this behavior with .ll code such as: | ||||||||
| 146 | // %buffer = alloca [4096 x i8] | ||||||||
| 147 | // %data = load [4096 x i8]* %argPtr | ||||||||
| 148 | // store [4096 x i8] %data, [4096 x i8]* %buffer | ||||||||
| 149 | static const unsigned MaxParallelChains = 64; | ||||||||
| 150 | |||||||||
| 151 | static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, | ||||||||
| 152 | const SDValue *Parts, unsigned NumParts, | ||||||||
| 153 | MVT PartVT, EVT ValueVT, const Value *V, | ||||||||
| 154 | std::optional<CallingConv::ID> CC); | ||||||||
| 155 | |||||||||
| 156 | /// getCopyFromParts - Create a value that contains the specified legal parts | ||||||||
| 157 | /// combined into the value they represent. If the parts combine to a type | ||||||||
| 158 | /// larger than ValueVT then AssertOp can be used to specify whether the extra | ||||||||
| 159 | /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT | ||||||||
| 160 | /// (ISD::AssertSext). | ||||||||
| 161 | static SDValue | ||||||||
| 162 | getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, | ||||||||
| 163 | unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, | ||||||||
| 164 | std::optional<CallingConv::ID> CC = std::nullopt, | ||||||||
| 165 | std::optional<ISD::NodeType> AssertOp = std::nullopt) { | ||||||||
| 166 | // Let the target assemble the parts if it wants to | ||||||||
| 167 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 168 | if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts, | ||||||||
| 169 | PartVT, ValueVT, CC)) | ||||||||
| 170 | return Val; | ||||||||
| 171 | |||||||||
| 172 | if (ValueVT.isVector()) | ||||||||
| 173 | return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V, | ||||||||
| 174 | CC); | ||||||||
| 175 | |||||||||
| 176 | assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!" ) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 176 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 177 | SDValue Val = Parts[0]; | ||||||||
| 178 | |||||||||
| 179 | if (NumParts > 1) { | ||||||||
| 180 | // Assemble the value from multiple parts. | ||||||||
| 181 | if (ValueVT.isInteger()) { | ||||||||
| 182 | unsigned PartBits = PartVT.getSizeInBits(); | ||||||||
| 183 | unsigned ValueBits = ValueVT.getSizeInBits(); | ||||||||
| 184 | |||||||||
| 185 | // Assemble the power of 2 part. | ||||||||
| 186 | unsigned RoundParts = llvm::bit_floor(NumParts); | ||||||||
| 187 | unsigned RoundBits = PartBits * RoundParts; | ||||||||
| 188 | EVT RoundVT = RoundBits == ValueBits ? | ||||||||
| 189 | ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); | ||||||||
| 190 | SDValue Lo, Hi; | ||||||||
| 191 | |||||||||
| 192 | EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); | ||||||||
| 193 | |||||||||
| 194 | if (RoundParts > 2) { | ||||||||
| 195 | Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, | ||||||||
| 196 | PartVT, HalfVT, V); | ||||||||
| 197 | Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, | ||||||||
| 198 | RoundParts / 2, PartVT, HalfVT, V); | ||||||||
| 199 | } else { | ||||||||
| 200 | Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); | ||||||||
| 201 | Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); | ||||||||
| 202 | } | ||||||||
| 203 | |||||||||
| 204 | if (DAG.getDataLayout().isBigEndian()) | ||||||||
| 205 | std::swap(Lo, Hi); | ||||||||
| 206 | |||||||||
| 207 | Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); | ||||||||
| 208 | |||||||||
| 209 | if (RoundParts < NumParts) { | ||||||||
| 210 | // Assemble the trailing non-power-of-2 part. | ||||||||
| 211 | unsigned OddParts = NumParts - RoundParts; | ||||||||
| 212 | EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); | ||||||||
| 213 | Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT, | ||||||||
| 214 | OddVT, V, CC); | ||||||||
| 215 | |||||||||
| 216 | // Combine the round and odd parts. | ||||||||
| 217 | Lo = Val; | ||||||||
| 218 | if (DAG.getDataLayout().isBigEndian()) | ||||||||
| 219 | std::swap(Lo, Hi); | ||||||||
| 220 | EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||
| 221 | Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); | ||||||||
| 222 | Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi, | ||||||||
| 223 | DAG.getConstant(Lo.getValueSizeInBits(), DL, | ||||||||
| 224 | TLI.getShiftAmountTy( | ||||||||
| 225 | TotalVT, DAG.getDataLayout()))); | ||||||||
| 226 | Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); | ||||||||
| 227 | Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); | ||||||||
| 228 | } | ||||||||
| 229 | } else if (PartVT.isFloatingPoint()) { | ||||||||
| 230 | // FP split into multiple FP parts (for ppcf128) | ||||||||
| 231 | assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && "Unexpected split") ? void (0) : __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 232 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 232 | "Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && "Unexpected split") ? void (0) : __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 232 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 233 | SDValue Lo, Hi; | ||||||||
| 234 | Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); | ||||||||
| 235 | Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); | ||||||||
| 236 | if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) | ||||||||
| 237 | std::swap(Lo, Hi); | ||||||||
| 238 | Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); | ||||||||
| 239 | } else { | ||||||||
| 240 | // FP split into integer parts (soft fp) | ||||||||
| 241 | assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && "Unexpected split" ) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 242 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 242 | !PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && "Unexpected split" ) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 242 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 243 | EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); | ||||||||
| 244 | Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC); | ||||||||
| 245 | } | ||||||||
| 246 | } | ||||||||
| 247 | |||||||||
| 248 | // There is now one part, held in Val. Correct it to match ValueVT. | ||||||||
| 249 | // PartEVT is the type of the register class that holds the value. | ||||||||
| 250 | // ValueVT is the type of the inline asm operation. | ||||||||
| 251 | EVT PartEVT = Val.getValueType(); | ||||||||
| 252 | |||||||||
| 253 | if (PartEVT == ValueVT) | ||||||||
| 254 | return Val; | ||||||||
| 255 | |||||||||
| 256 | if (PartEVT.isInteger() && ValueVT.isFloatingPoint() && | ||||||||
| 257 | ValueVT.bitsLT(PartEVT)) { | ||||||||
| 258 | // For an FP value in an integer part, we need to truncate to the right | ||||||||
| 259 | // width first. | ||||||||
| 260 | PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); | ||||||||
| 261 | Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val); | ||||||||
| 262 | } | ||||||||
| 263 | |||||||||
| 264 | // Handle types that have the same size. | ||||||||
| 265 | if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) | ||||||||
| 266 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||
| 267 | |||||||||
| 268 | // Handle types with different sizes. | ||||||||
| 269 | if (PartEVT.isInteger() && ValueVT.isInteger()) { | ||||||||
| 270 | if (ValueVT.bitsLT(PartEVT)) { | ||||||||
| 271 | // For a truncate, see if we have any information to | ||||||||
| 272 | // indicate whether the truncated bits will always be | ||||||||
| 273 | // zero or sign-extension. | ||||||||
| 274 | if (AssertOp) | ||||||||
| 275 | Val = DAG.getNode(*AssertOp, DL, PartEVT, Val, | ||||||||
| 276 | DAG.getValueType(ValueVT)); | ||||||||
| 277 | return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||
| 278 | } | ||||||||
| 279 | return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); | ||||||||
| 280 | } | ||||||||
| 281 | |||||||||
| 282 | if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { | ||||||||
| 283 | // FP_ROUND's are always exact here. | ||||||||
| 284 | if (ValueVT.bitsLT(Val.getValueType())) | ||||||||
| 285 | return DAG.getNode( | ||||||||
| 286 | ISD::FP_ROUND, DL, ValueVT, Val, | ||||||||
| 287 | DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 288 | |||||||||
| 289 | return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); | ||||||||
| 290 | } | ||||||||
| 291 | |||||||||
| 292 | // Handle MMX to a narrower integer type by bitcasting MMX to integer and | ||||||||
| 293 | // then truncating. | ||||||||
| 294 | if (PartEVT == MVT::x86mmx && ValueVT.isInteger() && | ||||||||
| 295 | ValueVT.bitsLT(PartEVT)) { | ||||||||
| 296 | Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val); | ||||||||
| 297 | return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||
| 298 | } | ||||||||
| 299 | |||||||||
| 300 | report_fatal_error("Unknown mismatch in getCopyFromParts!"); | ||||||||
| 301 | } | ||||||||
| 302 | |||||||||
| 303 | static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, | ||||||||
| 304 | const Twine &ErrMsg) { | ||||||||
| 305 | const Instruction *I = dyn_cast_or_null<Instruction>(V); | ||||||||
| 306 | if (!V) | ||||||||
| 307 | return Ctx.emitError(ErrMsg); | ||||||||
| 308 | |||||||||
| 309 | const char *AsmError = ", possible invalid constraint for vector type"; | ||||||||
| 310 | if (const CallInst *CI = dyn_cast<CallInst>(I)) | ||||||||
| 311 | if (CI->isInlineAsm()) | ||||||||
| 312 | return Ctx.emitError(I, ErrMsg + AsmError); | ||||||||
| 313 | |||||||||
| 314 | return Ctx.emitError(I, ErrMsg); | ||||||||
| 315 | } | ||||||||
| 316 | |||||||||
| 317 | /// getCopyFromPartsVector - Create a value that contains the specified legal | ||||||||
| 318 | /// parts combined into the value they represent. If the parts combine to a | ||||||||
| 319 | /// type larger than ValueVT then AssertOp can be used to specify whether the | ||||||||
| 320 | /// extra bits are known to be zero (ISD::AssertZext) or sign extended from | ||||||||
| 321 | /// ValueVT (ISD::AssertSext). | ||||||||
| 322 | static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, | ||||||||
| 323 | const SDValue *Parts, unsigned NumParts, | ||||||||
| 324 | MVT PartVT, EVT ValueVT, const Value *V, | ||||||||
| 325 | std::optional<CallingConv::ID> CallConv) { | ||||||||
| 326 | assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value" ) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 326 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 327 | assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!" ) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 327 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 328 | const bool IsABIRegCopy = CallConv.has_value(); | ||||||||
| 329 | |||||||||
| 330 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 331 | SDValue Val = Parts[0]; | ||||||||
| 332 | |||||||||
| 333 | // Handle a multi-element vector. | ||||||||
| 334 | if (NumParts > 1) { | ||||||||
| 335 | EVT IntermediateVT; | ||||||||
| 336 | MVT RegisterVT; | ||||||||
| 337 | unsigned NumIntermediates; | ||||||||
| 338 | unsigned NumRegs; | ||||||||
| 339 | |||||||||
| 340 | if (IsABIRegCopy) { | ||||||||
| 341 | NumRegs = TLI.getVectorTypeBreakdownForCallingConv( | ||||||||
| 342 | *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, | ||||||||
| 343 | NumIntermediates, RegisterVT); | ||||||||
| 344 | } else { | ||||||||
| 345 | NumRegs = | ||||||||
| 346 | TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, | ||||||||
| 347 | NumIntermediates, RegisterVT); | ||||||||
| 348 | } | ||||||||
| 349 | |||||||||
| 350 | assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 350 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 351 | NumParts = NumRegs; // Silence a compiler warning. | ||||||||
| 352 | assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 352 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 353 | assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts [0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!" ) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 354 | Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts [0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!" ) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 355 | "Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts [0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!" ) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 356 | |||||||||
| 357 | // Assemble the parts into intermediate operands. | ||||||||
| 358 | SmallVector<SDValue, 8> Ops(NumIntermediates); | ||||||||
| 359 | if (NumIntermediates == NumParts) { | ||||||||
| 360 | // If the register was not expanded, truncate or copy the value, | ||||||||
| 361 | // as appropriate. | ||||||||
| 362 | for (unsigned i = 0; i != NumParts; ++i) | ||||||||
| 363 | Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, | ||||||||
| 364 | PartVT, IntermediateVT, V, CallConv); | ||||||||
| 365 | } else if (NumParts > 0) { | ||||||||
| 366 | // If the intermediate type was expanded, build the intermediate | ||||||||
| 367 | // operands from the parts. | ||||||||
| 368 | assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 369 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 369 | "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 369 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 370 | unsigned Factor = NumParts / NumIntermediates; | ||||||||
| 371 | for (unsigned i = 0; i != NumIntermediates; ++i) | ||||||||
| 372 | Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, | ||||||||
| 373 | PartVT, IntermediateVT, V, CallConv); | ||||||||
| 374 | } | ||||||||
| 375 | |||||||||
| 376 | // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the | ||||||||
| 377 | // intermediate operands. | ||||||||
| 378 | EVT BuiltVectorTy = | ||||||||
| 379 | IntermediateVT.isVector() | ||||||||
| 380 | ? EVT::getVectorVT( | ||||||||
| 381 | *DAG.getContext(), IntermediateVT.getScalarType(), | ||||||||
| 382 | IntermediateVT.getVectorElementCount() * NumParts) | ||||||||
| 383 | : EVT::getVectorVT(*DAG.getContext(), | ||||||||
| 384 | IntermediateVT.getScalarType(), | ||||||||
| 385 | NumIntermediates); | ||||||||
| 386 | Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS | ||||||||
| 387 | : ISD::BUILD_VECTOR, | ||||||||
| 388 | DL, BuiltVectorTy, Ops); | ||||||||
| 389 | } | ||||||||
| 390 | |||||||||
| 391 | // There is now one part, held in Val. Correct it to match ValueVT. | ||||||||
| 392 | EVT PartEVT = Val.getValueType(); | ||||||||
| 393 | |||||||||
| 394 | if (PartEVT == ValueVT) | ||||||||
| 395 | return Val; | ||||||||
| 396 | |||||||||
| 397 | if (PartEVT.isVector()) { | ||||||||
| 398 | // Vector/Vector bitcast. | ||||||||
| 399 | if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) | ||||||||
| 400 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||
| 401 | |||||||||
| 402 | // If the parts vector has more elements than the value vector, then we | ||||||||
| 403 | // have a vector widening case (e.g. <2 x float> -> <4 x float>). | ||||||||
| 404 | // Extract the elements we want. | ||||||||
| 405 | if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) { | ||||||||
| 406 | assert((PartEVT.getVectorElementCount().getKnownMinValue() >(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 407 | ValueVT.getVectorElementCount().getKnownMinValue()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 408 | (PartEVT.getVectorElementCount().isScalable() ==(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 409 | ValueVT.getVectorElementCount().isScalable()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 410 | "Cannot narrow, it would be a lossy transformation")(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue () > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount ().isScalable()) && "Cannot narrow, it would be a lossy transformation" ) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 411 | PartEVT = | ||||||||
| 412 | EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(), | ||||||||
| 413 | ValueVT.getVectorElementCount()); | ||||||||
| 414 | Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val, | ||||||||
| 415 | DAG.getVectorIdxConstant(0, DL)); | ||||||||
| 416 | if (PartEVT == ValueVT) | ||||||||
| 417 | return Val; | ||||||||
| 418 | if (PartEVT.isInteger() && ValueVT.isFloatingPoint()) | ||||||||
| 419 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||
| 420 | } | ||||||||
| 421 | |||||||||
| 422 | // Promoted vector extract | ||||||||
| 423 | return DAG.getAnyExtOrTrunc(Val, DL, ValueVT); | ||||||||
| 424 | } | ||||||||
| 425 | |||||||||
| 426 | // Trivial bitcast if the types are the same size and the destination | ||||||||
| 427 | // vector type is legal. | ||||||||
| 428 | if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && | ||||||||
| 429 | TLI.isTypeLegal(ValueVT)) | ||||||||
| 430 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||
| 431 | |||||||||
| 432 | if (ValueVT.getVectorNumElements() != 1) { | ||||||||
| 433 | // Certain ABIs require that vectors are passed as integers. For vectors | ||||||||
| 434 | // are the same size, this is an obvious bitcast. | ||||||||
| 435 | if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) { | ||||||||
| 436 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||
| 437 | } else if (ValueVT.bitsLT(PartEVT)) { | ||||||||
| 438 | const uint64_t ValueSize = ValueVT.getFixedSizeInBits(); | ||||||||
| 439 | EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); | ||||||||
| 440 | // Drop the extra bits. | ||||||||
| 441 | Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); | ||||||||
| 442 | return DAG.getBitcast(ValueVT, Val); | ||||||||
| 443 | } | ||||||||
| 444 | |||||||||
| 445 | diagnosePossiblyInvalidConstraint( | ||||||||
| 446 | *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); | ||||||||
| 447 | return DAG.getUNDEF(ValueVT); | ||||||||
| 448 | } | ||||||||
| 449 | |||||||||
| 450 | // Handle cases such as i8 -> <1 x i1> | ||||||||
| 451 | EVT ValueSVT = ValueVT.getVectorElementType(); | ||||||||
| 452 | if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) { | ||||||||
| 453 | unsigned ValueSize = ValueSVT.getSizeInBits(); | ||||||||
| 454 | if (ValueSize == PartEVT.getSizeInBits()) { | ||||||||
| 455 | Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val); | ||||||||
| 456 | } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) { | ||||||||
| 457 | // It's possible a scalar floating point type gets softened to integer and | ||||||||
| 458 | // then promoted to a larger integer. If PartEVT is the larger integer | ||||||||
| 459 | // we need to truncate it and then bitcast to the FP type. | ||||||||
| 460 | assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types")(static_cast <bool> (ValueSVT.bitsLT(PartEVT) && "Unexpected types") ? void (0) : __assert_fail ("ValueSVT.bitsLT(PartEVT) && \"Unexpected types\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 460 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 461 | EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); | ||||||||
| 462 | Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); | ||||||||
| 463 | Val = DAG.getBitcast(ValueSVT, Val); | ||||||||
| 464 | } else { | ||||||||
| 465 | Val = ValueVT.isFloatingPoint() | ||||||||
| 466 | ? DAG.getFPExtendOrRound(Val, DL, ValueSVT) | ||||||||
| 467 | : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT); | ||||||||
| 468 | } | ||||||||
| 469 | } | ||||||||
| 470 | |||||||||
| 471 | return DAG.getBuildVector(ValueVT, DL, Val); | ||||||||
| 472 | } | ||||||||
| 473 | |||||||||
| 474 | static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, | ||||||||
| 475 | SDValue Val, SDValue *Parts, unsigned NumParts, | ||||||||
| 476 | MVT PartVT, const Value *V, | ||||||||
| 477 | std::optional<CallingConv::ID> CallConv); | ||||||||
| 478 | |||||||||
| 479 | /// getCopyToParts - Create a series of nodes that contain the specified value | ||||||||
| 480 | /// split into legal parts. If the parts contain more bits than Val, then, for | ||||||||
| 481 | /// integers, ExtendKind can be used to specify how to generate the extra bits. | ||||||||
| 482 | static void | ||||||||
| 483 | getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, | ||||||||
| 484 | unsigned NumParts, MVT PartVT, const Value *V, | ||||||||
| 485 | std::optional<CallingConv::ID> CallConv = std::nullopt, | ||||||||
| 486 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { | ||||||||
| 487 | // Let the target split the parts if it wants to | ||||||||
| 488 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 489 | if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT, | ||||||||
| 490 | CallConv)) | ||||||||
| 491 | return; | ||||||||
| 492 | EVT ValueVT = Val.getValueType(); | ||||||||
| 493 | |||||||||
| 494 | // Handle the vector case separately. | ||||||||
| 495 | if (ValueVT.isVector()) | ||||||||
| 496 | return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V, | ||||||||
| 497 | CallConv); | ||||||||
| 498 | |||||||||
| 499 | unsigned OrigNumParts = NumParts; | ||||||||
| 500 | assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal (PartVT) && "Copying to an illegal type!") ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 501 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 501 | "Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal (PartVT) && "Copying to an illegal type!") ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 501 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 502 | |||||||||
| 503 | if (NumParts == 0) | ||||||||
| 504 | return; | ||||||||
| 505 | |||||||||
| 506 | assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere" ) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 506 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 507 | EVT PartEVT = PartVT; | ||||||||
| 508 | if (PartEVT == ValueVT) { | ||||||||
| 509 | assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!" ) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 509 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 510 | Parts[0] = Val; | ||||||||
| 511 | return; | ||||||||
| 512 | } | ||||||||
| 513 | |||||||||
| 514 | unsigned PartBits = PartVT.getSizeInBits(); | ||||||||
| 515 | if (NumParts * PartBits > ValueVT.getSizeInBits()) { | ||||||||
| 516 | // If the parts cover more bits than the value has, promote the value. | ||||||||
| 517 | if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { | ||||||||
| 518 | assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!" ) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 518 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 519 | Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); | ||||||||
| 520 | } else { | ||||||||
| 521 | if (ValueVT.isFloatingPoint()) { | ||||||||
| 522 | // FP values need to be bitcast, then extended if they are being put | ||||||||
| 523 | // into a larger container. | ||||||||
| 524 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); | ||||||||
| 525 | Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); | ||||||||
| 526 | } | ||||||||
| 527 | assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 528 | ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 529 | "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 530 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||
| 531 | Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); | ||||||||
| 532 | if (PartVT == MVT::x86mmx) | ||||||||
| 533 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||
| 534 | } | ||||||||
| 535 | } else if (PartBits == ValueVT.getSizeInBits()) { | ||||||||
| 536 | // Different types of the same size. | ||||||||
| 537 | assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT != ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 537 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 538 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||
| 539 | } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { | ||||||||
| 540 | // If the parts cover less bits than value has, truncate the value. | ||||||||
| 541 | assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 542 | ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 543 | "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT ::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!" ) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 544 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||
| 545 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||
| 546 | if (PartVT == MVT::x86mmx) | ||||||||
| 547 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||
| 548 | } | ||||||||
| 549 | |||||||||
| 550 | // The value may have changed - recompute ValueVT. | ||||||||
| 551 | ValueVT = Val.getValueType(); | ||||||||
| 552 | assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits () && "Failed to tile the value with PartVT!") ? void (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 553 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 553 | "Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits () && "Failed to tile the value with PartVT!") ? void (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 553 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 554 | |||||||||
| 555 | if (NumParts == 1) { | ||||||||
| 556 | if (PartEVT != ValueVT) { | ||||||||
| 557 | diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, | ||||||||
| 558 | "scalar-to-vector conversion failed"); | ||||||||
| 559 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||
| 560 | } | ||||||||
| 561 | |||||||||
| 562 | Parts[0] = Val; | ||||||||
| 563 | return; | ||||||||
| 564 | } | ||||||||
| 565 | |||||||||
| 566 | // Expand the value into multiple parts. | ||||||||
| 567 | if (NumParts & (NumParts - 1)) { | ||||||||
| 568 | // The number of parts is not a power of 2. Split off and copy the tail. | ||||||||
| 569 | assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT .isInteger() && "Do not know what to expand to!") ? void (0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 570 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 570 | "Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT .isInteger() && "Do not know what to expand to!") ? void (0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 570 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 571 | unsigned RoundParts = llvm::bit_floor(NumParts); | ||||||||
| 572 | unsigned RoundBits = RoundParts * PartBits; | ||||||||
| 573 | unsigned OddParts = NumParts - RoundParts; | ||||||||
| 574 | SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, | ||||||||
| 575 | DAG.getShiftAmountConstant(RoundBits, ValueVT, DL)); | ||||||||
| 576 | |||||||||
| 577 | getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V, | ||||||||
| 578 | CallConv); | ||||||||
| 579 | |||||||||
| 580 | if (DAG.getDataLayout().isBigEndian()) | ||||||||
| 581 | // The odd parts were reversed by getCopyToParts - unreverse them. | ||||||||
| 582 | std::reverse(Parts + RoundParts, Parts + NumParts); | ||||||||
| 583 | |||||||||
| 584 | NumParts = RoundParts; | ||||||||
| 585 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); | ||||||||
| 586 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); | ||||||||
| 587 | } | ||||||||
| 588 | |||||||||
| 589 | // The number of parts is a power of 2. Repeatedly bisect the value using | ||||||||
| 590 | // EXTRACT_ELEMENT. | ||||||||
| 591 | Parts[0] = DAG.getNode(ISD::BITCAST, DL, | ||||||||
| 592 | EVT::getIntegerVT(*DAG.getContext(), | ||||||||
| 593 | ValueVT.getSizeInBits()), | ||||||||
| 594 | Val); | ||||||||
| 595 | |||||||||
| 596 | for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { | ||||||||
| 597 | for (unsigned i = 0; i < NumParts; i += StepSize) { | ||||||||
| 598 | unsigned ThisBits = StepSize * PartBits / 2; | ||||||||
| 599 | EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); | ||||||||
| 600 | SDValue &Part0 = Parts[i]; | ||||||||
| 601 | SDValue &Part1 = Parts[i+StepSize/2]; | ||||||||
| 602 | |||||||||
| 603 | Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, | ||||||||
| 604 | ThisVT, Part0, DAG.getIntPtrConstant(1, DL)); | ||||||||
| 605 | Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, | ||||||||
| 606 | ThisVT, Part0, DAG.getIntPtrConstant(0, DL)); | ||||||||
| 607 | |||||||||
| 608 | if (ThisBits == PartBits && ThisVT != PartVT) { | ||||||||
| 609 | Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); | ||||||||
| 610 | Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); | ||||||||
| 611 | } | ||||||||
| 612 | } | ||||||||
| 613 | } | ||||||||
| 614 | |||||||||
| 615 | if (DAG.getDataLayout().isBigEndian()) | ||||||||
| 616 | std::reverse(Parts, Parts + OrigNumParts); | ||||||||
| 617 | } | ||||||||
| 618 | |||||||||
| 619 | static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, | ||||||||
| 620 | const SDLoc &DL, EVT PartVT) { | ||||||||
| 621 | if (!PartVT.isVector()) | ||||||||
| 622 | return SDValue(); | ||||||||
| 623 | |||||||||
| 624 | EVT ValueVT = Val.getValueType(); | ||||||||
| 625 | ElementCount PartNumElts = PartVT.getVectorElementCount(); | ||||||||
| 626 | ElementCount ValueNumElts = ValueVT.getVectorElementCount(); | ||||||||
| 627 | |||||||||
| 628 | // We only support widening vectors with equivalent element types and | ||||||||
| 629 | // fixed/scalable properties. If a target needs to widen a fixed-length type | ||||||||
| 630 | // to a scalable one, it should be possible to use INSERT_SUBVECTOR below. | ||||||||
| 631 | if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) || | ||||||||
| 632 | PartNumElts.isScalable() != ValueNumElts.isScalable() || | ||||||||
| 633 | PartVT.getVectorElementType() != ValueVT.getVectorElementType()) | ||||||||
| 634 | return SDValue(); | ||||||||
| 635 | |||||||||
| 636 | // Widening a scalable vector to another scalable vector is done by inserting | ||||||||
| 637 | // the vector into a larger undef one. | ||||||||
| 638 | if (PartNumElts.isScalable()) | ||||||||
| 639 | return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), | ||||||||
| 640 | Val, DAG.getVectorIdxConstant(0, DL)); | ||||||||
| 641 | |||||||||
| 642 | EVT ElementVT = PartVT.getVectorElementType(); | ||||||||
| 643 | // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in | ||||||||
| 644 | // undef elements. | ||||||||
| 645 | SmallVector<SDValue, 16> Ops; | ||||||||
| 646 | DAG.ExtractVectorElements(Val, Ops); | ||||||||
| 647 | SDValue EltUndef = DAG.getUNDEF(ElementVT); | ||||||||
| 648 | Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef); | ||||||||
| 649 | |||||||||
| 650 | // FIXME: Use CONCAT for 2x -> 4x. | ||||||||
| 651 | return DAG.getBuildVector(PartVT, DL, Ops); | ||||||||
| 652 | } | ||||||||
| 653 | |||||||||
| 654 | /// getCopyToPartsVector - Create a series of nodes that contain the specified | ||||||||
| 655 | /// value split into legal parts. | ||||||||
| 656 | static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL, | ||||||||
| 657 | SDValue Val, SDValue *Parts, unsigned NumParts, | ||||||||
| 658 | MVT PartVT, const Value *V, | ||||||||
| 659 | std::optional<CallingConv::ID> CallConv) { | ||||||||
| 660 | EVT ValueVT = Val.getValueType(); | ||||||||
| 661 | assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector" ) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 661 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 662 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 663 | const bool IsABIRegCopy = CallConv.has_value(); | ||||||||
| 664 | |||||||||
| 665 | if (NumParts == 1) { | ||||||||
| 666 | EVT PartEVT = PartVT; | ||||||||
| 667 | if (PartEVT == ValueVT) { | ||||||||
| 668 | // Nothing to do. | ||||||||
| 669 | } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { | ||||||||
| 670 | // Bitconvert vector->vector case. | ||||||||
| 671 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); | ||||||||
| 672 | } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) { | ||||||||
| 673 | Val = Widened; | ||||||||
| 674 | } else if (PartVT.isVector() && | ||||||||
| 675 | PartEVT.getVectorElementType().bitsGE( | ||||||||
| 676 | ValueVT.getVectorElementType()) && | ||||||||
| 677 | PartEVT.getVectorElementCount() == | ||||||||
| 678 | ValueVT.getVectorElementCount()) { | ||||||||
| 679 | |||||||||
| 680 | // Promoted vector extract | ||||||||
| 681 | Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); | ||||||||
| 682 | } else if (PartEVT.isVector() && | ||||||||
| 683 | PartEVT.getVectorElementType() != | ||||||||
| 684 | ValueVT.getVectorElementType() && | ||||||||
| 685 | TLI.getTypeAction(*DAG.getContext(), ValueVT) == | ||||||||
| 686 | TargetLowering::TypeWidenVector) { | ||||||||
| 687 | // Combination of widening and promotion. | ||||||||
| 688 | EVT WidenVT = | ||||||||
| 689 | EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(), | ||||||||
| 690 | PartVT.getVectorElementCount()); | ||||||||
| 691 | SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT); | ||||||||
| 692 | Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT); | ||||||||
| 693 | } else { | ||||||||
| 694 | // Don't extract an integer from a float vector. This can happen if the | ||||||||
| 695 | // FP type gets softened to integer and then promoted. The promotion | ||||||||
| 696 | // prevents it from being picked up by the earlier bitcast case. | ||||||||
| 697 | if (ValueVT.getVectorElementCount().isScalar() && | ||||||||
| 698 | (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) { | ||||||||
| 699 | Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val, | ||||||||
| 700 | DAG.getVectorIdxConstant(0, DL)); | ||||||||
| 701 | } else { | ||||||||
| 702 | uint64_t ValueSize = ValueVT.getFixedSizeInBits(); | ||||||||
| 703 | assert(PartVT.getFixedSizeInBits() > ValueSize &&(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize && "lossy conversion of vector to scalar type") ? void (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 704 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 704 | "lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize && "lossy conversion of vector to scalar type") ? void (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 704 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 705 | EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); | ||||||||
| 706 | Val = DAG.getBitcast(IntermediateType, Val); | ||||||||
| 707 | Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); | ||||||||
| 708 | } | ||||||||
| 709 | } | ||||||||
| 710 | |||||||||
| 711 | assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT && "Unexpected vector part value type") ? void (0) : __assert_fail ("Val.getValueType() == PartVT && \"Unexpected vector part value type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 711 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 712 | Parts[0] = Val; | ||||||||
| 713 | return; | ||||||||
| 714 | } | ||||||||
| 715 | |||||||||
| 716 | // Handle a multi-element vector. | ||||||||
| 717 | EVT IntermediateVT; | ||||||||
| 718 | MVT RegisterVT; | ||||||||
| 719 | unsigned NumIntermediates; | ||||||||
| 720 | unsigned NumRegs; | ||||||||
| 721 | if (IsABIRegCopy) { | ||||||||
| 722 | NumRegs = TLI.getVectorTypeBreakdownForCallingConv( | ||||||||
| 723 | *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates, | ||||||||
| 724 | RegisterVT); | ||||||||
| 725 | } else { | ||||||||
| 726 | NumRegs = | ||||||||
| 727 | TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, | ||||||||
| 728 | NumIntermediates, RegisterVT); | ||||||||
| 729 | } | ||||||||
| 730 | |||||||||
| 731 | assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 731 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 732 | NumParts = NumRegs; // Silence a compiler warning. | ||||||||
| 733 | assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!" ) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 733 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 734 | |||||||||
| 735 | assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&(static_cast <bool> (IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts" ) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 736 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 736 | "Mixing scalable and fixed vectors when copying in parts")(static_cast <bool> (IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts" ) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 736 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 737 | |||||||||
| 738 | std::optional<ElementCount> DestEltCnt; | ||||||||
| 739 | |||||||||
| 740 | if (IntermediateVT.isVector()) | ||||||||
| 741 | DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates; | ||||||||
| 742 | else | ||||||||
| 743 | DestEltCnt = ElementCount::getFixed(NumIntermediates); | ||||||||
| 744 | |||||||||
| 745 | EVT BuiltVectorTy = EVT::getVectorVT( | ||||||||
| 746 | *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt); | ||||||||
| 747 | |||||||||
| 748 | if (ValueVT == BuiltVectorTy) { | ||||||||
| 749 | // Nothing to do. | ||||||||
| 750 | } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) { | ||||||||
| 751 | // Bitconvert vector->vector case. | ||||||||
| 752 | Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val); | ||||||||
| 753 | } else { | ||||||||
| 754 | if (BuiltVectorTy.getVectorElementType().bitsGT( | ||||||||
| 755 | ValueVT.getVectorElementType())) { | ||||||||
| 756 | // Integer promotion. | ||||||||
| 757 | ValueVT = EVT::getVectorVT(*DAG.getContext(), | ||||||||
| 758 | BuiltVectorTy.getVectorElementType(), | ||||||||
| 759 | ValueVT.getVectorElementCount()); | ||||||||
| 760 | Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); | ||||||||
| 761 | } | ||||||||
| 762 | |||||||||
| 763 | if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) { | ||||||||
| 764 | Val = Widened; | ||||||||
| 765 | } | ||||||||
| 766 | } | ||||||||
| 767 | |||||||||
| 768 | assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")(static_cast <bool> (Val.getValueType() == BuiltVectorTy && "Unexpected vector value type") ? void (0) : __assert_fail ("Val.getValueType() == BuiltVectorTy && \"Unexpected vector value type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 768 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 769 | |||||||||
| 770 | // Split the vector into intermediate operands. | ||||||||
| 771 | SmallVector<SDValue, 8> Ops(NumIntermediates); | ||||||||
| 772 | for (unsigned i = 0; i != NumIntermediates; ++i) { | ||||||||
| 773 | if (IntermediateVT.isVector()) { | ||||||||
| 774 | // This does something sensible for scalable vectors - see the | ||||||||
| 775 | // definition of EXTRACT_SUBVECTOR for further details. | ||||||||
| 776 | unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements(); | ||||||||
| 777 | Ops[i] = | ||||||||
| 778 | DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val, | ||||||||
| 779 | DAG.getVectorIdxConstant(i * IntermediateNumElts, DL)); | ||||||||
| 780 | } else { | ||||||||
| 781 | Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val, | ||||||||
| 782 | DAG.getVectorIdxConstant(i, DL)); | ||||||||
| 783 | } | ||||||||
| 784 | } | ||||||||
| 785 | |||||||||
| 786 | // Split the intermediate operands into legal parts. | ||||||||
| 787 | if (NumParts == NumIntermediates) { | ||||||||
| 788 | // If the register was not expanded, promote or copy the value, | ||||||||
| 789 | // as appropriate. | ||||||||
| 790 | for (unsigned i = 0; i != NumParts; ++i) | ||||||||
| 791 | getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv); | ||||||||
| 792 | } else if (NumParts > 0) { | ||||||||
| 793 | // If the intermediate type was expanded, split each the value into | ||||||||
| 794 | // legal parts. | ||||||||
| 795 | assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero" ) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 795 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 796 | assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 797 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 797 | "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!") ? void (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 797 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 798 | unsigned Factor = NumParts / NumIntermediates; | ||||||||
| 799 | for (unsigned i = 0; i != NumIntermediates; ++i) | ||||||||
| 800 | getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V, | ||||||||
| 801 | CallConv); | ||||||||
| 802 | } | ||||||||
| 803 | } | ||||||||
| 804 | |||||||||
| 805 | RegsForValue::RegsForValue(const SmallVector<unsigned, 4> ®s, MVT regvt, | ||||||||
| 806 | EVT valuevt, std::optional<CallingConv::ID> CC) | ||||||||
| 807 | : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs), | ||||||||
| 808 | RegCount(1, regs.size()), CallConv(CC) {} | ||||||||
| 809 | |||||||||
| 810 | RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI, | ||||||||
| 811 | const DataLayout &DL, unsigned Reg, Type *Ty, | ||||||||
| 812 | std::optional<CallingConv::ID> CC) { | ||||||||
| 813 | ComputeValueVTs(TLI, DL, Ty, ValueVTs); | ||||||||
| 814 | |||||||||
| 815 | CallConv = CC; | ||||||||
| 816 | |||||||||
| 817 | for (EVT ValueVT : ValueVTs) { | ||||||||
| 818 | unsigned NumRegs = | ||||||||
| 819 | isABIMangled() | ||||||||
| 820 | ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT) | ||||||||
| 821 | : TLI.getNumRegisters(Context, ValueVT); | ||||||||
| 822 | MVT RegisterVT = | ||||||||
| 823 | isABIMangled() | ||||||||
| 824 | ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT) | ||||||||
| 825 | : TLI.getRegisterType(Context, ValueVT); | ||||||||
| 826 | for (unsigned i = 0; i != NumRegs; ++i) | ||||||||
| 827 | Regs.push_back(Reg + i); | ||||||||
| 828 | RegVTs.push_back(RegisterVT); | ||||||||
| 829 | RegCount.push_back(NumRegs); | ||||||||
| 830 | Reg += NumRegs; | ||||||||
| 831 | } | ||||||||
| 832 | } | ||||||||
| 833 | |||||||||
| 834 | SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, | ||||||||
| 835 | FunctionLoweringInfo &FuncInfo, | ||||||||
| 836 | const SDLoc &dl, SDValue &Chain, | ||||||||
| 837 | SDValue *Glue, const Value *V) const { | ||||||||
| 838 | // A Value with type {} or [0 x %t] needs no registers. | ||||||||
| 839 | if (ValueVTs.empty()) | ||||||||
| 840 | return SDValue(); | ||||||||
| 841 | |||||||||
| 842 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 843 | |||||||||
| 844 | // Assemble the legal parts into the final values. | ||||||||
| 845 | SmallVector<SDValue, 4> Values(ValueVTs.size()); | ||||||||
| 846 | SmallVector<SDValue, 8> Parts; | ||||||||
| 847 | for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { | ||||||||
| 848 | // Copy the legal parts from the registers. | ||||||||
| 849 | EVT ValueVT = ValueVTs[Value]; | ||||||||
| 850 | unsigned NumRegs = RegCount[Value]; | ||||||||
| 851 | MVT RegisterVT = isABIMangled() | ||||||||
| 852 | ? TLI.getRegisterTypeForCallingConv( | ||||||||
| 853 | *DAG.getContext(), *CallConv, RegVTs[Value]) | ||||||||
| 854 | : RegVTs[Value]; | ||||||||
| 855 | |||||||||
| 856 | Parts.resize(NumRegs); | ||||||||
| 857 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||
| 858 | SDValue P; | ||||||||
| 859 | if (!Glue) { | ||||||||
| 860 | P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); | ||||||||
| 861 | } else { | ||||||||
| 862 | P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue); | ||||||||
| 863 | *Glue = P.getValue(2); | ||||||||
| 864 | } | ||||||||
| 865 | |||||||||
| 866 | Chain = P.getValue(1); | ||||||||
| 867 | Parts[i] = P; | ||||||||
| 868 | |||||||||
| 869 | // If the source register was virtual and if we know something about it, | ||||||||
| 870 | // add an assert node. | ||||||||
| 871 | if (!Register::isVirtualRegister(Regs[Part + i]) || | ||||||||
| 872 | !RegisterVT.isInteger()) | ||||||||
| 873 | continue; | ||||||||
| 874 | |||||||||
| 875 | const FunctionLoweringInfo::LiveOutInfo *LOI = | ||||||||
| 876 | FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); | ||||||||
| 877 | if (!LOI) | ||||||||
| 878 | continue; | ||||||||
| 879 | |||||||||
| 880 | unsigned RegSize = RegisterVT.getScalarSizeInBits(); | ||||||||
| 881 | unsigned NumSignBits = LOI->NumSignBits; | ||||||||
| 882 | unsigned NumZeroBits = LOI->Known.countMinLeadingZeros(); | ||||||||
| 883 | |||||||||
| 884 | if (NumZeroBits == RegSize) { | ||||||||
| 885 | // The current value is a zero. | ||||||||
| 886 | // Explicitly express that as it would be easier for | ||||||||
| 887 | // optimizations to kick in. | ||||||||
| 888 | Parts[i] = DAG.getConstant(0, dl, RegisterVT); | ||||||||
| 889 | continue; | ||||||||
| 890 | } | ||||||||
| 891 | |||||||||
| 892 | // FIXME: We capture more information than the dag can represent. For | ||||||||
| 893 | // now, just use the tightest assertzext/assertsext possible. | ||||||||
| 894 | bool isSExt; | ||||||||
| 895 | EVT FromVT(MVT::Other); | ||||||||
| 896 | if (NumZeroBits) { | ||||||||
| 897 | FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits); | ||||||||
| 898 | isSExt = false; | ||||||||
| 899 | } else if (NumSignBits > 1) { | ||||||||
| 900 | FromVT = | ||||||||
| 901 | EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1); | ||||||||
| 902 | isSExt = true; | ||||||||
| 903 | } else { | ||||||||
| 904 | continue; | ||||||||
| 905 | } | ||||||||
| 906 | // Add an assertion node. | ||||||||
| 907 | assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) : __assert_fail ("FromVT != MVT::Other", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 907, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 908 | Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, | ||||||||
| 909 | RegisterVT, P, DAG.getValueType(FromVT)); | ||||||||
| 910 | } | ||||||||
| 911 | |||||||||
| 912 | Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs, | ||||||||
| 913 | RegisterVT, ValueVT, V, CallConv); | ||||||||
| 914 | Part += NumRegs; | ||||||||
| 915 | Parts.clear(); | ||||||||
| 916 | } | ||||||||
| 917 | |||||||||
| 918 | return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values); | ||||||||
| 919 | } | ||||||||
| 920 | |||||||||
| 921 | void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, | ||||||||
| 922 | const SDLoc &dl, SDValue &Chain, SDValue *Glue, | ||||||||
| 923 | const Value *V, | ||||||||
| 924 | ISD::NodeType PreferredExtendType) const { | ||||||||
| 925 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 926 | ISD::NodeType ExtendKind = PreferredExtendType; | ||||||||
| 927 | |||||||||
| 928 | // Get the list of the values's legal parts. | ||||||||
| 929 | unsigned NumRegs = Regs.size(); | ||||||||
| 930 | SmallVector<SDValue, 8> Parts(NumRegs); | ||||||||
| 931 | for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { | ||||||||
| 932 | unsigned NumParts = RegCount[Value]; | ||||||||
| 933 | |||||||||
| 934 | MVT RegisterVT = isABIMangled() | ||||||||
| 935 | ? TLI.getRegisterTypeForCallingConv( | ||||||||
| 936 | *DAG.getContext(), *CallConv, RegVTs[Value]) | ||||||||
| 937 | : RegVTs[Value]; | ||||||||
| 938 | |||||||||
| 939 | if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT)) | ||||||||
| 940 | ExtendKind = ISD::ZERO_EXTEND; | ||||||||
| 941 | |||||||||
| 942 | getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part], | ||||||||
| 943 | NumParts, RegisterVT, V, CallConv, ExtendKind); | ||||||||
| 944 | Part += NumParts; | ||||||||
| 945 | } | ||||||||
| 946 | |||||||||
| 947 | // Copy the parts into the registers. | ||||||||
| 948 | SmallVector<SDValue, 8> Chains(NumRegs); | ||||||||
| 949 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||
| 950 | SDValue Part; | ||||||||
| 951 | if (!Glue) { | ||||||||
| 952 | Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); | ||||||||
| 953 | } else { | ||||||||
| 954 | Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue); | ||||||||
| 955 | *Glue = Part.getValue(1); | ||||||||
| 956 | } | ||||||||
| 957 | |||||||||
| 958 | Chains[i] = Part.getValue(0); | ||||||||
| 959 | } | ||||||||
| 960 | |||||||||
| 961 | if (NumRegs == 1 || Glue) | ||||||||
| 962 | // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is | ||||||||
| 963 | // flagged to it. That is the CopyToReg nodes and the user are considered | ||||||||
| 964 | // a single scheduling unit. If we create a TokenFactor and return it as | ||||||||
| 965 | // chain, then the TokenFactor is both a predecessor (operand) of the | ||||||||
| 966 | // user as well as a successor (the TF operands are flagged to the user). | ||||||||
| 967 | // c1, f1 = CopyToReg | ||||||||
| 968 | // c2, f2 = CopyToReg | ||||||||
| 969 | // c3 = TokenFactor c1, c2 | ||||||||
| 970 | // ... | ||||||||
| 971 | // = op c3, ..., f2 | ||||||||
| 972 | Chain = Chains[NumRegs-1]; | ||||||||
| 973 | else | ||||||||
| 974 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); | ||||||||
| 975 | } | ||||||||
| 976 | |||||||||
| 977 | void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, | ||||||||
| 978 | unsigned MatchingIdx, const SDLoc &dl, | ||||||||
| 979 | SelectionDAG &DAG, | ||||||||
| 980 | std::vector<SDValue> &Ops) const { | ||||||||
| 981 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 982 | |||||||||
| 983 | unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); | ||||||||
| 984 | if (HasMatching) | ||||||||
| 985 | Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); | ||||||||
| 986 | else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) { | ||||||||
| 987 | // Put the register class of the virtual registers in the flag word. That | ||||||||
| 988 | // way, later passes can recompute register class constraints for inline | ||||||||
| 989 | // assembly as well as normal instructions. | ||||||||
| 990 | // Don't do this for tied operands that can use the regclass information | ||||||||
| 991 | // from the def. | ||||||||
| 992 | const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); | ||||||||
| 993 | const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); | ||||||||
| 994 | Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); | ||||||||
| 995 | } | ||||||||
| 996 | |||||||||
| 997 | SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32); | ||||||||
| 998 | Ops.push_back(Res); | ||||||||
| 999 | |||||||||
| 1000 | if (Code == InlineAsm::Kind_Clobber) { | ||||||||
| 1001 | // Clobbers should always have a 1:1 mapping with registers, and may | ||||||||
| 1002 | // reference registers that have illegal (e.g. vector) types. Hence, we | ||||||||
| 1003 | // shouldn't try to apply any sort of splitting logic to them. | ||||||||
| 1004 | assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?" ) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1005 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1005 | "No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?" ) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1005 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1006 | Register SP = TLI.getStackPointerRegisterToSaveRestore(); | ||||||||
| 1007 | (void)SP; | ||||||||
| 1008 | for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) { | ||||||||
| 1009 | Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I])); | ||||||||
| 1010 | assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1011 | (Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1012 | DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1013 | "If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction ().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it." ) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1014 | } | ||||||||
| 1015 | return; | ||||||||
| 1016 | } | ||||||||
| 1017 | |||||||||
| 1018 | for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { | ||||||||
| 1019 | MVT RegisterVT = RegVTs[Value]; | ||||||||
| 1020 | unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], | ||||||||
| 1021 | RegisterVT); | ||||||||
| 1022 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||
| 1023 | assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected" ) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1023 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1024 | unsigned TheReg = Regs[Reg++]; | ||||||||
| 1025 | Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); | ||||||||
| 1026 | } | ||||||||
| 1027 | } | ||||||||
| 1028 | } | ||||||||
| 1029 | |||||||||
| 1030 | SmallVector<std::pair<unsigned, TypeSize>, 4> | ||||||||
| 1031 | RegsForValue::getRegsAndSizes() const { | ||||||||
| 1032 | SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec; | ||||||||
| 1033 | unsigned I = 0; | ||||||||
| 1034 | for (auto CountAndVT : zip_first(RegCount, RegVTs)) { | ||||||||
| 1035 | unsigned RegCount = std::get<0>(CountAndVT); | ||||||||
| 1036 | MVT RegisterVT = std::get<1>(CountAndVT); | ||||||||
| 1037 | TypeSize RegisterSize = RegisterVT.getSizeInBits(); | ||||||||
| 1038 | for (unsigned E = I + RegCount; I != E; ++I) | ||||||||
| 1039 | OutVec.push_back(std::make_pair(Regs[I], RegisterSize)); | ||||||||
| 1040 | } | ||||||||
| 1041 | return OutVec; | ||||||||
| 1042 | } | ||||||||
| 1043 | |||||||||
| 1044 | void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa, | ||||||||
| 1045 | AssumptionCache *ac, | ||||||||
| 1046 | const TargetLibraryInfo *li) { | ||||||||
| 1047 | AA = aa; | ||||||||
| 1048 | AC = ac; | ||||||||
| 1049 | GFI = gfi; | ||||||||
| 1050 | LibInfo = li; | ||||||||
| 1051 | Context = DAG.getContext(); | ||||||||
| 1052 | LPadToCallSiteMap.clear(); | ||||||||
| 1053 | SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout()); | ||||||||
| 1054 | AssignmentTrackingEnabled = isAssignmentTrackingEnabled( | ||||||||
| 1055 | *DAG.getMachineFunction().getFunction().getParent()); | ||||||||
| 1056 | } | ||||||||
| 1057 | |||||||||
| 1058 | void SelectionDAGBuilder::clear() { | ||||||||
| 1059 | NodeMap.clear(); | ||||||||
| 1060 | UnusedArgNodeMap.clear(); | ||||||||
| 1061 | PendingLoads.clear(); | ||||||||
| 1062 | PendingExports.clear(); | ||||||||
| 1063 | PendingConstrainedFP.clear(); | ||||||||
| 1064 | PendingConstrainedFPStrict.clear(); | ||||||||
| 1065 | CurInst = nullptr; | ||||||||
| 1066 | HasTailCall = false; | ||||||||
| 1067 | SDNodeOrder = LowestSDNodeOrder; | ||||||||
| 1068 | StatepointLowering.clear(); | ||||||||
| 1069 | } | ||||||||
| 1070 | |||||||||
| 1071 | void SelectionDAGBuilder::clearDanglingDebugInfo() { | ||||||||
| 1072 | DanglingDebugInfoMap.clear(); | ||||||||
| 1073 | } | ||||||||
| 1074 | |||||||||
| 1075 | // Update DAG root to include dependencies on Pending chains. | ||||||||
| 1076 | SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) { | ||||||||
| 1077 | SDValue Root = DAG.getRoot(); | ||||||||
| 1078 | |||||||||
| 1079 | if (Pending.empty()) | ||||||||
| 1080 | return Root; | ||||||||
| 1081 | |||||||||
| 1082 | // Add current root to PendingChains, unless we already indirectly | ||||||||
| 1083 | // depend on it. | ||||||||
| 1084 | if (Root.getOpcode() != ISD::EntryToken) { | ||||||||
| 1085 | unsigned i = 0, e = Pending.size(); | ||||||||
| 1086 | for (; i != e; ++i) { | ||||||||
| 1087 | assert(Pending[i].getNode()->getNumOperands() > 1)(static_cast <bool> (Pending[i].getNode()->getNumOperands () > 1) ? void (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1087 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1088 | if (Pending[i].getNode()->getOperand(0) == Root) | ||||||||
| 1089 | break; // Don't add the root if we already indirectly depend on it. | ||||||||
| 1090 | } | ||||||||
| 1091 | |||||||||
| 1092 | if (i == e) | ||||||||
| 1093 | Pending.push_back(Root); | ||||||||
| 1094 | } | ||||||||
| 1095 | |||||||||
| 1096 | if (Pending.size() == 1) | ||||||||
| 1097 | Root = Pending[0]; | ||||||||
| 1098 | else | ||||||||
| 1099 | Root = DAG.getTokenFactor(getCurSDLoc(), Pending); | ||||||||
| 1100 | |||||||||
| 1101 | DAG.setRoot(Root); | ||||||||
| 1102 | Pending.clear(); | ||||||||
| 1103 | return Root; | ||||||||
| 1104 | } | ||||||||
| 1105 | |||||||||
| 1106 | SDValue SelectionDAGBuilder::getMemoryRoot() { | ||||||||
| 1107 | return updateRoot(PendingLoads); | ||||||||
| 1108 | } | ||||||||
| 1109 | |||||||||
| 1110 | SDValue SelectionDAGBuilder::getRoot() { | ||||||||
| 1111 | // Chain up all pending constrained intrinsics together with all | ||||||||
| 1112 | // pending loads, by simply appending them to PendingLoads and | ||||||||
| 1113 | // then calling getMemoryRoot(). | ||||||||
| 1114 | PendingLoads.reserve(PendingLoads.size() + | ||||||||
| 1115 | PendingConstrainedFP.size() + | ||||||||
| 1116 | PendingConstrainedFPStrict.size()); | ||||||||
| 1117 | PendingLoads.append(PendingConstrainedFP.begin(), | ||||||||
| 1118 | PendingConstrainedFP.end()); | ||||||||
| 1119 | PendingLoads.append(PendingConstrainedFPStrict.begin(), | ||||||||
| 1120 | PendingConstrainedFPStrict.end()); | ||||||||
| 1121 | PendingConstrainedFP.clear(); | ||||||||
| 1122 | PendingConstrainedFPStrict.clear(); | ||||||||
| 1123 | return getMemoryRoot(); | ||||||||
| 1124 | } | ||||||||
| 1125 | |||||||||
| 1126 | SDValue SelectionDAGBuilder::getControlRoot() { | ||||||||
| 1127 | // We need to emit pending fpexcept.strict constrained intrinsics, | ||||||||
| 1128 | // so append them to the PendingExports list. | ||||||||
| 1129 | PendingExports.append(PendingConstrainedFPStrict.begin(), | ||||||||
| 1130 | PendingConstrainedFPStrict.end()); | ||||||||
| 1131 | PendingConstrainedFPStrict.clear(); | ||||||||
| 1132 | return updateRoot(PendingExports); | ||||||||
| 1133 | } | ||||||||
| 1134 | |||||||||
| 1135 | void SelectionDAGBuilder::visit(const Instruction &I) { | ||||||||
| 1136 | // Set up outgoing PHI node register values before emitting the terminator. | ||||||||
| 1137 | if (I.isTerminator()) { | ||||||||
| 1138 | HandlePHINodesInSuccessorBlocks(I.getParent()); | ||||||||
| 1139 | } | ||||||||
| 1140 | |||||||||
| 1141 | // Add SDDbgValue nodes for any var locs here. Do so before updating | ||||||||
| 1142 | // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}. | ||||||||
| 1143 | if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) { | ||||||||
| 1144 | // Add SDDbgValue nodes for any var locs here. Do so before updating | ||||||||
| 1145 | // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}. | ||||||||
| 1146 | for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I); | ||||||||
| 1147 | It != End; ++It) { | ||||||||
| 1148 | auto *Var = FnVarLocs->getDILocalVariable(It->VariableID); | ||||||||
| 1149 | dropDanglingDebugInfo(Var, It->Expr); | ||||||||
| 1150 | SmallVector<Value *> Values(It->Values.location_ops()); | ||||||||
| 1151 | if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder, | ||||||||
| 1152 | It->Values.hasArgList())) | ||||||||
| 1153 | addDanglingDebugInfo(It, SDNodeOrder); | ||||||||
| 1154 | } | ||||||||
| 1155 | } | ||||||||
| 1156 | |||||||||
| 1157 | // Increase the SDNodeOrder if dealing with a non-debug instruction. | ||||||||
| 1158 | if (!isa<DbgInfoIntrinsic>(I)) | ||||||||
| 1159 | ++SDNodeOrder; | ||||||||
| 1160 | |||||||||
| 1161 | CurInst = &I; | ||||||||
| 1162 | |||||||||
| 1163 | // Set inserted listener only if required. | ||||||||
| 1164 | bool NodeInserted = false; | ||||||||
| 1165 | std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener; | ||||||||
| 1166 | MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections); | ||||||||
| 1167 | if (PCSectionsMD) { | ||||||||
| 1168 | InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>( | ||||||||
| 1169 | DAG, [&](SDNode *) { NodeInserted = true; }); | ||||||||
| 1170 | } | ||||||||
| 1171 | |||||||||
| 1172 | visit(I.getOpcode(), I); | ||||||||
| 1173 | |||||||||
| 1174 | if (!I.isTerminator() && !HasTailCall && | ||||||||
| 1175 | !isa<GCStatepointInst>(I)) // statepoints handle their exports internally | ||||||||
| 1176 | CopyToExportRegsIfNeeded(&I); | ||||||||
| 1177 | |||||||||
| 1178 | // Handle metadata. | ||||||||
| 1179 | if (PCSectionsMD) { | ||||||||
| 1180 | auto It = NodeMap.find(&I); | ||||||||
| 1181 | if (It != NodeMap.end()) { | ||||||||
| 1182 | DAG.addPCSections(It->second.getNode(), PCSectionsMD); | ||||||||
| 1183 | } else if (NodeInserted) { | ||||||||
| 1184 | // This should not happen; if it does, don't let it go unnoticed so we can | ||||||||
| 1185 | // fix it. Relevant visit*() function is probably missing a setValue(). | ||||||||
| 1186 | errs() << "warning: loosing !pcsections metadata [" | ||||||||
| 1187 | << I.getModule()->getName() << "]\n"; | ||||||||
| 1188 | LLVM_DEBUG(I.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { I.dump(); } } while (false); | ||||||||
| 1189 | assert(false)(static_cast <bool> (false) ? void (0) : __assert_fail ( "false", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1189, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1190 | } | ||||||||
| 1191 | } | ||||||||
| 1192 | |||||||||
| 1193 | CurInst = nullptr; | ||||||||
| 1194 | } | ||||||||
| 1195 | |||||||||
| 1196 | void SelectionDAGBuilder::visitPHI(const PHINode &) { | ||||||||
| 1197 | llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1197 ); | ||||||||
| 1198 | } | ||||||||
| 1199 | |||||||||
| 1200 | void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { | ||||||||
| 1201 | // Note: this doesn't use InstVisitor, because it has to work with | ||||||||
| 1202 | // ConstantExpr's in addition to instructions. | ||||||||
| 1203 | switch (Opcode) { | ||||||||
| 1204 | default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1204 ); | ||||||||
| 1205 | // Build the switch statement using the Instruction.def file. | ||||||||
| 1206 | #define HANDLE_INST(NUM, OPCODE, CLASS) \ | ||||||||
| 1207 | case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; | ||||||||
| 1208 | #include "llvm/IR/Instruction.def" | ||||||||
| 1209 | } | ||||||||
| 1210 | } | ||||||||
| 1211 | |||||||||
| 1212 | static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, | ||||||||
| 1213 | DILocalVariable *Variable, | ||||||||
| 1214 | DebugLoc DL, unsigned Order, | ||||||||
| 1215 | RawLocationWrapper Values, | ||||||||
| 1216 | DIExpression *Expression) { | ||||||||
| 1217 | if (!Values.hasArgList()) | ||||||||
| 1218 | return false; | ||||||||
| 1219 | // For variadic dbg_values we will now insert an undef. | ||||||||
| 1220 | // FIXME: We can potentially recover these! | ||||||||
| 1221 | SmallVector<SDDbgOperand, 2> Locs; | ||||||||
| 1222 | for (const Value *V : Values.location_ops()) { | ||||||||
| 1223 | auto *Undef = UndefValue::get(V->getType()); | ||||||||
| 1224 | Locs.push_back(SDDbgOperand::fromConst(Undef)); | ||||||||
| 1225 | } | ||||||||
| 1226 | SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {}, | ||||||||
| 1227 | /*IsIndirect=*/false, DL, Order, | ||||||||
| 1228 | /*IsVariadic=*/true); | ||||||||
| 1229 | DAG.AddDbgValue(SDV, /*isParameter=*/false); | ||||||||
| 1230 | return true; | ||||||||
| 1231 | } | ||||||||
| 1232 | |||||||||
| 1233 | void SelectionDAGBuilder::addDanglingDebugInfo(const VarLocInfo *VarLoc, | ||||||||
| 1234 | unsigned Order) { | ||||||||
| 1235 | if (!handleDanglingVariadicDebugInfo( | ||||||||
| 1236 | DAG, | ||||||||
| 1237 | const_cast<DILocalVariable *>(DAG.getFunctionVarLocs() | ||||||||
| 1238 | ->getVariable(VarLoc->VariableID) | ||||||||
| 1239 | .getVariable()), | ||||||||
| 1240 | VarLoc->DL, Order, VarLoc->Values, VarLoc->Expr)) { | ||||||||
| 1241 | DanglingDebugInfoMap[VarLoc->Values.getVariableLocationOp(0)].emplace_back( | ||||||||
| 1242 | VarLoc, Order); | ||||||||
| 1243 | } | ||||||||
| 1244 | } | ||||||||
| 1245 | |||||||||
| 1246 | void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI, | ||||||||
| 1247 | unsigned Order) { | ||||||||
| 1248 | // We treat variadic dbg_values differently at this stage. | ||||||||
| 1249 | if (!handleDanglingVariadicDebugInfo( | ||||||||
| 1250 | DAG, DI->getVariable(), DI->getDebugLoc(), Order, | ||||||||
| 1251 | DI->getWrappedLocation(), DI->getExpression())) { | ||||||||
| 1252 | // TODO: Dangling debug info will eventually either be resolved or produce | ||||||||
| 1253 | // an Undef DBG_VALUE. However in the resolution case, a gap may appear | ||||||||
| 1254 | // between the original dbg.value location and its resolved DBG_VALUE, | ||||||||
| 1255 | // which we should ideally fill with an extra Undef DBG_VALUE. | ||||||||
| 1256 | assert(DI->getNumVariableLocationOps() == 1 &&(static_cast <bool> (DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location " "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1258 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1257 | "DbgValueInst without an ArgList should have a single location "(static_cast <bool> (DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location " "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1258 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1258 | "operand.")(static_cast <bool> (DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location " "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1258 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1259 | DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order); | ||||||||
| 1260 | } | ||||||||
| 1261 | } | ||||||||
| 1262 | |||||||||
| 1263 | void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable, | ||||||||
| 1264 | const DIExpression *Expr) { | ||||||||
| 1265 | auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) { | ||||||||
| 1266 | DIVariable *DanglingVariable = DDI.getVariable(DAG.getFunctionVarLocs()); | ||||||||
| 1267 | DIExpression *DanglingExpr = DDI.getExpression(); | ||||||||
| 1268 | if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) { | ||||||||
| 1269 | LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping dangling debug info for " << printDDI(DDI) << "\n"; } } while (false) | ||||||||
| 1270 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping dangling debug info for " << printDDI(DDI) << "\n"; } } while (false); | ||||||||
| 1271 | return true; | ||||||||
| 1272 | } | ||||||||
| 1273 | return false; | ||||||||
| 1274 | }; | ||||||||
| 1275 | |||||||||
| 1276 | for (auto &DDIMI : DanglingDebugInfoMap) { | ||||||||
| 1277 | DanglingDebugInfoVector &DDIV = DDIMI.second; | ||||||||
| 1278 | |||||||||
| 1279 | // If debug info is to be dropped, run it through final checks to see | ||||||||
| 1280 | // whether it can be salvaged. | ||||||||
| 1281 | for (auto &DDI : DDIV) | ||||||||
| 1282 | if (isMatchingDbgValue(DDI)) | ||||||||
| 1283 | salvageUnresolvedDbgValue(DDI); | ||||||||
| 1284 | |||||||||
| 1285 | erase_if(DDIV, isMatchingDbgValue); | ||||||||
| 1286 | } | ||||||||
| 1287 | } | ||||||||
| 1288 | |||||||||
| 1289 | // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, | ||||||||
| 1290 | // generate the debug data structures now that we've seen its definition. | ||||||||
| 1291 | void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, | ||||||||
| 1292 | SDValue Val) { | ||||||||
| 1293 | auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V); | ||||||||
| 1294 | if (DanglingDbgInfoIt == DanglingDebugInfoMap.end()) | ||||||||
| 1295 | return; | ||||||||
| 1296 | |||||||||
| 1297 | DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second; | ||||||||
| 1298 | for (auto &DDI : DDIV) { | ||||||||
| 1299 | DebugLoc DL = DDI.getDebugLoc(); | ||||||||
| 1300 | unsigned ValSDNodeOrder = Val.getNode()->getIROrder(); | ||||||||
| 1301 | unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); | ||||||||
| 1302 | DILocalVariable *Variable = DDI.getVariable(DAG.getFunctionVarLocs()); | ||||||||
| 1303 | DIExpression *Expr = DDI.getExpression(); | ||||||||
| 1304 | assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1305 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1305 | "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1305 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1306 | SDDbgValue *SDV; | ||||||||
| 1307 | if (Val.getNode()) { | ||||||||
| 1308 | // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a | ||||||||
| 1309 | // FuncArgumentDbgValue (it would be hoisted to the function entry, and if | ||||||||
| 1310 | // we couldn't resolve it directly when examining the DbgValue intrinsic | ||||||||
| 1311 | // in the first place we should not be more successful here). Unless we | ||||||||
| 1312 | // have some test case that prove this to be correct we should avoid | ||||||||
| 1313 | // calling EmitFuncArgumentDbgValue here. | ||||||||
| 1314 | if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL, | ||||||||
| 1315 | FuncArgumentDbgValueKind::Value, Val)) { | ||||||||
| 1316 | LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolve dangling debug info for " << printDDI(DDI) << "\n"; } } while (false) | ||||||||
| 1317 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolve dangling debug info for " << printDDI(DDI) << "\n"; } } while (false); | ||||||||
| 1318 | LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " By mapping to:\n "; Val.dump (); } } while (false); | ||||||||
| 1319 | // Increase the SDNodeOrder for the DbgValue here to make sure it is | ||||||||
| 1320 | // inserted after the definition of Val when emitting the instructions | ||||||||
| 1321 | // after ISel. An alternative could be to teach | ||||||||
| 1322 | // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly. | ||||||||
| 1323 | LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() << "changing SDNodeOrder from " << DbgSDNodeOrder << " to " << ValSDNodeOrder << "\n"; } } while (false ) | ||||||||
| 1324 | << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() << "changing SDNodeOrder from " << DbgSDNodeOrder << " to " << ValSDNodeOrder << "\n"; } } while (false ) | ||||||||
| 1325 | << ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() << "changing SDNodeOrder from " << DbgSDNodeOrder << " to " << ValSDNodeOrder << "\n"; } } while (false ); | ||||||||
| 1326 | SDV = getDbgValue(Val, Variable, Expr, DL, | ||||||||
| 1327 | std::max(DbgSDNodeOrder, ValSDNodeOrder)); | ||||||||
| 1328 | DAG.AddDbgValue(SDV, false); | ||||||||
| 1329 | } else | ||||||||
| 1330 | LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolved dangling debug info for " << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n" ; } } while (false) | ||||||||
| 1331 | << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Resolved dangling debug info for " << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n" ; } } while (false); | ||||||||
| 1332 | } else { | ||||||||
| 1333 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n"; } } while (false); | ||||||||
| 1334 | auto Undef = UndefValue::get(V->getType()); | ||||||||
| 1335 | auto SDV = | ||||||||
| 1336 | DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder); | ||||||||
| 1337 | DAG.AddDbgValue(SDV, false); | ||||||||
| 1338 | } | ||||||||
| 1339 | } | ||||||||
| 1340 | DDIV.clear(); | ||||||||
| 1341 | } | ||||||||
| 1342 | |||||||||
| 1343 | void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) { | ||||||||
| 1344 | // TODO: For the variadic implementation, instead of only checking the fail | ||||||||
| 1345 | // state of `handleDebugValue`, we need know specifically which values were | ||||||||
| 1346 | // invalid, so that we attempt to salvage only those values when processing | ||||||||
| 1347 | // a DIArgList. | ||||||||
| 1348 | Value *V = DDI.getVariableLocationOp(0); | ||||||||
| 1349 | Value *OrigV = V; | ||||||||
| 1350 | DILocalVariable *Var = DDI.getVariable(DAG.getFunctionVarLocs()); | ||||||||
| 1351 | DIExpression *Expr = DDI.getExpression(); | ||||||||
| 1352 | DebugLoc DL = DDI.getDebugLoc(); | ||||||||
| 1353 | unsigned SDOrder = DDI.getSDNodeOrder(); | ||||||||
| 1354 | |||||||||
| 1355 | // Currently we consider only dbg.value intrinsics -- we tell the salvager | ||||||||
| 1356 | // that DW_OP_stack_value is desired. | ||||||||
| 1357 | bool StackValue = true; | ||||||||
| 1358 | |||||||||
| 1359 | // Can this Value can be encoded without any further work? | ||||||||
| 1360 | if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) | ||||||||
| 1361 | return; | ||||||||
| 1362 | |||||||||
| 1363 | // Attempt to salvage back through as many instructions as possible. Bail if | ||||||||
| 1364 | // a non-instruction is seen, such as a constant expression or global | ||||||||
| 1365 | // variable. FIXME: Further work could recover those too. | ||||||||
| 1366 | while (isa<Instruction>(V)) { | ||||||||
| 1367 | Instruction &VAsInst = *cast<Instruction>(V); | ||||||||
| 1368 | // Temporary "0", awaiting real implementation. | ||||||||
| 1369 | SmallVector<uint64_t, 16> Ops; | ||||||||
| 1370 | SmallVector<Value *, 4> AdditionalValues; | ||||||||
| 1371 | V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops, | ||||||||
| 1372 | AdditionalValues); | ||||||||
| 1373 | // If we cannot salvage any further, and haven't yet found a suitable debug | ||||||||
| 1374 | // expression, bail out. | ||||||||
| 1375 | if (!V) | ||||||||
| 1376 | break; | ||||||||
| 1377 | |||||||||
| 1378 | // TODO: If AdditionalValues isn't empty, then the salvage can only be | ||||||||
| 1379 | // represented with a DBG_VALUE_LIST, so we give up. When we have support | ||||||||
| 1380 | // here for variadic dbg_values, remove that condition. | ||||||||
| 1381 | if (!AdditionalValues.empty()) | ||||||||
| 1382 | break; | ||||||||
| 1383 | |||||||||
| 1384 | // New value and expr now represent this debuginfo. | ||||||||
| 1385 | Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue); | ||||||||
| 1386 | |||||||||
| 1387 | // Some kind of simplification occurred: check whether the operand of the | ||||||||
| 1388 | // salvaged debug expression can be encoded in this DAG. | ||||||||
| 1389 | if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) { | ||||||||
| 1390 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Salvaged debug location info for:\n " << *Var << "\n" << *OrigV << "\nBy stripping back to:\n " << *V << "\n"; } } while (false) | ||||||||
| 1391 | dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Salvaged debug location info for:\n " << *Var << "\n" << *OrigV << "\nBy stripping back to:\n " << *V << "\n"; } } while (false) | ||||||||
| 1392 | << *OrigV << "\nBy stripping back to:\n " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Salvaged debug location info for:\n " << *Var << "\n" << *OrigV << "\nBy stripping back to:\n " << *V << "\n"; } } while (false); | ||||||||
| 1393 | return; | ||||||||
| 1394 | } | ||||||||
| 1395 | } | ||||||||
| 1396 | |||||||||
| 1397 | // This was the final opportunity to salvage this debug information, and it | ||||||||
| 1398 | // couldn't be done. Place an undef DBG_VALUE at this location to terminate | ||||||||
| 1399 | // any earlier variable location. | ||||||||
| 1400 | assert(OrigV && "V shouldn't be null")(static_cast <bool> (OrigV && "V shouldn't be null" ) ? void (0) : __assert_fail ("OrigV && \"V shouldn't be null\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1400 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1401 | auto *Undef = UndefValue::get(OrigV->getType()); | ||||||||
| 1402 | auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder); | ||||||||
| 1403 | DAG.AddDbgValue(SDV, false); | ||||||||
| 1404 | LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug value info for:\n " << printDDI(DDI) << "\n"; } } while (false) | ||||||||
| 1405 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug value info for:\n " << printDDI(DDI) << "\n"; } } while (false); | ||||||||
| 1406 | } | ||||||||
| 1407 | |||||||||
| 1408 | void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var, | ||||||||
| 1409 | DIExpression *Expr, | ||||||||
| 1410 | DebugLoc DbgLoc, | ||||||||
| 1411 | unsigned Order) { | ||||||||
| 1412 | Value *Poison = PoisonValue::get(Type::getInt1Ty(*Context)); | ||||||||
| 1413 | DIExpression *NewExpr = | ||||||||
| 1414 | const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr)); | ||||||||
| 1415 | handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order, | ||||||||
| 1416 | /*IsVariadic*/ false); | ||||||||
| 1417 | } | ||||||||
| 1418 | |||||||||
| 1419 | bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values, | ||||||||
| 1420 | DILocalVariable *Var, | ||||||||
| 1421 | DIExpression *Expr, DebugLoc DbgLoc, | ||||||||
| 1422 | unsigned Order, bool IsVariadic) { | ||||||||
| 1423 | if (Values.empty()) | ||||||||
| 1424 | return true; | ||||||||
| 1425 | SmallVector<SDDbgOperand> LocationOps; | ||||||||
| 1426 | SmallVector<SDNode *> Dependencies; | ||||||||
| 1427 | for (const Value *V : Values) { | ||||||||
| 1428 | // Constant value. | ||||||||
| 1429 | if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) || | ||||||||
| 1430 | isa<ConstantPointerNull>(V)) { | ||||||||
| 1431 | LocationOps.emplace_back(SDDbgOperand::fromConst(V)); | ||||||||
| 1432 | continue; | ||||||||
| 1433 | } | ||||||||
| 1434 | |||||||||
| 1435 | // Look through IntToPtr constants. | ||||||||
| 1436 | if (auto *CE = dyn_cast<ConstantExpr>(V)) | ||||||||
| 1437 | if (CE->getOpcode() == Instruction::IntToPtr) { | ||||||||
| 1438 | LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0))); | ||||||||
| 1439 | continue; | ||||||||
| 1440 | } | ||||||||
| 1441 | |||||||||
| 1442 | // If the Value is a frame index, we can create a FrameIndex debug value | ||||||||
| 1443 | // without relying on the DAG at all. | ||||||||
| 1444 | if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { | ||||||||
| 1445 | auto SI = FuncInfo.StaticAllocaMap.find(AI); | ||||||||
| 1446 | if (SI != FuncInfo.StaticAllocaMap.end()) { | ||||||||
| 1447 | LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second)); | ||||||||
| 1448 | continue; | ||||||||
| 1449 | } | ||||||||
| 1450 | } | ||||||||
| 1451 | |||||||||
| 1452 | // Do not use getValue() in here; we don't want to generate code at | ||||||||
| 1453 | // this point if it hasn't been done yet. | ||||||||
| 1454 | SDValue N = NodeMap[V]; | ||||||||
| 1455 | if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map. | ||||||||
| 1456 | N = UnusedArgNodeMap[V]; | ||||||||
| 1457 | if (N.getNode()) { | ||||||||
| 1458 | // Only emit func arg dbg value for non-variadic dbg.values for now. | ||||||||
| 1459 | if (!IsVariadic && | ||||||||
| 1460 | EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc, | ||||||||
| 1461 | FuncArgumentDbgValueKind::Value, N)) | ||||||||
| 1462 | return true; | ||||||||
| 1463 | if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { | ||||||||
| 1464 | // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can | ||||||||
| 1465 | // describe stack slot locations. | ||||||||
| 1466 | // | ||||||||
| 1467 | // Consider "int x = 0; int *px = &x;". There are two kinds of | ||||||||
| 1468 | // interesting debug values here after optimization: | ||||||||
| 1469 | // | ||||||||
| 1470 | // dbg.value(i32* %px, !"int *px", !DIExpression()), and | ||||||||
| 1471 | // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) | ||||||||
| 1472 | // | ||||||||
| 1473 | // Both describe the direct values of their associated variables. | ||||||||
| 1474 | Dependencies.push_back(N.getNode()); | ||||||||
| 1475 | LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex())); | ||||||||
| 1476 | continue; | ||||||||
| 1477 | } | ||||||||
| 1478 | LocationOps.emplace_back( | ||||||||
| 1479 | SDDbgOperand::fromNode(N.getNode(), N.getResNo())); | ||||||||
| 1480 | continue; | ||||||||
| 1481 | } | ||||||||
| 1482 | |||||||||
| 1483 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 1484 | // Special rules apply for the first dbg.values of parameter variables in a | ||||||||
| 1485 | // function. Identify them by the fact they reference Argument Values, that | ||||||||
| 1486 | // they're parameters, and they are parameters of the current function. We | ||||||||
| 1487 | // need to let them dangle until they get an SDNode. | ||||||||
| 1488 | bool IsParamOfFunc = | ||||||||
| 1489 | isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt(); | ||||||||
| 1490 | if (IsParamOfFunc) | ||||||||
| 1491 | return false; | ||||||||
| 1492 | |||||||||
| 1493 | // The value is not used in this block yet (or it would have an SDNode). | ||||||||
| 1494 | // We still want the value to appear for the user if possible -- if it has | ||||||||
| 1495 | // an associated VReg, we can refer to that instead. | ||||||||
| 1496 | auto VMI = FuncInfo.ValueMap.find(V); | ||||||||
| 1497 | if (VMI != FuncInfo.ValueMap.end()) { | ||||||||
| 1498 | unsigned Reg = VMI->second; | ||||||||
| 1499 | // If this is a PHI node, it may be split up into several MI PHI nodes | ||||||||
| 1500 | // (in FunctionLoweringInfo::set). | ||||||||
| 1501 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, | ||||||||
| 1502 | V->getType(), std::nullopt); | ||||||||
| 1503 | if (RFV.occupiesMultipleRegs()) { | ||||||||
| 1504 | // FIXME: We could potentially support variadic dbg_values here. | ||||||||
| 1505 | if (IsVariadic) | ||||||||
| 1506 | return false; | ||||||||
| 1507 | unsigned Offset = 0; | ||||||||
| 1508 | unsigned BitsToDescribe = 0; | ||||||||
| 1509 | if (auto VarSize = Var->getSizeInBits()) | ||||||||
| 1510 | BitsToDescribe = *VarSize; | ||||||||
| 1511 | if (auto Fragment = Expr->getFragmentInfo()) | ||||||||
| 1512 | BitsToDescribe = Fragment->SizeInBits; | ||||||||
| 1513 | for (const auto &RegAndSize : RFV.getRegsAndSizes()) { | ||||||||
| 1514 | // Bail out if all bits are described already. | ||||||||
| 1515 | if (Offset >= BitsToDescribe) | ||||||||
| 1516 | break; | ||||||||
| 1517 | // TODO: handle scalable vectors. | ||||||||
| 1518 | unsigned RegisterSize = RegAndSize.second; | ||||||||
| 1519 | unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe) | ||||||||
| 1520 | ? BitsToDescribe - Offset | ||||||||
| 1521 | : RegisterSize; | ||||||||
| 1522 | auto FragmentExpr = DIExpression::createFragmentExpression( | ||||||||
| 1523 | Expr, Offset, FragmentSize); | ||||||||
| 1524 | if (!FragmentExpr) | ||||||||
| 1525 | continue; | ||||||||
| 1526 | SDDbgValue *SDV = DAG.getVRegDbgValue( | ||||||||
| 1527 | Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder); | ||||||||
| 1528 | DAG.AddDbgValue(SDV, false); | ||||||||
| 1529 | Offset += RegisterSize; | ||||||||
| 1530 | } | ||||||||
| 1531 | return true; | ||||||||
| 1532 | } | ||||||||
| 1533 | // We can use simple vreg locations for variadic dbg_values as well. | ||||||||
| 1534 | LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg)); | ||||||||
| 1535 | continue; | ||||||||
| 1536 | } | ||||||||
| 1537 | // We failed to create a SDDbgOperand for V. | ||||||||
| 1538 | return false; | ||||||||
| 1539 | } | ||||||||
| 1540 | |||||||||
| 1541 | // We have created a SDDbgOperand for each Value in Values. | ||||||||
| 1542 | // Should use Order instead of SDNodeOrder? | ||||||||
| 1543 | assert(!LocationOps.empty())(static_cast <bool> (!LocationOps.empty()) ? void (0) : __assert_fail ("!LocationOps.empty()", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1543, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1544 | SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies, | ||||||||
| 1545 | /*IsIndirect=*/false, DbgLoc, | ||||||||
| 1546 | SDNodeOrder, IsVariadic); | ||||||||
| 1547 | DAG.AddDbgValue(SDV, /*isParameter=*/false); | ||||||||
| 1548 | return true; | ||||||||
| 1549 | } | ||||||||
| 1550 | |||||||||
| 1551 | void SelectionDAGBuilder::resolveOrClearDbgInfo() { | ||||||||
| 1552 | // Try to fixup any remaining dangling debug info -- and drop it if we can't. | ||||||||
| 1553 | for (auto &Pair : DanglingDebugInfoMap) | ||||||||
| 1554 | for (auto &DDI : Pair.second) | ||||||||
| 1555 | salvageUnresolvedDbgValue(DDI); | ||||||||
| 1556 | clearDanglingDebugInfo(); | ||||||||
| 1557 | } | ||||||||
| 1558 | |||||||||
| 1559 | /// getCopyFromRegs - If there was virtual register allocated for the value V | ||||||||
| 1560 | /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. | ||||||||
| 1561 | SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { | ||||||||
| 1562 | DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V); | ||||||||
| 1563 | SDValue Result; | ||||||||
| 1564 | |||||||||
| 1565 | if (It != FuncInfo.ValueMap.end()) { | ||||||||
| 1566 | Register InReg = It->second; | ||||||||
| 1567 | |||||||||
| 1568 | RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), | ||||||||
| 1569 | DAG.getDataLayout(), InReg, Ty, | ||||||||
| 1570 | std::nullopt); // This is not an ABI copy. | ||||||||
| 1571 | SDValue Chain = DAG.getEntryNode(); | ||||||||
| 1572 | Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, | ||||||||
| 1573 | V); | ||||||||
| 1574 | resolveDanglingDebugInfo(V, Result); | ||||||||
| 1575 | } | ||||||||
| 1576 | |||||||||
| 1577 | return Result; | ||||||||
| 1578 | } | ||||||||
| 1579 | |||||||||
| 1580 | /// getValue - Return an SDValue for the given Value. | ||||||||
| 1581 | SDValue SelectionDAGBuilder::getValue(const Value *V) { | ||||||||
| 1582 | // If we already have an SDValue for this value, use it. It's important | ||||||||
| 1583 | // to do this first, so that we don't create a CopyFromReg if we already | ||||||||
| 1584 | // have a regular SDValue. | ||||||||
| 1585 | SDValue &N = NodeMap[V]; | ||||||||
| 1586 | if (N.getNode()) return N; | ||||||||
| 1587 | |||||||||
| 1588 | // If there's a virtual register allocated and initialized for this | ||||||||
| 1589 | // value, use it. | ||||||||
| 1590 | if (SDValue copyFromReg = getCopyFromRegs(V, V->getType())) | ||||||||
| 1591 | return copyFromReg; | ||||||||
| 1592 | |||||||||
| 1593 | // Otherwise create a new SDValue and remember it. | ||||||||
| 1594 | SDValue Val = getValueImpl(V); | ||||||||
| 1595 | NodeMap[V] = Val; | ||||||||
| 1596 | resolveDanglingDebugInfo(V, Val); | ||||||||
| 1597 | return Val; | ||||||||
| 1598 | } | ||||||||
| 1599 | |||||||||
| 1600 | /// getNonRegisterValue - Return an SDValue for the given Value, but | ||||||||
| 1601 | /// don't look in FuncInfo.ValueMap for a virtual register. | ||||||||
| 1602 | SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { | ||||||||
| 1603 | // If we already have an SDValue for this value, use it. | ||||||||
| 1604 | SDValue &N = NodeMap[V]; | ||||||||
| 1605 | if (N.getNode()) { | ||||||||
| 1606 | if (isIntOrFPConstant(N)) { | ||||||||
| 1607 | // Remove the debug location from the node as the node is about to be used | ||||||||
| 1608 | // in a location which may differ from the original debug location. This | ||||||||
| 1609 | // is relevant to Constant and ConstantFP nodes because they can appear | ||||||||
| 1610 | // as constant expressions inside PHI nodes. | ||||||||
| 1611 | N->setDebugLoc(DebugLoc()); | ||||||||
| 1612 | } | ||||||||
| 1613 | return N; | ||||||||
| 1614 | } | ||||||||
| 1615 | |||||||||
| 1616 | // Otherwise create a new SDValue and remember it. | ||||||||
| 1617 | SDValue Val = getValueImpl(V); | ||||||||
| 1618 | NodeMap[V] = Val; | ||||||||
| 1619 | resolveDanglingDebugInfo(V, Val); | ||||||||
| 1620 | return Val; | ||||||||
| 1621 | } | ||||||||
| 1622 | |||||||||
| 1623 | /// getValueImpl - Helper function for getValue and getNonRegisterValue. | ||||||||
| 1624 | /// Create an SDValue for the given value. | ||||||||
| 1625 | SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { | ||||||||
| 1626 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 1627 | |||||||||
| 1628 | if (const Constant *C = dyn_cast<Constant>(V)) { | ||||||||
| 1629 | EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true); | ||||||||
| 1630 | |||||||||
| 1631 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) | ||||||||
| 1632 | return DAG.getConstant(*CI, getCurSDLoc(), VT); | ||||||||
| 1633 | |||||||||
| 1634 | if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) | ||||||||
| 1635 | return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); | ||||||||
| 1636 | |||||||||
| 1637 | if (isa<ConstantPointerNull>(C)) { | ||||||||
| 1638 | unsigned AS = V->getType()->getPointerAddressSpace(); | ||||||||
| 1639 | return DAG.getConstant(0, getCurSDLoc(), | ||||||||
| 1640 | TLI.getPointerTy(DAG.getDataLayout(), AS)); | ||||||||
| 1641 | } | ||||||||
| 1642 | |||||||||
| 1643 | if (match(C, m_VScale())) | ||||||||
| 1644 | return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)); | ||||||||
| 1645 | |||||||||
| 1646 | if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) | ||||||||
| 1647 | return DAG.getConstantFP(*CFP, getCurSDLoc(), VT); | ||||||||
| 1648 | |||||||||
| 1649 | if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) | ||||||||
| 1650 | return DAG.getUNDEF(VT); | ||||||||
| 1651 | |||||||||
| 1652 | if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { | ||||||||
| 1653 | visit(CE->getOpcode(), *CE); | ||||||||
| 1654 | SDValue N1 = NodeMap[V]; | ||||||||
| 1655 | assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!" ) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1655 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1656 | return N1; | ||||||||
| 1657 | } | ||||||||
| 1658 | |||||||||
| 1659 | if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { | ||||||||
| 1660 | SmallVector<SDValue, 4> Constants; | ||||||||
| 1661 | for (const Use &U : C->operands()) { | ||||||||
| 1662 | SDNode *Val = getValue(U).getNode(); | ||||||||
| 1663 | // If the operand is an empty aggregate, there are no values. | ||||||||
| 1664 | if (!Val) continue; | ||||||||
| 1665 | // Add each leaf value from the operand to the Constants list | ||||||||
| 1666 | // to form a flattened list of all the values. | ||||||||
| 1667 | for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) | ||||||||
| 1668 | Constants.push_back(SDValue(Val, i)); | ||||||||
| 1669 | } | ||||||||
| 1670 | |||||||||
| 1671 | return DAG.getMergeValues(Constants, getCurSDLoc()); | ||||||||
| 1672 | } | ||||||||
| 1673 | |||||||||
| 1674 | if (const ConstantDataSequential *CDS = | ||||||||
| 1675 | dyn_cast<ConstantDataSequential>(C)) { | ||||||||
| 1676 | SmallVector<SDValue, 4> Ops; | ||||||||
| 1677 | for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { | ||||||||
| 1678 | SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); | ||||||||
| 1679 | // Add each leaf value from the operand to the Constants list | ||||||||
| 1680 | // to form a flattened list of all the values. | ||||||||
| 1681 | for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) | ||||||||
| 1682 | Ops.push_back(SDValue(Val, i)); | ||||||||
| 1683 | } | ||||||||
| 1684 | |||||||||
| 1685 | if (isa<ArrayType>(CDS->getType())) | ||||||||
| 1686 | return DAG.getMergeValues(Ops, getCurSDLoc()); | ||||||||
| 1687 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); | ||||||||
| 1688 | } | ||||||||
| 1689 | |||||||||
| 1690 | if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { | ||||||||
| 1691 | assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>( C) || isa<UndefValue>(C)) && "Unknown struct or array constant!" ) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1692 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1692 | "Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>( C) || isa<UndefValue>(C)) && "Unknown struct or array constant!" ) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1692 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1693 | |||||||||
| 1694 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 1695 | ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs); | ||||||||
| 1696 | unsigned NumElts = ValueVTs.size(); | ||||||||
| 1697 | if (NumElts == 0) | ||||||||
| 1698 | return SDValue(); // empty struct | ||||||||
| 1699 | SmallVector<SDValue, 4> Constants(NumElts); | ||||||||
| 1700 | for (unsigned i = 0; i != NumElts; ++i) { | ||||||||
| 1701 | EVT EltVT = ValueVTs[i]; | ||||||||
| 1702 | if (isa<UndefValue>(C)) | ||||||||
| 1703 | Constants[i] = DAG.getUNDEF(EltVT); | ||||||||
| 1704 | else if (EltVT.isFloatingPoint()) | ||||||||
| 1705 | Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT); | ||||||||
| 1706 | else | ||||||||
| 1707 | Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT); | ||||||||
| 1708 | } | ||||||||
| 1709 | |||||||||
| 1710 | return DAG.getMergeValues(Constants, getCurSDLoc()); | ||||||||
| 1711 | } | ||||||||
| 1712 | |||||||||
| 1713 | if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) | ||||||||
| 1714 | return DAG.getBlockAddress(BA, VT); | ||||||||
| 1715 | |||||||||
| 1716 | if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C)) | ||||||||
| 1717 | return getValue(Equiv->getGlobalValue()); | ||||||||
| 1718 | |||||||||
| 1719 | if (const auto *NC = dyn_cast<NoCFIValue>(C)) | ||||||||
| 1720 | return getValue(NC->getGlobalValue()); | ||||||||
| 1721 | |||||||||
| 1722 | VectorType *VecTy = cast<VectorType>(V->getType()); | ||||||||
| 1723 | |||||||||
| 1724 | // Now that we know the number and type of the elements, get that number of | ||||||||
| 1725 | // elements into the Ops array based on what kind of constant it is. | ||||||||
| 1726 | if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { | ||||||||
| 1727 | SmallVector<SDValue, 16> Ops; | ||||||||
| 1728 | unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements(); | ||||||||
| 1729 | for (unsigned i = 0; i != NumElements; ++i) | ||||||||
| 1730 | Ops.push_back(getValue(CV->getOperand(i))); | ||||||||
| 1731 | |||||||||
| 1732 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); | ||||||||
| 1733 | } | ||||||||
| 1734 | |||||||||
| 1735 | if (isa<ConstantAggregateZero>(C)) { | ||||||||
| 1736 | EVT EltVT = | ||||||||
| 1737 | TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType()); | ||||||||
| 1738 | |||||||||
| 1739 | SDValue Op; | ||||||||
| 1740 | if (EltVT.isFloatingPoint()) | ||||||||
| 1741 | Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT); | ||||||||
| 1742 | else | ||||||||
| 1743 | Op = DAG.getConstant(0, getCurSDLoc(), EltVT); | ||||||||
| 1744 | |||||||||
| 1745 | return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op); | ||||||||
| 1746 | } | ||||||||
| 1747 | |||||||||
| 1748 | llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 1748); | ||||||||
| 1749 | } | ||||||||
| 1750 | |||||||||
| 1751 | // If this is a static alloca, generate it as the frameindex instead of | ||||||||
| 1752 | // computation. | ||||||||
| 1753 | if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { | ||||||||
| 1754 | DenseMap<const AllocaInst*, int>::iterator SI = | ||||||||
| 1755 | FuncInfo.StaticAllocaMap.find(AI); | ||||||||
| 1756 | if (SI != FuncInfo.StaticAllocaMap.end()) | ||||||||
| 1757 | return DAG.getFrameIndex( | ||||||||
| 1758 | SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType())); | ||||||||
| 1759 | } | ||||||||
| 1760 | |||||||||
| 1761 | // If this is an instruction which fast-isel has deferred, select it now. | ||||||||
| 1762 | if (const Instruction *Inst = dyn_cast<Instruction>(V)) { | ||||||||
| 1763 | Register InReg = FuncInfo.InitializeRegForValue(Inst); | ||||||||
| 1764 | |||||||||
| 1765 | RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg, | ||||||||
| 1766 | Inst->getType(), std::nullopt); | ||||||||
| 1767 | SDValue Chain = DAG.getEntryNode(); | ||||||||
| 1768 | return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); | ||||||||
| 1769 | } | ||||||||
| 1770 | |||||||||
| 1771 | if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) | ||||||||
| 1772 | return DAG.getMDNode(cast<MDNode>(MD->getMetadata())); | ||||||||
| 1773 | |||||||||
| 1774 | if (const auto *BB = dyn_cast<BasicBlock>(V)) | ||||||||
| 1775 | return DAG.getBasicBlock(FuncInfo.MBBMap[BB]); | ||||||||
| 1776 | |||||||||
| 1777 | llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1777 ); | ||||||||
| 1778 | } | ||||||||
| 1779 | |||||||||
| 1780 | void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) { | ||||||||
| 1781 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||
| 1782 | bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX; | ||||||||
| 1783 | bool IsCoreCLR = Pers == EHPersonality::CoreCLR; | ||||||||
| 1784 | bool IsSEH = isAsynchronousEHPersonality(Pers); | ||||||||
| 1785 | MachineBasicBlock *CatchPadMBB = FuncInfo.MBB; | ||||||||
| 1786 | if (!IsSEH) | ||||||||
| 1787 | CatchPadMBB->setIsEHScopeEntry(); | ||||||||
| 1788 | // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues. | ||||||||
| 1789 | if (IsMSVCCXX || IsCoreCLR) | ||||||||
| 1790 | CatchPadMBB->setIsEHFuncletEntry(); | ||||||||
| 1791 | } | ||||||||
| 1792 | |||||||||
| 1793 | void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) { | ||||||||
| 1794 | // Update machine-CFG edge. | ||||||||
| 1795 | MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()]; | ||||||||
| 1796 | FuncInfo.MBB->addSuccessor(TargetMBB); | ||||||||
| 1797 | TargetMBB->setIsEHCatchretTarget(true); | ||||||||
| 1798 | DAG.getMachineFunction().setHasEHCatchret(true); | ||||||||
| 1799 | |||||||||
| 1800 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||
| 1801 | bool IsSEH = isAsynchronousEHPersonality(Pers); | ||||||||
| 1802 | if (IsSEH) { | ||||||||
| 1803 | // If this is not a fall-through branch or optimizations are switched off, | ||||||||
| 1804 | // emit the branch. | ||||||||
| 1805 | if (TargetMBB != NextBlock(FuncInfo.MBB) || | ||||||||
| 1806 | TM.getOptLevel() == CodeGenOpt::None) | ||||||||
| 1807 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, | ||||||||
| 1808 | getControlRoot(), DAG.getBasicBlock(TargetMBB))); | ||||||||
| 1809 | return; | ||||||||
| 1810 | } | ||||||||
| 1811 | |||||||||
| 1812 | // Figure out the funclet membership for the catchret's successor. | ||||||||
| 1813 | // This will be used by the FuncletLayout pass to determine how to order the | ||||||||
| 1814 | // BB's. | ||||||||
| 1815 | // A 'catchret' returns to the outer scope's color. | ||||||||
| 1816 | Value *ParentPad = I.getCatchSwitchParentPad(); | ||||||||
| 1817 | const BasicBlock *SuccessorColor; | ||||||||
| 1818 | if (isa<ConstantTokenNone>(ParentPad)) | ||||||||
| 1819 | SuccessorColor = &FuncInfo.Fn->getEntryBlock(); | ||||||||
| 1820 | else | ||||||||
| 1821 | SuccessorColor = cast<Instruction>(ParentPad)->getParent(); | ||||||||
| 1822 | assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!" ) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1822 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1823 | MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor]; | ||||||||
| 1824 | assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!" ) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1824 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1825 | |||||||||
| 1826 | // Create the terminator node. | ||||||||
| 1827 | SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other, | ||||||||
| 1828 | getControlRoot(), DAG.getBasicBlock(TargetMBB), | ||||||||
| 1829 | DAG.getBasicBlock(SuccessorColorMBB)); | ||||||||
| 1830 | DAG.setRoot(Ret); | ||||||||
| 1831 | } | ||||||||
| 1832 | |||||||||
| 1833 | void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) { | ||||||||
| 1834 | // Don't emit any special code for the cleanuppad instruction. It just marks | ||||||||
| 1835 | // the start of an EH scope/funclet. | ||||||||
| 1836 | FuncInfo.MBB->setIsEHScopeEntry(); | ||||||||
| 1837 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||
| 1838 | if (Pers != EHPersonality::Wasm_CXX) { | ||||||||
| 1839 | FuncInfo.MBB->setIsEHFuncletEntry(); | ||||||||
| 1840 | FuncInfo.MBB->setIsCleanupFuncletEntry(); | ||||||||
| 1841 | } | ||||||||
| 1842 | } | ||||||||
| 1843 | |||||||||
| 1844 | // In wasm EH, even though a catchpad may not catch an exception if a tag does | ||||||||
| 1845 | // not match, it is OK to add only the first unwind destination catchpad to the | ||||||||
| 1846 | // successors, because there will be at least one invoke instruction within the | ||||||||
| 1847 | // catch scope that points to the next unwind destination, if one exists, so | ||||||||
| 1848 | // CFGSort cannot mess up with BB sorting order. | ||||||||
| 1849 | // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic | ||||||||
| 1850 | // call within them, and catchpads only consisting of 'catch (...)' have a | ||||||||
| 1851 | // '__cxa_end_catch' call within them, both of which generate invokes in case | ||||||||
| 1852 | // the next unwind destination exists, i.e., the next unwind destination is not | ||||||||
| 1853 | // the caller.) | ||||||||
| 1854 | // | ||||||||
| 1855 | // Having at most one EH pad successor is also simpler and helps later | ||||||||
| 1856 | // transformations. | ||||||||
| 1857 | // | ||||||||
| 1858 | // For example, | ||||||||
| 1859 | // current: | ||||||||
| 1860 | // invoke void @foo to ... unwind label %catch.dispatch | ||||||||
| 1861 | // catch.dispatch: | ||||||||
| 1862 | // %0 = catchswitch within ... [label %catch.start] unwind label %next | ||||||||
| 1863 | // catch.start: | ||||||||
| 1864 | // ... | ||||||||
| 1865 | // ... in this BB or some other child BB dominated by this BB there will be an | ||||||||
| 1866 | // invoke that points to 'next' BB as an unwind destination | ||||||||
| 1867 | // | ||||||||
| 1868 | // next: ; We don't need to add this to 'current' BB's successor | ||||||||
| 1869 | // ... | ||||||||
| 1870 | static void findWasmUnwindDestinations( | ||||||||
| 1871 | FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, | ||||||||
| 1872 | BranchProbability Prob, | ||||||||
| 1873 | SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> | ||||||||
| 1874 | &UnwindDests) { | ||||||||
| 1875 | while (EHPadBB) { | ||||||||
| 1876 | const Instruction *Pad = EHPadBB->getFirstNonPHI(); | ||||||||
| 1877 | if (isa<CleanupPadInst>(Pad)) { | ||||||||
| 1878 | // Stop on cleanup pads. | ||||||||
| 1879 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); | ||||||||
| 1880 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||
| 1881 | break; | ||||||||
| 1882 | } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { | ||||||||
| 1883 | // Add the catchpad handlers to the possible destinations. We don't | ||||||||
| 1884 | // continue to the unwind destination of the catchswitch for wasm. | ||||||||
| 1885 | for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { | ||||||||
| 1886 | UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); | ||||||||
| 1887 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||
| 1888 | } | ||||||||
| 1889 | break; | ||||||||
| 1890 | } else { | ||||||||
| 1891 | continue; | ||||||||
| 1892 | } | ||||||||
| 1893 | } | ||||||||
| 1894 | } | ||||||||
| 1895 | |||||||||
| 1896 | /// When an invoke or a cleanupret unwinds to the next EH pad, there are | ||||||||
| 1897 | /// many places it could ultimately go. In the IR, we have a single unwind | ||||||||
| 1898 | /// destination, but in the machine CFG, we enumerate all the possible blocks. | ||||||||
| 1899 | /// This function skips over imaginary basic blocks that hold catchswitch | ||||||||
| 1900 | /// instructions, and finds all the "real" machine | ||||||||
| 1901 | /// basic block destinations. As those destinations may not be successors of | ||||||||
| 1902 | /// EHPadBB, here we also calculate the edge probability to those destinations. | ||||||||
| 1903 | /// The passed-in Prob is the edge probability to EHPadBB. | ||||||||
| 1904 | static void findUnwindDestinations( | ||||||||
| 1905 | FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, | ||||||||
| 1906 | BranchProbability Prob, | ||||||||
| 1907 | SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> | ||||||||
| 1908 | &UnwindDests) { | ||||||||
| 1909 | EHPersonality Personality = | ||||||||
| 1910 | classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||
| 1911 | bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; | ||||||||
| 1912 | bool IsCoreCLR = Personality == EHPersonality::CoreCLR; | ||||||||
| 1913 | bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX; | ||||||||
| 1914 | bool IsSEH = isAsynchronousEHPersonality(Personality); | ||||||||
| 1915 | |||||||||
| 1916 | if (IsWasmCXX) { | ||||||||
| 1917 | findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests); | ||||||||
| 1918 | assert(UnwindDests.size() <= 1 &&(static_cast <bool> (UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm") ? void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1919 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1919 | "There should be at most one unwind destination for wasm")(static_cast <bool> (UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm") ? void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1919 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1920 | return; | ||||||||
| 1921 | } | ||||||||
| 1922 | |||||||||
| 1923 | while (EHPadBB) { | ||||||||
| 1924 | const Instruction *Pad = EHPadBB->getFirstNonPHI(); | ||||||||
| 1925 | BasicBlock *NewEHPadBB = nullptr; | ||||||||
| 1926 | if (isa<LandingPadInst>(Pad)) { | ||||||||
| 1927 | // Stop on landingpads. They are not funclets. | ||||||||
| 1928 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); | ||||||||
| 1929 | break; | ||||||||
| 1930 | } else if (isa<CleanupPadInst>(Pad)) { | ||||||||
| 1931 | // Stop on cleanup pads. Cleanups are always funclet entries for all known | ||||||||
| 1932 | // personalities. | ||||||||
| 1933 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); | ||||||||
| 1934 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||
| 1935 | UnwindDests.back().first->setIsEHFuncletEntry(); | ||||||||
| 1936 | break; | ||||||||
| 1937 | } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { | ||||||||
| 1938 | // Add the catchpad handlers to the possible destinations. | ||||||||
| 1939 | for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { | ||||||||
| 1940 | UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); | ||||||||
| 1941 | // For MSVC++ and the CLR, catchblocks are funclets and need prologues. | ||||||||
| 1942 | if (IsMSVCCXX || IsCoreCLR) | ||||||||
| 1943 | UnwindDests.back().first->setIsEHFuncletEntry(); | ||||||||
| 1944 | if (!IsSEH) | ||||||||
| 1945 | UnwindDests.back().first->setIsEHScopeEntry(); | ||||||||
| 1946 | } | ||||||||
| 1947 | NewEHPadBB = CatchSwitch->getUnwindDest(); | ||||||||
| 1948 | } else { | ||||||||
| 1949 | continue; | ||||||||
| 1950 | } | ||||||||
| 1951 | |||||||||
| 1952 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||
| 1953 | if (BPI && NewEHPadBB) | ||||||||
| 1954 | Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB); | ||||||||
| 1955 | EHPadBB = NewEHPadBB; | ||||||||
| 1956 | } | ||||||||
| 1957 | } | ||||||||
| 1958 | |||||||||
| 1959 | void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) { | ||||||||
| 1960 | // Update successor info. | ||||||||
| 1961 | SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; | ||||||||
| 1962 | auto UnwindDest = I.getUnwindDest(); | ||||||||
| 1963 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||
| 1964 | BranchProbability UnwindDestProb = | ||||||||
| 1965 | (BPI && UnwindDest) | ||||||||
| 1966 | ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest) | ||||||||
| 1967 | : BranchProbability::getZero(); | ||||||||
| 1968 | findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests); | ||||||||
| 1969 | for (auto &UnwindDest : UnwindDests) { | ||||||||
| 1970 | UnwindDest.first->setIsEHPad(); | ||||||||
| 1971 | addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second); | ||||||||
| 1972 | } | ||||||||
| 1973 | FuncInfo.MBB->normalizeSuccProbs(); | ||||||||
| 1974 | |||||||||
| 1975 | // Create the terminator node. | ||||||||
| 1976 | SDValue Ret = | ||||||||
| 1977 | DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot()); | ||||||||
| 1978 | DAG.setRoot(Ret); | ||||||||
| 1979 | } | ||||||||
| 1980 | |||||||||
| 1981 | void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) { | ||||||||
| 1982 | report_fatal_error("visitCatchSwitch not yet implemented!"); | ||||||||
| 1983 | } | ||||||||
| 1984 | |||||||||
| 1985 | void SelectionDAGBuilder::visitRet(const ReturnInst &I) { | ||||||||
| 1986 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 1987 | auto &DL = DAG.getDataLayout(); | ||||||||
| 1988 | SDValue Chain = getControlRoot(); | ||||||||
| 1989 | SmallVector<ISD::OutputArg, 8> Outs; | ||||||||
| 1990 | SmallVector<SDValue, 8> OutVals; | ||||||||
| 1991 | |||||||||
| 1992 | // Calls to @llvm.experimental.deoptimize don't generate a return value, so | ||||||||
| 1993 | // lower | ||||||||
| 1994 | // | ||||||||
| 1995 | // %val = call <ty> @llvm.experimental.deoptimize() | ||||||||
| 1996 | // ret <ty> %val | ||||||||
| 1997 | // | ||||||||
| 1998 | // differently. | ||||||||
| 1999 | if (I.getParent()->getTerminatingDeoptimizeCall()) { | ||||||||
| 2000 | LowerDeoptimizingReturn(); | ||||||||
| 2001 | return; | ||||||||
| 2002 | } | ||||||||
| 2003 | |||||||||
| 2004 | if (!FuncInfo.CanLowerReturn) { | ||||||||
| 2005 | unsigned DemoteReg = FuncInfo.DemoteRegister; | ||||||||
| 2006 | const Function *F = I.getParent()->getParent(); | ||||||||
| 2007 | |||||||||
| 2008 | // Emit a store of the return value through the virtual register. | ||||||||
| 2009 | // Leave Outs empty so that LowerReturn won't try to load return | ||||||||
| 2010 | // registers the usual way. | ||||||||
| 2011 | SmallVector<EVT, 1> PtrValueVTs; | ||||||||
| 2012 | ComputeValueVTs(TLI, DL, | ||||||||
| 2013 | F->getReturnType()->getPointerTo( | ||||||||
| 2014 | DAG.getDataLayout().getAllocaAddrSpace()), | ||||||||
| 2015 | PtrValueVTs); | ||||||||
| 2016 | |||||||||
| 2017 | SDValue RetPtr = | ||||||||
| 2018 | DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]); | ||||||||
| 2019 | SDValue RetOp = getValue(I.getOperand(0)); | ||||||||
| 2020 | |||||||||
| 2021 | SmallVector<EVT, 4> ValueVTs, MemVTs; | ||||||||
| 2022 | SmallVector<uint64_t, 4> Offsets; | ||||||||
| 2023 | ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs, | ||||||||
| 2024 | &Offsets); | ||||||||
| 2025 | unsigned NumValues = ValueVTs.size(); | ||||||||
| 2026 | |||||||||
| 2027 | SmallVector<SDValue, 4> Chains(NumValues); | ||||||||
| 2028 | Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType()); | ||||||||
| 2029 | for (unsigned i = 0; i != NumValues; ++i) { | ||||||||
| 2030 | // An aggregate return value cannot wrap around the address space, so | ||||||||
| 2031 | // offsets to its parts don't wrap either. | ||||||||
| 2032 | SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, | ||||||||
| 2033 | TypeSize::Fixed(Offsets[i])); | ||||||||
| 2034 | |||||||||
| 2035 | SDValue Val = RetOp.getValue(RetOp.getResNo() + i); | ||||||||
| 2036 | if (MemVTs[i] != ValueVTs[i]) | ||||||||
| 2037 | Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]); | ||||||||
| 2038 | Chains[i] = DAG.getStore( | ||||||||
| 2039 | Chain, getCurSDLoc(), Val, | ||||||||
| 2040 | // FIXME: better loc info would be nice. | ||||||||
| 2041 | Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), | ||||||||
| 2042 | commonAlignment(BaseAlign, Offsets[i])); | ||||||||
| 2043 | } | ||||||||
| 2044 | |||||||||
| 2045 | Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), | ||||||||
| 2046 | MVT::Other, Chains); | ||||||||
| 2047 | } else if (I.getNumOperands() != 0) { | ||||||||
| 2048 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 2049 | ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs); | ||||||||
| 2050 | unsigned NumValues = ValueVTs.size(); | ||||||||
| 2051 | if (NumValues) { | ||||||||
| 2052 | SDValue RetOp = getValue(I.getOperand(0)); | ||||||||
| 2053 | |||||||||
| 2054 | const Function *F = I.getParent()->getParent(); | ||||||||
| 2055 | |||||||||
| 2056 | bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( | ||||||||
| 2057 | I.getOperand(0)->getType(), F->getCallingConv(), | ||||||||
| 2058 | /*IsVarArg*/ false, DL); | ||||||||
| 2059 | |||||||||
| 2060 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND; | ||||||||
| 2061 | if (F->getAttributes().hasRetAttr(Attribute::SExt)) | ||||||||
| 2062 | ExtendKind = ISD::SIGN_EXTEND; | ||||||||
| 2063 | else if (F->getAttributes().hasRetAttr(Attribute::ZExt)) | ||||||||
| 2064 | ExtendKind = ISD::ZERO_EXTEND; | ||||||||
| 2065 | |||||||||
| 2066 | LLVMContext &Context = F->getContext(); | ||||||||
| 2067 | bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg); | ||||||||
| 2068 | |||||||||
| 2069 | for (unsigned j = 0; j != NumValues; ++j) { | ||||||||
| 2070 | EVT VT = ValueVTs[j]; | ||||||||
| 2071 | |||||||||
| 2072 | if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) | ||||||||
| 2073 | VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind); | ||||||||
| 2074 | |||||||||
| 2075 | CallingConv::ID CC = F->getCallingConv(); | ||||||||
| 2076 | |||||||||
| 2077 | unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT); | ||||||||
| 2078 | MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT); | ||||||||
| 2079 | SmallVector<SDValue, 4> Parts(NumParts); | ||||||||
| 2080 | getCopyToParts(DAG, getCurSDLoc(), | ||||||||
| 2081 | SDValue(RetOp.getNode(), RetOp.getResNo() + j), | ||||||||
| 2082 | &Parts[0], NumParts, PartVT, &I, CC, ExtendKind); | ||||||||
| 2083 | |||||||||
| 2084 | // 'inreg' on function refers to return value | ||||||||
| 2085 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); | ||||||||
| 2086 | if (RetInReg) | ||||||||
| 2087 | Flags.setInReg(); | ||||||||
| 2088 | |||||||||
| 2089 | if (I.getOperand(0)->getType()->isPointerTy()) { | ||||||||
| 2090 | Flags.setPointer(); | ||||||||
| 2091 | Flags.setPointerAddrSpace( | ||||||||
| 2092 | cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace()); | ||||||||
| 2093 | } | ||||||||
| 2094 | |||||||||
| 2095 | if (NeedsRegBlock) { | ||||||||
| 2096 | Flags.setInConsecutiveRegs(); | ||||||||
| 2097 | if (j == NumValues - 1) | ||||||||
| 2098 | Flags.setInConsecutiveRegsLast(); | ||||||||
| 2099 | } | ||||||||
| 2100 | |||||||||
| 2101 | // Propagate extension type if any | ||||||||
| 2102 | if (ExtendKind == ISD::SIGN_EXTEND) | ||||||||
| 2103 | Flags.setSExt(); | ||||||||
| 2104 | else if (ExtendKind == ISD::ZERO_EXTEND) | ||||||||
| 2105 | Flags.setZExt(); | ||||||||
| 2106 | |||||||||
| 2107 | for (unsigned i = 0; i < NumParts; ++i) { | ||||||||
| 2108 | Outs.push_back(ISD::OutputArg(Flags, | ||||||||
| 2109 | Parts[i].getValueType().getSimpleVT(), | ||||||||
| 2110 | VT, /*isfixed=*/true, 0, 0)); | ||||||||
| 2111 | OutVals.push_back(Parts[i]); | ||||||||
| 2112 | } | ||||||||
| 2113 | } | ||||||||
| 2114 | } | ||||||||
| 2115 | } | ||||||||
| 2116 | |||||||||
| 2117 | // Push in swifterror virtual register as the last element of Outs. This makes | ||||||||
| 2118 | // sure swifterror virtual register will be returned in the swifterror | ||||||||
| 2119 | // physical register. | ||||||||
| 2120 | const Function *F = I.getParent()->getParent(); | ||||||||
| 2121 | if (TLI.supportSwiftError() && | ||||||||
| 2122 | F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) { | ||||||||
| 2123 | assert(SwiftError.getFunctionArg() && "Need a swift error argument")(static_cast <bool> (SwiftError.getFunctionArg() && "Need a swift error argument") ? void (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2123 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2124 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); | ||||||||
| 2125 | Flags.setSwiftError(); | ||||||||
| 2126 | Outs.push_back(ISD::OutputArg( | ||||||||
| 2127 | Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)), | ||||||||
| 2128 | /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0)); | ||||||||
| 2129 | // Create SDNode for the swifterror virtual register. | ||||||||
| 2130 | OutVals.push_back( | ||||||||
| 2131 | DAG.getRegister(SwiftError.getOrCreateVRegUseAt( | ||||||||
| 2132 | &I, FuncInfo.MBB, SwiftError.getFunctionArg()), | ||||||||
| 2133 | EVT(TLI.getPointerTy(DL)))); | ||||||||
| 2134 | } | ||||||||
| 2135 | |||||||||
| 2136 | bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg(); | ||||||||
| 2137 | CallingConv::ID CallConv = | ||||||||
| 2138 | DAG.getMachineFunction().getFunction().getCallingConv(); | ||||||||
| 2139 | Chain = DAG.getTargetLoweringInfo().LowerReturn( | ||||||||
| 2140 | Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG); | ||||||||
| 2141 | |||||||||
| 2142 | // Verify that the target's LowerReturn behaved as expected. | ||||||||
| 2143 | assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType () == MVT::Other && "LowerReturn didn't return a valid chain!" ) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2144 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2144 | "LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType () == MVT::Other && "LowerReturn didn't return a valid chain!" ) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2144 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2145 | |||||||||
| 2146 | // Update the DAG with the new chain value resulting from return lowering. | ||||||||
| 2147 | DAG.setRoot(Chain); | ||||||||
| 2148 | } | ||||||||
| 2149 | |||||||||
| 2150 | /// CopyToExportRegsIfNeeded - If the given value has virtual registers | ||||||||
| 2151 | /// created for it, emit nodes to copy the value into the virtual | ||||||||
| 2152 | /// registers. | ||||||||
| 2153 | void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { | ||||||||
| 2154 | // Skip empty types | ||||||||
| 2155 | if (V->getType()->isEmptyTy()) | ||||||||
| 2156 | return; | ||||||||
| 2157 | |||||||||
| 2158 | DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V); | ||||||||
| 2159 | if (VMI != FuncInfo.ValueMap.end()) { | ||||||||
| 2160 | assert((!V->use_empty() || isa<CallBrInst>(V)) &&(static_cast <bool> ((!V->use_empty() || isa<CallBrInst >(V)) && "Unused value assigned virtual registers!" ) ? void (0) : __assert_fail ("(!V->use_empty() || isa<CallBrInst>(V)) && \"Unused value assigned virtual registers!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2161 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2161 | "Unused value assigned virtual registers!")(static_cast <bool> ((!V->use_empty() || isa<CallBrInst >(V)) && "Unused value assigned virtual registers!" ) ? void (0) : __assert_fail ("(!V->use_empty() || isa<CallBrInst>(V)) && \"Unused value assigned virtual registers!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2161 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2162 | CopyValueToVirtualRegister(V, VMI->second); | ||||||||
| 2163 | } | ||||||||
| 2164 | } | ||||||||
| 2165 | |||||||||
| 2166 | /// ExportFromCurrentBlock - If this condition isn't known to be exported from | ||||||||
| 2167 | /// the current basic block, add it to ValueMap now so that we'll get a | ||||||||
| 2168 | /// CopyTo/FromReg. | ||||||||
| 2169 | void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { | ||||||||
| 2170 | // No need to export constants. | ||||||||
| 2171 | if (!isa<Instruction>(V) && !isa<Argument>(V)) return; | ||||||||
| 2172 | |||||||||
| 2173 | // Already exported? | ||||||||
| 2174 | if (FuncInfo.isExportedInst(V)) return; | ||||||||
| 2175 | |||||||||
| 2176 | Register Reg = FuncInfo.InitializeRegForValue(V); | ||||||||
| 2177 | CopyValueToVirtualRegister(V, Reg); | ||||||||
| 2178 | } | ||||||||
| 2179 | |||||||||
| 2180 | bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, | ||||||||
| 2181 | const BasicBlock *FromBB) { | ||||||||
| 2182 | // The operands of the setcc have to be in this block. We don't know | ||||||||
| 2183 | // how to export them from some other block. | ||||||||
| 2184 | if (const Instruction *VI = dyn_cast<Instruction>(V)) { | ||||||||
| 2185 | // Can export from current BB. | ||||||||
| 2186 | if (VI->getParent() == FromBB) | ||||||||
| 2187 | return true; | ||||||||
| 2188 | |||||||||
| 2189 | // Is already exported, noop. | ||||||||
| 2190 | return FuncInfo.isExportedInst(V); | ||||||||
| 2191 | } | ||||||||
| 2192 | |||||||||
| 2193 | // If this is an argument, we can export it if the BB is the entry block or | ||||||||
| 2194 | // if it is already exported. | ||||||||
| 2195 | if (isa<Argument>(V)) { | ||||||||
| 2196 | if (FromBB->isEntryBlock()) | ||||||||
| 2197 | return true; | ||||||||
| 2198 | |||||||||
| 2199 | // Otherwise, can only export this if it is already exported. | ||||||||
| 2200 | return FuncInfo.isExportedInst(V); | ||||||||
| 2201 | } | ||||||||
| 2202 | |||||||||
| 2203 | // Otherwise, constants can always be exported. | ||||||||
| 2204 | return true; | ||||||||
| 2205 | } | ||||||||
| 2206 | |||||||||
| 2207 | /// Return branch probability calculated by BranchProbabilityInfo for IR blocks. | ||||||||
| 2208 | BranchProbability | ||||||||
| 2209 | SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src, | ||||||||
| 2210 | const MachineBasicBlock *Dst) const { | ||||||||
| 2211 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||
| 2212 | const BasicBlock *SrcBB = Src->getBasicBlock(); | ||||||||
| 2213 | const BasicBlock *DstBB = Dst->getBasicBlock(); | ||||||||
| 2214 | if (!BPI) { | ||||||||
| 2215 | // If BPI is not available, set the default probability as 1 / N, where N is | ||||||||
| 2216 | // the number of successors. | ||||||||
| 2217 | auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1); | ||||||||
| 2218 | return BranchProbability(1, SuccSize); | ||||||||
| 2219 | } | ||||||||
| 2220 | return BPI->getEdgeProbability(SrcBB, DstBB); | ||||||||
| 2221 | } | ||||||||
| 2222 | |||||||||
| 2223 | void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src, | ||||||||
| 2224 | MachineBasicBlock *Dst, | ||||||||
| 2225 | BranchProbability Prob) { | ||||||||
| 2226 | if (!FuncInfo.BPI) | ||||||||
| 2227 | Src->addSuccessorWithoutProb(Dst); | ||||||||
| 2228 | else { | ||||||||
| 2229 | if (Prob.isUnknown()) | ||||||||
| 2230 | Prob = getEdgeProbability(Src, Dst); | ||||||||
| 2231 | Src->addSuccessor(Dst, Prob); | ||||||||
| 2232 | } | ||||||||
| 2233 | } | ||||||||
| 2234 | |||||||||
| 2235 | static bool InBlock(const Value *V, const BasicBlock *BB) { | ||||||||
| 2236 | if (const Instruction *I = dyn_cast<Instruction>(V)) | ||||||||
| 2237 | return I->getParent() == BB; | ||||||||
| 2238 | return true; | ||||||||
| 2239 | } | ||||||||
| 2240 | |||||||||
| 2241 | /// EmitBranchForMergedCondition - Helper method for FindMergedConditions. | ||||||||
| 2242 | /// This function emits a branch and is used at the leaves of an OR or an | ||||||||
| 2243 | /// AND operator tree. | ||||||||
| 2244 | void | ||||||||
| 2245 | SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, | ||||||||
| 2246 | MachineBasicBlock *TBB, | ||||||||
| 2247 | MachineBasicBlock *FBB, | ||||||||
| 2248 | MachineBasicBlock *CurBB, | ||||||||
| 2249 | MachineBasicBlock *SwitchBB, | ||||||||
| 2250 | BranchProbability TProb, | ||||||||
| 2251 | BranchProbability FProb, | ||||||||
| 2252 | bool InvertCond) { | ||||||||
| 2253 | const BasicBlock *BB = CurBB->getBasicBlock(); | ||||||||
| 2254 | |||||||||
| 2255 | // If the leaf of the tree is a comparison, merge the condition into | ||||||||
| 2256 | // the caseblock. | ||||||||
| 2257 | if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { | ||||||||
| 2258 | // The operands of the cmp have to be in this block. We don't know | ||||||||
| 2259 | // how to export them from some other block. If this is the first block | ||||||||
| 2260 | // of the sequence, no exporting is needed. | ||||||||
| 2261 | if (CurBB == SwitchBB || | ||||||||
| 2262 | (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && | ||||||||
| 2263 | isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { | ||||||||
| 2264 | ISD::CondCode Condition; | ||||||||
| 2265 | if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { | ||||||||
| 2266 | ICmpInst::Predicate Pred = | ||||||||
| 2267 | InvertCond ? IC->getInversePredicate() : IC->getPredicate(); | ||||||||
| 2268 | Condition = getICmpCondCode(Pred); | ||||||||
| 2269 | } else { | ||||||||
| 2270 | const FCmpInst *FC = cast<FCmpInst>(Cond); | ||||||||
| 2271 | FCmpInst::Predicate Pred = | ||||||||
| 2272 | InvertCond ? FC->getInversePredicate() : FC->getPredicate(); | ||||||||
| 2273 | Condition = getFCmpCondCode(Pred); | ||||||||
| 2274 | if (TM.Options.NoNaNsFPMath) | ||||||||
| 2275 | Condition = getFCmpCodeWithoutNaN(Condition); | ||||||||
| 2276 | } | ||||||||
| 2277 | |||||||||
| 2278 | CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, | ||||||||
| 2279 | TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); | ||||||||
| 2280 | SL->SwitchCases.push_back(CB); | ||||||||
| 2281 | return; | ||||||||
| 2282 | } | ||||||||
| 2283 | } | ||||||||
| 2284 | |||||||||
| 2285 | // Create a CaseBlock record representing this branch. | ||||||||
| 2286 | ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ; | ||||||||
| 2287 | CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), | ||||||||
| 2288 | nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); | ||||||||
| 2289 | SL->SwitchCases.push_back(CB); | ||||||||
| 2290 | } | ||||||||
| 2291 | |||||||||
| 2292 | void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, | ||||||||
| 2293 | MachineBasicBlock *TBB, | ||||||||
| 2294 | MachineBasicBlock *FBB, | ||||||||
| 2295 | MachineBasicBlock *CurBB, | ||||||||
| 2296 | MachineBasicBlock *SwitchBB, | ||||||||
| 2297 | Instruction::BinaryOps Opc, | ||||||||
| 2298 | BranchProbability TProb, | ||||||||
| 2299 | BranchProbability FProb, | ||||||||
| 2300 | bool InvertCond) { | ||||||||
| 2301 | // Skip over not part of the tree and remember to invert op and operands at | ||||||||
| 2302 | // next level. | ||||||||
| 2303 | Value *NotCond; | ||||||||
| 2304 | if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) && | ||||||||
| 2305 | InBlock(NotCond, CurBB->getBasicBlock())) { | ||||||||
| 2306 | FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb, | ||||||||
| 2307 | !InvertCond); | ||||||||
| 2308 | return; | ||||||||
| 2309 | } | ||||||||
| 2310 | |||||||||
| 2311 | const Instruction *BOp = dyn_cast<Instruction>(Cond); | ||||||||
| 2312 | const Value *BOpOp0, *BOpOp1; | ||||||||
| 2313 | // Compute the effective opcode for Cond, taking into account whether it needs | ||||||||
| 2314 | // to be inverted, e.g. | ||||||||
| 2315 | // and (not (or A, B)), C | ||||||||
| 2316 | // gets lowered as | ||||||||
| 2317 | // and (and (not A, not B), C) | ||||||||
| 2318 | Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0; | ||||||||
| 2319 | if (BOp) { | ||||||||
| 2320 | BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1))) | ||||||||
| 2321 | ? Instruction::And | ||||||||
| 2322 | : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1))) | ||||||||
| 2323 | ? Instruction::Or | ||||||||
| 2324 | : (Instruction::BinaryOps)0); | ||||||||
| 2325 | if (InvertCond) { | ||||||||
| 2326 | if (BOpc == Instruction::And) | ||||||||
| 2327 | BOpc = Instruction::Or; | ||||||||
| 2328 | else if (BOpc == Instruction::Or) | ||||||||
| 2329 | BOpc = Instruction::And; | ||||||||
| 2330 | } | ||||||||
| 2331 | } | ||||||||
| 2332 | |||||||||
| 2333 | // If this node is not part of the or/and tree, emit it as a branch. | ||||||||
| 2334 | // Note that all nodes in the tree should have same opcode. | ||||||||
| 2335 | bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse(); | ||||||||
| 2336 | if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() || | ||||||||
| 2337 | !InBlock(BOpOp0, CurBB->getBasicBlock()) || | ||||||||
| 2338 | !InBlock(BOpOp1, CurBB->getBasicBlock())) { | ||||||||
| 2339 | EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, | ||||||||
| 2340 | TProb, FProb, InvertCond); | ||||||||
| 2341 | return; | ||||||||
| 2342 | } | ||||||||
| 2343 | |||||||||
| 2344 | // Create TmpBB after CurBB. | ||||||||
| 2345 | MachineFunction::iterator BBI(CurBB); | ||||||||
| 2346 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 2347 | MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); | ||||||||
| 2348 | CurBB->getParent()->insert(++BBI, TmpBB); | ||||||||
| 2349 | |||||||||
| 2350 | if (Opc == Instruction::Or) { | ||||||||
| 2351 | // Codegen X | Y as: | ||||||||
| 2352 | // BB1: | ||||||||
| 2353 | // jmp_if_X TBB | ||||||||
| 2354 | // jmp TmpBB | ||||||||
| 2355 | // TmpBB: | ||||||||
| 2356 | // jmp_if_Y TBB | ||||||||
| 2357 | // jmp FBB | ||||||||
| 2358 | // | ||||||||
| 2359 | |||||||||
| 2360 | // We have flexibility in setting Prob for BB1 and Prob for TmpBB. | ||||||||
| 2361 | // The requirement is that | ||||||||
| 2362 | // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) | ||||||||
| 2363 | // = TrueProb for original BB. | ||||||||
| 2364 | // Assuming the original probabilities are A and B, one choice is to set | ||||||||
| 2365 | // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to | ||||||||
| 2366 | // A/(1+B) and 2B/(1+B). This choice assumes that | ||||||||
| 2367 | // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. | ||||||||
| 2368 | // Another choice is to assume TrueProb for BB1 equals to TrueProb for | ||||||||
| 2369 | // TmpBB, but the math is more complicated. | ||||||||
| 2370 | |||||||||
| 2371 | auto NewTrueProb = TProb / 2; | ||||||||
| 2372 | auto NewFalseProb = TProb / 2 + FProb; | ||||||||
| 2373 | // Emit the LHS condition. | ||||||||
| 2374 | FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb, | ||||||||
| 2375 | NewFalseProb, InvertCond); | ||||||||
| 2376 | |||||||||
| 2377 | // Normalize A/2 and B to get A/(1+B) and 2B/(1+B). | ||||||||
| 2378 | SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb}; | ||||||||
| 2379 | BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); | ||||||||
| 2380 | // Emit the RHS condition into TmpBB. | ||||||||
| 2381 | FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], | ||||||||
| 2382 | Probs[1], InvertCond); | ||||||||
| 2383 | } else { | ||||||||
| 2384 | assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And && "Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2384 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2385 | // Codegen X & Y as: | ||||||||
| 2386 | // BB1: | ||||||||
| 2387 | // jmp_if_X TmpBB | ||||||||
| 2388 | // jmp FBB | ||||||||
| 2389 | // TmpBB: | ||||||||
| 2390 | // jmp_if_Y TBB | ||||||||
| 2391 | // jmp FBB | ||||||||
| 2392 | // | ||||||||
| 2393 | // This requires creation of TmpBB after CurBB. | ||||||||
| 2394 | |||||||||
| 2395 | // We have flexibility in setting Prob for BB1 and Prob for TmpBB. | ||||||||
| 2396 | // The requirement is that | ||||||||
| 2397 | // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) | ||||||||
| 2398 | // = FalseProb for original BB. | ||||||||
| 2399 | // Assuming the original probabilities are A and B, one choice is to set | ||||||||
| 2400 | // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to | ||||||||
| 2401 | // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 == | ||||||||
| 2402 | // TrueProb for BB1 * FalseProb for TmpBB. | ||||||||
| 2403 | |||||||||
| 2404 | auto NewTrueProb = TProb + FProb / 2; | ||||||||
| 2405 | auto NewFalseProb = FProb / 2; | ||||||||
| 2406 | // Emit the LHS condition. | ||||||||
| 2407 | FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb, | ||||||||
| 2408 | NewFalseProb, InvertCond); | ||||||||
| 2409 | |||||||||
| 2410 | // Normalize A and B/2 to get 2A/(1+A) and B/(1+A). | ||||||||
| 2411 | SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2}; | ||||||||
| 2412 | BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); | ||||||||
| 2413 | // Emit the RHS condition into TmpBB. | ||||||||
| 2414 | FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], | ||||||||
| 2415 | Probs[1], InvertCond); | ||||||||
| 2416 | } | ||||||||
| 2417 | } | ||||||||
| 2418 | |||||||||
| 2419 | /// If the set of cases should be emitted as a series of branches, return true. | ||||||||
| 2420 | /// If we should emit this as a bunch of and/or'd together conditions, return | ||||||||
| 2421 | /// false. | ||||||||
| 2422 | bool | ||||||||
| 2423 | SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { | ||||||||
| 2424 | if (Cases.size() != 2) return true; | ||||||||
| 2425 | |||||||||
| 2426 | // If this is two comparisons of the same values or'd or and'd together, they | ||||||||
| 2427 | // will get folded into a single comparison, so don't emit two blocks. | ||||||||
| 2428 | if ((Cases[0].CmpLHS == Cases[1].CmpLHS && | ||||||||
| 2429 | Cases[0].CmpRHS == Cases[1].CmpRHS) || | ||||||||
| 2430 | (Cases[0].CmpRHS == Cases[1].CmpLHS && | ||||||||
| 2431 | Cases[0].CmpLHS == Cases[1].CmpRHS)) { | ||||||||
| 2432 | return false; | ||||||||
| 2433 | } | ||||||||
| 2434 | |||||||||
| 2435 | // Handle: (X != null) | (Y != null) --> (X|Y) != 0 | ||||||||
| 2436 | // Handle: (X == null) & (Y == null) --> (X|Y) == 0 | ||||||||
| 2437 | if (Cases[0].CmpRHS == Cases[1].CmpRHS && | ||||||||
| 2438 | Cases[0].CC == Cases[1].CC && | ||||||||
| 2439 | isa<Constant>(Cases[0].CmpRHS) && | ||||||||
| 2440 | cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { | ||||||||
| 2441 | if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) | ||||||||
| 2442 | return false; | ||||||||
| 2443 | if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) | ||||||||
| 2444 | return false; | ||||||||
| 2445 | } | ||||||||
| 2446 | |||||||||
| 2447 | return true; | ||||||||
| 2448 | } | ||||||||
| 2449 | |||||||||
| 2450 | void SelectionDAGBuilder::visitBr(const BranchInst &I) { | ||||||||
| 2451 | MachineBasicBlock *BrMBB = FuncInfo.MBB; | ||||||||
| 2452 | |||||||||
| 2453 | // Update machine-CFG edges. | ||||||||
| 2454 | MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; | ||||||||
| 2455 | |||||||||
| 2456 | if (I.isUnconditional()) { | ||||||||
| 2457 | // Update machine-CFG edges. | ||||||||
| 2458 | BrMBB->addSuccessor(Succ0MBB); | ||||||||
| 2459 | |||||||||
| 2460 | // If this is not a fall-through branch or optimizations are switched off, | ||||||||
| 2461 | // emit the branch. | ||||||||
| 2462 | if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None) | ||||||||
| 2463 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), | ||||||||
| 2464 | MVT::Other, getControlRoot(), | ||||||||
| 2465 | DAG.getBasicBlock(Succ0MBB))); | ||||||||
| 2466 | |||||||||
| 2467 | return; | ||||||||
| 2468 | } | ||||||||
| 2469 | |||||||||
| 2470 | // If this condition is one of the special cases we handle, do special stuff | ||||||||
| 2471 | // now. | ||||||||
| 2472 | const Value *CondVal = I.getCondition(); | ||||||||
| 2473 | MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; | ||||||||
| 2474 | |||||||||
| 2475 | // If this is a series of conditions that are or'd or and'd together, emit | ||||||||
| 2476 | // this as a sequence of branches instead of setcc's with and/or operations. | ||||||||
| 2477 | // As long as jumps are not expensive (exceptions for multi-use logic ops, | ||||||||
| 2478 | // unpredictable branches, and vector extracts because those jumps are likely | ||||||||
| 2479 | // expensive for any target), this should improve performance. | ||||||||
| 2480 | // For example, instead of something like: | ||||||||
| 2481 | // cmp A, B | ||||||||
| 2482 | // C = seteq | ||||||||
| 2483 | // cmp D, E | ||||||||
| 2484 | // F = setle | ||||||||
| 2485 | // or C, F | ||||||||
| 2486 | // jnz foo | ||||||||
| 2487 | // Emit: | ||||||||
| 2488 | // cmp A, B | ||||||||
| 2489 | // je foo | ||||||||
| 2490 | // cmp D, E | ||||||||
| 2491 | // jle foo | ||||||||
| 2492 | const Instruction *BOp = dyn_cast<Instruction>(CondVal); | ||||||||
| 2493 | if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp && | ||||||||
| 2494 | BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) { | ||||||||
| 2495 | Value *Vec; | ||||||||
| 2496 | const Value *BOp0, *BOp1; | ||||||||
| 2497 | Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0; | ||||||||
| 2498 | if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1)))) | ||||||||
| 2499 | Opcode = Instruction::And; | ||||||||
| 2500 | else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1)))) | ||||||||
| 2501 | Opcode = Instruction::Or; | ||||||||
| 2502 | |||||||||
| 2503 | if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) && | ||||||||
| 2504 | match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) { | ||||||||
| 2505 | FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode, | ||||||||
| 2506 | getEdgeProbability(BrMBB, Succ0MBB), | ||||||||
| 2507 | getEdgeProbability(BrMBB, Succ1MBB), | ||||||||
| 2508 | /*InvertCond=*/false); | ||||||||
| 2509 | // If the compares in later blocks need to use values not currently | ||||||||
| 2510 | // exported from this block, export them now. This block should always | ||||||||
| 2511 | // be the first entry. | ||||||||
| 2512 | assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!") ? void (0) : __assert_fail ("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2512 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2513 | |||||||||
| 2514 | // Allow some cases to be rejected. | ||||||||
| 2515 | if (ShouldEmitAsBranches(SL->SwitchCases)) { | ||||||||
| 2516 | for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) { | ||||||||
| 2517 | ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS); | ||||||||
| 2518 | ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS); | ||||||||
| 2519 | } | ||||||||
| 2520 | |||||||||
| 2521 | // Emit the branch for this block. | ||||||||
| 2522 | visitSwitchCase(SL->SwitchCases[0], BrMBB); | ||||||||
| 2523 | SL->SwitchCases.erase(SL->SwitchCases.begin()); | ||||||||
| 2524 | return; | ||||||||
| 2525 | } | ||||||||
| 2526 | |||||||||
| 2527 | // Okay, we decided not to do this, remove any inserted MBB's and clear | ||||||||
| 2528 | // SwitchCases. | ||||||||
| 2529 | for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) | ||||||||
| 2530 | FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB); | ||||||||
| 2531 | |||||||||
| 2532 | SL->SwitchCases.clear(); | ||||||||
| 2533 | } | ||||||||
| 2534 | } | ||||||||
| 2535 | |||||||||
| 2536 | // Create a CaseBlock record representing this branch. | ||||||||
| 2537 | CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), | ||||||||
| 2538 | nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc()); | ||||||||
| 2539 | |||||||||
| 2540 | // Use visitSwitchCase to actually insert the fast branch sequence for this | ||||||||
| 2541 | // cond branch. | ||||||||
| 2542 | visitSwitchCase(CB, BrMBB); | ||||||||
| 2543 | } | ||||||||
| 2544 | |||||||||
| 2545 | /// visitSwitchCase - Emits the necessary code to represent a single node in | ||||||||
| 2546 | /// the binary search tree resulting from lowering a switch instruction. | ||||||||
| 2547 | void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, | ||||||||
| 2548 | MachineBasicBlock *SwitchBB) { | ||||||||
| 2549 | SDValue Cond; | ||||||||
| 2550 | SDValue CondLHS = getValue(CB.CmpLHS); | ||||||||
| 2551 | SDLoc dl = CB.DL; | ||||||||
| 2552 | |||||||||
| 2553 | if (CB.CC == ISD::SETTRUE) { | ||||||||
| 2554 | // Branch or fall through to TrueBB. | ||||||||
| 2555 | addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); | ||||||||
| 2556 | SwitchBB->normalizeSuccProbs(); | ||||||||
| 2557 | if (CB.TrueBB != NextBlock(SwitchBB)) { | ||||||||
| 2558 | DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(), | ||||||||
| 2559 | DAG.getBasicBlock(CB.TrueBB))); | ||||||||
| 2560 | } | ||||||||
| 2561 | return; | ||||||||
| 2562 | } | ||||||||
| 2563 | |||||||||
| 2564 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 2565 | EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType()); | ||||||||
| 2566 | |||||||||
| 2567 | // Build the setcc now. | ||||||||
| 2568 | if (!CB.CmpMHS) { | ||||||||
| 2569 | // Fold "(X == true)" to X and "(X == false)" to !X to | ||||||||
| 2570 | // handle common cases produced by branch lowering. | ||||||||
| 2571 | if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && | ||||||||
| 2572 | CB.CC == ISD::SETEQ) | ||||||||
| 2573 | Cond = CondLHS; | ||||||||
| 2574 | else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && | ||||||||
| 2575 | CB.CC == ISD::SETEQ) { | ||||||||
| 2576 | SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType()); | ||||||||
| 2577 | Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); | ||||||||
| 2578 | } else { | ||||||||
| 2579 | SDValue CondRHS = getValue(CB.CmpRHS); | ||||||||
| 2580 | |||||||||
| 2581 | // If a pointer's DAG type is larger than its memory type then the DAG | ||||||||
| 2582 | // values are zero-extended. This breaks signed comparisons so truncate | ||||||||
| 2583 | // back to the underlying type before doing the compare. | ||||||||
| 2584 | if (CondLHS.getValueType() != MemVT) { | ||||||||
| 2585 | CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT); | ||||||||
| 2586 | CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT); | ||||||||
| 2587 | } | ||||||||
| 2588 | Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC); | ||||||||
| 2589 | } | ||||||||
| 2590 | } else { | ||||||||
| 2591 | assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now" ) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2591 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2592 | |||||||||
| 2593 | const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); | ||||||||
| 2594 | const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); | ||||||||
| 2595 | |||||||||
| 2596 | SDValue CmpOp = getValue(CB.CmpMHS); | ||||||||
| 2597 | EVT VT = CmpOp.getValueType(); | ||||||||
| 2598 | |||||||||
| 2599 | if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { | ||||||||
| 2600 | Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT), | ||||||||
| 2601 | ISD::SETLE); | ||||||||
| 2602 | } else { | ||||||||
| 2603 | SDValue SUB = DAG.getNode(ISD::SUB, dl, | ||||||||
| 2604 | VT, CmpOp, DAG.getConstant(Low, dl, VT)); | ||||||||
| 2605 | Cond = DAG.getSetCC(dl, MVT::i1, SUB, | ||||||||
| 2606 | DAG.getConstant(High-Low, dl, VT), ISD::SETULE); | ||||||||
| 2607 | } | ||||||||
| 2608 | } | ||||||||
| 2609 | |||||||||
| 2610 | // Update successor info | ||||||||
| 2611 | addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); | ||||||||
| 2612 | // TrueBB and FalseBB are always different unless the incoming IR is | ||||||||
| 2613 | // degenerate. This only happens when running llc on weird IR. | ||||||||
| 2614 | if (CB.TrueBB != CB.FalseBB) | ||||||||
| 2615 | addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb); | ||||||||
| 2616 | SwitchBB->normalizeSuccProbs(); | ||||||||
| 2617 | |||||||||
| 2618 | // If the lhs block is the next block, invert the condition so that we can | ||||||||
| 2619 | // fall through to the lhs instead of the rhs block. | ||||||||
| 2620 | if (CB.TrueBB == NextBlock(SwitchBB)) { | ||||||||
| 2621 | std::swap(CB.TrueBB, CB.FalseBB); | ||||||||
| 2622 | SDValue True = DAG.getConstant(1, dl, Cond.getValueType()); | ||||||||
| 2623 | Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); | ||||||||
| 2624 | } | ||||||||
| 2625 | |||||||||
| 2626 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, | ||||||||
| 2627 | MVT::Other, getControlRoot(), Cond, | ||||||||
| 2628 | DAG.getBasicBlock(CB.TrueBB)); | ||||||||
| 2629 | |||||||||
| 2630 | setValue(CurInst, BrCond); | ||||||||
| 2631 | |||||||||
| 2632 | // Insert the false branch. Do this even if it's a fall through branch, | ||||||||
| 2633 | // this makes it easier to do DAG optimizations which require inverting | ||||||||
| 2634 | // the branch condition. | ||||||||
| 2635 | BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, | ||||||||
| 2636 | DAG.getBasicBlock(CB.FalseBB)); | ||||||||
| 2637 | |||||||||
| 2638 | DAG.setRoot(BrCond); | ||||||||
| 2639 | } | ||||||||
| 2640 | |||||||||
| 2641 | /// visitJumpTable - Emit JumpTable node in the current MBB | ||||||||
| 2642 | void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) { | ||||||||
| 2643 | // Emit the code for the jump table | ||||||||
| 2644 | assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!" ) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2644 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2645 | EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); | ||||||||
| 2646 | SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), | ||||||||
| 2647 | JT.Reg, PTy); | ||||||||
| 2648 | SDValue Table = DAG.getJumpTable(JT.JTI, PTy); | ||||||||
| 2649 | SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), | ||||||||
| 2650 | MVT::Other, Index.getValue(1), | ||||||||
| 2651 | Table, Index); | ||||||||
| 2652 | DAG.setRoot(BrJumpTable); | ||||||||
| 2653 | } | ||||||||
| 2654 | |||||||||
| 2655 | /// visitJumpTableHeader - This function emits necessary code to produce index | ||||||||
| 2656 | /// in the JumpTable from switch case. | ||||||||
| 2657 | void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT, | ||||||||
| 2658 | JumpTableHeader &JTH, | ||||||||
| 2659 | MachineBasicBlock *SwitchBB) { | ||||||||
| 2660 | SDLoc dl = getCurSDLoc(); | ||||||||
| 2661 | |||||||||
| 2662 | // Subtract the lowest switch case value from the value being switched on. | ||||||||
| 2663 | SDValue SwitchOp = getValue(JTH.SValue); | ||||||||
| 2664 | EVT VT = SwitchOp.getValueType(); | ||||||||
| 2665 | SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, | ||||||||
| 2666 | DAG.getConstant(JTH.First, dl, VT)); | ||||||||
| 2667 | |||||||||
| 2668 | // The SDNode we just created, which holds the value being switched on minus | ||||||||
| 2669 | // the smallest case value, needs to be copied to a virtual register so it | ||||||||
| 2670 | // can be used as an index into the jump table in a subsequent basic block. | ||||||||
| 2671 | // This value may be smaller or larger than the target's pointer type, and | ||||||||
| 2672 | // therefore require extension or truncating. | ||||||||
| 2673 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 2674 | SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 2675 | |||||||||
| 2676 | unsigned JumpTableReg = | ||||||||
| 2677 | FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 2678 | SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, | ||||||||
| 2679 | JumpTableReg, SwitchOp); | ||||||||
| 2680 | JT.Reg = JumpTableReg; | ||||||||
| 2681 | |||||||||
| 2682 | if (!JTH.FallthroughUnreachable) { | ||||||||
| 2683 | // Emit the range check for the jump table, and branch to the default block | ||||||||
| 2684 | // for the switch statement if the value being switched on exceeds the | ||||||||
| 2685 | // largest case in the switch. | ||||||||
| 2686 | SDValue CMP = DAG.getSetCC( | ||||||||
| 2687 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), | ||||||||
| 2688 | Sub.getValueType()), | ||||||||
| 2689 | Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); | ||||||||
| 2690 | |||||||||
| 2691 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, | ||||||||
| 2692 | MVT::Other, CopyTo, CMP, | ||||||||
| 2693 | DAG.getBasicBlock(JT.Default)); | ||||||||
| 2694 | |||||||||
| 2695 | // Avoid emitting unnecessary branches to the next block. | ||||||||
| 2696 | if (JT.MBB != NextBlock(SwitchBB)) | ||||||||
| 2697 | BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, | ||||||||
| 2698 | DAG.getBasicBlock(JT.MBB)); | ||||||||
| 2699 | |||||||||
| 2700 | DAG.setRoot(BrCond); | ||||||||
| 2701 | } else { | ||||||||
| 2702 | // Avoid emitting unnecessary branches to the next block. | ||||||||
| 2703 | if (JT.MBB != NextBlock(SwitchBB)) | ||||||||
| 2704 | DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo, | ||||||||
| 2705 | DAG.getBasicBlock(JT.MBB))); | ||||||||
| 2706 | else | ||||||||
| 2707 | DAG.setRoot(CopyTo); | ||||||||
| 2708 | } | ||||||||
| 2709 | } | ||||||||
| 2710 | |||||||||
| 2711 | /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global | ||||||||
| 2712 | /// variable if there exists one. | ||||||||
| 2713 | static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, | ||||||||
| 2714 | SDValue &Chain) { | ||||||||
| 2715 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 2716 | EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||
| 2717 | EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); | ||||||||
| 2718 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 2719 | Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent()); | ||||||||
| 2720 | MachineSDNode *Node = | ||||||||
| 2721 | DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain); | ||||||||
| 2722 | if (Global) { | ||||||||
| 2723 | MachinePointerInfo MPInfo(Global); | ||||||||
| 2724 | auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | | ||||||||
| 2725 | MachineMemOperand::MODereferenceable; | ||||||||
| 2726 | MachineMemOperand *MemRef = MF.getMachineMemOperand( | ||||||||
| 2727 | MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy)); | ||||||||
| 2728 | DAG.setNodeMemRefs(Node, {MemRef}); | ||||||||
| 2729 | } | ||||||||
| 2730 | if (PtrTy != PtrMemTy) | ||||||||
| 2731 | return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy); | ||||||||
| 2732 | return SDValue(Node, 0); | ||||||||
| 2733 | } | ||||||||
| 2734 | |||||||||
| 2735 | /// Codegen a new tail for a stack protector check ParentMBB which has had its | ||||||||
| 2736 | /// tail spliced into a stack protector check success bb. | ||||||||
| 2737 | /// | ||||||||
| 2738 | /// For a high level explanation of how this fits into the stack protector | ||||||||
| 2739 | /// generation see the comment on the declaration of class | ||||||||
| 2740 | /// StackProtectorDescriptor. | ||||||||
| 2741 | void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, | ||||||||
| 2742 | MachineBasicBlock *ParentBB) { | ||||||||
| 2743 | |||||||||
| 2744 | // First create the loads to the guard/stack slot for the comparison. | ||||||||
| 2745 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 2746 | EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||
| 2747 | EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); | ||||||||
| 2748 | |||||||||
| 2749 | MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo(); | ||||||||
| 2750 | int FI = MFI.getStackProtectorIndex(); | ||||||||
| 2751 | |||||||||
| 2752 | SDValue Guard; | ||||||||
| 2753 | SDLoc dl = getCurSDLoc(); | ||||||||
| 2754 | SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); | ||||||||
| 2755 | const Module &M = *ParentBB->getParent()->getFunction().getParent(); | ||||||||
| 2756 | Align Align = | ||||||||
| 2757 | DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext())); | ||||||||
| 2758 | |||||||||
| 2759 | // Generate code to load the content of the guard slot. | ||||||||
| 2760 | SDValue GuardVal = DAG.getLoad( | ||||||||
| 2761 | PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr, | ||||||||
| 2762 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align, | ||||||||
| 2763 | MachineMemOperand::MOVolatile); | ||||||||
| 2764 | |||||||||
| 2765 | if (TLI.useStackGuardXorFP()) | ||||||||
| 2766 | GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl); | ||||||||
| 2767 | |||||||||
| 2768 | // Retrieve guard check function, nullptr if instrumentation is inlined. | ||||||||
| 2769 | if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) { | ||||||||
| 2770 | // The target provides a guard check function to validate the guard value. | ||||||||
| 2771 | // Generate a call to that function with the content of the guard slot as | ||||||||
| 2772 | // argument. | ||||||||
| 2773 | FunctionType *FnTy = GuardCheckFn->getFunctionType(); | ||||||||
| 2774 | assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 && "Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2774 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2775 | |||||||||
| 2776 | TargetLowering::ArgListTy Args; | ||||||||
| 2777 | TargetLowering::ArgListEntry Entry; | ||||||||
| 2778 | Entry.Node = GuardVal; | ||||||||
| 2779 | Entry.Ty = FnTy->getParamType(0); | ||||||||
| 2780 | if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg)) | ||||||||
| 2781 | Entry.IsInReg = true; | ||||||||
| 2782 | Args.push_back(Entry); | ||||||||
| 2783 | |||||||||
| 2784 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||
| 2785 | CLI.setDebugLoc(getCurSDLoc()) | ||||||||
| 2786 | .setChain(DAG.getEntryNode()) | ||||||||
| 2787 | .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(), | ||||||||
| 2788 | getValue(GuardCheckFn), std::move(Args)); | ||||||||
| 2789 | |||||||||
| 2790 | std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); | ||||||||
| 2791 | DAG.setRoot(Result.second); | ||||||||
| 2792 | return; | ||||||||
| 2793 | } | ||||||||
| 2794 | |||||||||
| 2795 | // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD. | ||||||||
| 2796 | // Otherwise, emit a volatile load to retrieve the stack guard value. | ||||||||
| 2797 | SDValue Chain = DAG.getEntryNode(); | ||||||||
| 2798 | if (TLI.useLoadStackGuardNode()) { | ||||||||
| 2799 | Guard = getLoadStackGuard(DAG, dl, Chain); | ||||||||
| 2800 | } else { | ||||||||
| 2801 | const Value *IRGuard = TLI.getSDagStackGuard(M); | ||||||||
| 2802 | SDValue GuardPtr = getValue(IRGuard); | ||||||||
| 2803 | |||||||||
| 2804 | Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, | ||||||||
| 2805 | MachinePointerInfo(IRGuard, 0), Align, | ||||||||
| 2806 | MachineMemOperand::MOVolatile); | ||||||||
| 2807 | } | ||||||||
| 2808 | |||||||||
| 2809 | // Perform the comparison via a getsetcc. | ||||||||
| 2810 | SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(), | ||||||||
| 2811 | *DAG.getContext(), | ||||||||
| 2812 | Guard.getValueType()), | ||||||||
| 2813 | Guard, GuardVal, ISD::SETNE); | ||||||||
| 2814 | |||||||||
| 2815 | // If the guard/stackslot do not equal, branch to failure MBB. | ||||||||
| 2816 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, | ||||||||
| 2817 | MVT::Other, GuardVal.getOperand(0), | ||||||||
| 2818 | Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); | ||||||||
| 2819 | // Otherwise branch to success MBB. | ||||||||
| 2820 | SDValue Br = DAG.getNode(ISD::BR, dl, | ||||||||
| 2821 | MVT::Other, BrCond, | ||||||||
| 2822 | DAG.getBasicBlock(SPD.getSuccessMBB())); | ||||||||
| 2823 | |||||||||
| 2824 | DAG.setRoot(Br); | ||||||||
| 2825 | } | ||||||||
| 2826 | |||||||||
| 2827 | /// Codegen the failure basic block for a stack protector check. | ||||||||
| 2828 | /// | ||||||||
| 2829 | /// A failure stack protector machine basic block consists simply of a call to | ||||||||
| 2830 | /// __stack_chk_fail(). | ||||||||
| 2831 | /// | ||||||||
| 2832 | /// For a high level explanation of how this fits into the stack protector | ||||||||
| 2833 | /// generation see the comment on the declaration of class | ||||||||
| 2834 | /// StackProtectorDescriptor. | ||||||||
| 2835 | void | ||||||||
| 2836 | SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { | ||||||||
| 2837 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 2838 | TargetLowering::MakeLibCallOptions CallOptions; | ||||||||
| 2839 | CallOptions.setDiscardResult(true); | ||||||||
| 2840 | SDValue Chain = | ||||||||
| 2841 | TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid, | ||||||||
| 2842 | std::nullopt, CallOptions, getCurSDLoc()) | ||||||||
| 2843 | .second; | ||||||||
| 2844 | // On PS4/PS5, the "return address" must still be within the calling | ||||||||
| 2845 | // function, even if it's at the very end, so emit an explicit TRAP here. | ||||||||
| 2846 | // Passing 'true' for doesNotReturn above won't generate the trap for us. | ||||||||
| 2847 | if (TM.getTargetTriple().isPS()) | ||||||||
| 2848 | Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); | ||||||||
| 2849 | // WebAssembly needs an unreachable instruction after a non-returning call, | ||||||||
| 2850 | // because the function return type can be different from __stack_chk_fail's | ||||||||
| 2851 | // return type (void). | ||||||||
| 2852 | if (TM.getTargetTriple().isWasm()) | ||||||||
| 2853 | Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); | ||||||||
| 2854 | |||||||||
| 2855 | DAG.setRoot(Chain); | ||||||||
| 2856 | } | ||||||||
| 2857 | |||||||||
| 2858 | /// visitBitTestHeader - This function emits necessary code to produce value | ||||||||
| 2859 | /// suitable for "bit tests" | ||||||||
| 2860 | void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, | ||||||||
| 2861 | MachineBasicBlock *SwitchBB) { | ||||||||
| 2862 | SDLoc dl = getCurSDLoc(); | ||||||||
| 2863 | |||||||||
| 2864 | // Subtract the minimum value. | ||||||||
| 2865 | SDValue SwitchOp = getValue(B.SValue); | ||||||||
| 2866 | EVT VT = SwitchOp.getValueType(); | ||||||||
| 2867 | SDValue RangeSub = | ||||||||
| 2868 | DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT)); | ||||||||
| 2869 | |||||||||
| 2870 | // Determine the type of the test operands. | ||||||||
| 2871 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 2872 | bool UsePtrType = false; | ||||||||
| 2873 | if (!TLI.isTypeLegal(VT)) { | ||||||||
| 2874 | UsePtrType = true; | ||||||||
| 2875 | } else { | ||||||||
| 2876 | for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) | ||||||||
| 2877 | if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { | ||||||||
| 2878 | // Switch table case range are encoded into series of masks. | ||||||||
| 2879 | // Just use pointer type, it's guaranteed to fit. | ||||||||
| 2880 | UsePtrType = true; | ||||||||
| 2881 | break; | ||||||||
| 2882 | } | ||||||||
| 2883 | } | ||||||||
| 2884 | SDValue Sub = RangeSub; | ||||||||
| 2885 | if (UsePtrType) { | ||||||||
| 2886 | VT = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||
| 2887 | Sub = DAG.getZExtOrTrunc(Sub, dl, VT); | ||||||||
| 2888 | } | ||||||||
| 2889 | |||||||||
| 2890 | B.RegVT = VT.getSimpleVT(); | ||||||||
| 2891 | B.Reg = FuncInfo.CreateReg(B.RegVT); | ||||||||
| 2892 | SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub); | ||||||||
| 2893 | |||||||||
| 2894 | MachineBasicBlock* MBB = B.Cases[0].ThisBB; | ||||||||
| 2895 | |||||||||
| 2896 | if (!B.FallthroughUnreachable) | ||||||||
| 2897 | addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb); | ||||||||
| 2898 | addSuccessorWithProb(SwitchBB, MBB, B.Prob); | ||||||||
| 2899 | SwitchBB->normalizeSuccProbs(); | ||||||||
| 2900 | |||||||||
| 2901 | SDValue Root = CopyTo; | ||||||||
| 2902 | if (!B.FallthroughUnreachable) { | ||||||||
| 2903 | // Conditional branch to the default block. | ||||||||
| 2904 | SDValue RangeCmp = DAG.getSetCC(dl, | ||||||||
| 2905 | TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), | ||||||||
| 2906 | RangeSub.getValueType()), | ||||||||
| 2907 | RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()), | ||||||||
| 2908 | ISD::SETUGT); | ||||||||
| 2909 | |||||||||
| 2910 | Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp, | ||||||||
| 2911 | DAG.getBasicBlock(B.Default)); | ||||||||
| 2912 | } | ||||||||
| 2913 | |||||||||
| 2914 | // Avoid emitting unnecessary branches to the next block. | ||||||||
| 2915 | if (MBB != NextBlock(SwitchBB)) | ||||||||
| 2916 | Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB)); | ||||||||
| 2917 | |||||||||
| 2918 | DAG.setRoot(Root); | ||||||||
| 2919 | } | ||||||||
| 2920 | |||||||||
| 2921 | /// visitBitTestCase - this function produces one "bit test" | ||||||||
| 2922 | void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, | ||||||||
| 2923 | MachineBasicBlock* NextMBB, | ||||||||
| 2924 | BranchProbability BranchProbToNext, | ||||||||
| 2925 | unsigned Reg, | ||||||||
| 2926 | BitTestCase &B, | ||||||||
| 2927 | MachineBasicBlock *SwitchBB) { | ||||||||
| 2928 | SDLoc dl = getCurSDLoc(); | ||||||||
| 2929 | MVT VT = BB.RegVT; | ||||||||
| 2930 | SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT); | ||||||||
| 2931 | SDValue Cmp; | ||||||||
| 2932 | unsigned PopCount = llvm::popcount(B.Mask); | ||||||||
| 2933 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 2934 | if (PopCount == 1) { | ||||||||
| 2935 | // Testing for a single bit; just compare the shift count with what it | ||||||||
| 2936 | // would need to be to shift a 1 bit in that position. | ||||||||
| 2937 | Cmp = DAG.getSetCC( | ||||||||
| 2938 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), | ||||||||
| 2939 | ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT), | ||||||||
| 2940 | ISD::SETEQ); | ||||||||
| 2941 | } else if (PopCount == BB.Range) { | ||||||||
| 2942 | // There is only one zero bit in the range, test for it directly. | ||||||||
| 2943 | Cmp = DAG.getSetCC( | ||||||||
| 2944 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), | ||||||||
| 2945 | ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE); | ||||||||
| 2946 | } else { | ||||||||
| 2947 | // Make desired shift | ||||||||
| 2948 | SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT, | ||||||||
| 2949 | DAG.getConstant(1, dl, VT), ShiftOp); | ||||||||
| 2950 | |||||||||
| 2951 | // Emit bit tests and jumps | ||||||||
| 2952 | SDValue AndOp = DAG.getNode(ISD::AND, dl, | ||||||||
| 2953 | VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT)); | ||||||||
| 2954 | Cmp = DAG.getSetCC( | ||||||||
| 2955 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), | ||||||||
| 2956 | AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE); | ||||||||
| 2957 | } | ||||||||
| 2958 | |||||||||
| 2959 | // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb. | ||||||||
| 2960 | addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb); | ||||||||
| 2961 | // The branch probability from SwitchBB to NextMBB is BranchProbToNext. | ||||||||
| 2962 | addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext); | ||||||||
| 2963 | // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is | ||||||||
| 2964 | // one as they are relative probabilities (and thus work more like weights), | ||||||||
| 2965 | // and hence we need to normalize them to let the sum of them become one. | ||||||||
| 2966 | SwitchBB->normalizeSuccProbs(); | ||||||||
| 2967 | |||||||||
| 2968 | SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, | ||||||||
| 2969 | MVT::Other, getControlRoot(), | ||||||||
| 2970 | Cmp, DAG.getBasicBlock(B.TargetBB)); | ||||||||
| 2971 | |||||||||
| 2972 | // Avoid emitting unnecessary branches to the next block. | ||||||||
| 2973 | if (NextMBB != NextBlock(SwitchBB)) | ||||||||
| 2974 | BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, | ||||||||
| 2975 | DAG.getBasicBlock(NextMBB)); | ||||||||
| 2976 | |||||||||
| 2977 | DAG.setRoot(BrAnd); | ||||||||
| 2978 | } | ||||||||
| 2979 | |||||||||
| 2980 | void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { | ||||||||
| 2981 | MachineBasicBlock *InvokeMBB = FuncInfo.MBB; | ||||||||
| 2982 | |||||||||
| 2983 | // Retrieve successors. Look through artificial IR level blocks like | ||||||||
| 2984 | // catchswitch for successors. | ||||||||
| 2985 | MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; | ||||||||
| 2986 | const BasicBlock *EHPadBB = I.getSuccessor(1); | ||||||||
| 2987 | MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB]; | ||||||||
| 2988 | |||||||||
| 2989 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't | ||||||||
| 2990 | // have to do anything here to lower funclet bundles. | ||||||||
| 2991 | assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2992 | {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2993 | LLVMContext::OB_gc_live, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2994 | LLVMContext::OB_cfguardtarget,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2995 | LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2996 | "Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live , LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext ::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2997 | |||||||||
| 2998 | const Value *Callee(I.getCalledOperand()); | ||||||||
| 2999 | const Function *Fn = dyn_cast<Function>(Callee); | ||||||||
| 3000 | if (isa<InlineAsm>(Callee)) | ||||||||
| 3001 | visitInlineAsm(I, EHPadBB); | ||||||||
| 3002 | else if (Fn && Fn->isIntrinsic()) { | ||||||||
| 3003 | switch (Fn->getIntrinsicID()) { | ||||||||
| 3004 | default: | ||||||||
| 3005 | llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3005 ); | ||||||||
| 3006 | case Intrinsic::donothing: | ||||||||
| 3007 | // Ignore invokes to @llvm.donothing: jump directly to the next BB. | ||||||||
| 3008 | case Intrinsic::seh_try_begin: | ||||||||
| 3009 | case Intrinsic::seh_scope_begin: | ||||||||
| 3010 | case Intrinsic::seh_try_end: | ||||||||
| 3011 | case Intrinsic::seh_scope_end: | ||||||||
| 3012 | if (EHPadMBB) | ||||||||
| 3013 | // a block referenced by EH table | ||||||||
| 3014 | // so dtor-funclet not removed by opts | ||||||||
| 3015 | EHPadMBB->setMachineBlockAddressTaken(); | ||||||||
| 3016 | break; | ||||||||
| 3017 | case Intrinsic::experimental_patchpoint_void: | ||||||||
| 3018 | case Intrinsic::experimental_patchpoint_i64: | ||||||||
| 3019 | visitPatchpoint(I, EHPadBB); | ||||||||
| 3020 | break; | ||||||||
| 3021 | case Intrinsic::experimental_gc_statepoint: | ||||||||
| 3022 | LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB); | ||||||||
| 3023 | break; | ||||||||
| 3024 | case Intrinsic::wasm_rethrow: { | ||||||||
| 3025 | // This is usually done in visitTargetIntrinsic, but this intrinsic is | ||||||||
| 3026 | // special because it can be invoked, so we manually lower it to a DAG | ||||||||
| 3027 | // node here. | ||||||||
| 3028 | SmallVector<SDValue, 8> Ops; | ||||||||
| 3029 | Ops.push_back(getRoot()); // inchain | ||||||||
| 3030 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3031 | Ops.push_back( | ||||||||
| 3032 | DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(), | ||||||||
| 3033 | TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 3034 | SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain | ||||||||
| 3035 | DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops)); | ||||||||
| 3036 | break; | ||||||||
| 3037 | } | ||||||||
| 3038 | } | ||||||||
| 3039 | } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) { | ||||||||
| 3040 | // Currently we do not lower any intrinsic calls with deopt operand bundles. | ||||||||
| 3041 | // Eventually we will support lowering the @llvm.experimental.deoptimize | ||||||||
| 3042 | // intrinsic, and right now there are no plans to support other intrinsics | ||||||||
| 3043 | // with deopt state. | ||||||||
| 3044 | LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB); | ||||||||
| 3045 | } else { | ||||||||
| 3046 | LowerCallTo(I, getValue(Callee), false, false, EHPadBB); | ||||||||
| 3047 | } | ||||||||
| 3048 | |||||||||
| 3049 | // If the value of the invoke is used outside of its defining block, make it | ||||||||
| 3050 | // available as a virtual register. | ||||||||
| 3051 | // We already took care of the exported value for the statepoint instruction | ||||||||
| 3052 | // during call to the LowerStatepoint. | ||||||||
| 3053 | if (!isa<GCStatepointInst>(I)) { | ||||||||
| 3054 | CopyToExportRegsIfNeeded(&I); | ||||||||
| 3055 | } | ||||||||
| 3056 | |||||||||
| 3057 | SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; | ||||||||
| 3058 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||
| 3059 | BranchProbability EHPadBBProb = | ||||||||
| 3060 | BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB) | ||||||||
| 3061 | : BranchProbability::getZero(); | ||||||||
| 3062 | findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests); | ||||||||
| 3063 | |||||||||
| 3064 | // Update successor info. | ||||||||
| 3065 | addSuccessorWithProb(InvokeMBB, Return); | ||||||||
| 3066 | for (auto &UnwindDest : UnwindDests) { | ||||||||
| 3067 | UnwindDest.first->setIsEHPad(); | ||||||||
| 3068 | addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second); | ||||||||
| 3069 | } | ||||||||
| 3070 | InvokeMBB->normalizeSuccProbs(); | ||||||||
| 3071 | |||||||||
| 3072 | // Drop into normal successor. | ||||||||
| 3073 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), | ||||||||
| 3074 | DAG.getBasicBlock(Return))); | ||||||||
| 3075 | } | ||||||||
| 3076 | |||||||||
| 3077 | void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { | ||||||||
| 3078 | MachineBasicBlock *CallBrMBB = FuncInfo.MBB; | ||||||||
| 3079 | |||||||||
| 3080 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't | ||||||||
| 3081 | // have to do anything here to lower funclet bundles. | ||||||||
| 3082 | assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3084 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 3083 | {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3084 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 3084 | "Cannot lower callbrs with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3084 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3085 | |||||||||
| 3086 | assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")(static_cast <bool> (I.isInlineAsm() && "Only know how to handle inlineasm callbr" ) ? void (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3086 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3087 | visitInlineAsm(I); | ||||||||
| 3088 | CopyToExportRegsIfNeeded(&I); | ||||||||
| 3089 | |||||||||
| 3090 | // Retrieve successors. | ||||||||
| 3091 | SmallPtrSet<BasicBlock *, 8> Dests; | ||||||||
| 3092 | Dests.insert(I.getDefaultDest()); | ||||||||
| 3093 | MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()]; | ||||||||
| 3094 | |||||||||
| 3095 | // Update successor info. | ||||||||
| 3096 | addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); | ||||||||
| 3097 | for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { | ||||||||
| 3098 | BasicBlock *Dest = I.getIndirectDest(i); | ||||||||
| 3099 | MachineBasicBlock *Target = FuncInfo.MBBMap[Dest]; | ||||||||
| 3100 | Target->setIsInlineAsmBrIndirectTarget(); | ||||||||
| 3101 | Target->setMachineBlockAddressTaken(); | ||||||||
| 3102 | Target->setLabelMustBeEmitted(); | ||||||||
| 3103 | // Don't add duplicate machine successors. | ||||||||
| 3104 | if (Dests.insert(Dest).second) | ||||||||
| 3105 | addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero()); | ||||||||
| 3106 | } | ||||||||
| 3107 | CallBrMBB->normalizeSuccProbs(); | ||||||||
| 3108 | |||||||||
| 3109 | // Drop into default successor. | ||||||||
| 3110 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), | ||||||||
| 3111 | MVT::Other, getControlRoot(), | ||||||||
| 3112 | DAG.getBasicBlock(Return))); | ||||||||
| 3113 | } | ||||||||
| 3114 | |||||||||
| 3115 | void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { | ||||||||
| 3116 | llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3116 ); | ||||||||
| 3117 | } | ||||||||
| 3118 | |||||||||
| 3119 | void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { | ||||||||
| 3120 | assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!") ? void (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3121 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 3121 | "Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!") ? void (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3121 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3122 | |||||||||
| 3123 | // If there aren't registers to copy the values into (e.g., during SjLj | ||||||||
| 3124 | // exceptions), then don't bother to create these DAG nodes. | ||||||||
| 3125 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3126 | const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn(); | ||||||||
| 3127 | if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 && | ||||||||
| 3128 | TLI.getExceptionSelectorRegister(PersonalityFn) == 0) | ||||||||
| 3129 | return; | ||||||||
| 3130 | |||||||||
| 3131 | // If landingpad's return type is token type, we don't create DAG nodes | ||||||||
| 3132 | // for its exception pointer and selector value. The extraction of exception | ||||||||
| 3133 | // pointer or selector value from token type landingpads is not currently | ||||||||
| 3134 | // supported. | ||||||||
| 3135 | if (LP.getType()->isTokenTy()) | ||||||||
| 3136 | return; | ||||||||
| 3137 | |||||||||
| 3138 | SmallVector<EVT, 2> ValueVTs; | ||||||||
| 3139 | SDLoc dl = getCurSDLoc(); | ||||||||
| 3140 | ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs); | ||||||||
| 3141 | assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported" ) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3141 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3142 | |||||||||
| 3143 | // Get the two live-in registers as SDValues. The physregs have already been | ||||||||
| 3144 | // copied into virtual registers. | ||||||||
| 3145 | SDValue Ops[2]; | ||||||||
| 3146 | if (FuncInfo.ExceptionPointerVirtReg) { | ||||||||
| 3147 | Ops[0] = DAG.getZExtOrTrunc( | ||||||||
| 3148 | DAG.getCopyFromReg(DAG.getEntryNode(), dl, | ||||||||
| 3149 | FuncInfo.ExceptionPointerVirtReg, | ||||||||
| 3150 | TLI.getPointerTy(DAG.getDataLayout())), | ||||||||
| 3151 | dl, ValueVTs[0]); | ||||||||
| 3152 | } else { | ||||||||
| 3153 | Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 3154 | } | ||||||||
| 3155 | Ops[1] = DAG.getZExtOrTrunc( | ||||||||
| 3156 | DAG.getCopyFromReg(DAG.getEntryNode(), dl, | ||||||||
| 3157 | FuncInfo.ExceptionSelectorVirtReg, | ||||||||
| 3158 | TLI.getPointerTy(DAG.getDataLayout())), | ||||||||
| 3159 | dl, ValueVTs[1]); | ||||||||
| 3160 | |||||||||
| 3161 | // Merge into one. | ||||||||
| 3162 | SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, | ||||||||
| 3163 | DAG.getVTList(ValueVTs), Ops); | ||||||||
| 3164 | setValue(&LP, Res); | ||||||||
| 3165 | } | ||||||||
| 3166 | |||||||||
| 3167 | void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, | ||||||||
| 3168 | MachineBasicBlock *Last) { | ||||||||
| 3169 | // Update JTCases. | ||||||||
| 3170 | for (JumpTableBlock &JTB : SL->JTCases) | ||||||||
| 3171 | if (JTB.first.HeaderBB == First) | ||||||||
| 3172 | JTB.first.HeaderBB = Last; | ||||||||
| 3173 | |||||||||
| 3174 | // Update BitTestCases. | ||||||||
| 3175 | for (BitTestBlock &BTB : SL->BitTestCases) | ||||||||
| 3176 | if (BTB.Parent == First) | ||||||||
| 3177 | BTB.Parent = Last; | ||||||||
| 3178 | } | ||||||||
| 3179 | |||||||||
| 3180 | void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { | ||||||||
| 3181 | MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB; | ||||||||
| 3182 | |||||||||
| 3183 | // Update machine-CFG edges with unique successors. | ||||||||
| 3184 | SmallSet<BasicBlock*, 32> Done; | ||||||||
| 3185 | for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) { | ||||||||
| 3186 | BasicBlock *BB = I.getSuccessor(i); | ||||||||
| 3187 | bool Inserted = Done.insert(BB).second; | ||||||||
| 3188 | if (!Inserted) | ||||||||
| 3189 | continue; | ||||||||
| 3190 | |||||||||
| 3191 | MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; | ||||||||
| 3192 | addSuccessorWithProb(IndirectBrMBB, Succ); | ||||||||
| 3193 | } | ||||||||
| 3194 | IndirectBrMBB->normalizeSuccProbs(); | ||||||||
| 3195 | |||||||||
| 3196 | DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), | ||||||||
| 3197 | MVT::Other, getControlRoot(), | ||||||||
| 3198 | getValue(I.getAddress()))); | ||||||||
| 3199 | } | ||||||||
| 3200 | |||||||||
| 3201 | void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) { | ||||||||
| 3202 | if (!DAG.getTarget().Options.TrapUnreachable) | ||||||||
| 3203 | return; | ||||||||
| 3204 | |||||||||
| 3205 | // We may be able to ignore unreachable behind a noreturn call. | ||||||||
| 3206 | if (DAG.getTarget().Options.NoTrapAfterNoreturn) { | ||||||||
| 3207 | const BasicBlock &BB = *I.getParent(); | ||||||||
| 3208 | if (&I != &BB.front()) { | ||||||||
| 3209 | BasicBlock::const_iterator PredI = | ||||||||
| 3210 | std::prev(BasicBlock::const_iterator(&I)); | ||||||||
| 3211 | if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) { | ||||||||
| 3212 | if (Call->doesNotReturn()) | ||||||||
| 3213 | return; | ||||||||
| 3214 | } | ||||||||
| 3215 | } | ||||||||
| 3216 | } | ||||||||
| 3217 | |||||||||
| 3218 | DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot())); | ||||||||
| 3219 | } | ||||||||
| 3220 | |||||||||
| 3221 | void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) { | ||||||||
| 3222 | SDNodeFlags Flags; | ||||||||
| 3223 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) | ||||||||
| 3224 | Flags.copyFMF(*FPOp); | ||||||||
| 3225 | |||||||||
| 3226 | SDValue Op = getValue(I.getOperand(0)); | ||||||||
| 3227 | SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), | ||||||||
| 3228 | Op, Flags); | ||||||||
| 3229 | setValue(&I, UnNodeValue); | ||||||||
| 3230 | } | ||||||||
| 3231 | |||||||||
| 3232 | void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) { | ||||||||
| 3233 | SDNodeFlags Flags; | ||||||||
| 3234 | if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) { | ||||||||
| 3235 | Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap()); | ||||||||
| 3236 | Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap()); | ||||||||
| 3237 | } | ||||||||
| 3238 | if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) | ||||||||
| 3239 | Flags.setExact(ExactOp->isExact()); | ||||||||
| 3240 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) | ||||||||
| 3241 | Flags.copyFMF(*FPOp); | ||||||||
| 3242 | |||||||||
| 3243 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||
| 3244 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||
| 3245 | SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), | ||||||||
| 3246 | Op1, Op2, Flags); | ||||||||
| 3247 | setValue(&I, BinNodeValue); | ||||||||
| 3248 | } | ||||||||
| 3249 | |||||||||
| 3250 | void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { | ||||||||
| 3251 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||
| 3252 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||
| 3253 | |||||||||
| 3254 | EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy( | ||||||||
| 3255 | Op1.getValueType(), DAG.getDataLayout()); | ||||||||
| 3256 | |||||||||
| 3257 | // Coerce the shift amount to the right type if we can. This exposes the | ||||||||
| 3258 | // truncate or zext to optimization early. | ||||||||
| 3259 | if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { | ||||||||
| 3260 | assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&(static_cast <bool> (ShiftTy.getSizeInBits() >= Log2_32_Ceil (Op1.getValueSizeInBits()) && "Unexpected shift type" ) ? void (0) : __assert_fail ("ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) && \"Unexpected shift type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3261 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 3261 | "Unexpected shift type")(static_cast <bool> (ShiftTy.getSizeInBits() >= Log2_32_Ceil (Op1.getValueSizeInBits()) && "Unexpected shift type" ) ? void (0) : __assert_fail ("ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) && \"Unexpected shift type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3261 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3262 | Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy); | ||||||||
| 3263 | } | ||||||||
| 3264 | |||||||||
| 3265 | bool nuw = false; | ||||||||
| 3266 | bool nsw = false; | ||||||||
| 3267 | bool exact = false; | ||||||||
| 3268 | |||||||||
| 3269 | if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) { | ||||||||
| 3270 | |||||||||
| 3271 | if (const OverflowingBinaryOperator *OFBinOp = | ||||||||
| 3272 | dyn_cast<const OverflowingBinaryOperator>(&I)) { | ||||||||
| 3273 | nuw = OFBinOp->hasNoUnsignedWrap(); | ||||||||
| 3274 | nsw = OFBinOp->hasNoSignedWrap(); | ||||||||
| 3275 | } | ||||||||
| 3276 | if (const PossiblyExactOperator *ExactOp = | ||||||||
| 3277 | dyn_cast<const PossiblyExactOperator>(&I)) | ||||||||
| 3278 | exact = ExactOp->isExact(); | ||||||||
| 3279 | } | ||||||||
| 3280 | SDNodeFlags Flags; | ||||||||
| 3281 | Flags.setExact(exact); | ||||||||
| 3282 | Flags.setNoSignedWrap(nsw); | ||||||||
| 3283 | Flags.setNoUnsignedWrap(nuw); | ||||||||
| 3284 | SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, | ||||||||
| 3285 | Flags); | ||||||||
| 3286 | setValue(&I, Res); | ||||||||
| 3287 | } | ||||||||
| 3288 | |||||||||
| 3289 | void SelectionDAGBuilder::visitSDiv(const User &I) { | ||||||||
| 3290 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||
| 3291 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||
| 3292 | |||||||||
| 3293 | SDNodeFlags Flags; | ||||||||
| 3294 | Flags.setExact(isa<PossiblyExactOperator>(&I) && | ||||||||
| 3295 | cast<PossiblyExactOperator>(&I)->isExact()); | ||||||||
| 3296 | setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1, | ||||||||
| 3297 | Op2, Flags)); | ||||||||
| 3298 | } | ||||||||
| 3299 | |||||||||
| 3300 | void SelectionDAGBuilder::visitICmp(const User &I) { | ||||||||
| 3301 | ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; | ||||||||
| 3302 | if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) | ||||||||
| 3303 | predicate = IC->getPredicate(); | ||||||||
| 3304 | else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) | ||||||||
| 3305 | predicate = ICmpInst::Predicate(IC->getPredicate()); | ||||||||
| 3306 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||
| 3307 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||
| 3308 | ISD::CondCode Opcode = getICmpCondCode(predicate); | ||||||||
| 3309 | |||||||||
| 3310 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3311 | EVT MemVT = | ||||||||
| 3312 | TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); | ||||||||
| 3313 | |||||||||
| 3314 | // If a pointer's DAG type is larger than its memory type then the DAG values | ||||||||
| 3315 | // are zero-extended. This breaks signed comparisons so truncate back to the | ||||||||
| 3316 | // underlying type before doing the compare. | ||||||||
| 3317 | if (Op1.getValueType() != MemVT) { | ||||||||
| 3318 | Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT); | ||||||||
| 3319 | Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT); | ||||||||
| 3320 | } | ||||||||
| 3321 | |||||||||
| 3322 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3323 | I.getType()); | ||||||||
| 3324 | setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); | ||||||||
| 3325 | } | ||||||||
| 3326 | |||||||||
| 3327 | void SelectionDAGBuilder::visitFCmp(const User &I) { | ||||||||
| 3328 | FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; | ||||||||
| 3329 | if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) | ||||||||
| 3330 | predicate = FC->getPredicate(); | ||||||||
| 3331 | else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) | ||||||||
| 3332 | predicate = FCmpInst::Predicate(FC->getPredicate()); | ||||||||
| 3333 | SDValue Op1 = getValue(I.getOperand(0)); | ||||||||
| 3334 | SDValue Op2 = getValue(I.getOperand(1)); | ||||||||
| 3335 | |||||||||
| 3336 | ISD::CondCode Condition = getFCmpCondCode(predicate); | ||||||||
| 3337 | auto *FPMO = cast<FPMathOperator>(&I); | ||||||||
| 3338 | if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath) | ||||||||
| 3339 | Condition = getFCmpCodeWithoutNaN(Condition); | ||||||||
| 3340 | |||||||||
| 3341 | SDNodeFlags Flags; | ||||||||
| 3342 | Flags.copyFMF(*FPMO); | ||||||||
| 3343 | SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); | ||||||||
| 3344 | |||||||||
| 3345 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3346 | I.getType()); | ||||||||
| 3347 | setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); | ||||||||
| 3348 | } | ||||||||
| 3349 | |||||||||
| 3350 | // Check if the condition of the select has one use or two users that are both | ||||||||
| 3351 | // selects with the same condition. | ||||||||
| 3352 | static bool hasOnlySelectUsers(const Value *Cond) { | ||||||||
| 3353 | return llvm::all_of(Cond->users(), [](const Value *V) { | ||||||||
| 3354 | return isa<SelectInst>(V); | ||||||||
| 3355 | }); | ||||||||
| 3356 | } | ||||||||
| 3357 | |||||||||
| 3358 | void SelectionDAGBuilder::visitSelect(const User &I) { | ||||||||
| 3359 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 3360 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), | ||||||||
| 3361 | ValueVTs); | ||||||||
| 3362 | unsigned NumValues = ValueVTs.size(); | ||||||||
| 3363 | if (NumValues == 0) return; | ||||||||
| 3364 | |||||||||
| 3365 | SmallVector<SDValue, 4> Values(NumValues); | ||||||||
| 3366 | SDValue Cond = getValue(I.getOperand(0)); | ||||||||
| 3367 | SDValue LHSVal = getValue(I.getOperand(1)); | ||||||||
| 3368 | SDValue RHSVal = getValue(I.getOperand(2)); | ||||||||
| 3369 | SmallVector<SDValue, 1> BaseOps(1, Cond); | ||||||||
| 3370 | ISD::NodeType OpCode = | ||||||||
| 3371 | Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; | ||||||||
| 3372 | |||||||||
| 3373 | bool IsUnaryAbs = false; | ||||||||
| 3374 | bool Negate = false; | ||||||||
| 3375 | |||||||||
| 3376 | SDNodeFlags Flags; | ||||||||
| 3377 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) | ||||||||
| 3378 | Flags.copyFMF(*FPOp); | ||||||||
| 3379 | |||||||||
| 3380 | // Min/max matching is only viable if all output VTs are the same. | ||||||||
| 3381 | if (all_equal(ValueVTs)) { | ||||||||
| 3382 | EVT VT = ValueVTs[0]; | ||||||||
| 3383 | LLVMContext &Ctx = *DAG.getContext(); | ||||||||
| 3384 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3385 | |||||||||
| 3386 | // We care about the legality of the operation after it has been type | ||||||||
| 3387 | // legalized. | ||||||||
| 3388 | while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal) | ||||||||
| 3389 | VT = TLI.getTypeToTransformTo(Ctx, VT); | ||||||||
| 3390 | |||||||||
| 3391 | // If the vselect is legal, assume we want to leave this as a vector setcc + | ||||||||
| 3392 | // vselect. Otherwise, if this is going to be scalarized, we want to see if | ||||||||
| 3393 | // min/max is legal on the scalar type. | ||||||||
| 3394 | bool UseScalarMinMax = VT.isVector() && | ||||||||
| 3395 | !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT); | ||||||||
| 3396 | |||||||||
| 3397 | // ValueTracking's select pattern matching does not account for -0.0, | ||||||||
| 3398 | // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that | ||||||||
| 3399 | // -0.0 is less than +0.0. | ||||||||
| 3400 | Value *LHS, *RHS; | ||||||||
| 3401 | auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS); | ||||||||
| 3402 | ISD::NodeType Opc = ISD::DELETED_NODE; | ||||||||
| 3403 | switch (SPR.Flavor) { | ||||||||
| 3404 | case SPF_UMAX: Opc = ISD::UMAX; break; | ||||||||
| 3405 | case SPF_UMIN: Opc = ISD::UMIN; break; | ||||||||
| 3406 | case SPF_SMAX: Opc = ISD::SMAX; break; | ||||||||
| 3407 | case SPF_SMIN: Opc = ISD::SMIN; break; | ||||||||
| 3408 | case SPF_FMINNUM: | ||||||||
| 3409 | switch (SPR.NaNBehavior) { | ||||||||
| 3410 | case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3410 ); | ||||||||
| 3411 | case SPNB_RETURNS_NAN: break; | ||||||||
| 3412 | case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break; | ||||||||
| 3413 | case SPNB_RETURNS_ANY: | ||||||||
| 3414 | if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) || | ||||||||
| 3415 | (UseScalarMinMax && | ||||||||
| 3416 | TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()))) | ||||||||
| 3417 | Opc = ISD::FMINNUM; | ||||||||
| 3418 | break; | ||||||||
| 3419 | } | ||||||||
| 3420 | break; | ||||||||
| 3421 | case SPF_FMAXNUM: | ||||||||
| 3422 | switch (SPR.NaNBehavior) { | ||||||||
| 3423 | case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3423 ); | ||||||||
| 3424 | case SPNB_RETURNS_NAN: break; | ||||||||
| 3425 | case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break; | ||||||||
| 3426 | case SPNB_RETURNS_ANY: | ||||||||
| 3427 | if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) || | ||||||||
| 3428 | (UseScalarMinMax && | ||||||||
| 3429 | TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()))) | ||||||||
| 3430 | Opc = ISD::FMAXNUM; | ||||||||
| 3431 | break; | ||||||||
| 3432 | } | ||||||||
| 3433 | break; | ||||||||
| 3434 | case SPF_NABS: | ||||||||
| 3435 | Negate = true; | ||||||||
| 3436 | [[fallthrough]]; | ||||||||
| 3437 | case SPF_ABS: | ||||||||
| 3438 | IsUnaryAbs = true; | ||||||||
| 3439 | Opc = ISD::ABS; | ||||||||
| 3440 | break; | ||||||||
| 3441 | default: break; | ||||||||
| 3442 | } | ||||||||
| 3443 | |||||||||
| 3444 | if (!IsUnaryAbs && Opc != ISD::DELETED_NODE && | ||||||||
| 3445 | (TLI.isOperationLegalOrCustom(Opc, VT) || | ||||||||
| 3446 | (UseScalarMinMax && | ||||||||
| 3447 | TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) && | ||||||||
| 3448 | // If the underlying comparison instruction is used by any other | ||||||||
| 3449 | // instruction, the consumed instructions won't be destroyed, so it is | ||||||||
| 3450 | // not profitable to convert to a min/max. | ||||||||
| 3451 | hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) { | ||||||||
| 3452 | OpCode = Opc; | ||||||||
| 3453 | LHSVal = getValue(LHS); | ||||||||
| 3454 | RHSVal = getValue(RHS); | ||||||||
| 3455 | BaseOps.clear(); | ||||||||
| 3456 | } | ||||||||
| 3457 | |||||||||
| 3458 | if (IsUnaryAbs) { | ||||||||
| 3459 | OpCode = Opc; | ||||||||
| 3460 | LHSVal = getValue(LHS); | ||||||||
| 3461 | BaseOps.clear(); | ||||||||
| 3462 | } | ||||||||
| 3463 | } | ||||||||
| 3464 | |||||||||
| 3465 | if (IsUnaryAbs) { | ||||||||
| 3466 | for (unsigned i = 0; i != NumValues; ++i) { | ||||||||
| 3467 | SDLoc dl = getCurSDLoc(); | ||||||||
| 3468 | EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i); | ||||||||
| 3469 | Values[i] = | ||||||||
| 3470 | DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i)); | ||||||||
| 3471 | if (Negate) | ||||||||
| 3472 | Values[i] = DAG.getNegative(Values[i], dl, VT); | ||||||||
| 3473 | } | ||||||||
| 3474 | } else { | ||||||||
| 3475 | for (unsigned i = 0; i != NumValues; ++i) { | ||||||||
| 3476 | SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end()); | ||||||||
| 3477 | Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i)); | ||||||||
| 3478 | Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i)); | ||||||||
| 3479 | Values[i] = DAG.getNode( | ||||||||
| 3480 | OpCode, getCurSDLoc(), | ||||||||
| 3481 | LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags); | ||||||||
| 3482 | } | ||||||||
| 3483 | } | ||||||||
| 3484 | |||||||||
| 3485 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||
| 3486 | DAG.getVTList(ValueVTs), Values)); | ||||||||
| 3487 | } | ||||||||
| 3488 | |||||||||
| 3489 | void SelectionDAGBuilder::visitTrunc(const User &I) { | ||||||||
| 3490 | // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). | ||||||||
| 3491 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3492 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3493 | I.getType()); | ||||||||
| 3494 | setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); | ||||||||
| 3495 | } | ||||||||
| 3496 | |||||||||
| 3497 | void SelectionDAGBuilder::visitZExt(const User &I) { | ||||||||
| 3498 | // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). | ||||||||
| 3499 | // ZExt also can't be a cast to bool for same reason. So, nothing much to do | ||||||||
| 3500 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3501 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3502 | I.getType()); | ||||||||
| 3503 | setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N)); | ||||||||
| 3504 | } | ||||||||
| 3505 | |||||||||
| 3506 | void SelectionDAGBuilder::visitSExt(const User &I) { | ||||||||
| 3507 | // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). | ||||||||
| 3508 | // SExt also can't be a cast to bool for same reason. So, nothing much to do | ||||||||
| 3509 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3510 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3511 | I.getType()); | ||||||||
| 3512 | setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); | ||||||||
| 3513 | } | ||||||||
| 3514 | |||||||||
| 3515 | void SelectionDAGBuilder::visitFPTrunc(const User &I) { | ||||||||
| 3516 | // FPTrunc is never a no-op cast, no need to check | ||||||||
| 3517 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3518 | SDLoc dl = getCurSDLoc(); | ||||||||
| 3519 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3520 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 3521 | setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N, | ||||||||
| 3522 | DAG.getTargetConstant( | ||||||||
| 3523 | 0, dl, TLI.getPointerTy(DAG.getDataLayout())))); | ||||||||
| 3524 | } | ||||||||
| 3525 | |||||||||
| 3526 | void SelectionDAGBuilder::visitFPExt(const User &I) { | ||||||||
| 3527 | // FPExt is never a no-op cast, no need to check | ||||||||
| 3528 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3529 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3530 | I.getType()); | ||||||||
| 3531 | setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); | ||||||||
| 3532 | } | ||||||||
| 3533 | |||||||||
| 3534 | void SelectionDAGBuilder::visitFPToUI(const User &I) { | ||||||||
| 3535 | // FPToUI is never a no-op cast, no need to check | ||||||||
| 3536 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3537 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3538 | I.getType()); | ||||||||
| 3539 | setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); | ||||||||
| 3540 | } | ||||||||
| 3541 | |||||||||
| 3542 | void SelectionDAGBuilder::visitFPToSI(const User &I) { | ||||||||
| 3543 | // FPToSI is never a no-op cast, no need to check | ||||||||
| 3544 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3545 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3546 | I.getType()); | ||||||||
| 3547 | setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); | ||||||||
| 3548 | } | ||||||||
| 3549 | |||||||||
| 3550 | void SelectionDAGBuilder::visitUIToFP(const User &I) { | ||||||||
| 3551 | // UIToFP is never a no-op cast, no need to check | ||||||||
| 3552 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3553 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3554 | I.getType()); | ||||||||
| 3555 | setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N)); | ||||||||
| 3556 | } | ||||||||
| 3557 | |||||||||
| 3558 | void SelectionDAGBuilder::visitSIToFP(const User &I) { | ||||||||
| 3559 | // SIToFP is never a no-op cast, no need to check | ||||||||
| 3560 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3561 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3562 | I.getType()); | ||||||||
| 3563 | setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); | ||||||||
| 3564 | } | ||||||||
| 3565 | |||||||||
| 3566 | void SelectionDAGBuilder::visitPtrToInt(const User &I) { | ||||||||
| 3567 | // What to do depends on the size of the integer and the size of the pointer. | ||||||||
| 3568 | // We can either truncate, zero extend, or no-op, accordingly. | ||||||||
| 3569 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3570 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3571 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3572 | I.getType()); | ||||||||
| 3573 | EVT PtrMemVT = | ||||||||
| 3574 | TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); | ||||||||
| 3575 | N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT); | ||||||||
| 3576 | N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT); | ||||||||
| 3577 | setValue(&I, N); | ||||||||
| 3578 | } | ||||||||
| 3579 | |||||||||
| 3580 | void SelectionDAGBuilder::visitIntToPtr(const User &I) { | ||||||||
| 3581 | // What to do depends on the size of the integer and the size of the pointer. | ||||||||
| 3582 | // We can either truncate, zero extend, or no-op, accordingly. | ||||||||
| 3583 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3584 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3585 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 3586 | EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 3587 | N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT); | ||||||||
| 3588 | N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT); | ||||||||
| 3589 | setValue(&I, N); | ||||||||
| 3590 | } | ||||||||
| 3591 | |||||||||
| 3592 | void SelectionDAGBuilder::visitBitCast(const User &I) { | ||||||||
| 3593 | SDValue N = getValue(I.getOperand(0)); | ||||||||
| 3594 | SDLoc dl = getCurSDLoc(); | ||||||||
| 3595 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 3596 | I.getType()); | ||||||||
| 3597 | |||||||||
| 3598 | // BitCast assures us that source and destination are the same size so this is | ||||||||
| 3599 | // either a BITCAST or a no-op. | ||||||||
| 3600 | if (DestVT != N.getValueType()) | ||||||||
| 3601 | setValue(&I, DAG.getNode(ISD::BITCAST, dl, | ||||||||
| 3602 | DestVT, N)); // convert types. | ||||||||
| 3603 | // Check if the original LLVM IR Operand was a ConstantInt, because getValue() | ||||||||
| 3604 | // might fold any kind of constant expression to an integer constant and that | ||||||||
| 3605 | // is not what we are looking for. Only recognize a bitcast of a genuine | ||||||||
| 3606 | // constant integer as an opaque constant. | ||||||||
| 3607 | else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0))) | ||||||||
| 3608 | setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false, | ||||||||
| 3609 | /*isOpaque*/true)); | ||||||||
| 3610 | else | ||||||||
| 3611 | setValue(&I, N); // noop cast. | ||||||||
| 3612 | } | ||||||||
| 3613 | |||||||||
| 3614 | void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { | ||||||||
| 3615 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3616 | const Value *SV = I.getOperand(0); | ||||||||
| 3617 | SDValue N = getValue(SV); | ||||||||
| 3618 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 3619 | |||||||||
| 3620 | unsigned SrcAS = SV->getType()->getPointerAddressSpace(); | ||||||||
| 3621 | unsigned DestAS = I.getType()->getPointerAddressSpace(); | ||||||||
| 3622 | |||||||||
| 3623 | if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS)) | ||||||||
| 3624 | N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); | ||||||||
| 3625 | |||||||||
| 3626 | setValue(&I, N); | ||||||||
| 3627 | } | ||||||||
| 3628 | |||||||||
| 3629 | void SelectionDAGBuilder::visitInsertElement(const User &I) { | ||||||||
| 3630 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3631 | SDValue InVec = getValue(I.getOperand(0)); | ||||||||
| 3632 | SDValue InVal = getValue(I.getOperand(1)); | ||||||||
| 3633 | SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(), | ||||||||
| 3634 | TLI.getVectorIdxTy(DAG.getDataLayout())); | ||||||||
| 3635 | setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), | ||||||||
| 3636 | TLI.getValueType(DAG.getDataLayout(), I.getType()), | ||||||||
| 3637 | InVec, InVal, InIdx)); | ||||||||
| 3638 | } | ||||||||
| 3639 | |||||||||
| 3640 | void SelectionDAGBuilder::visitExtractElement(const User &I) { | ||||||||
| 3641 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3642 | SDValue InVec = getValue(I.getOperand(0)); | ||||||||
| 3643 | SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(), | ||||||||
| 3644 | TLI.getVectorIdxTy(DAG.getDataLayout())); | ||||||||
| 3645 | setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), | ||||||||
| 3646 | TLI.getValueType(DAG.getDataLayout(), I.getType()), | ||||||||
| 3647 | InVec, InIdx)); | ||||||||
| 3648 | } | ||||||||
| 3649 | |||||||||
| 3650 | void SelectionDAGBuilder::visitShuffleVector(const User &I) { | ||||||||
| 3651 | SDValue Src1 = getValue(I.getOperand(0)); | ||||||||
| 3652 | SDValue Src2 = getValue(I.getOperand(1)); | ||||||||
| 3653 | ArrayRef<int> Mask; | ||||||||
| 3654 | if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) | ||||||||
| 3655 | Mask = SVI->getShuffleMask(); | ||||||||
| 3656 | else | ||||||||
| 3657 | Mask = cast<ConstantExpr>(I).getShuffleMask(); | ||||||||
| 3658 | SDLoc DL = getCurSDLoc(); | ||||||||
| 3659 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3660 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 3661 | EVT SrcVT = Src1.getValueType(); | ||||||||
| 3662 | |||||||||
| 3663 | if (all_of(Mask, [](int Elem) { return Elem == 0; }) && | ||||||||
| 3664 | VT.isScalableVector()) { | ||||||||
| 3665 | // Canonical splat form of first element of first input vector. | ||||||||
| 3666 | SDValue FirstElt = | ||||||||
| 3667 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1, | ||||||||
| 3668 | DAG.getVectorIdxConstant(0, DL)); | ||||||||
| 3669 | setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt)); | ||||||||
| 3670 | return; | ||||||||
| 3671 | } | ||||||||
| 3672 | |||||||||
| 3673 | // For now, we only handle splats for scalable vectors. | ||||||||
| 3674 | // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation | ||||||||
| 3675 | // for targets that support a SPLAT_VECTOR for non-scalable vector types. | ||||||||
| 3676 | assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")(static_cast <bool> (!VT.isScalableVector() && "Unsupported scalable vector shuffle" ) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3676 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 3677 | |||||||||
| 3678 | unsigned SrcNumElts = SrcVT.getVectorNumElements(); | ||||||||
| 3679 | unsigned MaskNumElts = Mask.size(); | ||||||||
| 3680 | |||||||||
| 3681 | if (SrcNumElts == MaskNumElts) { | ||||||||
| 3682 | setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask)); | ||||||||
| 3683 | return; | ||||||||
| 3684 | } | ||||||||
| 3685 | |||||||||
| 3686 | // Normalize the shuffle vector since mask and vector length don't match. | ||||||||
| 3687 | if (SrcNumElts < MaskNumElts) { | ||||||||
| 3688 | // Mask is longer than the source vectors. We can use concatenate vector to | ||||||||
| 3689 | // make the mask and vectors lengths match. | ||||||||
| 3690 | |||||||||
| 3691 | if (MaskNumElts % SrcNumElts == 0) { | ||||||||
| 3692 | // Mask length is a multiple of the source vector length. | ||||||||
| 3693 | // Check if the shuffle is some kind of concatenation of the input | ||||||||
| 3694 | // vectors. | ||||||||
| 3695 | unsigned NumConcat = MaskNumElts / SrcNumElts; | ||||||||
| 3696 | bool IsConcat = true; | ||||||||
| 3697 | SmallVector<int, 8> ConcatSrcs(NumConcat, -1); | ||||||||
| 3698 | for (unsigned i = 0; i != MaskNumElts; ++i) { | ||||||||
| 3699 | int Idx = Mask[i]; | ||||||||
| 3700 | if (Idx < 0) | ||||||||
| 3701 | continue; | ||||||||
| 3702 | // Ensure the indices in each SrcVT sized piece are sequential and that | ||||||||
| 3703 | // the same source is used for the whole piece. | ||||||||
| 3704 | if ((Idx % SrcNumElts != (i % SrcNumElts)) || | ||||||||
| 3705 | (ConcatSrcs[i / SrcNumElts] >= 0 && | ||||||||
| 3706 | ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) { | ||||||||
| 3707 | IsConcat = false; | ||||||||
| 3708 | break; | ||||||||
| 3709 | } | ||||||||
| 3710 | // Remember which source this index came from. | ||||||||
| 3711 | ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts; | ||||||||
| 3712 | } | ||||||||
| 3713 | |||||||||
| 3714 | // The shuffle is concatenating multiple vectors together. Just emit | ||||||||
| 3715 | // a CONCAT_VECTORS operation. | ||||||||
| 3716 | if (IsConcat) { | ||||||||
| 3717 | SmallVector<SDValue, 8> ConcatOps; | ||||||||
| 3718 | for (auto Src : ConcatSrcs) { | ||||||||
| 3719 | if (Src < 0) | ||||||||
| 3720 | ConcatOps.push_back(DAG.getUNDEF(SrcVT)); | ||||||||
| 3721 | else if (Src == 0) | ||||||||
| 3722 | ConcatOps.push_back(Src1); | ||||||||
| 3723 | else | ||||||||
| 3724 | ConcatOps.push_back(Src2); | ||||||||
| 3725 | } | ||||||||
| 3726 | setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps)); | ||||||||
| 3727 | return; | ||||||||
| 3728 | } | ||||||||
| 3729 | } | ||||||||
| 3730 | |||||||||
| 3731 | unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts); | ||||||||
| 3732 | unsigned NumConcat = PaddedMaskNumElts / SrcNumElts; | ||||||||
| 3733 | EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), | ||||||||
| 3734 | PaddedMaskNumElts); | ||||||||
| 3735 | |||||||||
| 3736 | // Pad both vectors with undefs to make them the same length as the mask. | ||||||||
| 3737 | SDValue UndefVal = DAG.getUNDEF(SrcVT); | ||||||||
| 3738 | |||||||||
| 3739 | SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal); | ||||||||
| 3740 | SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); | ||||||||
| 3741 | MOps1[0] = Src1; | ||||||||
| 3742 | MOps2[0] = Src2; | ||||||||
| 3743 | |||||||||
| 3744 | Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1); | ||||||||
| 3745 | Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2); | ||||||||
| 3746 | |||||||||
| 3747 | // Readjust mask for new input vector length. | ||||||||
| 3748 | SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1); | ||||||||
| 3749 | for (unsigned i = 0; i != MaskNumElts; ++i) { | ||||||||
| 3750 | int Idx = Mask[i]; | ||||||||
| 3751 | if (Idx >= (int)SrcNumElts) | ||||||||
| 3752 | Idx -= SrcNumElts - PaddedMaskNumElts; | ||||||||
| 3753 | MappedOps[i] = Idx; | ||||||||
| 3754 | } | ||||||||
| 3755 | |||||||||
| 3756 | SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps); | ||||||||
| 3757 | |||||||||
| 3758 | // If the concatenated vector was padded, extract a subvector with the | ||||||||
| 3759 | // correct number of elements. | ||||||||
| 3760 | if (MaskNumElts != PaddedMaskNumElts) | ||||||||
| 3761 | Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result, | ||||||||
| 3762 | DAG.getVectorIdxConstant(0, DL)); | ||||||||
| 3763 | |||||||||
| 3764 | setValue(&I, Result); | ||||||||
| 3765 | return; | ||||||||
| 3766 | } | ||||||||
| 3767 | |||||||||
| 3768 | if (SrcNumElts > MaskNumElts) { | ||||||||
| 3769 | // Analyze the access pattern of the vector to see if we can extract | ||||||||
| 3770 | // two subvectors and do the shuffle. | ||||||||
| 3771 | int StartIdx[2] = { -1, -1 }; // StartIdx to extract from | ||||||||
| 3772 | bool CanExtract = true; | ||||||||
| 3773 | for (int Idx : Mask) { | ||||||||
| 3774 | unsigned Input = 0; | ||||||||
| 3775 | if (Idx < 0) | ||||||||
| 3776 | continue; | ||||||||
| 3777 | |||||||||
| 3778 | if (Idx >= (int)SrcNumElts) { | ||||||||
| 3779 | Input = 1; | ||||||||
| 3780 | Idx -= SrcNumElts; | ||||||||
| 3781 | } | ||||||||
| 3782 | |||||||||
| 3783 | // If all the indices come from the same MaskNumElts sized portion of | ||||||||
| 3784 | // the sources we can use extract. Also make sure the extract wouldn't | ||||||||
| 3785 | // extract past the end of the source. | ||||||||
| 3786 | int NewStartIdx = alignDown(Idx, MaskNumElts); | ||||||||
| 3787 | if (NewStartIdx + MaskNumElts > SrcNumElts || | ||||||||
| 3788 | (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx)) | ||||||||
| 3789 | CanExtract = false; | ||||||||
| 3790 | // Make sure we always update StartIdx as we use it to track if all | ||||||||
| 3791 | // elements are undef. | ||||||||
| 3792 | StartIdx[Input] = NewStartIdx; | ||||||||
| 3793 | } | ||||||||
| 3794 | |||||||||
| 3795 | if (StartIdx[0] < 0 && StartIdx[1] < 0) { | ||||||||
| 3796 | setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. | ||||||||
| 3797 | return; | ||||||||
| 3798 | } | ||||||||
| 3799 | if (CanExtract) { | ||||||||
| 3800 | // Extract appropriate subvector and generate a vector shuffle | ||||||||
| 3801 | for (unsigned Input = 0; Input < 2; ++Input) { | ||||||||
| 3802 | SDValue &Src = Input == 0 ? Src1 : Src2; | ||||||||
| 3803 | if (StartIdx[Input] < 0) | ||||||||
| 3804 | Src = DAG.getUNDEF(VT); | ||||||||
| 3805 | else { | ||||||||
| 3806 | Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src, | ||||||||
| 3807 | DAG.getVectorIdxConstant(StartIdx[Input], DL)); | ||||||||
| 3808 | } | ||||||||
| 3809 | } | ||||||||
| 3810 | |||||||||
| 3811 | // Calculate new mask. | ||||||||
| 3812 | SmallVector<int, 8> MappedOps(Mask); | ||||||||
| 3813 | for (int &Idx : MappedOps) { | ||||||||
| 3814 | if (Idx >= (int)SrcNumElts) | ||||||||
| 3815 | Idx -= SrcNumElts + StartIdx[1] - MaskNumElts; | ||||||||
| 3816 | else if (Idx >= 0) | ||||||||
| 3817 | Idx -= StartIdx[0]; | ||||||||
| 3818 | } | ||||||||
| 3819 | |||||||||
| 3820 | setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps)); | ||||||||
| 3821 | return; | ||||||||
| 3822 | } | ||||||||
| 3823 | } | ||||||||
| 3824 | |||||||||
| 3825 | // We can't use either concat vectors or extract subvectors so fall back to | ||||||||
| 3826 | // replacing the shuffle with extract and build vector. | ||||||||
| 3827 | // to insert and build vector. | ||||||||
| 3828 | EVT EltVT = VT.getVectorElementType(); | ||||||||
| 3829 | SmallVector<SDValue,8> Ops; | ||||||||
| 3830 | for (int Idx : Mask) { | ||||||||
| 3831 | SDValue Res; | ||||||||
| 3832 | |||||||||
| 3833 | if (Idx < 0) { | ||||||||
| 3834 | Res = DAG.getUNDEF(EltVT); | ||||||||
| 3835 | } else { | ||||||||
| 3836 | SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; | ||||||||
| 3837 | if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; | ||||||||
| 3838 | |||||||||
| 3839 | Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src, | ||||||||
| 3840 | DAG.getVectorIdxConstant(Idx, DL)); | ||||||||
| 3841 | } | ||||||||
| 3842 | |||||||||
| 3843 | Ops.push_back(Res); | ||||||||
| 3844 | } | ||||||||
| 3845 | |||||||||
| 3846 | setValue(&I, DAG.getBuildVector(VT, DL, Ops)); | ||||||||
| 3847 | } | ||||||||
| 3848 | |||||||||
| 3849 | void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { | ||||||||
| 3850 | ArrayRef<unsigned> Indices = I.getIndices(); | ||||||||
| 3851 | const Value *Op0 = I.getOperand(0); | ||||||||
| 3852 | const Value *Op1 = I.getOperand(1); | ||||||||
| 3853 | Type *AggTy = I.getType(); | ||||||||
| 3854 | Type *ValTy = Op1->getType(); | ||||||||
| 3855 | bool IntoUndef = isa<UndefValue>(Op0); | ||||||||
| 3856 | bool FromUndef = isa<UndefValue>(Op1); | ||||||||
| 3857 | |||||||||
| 3858 | unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); | ||||||||
| 3859 | |||||||||
| 3860 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3861 | SmallVector<EVT, 4> AggValueVTs; | ||||||||
| 3862 | ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs); | ||||||||
| 3863 | SmallVector<EVT, 4> ValValueVTs; | ||||||||
| 3864 | ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); | ||||||||
| 3865 | |||||||||
| 3866 | unsigned NumAggValues = AggValueVTs.size(); | ||||||||
| 3867 | unsigned NumValValues = ValValueVTs.size(); | ||||||||
| 3868 | SmallVector<SDValue, 4> Values(NumAggValues); | ||||||||
| 3869 | |||||||||
| 3870 | // Ignore an insertvalue that produces an empty object | ||||||||
| 3871 | if (!NumAggValues) { | ||||||||
| 3872 | setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); | ||||||||
| 3873 | return; | ||||||||
| 3874 | } | ||||||||
| 3875 | |||||||||
| 3876 | SDValue Agg = getValue(Op0); | ||||||||
| 3877 | unsigned i = 0; | ||||||||
| 3878 | // Copy the beginning value(s) from the original aggregate. | ||||||||
| 3879 | for (; i != LinearIndex; ++i) | ||||||||
| 3880 | Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : | ||||||||
| 3881 | SDValue(Agg.getNode(), Agg.getResNo() + i); | ||||||||
| 3882 | // Copy values from the inserted value(s). | ||||||||
| 3883 | if (NumValValues) { | ||||||||
| 3884 | SDValue Val = getValue(Op1); | ||||||||
| 3885 | for (; i != LinearIndex + NumValValues; ++i) | ||||||||
| 3886 | Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : | ||||||||
| 3887 | SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); | ||||||||
| 3888 | } | ||||||||
| 3889 | // Copy remaining value(s) from the original aggregate. | ||||||||
| 3890 | for (; i != NumAggValues; ++i) | ||||||||
| 3891 | Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : | ||||||||
| 3892 | SDValue(Agg.getNode(), Agg.getResNo() + i); | ||||||||
| 3893 | |||||||||
| 3894 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||
| 3895 | DAG.getVTList(AggValueVTs), Values)); | ||||||||
| 3896 | } | ||||||||
| 3897 | |||||||||
| 3898 | void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { | ||||||||
| 3899 | ArrayRef<unsigned> Indices = I.getIndices(); | ||||||||
| 3900 | const Value *Op0 = I.getOperand(0); | ||||||||
| 3901 | Type *AggTy = Op0->getType(); | ||||||||
| 3902 | Type *ValTy = I.getType(); | ||||||||
| 3903 | bool OutOfUndef = isa<UndefValue>(Op0); | ||||||||
| 3904 | |||||||||
| 3905 | unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); | ||||||||
| 3906 | |||||||||
| 3907 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3908 | SmallVector<EVT, 4> ValValueVTs; | ||||||||
| 3909 | ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); | ||||||||
| 3910 | |||||||||
| 3911 | unsigned NumValValues = ValValueVTs.size(); | ||||||||
| 3912 | |||||||||
| 3913 | // Ignore a extractvalue that produces an empty object | ||||||||
| 3914 | if (!NumValValues) { | ||||||||
| 3915 | setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); | ||||||||
| 3916 | return; | ||||||||
| 3917 | } | ||||||||
| 3918 | |||||||||
| 3919 | SmallVector<SDValue, 4> Values(NumValValues); | ||||||||
| 3920 | |||||||||
| 3921 | SDValue Agg = getValue(Op0); | ||||||||
| 3922 | // Copy out the selected value(s). | ||||||||
| 3923 | for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) | ||||||||
| 3924 | Values[i - LinearIndex] = | ||||||||
| 3925 | OutOfUndef ? | ||||||||
| 3926 | DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : | ||||||||
| 3927 | SDValue(Agg.getNode(), Agg.getResNo() + i); | ||||||||
| 3928 | |||||||||
| 3929 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||
| 3930 | DAG.getVTList(ValValueVTs), Values)); | ||||||||
| 3931 | } | ||||||||
| 3932 | |||||||||
| 3933 | void SelectionDAGBuilder::visitGetElementPtr(const User &I) { | ||||||||
| 3934 | Value *Op0 = I.getOperand(0); | ||||||||
| 3935 | // Note that the pointer operand may be a vector of pointers. Take the scalar | ||||||||
| 3936 | // element which holds a pointer. | ||||||||
| 3937 | unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||
| 3938 | SDValue N = getValue(Op0); | ||||||||
| 3939 | SDLoc dl = getCurSDLoc(); | ||||||||
| 3940 | auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 3941 | |||||||||
| 3942 | // Normalize Vector GEP - all scalar operands should be converted to the | ||||||||
| 3943 | // splat vector. | ||||||||
| 3944 | bool IsVectorGEP = I.getType()->isVectorTy(); | ||||||||
| 3945 | ElementCount VectorElementCount = | ||||||||
| 3946 | IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount() | ||||||||
| 3947 | : ElementCount::getFixed(0); | ||||||||
| 3948 | |||||||||
| 3949 | if (IsVectorGEP && !N.getValueType().isVector()) { | ||||||||
| 3950 | LLVMContext &Context = *DAG.getContext(); | ||||||||
| 3951 | EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount); | ||||||||
| 3952 | N = DAG.getSplat(VT, dl, N); | ||||||||
| 3953 | } | ||||||||
| 3954 | |||||||||
| 3955 | for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I); | ||||||||
| 3956 | GTI != E; ++GTI) { | ||||||||
| 3957 | const Value *Idx = GTI.getOperand(); | ||||||||
| 3958 | if (StructType *StTy = GTI.getStructTypeOrNull()) { | ||||||||
| 3959 | unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); | ||||||||
| 3960 | if (Field) { | ||||||||
| 3961 | // N = N + Offset | ||||||||
| 3962 | uint64_t Offset = | ||||||||
| 3963 | DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field); | ||||||||
| 3964 | |||||||||
| 3965 | // In an inbounds GEP with an offset that is nonnegative even when | ||||||||
| 3966 | // interpreted as signed, assume there is no unsigned overflow. | ||||||||
| 3967 | SDNodeFlags Flags; | ||||||||
| 3968 | if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds()) | ||||||||
| 3969 | Flags.setNoUnsignedWrap(true); | ||||||||
| 3970 | |||||||||
| 3971 | N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, | ||||||||
| 3972 | DAG.getConstant(Offset, dl, N.getValueType()), Flags); | ||||||||
| 3973 | } | ||||||||
| 3974 | } else { | ||||||||
| 3975 | // IdxSize is the width of the arithmetic according to IR semantics. | ||||||||
| 3976 | // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth | ||||||||
| 3977 | // (and fix up the result later). | ||||||||
| 3978 | unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS); | ||||||||
| 3979 | MVT IdxTy = MVT::getIntegerVT(IdxSize); | ||||||||
| 3980 | TypeSize ElementSize = | ||||||||
| 3981 | DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType()); | ||||||||
| 3982 | // We intentionally mask away the high bits here; ElementSize may not | ||||||||
| 3983 | // fit in IdxTy. | ||||||||
| 3984 | APInt ElementMul(IdxSize, ElementSize.getKnownMinValue()); | ||||||||
| 3985 | bool ElementScalable = ElementSize.isScalable(); | ||||||||
| 3986 | |||||||||
| 3987 | // If this is a scalar constant or a splat vector of constants, | ||||||||
| 3988 | // handle it quickly. | ||||||||
| 3989 | const auto *C = dyn_cast<Constant>(Idx); | ||||||||
| 3990 | if (C && isa<VectorType>(C->getType())) | ||||||||
| 3991 | C = C->getSplatValue(); | ||||||||
| 3992 | |||||||||
| 3993 | const auto *CI = dyn_cast_or_null<ConstantInt>(C); | ||||||||
| 3994 | if (CI && CI->isZero()) | ||||||||
| 3995 | continue; | ||||||||
| 3996 | if (CI && !ElementScalable) { | ||||||||
| 3997 | APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize); | ||||||||
| 3998 | LLVMContext &Context = *DAG.getContext(); | ||||||||
| 3999 | SDValue OffsVal; | ||||||||
| 4000 | if (IsVectorGEP) | ||||||||
| 4001 | OffsVal = DAG.getConstant( | ||||||||
| 4002 | Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount)); | ||||||||
| 4003 | else | ||||||||
| 4004 | OffsVal = DAG.getConstant(Offs, dl, IdxTy); | ||||||||
| 4005 | |||||||||
| 4006 | // In an inbounds GEP with an offset that is nonnegative even when | ||||||||
| 4007 | // interpreted as signed, assume there is no unsigned overflow. | ||||||||
| 4008 | SDNodeFlags Flags; | ||||||||
| 4009 | if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds()) | ||||||||
| 4010 | Flags.setNoUnsignedWrap(true); | ||||||||
| 4011 | |||||||||
| 4012 | OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType()); | ||||||||
| 4013 | |||||||||
| 4014 | N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags); | ||||||||
| 4015 | continue; | ||||||||
| 4016 | } | ||||||||
| 4017 | |||||||||
| 4018 | // N = N + Idx * ElementMul; | ||||||||
| 4019 | SDValue IdxN = getValue(Idx); | ||||||||
| 4020 | |||||||||
| 4021 | if (!IdxN.getValueType().isVector() && IsVectorGEP) { | ||||||||
| 4022 | EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), | ||||||||
| 4023 | VectorElementCount); | ||||||||
| 4024 | IdxN = DAG.getSplat(VT, dl, IdxN); | ||||||||
| 4025 | } | ||||||||
| 4026 | |||||||||
| 4027 | // If the index is smaller or larger than intptr_t, truncate or extend | ||||||||
| 4028 | // it. | ||||||||
| 4029 | IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType()); | ||||||||
| 4030 | |||||||||
| 4031 | if (ElementScalable) { | ||||||||
| 4032 | EVT VScaleTy = N.getValueType().getScalarType(); | ||||||||
| 4033 | SDValue VScale = DAG.getNode( | ||||||||
| 4034 | ISD::VSCALE, dl, VScaleTy, | ||||||||
| 4035 | DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy)); | ||||||||
| 4036 | if (IsVectorGEP) | ||||||||
| 4037 | VScale = DAG.getSplatVector(N.getValueType(), dl, VScale); | ||||||||
| 4038 | IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale); | ||||||||
| 4039 | } else { | ||||||||
| 4040 | // If this is a multiply by a power of two, turn it into a shl | ||||||||
| 4041 | // immediately. This is a very common case. | ||||||||
| 4042 | if (ElementMul != 1) { | ||||||||
| 4043 | if (ElementMul.isPowerOf2()) { | ||||||||
| 4044 | unsigned Amt = ElementMul.logBase2(); | ||||||||
| 4045 | IdxN = DAG.getNode(ISD::SHL, dl, | ||||||||
| 4046 | N.getValueType(), IdxN, | ||||||||
| 4047 | DAG.getConstant(Amt, dl, IdxN.getValueType())); | ||||||||
| 4048 | } else { | ||||||||
| 4049 | SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl, | ||||||||
| 4050 | IdxN.getValueType()); | ||||||||
| 4051 | IdxN = DAG.getNode(ISD::MUL, dl, | ||||||||
| 4052 | N.getValueType(), IdxN, Scale); | ||||||||
| 4053 | } | ||||||||
| 4054 | } | ||||||||
| 4055 | } | ||||||||
| 4056 | |||||||||
| 4057 | N = DAG.getNode(ISD::ADD, dl, | ||||||||
| 4058 | N.getValueType(), N, IdxN); | ||||||||
| 4059 | } | ||||||||
| 4060 | } | ||||||||
| 4061 | |||||||||
| 4062 | MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS); | ||||||||
| 4063 | MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS); | ||||||||
| 4064 | if (IsVectorGEP) { | ||||||||
| 4065 | PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount); | ||||||||
| 4066 | PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount); | ||||||||
| 4067 | } | ||||||||
| 4068 | |||||||||
| 4069 | if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds()) | ||||||||
| 4070 | N = DAG.getPtrExtendInReg(N, dl, PtrMemTy); | ||||||||
| 4071 | |||||||||
| 4072 | setValue(&I, N); | ||||||||
| 4073 | } | ||||||||
| 4074 | |||||||||
| 4075 | void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { | ||||||||
| 4076 | // If this is a fixed sized alloca in the entry block of the function, | ||||||||
| 4077 | // allocate it statically on the stack. | ||||||||
| 4078 | if (FuncInfo.StaticAllocaMap.count(&I)) | ||||||||
| 4079 | return; // getValue will auto-populate this. | ||||||||
| 4080 | |||||||||
| 4081 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4082 | Type *Ty = I.getAllocatedType(); | ||||||||
| 4083 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4084 | auto &DL = DAG.getDataLayout(); | ||||||||
| 4085 | TypeSize TySize = DL.getTypeAllocSize(Ty); | ||||||||
| 4086 | MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign()); | ||||||||
| 4087 | |||||||||
| 4088 | SDValue AllocSize = getValue(I.getArraySize()); | ||||||||
| 4089 | |||||||||
| 4090 | EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace()); | ||||||||
| 4091 | if (AllocSize.getValueType() != IntPtr) | ||||||||
| 4092 | AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr); | ||||||||
| 4093 | |||||||||
| 4094 | if (TySize.isScalable()) | ||||||||
| 4095 | AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, | ||||||||
| 4096 | DAG.getVScale(dl, IntPtr, | ||||||||
| 4097 | APInt(IntPtr.getScalarSizeInBits(), | ||||||||
| 4098 | TySize.getKnownMinValue()))); | ||||||||
| 4099 | else | ||||||||
| 4100 | AllocSize = | ||||||||
| 4101 | DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, | ||||||||
| 4102 | DAG.getConstant(TySize.getFixedValue(), dl, IntPtr)); | ||||||||
| 4103 | |||||||||
| 4104 | // Handle alignment. If the requested alignment is less than or equal to | ||||||||
| 4105 | // the stack alignment, ignore it. If the size is greater than or equal to | ||||||||
| 4106 | // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. | ||||||||
| 4107 | Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign(); | ||||||||
| 4108 | if (*Alignment <= StackAlign) | ||||||||
| 4109 | Alignment = std::nullopt; | ||||||||
| 4110 | |||||||||
| 4111 | const uint64_t StackAlignMask = StackAlign.value() - 1U; | ||||||||
| 4112 | // Round the size of the allocation up to the stack alignment size | ||||||||
| 4113 | // by add SA-1 to the size. This doesn't overflow because we're computing | ||||||||
| 4114 | // an address inside an alloca. | ||||||||
| 4115 | SDNodeFlags Flags; | ||||||||
| 4116 | Flags.setNoUnsignedWrap(true); | ||||||||
| 4117 | AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize, | ||||||||
| 4118 | DAG.getConstant(StackAlignMask, dl, IntPtr), Flags); | ||||||||
| 4119 | |||||||||
| 4120 | // Mask out the low bits for alignment purposes. | ||||||||
| 4121 | AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize, | ||||||||
| 4122 | DAG.getConstant(~StackAlignMask, dl, IntPtr)); | ||||||||
| 4123 | |||||||||
| 4124 | SDValue Ops[] = { | ||||||||
| 4125 | getRoot(), AllocSize, | ||||||||
| 4126 | DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)}; | ||||||||
| 4127 | SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); | ||||||||
| 4128 | SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops); | ||||||||
| 4129 | setValue(&I, DSA); | ||||||||
| 4130 | DAG.setRoot(DSA.getValue(1)); | ||||||||
| 4131 | |||||||||
| 4132 | assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects ()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4132 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4133 | } | ||||||||
| 4134 | |||||||||
| 4135 | void SelectionDAGBuilder::visitLoad(const LoadInst &I) { | ||||||||
| 4136 | if (I.isAtomic()) | ||||||||
| 4137 | return visitAtomicLoad(I); | ||||||||
| 4138 | |||||||||
| 4139 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4140 | const Value *SV = I.getOperand(0); | ||||||||
| 4141 | if (TLI.supportSwiftError()) { | ||||||||
| 4142 | // Swifterror values can come from either a function parameter with | ||||||||
| 4143 | // swifterror attribute or an alloca with swifterror attribute. | ||||||||
| 4144 | if (const Argument *Arg = dyn_cast<Argument>(SV)) { | ||||||||
| 4145 | if (Arg->hasSwiftErrorAttr()) | ||||||||
| 4146 | return visitLoadFromSwiftError(I); | ||||||||
| 4147 | } | ||||||||
| 4148 | |||||||||
| 4149 | if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) { | ||||||||
| 4150 | if (Alloca->isSwiftError()) | ||||||||
| 4151 | return visitLoadFromSwiftError(I); | ||||||||
| 4152 | } | ||||||||
| 4153 | } | ||||||||
| 4154 | |||||||||
| 4155 | SDValue Ptr = getValue(SV); | ||||||||
| 4156 | |||||||||
| 4157 | Type *Ty = I.getType(); | ||||||||
| 4158 | SmallVector<EVT, 4> ValueVTs, MemVTs; | ||||||||
| 4159 | SmallVector<uint64_t, 4> Offsets; | ||||||||
| 4160 | ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets); | ||||||||
| 4161 | unsigned NumValues = ValueVTs.size(); | ||||||||
| 4162 | if (NumValues == 0) | ||||||||
| 4163 | return; | ||||||||
| 4164 | |||||||||
| 4165 | Align Alignment = I.getAlign(); | ||||||||
| 4166 | AAMDNodes AAInfo = I.getAAMetadata(); | ||||||||
| 4167 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); | ||||||||
| 4168 | bool isVolatile = I.isVolatile(); | ||||||||
| 4169 | MachineMemOperand::Flags MMOFlags = | ||||||||
| 4170 | TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); | ||||||||
| 4171 | |||||||||
| 4172 | SDValue Root; | ||||||||
| 4173 | bool ConstantMemory = false; | ||||||||
| 4174 | if (isVolatile) | ||||||||
| 4175 | // Serialize volatile loads with other side effects. | ||||||||
| 4176 | Root = getRoot(); | ||||||||
| 4177 | else if (NumValues > MaxParallelChains) | ||||||||
| 4178 | Root = getMemoryRoot(); | ||||||||
| 4179 | else if (AA && | ||||||||
| 4180 | AA->pointsToConstantMemory(MemoryLocation( | ||||||||
| 4181 | SV, | ||||||||
| 4182 | LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), | ||||||||
| 4183 | AAInfo))) { | ||||||||
| 4184 | // Do not serialize (non-volatile) loads of constant memory with anything. | ||||||||
| 4185 | Root = DAG.getEntryNode(); | ||||||||
| 4186 | ConstantMemory = true; | ||||||||
| 4187 | MMOFlags |= MachineMemOperand::MOInvariant; | ||||||||
| 4188 | } else { | ||||||||
| 4189 | // Do not serialize non-volatile loads against each other. | ||||||||
| 4190 | Root = DAG.getRoot(); | ||||||||
| 4191 | } | ||||||||
| 4192 | |||||||||
| 4193 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4194 | |||||||||
| 4195 | if (isVolatile) | ||||||||
| 4196 | Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG); | ||||||||
| 4197 | |||||||||
| 4198 | // An aggregate load cannot wrap around the address space, so offsets to its | ||||||||
| 4199 | // parts don't wrap either. | ||||||||
| 4200 | SDNodeFlags Flags; | ||||||||
| 4201 | Flags.setNoUnsignedWrap(true); | ||||||||
| 4202 | |||||||||
| 4203 | SmallVector<SDValue, 4> Values(NumValues); | ||||||||
| 4204 | SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); | ||||||||
| 4205 | EVT PtrVT = Ptr.getValueType(); | ||||||||
| 4206 | |||||||||
| 4207 | unsigned ChainI = 0; | ||||||||
| 4208 | for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { | ||||||||
| 4209 | // Serializing loads here may result in excessive register pressure, and | ||||||||
| 4210 | // TokenFactor places arbitrary choke points on the scheduler. SD scheduling | ||||||||
| 4211 | // could recover a bit by hoisting nodes upward in the chain by recognizing | ||||||||
| 4212 | // they are side-effect free or do not alias. The optimizer should really | ||||||||
| 4213 | // avoid this case by converting large object/array copies to llvm.memcpy | ||||||||
| 4214 | // (MaxParallelChains should always remain as failsafe). | ||||||||
| 4215 | if (ChainI == MaxParallelChains) { | ||||||||
| 4216 | assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first" ) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4216 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4217 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||
| 4218 | ArrayRef(Chains.data(), ChainI)); | ||||||||
| 4219 | Root = Chain; | ||||||||
| 4220 | ChainI = 0; | ||||||||
| 4221 | } | ||||||||
| 4222 | SDValue A = DAG.getNode(ISD::ADD, dl, | ||||||||
| 4223 | PtrVT, Ptr, | ||||||||
| 4224 | DAG.getConstant(Offsets[i], dl, PtrVT), | ||||||||
| 4225 | Flags); | ||||||||
| 4226 | |||||||||
| 4227 | SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, | ||||||||
| 4228 | MachinePointerInfo(SV, Offsets[i]), Alignment, | ||||||||
| 4229 | MMOFlags, AAInfo, Ranges); | ||||||||
| 4230 | Chains[ChainI] = L.getValue(1); | ||||||||
| 4231 | |||||||||
| 4232 | if (MemVTs[i] != ValueVTs[i]) | ||||||||
| 4233 | L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]); | ||||||||
| 4234 | |||||||||
| 4235 | Values[i] = L; | ||||||||
| 4236 | } | ||||||||
| 4237 | |||||||||
| 4238 | if (!ConstantMemory) { | ||||||||
| 4239 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||
| 4240 | ArrayRef(Chains.data(), ChainI)); | ||||||||
| 4241 | if (isVolatile) | ||||||||
| 4242 | DAG.setRoot(Chain); | ||||||||
| 4243 | else | ||||||||
| 4244 | PendingLoads.push_back(Chain); | ||||||||
| 4245 | } | ||||||||
| 4246 | |||||||||
| 4247 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl, | ||||||||
| 4248 | DAG.getVTList(ValueVTs), Values)); | ||||||||
| 4249 | } | ||||||||
| 4250 | |||||||||
| 4251 | void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) { | ||||||||
| 4252 | assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitStoreToSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4253 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4253 | "call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitStoreToSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4253 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4254 | |||||||||
| 4255 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 4256 | SmallVector<uint64_t, 4> Offsets; | ||||||||
| 4257 | const Value *SrcV = I.getOperand(0); | ||||||||
| 4258 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), | ||||||||
| 4259 | SrcV->getType(), ValueVTs, &Offsets); | ||||||||
| 4260 | assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4261 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4261 | "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4261 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4262 | |||||||||
| 4263 | SDValue Src = getValue(SrcV); | ||||||||
| 4264 | // Create a virtual register, then update the virtual register. | ||||||||
| 4265 | Register VReg = | ||||||||
| 4266 | SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand()); | ||||||||
| 4267 | // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue | ||||||||
| 4268 | // Chain can be getRoot or getControlRoot. | ||||||||
| 4269 | SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg, | ||||||||
| 4270 | SDValue(Src.getNode(), Src.getResNo())); | ||||||||
| 4271 | DAG.setRoot(CopyNode); | ||||||||
| 4272 | } | ||||||||
| 4273 | |||||||||
| 4274 | void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { | ||||||||
| 4275 | assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitLoadFromSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4276 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4276 | "call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError () && "call visitLoadFromSwiftError when backend supports swifterror" ) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4276 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4277 | |||||||||
| 4278 | assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4279 | !I.hasMetadata(LLVMContext::MD_nontemporal) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4280 | !I.hasMetadata(LLVMContext::MD_invariant_load) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4281 | "Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && !I.hasMetadata (LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext ::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error" ) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4282 | |||||||||
| 4283 | const Value *SV = I.getOperand(0); | ||||||||
| 4284 | Type *Ty = I.getType(); | ||||||||
| 4285 | assert((static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4286 | (!AA ||(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4287 | !AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4288 | SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4289 | I.getAAMetadata()))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4290 | "load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory (MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout( ).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory" ) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4291 | |||||||||
| 4292 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 4293 | SmallVector<uint64_t, 4> Offsets; | ||||||||
| 4294 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty, | ||||||||
| 4295 | ValueVTs, &Offsets); | ||||||||
| 4296 | assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4297 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4297 | "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets [0] == 0 && "expect a single EVT for swifterror") ? void (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4297 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4298 | |||||||||
| 4299 | // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT | ||||||||
| 4300 | SDValue L = DAG.getCopyFromReg( | ||||||||
| 4301 | getRoot(), getCurSDLoc(), | ||||||||
| 4302 | SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]); | ||||||||
| 4303 | |||||||||
| 4304 | setValue(&I, L); | ||||||||
| 4305 | } | ||||||||
| 4306 | |||||||||
| 4307 | void SelectionDAGBuilder::visitStore(const StoreInst &I) { | ||||||||
| 4308 | if (I.isAtomic()) | ||||||||
| 4309 | return visitAtomicStore(I); | ||||||||
| 4310 | |||||||||
| 4311 | const Value *SrcV = I.getOperand(0); | ||||||||
| 4312 | const Value *PtrV = I.getOperand(1); | ||||||||
| 4313 | |||||||||
| 4314 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4315 | if (TLI.supportSwiftError()) { | ||||||||
| 4316 | // Swifterror values can come from either a function parameter with | ||||||||
| 4317 | // swifterror attribute or an alloca with swifterror attribute. | ||||||||
| 4318 | if (const Argument *Arg = dyn_cast<Argument>(PtrV)) { | ||||||||
| 4319 | if (Arg->hasSwiftErrorAttr()) | ||||||||
| 4320 | return visitStoreToSwiftError(I); | ||||||||
| 4321 | } | ||||||||
| 4322 | |||||||||
| 4323 | if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) { | ||||||||
| 4324 | if (Alloca->isSwiftError()) | ||||||||
| 4325 | return visitStoreToSwiftError(I); | ||||||||
| 4326 | } | ||||||||
| 4327 | } | ||||||||
| 4328 | |||||||||
| 4329 | SmallVector<EVT, 4> ValueVTs, MemVTs; | ||||||||
| 4330 | SmallVector<uint64_t, 4> Offsets; | ||||||||
| 4331 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), | ||||||||
| 4332 | SrcV->getType(), ValueVTs, &MemVTs, &Offsets); | ||||||||
| 4333 | unsigned NumValues = ValueVTs.size(); | ||||||||
| 4334 | if (NumValues == 0) | ||||||||
| 4335 | return; | ||||||||
| 4336 | |||||||||
| 4337 | // Get the lowered operands. Note that we do this after | ||||||||
| 4338 | // checking if NumResults is zero, because with zero results | ||||||||
| 4339 | // the operands won't have values in the map. | ||||||||
| 4340 | SDValue Src = getValue(SrcV); | ||||||||
| 4341 | SDValue Ptr = getValue(PtrV); | ||||||||
| 4342 | |||||||||
| 4343 | SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot(); | ||||||||
| 4344 | SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); | ||||||||
| 4345 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4346 | Align Alignment = I.getAlign(); | ||||||||
| 4347 | AAMDNodes AAInfo = I.getAAMetadata(); | ||||||||
| 4348 | |||||||||
| 4349 | auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); | ||||||||
| 4350 | |||||||||
| 4351 | // An aggregate load cannot wrap around the address space, so offsets to its | ||||||||
| 4352 | // parts don't wrap either. | ||||||||
| 4353 | SDNodeFlags Flags; | ||||||||
| 4354 | Flags.setNoUnsignedWrap(true); | ||||||||
| 4355 | |||||||||
| 4356 | unsigned ChainI = 0; | ||||||||
| 4357 | for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { | ||||||||
| 4358 | // See visitLoad comments. | ||||||||
| 4359 | if (ChainI == MaxParallelChains) { | ||||||||
| 4360 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||
| 4361 | ArrayRef(Chains.data(), ChainI)); | ||||||||
| 4362 | Root = Chain; | ||||||||
| 4363 | ChainI = 0; | ||||||||
| 4364 | } | ||||||||
| 4365 | SDValue Add = | ||||||||
| 4366 | DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); | ||||||||
| 4367 | SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); | ||||||||
| 4368 | if (MemVTs[i] != ValueVTs[i]) | ||||||||
| 4369 | Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); | ||||||||
| 4370 | SDValue St = | ||||||||
| 4371 | DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]), | ||||||||
| 4372 | Alignment, MMOFlags, AAInfo); | ||||||||
| 4373 | Chains[ChainI] = St; | ||||||||
| 4374 | } | ||||||||
| 4375 | |||||||||
| 4376 | SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, | ||||||||
| 4377 | ArrayRef(Chains.data(), ChainI)); | ||||||||
| 4378 | setValue(&I, StoreNode); | ||||||||
| 4379 | DAG.setRoot(StoreNode); | ||||||||
| 4380 | } | ||||||||
| 4381 | |||||||||
| 4382 | void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, | ||||||||
| 4383 | bool IsCompressing) { | ||||||||
| 4384 | SDLoc sdl = getCurSDLoc(); | ||||||||
| 4385 | |||||||||
| 4386 | auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||
| 4387 | MaybeAlign &Alignment) { | ||||||||
| 4388 | // llvm.masked.store.*(Src0, Ptr, alignment, Mask) | ||||||||
| 4389 | Src0 = I.getArgOperand(0); | ||||||||
| 4390 | Ptr = I.getArgOperand(1); | ||||||||
| 4391 | Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue(); | ||||||||
| 4392 | Mask = I.getArgOperand(3); | ||||||||
| 4393 | }; | ||||||||
| 4394 | auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||
| 4395 | MaybeAlign &Alignment) { | ||||||||
| 4396 | // llvm.masked.compressstore.*(Src0, Ptr, Mask) | ||||||||
| 4397 | Src0 = I.getArgOperand(0); | ||||||||
| 4398 | Ptr = I.getArgOperand(1); | ||||||||
| 4399 | Mask = I.getArgOperand(2); | ||||||||
| 4400 | Alignment = std::nullopt; | ||||||||
| 4401 | }; | ||||||||
| 4402 | |||||||||
| 4403 | Value *PtrOperand, *MaskOperand, *Src0Operand; | ||||||||
| 4404 | MaybeAlign Alignment; | ||||||||
| 4405 | if (IsCompressing) | ||||||||
| 4406 | getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||
| 4407 | else | ||||||||
| 4408 | getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||
| 4409 | |||||||||
| 4410 | SDValue Ptr = getValue(PtrOperand); | ||||||||
| 4411 | SDValue Src0 = getValue(Src0Operand); | ||||||||
| 4412 | SDValue Mask = getValue(MaskOperand); | ||||||||
| 4413 | SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); | ||||||||
| 4414 | |||||||||
| 4415 | EVT VT = Src0.getValueType(); | ||||||||
| 4416 | if (!Alignment) | ||||||||
| 4417 | Alignment = DAG.getEVTAlign(VT); | ||||||||
| 4418 | |||||||||
| 4419 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 4420 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, | ||||||||
| 4421 | MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata()); | ||||||||
| 4422 | SDValue StoreNode = | ||||||||
| 4423 | DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO, | ||||||||
| 4424 | ISD::UNINDEXED, false /* Truncating */, IsCompressing); | ||||||||
| 4425 | DAG.setRoot(StoreNode); | ||||||||
| 4426 | setValue(&I, StoreNode); | ||||||||
| 4427 | } | ||||||||
| 4428 | |||||||||
| 4429 | // Get a uniform base for the Gather/Scatter intrinsic. | ||||||||
| 4430 | // The first argument of the Gather/Scatter intrinsic is a vector of pointers. | ||||||||
| 4431 | // We try to represent it as a base pointer + vector of indices. | ||||||||
| 4432 | // Usually, the vector of pointers comes from a 'getelementptr' instruction. | ||||||||
| 4433 | // The first operand of the GEP may be a single pointer or a vector of pointers | ||||||||
| 4434 | // Example: | ||||||||
| 4435 | // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind | ||||||||
| 4436 | // or | ||||||||
| 4437 | // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind | ||||||||
| 4438 | // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, .. | ||||||||
| 4439 | // | ||||||||
| 4440 | // When the first GEP operand is a single pointer - it is the uniform base we | ||||||||
| 4441 | // are looking for. If first operand of the GEP is a splat vector - we | ||||||||
| 4442 | // extract the splat value and use it as a uniform base. | ||||||||
| 4443 | // In all other cases the function returns 'false'. | ||||||||
| 4444 | static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, | ||||||||
| 4445 | ISD::MemIndexType &IndexType, SDValue &Scale, | ||||||||
| 4446 | SelectionDAGBuilder *SDB, const BasicBlock *CurBB, | ||||||||
| 4447 | uint64_t ElemSize) { | ||||||||
| 4448 | SelectionDAG& DAG = SDB->DAG; | ||||||||
| 4449 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4450 | const DataLayout &DL = DAG.getDataLayout(); | ||||||||
| 4451 | |||||||||
| 4452 | assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy() && "Unexpected pointer type") ? void (0) : __assert_fail ("Ptr->getType()->isVectorTy() && \"Unexpected pointer type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4452 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4453 | |||||||||
| 4454 | // Handle splat constant pointer. | ||||||||
| 4455 | if (auto *C = dyn_cast<Constant>(Ptr)) { | ||||||||
| 4456 | C = C->getSplatValue(); | ||||||||
| 4457 | if (!C) | ||||||||
| 4458 | return false; | ||||||||
| 4459 | |||||||||
| 4460 | Base = SDB->getValue(C); | ||||||||
| 4461 | |||||||||
| 4462 | ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount(); | ||||||||
| 4463 | EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts); | ||||||||
| 4464 | Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT); | ||||||||
| 4465 | IndexType = ISD::SIGNED_SCALED; | ||||||||
| 4466 | Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); | ||||||||
| 4467 | return true; | ||||||||
| 4468 | } | ||||||||
| 4469 | |||||||||
| 4470 | const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr); | ||||||||
| 4471 | if (!GEP || GEP->getParent() != CurBB) | ||||||||
| 4472 | return false; | ||||||||
| 4473 | |||||||||
| 4474 | if (GEP->getNumOperands() != 2) | ||||||||
| 4475 | return false; | ||||||||
| 4476 | |||||||||
| 4477 | const Value *BasePtr = GEP->getPointerOperand(); | ||||||||
| 4478 | const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1); | ||||||||
| 4479 | |||||||||
| 4480 | // Make sure the base is scalar and the index is a vector. | ||||||||
| 4481 | if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy()) | ||||||||
| 4482 | return false; | ||||||||
| 4483 | |||||||||
| 4484 | uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType()); | ||||||||
| 4485 | |||||||||
| 4486 | // Target may not support the required addressing mode. | ||||||||
| 4487 | if (ScaleVal != 1 && | ||||||||
| 4488 | !TLI.isLegalScaleForGatherScatter(ScaleVal, ElemSize)) | ||||||||
| 4489 | return false; | ||||||||
| 4490 | |||||||||
| 4491 | Base = SDB->getValue(BasePtr); | ||||||||
| 4492 | Index = SDB->getValue(IndexVal); | ||||||||
| 4493 | IndexType = ISD::SIGNED_SCALED; | ||||||||
| 4494 | |||||||||
| 4495 | Scale = | ||||||||
| 4496 | DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); | ||||||||
| 4497 | return true; | ||||||||
| 4498 | } | ||||||||
| 4499 | |||||||||
| 4500 | void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { | ||||||||
| 4501 | SDLoc sdl = getCurSDLoc(); | ||||||||
| 4502 | |||||||||
| 4503 | // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask) | ||||||||
| 4504 | const Value *Ptr = I.getArgOperand(1); | ||||||||
| 4505 | SDValue Src0 = getValue(I.getArgOperand(0)); | ||||||||
| 4506 | SDValue Mask = getValue(I.getArgOperand(3)); | ||||||||
| 4507 | EVT VT = Src0.getValueType(); | ||||||||
| 4508 | Align Alignment = cast<ConstantInt>(I.getArgOperand(2)) | ||||||||
| 4509 | ->getMaybeAlignValue() | ||||||||
| 4510 | .value_or(DAG.getEVTAlign(VT.getScalarType())); | ||||||||
| 4511 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4512 | |||||||||
| 4513 | SDValue Base; | ||||||||
| 4514 | SDValue Index; | ||||||||
| 4515 | ISD::MemIndexType IndexType; | ||||||||
| 4516 | SDValue Scale; | ||||||||
| 4517 | bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, | ||||||||
| 4518 | I.getParent(), VT.getScalarStoreSize()); | ||||||||
| 4519 | |||||||||
| 4520 | unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||
| 4521 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 4522 | MachinePointerInfo(AS), MachineMemOperand::MOStore, | ||||||||
| 4523 | // TODO: Make MachineMemOperands aware of scalable | ||||||||
| 4524 | // vectors. | ||||||||
| 4525 | MemoryLocation::UnknownSize, Alignment, I.getAAMetadata()); | ||||||||
| 4526 | if (!UniformBase) { | ||||||||
| 4527 | Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 4528 | Index = getValue(Ptr); | ||||||||
| 4529 | IndexType = ISD::SIGNED_SCALED; | ||||||||
| 4530 | Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 4531 | } | ||||||||
| 4532 | |||||||||
| 4533 | EVT IdxVT = Index.getValueType(); | ||||||||
| 4534 | EVT EltTy = IdxVT.getVectorElementType(); | ||||||||
| 4535 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { | ||||||||
| 4536 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); | ||||||||
| 4537 | Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); | ||||||||
| 4538 | } | ||||||||
| 4539 | |||||||||
| 4540 | SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale }; | ||||||||
| 4541 | SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl, | ||||||||
| 4542 | Ops, MMO, IndexType, false); | ||||||||
| 4543 | DAG.setRoot(Scatter); | ||||||||
| 4544 | setValue(&I, Scatter); | ||||||||
| 4545 | } | ||||||||
| 4546 | |||||||||
| 4547 | void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { | ||||||||
| 4548 | SDLoc sdl = getCurSDLoc(); | ||||||||
| 4549 | |||||||||
| 4550 | auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||
| 4551 | MaybeAlign &Alignment) { | ||||||||
| 4552 | // @llvm.masked.load.*(Ptr, alignment, Mask, Src0) | ||||||||
| 4553 | Ptr = I.getArgOperand(0); | ||||||||
| 4554 | Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue(); | ||||||||
| 4555 | Mask = I.getArgOperand(2); | ||||||||
| 4556 | Src0 = I.getArgOperand(3); | ||||||||
| 4557 | }; | ||||||||
| 4558 | auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, | ||||||||
| 4559 | MaybeAlign &Alignment) { | ||||||||
| 4560 | // @llvm.masked.expandload.*(Ptr, Mask, Src0) | ||||||||
| 4561 | Ptr = I.getArgOperand(0); | ||||||||
| 4562 | Alignment = std::nullopt; | ||||||||
| 4563 | Mask = I.getArgOperand(1); | ||||||||
| 4564 | Src0 = I.getArgOperand(2); | ||||||||
| 4565 | }; | ||||||||
| 4566 | |||||||||
| 4567 | Value *PtrOperand, *MaskOperand, *Src0Operand; | ||||||||
| 4568 | MaybeAlign Alignment; | ||||||||
| 4569 | if (IsExpanding) | ||||||||
| 4570 | getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||
| 4571 | else | ||||||||
| 4572 | getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); | ||||||||
| 4573 | |||||||||
| 4574 | SDValue Ptr = getValue(PtrOperand); | ||||||||
| 4575 | SDValue Src0 = getValue(Src0Operand); | ||||||||
| 4576 | SDValue Mask = getValue(MaskOperand); | ||||||||
| 4577 | SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); | ||||||||
| 4578 | |||||||||
| 4579 | EVT VT = Src0.getValueType(); | ||||||||
| 4580 | if (!Alignment) | ||||||||
| 4581 | Alignment = DAG.getEVTAlign(VT); | ||||||||
| 4582 | |||||||||
| 4583 | AAMDNodes AAInfo = I.getAAMetadata(); | ||||||||
| 4584 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); | ||||||||
| 4585 | |||||||||
| 4586 | // Do not serialize masked loads of constant memory with anything. | ||||||||
| 4587 | MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); | ||||||||
| 4588 | bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); | ||||||||
| 4589 | |||||||||
| 4590 | SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); | ||||||||
| 4591 | |||||||||
| 4592 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 4593 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, | ||||||||
| 4594 | MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); | ||||||||
| 4595 | |||||||||
| 4596 | SDValue Load = | ||||||||
| 4597 | DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO, | ||||||||
| 4598 | ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding); | ||||||||
| 4599 | if (AddToChain) | ||||||||
| 4600 | PendingLoads.push_back(Load.getValue(1)); | ||||||||
| 4601 | setValue(&I, Load); | ||||||||
| 4602 | } | ||||||||
| 4603 | |||||||||
| 4604 | void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { | ||||||||
| 4605 | SDLoc sdl = getCurSDLoc(); | ||||||||
| 4606 | |||||||||
| 4607 | // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) | ||||||||
| 4608 | const Value *Ptr = I.getArgOperand(0); | ||||||||
| 4609 | SDValue Src0 = getValue(I.getArgOperand(3)); | ||||||||
| 4610 | SDValue Mask = getValue(I.getArgOperand(2)); | ||||||||
| 4611 | |||||||||
| 4612 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4613 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 4614 | Align Alignment = cast<ConstantInt>(I.getArgOperand(1)) | ||||||||
| 4615 | ->getMaybeAlignValue() | ||||||||
| 4616 | .value_or(DAG.getEVTAlign(VT.getScalarType())); | ||||||||
| 4617 | |||||||||
| 4618 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); | ||||||||
| 4619 | |||||||||
| 4620 | SDValue Root = DAG.getRoot(); | ||||||||
| 4621 | SDValue Base; | ||||||||
| 4622 | SDValue Index; | ||||||||
| 4623 | ISD::MemIndexType IndexType; | ||||||||
| 4624 | SDValue Scale; | ||||||||
| 4625 | bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, | ||||||||
| 4626 | I.getParent(), VT.getScalarStoreSize()); | ||||||||
| 4627 | unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||
| 4628 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 4629 | MachinePointerInfo(AS), MachineMemOperand::MOLoad, | ||||||||
| 4630 | // TODO: Make MachineMemOperands aware of scalable | ||||||||
| 4631 | // vectors. | ||||||||
| 4632 | MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges); | ||||||||
| 4633 | |||||||||
| 4634 | if (!UniformBase) { | ||||||||
| 4635 | Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 4636 | Index = getValue(Ptr); | ||||||||
| 4637 | IndexType = ISD::SIGNED_SCALED; | ||||||||
| 4638 | Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 4639 | } | ||||||||
| 4640 | |||||||||
| 4641 | EVT IdxVT = Index.getValueType(); | ||||||||
| 4642 | EVT EltTy = IdxVT.getVectorElementType(); | ||||||||
| 4643 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { | ||||||||
| 4644 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); | ||||||||
| 4645 | Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); | ||||||||
| 4646 | } | ||||||||
| 4647 | |||||||||
| 4648 | SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale }; | ||||||||
| 4649 | SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, | ||||||||
| 4650 | Ops, MMO, IndexType, ISD::NON_EXTLOAD); | ||||||||
| 4651 | |||||||||
| 4652 | PendingLoads.push_back(Gather.getValue(1)); | ||||||||
| 4653 | setValue(&I, Gather); | ||||||||
| 4654 | } | ||||||||
| 4655 | |||||||||
| 4656 | void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { | ||||||||
| 4657 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4658 | AtomicOrdering SuccessOrdering = I.getSuccessOrdering(); | ||||||||
| 4659 | AtomicOrdering FailureOrdering = I.getFailureOrdering(); | ||||||||
| 4660 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||
| 4661 | |||||||||
| 4662 | SDValue InChain = getRoot(); | ||||||||
| 4663 | |||||||||
| 4664 | MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType(); | ||||||||
| 4665 | SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); | ||||||||
| 4666 | |||||||||
| 4667 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4668 | auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); | ||||||||
| 4669 | |||||||||
| 4670 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 4671 | MachineMemOperand *MMO = MF.getMachineMemOperand( | ||||||||
| 4672 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||
| 4673 | DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering, | ||||||||
| 4674 | FailureOrdering); | ||||||||
| 4675 | |||||||||
| 4676 | SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, | ||||||||
| 4677 | dl, MemVT, VTs, InChain, | ||||||||
| 4678 | getValue(I.getPointerOperand()), | ||||||||
| 4679 | getValue(I.getCompareOperand()), | ||||||||
| 4680 | getValue(I.getNewValOperand()), MMO); | ||||||||
| 4681 | |||||||||
| 4682 | SDValue OutChain = L.getValue(2); | ||||||||
| 4683 | |||||||||
| 4684 | setValue(&I, L); | ||||||||
| 4685 | DAG.setRoot(OutChain); | ||||||||
| 4686 | } | ||||||||
| 4687 | |||||||||
| 4688 | void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { | ||||||||
| 4689 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4690 | ISD::NodeType NT; | ||||||||
| 4691 | switch (I.getOperation()) { | ||||||||
| 4692 | default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4692 ); | ||||||||
| 4693 | case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; | ||||||||
| 4694 | case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; | ||||||||
| 4695 | case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; | ||||||||
| 4696 | case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; | ||||||||
| 4697 | case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; | ||||||||
| 4698 | case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; | ||||||||
| 4699 | case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; | ||||||||
| 4700 | case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; | ||||||||
| 4701 | case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; | ||||||||
| 4702 | case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; | ||||||||
| 4703 | case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; | ||||||||
| 4704 | case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break; | ||||||||
| 4705 | case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break; | ||||||||
| 4706 | case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break; | ||||||||
| 4707 | case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break; | ||||||||
| 4708 | case AtomicRMWInst::UIncWrap: | ||||||||
| 4709 | NT = ISD::ATOMIC_LOAD_UINC_WRAP; | ||||||||
| 4710 | break; | ||||||||
| 4711 | case AtomicRMWInst::UDecWrap: | ||||||||
| 4712 | NT = ISD::ATOMIC_LOAD_UDEC_WRAP; | ||||||||
| 4713 | break; | ||||||||
| 4714 | } | ||||||||
| 4715 | AtomicOrdering Ordering = I.getOrdering(); | ||||||||
| 4716 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||
| 4717 | |||||||||
| 4718 | SDValue InChain = getRoot(); | ||||||||
| 4719 | |||||||||
| 4720 | auto MemVT = getValue(I.getValOperand()).getSimpleValueType(); | ||||||||
| 4721 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4722 | auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); | ||||||||
| 4723 | |||||||||
| 4724 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 4725 | MachineMemOperand *MMO = MF.getMachineMemOperand( | ||||||||
| 4726 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||
| 4727 | DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering); | ||||||||
| 4728 | |||||||||
| 4729 | SDValue L = | ||||||||
| 4730 | DAG.getAtomic(NT, dl, MemVT, InChain, | ||||||||
| 4731 | getValue(I.getPointerOperand()), getValue(I.getValOperand()), | ||||||||
| 4732 | MMO); | ||||||||
| 4733 | |||||||||
| 4734 | SDValue OutChain = L.getValue(1); | ||||||||
| 4735 | |||||||||
| 4736 | setValue(&I, L); | ||||||||
| 4737 | DAG.setRoot(OutChain); | ||||||||
| 4738 | } | ||||||||
| 4739 | |||||||||
| 4740 | void SelectionDAGBuilder::visitFence(const FenceInst &I) { | ||||||||
| 4741 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4742 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4743 | SDValue Ops[3]; | ||||||||
| 4744 | Ops[0] = getRoot(); | ||||||||
| 4745 | Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl, | ||||||||
| 4746 | TLI.getFenceOperandTy(DAG.getDataLayout())); | ||||||||
| 4747 | Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl, | ||||||||
| 4748 | TLI.getFenceOperandTy(DAG.getDataLayout())); | ||||||||
| 4749 | SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops); | ||||||||
| 4750 | setValue(&I, N); | ||||||||
| 4751 | DAG.setRoot(N); | ||||||||
| 4752 | } | ||||||||
| 4753 | |||||||||
| 4754 | void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { | ||||||||
| 4755 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4756 | AtomicOrdering Order = I.getOrdering(); | ||||||||
| 4757 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||
| 4758 | |||||||||
| 4759 | SDValue InChain = getRoot(); | ||||||||
| 4760 | |||||||||
| 4761 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4762 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 4763 | EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 4764 | |||||||||
| 4765 | if (!TLI.supportsUnalignedAtomics() && | ||||||||
| 4766 | I.getAlign().value() < MemVT.getSizeInBits() / 8) | ||||||||
| 4767 | report_fatal_error("Cannot generate unaligned atomic load"); | ||||||||
| 4768 | |||||||||
| 4769 | auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); | ||||||||
| 4770 | |||||||||
| 4771 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 4772 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||
| 4773 | I.getAlign(), AAMDNodes(), nullptr, SSID, Order); | ||||||||
| 4774 | |||||||||
| 4775 | InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); | ||||||||
| 4776 | |||||||||
| 4777 | SDValue Ptr = getValue(I.getPointerOperand()); | ||||||||
| 4778 | |||||||||
| 4779 | if (TLI.lowerAtomicLoadAsLoadSDNode(I)) { | ||||||||
| 4780 | // TODO: Once this is better exercised by tests, it should be merged with | ||||||||
| 4781 | // the normal path for loads to prevent future divergence. | ||||||||
| 4782 | SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO); | ||||||||
| 4783 | if (MemVT != VT) | ||||||||
| 4784 | L = DAG.getPtrExtOrTrunc(L, dl, VT); | ||||||||
| 4785 | |||||||||
| 4786 | setValue(&I, L); | ||||||||
| 4787 | SDValue OutChain = L.getValue(1); | ||||||||
| 4788 | if (!I.isUnordered()) | ||||||||
| 4789 | DAG.setRoot(OutChain); | ||||||||
| 4790 | else | ||||||||
| 4791 | PendingLoads.push_back(OutChain); | ||||||||
| 4792 | return; | ||||||||
| 4793 | } | ||||||||
| 4794 | |||||||||
| 4795 | SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain, | ||||||||
| 4796 | Ptr, MMO); | ||||||||
| 4797 | |||||||||
| 4798 | SDValue OutChain = L.getValue(1); | ||||||||
| 4799 | if (MemVT != VT) | ||||||||
| 4800 | L = DAG.getPtrExtOrTrunc(L, dl, VT); | ||||||||
| 4801 | |||||||||
| 4802 | setValue(&I, L); | ||||||||
| 4803 | DAG.setRoot(OutChain); | ||||||||
| 4804 | } | ||||||||
| 4805 | |||||||||
| 4806 | void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { | ||||||||
| 4807 | SDLoc dl = getCurSDLoc(); | ||||||||
| 4808 | |||||||||
| 4809 | AtomicOrdering Ordering = I.getOrdering(); | ||||||||
| 4810 | SyncScope::ID SSID = I.getSyncScopeID(); | ||||||||
| 4811 | |||||||||
| 4812 | SDValue InChain = getRoot(); | ||||||||
| 4813 | |||||||||
| 4814 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4815 | EVT MemVT = | ||||||||
| 4816 | TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); | ||||||||
| 4817 | |||||||||
| 4818 | if (!TLI.supportsUnalignedAtomics() && | ||||||||
| 4819 | I.getAlign().value() < MemVT.getSizeInBits() / 8) | ||||||||
| 4820 | report_fatal_error("Cannot generate unaligned atomic store"); | ||||||||
| 4821 | |||||||||
| 4822 | auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); | ||||||||
| 4823 | |||||||||
| 4824 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 4825 | MachineMemOperand *MMO = MF.getMachineMemOperand( | ||||||||
| 4826 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), | ||||||||
| 4827 | I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering); | ||||||||
| 4828 | |||||||||
| 4829 | SDValue Val = getValue(I.getValueOperand()); | ||||||||
| 4830 | if (Val.getValueType() != MemVT) | ||||||||
| 4831 | Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT); | ||||||||
| 4832 | SDValue Ptr = getValue(I.getPointerOperand()); | ||||||||
| 4833 | |||||||||
| 4834 | if (TLI.lowerAtomicStoreAsStoreSDNode(I)) { | ||||||||
| 4835 | // TODO: Once this is better exercised by tests, it should be merged with | ||||||||
| 4836 | // the normal path for stores to prevent future divergence. | ||||||||
| 4837 | SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO); | ||||||||
| 4838 | setValue(&I, S); | ||||||||
| 4839 | DAG.setRoot(S); | ||||||||
| 4840 | return; | ||||||||
| 4841 | } | ||||||||
| 4842 | SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, | ||||||||
| 4843 | Ptr, Val, MMO); | ||||||||
| 4844 | |||||||||
| 4845 | setValue(&I, OutChain); | ||||||||
| 4846 | DAG.setRoot(OutChain); | ||||||||
| 4847 | } | ||||||||
| 4848 | |||||||||
| 4849 | /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC | ||||||||
| 4850 | /// node. | ||||||||
| 4851 | void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, | ||||||||
| 4852 | unsigned Intrinsic) { | ||||||||
| 4853 | // Ignore the callsite's attributes. A specific call site may be marked with | ||||||||
| 4854 | // readnone, but the lowering code will expect the chain based on the | ||||||||
| 4855 | // definition. | ||||||||
| 4856 | const Function *F = I.getCalledFunction(); | ||||||||
| 4857 | bool HasChain = !F->doesNotAccessMemory(); | ||||||||
| 4858 | bool OnlyLoad = HasChain && F->onlyReadsMemory(); | ||||||||
| 4859 | |||||||||
| 4860 | // Build the operand list. | ||||||||
| 4861 | SmallVector<SDValue, 8> Ops; | ||||||||
| 4862 | if (HasChain) { // If this intrinsic has side-effects, chainify it. | ||||||||
| 4863 | if (OnlyLoad) { | ||||||||
| 4864 | // We don't need to serialize loads against other loads. | ||||||||
| 4865 | Ops.push_back(DAG.getRoot()); | ||||||||
| 4866 | } else { | ||||||||
| 4867 | Ops.push_back(getRoot()); | ||||||||
| 4868 | } | ||||||||
| 4869 | } | ||||||||
| 4870 | |||||||||
| 4871 | // Info is set by getTgtMemIntrinsic | ||||||||
| 4872 | TargetLowering::IntrinsicInfo Info; | ||||||||
| 4873 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 4874 | bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, | ||||||||
| 4875 | DAG.getMachineFunction(), | ||||||||
| 4876 | Intrinsic); | ||||||||
| 4877 | |||||||||
| 4878 | // Add the intrinsic ID as an integer operand if it's not a target intrinsic. | ||||||||
| 4879 | if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || | ||||||||
| 4880 | Info.opc == ISD::INTRINSIC_W_CHAIN) | ||||||||
| 4881 | Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(), | ||||||||
| 4882 | TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 4883 | |||||||||
| 4884 | // Add all operands of the call to the operand list. | ||||||||
| 4885 | for (unsigned i = 0, e = I.arg_size(); i != e; ++i) { | ||||||||
| 4886 | const Value *Arg = I.getArgOperand(i); | ||||||||
| 4887 | if (!I.paramHasAttr(i, Attribute::ImmArg)) { | ||||||||
| 4888 | Ops.push_back(getValue(Arg)); | ||||||||
| 4889 | continue; | ||||||||
| 4890 | } | ||||||||
| 4891 | |||||||||
| 4892 | // Use TargetConstant instead of a regular constant for immarg. | ||||||||
| 4893 | EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true); | ||||||||
| 4894 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) { | ||||||||
| 4895 | assert(CI->getBitWidth() <= 64 &&(static_cast <bool> (CI->getBitWidth() <= 64 && "large intrinsic immediates not handled") ? void (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4896 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 4896 | "large intrinsic immediates not handled")(static_cast <bool> (CI->getBitWidth() <= 64 && "large intrinsic immediates not handled") ? void (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4896 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 4897 | Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT)); | ||||||||
| 4898 | } else { | ||||||||
| 4899 | Ops.push_back( | ||||||||
| 4900 | DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT)); | ||||||||
| 4901 | } | ||||||||
| 4902 | } | ||||||||
| 4903 | |||||||||
| 4904 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 4905 | ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); | ||||||||
| 4906 | |||||||||
| 4907 | if (HasChain) | ||||||||
| 4908 | ValueVTs.push_back(MVT::Other); | ||||||||
| 4909 | |||||||||
| 4910 | SDVTList VTs = DAG.getVTList(ValueVTs); | ||||||||
| 4911 | |||||||||
| 4912 | // Propagate fast-math-flags from IR to node(s). | ||||||||
| 4913 | SDNodeFlags Flags; | ||||||||
| 4914 | if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) | ||||||||
| 4915 | Flags.copyFMF(*FPMO); | ||||||||
| 4916 | SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); | ||||||||
| 4917 | |||||||||
| 4918 | // Create the node. | ||||||||
| 4919 | SDValue Result; | ||||||||
| 4920 | // In some cases, custom collection of operands from CallInst I may be needed. | ||||||||
| 4921 | TLI.CollectTargetIntrinsicOperands(I, Ops, DAG); | ||||||||
| 4922 | if (IsTgtIntrinsic) { | ||||||||
| 4923 | // This is target intrinsic that touches memory | ||||||||
| 4924 | // | ||||||||
| 4925 | // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic | ||||||||
| 4926 | // didn't yield anything useful. | ||||||||
| 4927 | MachinePointerInfo MPI; | ||||||||
| 4928 | if (Info.ptrVal) | ||||||||
| 4929 | MPI = MachinePointerInfo(Info.ptrVal, Info.offset); | ||||||||
| 4930 | else if (Info.fallbackAddressSpace) | ||||||||
| 4931 | MPI = MachinePointerInfo(*Info.fallbackAddressSpace); | ||||||||
| 4932 | Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, | ||||||||
| 4933 | Info.memVT, MPI, Info.align, Info.flags, | ||||||||
| 4934 | Info.size, I.getAAMetadata()); | ||||||||
| 4935 | } else if (!HasChain) { | ||||||||
| 4936 | Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); | ||||||||
| 4937 | } else if (!I.getType()->isVoidTy()) { | ||||||||
| 4938 | Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); | ||||||||
| 4939 | } else { | ||||||||
| 4940 | Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); | ||||||||
| 4941 | } | ||||||||
| 4942 | |||||||||
| 4943 | if (HasChain) { | ||||||||
| 4944 | SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); | ||||||||
| 4945 | if (OnlyLoad) | ||||||||
| 4946 | PendingLoads.push_back(Chain); | ||||||||
| 4947 | else | ||||||||
| 4948 | DAG.setRoot(Chain); | ||||||||
| 4949 | } | ||||||||
| 4950 | |||||||||
| 4951 | if (!I.getType()->isVoidTy()) { | ||||||||
| 4952 | if (!isa<VectorType>(I.getType())) | ||||||||
| 4953 | Result = lowerRangeToAssertZExt(DAG, I, Result); | ||||||||
| 4954 | |||||||||
| 4955 | MaybeAlign Alignment = I.getRetAlign(); | ||||||||
| 4956 | |||||||||
| 4957 | // Insert `assertalign` node if there's an alignment. | ||||||||
| 4958 | if (InsertAssertAlign && Alignment) { | ||||||||
| 4959 | Result = | ||||||||
| 4960 | DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); | ||||||||
| 4961 | } | ||||||||
| 4962 | |||||||||
| 4963 | setValue(&I, Result); | ||||||||
| 4964 | } | ||||||||
| 4965 | } | ||||||||
| 4966 | |||||||||
| 4967 | /// GetSignificand - Get the significand and build it into a floating-point | ||||||||
| 4968 | /// number with exponent of 1: | ||||||||
| 4969 | /// | ||||||||
| 4970 | /// Op = (Op & 0x007fffff) | 0x3f800000; | ||||||||
| 4971 | /// | ||||||||
| 4972 | /// where Op is the hexadecimal representation of floating point value. | ||||||||
| 4973 | static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) { | ||||||||
| 4974 | SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, | ||||||||
| 4975 | DAG.getConstant(0x007fffff, dl, MVT::i32)); | ||||||||
| 4976 | SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, | ||||||||
| 4977 | DAG.getConstant(0x3f800000, dl, MVT::i32)); | ||||||||
| 4978 | return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); | ||||||||
| 4979 | } | ||||||||
| 4980 | |||||||||
| 4981 | /// GetExponent - Get the exponent: | ||||||||
| 4982 | /// | ||||||||
| 4983 | /// (float)(int)(((Op & 0x7f800000) >> 23) - 127); | ||||||||
| 4984 | /// | ||||||||
| 4985 | /// where Op is the hexadecimal representation of floating point value. | ||||||||
| 4986 | static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, | ||||||||
| 4987 | const TargetLowering &TLI, const SDLoc &dl) { | ||||||||
| 4988 | SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, | ||||||||
| 4989 | DAG.getConstant(0x7f800000, dl, MVT::i32)); | ||||||||
| 4990 | SDValue t1 = DAG.getNode( | ||||||||
| 4991 | ISD::SRL, dl, MVT::i32, t0, | ||||||||
| 4992 | DAG.getConstant(23, dl, | ||||||||
| 4993 | TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout()))); | ||||||||
| 4994 | SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, | ||||||||
| 4995 | DAG.getConstant(127, dl, MVT::i32)); | ||||||||
| 4996 | return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); | ||||||||
| 4997 | } | ||||||||
| 4998 | |||||||||
| 4999 | /// getF32Constant - Get 32-bit floating point constant. | ||||||||
| 5000 | static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, | ||||||||
| 5001 | const SDLoc &dl) { | ||||||||
| 5002 | return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl, | ||||||||
| 5003 | MVT::f32); | ||||||||
| 5004 | } | ||||||||
| 5005 | |||||||||
| 5006 | static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, | ||||||||
| 5007 | SelectionDAG &DAG) { | ||||||||
| 5008 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||
| 5009 | |||||||||
| 5010 | // IntegerPartOfX = ((int32_t)(t0); | ||||||||
| 5011 | SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); | ||||||||
| 5012 | |||||||||
| 5013 | // FractionalPartOfX = t0 - (float)IntegerPartOfX; | ||||||||
| 5014 | SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); | ||||||||
| 5015 | SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); | ||||||||
| 5016 | |||||||||
| 5017 | // IntegerPartOfX <<= 23; | ||||||||
| 5018 | IntegerPartOfX = | ||||||||
| 5019 | DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, | ||||||||
| 5020 | DAG.getConstant(23, dl, | ||||||||
| 5021 | DAG.getTargetLoweringInfo().getShiftAmountTy( | ||||||||
| 5022 | MVT::i32, DAG.getDataLayout()))); | ||||||||
| 5023 | |||||||||
| 5024 | SDValue TwoToFractionalPartOfX; | ||||||||
| 5025 | if (LimitFloatPrecision <= 6) { | ||||||||
| 5026 | // For floating-point precision of 6: | ||||||||
| 5027 | // | ||||||||
| 5028 | // TwoToFractionalPartOfX = | ||||||||
| 5029 | // 0.997535578f + | ||||||||
| 5030 | // (0.735607626f + 0.252464424f * x) * x; | ||||||||
| 5031 | // | ||||||||
| 5032 | // error 0.0144103317, which is 6 bits | ||||||||
| 5033 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5034 | getF32Constant(DAG, 0x3e814304, dl)); | ||||||||
| 5035 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||
| 5036 | getF32Constant(DAG, 0x3f3c50c8, dl)); | ||||||||
| 5037 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5038 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||
| 5039 | getF32Constant(DAG, 0x3f7f5e7e, dl)); | ||||||||
| 5040 | } else if (LimitFloatPrecision <= 12) { | ||||||||
| 5041 | // For floating-point precision of 12: | ||||||||
| 5042 | // | ||||||||
| 5043 | // TwoToFractionalPartOfX = | ||||||||
| 5044 | // 0.999892986f + | ||||||||
| 5045 | // (0.696457318f + | ||||||||
| 5046 | // (0.224338339f + 0.792043434e-1f * x) * x) * x; | ||||||||
| 5047 | // | ||||||||
| 5048 | // error 0.000107046256, which is 13 to 14 bits | ||||||||
| 5049 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5050 | getF32Constant(DAG, 0x3da235e3, dl)); | ||||||||
| 5051 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||
| 5052 | getF32Constant(DAG, 0x3e65b8f3, dl)); | ||||||||
| 5053 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5054 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||
| 5055 | getF32Constant(DAG, 0x3f324b07, dl)); | ||||||||
| 5056 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||
| 5057 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, | ||||||||
| 5058 | getF32Constant(DAG, 0x3f7ff8fd, dl)); | ||||||||
| 5059 | } else { // LimitFloatPrecision <= 18 | ||||||||
| 5060 | // For floating-point precision of 18: | ||||||||
| 5061 | // | ||||||||
| 5062 | // TwoToFractionalPartOfX = | ||||||||
| 5063 | // 0.999999982f + | ||||||||
| 5064 | // (0.693148872f + | ||||||||
| 5065 | // (0.240227044f + | ||||||||
| 5066 | // (0.554906021e-1f + | ||||||||
| 5067 | // (0.961591928e-2f + | ||||||||
| 5068 | // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; | ||||||||
| 5069 | // error 2.47208000*10^(-7), which is better than 18 bits | ||||||||
| 5070 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5071 | getF32Constant(DAG, 0x3924b03e, dl)); | ||||||||
| 5072 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||
| 5073 | getF32Constant(DAG, 0x3ab24b87, dl)); | ||||||||
| 5074 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5075 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||
| 5076 | getF32Constant(DAG, 0x3c1d8c17, dl)); | ||||||||
| 5077 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||
| 5078 | SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, | ||||||||
| 5079 | getF32Constant(DAG, 0x3d634a1d, dl)); | ||||||||
| 5080 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||
| 5081 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, | ||||||||
| 5082 | getF32Constant(DAG, 0x3e75fe14, dl)); | ||||||||
| 5083 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); | ||||||||
| 5084 | SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, | ||||||||
| 5085 | getF32Constant(DAG, 0x3f317234, dl)); | ||||||||
| 5086 | SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); | ||||||||
| 5087 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, | ||||||||
| 5088 | getF32Constant(DAG, 0x3f800000, dl)); | ||||||||
| 5089 | } | ||||||||
| 5090 | |||||||||
| 5091 | // Add the exponent into the result in integer domain. | ||||||||
| 5092 | SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX); | ||||||||
| 5093 | return DAG.getNode(ISD::BITCAST, dl, MVT::f32, | ||||||||
| 5094 | DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX)); | ||||||||
| 5095 | } | ||||||||
| 5096 | |||||||||
| 5097 | /// expandExp - Lower an exp intrinsic. Handles the special sequences for | ||||||||
| 5098 | /// limited-precision mode. | ||||||||
| 5099 | static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||
| 5100 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||
| 5101 | if (Op.getValueType() == MVT::f32 && | ||||||||
| 5102 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||
| 5103 | |||||||||
| 5104 | // Put the exponent in the right bit position for later addition to the | ||||||||
| 5105 | // final result: | ||||||||
| 5106 | // | ||||||||
| 5107 | // t0 = Op * log2(e) | ||||||||
| 5108 | |||||||||
| 5109 | // TODO: What fast-math-flags should be set here? | ||||||||
| 5110 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, | ||||||||
| 5111 | DAG.getConstantFP(numbers::log2ef, dl, MVT::f32)); | ||||||||
| 5112 | return getLimitedPrecisionExp2(t0, dl, DAG); | ||||||||
| 5113 | } | ||||||||
| 5114 | |||||||||
| 5115 | // No special expansion. | ||||||||
| 5116 | return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags); | ||||||||
| 5117 | } | ||||||||
| 5118 | |||||||||
| 5119 | /// expandLog - Lower a log intrinsic. Handles the special sequences for | ||||||||
| 5120 | /// limited-precision mode. | ||||||||
| 5121 | static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||
| 5122 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||
| 5123 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||
| 5124 | |||||||||
| 5125 | if (Op.getValueType() == MVT::f32 && | ||||||||
| 5126 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||
| 5127 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); | ||||||||
| 5128 | |||||||||
| 5129 | // Scale the exponent by log(2). | ||||||||
| 5130 | SDValue Exp = GetExponent(DAG, Op1, TLI, dl); | ||||||||
| 5131 | SDValue LogOfExponent = | ||||||||
| 5132 | DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, | ||||||||
| 5133 | DAG.getConstantFP(numbers::ln2f, dl, MVT::f32)); | ||||||||
| 5134 | |||||||||
| 5135 | // Get the significand and build it into a floating-point number with | ||||||||
| 5136 | // exponent of 1. | ||||||||
| 5137 | SDValue X = GetSignificand(DAG, Op1, dl); | ||||||||
| 5138 | |||||||||
| 5139 | SDValue LogOfMantissa; | ||||||||
| 5140 | if (LimitFloatPrecision <= 6) { | ||||||||
| 5141 | // For floating-point precision of 6: | ||||||||
| 5142 | // | ||||||||
| 5143 | // LogofMantissa = | ||||||||
| 5144 | // -1.1609546f + | ||||||||
| 5145 | // (1.4034025f - 0.23903021f * x) * x; | ||||||||
| 5146 | // | ||||||||
| 5147 | // error 0.0034276066, which is better than 8 bits | ||||||||
| 5148 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5149 | getF32Constant(DAG, 0xbe74c456, dl)); | ||||||||
| 5150 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||
| 5151 | getF32Constant(DAG, 0x3fb3a2b1, dl)); | ||||||||
| 5152 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5153 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||
| 5154 | getF32Constant(DAG, 0x3f949a29, dl)); | ||||||||
| 5155 | } else if (LimitFloatPrecision <= 12) { | ||||||||
| 5156 | // For floating-point precision of 12: | ||||||||
| 5157 | // | ||||||||
| 5158 | // LogOfMantissa = | ||||||||
| 5159 | // -1.7417939f + | ||||||||
| 5160 | // (2.8212026f + | ||||||||
| 5161 | // (-1.4699568f + | ||||||||
| 5162 | // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; | ||||||||
| 5163 | // | ||||||||
| 5164 | // error 0.000061011436, which is 14 bits | ||||||||
| 5165 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5166 | getF32Constant(DAG, 0xbd67b6d6, dl)); | ||||||||
| 5167 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||
| 5168 | getF32Constant(DAG, 0x3ee4f4b8, dl)); | ||||||||
| 5169 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5170 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||
| 5171 | getF32Constant(DAG, 0x3fbc278b, dl)); | ||||||||
| 5172 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5173 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||
| 5174 | getF32Constant(DAG, 0x40348e95, dl)); | ||||||||
| 5175 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||
| 5176 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||
| 5177 | getF32Constant(DAG, 0x3fdef31a, dl)); | ||||||||
| 5178 | } else { // LimitFloatPrecision <= 18 | ||||||||
| 5179 | // For floating-point precision of 18: | ||||||||
| 5180 | // | ||||||||
| 5181 | // LogOfMantissa = | ||||||||
| 5182 | // -2.1072184f + | ||||||||
| 5183 | // (4.2372794f + | ||||||||
| 5184 | // (-3.7029485f + | ||||||||
| 5185 | // (2.2781945f + | ||||||||
| 5186 | // (-0.87823314f + | ||||||||
| 5187 | // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; | ||||||||
| 5188 | // | ||||||||
| 5189 | // error 0.0000023660568, which is better than 18 bits | ||||||||
| 5190 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5191 | getF32Constant(DAG, 0xbc91e5ac, dl)); | ||||||||
| 5192 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||
| 5193 | getF32Constant(DAG, 0x3e4350aa, dl)); | ||||||||
| 5194 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5195 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||
| 5196 | getF32Constant(DAG, 0x3f60d3e3, dl)); | ||||||||
| 5197 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5198 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||
| 5199 | getF32Constant(DAG, 0x4011cdf0, dl)); | ||||||||
| 5200 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||
| 5201 | SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||
| 5202 | getF32Constant(DAG, 0x406cfd1c, dl)); | ||||||||
| 5203 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||
| 5204 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, | ||||||||
| 5205 | getF32Constant(DAG, 0x408797cb, dl)); | ||||||||
| 5206 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); | ||||||||
| 5207 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, | ||||||||
| 5208 | getF32Constant(DAG, 0x4006dcab, dl)); | ||||||||
| 5209 | } | ||||||||
| 5210 | |||||||||
| 5211 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); | ||||||||
| 5212 | } | ||||||||
| 5213 | |||||||||
| 5214 | // No special expansion. | ||||||||
| 5215 | return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags); | ||||||||
| 5216 | } | ||||||||
| 5217 | |||||||||
| 5218 | /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for | ||||||||
| 5219 | /// limited-precision mode. | ||||||||
| 5220 | static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||
| 5221 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||
| 5222 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||
| 5223 | |||||||||
| 5224 | if (Op.getValueType() == MVT::f32 && | ||||||||
| 5225 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||
| 5226 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); | ||||||||
| 5227 | |||||||||
| 5228 | // Get the exponent. | ||||||||
| 5229 | SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); | ||||||||
| 5230 | |||||||||
| 5231 | // Get the significand and build it into a floating-point number with | ||||||||
| 5232 | // exponent of 1. | ||||||||
| 5233 | SDValue X = GetSignificand(DAG, Op1, dl); | ||||||||
| 5234 | |||||||||
| 5235 | // Different possible minimax approximations of significand in | ||||||||
| 5236 | // floating-point for various degrees of accuracy over [1,2]. | ||||||||
| 5237 | SDValue Log2ofMantissa; | ||||||||
| 5238 | if (LimitFloatPrecision <= 6) { | ||||||||
| 5239 | // For floating-point precision of 6: | ||||||||
| 5240 | // | ||||||||
| 5241 | // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; | ||||||||
| 5242 | // | ||||||||
| 5243 | // error 0.0049451742, which is more than 7 bits | ||||||||
| 5244 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5245 | getF32Constant(DAG, 0xbeb08fe0, dl)); | ||||||||
| 5246 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||
| 5247 | getF32Constant(DAG, 0x40019463, dl)); | ||||||||
| 5248 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5249 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||
| 5250 | getF32Constant(DAG, 0x3fd6633d, dl)); | ||||||||
| 5251 | } else if (LimitFloatPrecision <= 12) { | ||||||||
| 5252 | // For floating-point precision of 12: | ||||||||
| 5253 | // | ||||||||
| 5254 | // Log2ofMantissa = | ||||||||
| 5255 | // -2.51285454f + | ||||||||
| 5256 | // (4.07009056f + | ||||||||
| 5257 | // (-2.12067489f + | ||||||||
| 5258 | // (.645142248f - 0.816157886e-1f * x) * x) * x) * x; | ||||||||
| 5259 | // | ||||||||
| 5260 | // error 0.0000876136000, which is better than 13 bits | ||||||||
| 5261 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5262 | getF32Constant(DAG, 0xbda7262e, dl)); | ||||||||
| 5263 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||
| 5264 | getF32Constant(DAG, 0x3f25280b, dl)); | ||||||||
| 5265 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5266 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||
| 5267 | getF32Constant(DAG, 0x4007b923, dl)); | ||||||||
| 5268 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5269 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||
| 5270 | getF32Constant(DAG, 0x40823e2f, dl)); | ||||||||
| 5271 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||
| 5272 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||
| 5273 | getF32Constant(DAG, 0x4020d29c, dl)); | ||||||||
| 5274 | } else { // LimitFloatPrecision <= 18 | ||||||||
| 5275 | // For floating-point precision of 18: | ||||||||
| 5276 | // | ||||||||
| 5277 | // Log2ofMantissa = | ||||||||
| 5278 | // -3.0400495f + | ||||||||
| 5279 | // (6.1129976f + | ||||||||
| 5280 | // (-5.3420409f + | ||||||||
| 5281 | // (3.2865683f + | ||||||||
| 5282 | // (-1.2669343f + | ||||||||
| 5283 | // (0.27515199f - | ||||||||
| 5284 | // 0.25691327e-1f * x) * x) * x) * x) * x) * x; | ||||||||
| 5285 | // | ||||||||
| 5286 | // error 0.0000018516, which is better than 18 bits | ||||||||
| 5287 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5288 | getF32Constant(DAG, 0xbcd2769e, dl)); | ||||||||
| 5289 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||
| 5290 | getF32Constant(DAG, 0x3e8ce0b9, dl)); | ||||||||
| 5291 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5292 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||
| 5293 | getF32Constant(DAG, 0x3fa22ae7, dl)); | ||||||||
| 5294 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5295 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, | ||||||||
| 5296 | getF32Constant(DAG, 0x40525723, dl)); | ||||||||
| 5297 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||
| 5298 | SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, | ||||||||
| 5299 | getF32Constant(DAG, 0x40aaf200, dl)); | ||||||||
| 5300 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||
| 5301 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, | ||||||||
| 5302 | getF32Constant(DAG, 0x40c39dad, dl)); | ||||||||
| 5303 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); | ||||||||
| 5304 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, | ||||||||
| 5305 | getF32Constant(DAG, 0x4042902c, dl)); | ||||||||
| 5306 | } | ||||||||
| 5307 | |||||||||
| 5308 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); | ||||||||
| 5309 | } | ||||||||
| 5310 | |||||||||
| 5311 | // No special expansion. | ||||||||
| 5312 | return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags); | ||||||||
| 5313 | } | ||||||||
| 5314 | |||||||||
| 5315 | /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for | ||||||||
| 5316 | /// limited-precision mode. | ||||||||
| 5317 | static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||
| 5318 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||
| 5319 | // TODO: What fast-math-flags should be set on the floating-point nodes? | ||||||||
| 5320 | |||||||||
| 5321 | if (Op.getValueType() == MVT::f32 && | ||||||||
| 5322 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||
| 5323 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); | ||||||||
| 5324 | |||||||||
| 5325 | // Scale the exponent by log10(2) [0.30102999f]. | ||||||||
| 5326 | SDValue Exp = GetExponent(DAG, Op1, TLI, dl); | ||||||||
| 5327 | SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, | ||||||||
| 5328 | getF32Constant(DAG, 0x3e9a209a, dl)); | ||||||||
| 5329 | |||||||||
| 5330 | // Get the significand and build it into a floating-point number with | ||||||||
| 5331 | // exponent of 1. | ||||||||
| 5332 | SDValue X = GetSignificand(DAG, Op1, dl); | ||||||||
| 5333 | |||||||||
| 5334 | SDValue Log10ofMantissa; | ||||||||
| 5335 | if (LimitFloatPrecision <= 6) { | ||||||||
| 5336 | // For floating-point precision of 6: | ||||||||
| 5337 | // | ||||||||
| 5338 | // Log10ofMantissa = | ||||||||
| 5339 | // -0.50419619f + | ||||||||
| 5340 | // (0.60948995f - 0.10380950f * x) * x; | ||||||||
| 5341 | // | ||||||||
| 5342 | // error 0.0014886165, which is 6 bits | ||||||||
| 5343 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5344 | getF32Constant(DAG, 0xbdd49a13, dl)); | ||||||||
| 5345 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, | ||||||||
| 5346 | getF32Constant(DAG, 0x3f1c0789, dl)); | ||||||||
| 5347 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5348 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, | ||||||||
| 5349 | getF32Constant(DAG, 0x3f011300, dl)); | ||||||||
| 5350 | } else if (LimitFloatPrecision <= 12) { | ||||||||
| 5351 | // For floating-point precision of 12: | ||||||||
| 5352 | // | ||||||||
| 5353 | // Log10ofMantissa = | ||||||||
| 5354 | // -0.64831180f + | ||||||||
| 5355 | // (0.91751397f + | ||||||||
| 5356 | // (-0.31664806f + 0.47637168e-1f * x) * x) * x; | ||||||||
| 5357 | // | ||||||||
| 5358 | // error 0.00019228036, which is better than 12 bits | ||||||||
| 5359 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5360 | getF32Constant(DAG, 0x3d431f31, dl)); | ||||||||
| 5361 | SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, | ||||||||
| 5362 | getF32Constant(DAG, 0x3ea21fb2, dl)); | ||||||||
| 5363 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5364 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||
| 5365 | getF32Constant(DAG, 0x3f6ae232, dl)); | ||||||||
| 5366 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5367 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, | ||||||||
| 5368 | getF32Constant(DAG, 0x3f25f7c3, dl)); | ||||||||
| 5369 | } else { // LimitFloatPrecision <= 18 | ||||||||
| 5370 | // For floating-point precision of 18: | ||||||||
| 5371 | // | ||||||||
| 5372 | // Log10ofMantissa = | ||||||||
| 5373 | // -0.84299375f + | ||||||||
| 5374 | // (1.5327582f + | ||||||||
| 5375 | // (-1.0688956f + | ||||||||
| 5376 | // (0.49102474f + | ||||||||
| 5377 | // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; | ||||||||
| 5378 | // | ||||||||
| 5379 | // error 0.0000037995730, which is better than 18 bits | ||||||||
| 5380 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, | ||||||||
| 5381 | getF32Constant(DAG, 0x3c5d51ce, dl)); | ||||||||
| 5382 | SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, | ||||||||
| 5383 | getF32Constant(DAG, 0x3e00685a, dl)); | ||||||||
| 5384 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); | ||||||||
| 5385 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, | ||||||||
| 5386 | getF32Constant(DAG, 0x3efb6798, dl)); | ||||||||
| 5387 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); | ||||||||
| 5388 | SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, | ||||||||
| 5389 | getF32Constant(DAG, 0x3f88d192, dl)); | ||||||||
| 5390 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); | ||||||||
| 5391 | SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, | ||||||||
| 5392 | getF32Constant(DAG, 0x3fc4316c, dl)); | ||||||||
| 5393 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); | ||||||||
| 5394 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, | ||||||||
| 5395 | getF32Constant(DAG, 0x3f57ce70, dl)); | ||||||||
| 5396 | } | ||||||||
| 5397 | |||||||||
| 5398 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); | ||||||||
| 5399 | } | ||||||||
| 5400 | |||||||||
| 5401 | // No special expansion. | ||||||||
| 5402 | return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags); | ||||||||
| 5403 | } | ||||||||
| 5404 | |||||||||
| 5405 | /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for | ||||||||
| 5406 | /// limited-precision mode. | ||||||||
| 5407 | static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, | ||||||||
| 5408 | const TargetLowering &TLI, SDNodeFlags Flags) { | ||||||||
| 5409 | if (Op.getValueType() == MVT::f32 && | ||||||||
| 5410 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) | ||||||||
| 5411 | return getLimitedPrecisionExp2(Op, dl, DAG); | ||||||||
| 5412 | |||||||||
| 5413 | // No special expansion. | ||||||||
| 5414 | return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags); | ||||||||
| 5415 | } | ||||||||
| 5416 | |||||||||
| 5417 | /// visitPow - Lower a pow intrinsic. Handles the special sequences for | ||||||||
| 5418 | /// limited-precision mode with x == 10.0f. | ||||||||
| 5419 | static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, | ||||||||
| 5420 | SelectionDAG &DAG, const TargetLowering &TLI, | ||||||||
| 5421 | SDNodeFlags Flags) { | ||||||||
| 5422 | bool IsExp10 = false; | ||||||||
| 5423 | if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 && | ||||||||
| 5424 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { | ||||||||
| 5425 | if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { | ||||||||
| 5426 | APFloat Ten(10.0f); | ||||||||
| 5427 | IsExp10 = LHSC->isExactlyValue(Ten); | ||||||||
| 5428 | } | ||||||||
| 5429 | } | ||||||||
| 5430 | |||||||||
| 5431 | // TODO: What fast-math-flags should be set on the FMUL node? | ||||||||
| 5432 | if (IsExp10) { | ||||||||
| 5433 | // Put the exponent in the right bit position for later addition to the | ||||||||
| 5434 | // final result: | ||||||||
| 5435 | // | ||||||||
| 5436 | // #define LOG2OF10 3.3219281f | ||||||||
| 5437 | // t0 = Op * LOG2OF10; | ||||||||
| 5438 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, | ||||||||
| 5439 | getF32Constant(DAG, 0x40549a78, dl)); | ||||||||
| 5440 | return getLimitedPrecisionExp2(t0, dl, DAG); | ||||||||
| 5441 | } | ||||||||
| 5442 | |||||||||
| 5443 | // No special expansion. | ||||||||
| 5444 | return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags); | ||||||||
| 5445 | } | ||||||||
| 5446 | |||||||||
| 5447 | /// ExpandPowI - Expand a llvm.powi intrinsic. | ||||||||
| 5448 | static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, | ||||||||
| 5449 | SelectionDAG &DAG) { | ||||||||
| 5450 | // If RHS is a constant, we can expand this out to a multiplication tree if | ||||||||
| 5451 | // it's beneficial on the target, otherwise we end up lowering to a call to | ||||||||
| 5452 | // __powidf2 (for example). | ||||||||
| 5453 | if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { | ||||||||
| 5454 | unsigned Val = RHSC->getSExtValue(); | ||||||||
| 5455 | |||||||||
| 5456 | // powi(x, 0) -> 1.0 | ||||||||
| 5457 | if (Val == 0) | ||||||||
| 5458 | return DAG.getConstantFP(1.0, DL, LHS.getValueType()); | ||||||||
| 5459 | |||||||||
| 5460 | if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI( | ||||||||
| 5461 | Val, DAG.shouldOptForSize())) { | ||||||||
| 5462 | // Get the exponent as a positive value. | ||||||||
| 5463 | if ((int)Val < 0) | ||||||||
| 5464 | Val = -Val; | ||||||||
| 5465 | // We use the simple binary decomposition method to generate the multiply | ||||||||
| 5466 | // sequence. There are more optimal ways to do this (for example, | ||||||||
| 5467 | // powi(x,15) generates one more multiply than it should), but this has | ||||||||
| 5468 | // the benefit of being both really simple and much better than a libcall. | ||||||||
| 5469 | SDValue Res; // Logically starts equal to 1.0 | ||||||||
| 5470 | SDValue CurSquare = LHS; | ||||||||
| 5471 | // TODO: Intrinsics should have fast-math-flags that propagate to these | ||||||||
| 5472 | // nodes. | ||||||||
| 5473 | while (Val) { | ||||||||
| 5474 | if (Val & 1) { | ||||||||
| 5475 | if (Res.getNode()) | ||||||||
| 5476 | Res = | ||||||||
| 5477 | DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare); | ||||||||
| 5478 | else | ||||||||
| 5479 | Res = CurSquare; // 1.0*CurSquare. | ||||||||
| 5480 | } | ||||||||
| 5481 | |||||||||
| 5482 | CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), | ||||||||
| 5483 | CurSquare, CurSquare); | ||||||||
| 5484 | Val >>= 1; | ||||||||
| 5485 | } | ||||||||
| 5486 | |||||||||
| 5487 | // If the original was negative, invert the result, producing 1/(x*x*x). | ||||||||
| 5488 | if (RHSC->getSExtValue() < 0) | ||||||||
| 5489 | Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), | ||||||||
| 5490 | DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res); | ||||||||
| 5491 | return Res; | ||||||||
| 5492 | } | ||||||||
| 5493 | } | ||||||||
| 5494 | |||||||||
| 5495 | // Otherwise, expand to a libcall. | ||||||||
| 5496 | return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); | ||||||||
| 5497 | } | ||||||||
| 5498 | |||||||||
| 5499 | static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, | ||||||||
| 5500 | SDValue LHS, SDValue RHS, SDValue Scale, | ||||||||
| 5501 | SelectionDAG &DAG, const TargetLowering &TLI) { | ||||||||
| 5502 | EVT VT = LHS.getValueType(); | ||||||||
| 5503 | bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; | ||||||||
| 5504 | bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; | ||||||||
| 5505 | LLVMContext &Ctx = *DAG.getContext(); | ||||||||
| 5506 | |||||||||
| 5507 | // If the type is legal but the operation isn't, this node might survive all | ||||||||
| 5508 | // the way to operation legalization. If we end up there and we do not have | ||||||||
| 5509 | // the ability to widen the type (if VT*2 is not legal), we cannot expand the | ||||||||
| 5510 | // node. | ||||||||
| 5511 | |||||||||
| 5512 | // Coax the legalizer into expanding the node during type legalization instead | ||||||||
| 5513 | // by bumping the size by one bit. This will force it to Promote, enabling the | ||||||||
| 5514 | // early expansion and avoiding the need to expand later. | ||||||||
| 5515 | |||||||||
| 5516 | // We don't have to do this if Scale is 0; that can always be expanded, unless | ||||||||
| 5517 | // it's a saturating signed operation. Those can experience true integer | ||||||||
| 5518 | // division overflow, a case which we must avoid. | ||||||||
| 5519 | |||||||||
| 5520 | // FIXME: We wouldn't have to do this (or any of the early | ||||||||
| 5521 | // expansion/promotion) if it was possible to expand a libcall of an | ||||||||
| 5522 | // illegal type during operation legalization. But it's not, so things | ||||||||
| 5523 | // get a bit hacky. | ||||||||
| 5524 | unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue(); | ||||||||
| 5525 | if ((ScaleInt > 0 || (Saturating && Signed)) && | ||||||||
| 5526 | (TLI.isTypeLegal(VT) || | ||||||||
| 5527 | (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) { | ||||||||
| 5528 | TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction( | ||||||||
| 5529 | Opcode, VT, ScaleInt); | ||||||||
| 5530 | if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) { | ||||||||
| 5531 | EVT PromVT; | ||||||||
| 5532 | if (VT.isScalarInteger()) | ||||||||
| 5533 | PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1); | ||||||||
| 5534 | else if (VT.isVector()) { | ||||||||
| 5535 | PromVT = VT.getVectorElementType(); | ||||||||
| 5536 | PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1); | ||||||||
| 5537 | PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount()); | ||||||||
| 5538 | } else | ||||||||
| 5539 | llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 5539); | ||||||||
| 5540 | if (Signed) { | ||||||||
| 5541 | LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT); | ||||||||
| 5542 | RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT); | ||||||||
| 5543 | } else { | ||||||||
| 5544 | LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT); | ||||||||
| 5545 | RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT); | ||||||||
| 5546 | } | ||||||||
| 5547 | EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout()); | ||||||||
| 5548 | // For saturating operations, we need to shift up the LHS to get the | ||||||||
| 5549 | // proper saturation width, and then shift down again afterwards. | ||||||||
| 5550 | if (Saturating) | ||||||||
| 5551 | LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS, | ||||||||
| 5552 | DAG.getConstant(1, DL, ShiftTy)); | ||||||||
| 5553 | SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale); | ||||||||
| 5554 | if (Saturating) | ||||||||
| 5555 | Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res, | ||||||||
| 5556 | DAG.getConstant(1, DL, ShiftTy)); | ||||||||
| 5557 | return DAG.getZExtOrTrunc(Res, DL, VT); | ||||||||
| 5558 | } | ||||||||
| 5559 | } | ||||||||
| 5560 | |||||||||
| 5561 | return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale); | ||||||||
| 5562 | } | ||||||||
| 5563 | |||||||||
| 5564 | // getUnderlyingArgRegs - Find underlying registers used for a truncated, | ||||||||
| 5565 | // bitcasted, or split argument. Returns a list of <Register, size in bits> | ||||||||
| 5566 | static void | ||||||||
| 5567 | getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs, | ||||||||
| 5568 | const SDValue &N) { | ||||||||
| 5569 | switch (N.getOpcode()) { | ||||||||
| 5570 | case ISD::CopyFromReg: { | ||||||||
| 5571 | SDValue Op = N.getOperand(1); | ||||||||
| 5572 | Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(), | ||||||||
| 5573 | Op.getValueType().getSizeInBits()); | ||||||||
| 5574 | return; | ||||||||
| 5575 | } | ||||||||
| 5576 | case ISD::BITCAST: | ||||||||
| 5577 | case ISD::AssertZext: | ||||||||
| 5578 | case ISD::AssertSext: | ||||||||
| 5579 | case ISD::TRUNCATE: | ||||||||
| 5580 | getUnderlyingArgRegs(Regs, N.getOperand(0)); | ||||||||
| 5581 | return; | ||||||||
| 5582 | case ISD::BUILD_PAIR: | ||||||||
| 5583 | case ISD::BUILD_VECTOR: | ||||||||
| 5584 | case ISD::CONCAT_VECTORS: | ||||||||
| 5585 | for (SDValue Op : N->op_values()) | ||||||||
| 5586 | getUnderlyingArgRegs(Regs, Op); | ||||||||
| 5587 | return; | ||||||||
| 5588 | default: | ||||||||
| 5589 | return; | ||||||||
| 5590 | } | ||||||||
| 5591 | } | ||||||||
| 5592 | |||||||||
| 5593 | /// If the DbgValueInst is a dbg_value of a function argument, create the | ||||||||
| 5594 | /// corresponding DBG_VALUE machine instruction for it now. At the end of | ||||||||
| 5595 | /// instruction selection, they will be inserted to the entry BB. | ||||||||
| 5596 | /// We don't currently support this for variadic dbg_values, as they shouldn't | ||||||||
| 5597 | /// appear for function arguments or in the prologue. | ||||||||
| 5598 | bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( | ||||||||
| 5599 | const Value *V, DILocalVariable *Variable, DIExpression *Expr, | ||||||||
| 5600 | DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) { | ||||||||
| 5601 | const Argument *Arg = dyn_cast<Argument>(V); | ||||||||
| 5602 | if (!Arg) | ||||||||
| 5603 | return false; | ||||||||
| 5604 | |||||||||
| 5605 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 5606 | const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); | ||||||||
| 5607 | |||||||||
| 5608 | // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind | ||||||||
| 5609 | // we've been asked to pursue. | ||||||||
| 5610 | auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr, | ||||||||
| 5611 | bool Indirect) { | ||||||||
| 5612 | if (Reg.isVirtual() && MF.useDebugInstrRef()) { | ||||||||
| 5613 | // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF | ||||||||
| 5614 | // pointing at the VReg, which will be patched up later. | ||||||||
| 5615 | auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF); | ||||||||
| 5616 | SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg( | ||||||||
| 5617 | /* Reg */ Reg, /* isDef */ false, /* isImp */ false, | ||||||||
| 5618 | /* isKill */ false, /* isDead */ false, | ||||||||
| 5619 | /* isUndef */ false, /* isEarlyClobber */ false, | ||||||||
| 5620 | /* SubReg */ 0, /* isDebug */ true)}); | ||||||||
| 5621 | |||||||||
| 5622 | auto *NewDIExpr = FragExpr; | ||||||||
| 5623 | // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into | ||||||||
| 5624 | // the DIExpression. | ||||||||
| 5625 | if (Indirect) | ||||||||
| 5626 | NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore); | ||||||||
| 5627 | SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0}); | ||||||||
| 5628 | NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops); | ||||||||
| 5629 | return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr); | ||||||||
| 5630 | } else { | ||||||||
| 5631 | // Create a completely standard DBG_VALUE. | ||||||||
| 5632 | auto &Inst = TII->get(TargetOpcode::DBG_VALUE); | ||||||||
| 5633 | return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr); | ||||||||
| 5634 | } | ||||||||
| 5635 | }; | ||||||||
| 5636 | |||||||||
| 5637 | if (Kind == FuncArgumentDbgValueKind::Value) { | ||||||||
| 5638 | // ArgDbgValues are hoisted to the beginning of the entry block. So we | ||||||||
| 5639 | // should only emit as ArgDbgValue if the dbg.value intrinsic is found in | ||||||||
| 5640 | // the entry block. | ||||||||
| 5641 | bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front(); | ||||||||
| 5642 | if (!IsInEntryBlock) | ||||||||
| 5643 | return false; | ||||||||
| 5644 | |||||||||
| 5645 | // ArgDbgValues are hoisted to the beginning of the entry block. So we | ||||||||
| 5646 | // should only emit as ArgDbgValue if the dbg.value intrinsic describes a | ||||||||
| 5647 | // variable that also is a param. | ||||||||
| 5648 | // | ||||||||
| 5649 | // Although, if we are at the top of the entry block already, we can still | ||||||||
| 5650 | // emit using ArgDbgValue. This might catch some situations when the | ||||||||
| 5651 | // dbg.value refers to an argument that isn't used in the entry block, so | ||||||||
| 5652 | // any CopyToReg node would be optimized out and the only way to express | ||||||||
| 5653 | // this DBG_VALUE is by using the physical reg (or FI) as done in this | ||||||||
| 5654 | // method. ArgDbgValues are hoisted to the beginning of the entry block. So | ||||||||
| 5655 | // we should only emit as ArgDbgValue if the Variable is an argument to the | ||||||||
| 5656 | // current function, and the dbg.value intrinsic is found in the entry | ||||||||
| 5657 | // block. | ||||||||
| 5658 | bool VariableIsFunctionInputArg = Variable->isParameter() && | ||||||||
| 5659 | !DL->getInlinedAt(); | ||||||||
| 5660 | bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder; | ||||||||
| 5661 | if (!IsInPrologue && !VariableIsFunctionInputArg) | ||||||||
| 5662 | return false; | ||||||||
| 5663 | |||||||||
| 5664 | // Here we assume that a function argument on IR level only can be used to | ||||||||
| 5665 | // describe one input parameter on source level. If we for example have | ||||||||
| 5666 | // source code like this | ||||||||
| 5667 | // | ||||||||
| 5668 | // struct A { long x, y; }; | ||||||||
| 5669 | // void foo(struct A a, long b) { | ||||||||
| 5670 | // ... | ||||||||
| 5671 | // b = a.x; | ||||||||
| 5672 | // ... | ||||||||
| 5673 | // } | ||||||||
| 5674 | // | ||||||||
| 5675 | // and IR like this | ||||||||
| 5676 | // | ||||||||
| 5677 | // define void @foo(i32 %a1, i32 %a2, i32 %b) { | ||||||||
| 5678 | // entry: | ||||||||
| 5679 | // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment | ||||||||
| 5680 | // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment | ||||||||
| 5681 | // call void @llvm.dbg.value(metadata i32 %b, "b", | ||||||||
| 5682 | // ... | ||||||||
| 5683 | // call void @llvm.dbg.value(metadata i32 %a1, "b" | ||||||||
| 5684 | // ... | ||||||||
| 5685 | // | ||||||||
| 5686 | // then the last dbg.value is describing a parameter "b" using a value that | ||||||||
| 5687 | // is an argument. But since we already has used %a1 to describe a parameter | ||||||||
| 5688 | // we should not handle that last dbg.value here (that would result in an | ||||||||
| 5689 | // incorrect hoisting of the DBG_VALUE to the function entry). | ||||||||
| 5690 | // Notice that we allow one dbg.value per IR level argument, to accommodate | ||||||||
| 5691 | // for the situation with fragments above. | ||||||||
| 5692 | if (VariableIsFunctionInputArg) { | ||||||||
| 5693 | unsigned ArgNo = Arg->getArgNo(); | ||||||||
| 5694 | if (ArgNo >= FuncInfo.DescribedArgs.size()) | ||||||||
| 5695 | FuncInfo.DescribedArgs.resize(ArgNo + 1, false); | ||||||||
| 5696 | else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo)) | ||||||||
| 5697 | return false; | ||||||||
| 5698 | FuncInfo.DescribedArgs.set(ArgNo); | ||||||||
| 5699 | } | ||||||||
| 5700 | } | ||||||||
| 5701 | |||||||||
| 5702 | bool IsIndirect = false; | ||||||||
| 5703 | std::optional<MachineOperand> Op; | ||||||||
| 5704 | // Some arguments' frame index is recorded during argument lowering. | ||||||||
| 5705 | int FI = FuncInfo.getArgumentFrameIndex(Arg); | ||||||||
| 5706 | if (FI != std::numeric_limits<int>::max()) | ||||||||
| 5707 | Op = MachineOperand::CreateFI(FI); | ||||||||
| 5708 | |||||||||
| 5709 | SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes; | ||||||||
| 5710 | if (!Op && N.getNode()) { | ||||||||
| 5711 | getUnderlyingArgRegs(ArgRegsAndSizes, N); | ||||||||
| 5712 | Register Reg; | ||||||||
| 5713 | if (ArgRegsAndSizes.size() == 1) | ||||||||
| 5714 | Reg = ArgRegsAndSizes.front().first; | ||||||||
| 5715 | |||||||||
| 5716 | if (Reg && Reg.isVirtual()) { | ||||||||
| 5717 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | ||||||||
| 5718 | Register PR = RegInfo.getLiveInPhysReg(Reg); | ||||||||
| 5719 | if (PR) | ||||||||
| 5720 | Reg = PR; | ||||||||
| 5721 | } | ||||||||
| 5722 | if (Reg) { | ||||||||
| 5723 | Op = MachineOperand::CreateReg(Reg, false); | ||||||||
| 5724 | IsIndirect = Kind != FuncArgumentDbgValueKind::Value; | ||||||||
| 5725 | } | ||||||||
| 5726 | } | ||||||||
| 5727 | |||||||||
| 5728 | if (!Op && N.getNode()) { | ||||||||
| 5729 | // Check if frame index is available. | ||||||||
| 5730 | SDValue LCandidate = peekThroughBitcasts(N); | ||||||||
| 5731 | if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode())) | ||||||||
| 5732 | if (FrameIndexSDNode *FINode = | ||||||||
| 5733 | dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) | ||||||||
| 5734 | Op = MachineOperand::CreateFI(FINode->getIndex()); | ||||||||
| 5735 | } | ||||||||
| 5736 | |||||||||
| 5737 | if (!Op) { | ||||||||
| 5738 | // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg | ||||||||
| 5739 | auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>> | ||||||||
| 5740 | SplitRegs) { | ||||||||
| 5741 | unsigned Offset = 0; | ||||||||
| 5742 | for (const auto &RegAndSize : SplitRegs) { | ||||||||
| 5743 | // If the expression is already a fragment, the current register | ||||||||
| 5744 | // offset+size might extend beyond the fragment. In this case, only | ||||||||
| 5745 | // the register bits that are inside the fragment are relevant. | ||||||||
| 5746 | int RegFragmentSizeInBits = RegAndSize.second; | ||||||||
| 5747 | if (auto ExprFragmentInfo = Expr->getFragmentInfo()) { | ||||||||
| 5748 | uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits; | ||||||||
| 5749 | // The register is entirely outside the expression fragment, | ||||||||
| 5750 | // so is irrelevant for debug info. | ||||||||
| 5751 | if (Offset >= ExprFragmentSizeInBits) | ||||||||
| 5752 | break; | ||||||||
| 5753 | // The register is partially outside the expression fragment, only | ||||||||
| 5754 | // the low bits within the fragment are relevant for debug info. | ||||||||
| 5755 | if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) { | ||||||||
| 5756 | RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset; | ||||||||
| 5757 | } | ||||||||
| 5758 | } | ||||||||
| 5759 | |||||||||
| 5760 | auto FragmentExpr = DIExpression::createFragmentExpression( | ||||||||
| 5761 | Expr, Offset, RegFragmentSizeInBits); | ||||||||
| 5762 | Offset += RegAndSize.second; | ||||||||
| 5763 | // If a valid fragment expression cannot be created, the variable's | ||||||||
| 5764 | // correct value cannot be determined and so it is set as Undef. | ||||||||
| 5765 | if (!FragmentExpr) { | ||||||||
| 5766 | SDDbgValue *SDV = DAG.getConstantDbgValue( | ||||||||
| 5767 | Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder); | ||||||||
| 5768 | DAG.AddDbgValue(SDV, false); | ||||||||
| 5769 | continue; | ||||||||
| 5770 | } | ||||||||
| 5771 | MachineInstr *NewMI = | ||||||||
| 5772 | MakeVRegDbgValue(RegAndSize.first, *FragmentExpr, | ||||||||
| 5773 | Kind != FuncArgumentDbgValueKind::Value); | ||||||||
| 5774 | FuncInfo.ArgDbgValues.push_back(NewMI); | ||||||||
| 5775 | } | ||||||||
| 5776 | }; | ||||||||
| 5777 | |||||||||
| 5778 | // Check if ValueMap has reg number. | ||||||||
| 5779 | DenseMap<const Value *, Register>::const_iterator | ||||||||
| 5780 | VMI = FuncInfo.ValueMap.find(V); | ||||||||
| 5781 | if (VMI != FuncInfo.ValueMap.end()) { | ||||||||
| 5782 | const auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 5783 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second, | ||||||||
| 5784 | V->getType(), std::nullopt); | ||||||||
| 5785 | if (RFV.occupiesMultipleRegs()) { | ||||||||
| 5786 | splitMultiRegDbgValue(RFV.getRegsAndSizes()); | ||||||||
| 5787 | return true; | ||||||||
| 5788 | } | ||||||||
| 5789 | |||||||||
| 5790 | Op = MachineOperand::CreateReg(VMI->second, false); | ||||||||
| 5791 | IsIndirect = Kind != FuncArgumentDbgValueKind::Value; | ||||||||
| 5792 | } else if (ArgRegsAndSizes.size() > 1) { | ||||||||
| 5793 | // This was split due to the calling convention, and no virtual register | ||||||||
| 5794 | // mapping exists for the value. | ||||||||
| 5795 | splitMultiRegDbgValue(ArgRegsAndSizes); | ||||||||
| 5796 | return true; | ||||||||
| 5797 | } | ||||||||
| 5798 | } | ||||||||
| 5799 | |||||||||
| 5800 | if (!Op) | ||||||||
| 5801 | return false; | ||||||||
| 5802 | |||||||||
| 5803 | assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5804 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 5804 | "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? void (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5804 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 5805 | MachineInstr *NewMI = nullptr; | ||||||||
| 5806 | |||||||||
| 5807 | if (Op->isReg()) | ||||||||
| 5808 | NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect); | ||||||||
| 5809 | else | ||||||||
| 5810 | NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op, | ||||||||
| 5811 | Variable, Expr); | ||||||||
| 5812 | |||||||||
| 5813 | // Otherwise, use ArgDbgValues. | ||||||||
| 5814 | FuncInfo.ArgDbgValues.push_back(NewMI); | ||||||||
| 5815 | return true; | ||||||||
| 5816 | } | ||||||||
| 5817 | |||||||||
| 5818 | /// Return the appropriate SDDbgValue based on N. | ||||||||
| 5819 | SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N, | ||||||||
| 5820 | DILocalVariable *Variable, | ||||||||
| 5821 | DIExpression *Expr, | ||||||||
| 5822 | const DebugLoc &dl, | ||||||||
| 5823 | unsigned DbgSDNodeOrder) { | ||||||||
| 5824 | if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { | ||||||||
| 5825 | // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe | ||||||||
| 5826 | // stack slot locations. | ||||||||
| 5827 | // | ||||||||
| 5828 | // Consider "int x = 0; int *px = &x;". There are two kinds of interesting | ||||||||
| 5829 | // debug values here after optimization: | ||||||||
| 5830 | // | ||||||||
| 5831 | // dbg.value(i32* %px, !"int *px", !DIExpression()), and | ||||||||
| 5832 | // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) | ||||||||
| 5833 | // | ||||||||
| 5834 | // Both describe the direct values of their associated variables. | ||||||||
| 5835 | return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), | ||||||||
| 5836 | /*IsIndirect*/ false, dl, DbgSDNodeOrder); | ||||||||
| 5837 | } | ||||||||
| 5838 | return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), | ||||||||
| 5839 | /*IsIndirect*/ false, dl, DbgSDNodeOrder); | ||||||||
| 5840 | } | ||||||||
| 5841 | |||||||||
| 5842 | static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) { | ||||||||
| 5843 | switch (Intrinsic) { | ||||||||
| 5844 | case Intrinsic::smul_fix: | ||||||||
| 5845 | return ISD::SMULFIX; | ||||||||
| 5846 | case Intrinsic::umul_fix: | ||||||||
| 5847 | return ISD::UMULFIX; | ||||||||
| 5848 | case Intrinsic::smul_fix_sat: | ||||||||
| 5849 | return ISD::SMULFIXSAT; | ||||||||
| 5850 | case Intrinsic::umul_fix_sat: | ||||||||
| 5851 | return ISD::UMULFIXSAT; | ||||||||
| 5852 | case Intrinsic::sdiv_fix: | ||||||||
| 5853 | return ISD::SDIVFIX; | ||||||||
| 5854 | case Intrinsic::udiv_fix: | ||||||||
| 5855 | return ISD::UDIVFIX; | ||||||||
| 5856 | case Intrinsic::sdiv_fix_sat: | ||||||||
| 5857 | return ISD::SDIVFIXSAT; | ||||||||
| 5858 | case Intrinsic::udiv_fix_sat: | ||||||||
| 5859 | return ISD::UDIVFIXSAT; | ||||||||
| 5860 | default: | ||||||||
| 5861 | llvm_unreachable("Unhandled fixed point intrinsic")::llvm::llvm_unreachable_internal("Unhandled fixed point intrinsic" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5861 ); | ||||||||
| 5862 | } | ||||||||
| 5863 | } | ||||||||
| 5864 | |||||||||
| 5865 | void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I, | ||||||||
| 5866 | const char *FunctionName) { | ||||||||
| 5867 | assert(FunctionName && "FunctionName must not be nullptr")(static_cast <bool> (FunctionName && "FunctionName must not be nullptr" ) ? void (0) : __assert_fail ("FunctionName && \"FunctionName must not be nullptr\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5867 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 5868 | SDValue Callee = DAG.getExternalSymbol( | ||||||||
| 5869 | FunctionName, | ||||||||
| 5870 | DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); | ||||||||
| 5871 | LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall()); | ||||||||
| 5872 | } | ||||||||
| 5873 | |||||||||
| 5874 | /// Given a @llvm.call.preallocated.setup, return the corresponding | ||||||||
| 5875 | /// preallocated call. | ||||||||
| 5876 | static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) { | ||||||||
| 5877 | assert(cast<CallBase>(PreallocatedSetup)(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 5878 | ->getCalledFunction()(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 5879 | ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 5880 | "expected call_preallocated_setup Value")(static_cast <bool> (cast<CallBase>(PreallocatedSetup ) ->getCalledFunction() ->getIntrinsicID() == Intrinsic ::call_preallocated_setup && "expected call_preallocated_setup Value" ) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 5881 | for (const auto *U : PreallocatedSetup->users()) { | ||||||||
| 5882 | auto *UseCall = cast<CallBase>(U); | ||||||||
| 5883 | const Function *Fn = UseCall->getCalledFunction(); | ||||||||
| 5884 | if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) { | ||||||||
| 5885 | return UseCall; | ||||||||
| 5886 | } | ||||||||
| 5887 | } | ||||||||
| 5888 | llvm_unreachable("expected corresponding call to preallocated setup/arg")::llvm::llvm_unreachable_internal("expected corresponding call to preallocated setup/arg" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5888 ); | ||||||||
| 5889 | } | ||||||||
| 5890 | |||||||||
| 5891 | /// Lower the call to the specified intrinsic function. | ||||||||
| 5892 | void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, | ||||||||
| 5893 | unsigned Intrinsic) { | ||||||||
| 5894 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 5895 | SDLoc sdl = getCurSDLoc(); | ||||||||
| 5896 | DebugLoc dl = getCurDebugLoc(); | ||||||||
| 5897 | SDValue Res; | ||||||||
| 5898 | |||||||||
| 5899 | SDNodeFlags Flags; | ||||||||
| 5900 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) | ||||||||
| 5901 | Flags.copyFMF(*FPOp); | ||||||||
| 5902 | |||||||||
| 5903 | switch (Intrinsic) { | ||||||||
| 5904 | default: | ||||||||
| 5905 | // By default, turn this into a target intrinsic node. | ||||||||
| 5906 | visitTargetIntrinsic(I, Intrinsic); | ||||||||
| 5907 | return; | ||||||||
| 5908 | case Intrinsic::vscale: { | ||||||||
| 5909 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 5910 | setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1))); | ||||||||
| 5911 | return; | ||||||||
| 5912 | } | ||||||||
| 5913 | case Intrinsic::vastart: visitVAStart(I); return; | ||||||||
| 5914 | case Intrinsic::vaend: visitVAEnd(I); return; | ||||||||
| 5915 | case Intrinsic::vacopy: visitVACopy(I); return; | ||||||||
| 5916 | case Intrinsic::returnaddress: | ||||||||
| 5917 | setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, | ||||||||
| 5918 | TLI.getValueType(DAG.getDataLayout(), I.getType()), | ||||||||
| 5919 | getValue(I.getArgOperand(0)))); | ||||||||
| 5920 | return; | ||||||||
| 5921 | case Intrinsic::addressofreturnaddress: | ||||||||
| 5922 | setValue(&I, | ||||||||
| 5923 | DAG.getNode(ISD::ADDROFRETURNADDR, sdl, | ||||||||
| 5924 | TLI.getValueType(DAG.getDataLayout(), I.getType()))); | ||||||||
| 5925 | return; | ||||||||
| 5926 | case Intrinsic::sponentry: | ||||||||
| 5927 | setValue(&I, | ||||||||
| 5928 | DAG.getNode(ISD::SPONENTRY, sdl, | ||||||||
| 5929 | TLI.getValueType(DAG.getDataLayout(), I.getType()))); | ||||||||
| 5930 | return; | ||||||||
| 5931 | case Intrinsic::frameaddress: | ||||||||
| 5932 | setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, | ||||||||
| 5933 | TLI.getFrameIndexTy(DAG.getDataLayout()), | ||||||||
| 5934 | getValue(I.getArgOperand(0)))); | ||||||||
| 5935 | return; | ||||||||
| 5936 | case Intrinsic::read_volatile_register: | ||||||||
| 5937 | case Intrinsic::read_register: { | ||||||||
| 5938 | Value *Reg = I.getArgOperand(0); | ||||||||
| 5939 | SDValue Chain = getRoot(); | ||||||||
| 5940 | SDValue RegName = | ||||||||
| 5941 | DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); | ||||||||
| 5942 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 5943 | Res = DAG.getNode(ISD::READ_REGISTER, sdl, | ||||||||
| 5944 | DAG.getVTList(VT, MVT::Other), Chain, RegName); | ||||||||
| 5945 | setValue(&I, Res); | ||||||||
| 5946 | DAG.setRoot(Res.getValue(1)); | ||||||||
| 5947 | return; | ||||||||
| 5948 | } | ||||||||
| 5949 | case Intrinsic::write_register: { | ||||||||
| 5950 | Value *Reg = I.getArgOperand(0); | ||||||||
| 5951 | Value *RegValue = I.getArgOperand(1); | ||||||||
| 5952 | SDValue Chain = getRoot(); | ||||||||
| 5953 | SDValue RegName = | ||||||||
| 5954 | DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); | ||||||||
| 5955 | DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, | ||||||||
| 5956 | RegName, getValue(RegValue))); | ||||||||
| 5957 | return; | ||||||||
| 5958 | } | ||||||||
| 5959 | case Intrinsic::memcpy: { | ||||||||
| 5960 | const auto &MCI = cast<MemCpyInst>(I); | ||||||||
| 5961 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 5962 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 5963 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||
| 5964 | // @llvm.memcpy defines 0 and 1 to both mean no alignment. | ||||||||
| 5965 | Align DstAlign = MCI.getDestAlign().valueOrOne(); | ||||||||
| 5966 | Align SrcAlign = MCI.getSourceAlign().valueOrOne(); | ||||||||
| 5967 | Align Alignment = std::min(DstAlign, SrcAlign); | ||||||||
| 5968 | bool isVol = MCI.isVolatile(); | ||||||||
| 5969 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 5970 | // FIXME: Support passing different dest/src alignments to the memcpy DAG | ||||||||
| 5971 | // node. | ||||||||
| 5972 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||
| 5973 | SDValue MC = DAG.getMemcpy( | ||||||||
| 5974 | Root, sdl, Op1, Op2, Op3, Alignment, isVol, | ||||||||
| 5975 | /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)), | ||||||||
| 5976 | MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA); | ||||||||
| 5977 | updateDAGForMaybeTailCall(MC); | ||||||||
| 5978 | return; | ||||||||
| 5979 | } | ||||||||
| 5980 | case Intrinsic::memcpy_inline: { | ||||||||
| 5981 | const auto &MCI = cast<MemCpyInlineInst>(I); | ||||||||
| 5982 | SDValue Dst = getValue(I.getArgOperand(0)); | ||||||||
| 5983 | SDValue Src = getValue(I.getArgOperand(1)); | ||||||||
| 5984 | SDValue Size = getValue(I.getArgOperand(2)); | ||||||||
| 5985 | assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size")(static_cast <bool> (isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Size) && \"memcpy_inline needs constant size\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5985 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 5986 | // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment. | ||||||||
| 5987 | Align DstAlign = MCI.getDestAlign().valueOrOne(); | ||||||||
| 5988 | Align SrcAlign = MCI.getSourceAlign().valueOrOne(); | ||||||||
| 5989 | Align Alignment = std::min(DstAlign, SrcAlign); | ||||||||
| 5990 | bool isVol = MCI.isVolatile(); | ||||||||
| 5991 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 5992 | // FIXME: Support passing different dest/src alignments to the memcpy DAG | ||||||||
| 5993 | // node. | ||||||||
| 5994 | SDValue MC = DAG.getMemcpy( | ||||||||
| 5995 | getRoot(), sdl, Dst, Src, Size, Alignment, isVol, | ||||||||
| 5996 | /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)), | ||||||||
| 5997 | MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA); | ||||||||
| 5998 | updateDAGForMaybeTailCall(MC); | ||||||||
| 5999 | return; | ||||||||
| 6000 | } | ||||||||
| 6001 | case Intrinsic::memset: { | ||||||||
| 6002 | const auto &MSI = cast<MemSetInst>(I); | ||||||||
| 6003 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6004 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6005 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||
| 6006 | // @llvm.memset defines 0 and 1 to both mean no alignment. | ||||||||
| 6007 | Align Alignment = MSI.getDestAlign().valueOrOne(); | ||||||||
| 6008 | bool isVol = MSI.isVolatile(); | ||||||||
| 6009 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 6010 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||
| 6011 | SDValue MS = DAG.getMemset( | ||||||||
| 6012 | Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false, | ||||||||
| 6013 | isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata()); | ||||||||
| 6014 | updateDAGForMaybeTailCall(MS); | ||||||||
| 6015 | return; | ||||||||
| 6016 | } | ||||||||
| 6017 | case Intrinsic::memset_inline: { | ||||||||
| 6018 | const auto &MSII = cast<MemSetInlineInst>(I); | ||||||||
| 6019 | SDValue Dst = getValue(I.getArgOperand(0)); | ||||||||
| 6020 | SDValue Value = getValue(I.getArgOperand(1)); | ||||||||
| 6021 | SDValue Size = getValue(I.getArgOperand(2)); | ||||||||
| 6022 | assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size")(static_cast <bool> (isa<ConstantSDNode>(Size) && "memset_inline needs constant size") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Size) && \"memset_inline needs constant size\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6022 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6023 | // @llvm.memset defines 0 and 1 to both mean no alignment. | ||||||||
| 6024 | Align DstAlign = MSII.getDestAlign().valueOrOne(); | ||||||||
| 6025 | bool isVol = MSII.isVolatile(); | ||||||||
| 6026 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 6027 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||
| 6028 | SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol, | ||||||||
| 6029 | /* AlwaysInline */ true, isTC, | ||||||||
| 6030 | MachinePointerInfo(I.getArgOperand(0)), | ||||||||
| 6031 | I.getAAMetadata()); | ||||||||
| 6032 | updateDAGForMaybeTailCall(MC); | ||||||||
| 6033 | return; | ||||||||
| 6034 | } | ||||||||
| 6035 | case Intrinsic::memmove: { | ||||||||
| 6036 | const auto &MMI = cast<MemMoveInst>(I); | ||||||||
| 6037 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6038 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6039 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||
| 6040 | // @llvm.memmove defines 0 and 1 to both mean no alignment. | ||||||||
| 6041 | Align DstAlign = MMI.getDestAlign().valueOrOne(); | ||||||||
| 6042 | Align SrcAlign = MMI.getSourceAlign().valueOrOne(); | ||||||||
| 6043 | Align Alignment = std::min(DstAlign, SrcAlign); | ||||||||
| 6044 | bool isVol = MMI.isVolatile(); | ||||||||
| 6045 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 6046 | // FIXME: Support passing different dest/src alignments to the memmove DAG | ||||||||
| 6047 | // node. | ||||||||
| 6048 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||
| 6049 | SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, | ||||||||
| 6050 | isTC, MachinePointerInfo(I.getArgOperand(0)), | ||||||||
| 6051 | MachinePointerInfo(I.getArgOperand(1)), | ||||||||
| 6052 | I.getAAMetadata(), AA); | ||||||||
| 6053 | updateDAGForMaybeTailCall(MM); | ||||||||
| 6054 | return; | ||||||||
| 6055 | } | ||||||||
| 6056 | case Intrinsic::memcpy_element_unordered_atomic: { | ||||||||
| 6057 | const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I); | ||||||||
| 6058 | SDValue Dst = getValue(MI.getRawDest()); | ||||||||
| 6059 | SDValue Src = getValue(MI.getRawSource()); | ||||||||
| 6060 | SDValue Length = getValue(MI.getLength()); | ||||||||
| 6061 | |||||||||
| 6062 | Type *LengthTy = MI.getLength()->getType(); | ||||||||
| 6063 | unsigned ElemSz = MI.getElementSizeInBytes(); | ||||||||
| 6064 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 6065 | SDValue MC = | ||||||||
| 6066 | DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz, | ||||||||
| 6067 | isTC, MachinePointerInfo(MI.getRawDest()), | ||||||||
| 6068 | MachinePointerInfo(MI.getRawSource())); | ||||||||
| 6069 | updateDAGForMaybeTailCall(MC); | ||||||||
| 6070 | return; | ||||||||
| 6071 | } | ||||||||
| 6072 | case Intrinsic::memmove_element_unordered_atomic: { | ||||||||
| 6073 | auto &MI = cast<AtomicMemMoveInst>(I); | ||||||||
| 6074 | SDValue Dst = getValue(MI.getRawDest()); | ||||||||
| 6075 | SDValue Src = getValue(MI.getRawSource()); | ||||||||
| 6076 | SDValue Length = getValue(MI.getLength()); | ||||||||
| 6077 | |||||||||
| 6078 | Type *LengthTy = MI.getLength()->getType(); | ||||||||
| 6079 | unsigned ElemSz = MI.getElementSizeInBytes(); | ||||||||
| 6080 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 6081 | SDValue MC = | ||||||||
| 6082 | DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz, | ||||||||
| 6083 | isTC, MachinePointerInfo(MI.getRawDest()), | ||||||||
| 6084 | MachinePointerInfo(MI.getRawSource())); | ||||||||
| 6085 | updateDAGForMaybeTailCall(MC); | ||||||||
| 6086 | return; | ||||||||
| 6087 | } | ||||||||
| 6088 | case Intrinsic::memset_element_unordered_atomic: { | ||||||||
| 6089 | auto &MI = cast<AtomicMemSetInst>(I); | ||||||||
| 6090 | SDValue Dst = getValue(MI.getRawDest()); | ||||||||
| 6091 | SDValue Val = getValue(MI.getValue()); | ||||||||
| 6092 | SDValue Length = getValue(MI.getLength()); | ||||||||
| 6093 | |||||||||
| 6094 | Type *LengthTy = MI.getLength()->getType(); | ||||||||
| 6095 | unsigned ElemSz = MI.getElementSizeInBytes(); | ||||||||
| 6096 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); | ||||||||
| 6097 | SDValue MC = | ||||||||
| 6098 | DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz, | ||||||||
| 6099 | isTC, MachinePointerInfo(MI.getRawDest())); | ||||||||
| 6100 | updateDAGForMaybeTailCall(MC); | ||||||||
| 6101 | return; | ||||||||
| 6102 | } | ||||||||
| 6103 | case Intrinsic::call_preallocated_setup: { | ||||||||
| 6104 | const CallBase *PreallocatedCall = FindPreallocatedCall(&I); | ||||||||
| 6105 | SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); | ||||||||
| 6106 | SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other, | ||||||||
| 6107 | getRoot(), SrcValue); | ||||||||
| 6108 | setValue(&I, Res); | ||||||||
| 6109 | DAG.setRoot(Res); | ||||||||
| 6110 | return; | ||||||||
| 6111 | } | ||||||||
| 6112 | case Intrinsic::call_preallocated_arg: { | ||||||||
| 6113 | const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0)); | ||||||||
| 6114 | SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); | ||||||||
| 6115 | SDValue Ops[3]; | ||||||||
| 6116 | Ops[0] = getRoot(); | ||||||||
| 6117 | Ops[1] = SrcValue; | ||||||||
| 6118 | Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl, | ||||||||
| 6119 | MVT::i32); // arg index | ||||||||
| 6120 | SDValue Res = DAG.getNode( | ||||||||
| 6121 | ISD::PREALLOCATED_ARG, sdl, | ||||||||
| 6122 | DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops); | ||||||||
| 6123 | setValue(&I, Res); | ||||||||
| 6124 | DAG.setRoot(Res.getValue(1)); | ||||||||
| 6125 | return; | ||||||||
| 6126 | } | ||||||||
| 6127 | case Intrinsic::dbg_declare: { | ||||||||
| 6128 | const auto &DI = cast<DbgDeclareInst>(I); | ||||||||
| 6129 | // Debug intrinsics are handled separately in assignment tracking mode. | ||||||||
| 6130 | // Some intrinsics are handled right after Argument lowering. | ||||||||
| 6131 | if (AssignmentTrackingEnabled || | ||||||||
| 6132 | FuncInfo.PreprocessedDbgDeclares.count(&DI)) | ||||||||
| 6133 | return; | ||||||||
| 6134 | // Assume dbg.declare can not currently use DIArgList, i.e. | ||||||||
| 6135 | // it is non-variadic. | ||||||||
| 6136 | assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList")(static_cast <bool> (!DI.hasArgList() && "Only dbg.value should currently use DIArgList" ) ? void (0) : __assert_fail ("!DI.hasArgList() && \"Only dbg.value should currently use DIArgList\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6136 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6137 | DILocalVariable *Variable = DI.getVariable(); | ||||||||
| 6138 | DIExpression *Expression = DI.getExpression(); | ||||||||
| 6139 | dropDanglingDebugInfo(Variable, Expression); | ||||||||
| 6140 | assert(Variable && "Missing variable")(static_cast <bool> (Variable && "Missing variable" ) ? void (0) : __assert_fail ("Variable && \"Missing variable\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6140 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6141 | LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: " << DI << "\n"; } } while (false) | ||||||||
| 6142 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: " << DI << "\n"; } } while (false); | ||||||||
| 6143 | // Check if address has undef value. | ||||||||
| 6144 | const Value *Address = DI.getVariableLocationOp(0); | ||||||||
| 6145 | if (!Address || isa<UndefValue>(Address) || | ||||||||
| 6146 | (Address->use_empty() && !isa<Argument>(Address))) { | ||||||||
| 6147 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (bad/undef/unused-arg address)\n"; } } while ( false) | ||||||||
| 6148 | << " (bad/undef/unused-arg address)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (bad/undef/unused-arg address)\n"; } } while ( false); | ||||||||
| 6149 | return; | ||||||||
| 6150 | } | ||||||||
| 6151 | |||||||||
| 6152 | bool isParameter = Variable->isParameter() || isa<Argument>(Address); | ||||||||
| 6153 | |||||||||
| 6154 | SDValue &N = NodeMap[Address]; | ||||||||
| 6155 | if (!N.getNode() && isa<Argument>(Address)) | ||||||||
| 6156 | // Check unused arguments map. | ||||||||
| 6157 | N = UnusedArgNodeMap[Address]; | ||||||||
| 6158 | SDDbgValue *SDV; | ||||||||
| 6159 | if (N.getNode()) { | ||||||||
| 6160 | if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) | ||||||||
| 6161 | Address = BCI->getOperand(0); | ||||||||
| 6162 | // Parameters are handled specially. | ||||||||
| 6163 | auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); | ||||||||
| 6164 | if (isParameter && FINode) { | ||||||||
| 6165 | // Byval parameter. We have a frame index at this point. | ||||||||
| 6166 | SDV = | ||||||||
| 6167 | DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(), | ||||||||
| 6168 | /*IsIndirect*/ true, dl, SDNodeOrder); | ||||||||
| 6169 | } else if (isa<Argument>(Address)) { | ||||||||
| 6170 | // Address is an argument, so try to emit its dbg value using | ||||||||
| 6171 | // virtual register info from the FuncInfo.ValueMap. | ||||||||
| 6172 | EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, | ||||||||
| 6173 | FuncArgumentDbgValueKind::Declare, N); | ||||||||
| 6174 | return; | ||||||||
| 6175 | } else { | ||||||||
| 6176 | SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), | ||||||||
| 6177 | true, dl, SDNodeOrder); | ||||||||
| 6178 | } | ||||||||
| 6179 | DAG.AddDbgValue(SDV, isParameter); | ||||||||
| 6180 | } else { | ||||||||
| 6181 | // If Address is an argument then try to emit its dbg value using | ||||||||
| 6182 | // virtual register info from the FuncInfo.ValueMap. | ||||||||
| 6183 | if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, | ||||||||
| 6184 | FuncArgumentDbgValueKind::Declare, N)) { | ||||||||
| 6185 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (could not emit func-arg dbg_value)\n"; } } while (false) | ||||||||
| 6186 | << " (could not emit func-arg dbg_value)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for " << DI << " (could not emit func-arg dbg_value)\n"; } } while (false); | ||||||||
| 6187 | } | ||||||||
| 6188 | } | ||||||||
| 6189 | return; | ||||||||
| 6190 | } | ||||||||
| 6191 | case Intrinsic::dbg_label: { | ||||||||
| 6192 | const DbgLabelInst &DI = cast<DbgLabelInst>(I); | ||||||||
| 6193 | DILabel *Label = DI.getLabel(); | ||||||||
| 6194 | assert(Label && "Missing label")(static_cast <bool> (Label && "Missing label") ? void (0) : __assert_fail ("Label && \"Missing label\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6194 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6195 | |||||||||
| 6196 | SDDbgLabel *SDV; | ||||||||
| 6197 | SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder); | ||||||||
| 6198 | DAG.AddDbgLabel(SDV); | ||||||||
| 6199 | return; | ||||||||
| 6200 | } | ||||||||
| 6201 | case Intrinsic::dbg_assign: { | ||||||||
| 6202 | // Debug intrinsics are handled seperately in assignment tracking mode. | ||||||||
| 6203 | if (AssignmentTrackingEnabled) | ||||||||
| 6204 | return; | ||||||||
| 6205 | // If assignment tracking hasn't been enabled then fall through and treat | ||||||||
| 6206 | // the dbg.assign as a dbg.value. | ||||||||
| 6207 | [[fallthrough]]; | ||||||||
| 6208 | } | ||||||||
| 6209 | case Intrinsic::dbg_value: { | ||||||||
| 6210 | // Debug intrinsics are handled seperately in assignment tracking mode. | ||||||||
| 6211 | if (AssignmentTrackingEnabled) | ||||||||
| 6212 | return; | ||||||||
| 6213 | const DbgValueInst &DI = cast<DbgValueInst>(I); | ||||||||
| 6214 | assert(DI.getVariable() && "Missing variable")(static_cast <bool> (DI.getVariable() && "Missing variable" ) ? void (0) : __assert_fail ("DI.getVariable() && \"Missing variable\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6214 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6215 | |||||||||
| 6216 | DILocalVariable *Variable = DI.getVariable(); | ||||||||
| 6217 | DIExpression *Expression = DI.getExpression(); | ||||||||
| 6218 | dropDanglingDebugInfo(Variable, Expression); | ||||||||
| 6219 | |||||||||
| 6220 | if (DI.isKillLocation()) { | ||||||||
| 6221 | handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder); | ||||||||
| 6222 | return; | ||||||||
| 6223 | } | ||||||||
| 6224 | |||||||||
| 6225 | SmallVector<Value *, 4> Values(DI.getValues()); | ||||||||
| 6226 | if (Values.empty()) | ||||||||
| 6227 | return; | ||||||||
| 6228 | |||||||||
| 6229 | bool IsVariadic = DI.hasArgList(); | ||||||||
| 6230 | if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(), | ||||||||
| 6231 | SDNodeOrder, IsVariadic)) | ||||||||
| 6232 | addDanglingDebugInfo(&DI, SDNodeOrder); | ||||||||
| 6233 | return; | ||||||||
| 6234 | } | ||||||||
| 6235 | |||||||||
| 6236 | case Intrinsic::eh_typeid_for: { | ||||||||
| 6237 | // Find the type id for the given typeinfo. | ||||||||
| 6238 | GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0)); | ||||||||
| 6239 | unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV); | ||||||||
| 6240 | Res = DAG.getConstant(TypeID, sdl, MVT::i32); | ||||||||
| 6241 | setValue(&I, Res); | ||||||||
| 6242 | return; | ||||||||
| 6243 | } | ||||||||
| 6244 | |||||||||
| 6245 | case Intrinsic::eh_return_i32: | ||||||||
| 6246 | case Intrinsic::eh_return_i64: | ||||||||
| 6247 | DAG.getMachineFunction().setCallsEHReturn(true); | ||||||||
| 6248 | DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, | ||||||||
| 6249 | MVT::Other, | ||||||||
| 6250 | getControlRoot(), | ||||||||
| 6251 | getValue(I.getArgOperand(0)), | ||||||||
| 6252 | getValue(I.getArgOperand(1)))); | ||||||||
| 6253 | return; | ||||||||
| 6254 | case Intrinsic::eh_unwind_init: | ||||||||
| 6255 | DAG.getMachineFunction().setCallsUnwindInit(true); | ||||||||
| 6256 | return; | ||||||||
| 6257 | case Intrinsic::eh_dwarf_cfa: | ||||||||
| 6258 | setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl, | ||||||||
| 6259 | TLI.getPointerTy(DAG.getDataLayout()), | ||||||||
| 6260 | getValue(I.getArgOperand(0)))); | ||||||||
| 6261 | return; | ||||||||
| 6262 | case Intrinsic::eh_sjlj_callsite: { | ||||||||
| 6263 | MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); | ||||||||
| 6264 | ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0)); | ||||||||
| 6265 | assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!")(static_cast <bool> (MMI.getCurrentCallSite() == 0 && "Overlapping call sites!") ? void (0) : __assert_fail ("MMI.getCurrentCallSite() == 0 && \"Overlapping call sites!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6265 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6266 | |||||||||
| 6267 | MMI.setCurrentCallSite(CI->getZExtValue()); | ||||||||
| 6268 | return; | ||||||||
| 6269 | } | ||||||||
| 6270 | case Intrinsic::eh_sjlj_functioncontext: { | ||||||||
| 6271 | // Get and store the index of the function context. | ||||||||
| 6272 | MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); | ||||||||
| 6273 | AllocaInst *FnCtx = | ||||||||
| 6274 | cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); | ||||||||
| 6275 | int FI = FuncInfo.StaticAllocaMap[FnCtx]; | ||||||||
| 6276 | MFI.setFunctionContextIndex(FI); | ||||||||
| 6277 | return; | ||||||||
| 6278 | } | ||||||||
| 6279 | case Intrinsic::eh_sjlj_setjmp: { | ||||||||
| 6280 | SDValue Ops[2]; | ||||||||
| 6281 | Ops[0] = getRoot(); | ||||||||
| 6282 | Ops[1] = getValue(I.getArgOperand(0)); | ||||||||
| 6283 | SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl, | ||||||||
| 6284 | DAG.getVTList(MVT::i32, MVT::Other), Ops); | ||||||||
| 6285 | setValue(&I, Op.getValue(0)); | ||||||||
| 6286 | DAG.setRoot(Op.getValue(1)); | ||||||||
| 6287 | return; | ||||||||
| 6288 | } | ||||||||
| 6289 | case Intrinsic::eh_sjlj_longjmp: | ||||||||
| 6290 | DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, | ||||||||
| 6291 | getRoot(), getValue(I.getArgOperand(0)))); | ||||||||
| 6292 | return; | ||||||||
| 6293 | case Intrinsic::eh_sjlj_setup_dispatch: | ||||||||
| 6294 | DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, | ||||||||
| 6295 | getRoot())); | ||||||||
| 6296 | return; | ||||||||
| 6297 | case Intrinsic::masked_gather: | ||||||||
| 6298 | visitMaskedGather(I); | ||||||||
| 6299 | return; | ||||||||
| 6300 | case Intrinsic::masked_load: | ||||||||
| 6301 | visitMaskedLoad(I); | ||||||||
| 6302 | return; | ||||||||
| 6303 | case Intrinsic::masked_scatter: | ||||||||
| 6304 | visitMaskedScatter(I); | ||||||||
| 6305 | return; | ||||||||
| 6306 | case Intrinsic::masked_store: | ||||||||
| 6307 | visitMaskedStore(I); | ||||||||
| 6308 | return; | ||||||||
| 6309 | case Intrinsic::masked_expandload: | ||||||||
| 6310 | visitMaskedLoad(I, true /* IsExpanding */); | ||||||||
| 6311 | return; | ||||||||
| 6312 | case Intrinsic::masked_compressstore: | ||||||||
| 6313 | visitMaskedStore(I, true /* IsCompressing */); | ||||||||
| 6314 | return; | ||||||||
| 6315 | case Intrinsic::powi: | ||||||||
| 6316 | setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), | ||||||||
| 6317 | getValue(I.getArgOperand(1)), DAG)); | ||||||||
| 6318 | return; | ||||||||
| 6319 | case Intrinsic::log: | ||||||||
| 6320 | setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||
| 6321 | return; | ||||||||
| 6322 | case Intrinsic::log2: | ||||||||
| 6323 | setValue(&I, | ||||||||
| 6324 | expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||
| 6325 | return; | ||||||||
| 6326 | case Intrinsic::log10: | ||||||||
| 6327 | setValue(&I, | ||||||||
| 6328 | expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||
| 6329 | return; | ||||||||
| 6330 | case Intrinsic::exp: | ||||||||
| 6331 | setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||
| 6332 | return; | ||||||||
| 6333 | case Intrinsic::exp2: | ||||||||
| 6334 | setValue(&I, | ||||||||
| 6335 | expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); | ||||||||
| 6336 | return; | ||||||||
| 6337 | case Intrinsic::pow: | ||||||||
| 6338 | setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), | ||||||||
| 6339 | getValue(I.getArgOperand(1)), DAG, TLI, Flags)); | ||||||||
| 6340 | return; | ||||||||
| 6341 | case Intrinsic::sqrt: | ||||||||
| 6342 | case Intrinsic::fabs: | ||||||||
| 6343 | case Intrinsic::sin: | ||||||||
| 6344 | case Intrinsic::cos: | ||||||||
| 6345 | case Intrinsic::floor: | ||||||||
| 6346 | case Intrinsic::ceil: | ||||||||
| 6347 | case Intrinsic::trunc: | ||||||||
| 6348 | case Intrinsic::rint: | ||||||||
| 6349 | case Intrinsic::nearbyint: | ||||||||
| 6350 | case Intrinsic::round: | ||||||||
| 6351 | case Intrinsic::roundeven: | ||||||||
| 6352 | case Intrinsic::canonicalize: { | ||||||||
| 6353 | unsigned Opcode; | ||||||||
| 6354 | switch (Intrinsic) { | ||||||||
| 6355 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6355); // Can't reach here. | ||||||||
| 6356 | case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; | ||||||||
| 6357 | case Intrinsic::fabs: Opcode = ISD::FABS; break; | ||||||||
| 6358 | case Intrinsic::sin: Opcode = ISD::FSIN; break; | ||||||||
| 6359 | case Intrinsic::cos: Opcode = ISD::FCOS; break; | ||||||||
| 6360 | case Intrinsic::floor: Opcode = ISD::FFLOOR; break; | ||||||||
| 6361 | case Intrinsic::ceil: Opcode = ISD::FCEIL; break; | ||||||||
| 6362 | case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; | ||||||||
| 6363 | case Intrinsic::rint: Opcode = ISD::FRINT; break; | ||||||||
| 6364 | case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; | ||||||||
| 6365 | case Intrinsic::round: Opcode = ISD::FROUND; break; | ||||||||
| 6366 | case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break; | ||||||||
| 6367 | case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break; | ||||||||
| 6368 | } | ||||||||
| 6369 | |||||||||
| 6370 | setValue(&I, DAG.getNode(Opcode, sdl, | ||||||||
| 6371 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6372 | getValue(I.getArgOperand(0)), Flags)); | ||||||||
| 6373 | return; | ||||||||
| 6374 | } | ||||||||
| 6375 | case Intrinsic::lround: | ||||||||
| 6376 | case Intrinsic::llround: | ||||||||
| 6377 | case Intrinsic::lrint: | ||||||||
| 6378 | case Intrinsic::llrint: { | ||||||||
| 6379 | unsigned Opcode; | ||||||||
| 6380 | switch (Intrinsic) { | ||||||||
| 6381 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6381); // Can't reach here. | ||||||||
| 6382 | case Intrinsic::lround: Opcode = ISD::LROUND; break; | ||||||||
| 6383 | case Intrinsic::llround: Opcode = ISD::LLROUND; break; | ||||||||
| 6384 | case Intrinsic::lrint: Opcode = ISD::LRINT; break; | ||||||||
| 6385 | case Intrinsic::llrint: Opcode = ISD::LLRINT; break; | ||||||||
| 6386 | } | ||||||||
| 6387 | |||||||||
| 6388 | EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6389 | setValue(&I, DAG.getNode(Opcode, sdl, RetVT, | ||||||||
| 6390 | getValue(I.getArgOperand(0)))); | ||||||||
| 6391 | return; | ||||||||
| 6392 | } | ||||||||
| 6393 | case Intrinsic::minnum: | ||||||||
| 6394 | setValue(&I, DAG.getNode(ISD::FMINNUM, sdl, | ||||||||
| 6395 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6396 | getValue(I.getArgOperand(0)), | ||||||||
| 6397 | getValue(I.getArgOperand(1)), Flags)); | ||||||||
| 6398 | return; | ||||||||
| 6399 | case Intrinsic::maxnum: | ||||||||
| 6400 | setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl, | ||||||||
| 6401 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6402 | getValue(I.getArgOperand(0)), | ||||||||
| 6403 | getValue(I.getArgOperand(1)), Flags)); | ||||||||
| 6404 | return; | ||||||||
| 6405 | case Intrinsic::minimum: | ||||||||
| 6406 | setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl, | ||||||||
| 6407 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6408 | getValue(I.getArgOperand(0)), | ||||||||
| 6409 | getValue(I.getArgOperand(1)), Flags)); | ||||||||
| 6410 | return; | ||||||||
| 6411 | case Intrinsic::maximum: | ||||||||
| 6412 | setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl, | ||||||||
| 6413 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6414 | getValue(I.getArgOperand(0)), | ||||||||
| 6415 | getValue(I.getArgOperand(1)), Flags)); | ||||||||
| 6416 | return; | ||||||||
| 6417 | case Intrinsic::copysign: | ||||||||
| 6418 | setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, | ||||||||
| 6419 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6420 | getValue(I.getArgOperand(0)), | ||||||||
| 6421 | getValue(I.getArgOperand(1)), Flags)); | ||||||||
| 6422 | return; | ||||||||
| 6423 | case Intrinsic::arithmetic_fence: { | ||||||||
| 6424 | setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl, | ||||||||
| 6425 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6426 | getValue(I.getArgOperand(0)), Flags)); | ||||||||
| 6427 | return; | ||||||||
| 6428 | } | ||||||||
| 6429 | case Intrinsic::fma: | ||||||||
| 6430 | setValue(&I, DAG.getNode( | ||||||||
| 6431 | ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6432 | getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), | ||||||||
| 6433 | getValue(I.getArgOperand(2)), Flags)); | ||||||||
| 6434 | return; | ||||||||
| 6435 | #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ | ||||||||
| 6436 | case Intrinsic::INTRINSIC: | ||||||||
| 6437 | #include "llvm/IR/ConstrainedOps.def" | ||||||||
| 6438 | visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I)); | ||||||||
| 6439 | return; | ||||||||
| 6440 | #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID: | ||||||||
| 6441 | #include "llvm/IR/VPIntrinsics.def" | ||||||||
| 6442 | visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I)); | ||||||||
| 6443 | return; | ||||||||
| 6444 | case Intrinsic::fptrunc_round: { | ||||||||
| 6445 | // Get the last argument, the metadata and convert it to an integer in the | ||||||||
| 6446 | // call | ||||||||
| 6447 | Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata(); | ||||||||
| 6448 | std::optional<RoundingMode> RoundMode = | ||||||||
| 6449 | convertStrToRoundingMode(cast<MDString>(MD)->getString()); | ||||||||
| 6450 | |||||||||
| 6451 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6452 | |||||||||
| 6453 | // Propagate fast-math-flags from IR to node(s). | ||||||||
| 6454 | SDNodeFlags Flags; | ||||||||
| 6455 | Flags.copyFMF(*cast<FPMathOperator>(&I)); | ||||||||
| 6456 | SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); | ||||||||
| 6457 | |||||||||
| 6458 | SDValue Result; | ||||||||
| 6459 | Result = DAG.getNode( | ||||||||
| 6460 | ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)), | ||||||||
| 6461 | DAG.getTargetConstant((int)*RoundMode, sdl, | ||||||||
| 6462 | TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 6463 | setValue(&I, Result); | ||||||||
| 6464 | |||||||||
| 6465 | return; | ||||||||
| 6466 | } | ||||||||
| 6467 | case Intrinsic::fmuladd: { | ||||||||
| 6468 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6469 | if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && | ||||||||
| 6470 | TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { | ||||||||
| 6471 | setValue(&I, DAG.getNode(ISD::FMA, sdl, | ||||||||
| 6472 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6473 | getValue(I.getArgOperand(0)), | ||||||||
| 6474 | getValue(I.getArgOperand(1)), | ||||||||
| 6475 | getValue(I.getArgOperand(2)), Flags)); | ||||||||
| 6476 | } else { | ||||||||
| 6477 | // TODO: Intrinsic calls should have fast-math-flags. | ||||||||
| 6478 | SDValue Mul = DAG.getNode( | ||||||||
| 6479 | ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6480 | getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags); | ||||||||
| 6481 | SDValue Add = DAG.getNode(ISD::FADD, sdl, | ||||||||
| 6482 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6483 | Mul, getValue(I.getArgOperand(2)), Flags); | ||||||||
| 6484 | setValue(&I, Add); | ||||||||
| 6485 | } | ||||||||
| 6486 | return; | ||||||||
| 6487 | } | ||||||||
| 6488 | case Intrinsic::convert_to_fp16: | ||||||||
| 6489 | setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16, | ||||||||
| 6490 | DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16, | ||||||||
| 6491 | getValue(I.getArgOperand(0)), | ||||||||
| 6492 | DAG.getTargetConstant(0, sdl, | ||||||||
| 6493 | MVT::i32)))); | ||||||||
| 6494 | return; | ||||||||
| 6495 | case Intrinsic::convert_from_fp16: | ||||||||
| 6496 | setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl, | ||||||||
| 6497 | TLI.getValueType(DAG.getDataLayout(), I.getType()), | ||||||||
| 6498 | DAG.getNode(ISD::BITCAST, sdl, MVT::f16, | ||||||||
| 6499 | getValue(I.getArgOperand(0))))); | ||||||||
| 6500 | return; | ||||||||
| 6501 | case Intrinsic::fptosi_sat: { | ||||||||
| 6502 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6503 | setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT, | ||||||||
| 6504 | getValue(I.getArgOperand(0)), | ||||||||
| 6505 | DAG.getValueType(VT.getScalarType()))); | ||||||||
| 6506 | return; | ||||||||
| 6507 | } | ||||||||
| 6508 | case Intrinsic::fptoui_sat: { | ||||||||
| 6509 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6510 | setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT, | ||||||||
| 6511 | getValue(I.getArgOperand(0)), | ||||||||
| 6512 | DAG.getValueType(VT.getScalarType()))); | ||||||||
| 6513 | return; | ||||||||
| 6514 | } | ||||||||
| 6515 | case Intrinsic::set_rounding: | ||||||||
| 6516 | Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other, | ||||||||
| 6517 | {getRoot(), getValue(I.getArgOperand(0))}); | ||||||||
| 6518 | setValue(&I, Res); | ||||||||
| 6519 | DAG.setRoot(Res.getValue(0)); | ||||||||
| 6520 | return; | ||||||||
| 6521 | case Intrinsic::is_fpclass: { | ||||||||
| 6522 | const DataLayout DLayout = DAG.getDataLayout(); | ||||||||
| 6523 | EVT DestVT = TLI.getValueType(DLayout, I.getType()); | ||||||||
| 6524 | EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType()); | ||||||||
| 6525 | FPClassTest Test = static_cast<FPClassTest>( | ||||||||
| 6526 | cast<ConstantInt>(I.getArgOperand(1))->getZExtValue()); | ||||||||
| 6527 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 6528 | const Function &F = MF.getFunction(); | ||||||||
| 6529 | SDValue Op = getValue(I.getArgOperand(0)); | ||||||||
| 6530 | SDNodeFlags Flags; | ||||||||
| 6531 | Flags.setNoFPExcept( | ||||||||
| 6532 | !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP)); | ||||||||
| 6533 | // If ISD::IS_FPCLASS should be expanded, do it right now, because the | ||||||||
| 6534 | // expansion can use illegal types. Making expansion early allows | ||||||||
| 6535 | // legalizing these types prior to selection. | ||||||||
| 6536 | if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) { | ||||||||
| 6537 | SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG); | ||||||||
| 6538 | setValue(&I, Result); | ||||||||
| 6539 | return; | ||||||||
| 6540 | } | ||||||||
| 6541 | |||||||||
| 6542 | SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32); | ||||||||
| 6543 | SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags); | ||||||||
| 6544 | setValue(&I, V); | ||||||||
| 6545 | return; | ||||||||
| 6546 | } | ||||||||
| 6547 | case Intrinsic::pcmarker: { | ||||||||
| 6548 | SDValue Tmp = getValue(I.getArgOperand(0)); | ||||||||
| 6549 | DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); | ||||||||
| 6550 | return; | ||||||||
| 6551 | } | ||||||||
| 6552 | case Intrinsic::readcyclecounter: { | ||||||||
| 6553 | SDValue Op = getRoot(); | ||||||||
| 6554 | Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl, | ||||||||
| 6555 | DAG.getVTList(MVT::i64, MVT::Other), Op); | ||||||||
| 6556 | setValue(&I, Res); | ||||||||
| 6557 | DAG.setRoot(Res.getValue(1)); | ||||||||
| 6558 | return; | ||||||||
| 6559 | } | ||||||||
| 6560 | case Intrinsic::bitreverse: | ||||||||
| 6561 | setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl, | ||||||||
| 6562 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6563 | getValue(I.getArgOperand(0)))); | ||||||||
| 6564 | return; | ||||||||
| 6565 | case Intrinsic::bswap: | ||||||||
| 6566 | setValue(&I, DAG.getNode(ISD::BSWAP, sdl, | ||||||||
| 6567 | getValue(I.getArgOperand(0)).getValueType(), | ||||||||
| 6568 | getValue(I.getArgOperand(0)))); | ||||||||
| 6569 | return; | ||||||||
| 6570 | case Intrinsic::cttz: { | ||||||||
| 6571 | SDValue Arg = getValue(I.getArgOperand(0)); | ||||||||
| 6572 | ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); | ||||||||
| 6573 | EVT Ty = Arg.getValueType(); | ||||||||
| 6574 | setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, | ||||||||
| 6575 | sdl, Ty, Arg)); | ||||||||
| 6576 | return; | ||||||||
| 6577 | } | ||||||||
| 6578 | case Intrinsic::ctlz: { | ||||||||
| 6579 | SDValue Arg = getValue(I.getArgOperand(0)); | ||||||||
| 6580 | ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); | ||||||||
| 6581 | EVT Ty = Arg.getValueType(); | ||||||||
| 6582 | setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, | ||||||||
| 6583 | sdl, Ty, Arg)); | ||||||||
| 6584 | return; | ||||||||
| 6585 | } | ||||||||
| 6586 | case Intrinsic::ctpop: { | ||||||||
| 6587 | SDValue Arg = getValue(I.getArgOperand(0)); | ||||||||
| 6588 | EVT Ty = Arg.getValueType(); | ||||||||
| 6589 | setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); | ||||||||
| 6590 | return; | ||||||||
| 6591 | } | ||||||||
| 6592 | case Intrinsic::fshl: | ||||||||
| 6593 | case Intrinsic::fshr: { | ||||||||
| 6594 | bool IsFSHL = Intrinsic == Intrinsic::fshl; | ||||||||
| 6595 | SDValue X = getValue(I.getArgOperand(0)); | ||||||||
| 6596 | SDValue Y = getValue(I.getArgOperand(1)); | ||||||||
| 6597 | SDValue Z = getValue(I.getArgOperand(2)); | ||||||||
| 6598 | EVT VT = X.getValueType(); | ||||||||
| 6599 | |||||||||
| 6600 | if (X == Y) { | ||||||||
| 6601 | auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR; | ||||||||
| 6602 | setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z)); | ||||||||
| 6603 | } else { | ||||||||
| 6604 | auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR; | ||||||||
| 6605 | setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z)); | ||||||||
| 6606 | } | ||||||||
| 6607 | return; | ||||||||
| 6608 | } | ||||||||
| 6609 | case Intrinsic::sadd_sat: { | ||||||||
| 6610 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6611 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6612 | setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6613 | return; | ||||||||
| 6614 | } | ||||||||
| 6615 | case Intrinsic::uadd_sat: { | ||||||||
| 6616 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6617 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6618 | setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6619 | return; | ||||||||
| 6620 | } | ||||||||
| 6621 | case Intrinsic::ssub_sat: { | ||||||||
| 6622 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6623 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6624 | setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6625 | return; | ||||||||
| 6626 | } | ||||||||
| 6627 | case Intrinsic::usub_sat: { | ||||||||
| 6628 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6629 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6630 | setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6631 | return; | ||||||||
| 6632 | } | ||||||||
| 6633 | case Intrinsic::sshl_sat: { | ||||||||
| 6634 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6635 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6636 | setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6637 | return; | ||||||||
| 6638 | } | ||||||||
| 6639 | case Intrinsic::ushl_sat: { | ||||||||
| 6640 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6641 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6642 | setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6643 | return; | ||||||||
| 6644 | } | ||||||||
| 6645 | case Intrinsic::smul_fix: | ||||||||
| 6646 | case Intrinsic::umul_fix: | ||||||||
| 6647 | case Intrinsic::smul_fix_sat: | ||||||||
| 6648 | case Intrinsic::umul_fix_sat: { | ||||||||
| 6649 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6650 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6651 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||
| 6652 | setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl, | ||||||||
| 6653 | Op1.getValueType(), Op1, Op2, Op3)); | ||||||||
| 6654 | return; | ||||||||
| 6655 | } | ||||||||
| 6656 | case Intrinsic::sdiv_fix: | ||||||||
| 6657 | case Intrinsic::udiv_fix: | ||||||||
| 6658 | case Intrinsic::sdiv_fix_sat: | ||||||||
| 6659 | case Intrinsic::udiv_fix_sat: { | ||||||||
| 6660 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6661 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6662 | SDValue Op3 = getValue(I.getArgOperand(2)); | ||||||||
| 6663 | setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl, | ||||||||
| 6664 | Op1, Op2, Op3, DAG, TLI)); | ||||||||
| 6665 | return; | ||||||||
| 6666 | } | ||||||||
| 6667 | case Intrinsic::smax: { | ||||||||
| 6668 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6669 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6670 | setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6671 | return; | ||||||||
| 6672 | } | ||||||||
| 6673 | case Intrinsic::smin: { | ||||||||
| 6674 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6675 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6676 | setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6677 | return; | ||||||||
| 6678 | } | ||||||||
| 6679 | case Intrinsic::umax: { | ||||||||
| 6680 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6681 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6682 | setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6683 | return; | ||||||||
| 6684 | } | ||||||||
| 6685 | case Intrinsic::umin: { | ||||||||
| 6686 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6687 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6688 | setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2)); | ||||||||
| 6689 | return; | ||||||||
| 6690 | } | ||||||||
| 6691 | case Intrinsic::abs: { | ||||||||
| 6692 | // TODO: Preserve "int min is poison" arg in SDAG? | ||||||||
| 6693 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6694 | setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1)); | ||||||||
| 6695 | return; | ||||||||
| 6696 | } | ||||||||
| 6697 | case Intrinsic::stacksave: { | ||||||||
| 6698 | SDValue Op = getRoot(); | ||||||||
| 6699 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6700 | Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op); | ||||||||
| 6701 | setValue(&I, Res); | ||||||||
| 6702 | DAG.setRoot(Res.getValue(1)); | ||||||||
| 6703 | return; | ||||||||
| 6704 | } | ||||||||
| 6705 | case Intrinsic::stackrestore: | ||||||||
| 6706 | Res = getValue(I.getArgOperand(0)); | ||||||||
| 6707 | DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); | ||||||||
| 6708 | return; | ||||||||
| 6709 | case Intrinsic::get_dynamic_area_offset: { | ||||||||
| 6710 | SDValue Op = getRoot(); | ||||||||
| 6711 | EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); | ||||||||
| 6712 | EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6713 | // Result type for @llvm.get.dynamic.area.offset should match PtrTy for | ||||||||
| 6714 | // target. | ||||||||
| 6715 | if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits()) | ||||||||
| 6716 | report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset" | ||||||||
| 6717 | " intrinsic!"); | ||||||||
| 6718 | Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy), | ||||||||
| 6719 | Op); | ||||||||
| 6720 | DAG.setRoot(Op); | ||||||||
| 6721 | setValue(&I, Res); | ||||||||
| 6722 | return; | ||||||||
| 6723 | } | ||||||||
| 6724 | case Intrinsic::stackguard: { | ||||||||
| 6725 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 6726 | const Module &M = *MF.getFunction().getParent(); | ||||||||
| 6727 | SDValue Chain = getRoot(); | ||||||||
| 6728 | if (TLI.useLoadStackGuardNode()) { | ||||||||
| 6729 | Res = getLoadStackGuard(DAG, sdl, Chain); | ||||||||
| 6730 | } else { | ||||||||
| 6731 | EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 6732 | const Value *Global = TLI.getSDagStackGuard(M); | ||||||||
| 6733 | Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType()); | ||||||||
| 6734 | Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global), | ||||||||
| 6735 | MachinePointerInfo(Global, 0), Align, | ||||||||
| 6736 | MachineMemOperand::MOVolatile); | ||||||||
| 6737 | } | ||||||||
| 6738 | if (TLI.useStackGuardXorFP()) | ||||||||
| 6739 | Res = TLI.emitStackGuardXorFP(DAG, Res, sdl); | ||||||||
| 6740 | DAG.setRoot(Chain); | ||||||||
| 6741 | setValue(&I, Res); | ||||||||
| 6742 | return; | ||||||||
| 6743 | } | ||||||||
| 6744 | case Intrinsic::stackprotector: { | ||||||||
| 6745 | // Emit code into the DAG to store the stack guard onto the stack. | ||||||||
| 6746 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 6747 | MachineFrameInfo &MFI = MF.getFrameInfo(); | ||||||||
| 6748 | SDValue Src, Chain = getRoot(); | ||||||||
| 6749 | |||||||||
| 6750 | if (TLI.useLoadStackGuardNode()) | ||||||||
| 6751 | Src = getLoadStackGuard(DAG, sdl, Chain); | ||||||||
| 6752 | else | ||||||||
| 6753 | Src = getValue(I.getArgOperand(0)); // The guard's value. | ||||||||
| 6754 | |||||||||
| 6755 | AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); | ||||||||
| 6756 | |||||||||
| 6757 | int FI = FuncInfo.StaticAllocaMap[Slot]; | ||||||||
| 6758 | MFI.setStackProtectorIndex(FI); | ||||||||
| 6759 | EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); | ||||||||
| 6760 | |||||||||
| 6761 | SDValue FIN = DAG.getFrameIndex(FI, PtrTy); | ||||||||
| 6762 | |||||||||
| 6763 | // Store the stack protector onto the stack. | ||||||||
| 6764 | Res = DAG.getStore( | ||||||||
| 6765 | Chain, sdl, Src, FIN, | ||||||||
| 6766 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), | ||||||||
| 6767 | MaybeAlign(), MachineMemOperand::MOVolatile); | ||||||||
| 6768 | setValue(&I, Res); | ||||||||
| 6769 | DAG.setRoot(Res); | ||||||||
| 6770 | return; | ||||||||
| 6771 | } | ||||||||
| 6772 | case Intrinsic::objectsize: | ||||||||
| 6773 | llvm_unreachable("llvm.objectsize.* should have been lowered already")::llvm::llvm_unreachable_internal("llvm.objectsize.* should have been lowered already" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6773 ); | ||||||||
| 6774 | |||||||||
| 6775 | case Intrinsic::is_constant: | ||||||||
| 6776 | llvm_unreachable("llvm.is.constant.* should have been lowered already")::llvm::llvm_unreachable_internal("llvm.is.constant.* should have been lowered already" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6776 ); | ||||||||
| 6777 | |||||||||
| 6778 | case Intrinsic::annotation: | ||||||||
| 6779 | case Intrinsic::ptr_annotation: | ||||||||
| 6780 | case Intrinsic::launder_invariant_group: | ||||||||
| 6781 | case Intrinsic::strip_invariant_group: | ||||||||
| 6782 | // Drop the intrinsic, but forward the value | ||||||||
| 6783 | setValue(&I, getValue(I.getOperand(0))); | ||||||||
| 6784 | return; | ||||||||
| 6785 | |||||||||
| 6786 | case Intrinsic::assume: | ||||||||
| 6787 | case Intrinsic::experimental_noalias_scope_decl: | ||||||||
| 6788 | case Intrinsic::var_annotation: | ||||||||
| 6789 | case Intrinsic::sideeffect: | ||||||||
| 6790 | // Discard annotate attributes, noalias scope declarations, assumptions, and | ||||||||
| 6791 | // artificial side-effects. | ||||||||
| 6792 | return; | ||||||||
| 6793 | |||||||||
| 6794 | case Intrinsic::codeview_annotation: { | ||||||||
| 6795 | // Emit a label associated with this metadata. | ||||||||
| 6796 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 6797 | MCSymbol *Label = | ||||||||
| 6798 | MF.getMMI().getContext().createTempSymbol("annotation", true); | ||||||||
| 6799 | Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata(); | ||||||||
| 6800 | MF.addCodeViewAnnotation(Label, cast<MDNode>(MD)); | ||||||||
| 6801 | Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label); | ||||||||
| 6802 | DAG.setRoot(Res); | ||||||||
| 6803 | return; | ||||||||
| 6804 | } | ||||||||
| 6805 | |||||||||
| 6806 | case Intrinsic::init_trampoline: { | ||||||||
| 6807 | const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts()); | ||||||||
| 6808 | |||||||||
| 6809 | SDValue Ops[6]; | ||||||||
| 6810 | Ops[0] = getRoot(); | ||||||||
| 6811 | Ops[1] = getValue(I.getArgOperand(0)); | ||||||||
| 6812 | Ops[2] = getValue(I.getArgOperand(1)); | ||||||||
| 6813 | Ops[3] = getValue(I.getArgOperand(2)); | ||||||||
| 6814 | Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); | ||||||||
| 6815 | Ops[5] = DAG.getSrcValue(F); | ||||||||
| 6816 | |||||||||
| 6817 | Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops); | ||||||||
| 6818 | |||||||||
| 6819 | DAG.setRoot(Res); | ||||||||
| 6820 | return; | ||||||||
| 6821 | } | ||||||||
| 6822 | case Intrinsic::adjust_trampoline: | ||||||||
| 6823 | setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, | ||||||||
| 6824 | TLI.getPointerTy(DAG.getDataLayout()), | ||||||||
| 6825 | getValue(I.getArgOperand(0)))); | ||||||||
| 6826 | return; | ||||||||
| 6827 | case Intrinsic::gcroot: { | ||||||||
| 6828 | assert(DAG.getMachineFunction().getFunction().hasGC() &&(static_cast <bool> (DAG.getMachineFunction().getFunction ().hasGC() && "only valid in functions with gc specified, enforced by Verifier" ) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6829 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 6829 | "only valid in functions with gc specified, enforced by Verifier")(static_cast <bool> (DAG.getMachineFunction().getFunction ().hasGC() && "only valid in functions with gc specified, enforced by Verifier" ) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6829 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6830 | assert(GFI && "implied by previous")(static_cast <bool> (GFI && "implied by previous" ) ? void (0) : __assert_fail ("GFI && \"implied by previous\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6830 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 6831 | const Value *Alloca = I.getArgOperand(0)->stripPointerCasts(); | ||||||||
| 6832 | const Constant *TypeMap = cast<Constant>(I.getArgOperand(1)); | ||||||||
| 6833 | |||||||||
| 6834 | FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode()); | ||||||||
| 6835 | GFI->addStackRoot(FI->getIndex(), TypeMap); | ||||||||
| 6836 | return; | ||||||||
| 6837 | } | ||||||||
| 6838 | case Intrinsic::gcread: | ||||||||
| 6839 | case Intrinsic::gcwrite: | ||||||||
| 6840 | llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!")::llvm::llvm_unreachable_internal("GC failed to lower gcread/gcwrite intrinsics!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6840 ); | ||||||||
| 6841 | case Intrinsic::get_rounding: | ||||||||
| 6842 | Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot()); | ||||||||
| 6843 | setValue(&I, Res); | ||||||||
| 6844 | DAG.setRoot(Res.getValue(1)); | ||||||||
| 6845 | return; | ||||||||
| 6846 | |||||||||
| 6847 | case Intrinsic::expect: | ||||||||
| 6848 | // Just replace __builtin_expect(exp, c) with EXP. | ||||||||
| 6849 | setValue(&I, getValue(I.getArgOperand(0))); | ||||||||
| 6850 | return; | ||||||||
| 6851 | |||||||||
| 6852 | case Intrinsic::ubsantrap: | ||||||||
| 6853 | case Intrinsic::debugtrap: | ||||||||
| 6854 | case Intrinsic::trap: { | ||||||||
| 6855 | StringRef TrapFuncName = | ||||||||
| 6856 | I.getAttributes().getFnAttr("trap-func-name").getValueAsString(); | ||||||||
| 6857 | if (TrapFuncName.empty()) { | ||||||||
| 6858 | switch (Intrinsic) { | ||||||||
| 6859 | case Intrinsic::trap: | ||||||||
| 6860 | DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot())); | ||||||||
| 6861 | break; | ||||||||
| 6862 | case Intrinsic::debugtrap: | ||||||||
| 6863 | DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot())); | ||||||||
| 6864 | break; | ||||||||
| 6865 | case Intrinsic::ubsantrap: | ||||||||
| 6866 | DAG.setRoot(DAG.getNode( | ||||||||
| 6867 | ISD::UBSANTRAP, sdl, MVT::Other, getRoot(), | ||||||||
| 6868 | DAG.getTargetConstant( | ||||||||
| 6869 | cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl, | ||||||||
| 6870 | MVT::i32))); | ||||||||
| 6871 | break; | ||||||||
| 6872 | default: llvm_unreachable("unknown trap intrinsic")::llvm::llvm_unreachable_internal("unknown trap intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6872); | ||||||||
| 6873 | } | ||||||||
| 6874 | return; | ||||||||
| 6875 | } | ||||||||
| 6876 | TargetLowering::ArgListTy Args; | ||||||||
| 6877 | if (Intrinsic == Intrinsic::ubsantrap) { | ||||||||
| 6878 | Args.push_back(TargetLoweringBase::ArgListEntry()); | ||||||||
| 6879 | Args[0].Val = I.getArgOperand(0); | ||||||||
| 6880 | Args[0].Node = getValue(Args[0].Val); | ||||||||
| 6881 | Args[0].Ty = Args[0].Val->getType(); | ||||||||
| 6882 | } | ||||||||
| 6883 | |||||||||
| 6884 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||
| 6885 | CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee( | ||||||||
| 6886 | CallingConv::C, I.getType(), | ||||||||
| 6887 | DAG.getExternalSymbol(TrapFuncName.data(), | ||||||||
| 6888 | TLI.getPointerTy(DAG.getDataLayout())), | ||||||||
| 6889 | std::move(Args)); | ||||||||
| 6890 | |||||||||
| 6891 | std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); | ||||||||
| 6892 | DAG.setRoot(Result.second); | ||||||||
| 6893 | return; | ||||||||
| 6894 | } | ||||||||
| 6895 | |||||||||
| 6896 | case Intrinsic::uadd_with_overflow: | ||||||||
| 6897 | case Intrinsic::sadd_with_overflow: | ||||||||
| 6898 | case Intrinsic::usub_with_overflow: | ||||||||
| 6899 | case Intrinsic::ssub_with_overflow: | ||||||||
| 6900 | case Intrinsic::umul_with_overflow: | ||||||||
| 6901 | case Intrinsic::smul_with_overflow: { | ||||||||
| 6902 | ISD::NodeType Op; | ||||||||
| 6903 | switch (Intrinsic) { | ||||||||
| 6904 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 6904); // Can't reach here. | ||||||||
| 6905 | case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break; | ||||||||
| 6906 | case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break; | ||||||||
| 6907 | case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break; | ||||||||
| 6908 | case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break; | ||||||||
| 6909 | case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break; | ||||||||
| 6910 | case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break; | ||||||||
| 6911 | } | ||||||||
| 6912 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 6913 | SDValue Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 6914 | |||||||||
| 6915 | EVT ResultVT = Op1.getValueType(); | ||||||||
| 6916 | EVT OverflowVT = MVT::i1; | ||||||||
| 6917 | if (ResultVT.isVector()) | ||||||||
| 6918 | OverflowVT = EVT::getVectorVT( | ||||||||
| 6919 | *Context, OverflowVT, ResultVT.getVectorElementCount()); | ||||||||
| 6920 | |||||||||
| 6921 | SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT); | ||||||||
| 6922 | setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); | ||||||||
| 6923 | return; | ||||||||
| 6924 | } | ||||||||
| 6925 | case Intrinsic::prefetch: { | ||||||||
| 6926 | SDValue Ops[5]; | ||||||||
| 6927 | unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); | ||||||||
| 6928 | auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore; | ||||||||
| 6929 | Ops[0] = DAG.getRoot(); | ||||||||
| 6930 | Ops[1] = getValue(I.getArgOperand(0)); | ||||||||
| 6931 | Ops[2] = getValue(I.getArgOperand(1)); | ||||||||
| 6932 | Ops[3] = getValue(I.getArgOperand(2)); | ||||||||
| 6933 | Ops[4] = getValue(I.getArgOperand(3)); | ||||||||
| 6934 | SDValue Result = DAG.getMemIntrinsicNode( | ||||||||
| 6935 | ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops, | ||||||||
| 6936 | EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)), | ||||||||
| 6937 | /* align */ std::nullopt, Flags); | ||||||||
| 6938 | |||||||||
| 6939 | // Chain the prefetch in parallell with any pending loads, to stay out of | ||||||||
| 6940 | // the way of later optimizations. | ||||||||
| 6941 | PendingLoads.push_back(Result); | ||||||||
| 6942 | Result = getRoot(); | ||||||||
| 6943 | DAG.setRoot(Result); | ||||||||
| 6944 | return; | ||||||||
| 6945 | } | ||||||||
| 6946 | case Intrinsic::lifetime_start: | ||||||||
| 6947 | case Intrinsic::lifetime_end: { | ||||||||
| 6948 | bool IsStart = (Intrinsic == Intrinsic::lifetime_start); | ||||||||
| 6949 | // Stack coloring is not enabled in O0, discard region information. | ||||||||
| 6950 | if (TM.getOptLevel() == CodeGenOpt::None) | ||||||||
| 6951 | return; | ||||||||
| 6952 | |||||||||
| 6953 | const int64_t ObjectSize = | ||||||||
| 6954 | cast<ConstantInt>(I.getArgOperand(0))->getSExtValue(); | ||||||||
| 6955 | Value *const ObjectPtr = I.getArgOperand(1); | ||||||||
| 6956 | SmallVector<const Value *, 4> Allocas; | ||||||||
| 6957 | getUnderlyingObjects(ObjectPtr, Allocas); | ||||||||
| 6958 | |||||||||
| 6959 | for (const Value *Alloca : Allocas) { | ||||||||
| 6960 | const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca); | ||||||||
| 6961 | |||||||||
| 6962 | // Could not find an Alloca. | ||||||||
| 6963 | if (!LifetimeObject) | ||||||||
| 6964 | continue; | ||||||||
| 6965 | |||||||||
| 6966 | // First check that the Alloca is static, otherwise it won't have a | ||||||||
| 6967 | // valid frame index. | ||||||||
| 6968 | auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject); | ||||||||
| 6969 | if (SI == FuncInfo.StaticAllocaMap.end()) | ||||||||
| 6970 | return; | ||||||||
| 6971 | |||||||||
| 6972 | const int FrameIndex = SI->second; | ||||||||
| 6973 | int64_t Offset; | ||||||||
| 6974 | if (GetPointerBaseWithConstantOffset( | ||||||||
| 6975 | ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject) | ||||||||
| 6976 | Offset = -1; // Cannot determine offset from alloca to lifetime object. | ||||||||
| 6977 | Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize, | ||||||||
| 6978 | Offset); | ||||||||
| 6979 | DAG.setRoot(Res); | ||||||||
| 6980 | } | ||||||||
| 6981 | return; | ||||||||
| 6982 | } | ||||||||
| 6983 | case Intrinsic::pseudoprobe: { | ||||||||
| 6984 | auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(); | ||||||||
| 6985 | auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); | ||||||||
| 6986 | auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue(); | ||||||||
| 6987 | Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr); | ||||||||
| 6988 | DAG.setRoot(Res); | ||||||||
| 6989 | return; | ||||||||
| 6990 | } | ||||||||
| 6991 | case Intrinsic::invariant_start: | ||||||||
| 6992 | // Discard region information. | ||||||||
| 6993 | setValue(&I, | ||||||||
| 6994 | DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType()))); | ||||||||
| 6995 | return; | ||||||||
| 6996 | case Intrinsic::invariant_end: | ||||||||
| 6997 | // Discard region information. | ||||||||
| 6998 | return; | ||||||||
| 6999 | case Intrinsic::clear_cache: | ||||||||
| 7000 | /// FunctionName may be null. | ||||||||
| 7001 | if (const char *FunctionName = TLI.getClearCacheBuiltinName()) | ||||||||
| 7002 | lowerCallToExternalSymbol(I, FunctionName); | ||||||||
| 7003 | return; | ||||||||
| 7004 | case Intrinsic::donothing: | ||||||||
| 7005 | case Intrinsic::seh_try_begin: | ||||||||
| 7006 | case Intrinsic::seh_scope_begin: | ||||||||
| 7007 | case Intrinsic::seh_try_end: | ||||||||
| 7008 | case Intrinsic::seh_scope_end: | ||||||||
| 7009 | // ignore | ||||||||
| 7010 | return; | ||||||||
| 7011 | case Intrinsic::experimental_stackmap: | ||||||||
| 7012 | visitStackmap(I); | ||||||||
| 7013 | return; | ||||||||
| 7014 | case Intrinsic::experimental_patchpoint_void: | ||||||||
| 7015 | case Intrinsic::experimental_patchpoint_i64: | ||||||||
| 7016 | visitPatchpoint(I); | ||||||||
| 7017 | return; | ||||||||
| 7018 | case Intrinsic::experimental_gc_statepoint: | ||||||||
| 7019 | LowerStatepoint(cast<GCStatepointInst>(I)); | ||||||||
| 7020 | return; | ||||||||
| 7021 | case Intrinsic::experimental_gc_result: | ||||||||
| 7022 | visitGCResult(cast<GCResultInst>(I)); | ||||||||
| 7023 | return; | ||||||||
| 7024 | case Intrinsic::experimental_gc_relocate: | ||||||||
| 7025 | visitGCRelocate(cast<GCRelocateInst>(I)); | ||||||||
| 7026 | return; | ||||||||
| 7027 | case Intrinsic::instrprof_cover: | ||||||||
| 7028 | llvm_unreachable("instrprof failed to lower a cover")::llvm::llvm_unreachable_internal("instrprof failed to lower a cover" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7028 ); | ||||||||
| 7029 | case Intrinsic::instrprof_increment: | ||||||||
| 7030 | llvm_unreachable("instrprof failed to lower an increment")::llvm::llvm_unreachable_internal("instrprof failed to lower an increment" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7030 ); | ||||||||
| 7031 | case Intrinsic::instrprof_timestamp: | ||||||||
| 7032 | llvm_unreachable("instrprof failed to lower a timestamp")::llvm::llvm_unreachable_internal("instrprof failed to lower a timestamp" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7032 ); | ||||||||
| 7033 | case Intrinsic::instrprof_value_profile: | ||||||||
| 7034 | llvm_unreachable("instrprof failed to lower a value profiling call")::llvm::llvm_unreachable_internal("instrprof failed to lower a value profiling call" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7034 ); | ||||||||
| 7035 | case Intrinsic::localescape: { | ||||||||
| 7036 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 7037 | const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); | ||||||||
| 7038 | |||||||||
| 7039 | // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission | ||||||||
| 7040 | // is the same on all targets. | ||||||||
| 7041 | for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) { | ||||||||
| 7042 | Value *Arg = I.getArgOperand(Idx)->stripPointerCasts(); | ||||||||
| 7043 | if (isa<ConstantPointerNull>(Arg)) | ||||||||
| 7044 | continue; // Skip null pointers. They represent a hole in index space. | ||||||||
| 7045 | AllocaInst *Slot = cast<AllocaInst>(Arg); | ||||||||
| 7046 | assert(FuncInfo.StaticAllocaMap.count(Slot) &&(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot ) && "can only escape static allocas") ? void (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7047 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 7047 | "can only escape static allocas")(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot ) && "can only escape static allocas") ? void (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7047 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7048 | int FI = FuncInfo.StaticAllocaMap[Slot]; | ||||||||
| 7049 | MCSymbol *FrameAllocSym = | ||||||||
| 7050 | MF.getMMI().getContext().getOrCreateFrameAllocSymbol( | ||||||||
| 7051 | GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx); | ||||||||
| 7052 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl, | ||||||||
| 7053 | TII->get(TargetOpcode::LOCAL_ESCAPE)) | ||||||||
| 7054 | .addSym(FrameAllocSym) | ||||||||
| 7055 | .addFrameIndex(FI); | ||||||||
| 7056 | } | ||||||||
| 7057 | |||||||||
| 7058 | return; | ||||||||
| 7059 | } | ||||||||
| 7060 | |||||||||
| 7061 | case Intrinsic::localrecover: { | ||||||||
| 7062 | // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx) | ||||||||
| 7063 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 7064 | |||||||||
| 7065 | // Get the symbol that defines the frame offset. | ||||||||
| 7066 | auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts()); | ||||||||
| 7067 | auto *Idx = cast<ConstantInt>(I.getArgOperand(2)); | ||||||||
| 7068 | unsigned IdxVal = | ||||||||
| 7069 | unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max())); | ||||||||
| 7070 | MCSymbol *FrameAllocSym = | ||||||||
| 7071 | MF.getMMI().getContext().getOrCreateFrameAllocSymbol( | ||||||||
| 7072 | GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal); | ||||||||
| 7073 | |||||||||
| 7074 | Value *FP = I.getArgOperand(1); | ||||||||
| 7075 | SDValue FPVal = getValue(FP); | ||||||||
| 7076 | EVT PtrVT = FPVal.getValueType(); | ||||||||
| 7077 | |||||||||
| 7078 | // Create a MCSymbol for the label to avoid any target lowering | ||||||||
| 7079 | // that would make this PC relative. | ||||||||
| 7080 | SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT); | ||||||||
| 7081 | SDValue OffsetVal = | ||||||||
| 7082 | DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym); | ||||||||
| 7083 | |||||||||
| 7084 | // Add the offset to the FP. | ||||||||
| 7085 | SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl); | ||||||||
| 7086 | setValue(&I, Add); | ||||||||
| 7087 | |||||||||
| 7088 | return; | ||||||||
| 7089 | } | ||||||||
| 7090 | |||||||||
| 7091 | case Intrinsic::eh_exceptionpointer: | ||||||||
| 7092 | case Intrinsic::eh_exceptioncode: { | ||||||||
| 7093 | // Get the exception pointer vreg, copy from it, and resize it to fit. | ||||||||
| 7094 | const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0)); | ||||||||
| 7095 | MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); | ||||||||
| 7096 | const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT); | ||||||||
| 7097 | unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC); | ||||||||
| 7098 | SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT); | ||||||||
| 7099 | if (Intrinsic == Intrinsic::eh_exceptioncode) | ||||||||
| 7100 | N = DAG.getZExtOrTrunc(N, sdl, MVT::i32); | ||||||||
| 7101 | setValue(&I, N); | ||||||||
| 7102 | return; | ||||||||
| 7103 | } | ||||||||
| 7104 | case Intrinsic::xray_customevent: { | ||||||||
| 7105 | // Here we want to make sure that the intrinsic behaves as if it has a | ||||||||
| 7106 | // specific calling convention, and only for x86_64. | ||||||||
| 7107 | // FIXME: Support other platforms later. | ||||||||
| 7108 | const auto &Triple = DAG.getTarget().getTargetTriple(); | ||||||||
| 7109 | if (Triple.getArch() != Triple::x86_64) | ||||||||
| 7110 | return; | ||||||||
| 7111 | |||||||||
| 7112 | SmallVector<SDValue, 8> Ops; | ||||||||
| 7113 | |||||||||
| 7114 | // We want to say that we always want the arguments in registers. | ||||||||
| 7115 | SDValue LogEntryVal = getValue(I.getArgOperand(0)); | ||||||||
| 7116 | SDValue StrSizeVal = getValue(I.getArgOperand(1)); | ||||||||
| 7117 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||
| 7118 | SDValue Chain = getRoot(); | ||||||||
| 7119 | Ops.push_back(LogEntryVal); | ||||||||
| 7120 | Ops.push_back(StrSizeVal); | ||||||||
| 7121 | Ops.push_back(Chain); | ||||||||
| 7122 | |||||||||
| 7123 | // We need to enforce the calling convention for the callsite, so that | ||||||||
| 7124 | // argument ordering is enforced correctly, and that register allocation can | ||||||||
| 7125 | // see that some registers may be assumed clobbered and have to preserve | ||||||||
| 7126 | // them across calls to the intrinsic. | ||||||||
| 7127 | MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL, | ||||||||
| 7128 | sdl, NodeTys, Ops); | ||||||||
| 7129 | SDValue patchableNode = SDValue(MN, 0); | ||||||||
| 7130 | DAG.setRoot(patchableNode); | ||||||||
| 7131 | setValue(&I, patchableNode); | ||||||||
| 7132 | return; | ||||||||
| 7133 | } | ||||||||
| 7134 | case Intrinsic::xray_typedevent: { | ||||||||
| 7135 | // Here we want to make sure that the intrinsic behaves as if it has a | ||||||||
| 7136 | // specific calling convention, and only for x86_64. | ||||||||
| 7137 | // FIXME: Support other platforms later. | ||||||||
| 7138 | const auto &Triple = DAG.getTarget().getTargetTriple(); | ||||||||
| 7139 | if (Triple.getArch() != Triple::x86_64) | ||||||||
| 7140 | return; | ||||||||
| 7141 | |||||||||
| 7142 | SmallVector<SDValue, 8> Ops; | ||||||||
| 7143 | |||||||||
| 7144 | // We want to say that we always want the arguments in registers. | ||||||||
| 7145 | // It's unclear to me how manipulating the selection DAG here forces callers | ||||||||
| 7146 | // to provide arguments in registers instead of on the stack. | ||||||||
| 7147 | SDValue LogTypeId = getValue(I.getArgOperand(0)); | ||||||||
| 7148 | SDValue LogEntryVal = getValue(I.getArgOperand(1)); | ||||||||
| 7149 | SDValue StrSizeVal = getValue(I.getArgOperand(2)); | ||||||||
| 7150 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||
| 7151 | SDValue Chain = getRoot(); | ||||||||
| 7152 | Ops.push_back(LogTypeId); | ||||||||
| 7153 | Ops.push_back(LogEntryVal); | ||||||||
| 7154 | Ops.push_back(StrSizeVal); | ||||||||
| 7155 | Ops.push_back(Chain); | ||||||||
| 7156 | |||||||||
| 7157 | // We need to enforce the calling convention for the callsite, so that | ||||||||
| 7158 | // argument ordering is enforced correctly, and that register allocation can | ||||||||
| 7159 | // see that some registers may be assumed clobbered and have to preserve | ||||||||
| 7160 | // them across calls to the intrinsic. | ||||||||
| 7161 | MachineSDNode *MN = DAG.getMachineNode( | ||||||||
| 7162 | TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops); | ||||||||
| 7163 | SDValue patchableNode = SDValue(MN, 0); | ||||||||
| 7164 | DAG.setRoot(patchableNode); | ||||||||
| 7165 | setValue(&I, patchableNode); | ||||||||
| 7166 | return; | ||||||||
| 7167 | } | ||||||||
| 7168 | case Intrinsic::experimental_deoptimize: | ||||||||
| 7169 | LowerDeoptimizeCall(&I); | ||||||||
| 7170 | return; | ||||||||
| 7171 | case Intrinsic::experimental_stepvector: | ||||||||
| 7172 | visitStepVector(I); | ||||||||
| 7173 | return; | ||||||||
| 7174 | case Intrinsic::vector_reduce_fadd: | ||||||||
| 7175 | case Intrinsic::vector_reduce_fmul: | ||||||||
| 7176 | case Intrinsic::vector_reduce_add: | ||||||||
| 7177 | case Intrinsic::vector_reduce_mul: | ||||||||
| 7178 | case Intrinsic::vector_reduce_and: | ||||||||
| 7179 | case Intrinsic::vector_reduce_or: | ||||||||
| 7180 | case Intrinsic::vector_reduce_xor: | ||||||||
| 7181 | case Intrinsic::vector_reduce_smax: | ||||||||
| 7182 | case Intrinsic::vector_reduce_smin: | ||||||||
| 7183 | case Intrinsic::vector_reduce_umax: | ||||||||
| 7184 | case Intrinsic::vector_reduce_umin: | ||||||||
| 7185 | case Intrinsic::vector_reduce_fmax: | ||||||||
| 7186 | case Intrinsic::vector_reduce_fmin: | ||||||||
| 7187 | visitVectorReduce(I, Intrinsic); | ||||||||
| 7188 | return; | ||||||||
| 7189 | |||||||||
| 7190 | case Intrinsic::icall_branch_funnel: { | ||||||||
| 7191 | SmallVector<SDValue, 16> Ops; | ||||||||
| 7192 | Ops.push_back(getValue(I.getArgOperand(0))); | ||||||||
| 7193 | |||||||||
| 7194 | int64_t Offset; | ||||||||
| 7195 | auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset( | ||||||||
| 7196 | I.getArgOperand(1), Offset, DAG.getDataLayout())); | ||||||||
| 7197 | if (!Base) | ||||||||
| 7198 | report_fatal_error( | ||||||||
| 7199 | "llvm.icall.branch.funnel operand must be a GlobalValue"); | ||||||||
| 7200 | Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0)); | ||||||||
| 7201 | |||||||||
| 7202 | struct BranchFunnelTarget { | ||||||||
| 7203 | int64_t Offset; | ||||||||
| 7204 | SDValue Target; | ||||||||
| 7205 | }; | ||||||||
| 7206 | SmallVector<BranchFunnelTarget, 8> Targets; | ||||||||
| 7207 | |||||||||
| 7208 | for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) { | ||||||||
| 7209 | auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset( | ||||||||
| 7210 | I.getArgOperand(Op), Offset, DAG.getDataLayout())); | ||||||||
| 7211 | if (ElemBase != Base) | ||||||||
| 7212 | report_fatal_error("all llvm.icall.branch.funnel operands must refer " | ||||||||
| 7213 | "to the same GlobalValue"); | ||||||||
| 7214 | |||||||||
| 7215 | SDValue Val = getValue(I.getArgOperand(Op + 1)); | ||||||||
| 7216 | auto *GA = dyn_cast<GlobalAddressSDNode>(Val); | ||||||||
| 7217 | if (!GA) | ||||||||
| 7218 | report_fatal_error( | ||||||||
| 7219 | "llvm.icall.branch.funnel operand must be a GlobalValue"); | ||||||||
| 7220 | Targets.push_back({Offset, DAG.getTargetGlobalAddress( | ||||||||
| 7221 | GA->getGlobal(), sdl, Val.getValueType(), | ||||||||
| 7222 | GA->getOffset())}); | ||||||||
| 7223 | } | ||||||||
| 7224 | llvm::sort(Targets, | ||||||||
| 7225 | [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) { | ||||||||
| 7226 | return T1.Offset < T2.Offset; | ||||||||
| 7227 | }); | ||||||||
| 7228 | |||||||||
| 7229 | for (auto &T : Targets) { | ||||||||
| 7230 | Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32)); | ||||||||
| 7231 | Ops.push_back(T.Target); | ||||||||
| 7232 | } | ||||||||
| 7233 | |||||||||
| 7234 | Ops.push_back(DAG.getRoot()); // Chain | ||||||||
| 7235 | SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl, | ||||||||
| 7236 | MVT::Other, Ops), | ||||||||
| 7237 | 0); | ||||||||
| 7238 | DAG.setRoot(N); | ||||||||
| 7239 | setValue(&I, N); | ||||||||
| 7240 | HasTailCall = true; | ||||||||
| 7241 | return; | ||||||||
| 7242 | } | ||||||||
| 7243 | |||||||||
| 7244 | case Intrinsic::wasm_landingpad_index: | ||||||||
| 7245 | // Information this intrinsic contained has been transferred to | ||||||||
| 7246 | // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely | ||||||||
| 7247 | // delete it now. | ||||||||
| 7248 | return; | ||||||||
| 7249 | |||||||||
| 7250 | case Intrinsic::aarch64_settag: | ||||||||
| 7251 | case Intrinsic::aarch64_settag_zero: { | ||||||||
| 7252 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||
| 7253 | bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero; | ||||||||
| 7254 | SDValue Val = TSI.EmitTargetCodeForSetTag( | ||||||||
| 7255 | DAG, sdl, getRoot(), getValue(I.getArgOperand(0)), | ||||||||
| 7256 | getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)), | ||||||||
| 7257 | ZeroMemory); | ||||||||
| 7258 | DAG.setRoot(Val); | ||||||||
| 7259 | setValue(&I, Val); | ||||||||
| 7260 | return; | ||||||||
| 7261 | } | ||||||||
| 7262 | case Intrinsic::ptrmask: { | ||||||||
| 7263 | SDValue Ptr = getValue(I.getOperand(0)); | ||||||||
| 7264 | SDValue Const = getValue(I.getOperand(1)); | ||||||||
| 7265 | |||||||||
| 7266 | EVT PtrVT = Ptr.getValueType(); | ||||||||
| 7267 | setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, | ||||||||
| 7268 | DAG.getZExtOrTrunc(Const, sdl, PtrVT))); | ||||||||
| 7269 | return; | ||||||||
| 7270 | } | ||||||||
| 7271 | case Intrinsic::threadlocal_address: { | ||||||||
| 7272 | setValue(&I, getValue(I.getOperand(0))); | ||||||||
| 7273 | return; | ||||||||
| 7274 | } | ||||||||
| 7275 | case Intrinsic::get_active_lane_mask: { | ||||||||
| 7276 | EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 7277 | SDValue Index = getValue(I.getOperand(0)); | ||||||||
| 7278 | EVT ElementVT = Index.getValueType(); | ||||||||
| 7279 | |||||||||
| 7280 | if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) { | ||||||||
| 7281 | visitTargetIntrinsic(I, Intrinsic); | ||||||||
| 7282 | return; | ||||||||
| 7283 | } | ||||||||
| 7284 | |||||||||
| 7285 | SDValue TripCount = getValue(I.getOperand(1)); | ||||||||
| 7286 | auto VecTy = CCVT.changeVectorElementType(ElementVT); | ||||||||
| 7287 | |||||||||
| 7288 | SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index); | ||||||||
| 7289 | SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount); | ||||||||
| 7290 | SDValue VectorStep = DAG.getStepVector(sdl, VecTy); | ||||||||
| 7291 | SDValue VectorInduction = DAG.getNode( | ||||||||
| 7292 | ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep); | ||||||||
| 7293 | SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction, | ||||||||
| 7294 | VectorTripCount, ISD::CondCode::SETULT); | ||||||||
| 7295 | setValue(&I, SetCC); | ||||||||
| 7296 | return; | ||||||||
| 7297 | } | ||||||||
| 7298 | case Intrinsic::vector_insert: { | ||||||||
| 7299 | SDValue Vec = getValue(I.getOperand(0)); | ||||||||
| 7300 | SDValue SubVec = getValue(I.getOperand(1)); | ||||||||
| 7301 | SDValue Index = getValue(I.getOperand(2)); | ||||||||
| 7302 | |||||||||
| 7303 | // The intrinsic's index type is i64, but the SDNode requires an index type | ||||||||
| 7304 | // suitable for the target. Convert the index as required. | ||||||||
| 7305 | MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); | ||||||||
| 7306 | if (Index.getValueType() != VectorIdxTy) | ||||||||
| 7307 | Index = DAG.getVectorIdxConstant( | ||||||||
| 7308 | cast<ConstantSDNode>(Index)->getZExtValue(), sdl); | ||||||||
| 7309 | |||||||||
| 7310 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 7311 | setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec, | ||||||||
| 7312 | Index)); | ||||||||
| 7313 | return; | ||||||||
| 7314 | } | ||||||||
| 7315 | case Intrinsic::vector_extract: { | ||||||||
| 7316 | SDValue Vec = getValue(I.getOperand(0)); | ||||||||
| 7317 | SDValue Index = getValue(I.getOperand(1)); | ||||||||
| 7318 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 7319 | |||||||||
| 7320 | // The intrinsic's index type is i64, but the SDNode requires an index type | ||||||||
| 7321 | // suitable for the target. Convert the index as required. | ||||||||
| 7322 | MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); | ||||||||
| 7323 | if (Index.getValueType() != VectorIdxTy) | ||||||||
| 7324 | Index = DAG.getVectorIdxConstant( | ||||||||
| 7325 | cast<ConstantSDNode>(Index)->getZExtValue(), sdl); | ||||||||
| 7326 | |||||||||
| 7327 | setValue(&I, | ||||||||
| 7328 | DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index)); | ||||||||
| 7329 | return; | ||||||||
| 7330 | } | ||||||||
| 7331 | case Intrinsic::experimental_vector_reverse: | ||||||||
| 7332 | visitVectorReverse(I); | ||||||||
| 7333 | return; | ||||||||
| 7334 | case Intrinsic::experimental_vector_splice: | ||||||||
| 7335 | visitVectorSplice(I); | ||||||||
| 7336 | return; | ||||||||
| 7337 | case Intrinsic::callbr_landingpad: | ||||||||
| 7338 | visitCallBrLandingPad(I); | ||||||||
| 7339 | return; | ||||||||
| 7340 | case Intrinsic::experimental_vector_interleave2: | ||||||||
| 7341 | visitVectorInterleave(I); | ||||||||
| 7342 | return; | ||||||||
| 7343 | case Intrinsic::experimental_vector_deinterleave2: | ||||||||
| 7344 | visitVectorDeinterleave(I); | ||||||||
| 7345 | return; | ||||||||
| 7346 | } | ||||||||
| 7347 | } | ||||||||
| 7348 | |||||||||
| 7349 | void SelectionDAGBuilder::visitConstrainedFPIntrinsic( | ||||||||
| 7350 | const ConstrainedFPIntrinsic &FPI) { | ||||||||
| 7351 | SDLoc sdl = getCurSDLoc(); | ||||||||
| 7352 | |||||||||
| 7353 | // We do not need to serialize constrained FP intrinsics against | ||||||||
| 7354 | // each other or against (nonvolatile) loads, so they can be | ||||||||
| 7355 | // chained like loads. | ||||||||
| 7356 | SDValue Chain = DAG.getRoot(); | ||||||||
| 7357 | SmallVector<SDValue, 4> Opers; | ||||||||
| 7358 | Opers.push_back(Chain); | ||||||||
| 7359 | if (FPI.isUnaryOp()) { | ||||||||
| 7360 | Opers.push_back(getValue(FPI.getArgOperand(0))); | ||||||||
| 7361 | } else if (FPI.isTernaryOp()) { | ||||||||
| 7362 | Opers.push_back(getValue(FPI.getArgOperand(0))); | ||||||||
| 7363 | Opers.push_back(getValue(FPI.getArgOperand(1))); | ||||||||
| 7364 | Opers.push_back(getValue(FPI.getArgOperand(2))); | ||||||||
| 7365 | } else { | ||||||||
| 7366 | Opers.push_back(getValue(FPI.getArgOperand(0))); | ||||||||
| 7367 | Opers.push_back(getValue(FPI.getArgOperand(1))); | ||||||||
| 7368 | } | ||||||||
| 7369 | |||||||||
| 7370 | auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) { | ||||||||
| 7371 | assert(Result.getNode()->getNumValues() == 2)(static_cast <bool> (Result.getNode()->getNumValues( ) == 2) ? void (0) : __assert_fail ("Result.getNode()->getNumValues() == 2" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7371 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7372 | |||||||||
| 7373 | // Push node to the appropriate list so that future instructions can be | ||||||||
| 7374 | // chained up correctly. | ||||||||
| 7375 | SDValue OutChain = Result.getValue(1); | ||||||||
| 7376 | switch (EB) { | ||||||||
| 7377 | case fp::ExceptionBehavior::ebIgnore: | ||||||||
| 7378 | // The only reason why ebIgnore nodes still need to be chained is that | ||||||||
| 7379 | // they might depend on the current rounding mode, and therefore must | ||||||||
| 7380 | // not be moved across instruction that may change that mode. | ||||||||
| 7381 | [[fallthrough]]; | ||||||||
| 7382 | case fp::ExceptionBehavior::ebMayTrap: | ||||||||
| 7383 | // These must not be moved across calls or instructions that may change | ||||||||
| 7384 | // floating-point exception masks. | ||||||||
| 7385 | PendingConstrainedFP.push_back(OutChain); | ||||||||
| 7386 | break; | ||||||||
| 7387 | case fp::ExceptionBehavior::ebStrict: | ||||||||
| 7388 | // These must not be moved across calls or instructions that may change | ||||||||
| 7389 | // floating-point exception masks or read floating-point exception flags. | ||||||||
| 7390 | // In addition, they cannot be optimized out even if unused. | ||||||||
| 7391 | PendingConstrainedFPStrict.push_back(OutChain); | ||||||||
| 7392 | break; | ||||||||
| 7393 | } | ||||||||
| 7394 | }; | ||||||||
| 7395 | |||||||||
| 7396 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 7397 | EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType()); | ||||||||
| 7398 | SDVTList VTs = DAG.getVTList(VT, MVT::Other); | ||||||||
| 7399 | fp::ExceptionBehavior EB = *FPI.getExceptionBehavior(); | ||||||||
| 7400 | |||||||||
| 7401 | SDNodeFlags Flags; | ||||||||
| 7402 | if (EB == fp::ExceptionBehavior::ebIgnore) | ||||||||
| 7403 | Flags.setNoFPExcept(true); | ||||||||
| 7404 | |||||||||
| 7405 | if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI)) | ||||||||
| 7406 | Flags.copyFMF(*FPOp); | ||||||||
| 7407 | |||||||||
| 7408 | unsigned Opcode; | ||||||||
| 7409 | switch (FPI.getIntrinsicID()) { | ||||||||
| 7410 | default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7410); // Can't reach here. | ||||||||
| 7411 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | ||||||||
| 7412 | case Intrinsic::INTRINSIC: \ | ||||||||
| 7413 | Opcode = ISD::STRICT_##DAGN; \ | ||||||||
| 7414 | break; | ||||||||
| 7415 | #include "llvm/IR/ConstrainedOps.def" | ||||||||
| 7416 | case Intrinsic::experimental_constrained_fmuladd: { | ||||||||
| 7417 | Opcode = ISD::STRICT_FMA; | ||||||||
| 7418 | // Break fmuladd into fmul and fadd. | ||||||||
| 7419 | if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict || | ||||||||
| 7420 | !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { | ||||||||
| 7421 | Opers.pop_back(); | ||||||||
| 7422 | SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags); | ||||||||
| 7423 | pushOutChain(Mul, EB); | ||||||||
| 7424 | Opcode = ISD::STRICT_FADD; | ||||||||
| 7425 | Opers.clear(); | ||||||||
| 7426 | Opers.push_back(Mul.getValue(1)); | ||||||||
| 7427 | Opers.push_back(Mul.getValue(0)); | ||||||||
| 7428 | Opers.push_back(getValue(FPI.getArgOperand(2))); | ||||||||
| 7429 | } | ||||||||
| 7430 | break; | ||||||||
| 7431 | } | ||||||||
| 7432 | } | ||||||||
| 7433 | |||||||||
| 7434 | // A few strict DAG nodes carry additional operands that are not | ||||||||
| 7435 | // set up by the default code above. | ||||||||
| 7436 | switch (Opcode) { | ||||||||
| 7437 | default: break; | ||||||||
| 7438 | case ISD::STRICT_FP_ROUND: | ||||||||
| 7439 | Opers.push_back( | ||||||||
| 7440 | DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 7441 | break; | ||||||||
| 7442 | case ISD::STRICT_FSETCC: | ||||||||
| 7443 | case ISD::STRICT_FSETCCS: { | ||||||||
| 7444 | auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI); | ||||||||
| 7445 | ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate()); | ||||||||
| 7446 | if (TM.Options.NoNaNsFPMath) | ||||||||
| 7447 | Condition = getFCmpCodeWithoutNaN(Condition); | ||||||||
| 7448 | Opers.push_back(DAG.getCondCode(Condition)); | ||||||||
| 7449 | break; | ||||||||
| 7450 | } | ||||||||
| 7451 | } | ||||||||
| 7452 | |||||||||
| 7453 | SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags); | ||||||||
| 7454 | pushOutChain(Result, EB); | ||||||||
| 7455 | |||||||||
| 7456 | SDValue FPResult = Result.getValue(0); | ||||||||
| 7457 | setValue(&FPI, FPResult); | ||||||||
| 7458 | } | ||||||||
| 7459 | |||||||||
| 7460 | static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) { | ||||||||
| 7461 | std::optional<unsigned> ResOPC; | ||||||||
| 7462 | switch (VPIntrin.getIntrinsicID()) { | ||||||||
| 7463 | case Intrinsic::vp_ctlz: { | ||||||||
| 7464 | bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne(); | ||||||||
| 7465 | ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ; | ||||||||
| 7466 | break; | ||||||||
| 7467 | } | ||||||||
| 7468 | case Intrinsic::vp_cttz: { | ||||||||
| 7469 | bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne(); | ||||||||
| 7470 | ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ; | ||||||||
| 7471 | break; | ||||||||
| 7472 | } | ||||||||
| 7473 | #define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \ | ||||||||
| 7474 | case Intrinsic::VPID: \ | ||||||||
| 7475 | ResOPC = ISD::VPSD; \ | ||||||||
| 7476 | break; | ||||||||
| 7477 | #include "llvm/IR/VPIntrinsics.def" | ||||||||
| 7478 | } | ||||||||
| 7479 | |||||||||
| 7480 | if (!ResOPC) | ||||||||
| 7481 | llvm_unreachable(::llvm::llvm_unreachable_internal("Inconsistency: no SDNode available for this VPIntrinsic!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7482 ) | ||||||||
| 7482 | "Inconsistency: no SDNode available for this VPIntrinsic!")::llvm::llvm_unreachable_internal("Inconsistency: no SDNode available for this VPIntrinsic!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7482 ); | ||||||||
| 7483 | |||||||||
| 7484 | if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD || | ||||||||
| 7485 | *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) { | ||||||||
| 7486 | if (VPIntrin.getFastMathFlags().allowReassoc()) | ||||||||
| 7487 | return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD | ||||||||
| 7488 | : ISD::VP_REDUCE_FMUL; | ||||||||
| 7489 | } | ||||||||
| 7490 | |||||||||
| 7491 | return *ResOPC; | ||||||||
| 7492 | } | ||||||||
| 7493 | |||||||||
| 7494 | void SelectionDAGBuilder::visitVPLoad( | ||||||||
| 7495 | const VPIntrinsic &VPIntrin, EVT VT, | ||||||||
| 7496 | const SmallVectorImpl<SDValue> &OpValues) { | ||||||||
| 7497 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7498 | Value *PtrOperand = VPIntrin.getArgOperand(0); | ||||||||
| 7499 | MaybeAlign Alignment = VPIntrin.getPointerAlignment(); | ||||||||
| 7500 | AAMDNodes AAInfo = VPIntrin.getAAMetadata(); | ||||||||
| 7501 | const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); | ||||||||
| 7502 | SDValue LD; | ||||||||
| 7503 | // Do not serialize variable-length loads of constant memory with | ||||||||
| 7504 | // anything. | ||||||||
| 7505 | if (!Alignment) | ||||||||
| 7506 | Alignment = DAG.getEVTAlign(VT); | ||||||||
| 7507 | MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); | ||||||||
| 7508 | bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); | ||||||||
| 7509 | SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); | ||||||||
| 7510 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 7511 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, | ||||||||
| 7512 | MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); | ||||||||
| 7513 | LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2], | ||||||||
| 7514 | MMO, false /*IsExpanding */); | ||||||||
| 7515 | if (AddToChain) | ||||||||
| 7516 | PendingLoads.push_back(LD.getValue(1)); | ||||||||
| 7517 | setValue(&VPIntrin, LD); | ||||||||
| 7518 | } | ||||||||
| 7519 | |||||||||
| 7520 | void SelectionDAGBuilder::visitVPGather( | ||||||||
| 7521 | const VPIntrinsic &VPIntrin, EVT VT, | ||||||||
| 7522 | const SmallVectorImpl<SDValue> &OpValues) { | ||||||||
| 7523 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7524 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 7525 | Value *PtrOperand = VPIntrin.getArgOperand(0); | ||||||||
| 7526 | MaybeAlign Alignment = VPIntrin.getPointerAlignment(); | ||||||||
| 7527 | AAMDNodes AAInfo = VPIntrin.getAAMetadata(); | ||||||||
| 7528 | const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); | ||||||||
| 7529 | SDValue LD; | ||||||||
| 7530 | if (!Alignment) | ||||||||
| 7531 | Alignment = DAG.getEVTAlign(VT.getScalarType()); | ||||||||
| 7532 | unsigned AS = | ||||||||
| 7533 | PtrOperand->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||
| 7534 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 7535 | MachinePointerInfo(AS), MachineMemOperand::MOLoad, | ||||||||
| 7536 | MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); | ||||||||
| 7537 | SDValue Base, Index, Scale; | ||||||||
| 7538 | ISD::MemIndexType IndexType; | ||||||||
| 7539 | bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale, | ||||||||
| 7540 | this, VPIntrin.getParent(), | ||||||||
| 7541 | VT.getScalarStoreSize()); | ||||||||
| 7542 | if (!UniformBase) { | ||||||||
| 7543 | Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 7544 | Index = getValue(PtrOperand); | ||||||||
| 7545 | IndexType = ISD::SIGNED_SCALED; | ||||||||
| 7546 | Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 7547 | } | ||||||||
| 7548 | EVT IdxVT = Index.getValueType(); | ||||||||
| 7549 | EVT EltTy = IdxVT.getVectorElementType(); | ||||||||
| 7550 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { | ||||||||
| 7551 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); | ||||||||
| 7552 | Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index); | ||||||||
| 7553 | } | ||||||||
| 7554 | LD = DAG.getGatherVP( | ||||||||
| 7555 | DAG.getVTList(VT, MVT::Other), VT, DL, | ||||||||
| 7556 | {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO, | ||||||||
| 7557 | IndexType); | ||||||||
| 7558 | PendingLoads.push_back(LD.getValue(1)); | ||||||||
| 7559 | setValue(&VPIntrin, LD); | ||||||||
| 7560 | } | ||||||||
| 7561 | |||||||||
| 7562 | void SelectionDAGBuilder::visitVPStore( | ||||||||
| 7563 | const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) { | ||||||||
| 7564 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7565 | Value *PtrOperand = VPIntrin.getArgOperand(1); | ||||||||
| 7566 | EVT VT = OpValues[0].getValueType(); | ||||||||
| 7567 | MaybeAlign Alignment = VPIntrin.getPointerAlignment(); | ||||||||
| 7568 | AAMDNodes AAInfo = VPIntrin.getAAMetadata(); | ||||||||
| 7569 | SDValue ST; | ||||||||
| 7570 | if (!Alignment) | ||||||||
| 7571 | Alignment = DAG.getEVTAlign(VT); | ||||||||
| 7572 | SDValue Ptr = OpValues[1]; | ||||||||
| 7573 | SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); | ||||||||
| 7574 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 7575 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, | ||||||||
| 7576 | MemoryLocation::UnknownSize, *Alignment, AAInfo); | ||||||||
| 7577 | ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset, | ||||||||
| 7578 | OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED, | ||||||||
| 7579 | /* IsTruncating */ false, /*IsCompressing*/ false); | ||||||||
| 7580 | DAG.setRoot(ST); | ||||||||
| 7581 | setValue(&VPIntrin, ST); | ||||||||
| 7582 | } | ||||||||
| 7583 | |||||||||
| 7584 | void SelectionDAGBuilder::visitVPScatter( | ||||||||
| 7585 | const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) { | ||||||||
| 7586 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7587 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 7588 | Value *PtrOperand = VPIntrin.getArgOperand(1); | ||||||||
| 7589 | EVT VT = OpValues[0].getValueType(); | ||||||||
| 7590 | MaybeAlign Alignment = VPIntrin.getPointerAlignment(); | ||||||||
| 7591 | AAMDNodes AAInfo = VPIntrin.getAAMetadata(); | ||||||||
| 7592 | SDValue ST; | ||||||||
| 7593 | if (!Alignment) | ||||||||
| 7594 | Alignment = DAG.getEVTAlign(VT.getScalarType()); | ||||||||
| 7595 | unsigned AS = | ||||||||
| 7596 | PtrOperand->getType()->getScalarType()->getPointerAddressSpace(); | ||||||||
| 7597 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 7598 | MachinePointerInfo(AS), MachineMemOperand::MOStore, | ||||||||
| 7599 | MemoryLocation::UnknownSize, *Alignment, AAInfo); | ||||||||
| 7600 | SDValue Base, Index, Scale; | ||||||||
| 7601 | ISD::MemIndexType IndexType; | ||||||||
| 7602 | bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale, | ||||||||
| 7603 | this, VPIntrin.getParent(), | ||||||||
| 7604 | VT.getScalarStoreSize()); | ||||||||
| 7605 | if (!UniformBase) { | ||||||||
| 7606 | Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 7607 | Index = getValue(PtrOperand); | ||||||||
| 7608 | IndexType = ISD::SIGNED_SCALED; | ||||||||
| 7609 | Scale = | ||||||||
| 7610 | DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 7611 | } | ||||||||
| 7612 | EVT IdxVT = Index.getValueType(); | ||||||||
| 7613 | EVT EltTy = IdxVT.getVectorElementType(); | ||||||||
| 7614 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { | ||||||||
| 7615 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); | ||||||||
| 7616 | Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index); | ||||||||
| 7617 | } | ||||||||
| 7618 | ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL, | ||||||||
| 7619 | {getMemoryRoot(), OpValues[0], Base, Index, Scale, | ||||||||
| 7620 | OpValues[2], OpValues[3]}, | ||||||||
| 7621 | MMO, IndexType); | ||||||||
| 7622 | DAG.setRoot(ST); | ||||||||
| 7623 | setValue(&VPIntrin, ST); | ||||||||
| 7624 | } | ||||||||
| 7625 | |||||||||
| 7626 | void SelectionDAGBuilder::visitVPStridedLoad( | ||||||||
| 7627 | const VPIntrinsic &VPIntrin, EVT VT, | ||||||||
| 7628 | const SmallVectorImpl<SDValue> &OpValues) { | ||||||||
| 7629 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7630 | Value *PtrOperand = VPIntrin.getArgOperand(0); | ||||||||
| 7631 | MaybeAlign Alignment = VPIntrin.getPointerAlignment(); | ||||||||
| 7632 | if (!Alignment) | ||||||||
| 7633 | Alignment = DAG.getEVTAlign(VT.getScalarType()); | ||||||||
| 7634 | AAMDNodes AAInfo = VPIntrin.getAAMetadata(); | ||||||||
| 7635 | const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); | ||||||||
| 7636 | MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); | ||||||||
| 7637 | bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); | ||||||||
| 7638 | SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); | ||||||||
| 7639 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 7640 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, | ||||||||
| 7641 | MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); | ||||||||
| 7642 | |||||||||
| 7643 | SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], | ||||||||
| 7644 | OpValues[2], OpValues[3], MMO, | ||||||||
| 7645 | false /*IsExpanding*/); | ||||||||
| 7646 | |||||||||
| 7647 | if (AddToChain) | ||||||||
| 7648 | PendingLoads.push_back(LD.getValue(1)); | ||||||||
| 7649 | setValue(&VPIntrin, LD); | ||||||||
| 7650 | } | ||||||||
| 7651 | |||||||||
| 7652 | void SelectionDAGBuilder::visitVPStridedStore( | ||||||||
| 7653 | const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) { | ||||||||
| 7654 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7655 | Value *PtrOperand = VPIntrin.getArgOperand(1); | ||||||||
| 7656 | EVT VT = OpValues[0].getValueType(); | ||||||||
| 7657 | MaybeAlign Alignment = VPIntrin.getPointerAlignment(); | ||||||||
| 7658 | if (!Alignment) | ||||||||
| 7659 | Alignment = DAG.getEVTAlign(VT.getScalarType()); | ||||||||
| 7660 | AAMDNodes AAInfo = VPIntrin.getAAMetadata(); | ||||||||
| 7661 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( | ||||||||
| 7662 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, | ||||||||
| 7663 | MemoryLocation::UnknownSize, *Alignment, AAInfo); | ||||||||
| 7664 | |||||||||
| 7665 | SDValue ST = DAG.getStridedStoreVP( | ||||||||
| 7666 | getMemoryRoot(), DL, OpValues[0], OpValues[1], | ||||||||
| 7667 | DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3], | ||||||||
| 7668 | OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false, | ||||||||
| 7669 | /*IsCompressing*/ false); | ||||||||
| 7670 | |||||||||
| 7671 | DAG.setRoot(ST); | ||||||||
| 7672 | setValue(&VPIntrin, ST); | ||||||||
| 7673 | } | ||||||||
| 7674 | |||||||||
| 7675 | void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) { | ||||||||
| 7676 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 7677 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7678 | |||||||||
| 7679 | ISD::CondCode Condition; | ||||||||
| 7680 | CmpInst::Predicate CondCode = VPIntrin.getPredicate(); | ||||||||
| 7681 | bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy(); | ||||||||
| 7682 | if (IsFP) { | ||||||||
| 7683 | // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan) | ||||||||
| 7684 | // flags, but calls that don't return floating-point types can't be | ||||||||
| 7685 | // FPMathOperators, like vp.fcmp. This affects constrained fcmp too. | ||||||||
| 7686 | Condition = getFCmpCondCode(CondCode); | ||||||||
| 7687 | if (TM.Options.NoNaNsFPMath) | ||||||||
| 7688 | Condition = getFCmpCodeWithoutNaN(Condition); | ||||||||
| 7689 | } else { | ||||||||
| 7690 | Condition = getICmpCondCode(CondCode); | ||||||||
| 7691 | } | ||||||||
| 7692 | |||||||||
| 7693 | SDValue Op1 = getValue(VPIntrin.getOperand(0)); | ||||||||
| 7694 | SDValue Op2 = getValue(VPIntrin.getOperand(1)); | ||||||||
| 7695 | // #2 is the condition code | ||||||||
| 7696 | SDValue MaskOp = getValue(VPIntrin.getOperand(3)); | ||||||||
| 7697 | SDValue EVL = getValue(VPIntrin.getOperand(4)); | ||||||||
| 7698 | MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy(); | ||||||||
| 7699 | assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&(static_cast <bool> (EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type" ) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7700 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 7700 | "Unexpected target EVL type")(static_cast <bool> (EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type" ) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7700 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7701 | EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL); | ||||||||
| 7702 | |||||||||
| 7703 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 7704 | VPIntrin.getType()); | ||||||||
| 7705 | setValue(&VPIntrin, | ||||||||
| 7706 | DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL)); | ||||||||
| 7707 | } | ||||||||
| 7708 | |||||||||
| 7709 | void SelectionDAGBuilder::visitVectorPredicationIntrinsic( | ||||||||
| 7710 | const VPIntrinsic &VPIntrin) { | ||||||||
| 7711 | SDLoc DL = getCurSDLoc(); | ||||||||
| 7712 | unsigned Opcode = getISDForVPIntrinsic(VPIntrin); | ||||||||
| 7713 | |||||||||
| 7714 | auto IID = VPIntrin.getIntrinsicID(); | ||||||||
| 7715 | |||||||||
| 7716 | if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin)) | ||||||||
| 7717 | return visitVPCmp(*CmpI); | ||||||||
| 7718 | |||||||||
| 7719 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 7720 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 7721 | ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs); | ||||||||
| 7722 | SDVTList VTs = DAG.getVTList(ValueVTs); | ||||||||
| 7723 | |||||||||
| 7724 | auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID); | ||||||||
| 7725 | |||||||||
| 7726 | MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy(); | ||||||||
| 7727 | assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&(static_cast <bool> (EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type" ) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7728 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 7728 | "Unexpected target EVL type")(static_cast <bool> (EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type" ) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7728 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7729 | |||||||||
| 7730 | // Request operands. | ||||||||
| 7731 | SmallVector<SDValue, 7> OpValues; | ||||||||
| 7732 | for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) { | ||||||||
| 7733 | auto Op = getValue(VPIntrin.getArgOperand(I)); | ||||||||
| 7734 | if (I == EVLParamPos) | ||||||||
| 7735 | Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op); | ||||||||
| 7736 | OpValues.push_back(Op); | ||||||||
| 7737 | } | ||||||||
| 7738 | |||||||||
| 7739 | switch (Opcode) { | ||||||||
| 7740 | default: { | ||||||||
| 7741 | SDNodeFlags SDFlags; | ||||||||
| 7742 | if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin)) | ||||||||
| 7743 | SDFlags.copyFMF(*FPMO); | ||||||||
| 7744 | SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags); | ||||||||
| 7745 | setValue(&VPIntrin, Result); | ||||||||
| 7746 | break; | ||||||||
| 7747 | } | ||||||||
| 7748 | case ISD::VP_LOAD: | ||||||||
| 7749 | visitVPLoad(VPIntrin, ValueVTs[0], OpValues); | ||||||||
| 7750 | break; | ||||||||
| 7751 | case ISD::VP_GATHER: | ||||||||
| 7752 | visitVPGather(VPIntrin, ValueVTs[0], OpValues); | ||||||||
| 7753 | break; | ||||||||
| 7754 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: | ||||||||
| 7755 | visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues); | ||||||||
| 7756 | break; | ||||||||
| 7757 | case ISD::VP_STORE: | ||||||||
| 7758 | visitVPStore(VPIntrin, OpValues); | ||||||||
| 7759 | break; | ||||||||
| 7760 | case ISD::VP_SCATTER: | ||||||||
| 7761 | visitVPScatter(VPIntrin, OpValues); | ||||||||
| 7762 | break; | ||||||||
| 7763 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: | ||||||||
| 7764 | visitVPStridedStore(VPIntrin, OpValues); | ||||||||
| 7765 | break; | ||||||||
| 7766 | case ISD::VP_FMULADD: { | ||||||||
| 7767 | assert(OpValues.size() == 5 && "Unexpected number of operands")(static_cast <bool> (OpValues.size() == 5 && "Unexpected number of operands" ) ? void (0) : __assert_fail ("OpValues.size() == 5 && \"Unexpected number of operands\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7767 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7768 | SDNodeFlags SDFlags; | ||||||||
| 7769 | if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin)) | ||||||||
| 7770 | SDFlags.copyFMF(*FPMO); | ||||||||
| 7771 | if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && | ||||||||
| 7772 | TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), ValueVTs[0])) { | ||||||||
| 7773 | setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags)); | ||||||||
| 7774 | } else { | ||||||||
| 7775 | SDValue Mul = DAG.getNode( | ||||||||
| 7776 | ISD::VP_FMUL, DL, VTs, | ||||||||
| 7777 | {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags); | ||||||||
| 7778 | SDValue Add = | ||||||||
| 7779 | DAG.getNode(ISD::VP_FADD, DL, VTs, | ||||||||
| 7780 | {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags); | ||||||||
| 7781 | setValue(&VPIntrin, Add); | ||||||||
| 7782 | } | ||||||||
| 7783 | break; | ||||||||
| 7784 | } | ||||||||
| 7785 | case ISD::VP_INTTOPTR: { | ||||||||
| 7786 | SDValue N = OpValues[0]; | ||||||||
| 7787 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType()); | ||||||||
| 7788 | EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType()); | ||||||||
| 7789 | N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1], | ||||||||
| 7790 | OpValues[2]); | ||||||||
| 7791 | N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1], | ||||||||
| 7792 | OpValues[2]); | ||||||||
| 7793 | setValue(&VPIntrin, N); | ||||||||
| 7794 | break; | ||||||||
| 7795 | } | ||||||||
| 7796 | case ISD::VP_PTRTOINT: { | ||||||||
| 7797 | SDValue N = OpValues[0]; | ||||||||
| 7798 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 7799 | VPIntrin.getType()); | ||||||||
| 7800 | EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), | ||||||||
| 7801 | VPIntrin.getOperand(0)->getType()); | ||||||||
| 7802 | N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1], | ||||||||
| 7803 | OpValues[2]); | ||||||||
| 7804 | N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1], | ||||||||
| 7805 | OpValues[2]); | ||||||||
| 7806 | setValue(&VPIntrin, N); | ||||||||
| 7807 | break; | ||||||||
| 7808 | } | ||||||||
| 7809 | case ISD::VP_ABS: | ||||||||
| 7810 | case ISD::VP_CTLZ: | ||||||||
| 7811 | case ISD::VP_CTLZ_ZERO_UNDEF: | ||||||||
| 7812 | case ISD::VP_CTTZ: | ||||||||
| 7813 | case ISD::VP_CTTZ_ZERO_UNDEF: { | ||||||||
| 7814 | SDValue Result = | ||||||||
| 7815 | DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]}); | ||||||||
| 7816 | setValue(&VPIntrin, Result); | ||||||||
| 7817 | break; | ||||||||
| 7818 | } | ||||||||
| 7819 | } | ||||||||
| 7820 | } | ||||||||
| 7821 | |||||||||
| 7822 | SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain, | ||||||||
| 7823 | const BasicBlock *EHPadBB, | ||||||||
| 7824 | MCSymbol *&BeginLabel) { | ||||||||
| 7825 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 7826 | MachineModuleInfo &MMI = MF.getMMI(); | ||||||||
| 7827 | |||||||||
| 7828 | // Insert a label before the invoke call to mark the try range. This can be | ||||||||
| 7829 | // used to detect deletion of the invoke via the MachineModuleInfo. | ||||||||
| 7830 | BeginLabel = MMI.getContext().createTempSymbol(); | ||||||||
| 7831 | |||||||||
| 7832 | // For SjLj, keep track of which landing pads go with which invokes | ||||||||
| 7833 | // so as to maintain the ordering of pads in the LSDA. | ||||||||
| 7834 | unsigned CallSiteIndex = MMI.getCurrentCallSite(); | ||||||||
| 7835 | if (CallSiteIndex) { | ||||||||
| 7836 | MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); | ||||||||
| 7837 | LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex); | ||||||||
| 7838 | |||||||||
| 7839 | // Now that the call site is handled, stop tracking it. | ||||||||
| 7840 | MMI.setCurrentCallSite(0); | ||||||||
| 7841 | } | ||||||||
| 7842 | |||||||||
| 7843 | return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel); | ||||||||
| 7844 | } | ||||||||
| 7845 | |||||||||
| 7846 | SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II, | ||||||||
| 7847 | const BasicBlock *EHPadBB, | ||||||||
| 7848 | MCSymbol *BeginLabel) { | ||||||||
| 7849 | assert(BeginLabel && "BeginLabel should've been set")(static_cast <bool> (BeginLabel && "BeginLabel should've been set" ) ? void (0) : __assert_fail ("BeginLabel && \"BeginLabel should've been set\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7849 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7850 | |||||||||
| 7851 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 7852 | MachineModuleInfo &MMI = MF.getMMI(); | ||||||||
| 7853 | |||||||||
| 7854 | // Insert a label at the end of the invoke call to mark the try range. This | ||||||||
| 7855 | // can be used to detect deletion of the invoke via the MachineModuleInfo. | ||||||||
| 7856 | MCSymbol *EndLabel = MMI.getContext().createTempSymbol(); | ||||||||
| 7857 | Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel); | ||||||||
| 7858 | |||||||||
| 7859 | // Inform MachineModuleInfo of range. | ||||||||
| 7860 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); | ||||||||
| 7861 | // There is a platform (e.g. wasm) that uses funclet style IR but does not | ||||||||
| 7862 | // actually use outlined funclets and their LSDA info style. | ||||||||
| 7863 | if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) { | ||||||||
| 7864 | assert(II && "II should've been set")(static_cast <bool> (II && "II should've been set" ) ? void (0) : __assert_fail ("II && \"II should've been set\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7864 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7865 | WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo(); | ||||||||
| 7866 | EHInfo->addIPToStateRange(II, BeginLabel, EndLabel); | ||||||||
| 7867 | } else if (!isScopedEHPersonality(Pers)) { | ||||||||
| 7868 | assert(EHPadBB)(static_cast <bool> (EHPadBB) ? void (0) : __assert_fail ("EHPadBB", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 7868, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7869 | MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel); | ||||||||
| 7870 | } | ||||||||
| 7871 | |||||||||
| 7872 | return Chain; | ||||||||
| 7873 | } | ||||||||
| 7874 | |||||||||
| 7875 | std::pair<SDValue, SDValue> | ||||||||
| 7876 | SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI, | ||||||||
| 7877 | const BasicBlock *EHPadBB) { | ||||||||
| 7878 | MCSymbol *BeginLabel = nullptr; | ||||||||
| 7879 | |||||||||
| 7880 | if (EHPadBB) { | ||||||||
| 7881 | // Both PendingLoads and PendingExports must be flushed here; | ||||||||
| 7882 | // this call might not return. | ||||||||
| 7883 | (void)getRoot(); | ||||||||
| 7884 | DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel)); | ||||||||
| 7885 | CLI.setChain(getRoot()); | ||||||||
| 7886 | } | ||||||||
| 7887 | |||||||||
| 7888 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 7889 | std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); | ||||||||
| 7890 | |||||||||
| 7891 | assert((CLI.IsTailCall || Result.second.getNode()) &&(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode ()) && "Non-null chain expected with non-tail call!") ? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7892 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 7892 | "Non-null chain expected with non-tail call!")(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode ()) && "Non-null chain expected with non-tail call!") ? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7892 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7893 | assert((Result.second.getNode() || !Result.first.getNode()) &&(static_cast <bool> ((Result.second.getNode() || !Result .first.getNode()) && "Null value expected with tail call!" ) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7894 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 7894 | "Null value expected with tail call!")(static_cast <bool> ((Result.second.getNode() || !Result .first.getNode()) && "Null value expected with tail call!" ) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7894 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 7895 | |||||||||
| 7896 | if (!Result.second.getNode()) { | ||||||||
| 7897 | // As a special case, a null chain means that a tail call has been emitted | ||||||||
| 7898 | // and the DAG root is already updated. | ||||||||
| 7899 | HasTailCall = true; | ||||||||
| 7900 | |||||||||
| 7901 | // Since there's no actual continuation from this block, nothing can be | ||||||||
| 7902 | // relying on us setting vregs for them. | ||||||||
| 7903 | PendingExports.clear(); | ||||||||
| 7904 | } else { | ||||||||
| 7905 | DAG.setRoot(Result.second); | ||||||||
| 7906 | } | ||||||||
| 7907 | |||||||||
| 7908 | if (EHPadBB) { | ||||||||
| 7909 | DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB, | ||||||||
| 7910 | BeginLabel)); | ||||||||
| 7911 | } | ||||||||
| 7912 | |||||||||
| 7913 | return Result; | ||||||||
| 7914 | } | ||||||||
| 7915 | |||||||||
| 7916 | void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, | ||||||||
| 7917 | bool isTailCall, | ||||||||
| 7918 | bool isMustTailCall, | ||||||||
| 7919 | const BasicBlock *EHPadBB) { | ||||||||
| 7920 | auto &DL = DAG.getDataLayout(); | ||||||||
| 7921 | FunctionType *FTy = CB.getFunctionType(); | ||||||||
| 7922 | Type *RetTy = CB.getType(); | ||||||||
| 7923 | |||||||||
| 7924 | TargetLowering::ArgListTy Args; | ||||||||
| 7925 | Args.reserve(CB.arg_size()); | ||||||||
| 7926 | |||||||||
| 7927 | const Value *SwiftErrorVal = nullptr; | ||||||||
| 7928 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 7929 | |||||||||
| 7930 | if (isTailCall) { | ||||||||
| 7931 | // Avoid emitting tail calls in functions with the disable-tail-calls | ||||||||
| 7932 | // attribute. | ||||||||
| 7933 | auto *Caller = CB.getParent()->getParent(); | ||||||||
| 7934 | if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() == | ||||||||
| 7935 | "true" && !isMustTailCall) | ||||||||
| 7936 | isTailCall = false; | ||||||||
| 7937 | |||||||||
| 7938 | // We can't tail call inside a function with a swifterror argument. Lowering | ||||||||
| 7939 | // does not support this yet. It would have to move into the swifterror | ||||||||
| 7940 | // register before the call. | ||||||||
| 7941 | if (TLI.supportSwiftError() && | ||||||||
| 7942 | Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) | ||||||||
| 7943 | isTailCall = false; | ||||||||
| 7944 | } | ||||||||
| 7945 | |||||||||
| 7946 | for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) { | ||||||||
| 7947 | TargetLowering::ArgListEntry Entry; | ||||||||
| 7948 | const Value *V = *I; | ||||||||
| 7949 | |||||||||
| 7950 | // Skip empty types | ||||||||
| 7951 | if (V->getType()->isEmptyTy()) | ||||||||
| 7952 | continue; | ||||||||
| 7953 | |||||||||
| 7954 | SDValue ArgNode = getValue(V); | ||||||||
| 7955 | Entry.Node = ArgNode; Entry.Ty = V->getType(); | ||||||||
| 7956 | |||||||||
| 7957 | Entry.setAttributes(&CB, I - CB.arg_begin()); | ||||||||
| 7958 | |||||||||
| 7959 | // Use swifterror virtual register as input to the call. | ||||||||
| 7960 | if (Entry.IsSwiftError && TLI.supportSwiftError()) { | ||||||||
| 7961 | SwiftErrorVal = V; | ||||||||
| 7962 | // We find the virtual register for the actual swifterror argument. | ||||||||
| 7963 | // Instead of using the Value, we use the virtual register instead. | ||||||||
| 7964 | Entry.Node = | ||||||||
| 7965 | DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V), | ||||||||
| 7966 | EVT(TLI.getPointerTy(DL))); | ||||||||
| 7967 | } | ||||||||
| 7968 | |||||||||
| 7969 | Args.push_back(Entry); | ||||||||
| 7970 | |||||||||
| 7971 | // If we have an explicit sret argument that is an Instruction, (i.e., it | ||||||||
| 7972 | // might point to function-local memory), we can't meaningfully tail-call. | ||||||||
| 7973 | if (Entry.IsSRet && isa<Instruction>(V)) | ||||||||
| 7974 | isTailCall = false; | ||||||||
| 7975 | } | ||||||||
| 7976 | |||||||||
| 7977 | // If call site has a cfguardtarget operand bundle, create and add an | ||||||||
| 7978 | // additional ArgListEntry. | ||||||||
| 7979 | if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) { | ||||||||
| 7980 | TargetLowering::ArgListEntry Entry; | ||||||||
| 7981 | Value *V = Bundle->Inputs[0]; | ||||||||
| 7982 | SDValue ArgNode = getValue(V); | ||||||||
| 7983 | Entry.Node = ArgNode; | ||||||||
| 7984 | Entry.Ty = V->getType(); | ||||||||
| 7985 | Entry.IsCFGuardTarget = true; | ||||||||
| 7986 | Args.push_back(Entry); | ||||||||
| 7987 | } | ||||||||
| 7988 | |||||||||
| 7989 | // Check if target-independent constraints permit a tail call here. | ||||||||
| 7990 | // Target-dependent constraints are checked within TLI->LowerCallTo. | ||||||||
| 7991 | if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget())) | ||||||||
| 7992 | isTailCall = false; | ||||||||
| 7993 | |||||||||
| 7994 | // Disable tail calls if there is an swifterror argument. Targets have not | ||||||||
| 7995 | // been updated to support tail calls. | ||||||||
| 7996 | if (TLI.supportSwiftError() && SwiftErrorVal) | ||||||||
| 7997 | isTailCall = false; | ||||||||
| 7998 | |||||||||
| 7999 | ConstantInt *CFIType = nullptr; | ||||||||
| 8000 | if (CB.isIndirectCall()) { | ||||||||
| 8001 | if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) { | ||||||||
| 8002 | if (!TLI.supportKCFIBundles()) | ||||||||
| 8003 | report_fatal_error( | ||||||||
| 8004 | "Target doesn't support calls with kcfi operand bundles."); | ||||||||
| 8005 | CFIType = cast<ConstantInt>(Bundle->Inputs[0]); | ||||||||
| 8006 | assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type")(static_cast <bool> (CFIType->getType()->isIntegerTy (32) && "Invalid CFI type") ? void (0) : __assert_fail ("CFIType->getType()->isIntegerTy(32) && \"Invalid CFI type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8006 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 8007 | } | ||||||||
| 8008 | } | ||||||||
| 8009 | |||||||||
| 8010 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||
| 8011 | CLI.setDebugLoc(getCurSDLoc()) | ||||||||
| 8012 | .setChain(getRoot()) | ||||||||
| 8013 | .setCallee(RetTy, FTy, Callee, std::move(Args), CB) | ||||||||
| 8014 | .setTailCall(isTailCall) | ||||||||
| 8015 | .setConvergent(CB.isConvergent()) | ||||||||
| 8016 | .setIsPreallocated( | ||||||||
| 8017 | CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) | ||||||||
| 8018 | .setCFIType(CFIType); | ||||||||
| 8019 | std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB); | ||||||||
| 8020 | |||||||||
| 8021 | if (Result.first.getNode()) { | ||||||||
| 8022 | Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first); | ||||||||
| 8023 | setValue(&CB, Result.first); | ||||||||
| 8024 | } | ||||||||
| 8025 | |||||||||
| 8026 | // The last element of CLI.InVals has the SDValue for swifterror return. | ||||||||
| 8027 | // Here we copy it to a virtual register and update SwiftErrorMap for | ||||||||
| 8028 | // book-keeping. | ||||||||
| 8029 | if (SwiftErrorVal && TLI.supportSwiftError()) { | ||||||||
| 8030 | // Get the last element of InVals. | ||||||||
| 8031 | SDValue Src = CLI.InVals.back(); | ||||||||
| 8032 | Register VReg = | ||||||||
| 8033 | SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal); | ||||||||
| 8034 | SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src); | ||||||||
| 8035 | DAG.setRoot(CopyNode); | ||||||||
| 8036 | } | ||||||||
| 8037 | } | ||||||||
| 8038 | |||||||||
| 8039 | static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, | ||||||||
| 8040 | SelectionDAGBuilder &Builder) { | ||||||||
| 8041 | // Check to see if this load can be trivially constant folded, e.g. if the | ||||||||
| 8042 | // input is from a string literal. | ||||||||
| 8043 | if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { | ||||||||
| 8044 | // Cast pointer to the type we really want to load. | ||||||||
| 8045 | Type *LoadTy = | ||||||||
| 8046 | Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits()); | ||||||||
| 8047 | if (LoadVT.isVector()) | ||||||||
| 8048 | LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements()); | ||||||||
| 8049 | |||||||||
| 8050 | LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput), | ||||||||
| 8051 | PointerType::getUnqual(LoadTy)); | ||||||||
| 8052 | |||||||||
| 8053 | if (const Constant *LoadCst = | ||||||||
| 8054 | ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput), | ||||||||
| 8055 | LoadTy, Builder.DAG.getDataLayout())) | ||||||||
| 8056 | return Builder.getValue(LoadCst); | ||||||||
| 8057 | } | ||||||||
| 8058 | |||||||||
| 8059 | // Otherwise, we have to emit the load. If the pointer is to unfoldable but | ||||||||
| 8060 | // still constant memory, the input chain can be the entry node. | ||||||||
| 8061 | SDValue Root; | ||||||||
| 8062 | bool ConstantMemory = false; | ||||||||
| 8063 | |||||||||
| 8064 | // Do not serialize (non-volatile) loads of constant memory with anything. | ||||||||
| 8065 | if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) { | ||||||||
| 8066 | Root = Builder.DAG.getEntryNode(); | ||||||||
| 8067 | ConstantMemory = true; | ||||||||
| 8068 | } else { | ||||||||
| 8069 | // Do not serialize non-volatile loads against each other. | ||||||||
| 8070 | Root = Builder.DAG.getRoot(); | ||||||||
| 8071 | } | ||||||||
| 8072 | |||||||||
| 8073 | SDValue Ptr = Builder.getValue(PtrVal); | ||||||||
| 8074 | SDValue LoadVal = | ||||||||
| 8075 | Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr, | ||||||||
| 8076 | MachinePointerInfo(PtrVal), Align(1)); | ||||||||
| 8077 | |||||||||
| 8078 | if (!ConstantMemory) | ||||||||
| 8079 | Builder.PendingLoads.push_back(LoadVal.getValue(1)); | ||||||||
| 8080 | return LoadVal; | ||||||||
| 8081 | } | ||||||||
| 8082 | |||||||||
| 8083 | /// Record the value for an instruction that produces an integer result, | ||||||||
| 8084 | /// converting the type where necessary. | ||||||||
| 8085 | void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, | ||||||||
| 8086 | SDValue Value, | ||||||||
| 8087 | bool IsSigned) { | ||||||||
| 8088 | EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 8089 | I.getType(), true); | ||||||||
| 8090 | if (IsSigned) | ||||||||
| 8091 | Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT); | ||||||||
| 8092 | else | ||||||||
| 8093 | Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT); | ||||||||
| 8094 | setValue(&I, Value); | ||||||||
| 8095 | } | ||||||||
| 8096 | |||||||||
| 8097 | /// See if we can lower a memcmp/bcmp call into an optimized form. If so, return | ||||||||
| 8098 | /// true and lower it. Otherwise return false, and it will be lowered like a | ||||||||
| 8099 | /// normal call. | ||||||||
| 8100 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8101 | /// correct prototype. | ||||||||
| 8102 | bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) { | ||||||||
| 8103 | const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); | ||||||||
| 8104 | const Value *Size = I.getArgOperand(2); | ||||||||
| 8105 | const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size)); | ||||||||
| 8106 | if (CSize && CSize->getZExtValue() == 0) { | ||||||||
| 8107 | EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), | ||||||||
| 8108 | I.getType(), true); | ||||||||
| 8109 | setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT)); | ||||||||
| 8110 | return true; | ||||||||
| 8111 | } | ||||||||
| 8112 | |||||||||
| 8113 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||
| 8114 | std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp( | ||||||||
| 8115 | DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS), | ||||||||
| 8116 | getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS)); | ||||||||
| 8117 | if (Res.first.getNode()) { | ||||||||
| 8118 | processIntegerCallValue(I, Res.first, true); | ||||||||
| 8119 | PendingLoads.push_back(Res.second); | ||||||||
| 8120 | return true; | ||||||||
| 8121 | } | ||||||||
| 8122 | |||||||||
| 8123 | // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 | ||||||||
| 8124 | // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 | ||||||||
| 8125 | if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I)) | ||||||||
| 8126 | return false; | ||||||||
| 8127 | |||||||||
| 8128 | // If the target has a fast compare for the given size, it will return a | ||||||||
| 8129 | // preferred load type for that size. Require that the load VT is legal and | ||||||||
| 8130 | // that the target supports unaligned loads of that type. Otherwise, return | ||||||||
| 8131 | // INVALID. | ||||||||
| 8132 | auto hasFastLoadsAndCompare = [&](unsigned NumBits) { | ||||||||
| 8133 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 8134 | MVT LVT = TLI.hasFastEqualityCompare(NumBits); | ||||||||
| 8135 | if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) { | ||||||||
| 8136 | // TODO: Handle 5 byte compare as 4-byte + 1 byte. | ||||||||
| 8137 | // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. | ||||||||
| 8138 | // TODO: Check alignment of src and dest ptrs. | ||||||||
| 8139 | unsigned DstAS = LHS->getType()->getPointerAddressSpace(); | ||||||||
| 8140 | unsigned SrcAS = RHS->getType()->getPointerAddressSpace(); | ||||||||
| 8141 | if (!TLI.isTypeLegal(LVT) || | ||||||||
| 8142 | !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) || | ||||||||
| 8143 | !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS)) | ||||||||
| 8144 | LVT = MVT::INVALID_SIMPLE_VALUE_TYPE; | ||||||||
| 8145 | } | ||||||||
| 8146 | |||||||||
| 8147 | return LVT; | ||||||||
| 8148 | }; | ||||||||
| 8149 | |||||||||
| 8150 | // This turns into unaligned loads. We only do this if the target natively | ||||||||
| 8151 | // supports the MVT we'll be loading or if it is small enough (<= 4) that | ||||||||
| 8152 | // we'll only produce a small number of byte loads. | ||||||||
| 8153 | MVT LoadVT; | ||||||||
| 8154 | unsigned NumBitsToCompare = CSize->getZExtValue() * 8; | ||||||||
| 8155 | switch (NumBitsToCompare) { | ||||||||
| 8156 | default: | ||||||||
| 8157 | return false; | ||||||||
| 8158 | case 16: | ||||||||
| 8159 | LoadVT = MVT::i16; | ||||||||
| 8160 | break; | ||||||||
| 8161 | case 32: | ||||||||
| 8162 | LoadVT = MVT::i32; | ||||||||
| 8163 | break; | ||||||||
| 8164 | case 64: | ||||||||
| 8165 | case 128: | ||||||||
| 8166 | case 256: | ||||||||
| 8167 | LoadVT = hasFastLoadsAndCompare(NumBitsToCompare); | ||||||||
| 8168 | break; | ||||||||
| 8169 | } | ||||||||
| 8170 | |||||||||
| 8171 | if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE) | ||||||||
| 8172 | return false; | ||||||||
| 8173 | |||||||||
| 8174 | SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this); | ||||||||
| 8175 | SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this); | ||||||||
| 8176 | |||||||||
| 8177 | // Bitcast to a wide integer type if the loads are vectors. | ||||||||
| 8178 | if (LoadVT.isVector()) { | ||||||||
| 8179 | EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits()); | ||||||||
| 8180 | LoadL = DAG.getBitcast(CmpVT, LoadL); | ||||||||
| 8181 | LoadR = DAG.getBitcast(CmpVT, LoadR); | ||||||||
| 8182 | } | ||||||||
| 8183 | |||||||||
| 8184 | SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE); | ||||||||
| 8185 | processIntegerCallValue(I, Cmp, false); | ||||||||
| 8186 | return true; | ||||||||
| 8187 | } | ||||||||
| 8188 | |||||||||
| 8189 | /// See if we can lower a memchr call into an optimized form. If so, return | ||||||||
| 8190 | /// true and lower it. Otherwise return false, and it will be lowered like a | ||||||||
| 8191 | /// normal call. | ||||||||
| 8192 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8193 | /// correct prototype. | ||||||||
| 8194 | bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { | ||||||||
| 8195 | const Value *Src = I.getArgOperand(0); | ||||||||
| 8196 | const Value *Char = I.getArgOperand(1); | ||||||||
| 8197 | const Value *Length = I.getArgOperand(2); | ||||||||
| 8198 | |||||||||
| 8199 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||
| 8200 | std::pair<SDValue, SDValue> Res = | ||||||||
| 8201 | TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||
| 8202 | getValue(Src), getValue(Char), getValue(Length), | ||||||||
| 8203 | MachinePointerInfo(Src)); | ||||||||
| 8204 | if (Res.first.getNode()) { | ||||||||
| 8205 | setValue(&I, Res.first); | ||||||||
| 8206 | PendingLoads.push_back(Res.second); | ||||||||
| 8207 | return true; | ||||||||
| 8208 | } | ||||||||
| 8209 | |||||||||
| 8210 | return false; | ||||||||
| 8211 | } | ||||||||
| 8212 | |||||||||
| 8213 | /// See if we can lower a mempcpy call into an optimized form. If so, return | ||||||||
| 8214 | /// true and lower it. Otherwise return false, and it will be lowered like a | ||||||||
| 8215 | /// normal call. | ||||||||
| 8216 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8217 | /// correct prototype. | ||||||||
| 8218 | bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) { | ||||||||
| 8219 | SDValue Dst = getValue(I.getArgOperand(0)); | ||||||||
| 8220 | SDValue Src = getValue(I.getArgOperand(1)); | ||||||||
| 8221 | SDValue Size = getValue(I.getArgOperand(2)); | ||||||||
| 8222 | |||||||||
| 8223 | Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne(); | ||||||||
| 8224 | Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne(); | ||||||||
| 8225 | // DAG::getMemcpy needs Alignment to be defined. | ||||||||
| 8226 | Align Alignment = std::min(DstAlign, SrcAlign); | ||||||||
| 8227 | |||||||||
| 8228 | bool isVol = false; | ||||||||
| 8229 | SDLoc sdl = getCurSDLoc(); | ||||||||
| 8230 | |||||||||
| 8231 | // In the mempcpy context we need to pass in a false value for isTailCall | ||||||||
| 8232 | // because the return pointer needs to be adjusted by the size of | ||||||||
| 8233 | // the copied memory. | ||||||||
| 8234 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); | ||||||||
| 8235 | SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false, | ||||||||
| 8236 | /*isTailCall=*/false, | ||||||||
| 8237 | MachinePointerInfo(I.getArgOperand(0)), | ||||||||
| 8238 | MachinePointerInfo(I.getArgOperand(1)), | ||||||||
| 8239 | I.getAAMetadata()); | ||||||||
| 8240 | assert(MC.getNode() != nullptr &&(static_cast <bool> (MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **" ) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8241 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8241 | "** memcpy should not be lowered as TailCall in mempcpy context **")(static_cast <bool> (MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **" ) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8241 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 8242 | DAG.setRoot(MC); | ||||||||
| 8243 | |||||||||
| 8244 | // Check if Size needs to be truncated or extended. | ||||||||
| 8245 | Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType()); | ||||||||
| 8246 | |||||||||
| 8247 | // Adjust return pointer to point just past the last dst byte. | ||||||||
| 8248 | SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(), | ||||||||
| 8249 | Dst, Size); | ||||||||
| 8250 | setValue(&I, DstPlusSize); | ||||||||
| 8251 | return true; | ||||||||
| 8252 | } | ||||||||
| 8253 | |||||||||
| 8254 | /// See if we can lower a strcpy call into an optimized form. If so, return | ||||||||
| 8255 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||
| 8256 | /// normal call. | ||||||||
| 8257 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8258 | /// correct prototype. | ||||||||
| 8259 | bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { | ||||||||
| 8260 | const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); | ||||||||
| 8261 | |||||||||
| 8262 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||
| 8263 | std::pair<SDValue, SDValue> Res = | ||||||||
| 8264 | TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(), | ||||||||
| 8265 | getValue(Arg0), getValue(Arg1), | ||||||||
| 8266 | MachinePointerInfo(Arg0), | ||||||||
| 8267 | MachinePointerInfo(Arg1), isStpcpy); | ||||||||
| 8268 | if (Res.first.getNode()) { | ||||||||
| 8269 | setValue(&I, Res.first); | ||||||||
| 8270 | DAG.setRoot(Res.second); | ||||||||
| 8271 | return true; | ||||||||
| 8272 | } | ||||||||
| 8273 | |||||||||
| 8274 | return false; | ||||||||
| 8275 | } | ||||||||
| 8276 | |||||||||
| 8277 | /// See if we can lower a strcmp call into an optimized form. If so, return | ||||||||
| 8278 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||
| 8279 | /// normal call. | ||||||||
| 8280 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8281 | /// correct prototype. | ||||||||
| 8282 | bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { | ||||||||
| 8283 | const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); | ||||||||
| 8284 | |||||||||
| 8285 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||
| 8286 | std::pair<SDValue, SDValue> Res = | ||||||||
| 8287 | TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||
| 8288 | getValue(Arg0), getValue(Arg1), | ||||||||
| 8289 | MachinePointerInfo(Arg0), | ||||||||
| 8290 | MachinePointerInfo(Arg1)); | ||||||||
| 8291 | if (Res.first.getNode()) { | ||||||||
| 8292 | processIntegerCallValue(I, Res.first, true); | ||||||||
| 8293 | PendingLoads.push_back(Res.second); | ||||||||
| 8294 | return true; | ||||||||
| 8295 | } | ||||||||
| 8296 | |||||||||
| 8297 | return false; | ||||||||
| 8298 | } | ||||||||
| 8299 | |||||||||
| 8300 | /// See if we can lower a strlen call into an optimized form. If so, return | ||||||||
| 8301 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||
| 8302 | /// normal call. | ||||||||
| 8303 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8304 | /// correct prototype. | ||||||||
| 8305 | bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { | ||||||||
| 8306 | const Value *Arg0 = I.getArgOperand(0); | ||||||||
| 8307 | |||||||||
| 8308 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||
| 8309 | std::pair<SDValue, SDValue> Res = | ||||||||
| 8310 | TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||
| 8311 | getValue(Arg0), MachinePointerInfo(Arg0)); | ||||||||
| 8312 | if (Res.first.getNode()) { | ||||||||
| 8313 | processIntegerCallValue(I, Res.first, false); | ||||||||
| 8314 | PendingLoads.push_back(Res.second); | ||||||||
| 8315 | return true; | ||||||||
| 8316 | } | ||||||||
| 8317 | |||||||||
| 8318 | return false; | ||||||||
| 8319 | } | ||||||||
| 8320 | |||||||||
| 8321 | /// See if we can lower a strnlen call into an optimized form. If so, return | ||||||||
| 8322 | /// true and lower it, otherwise return false and it will be lowered like a | ||||||||
| 8323 | /// normal call. | ||||||||
| 8324 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8325 | /// correct prototype. | ||||||||
| 8326 | bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { | ||||||||
| 8327 | const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); | ||||||||
| 8328 | |||||||||
| 8329 | const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); | ||||||||
| 8330 | std::pair<SDValue, SDValue> Res = | ||||||||
| 8331 | TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), | ||||||||
| 8332 | getValue(Arg0), getValue(Arg1), | ||||||||
| 8333 | MachinePointerInfo(Arg0)); | ||||||||
| 8334 | if (Res.first.getNode()) { | ||||||||
| 8335 | processIntegerCallValue(I, Res.first, false); | ||||||||
| 8336 | PendingLoads.push_back(Res.second); | ||||||||
| 8337 | return true; | ||||||||
| 8338 | } | ||||||||
| 8339 | |||||||||
| 8340 | return false; | ||||||||
| 8341 | } | ||||||||
| 8342 | |||||||||
| 8343 | /// See if we can lower a unary floating-point operation into an SDNode with | ||||||||
| 8344 | /// the specified Opcode. If so, return true and lower it, otherwise return | ||||||||
| 8345 | /// false and it will be lowered like a normal call. | ||||||||
| 8346 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8347 | /// correct prototype. | ||||||||
| 8348 | bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, | ||||||||
| 8349 | unsigned Opcode) { | ||||||||
| 8350 | // We already checked this call's prototype; verify it doesn't modify errno. | ||||||||
| 8351 | if (!I.onlyReadsMemory()) | ||||||||
| 8352 | return false; | ||||||||
| 8353 | |||||||||
| 8354 | SDNodeFlags Flags; | ||||||||
| 8355 | Flags.copyFMF(cast<FPMathOperator>(I)); | ||||||||
| 8356 | |||||||||
| 8357 | SDValue Tmp = getValue(I.getArgOperand(0)); | ||||||||
| 8358 | setValue(&I, | ||||||||
| 8359 | DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags)); | ||||||||
| 8360 | return true; | ||||||||
| 8361 | } | ||||||||
| 8362 | |||||||||
| 8363 | /// See if we can lower a binary floating-point operation into an SDNode with | ||||||||
| 8364 | /// the specified Opcode. If so, return true and lower it. Otherwise return | ||||||||
| 8365 | /// false, and it will be lowered like a normal call. | ||||||||
| 8366 | /// The caller already checked that \p I calls the appropriate LibFunc with a | ||||||||
| 8367 | /// correct prototype. | ||||||||
| 8368 | bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I, | ||||||||
| 8369 | unsigned Opcode) { | ||||||||
| 8370 | // We already checked this call's prototype; verify it doesn't modify errno. | ||||||||
| 8371 | if (!I.onlyReadsMemory()) | ||||||||
| 8372 | return false; | ||||||||
| 8373 | |||||||||
| 8374 | SDNodeFlags Flags; | ||||||||
| 8375 | Flags.copyFMF(cast<FPMathOperator>(I)); | ||||||||
| 8376 | |||||||||
| 8377 | SDValue Tmp0 = getValue(I.getArgOperand(0)); | ||||||||
| 8378 | SDValue Tmp1 = getValue(I.getArgOperand(1)); | ||||||||
| 8379 | EVT VT = Tmp0.getValueType(); | ||||||||
| 8380 | setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags)); | ||||||||
| 8381 | return true; | ||||||||
| 8382 | } | ||||||||
| 8383 | |||||||||
| 8384 | void SelectionDAGBuilder::visitCall(const CallInst &I) { | ||||||||
| 8385 | // Handle inline assembly differently. | ||||||||
| 8386 | if (I.isInlineAsm()) { | ||||||||
| 8387 | visitInlineAsm(I); | ||||||||
| 8388 | return; | ||||||||
| 8389 | } | ||||||||
| 8390 | |||||||||
| 8391 | diagnoseDontCall(I); | ||||||||
| 8392 | |||||||||
| 8393 | if (Function *F = I.getCalledFunction()) { | ||||||||
| 8394 | if (F->isDeclaration()) { | ||||||||
| 8395 | // Is this an LLVM intrinsic or a target-specific intrinsic? | ||||||||
| 8396 | unsigned IID = F->getIntrinsicID(); | ||||||||
| 8397 | if (!IID) | ||||||||
| 8398 | if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) | ||||||||
| 8399 | IID = II->getIntrinsicID(F); | ||||||||
| 8400 | |||||||||
| 8401 | if (IID) { | ||||||||
| 8402 | visitIntrinsicCall(I, IID); | ||||||||
| 8403 | return; | ||||||||
| 8404 | } | ||||||||
| 8405 | } | ||||||||
| 8406 | |||||||||
| 8407 | // Check for well-known libc/libm calls. If the function is internal, it | ||||||||
| 8408 | // can't be a library call. Don't do the check if marked as nobuiltin for | ||||||||
| 8409 | // some reason or the call site requires strict floating point semantics. | ||||||||
| 8410 | LibFunc Func; | ||||||||
| 8411 | if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() && | ||||||||
| 8412 | F->hasName() && LibInfo->getLibFunc(*F, Func) && | ||||||||
| 8413 | LibInfo->hasOptimizedCodeGen(Func)) { | ||||||||
| 8414 | switch (Func) { | ||||||||
| 8415 | default: break; | ||||||||
| 8416 | case LibFunc_bcmp: | ||||||||
| 8417 | if (visitMemCmpBCmpCall(I)) | ||||||||
| 8418 | return; | ||||||||
| 8419 | break; | ||||||||
| 8420 | case LibFunc_copysign: | ||||||||
| 8421 | case LibFunc_copysignf: | ||||||||
| 8422 | case LibFunc_copysignl: | ||||||||
| 8423 | // We already checked this call's prototype; verify it doesn't modify | ||||||||
| 8424 | // errno. | ||||||||
| 8425 | if (I.onlyReadsMemory()) { | ||||||||
| 8426 | SDValue LHS = getValue(I.getArgOperand(0)); | ||||||||
| 8427 | SDValue RHS = getValue(I.getArgOperand(1)); | ||||||||
| 8428 | setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(), | ||||||||
| 8429 | LHS.getValueType(), LHS, RHS)); | ||||||||
| 8430 | return; | ||||||||
| 8431 | } | ||||||||
| 8432 | break; | ||||||||
| 8433 | case LibFunc_fabs: | ||||||||
| 8434 | case LibFunc_fabsf: | ||||||||
| 8435 | case LibFunc_fabsl: | ||||||||
| 8436 | if (visitUnaryFloatCall(I, ISD::FABS)) | ||||||||
| 8437 | return; | ||||||||
| 8438 | break; | ||||||||
| 8439 | case LibFunc_fmin: | ||||||||
| 8440 | case LibFunc_fminf: | ||||||||
| 8441 | case LibFunc_fminl: | ||||||||
| 8442 | if (visitBinaryFloatCall(I, ISD::FMINNUM)) | ||||||||
| 8443 | return; | ||||||||
| 8444 | break; | ||||||||
| 8445 | case LibFunc_fmax: | ||||||||
| 8446 | case LibFunc_fmaxf: | ||||||||
| 8447 | case LibFunc_fmaxl: | ||||||||
| 8448 | if (visitBinaryFloatCall(I, ISD::FMAXNUM)) | ||||||||
| 8449 | return; | ||||||||
| 8450 | break; | ||||||||
| 8451 | case LibFunc_sin: | ||||||||
| 8452 | case LibFunc_sinf: | ||||||||
| 8453 | case LibFunc_sinl: | ||||||||
| 8454 | if (visitUnaryFloatCall(I, ISD::FSIN)) | ||||||||
| 8455 | return; | ||||||||
| 8456 | break; | ||||||||
| 8457 | case LibFunc_cos: | ||||||||
| 8458 | case LibFunc_cosf: | ||||||||
| 8459 | case LibFunc_cosl: | ||||||||
| 8460 | if (visitUnaryFloatCall(I, ISD::FCOS)) | ||||||||
| 8461 | return; | ||||||||
| 8462 | break; | ||||||||
| 8463 | case LibFunc_sqrt: | ||||||||
| 8464 | case LibFunc_sqrtf: | ||||||||
| 8465 | case LibFunc_sqrtl: | ||||||||
| 8466 | case LibFunc_sqrt_finite: | ||||||||
| 8467 | case LibFunc_sqrtf_finite: | ||||||||
| 8468 | case LibFunc_sqrtl_finite: | ||||||||
| 8469 | if (visitUnaryFloatCall(I, ISD::FSQRT)) | ||||||||
| 8470 | return; | ||||||||
| 8471 | break; | ||||||||
| 8472 | case LibFunc_floor: | ||||||||
| 8473 | case LibFunc_floorf: | ||||||||
| 8474 | case LibFunc_floorl: | ||||||||
| 8475 | if (visitUnaryFloatCall(I, ISD::FFLOOR)) | ||||||||
| 8476 | return; | ||||||||
| 8477 | break; | ||||||||
| 8478 | case LibFunc_nearbyint: | ||||||||
| 8479 | case LibFunc_nearbyintf: | ||||||||
| 8480 | case LibFunc_nearbyintl: | ||||||||
| 8481 | if (visitUnaryFloatCall(I, ISD::FNEARBYINT)) | ||||||||
| 8482 | return; | ||||||||
| 8483 | break; | ||||||||
| 8484 | case LibFunc_ceil: | ||||||||
| 8485 | case LibFunc_ceilf: | ||||||||
| 8486 | case LibFunc_ceill: | ||||||||
| 8487 | if (visitUnaryFloatCall(I, ISD::FCEIL)) | ||||||||
| 8488 | return; | ||||||||
| 8489 | break; | ||||||||
| 8490 | case LibFunc_rint: | ||||||||
| 8491 | case LibFunc_rintf: | ||||||||
| 8492 | case LibFunc_rintl: | ||||||||
| 8493 | if (visitUnaryFloatCall(I, ISD::FRINT)) | ||||||||
| 8494 | return; | ||||||||
| 8495 | break; | ||||||||
| 8496 | case LibFunc_round: | ||||||||
| 8497 | case LibFunc_roundf: | ||||||||
| 8498 | case LibFunc_roundl: | ||||||||
| 8499 | if (visitUnaryFloatCall(I, ISD::FROUND)) | ||||||||
| 8500 | return; | ||||||||
| 8501 | break; | ||||||||
| 8502 | case LibFunc_trunc: | ||||||||
| 8503 | case LibFunc_truncf: | ||||||||
| 8504 | case LibFunc_truncl: | ||||||||
| 8505 | if (visitUnaryFloatCall(I, ISD::FTRUNC)) | ||||||||
| 8506 | return; | ||||||||
| 8507 | break; | ||||||||
| 8508 | case LibFunc_log2: | ||||||||
| 8509 | case LibFunc_log2f: | ||||||||
| 8510 | case LibFunc_log2l: | ||||||||
| 8511 | if (visitUnaryFloatCall(I, ISD::FLOG2)) | ||||||||
| 8512 | return; | ||||||||
| 8513 | break; | ||||||||
| 8514 | case LibFunc_exp2: | ||||||||
| 8515 | case LibFunc_exp2f: | ||||||||
| 8516 | case LibFunc_exp2l: | ||||||||
| 8517 | if (visitUnaryFloatCall(I, ISD::FEXP2)) | ||||||||
| 8518 | return; | ||||||||
| 8519 | break; | ||||||||
| 8520 | case LibFunc_memcmp: | ||||||||
| 8521 | if (visitMemCmpBCmpCall(I)) | ||||||||
| 8522 | return; | ||||||||
| 8523 | break; | ||||||||
| 8524 | case LibFunc_mempcpy: | ||||||||
| 8525 | if (visitMemPCpyCall(I)) | ||||||||
| 8526 | return; | ||||||||
| 8527 | break; | ||||||||
| 8528 | case LibFunc_memchr: | ||||||||
| 8529 | if (visitMemChrCall(I)) | ||||||||
| 8530 | return; | ||||||||
| 8531 | break; | ||||||||
| 8532 | case LibFunc_strcpy: | ||||||||
| 8533 | if (visitStrCpyCall(I, false)) | ||||||||
| 8534 | return; | ||||||||
| 8535 | break; | ||||||||
| 8536 | case LibFunc_stpcpy: | ||||||||
| 8537 | if (visitStrCpyCall(I, true)) | ||||||||
| 8538 | return; | ||||||||
| 8539 | break; | ||||||||
| 8540 | case LibFunc_strcmp: | ||||||||
| 8541 | if (visitStrCmpCall(I)) | ||||||||
| 8542 | return; | ||||||||
| 8543 | break; | ||||||||
| 8544 | case LibFunc_strlen: | ||||||||
| 8545 | if (visitStrLenCall(I)) | ||||||||
| 8546 | return; | ||||||||
| 8547 | break; | ||||||||
| 8548 | case LibFunc_strnlen: | ||||||||
| 8549 | if (visitStrNLenCall(I)) | ||||||||
| 8550 | return; | ||||||||
| 8551 | break; | ||||||||
| 8552 | } | ||||||||
| 8553 | } | ||||||||
| 8554 | } | ||||||||
| 8555 | |||||||||
| 8556 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't | ||||||||
| 8557 | // have to do anything here to lower funclet bundles. | ||||||||
| 8558 | // CFGuardTarget bundles are lowered in LowerCallTo. | ||||||||
| 8559 | assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall , LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8560 | {LLVMContext::OB_deopt, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall , LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8561 | LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall , LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8562 | LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall , LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8563 | "Cannot lower calls with arbitrary operand bundles!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext ::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget , LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall , LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!" ) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 8564 | |||||||||
| 8565 | SDValue Callee = getValue(I.getCalledOperand()); | ||||||||
| 8566 | |||||||||
| 8567 | if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) | ||||||||
| 8568 | LowerCallSiteWithDeoptBundle(&I, Callee, nullptr); | ||||||||
| 8569 | else | ||||||||
| 8570 | // Check if we can potentially perform a tail call. More detailed checking | ||||||||
| 8571 | // is be done within LowerCallTo, after more information about the call is | ||||||||
| 8572 | // known. | ||||||||
| 8573 | LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall()); | ||||||||
| 8574 | } | ||||||||
| 8575 | |||||||||
| 8576 | namespace { | ||||||||
| 8577 | |||||||||
| 8578 | /// AsmOperandInfo - This contains information for each constraint that we are | ||||||||
| 8579 | /// lowering. | ||||||||
| 8580 | class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { | ||||||||
| 8581 | public: | ||||||||
| 8582 | /// CallOperand - If this is the result output operand or a clobber | ||||||||
| 8583 | /// this is null, otherwise it is the incoming operand to the CallInst. | ||||||||
| 8584 | /// This gets modified as the asm is processed. | ||||||||
| 8585 | SDValue CallOperand; | ||||||||
| 8586 | |||||||||
| 8587 | /// AssignedRegs - If this is a register or register class operand, this | ||||||||
| 8588 | /// contains the set of register corresponding to the operand. | ||||||||
| 8589 | RegsForValue AssignedRegs; | ||||||||
| 8590 | |||||||||
| 8591 | explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info) | ||||||||
| 8592 | : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) { | ||||||||
| 8593 | } | ||||||||
| 8594 | |||||||||
| 8595 | /// Whether or not this operand accesses memory | ||||||||
| 8596 | bool hasMemory(const TargetLowering &TLI) const { | ||||||||
| 8597 | // Indirect operand accesses access memory. | ||||||||
| 8598 | if (isIndirect) | ||||||||
| 8599 | return true; | ||||||||
| 8600 | |||||||||
| 8601 | for (const auto &Code : Codes) | ||||||||
| 8602 | if (TLI.getConstraintType(Code) == TargetLowering::C_Memory) | ||||||||
| 8603 | return true; | ||||||||
| 8604 | |||||||||
| 8605 | return false; | ||||||||
| 8606 | } | ||||||||
| 8607 | }; | ||||||||
| 8608 | |||||||||
| 8609 | |||||||||
| 8610 | } // end anonymous namespace | ||||||||
| 8611 | |||||||||
| 8612 | /// Make sure that the output operand \p OpInfo and its corresponding input | ||||||||
| 8613 | /// operand \p MatchingOpInfo have compatible constraint types (otherwise error | ||||||||
| 8614 | /// out). | ||||||||
| 8615 | static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, | ||||||||
| 8616 | SDISelAsmOperandInfo &MatchingOpInfo, | ||||||||
| 8617 | SelectionDAG &DAG) { | ||||||||
| 8618 | if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT) | ||||||||
| 8619 | return; | ||||||||
| 8620 | |||||||||
| 8621 | const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo(); | ||||||||
| 8622 | const auto &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 8623 | |||||||||
| 8624 | std::pair<unsigned, const TargetRegisterClass *> MatchRC = | ||||||||
| 8625 | TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode, | ||||||||
| 8626 | OpInfo.ConstraintVT); | ||||||||
| 8627 | std::pair<unsigned, const TargetRegisterClass *> InputRC = | ||||||||
| 8628 | TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode, | ||||||||
| 8629 | MatchingOpInfo.ConstraintVT); | ||||||||
| 8630 | if ((OpInfo.ConstraintVT.isInteger() != | ||||||||
| 8631 | MatchingOpInfo.ConstraintVT.isInteger()) || | ||||||||
| 8632 | (MatchRC.second != InputRC.second)) { | ||||||||
| 8633 | // FIXME: error out in a more elegant fashion | ||||||||
| 8634 | report_fatal_error("Unsupported asm: input constraint" | ||||||||
| 8635 | " with a matching output constraint of" | ||||||||
| 8636 | " incompatible type!"); | ||||||||
| 8637 | } | ||||||||
| 8638 | MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT; | ||||||||
| 8639 | } | ||||||||
| 8640 | |||||||||
| 8641 | /// Get a direct memory input to behave well as an indirect operand. | ||||||||
| 8642 | /// This may introduce stores, hence the need for a \p Chain. | ||||||||
| 8643 | /// \return The (possibly updated) chain. | ||||||||
| 8644 | static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, | ||||||||
| 8645 | SDISelAsmOperandInfo &OpInfo, | ||||||||
| 8646 | SelectionDAG &DAG) { | ||||||||
| 8647 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 8648 | |||||||||
| 8649 | // If we don't have an indirect input, put it in the constpool if we can, | ||||||||
| 8650 | // otherwise spill it to a stack slot. | ||||||||
| 8651 | // TODO: This isn't quite right. We need to handle these according to | ||||||||
| 8652 | // the addressing mode that the constraint wants. Also, this may take | ||||||||
| 8653 | // an additional register for the computation and we don't want that | ||||||||
| 8654 | // either. | ||||||||
| 8655 | |||||||||
| 8656 | // If the operand is a float, integer, or vector constant, spill to a | ||||||||
| 8657 | // constant pool entry to get its address. | ||||||||
| 8658 | const Value *OpVal = OpInfo.CallOperandVal; | ||||||||
| 8659 | if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) || | ||||||||
| 8660 | isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) { | ||||||||
| 8661 | OpInfo.CallOperand = DAG.getConstantPool( | ||||||||
| 8662 | cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout())); | ||||||||
| 8663 | return Chain; | ||||||||
| 8664 | } | ||||||||
| 8665 | |||||||||
| 8666 | // Otherwise, create a stack slot and emit a store to it before the asm. | ||||||||
| 8667 | Type *Ty = OpVal->getType(); | ||||||||
| 8668 | auto &DL = DAG.getDataLayout(); | ||||||||
| 8669 | uint64_t TySize = DL.getTypeAllocSize(Ty); | ||||||||
| 8670 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 8671 | int SSFI = MF.getFrameInfo().CreateStackObject( | ||||||||
| 8672 | TySize, DL.getPrefTypeAlign(Ty), false); | ||||||||
| 8673 | SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL)); | ||||||||
| 8674 | Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot, | ||||||||
| 8675 | MachinePointerInfo::getFixedStack(MF, SSFI), | ||||||||
| 8676 | TLI.getMemValueType(DL, Ty)); | ||||||||
| 8677 | OpInfo.CallOperand = StackSlot; | ||||||||
| 8678 | |||||||||
| 8679 | return Chain; | ||||||||
| 8680 | } | ||||||||
| 8681 | |||||||||
| 8682 | /// GetRegistersForValue - Assign registers (virtual or physical) for the | ||||||||
| 8683 | /// specified operand. We prefer to assign virtual registers, to allow the | ||||||||
| 8684 | /// register allocator to handle the assignment process. However, if the asm | ||||||||
| 8685 | /// uses features that we can't model on machineinstrs, we have SDISel do the | ||||||||
| 8686 | /// allocation. This produces generally horrible, but correct, code. | ||||||||
| 8687 | /// | ||||||||
| 8688 | /// OpInfo describes the operand | ||||||||
| 8689 | /// RefOpInfo describes the matching operand if any, the operand otherwise | ||||||||
| 8690 | static std::optional<unsigned> | ||||||||
| 8691 | getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL, | ||||||||
| 8692 | SDISelAsmOperandInfo &OpInfo, | ||||||||
| 8693 | SDISelAsmOperandInfo &RefOpInfo) { | ||||||||
| 8694 | LLVMContext &Context = *DAG.getContext(); | ||||||||
| 8695 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 8696 | |||||||||
| 8697 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 8698 | SmallVector<unsigned, 4> Regs; | ||||||||
| 8699 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | ||||||||
| 8700 | |||||||||
| 8701 | // No work to do for memory/address operands. | ||||||||
| 8702 | if (OpInfo.ConstraintType == TargetLowering::C_Memory || | ||||||||
| 8703 | OpInfo.ConstraintType == TargetLowering::C_Address) | ||||||||
| 8704 | return std::nullopt; | ||||||||
| 8705 | |||||||||
| 8706 | // If this is a constraint for a single physreg, or a constraint for a | ||||||||
| 8707 | // register class, find it. | ||||||||
| 8708 | unsigned AssignedReg; | ||||||||
| 8709 | const TargetRegisterClass *RC; | ||||||||
| 8710 | std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint( | ||||||||
| 8711 | &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT); | ||||||||
| 8712 | // RC is unset only on failure. Return immediately. | ||||||||
| 8713 | if (!RC) | ||||||||
| 8714 | return std::nullopt; | ||||||||
| 8715 | |||||||||
| 8716 | // Get the actual register value type. This is important, because the user | ||||||||
| 8717 | // may have asked for (e.g.) the AX register in i32 type. We need to | ||||||||
| 8718 | // remember that AX is actually i16 to get the right extension. | ||||||||
| 8719 | const MVT RegVT = *TRI.legalclasstypes_begin(*RC); | ||||||||
| 8720 | |||||||||
| 8721 | if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) { | ||||||||
| 8722 | // If this is an FP operand in an integer register (or visa versa), or more | ||||||||
| 8723 | // generally if the operand value disagrees with the register class we plan | ||||||||
| 8724 | // to stick it in, fix the operand type. | ||||||||
| 8725 | // | ||||||||
| 8726 | // If this is an input value, the bitcast to the new type is done now. | ||||||||
| 8727 | // Bitcast for output value is done at the end of visitInlineAsm(). | ||||||||
| 8728 | if ((OpInfo.Type == InlineAsm::isOutput || | ||||||||
| 8729 | OpInfo.Type == InlineAsm::isInput) && | ||||||||
| 8730 | !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) { | ||||||||
| 8731 | // Try to convert to the first EVT that the reg class contains. If the | ||||||||
| 8732 | // types are identical size, use a bitcast to convert (e.g. two differing | ||||||||
| 8733 | // vector types). Note: output bitcast is done at the end of | ||||||||
| 8734 | // visitInlineAsm(). | ||||||||
| 8735 | if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) { | ||||||||
| 8736 | // Exclude indirect inputs while they are unsupported because the code | ||||||||
| 8737 | // to perform the load is missing and thus OpInfo.CallOperand still | ||||||||
| 8738 | // refers to the input address rather than the pointed-to value. | ||||||||
| 8739 | if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect) | ||||||||
| 8740 | OpInfo.CallOperand = | ||||||||
| 8741 | DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand); | ||||||||
| 8742 | OpInfo.ConstraintVT = RegVT; | ||||||||
| 8743 | // If the operand is an FP value and we want it in integer registers, | ||||||||
| 8744 | // use the corresponding integer type. This turns an f64 value into | ||||||||
| 8745 | // i64, which can be passed with two i32 values on a 32-bit machine. | ||||||||
| 8746 | } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) { | ||||||||
| 8747 | MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits()); | ||||||||
| 8748 | if (OpInfo.Type == InlineAsm::isInput) | ||||||||
| 8749 | OpInfo.CallOperand = | ||||||||
| 8750 | DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand); | ||||||||
| 8751 | OpInfo.ConstraintVT = VT; | ||||||||
| 8752 | } | ||||||||
| 8753 | } | ||||||||
| 8754 | } | ||||||||
| 8755 | |||||||||
| 8756 | // No need to allocate a matching input constraint since the constraint it's | ||||||||
| 8757 | // matching to has already been allocated. | ||||||||
| 8758 | if (OpInfo.isMatchingInputConstraint()) | ||||||||
| 8759 | return std::nullopt; | ||||||||
| 8760 | |||||||||
| 8761 | EVT ValueVT = OpInfo.ConstraintVT; | ||||||||
| 8762 | if (OpInfo.ConstraintVT == MVT::Other) | ||||||||
| 8763 | ValueVT = RegVT; | ||||||||
| 8764 | |||||||||
| 8765 | // Initialize NumRegs. | ||||||||
| 8766 | unsigned NumRegs = 1; | ||||||||
| 8767 | if (OpInfo.ConstraintVT != MVT::Other) | ||||||||
| 8768 | NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT); | ||||||||
| 8769 | |||||||||
| 8770 | // If this is a constraint for a specific physical register, like {r17}, | ||||||||
| 8771 | // assign it now. | ||||||||
| 8772 | |||||||||
| 8773 | // If this associated to a specific register, initialize iterator to correct | ||||||||
| 8774 | // place. If virtual, make sure we have enough registers | ||||||||
| 8775 | |||||||||
| 8776 | // Initialize iterator if necessary | ||||||||
| 8777 | TargetRegisterClass::iterator I = RC->begin(); | ||||||||
| 8778 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); | ||||||||
| 8779 | |||||||||
| 8780 | // Do not check for single registers. | ||||||||
| 8781 | if (AssignedReg) { | ||||||||
| 8782 | I = std::find(I, RC->end(), AssignedReg); | ||||||||
| 8783 | if (I == RC->end()) { | ||||||||
| 8784 | // RC does not contain the selected register, which indicates a | ||||||||
| 8785 | // mismatch between the register and the required type/bitwidth. | ||||||||
| 8786 | return {AssignedReg}; | ||||||||
| 8787 | } | ||||||||
| 8788 | } | ||||||||
| 8789 | |||||||||
| 8790 | for (; NumRegs; --NumRegs, ++I) { | ||||||||
| 8791 | assert(I != RC->end() && "Ran out of registers to allocate!")(static_cast <bool> (I != RC->end() && "Ran out of registers to allocate!" ) ? void (0) : __assert_fail ("I != RC->end() && \"Ran out of registers to allocate!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8791 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 8792 | Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC); | ||||||||
| 8793 | Regs.push_back(R); | ||||||||
| 8794 | } | ||||||||
| 8795 | |||||||||
| 8796 | OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); | ||||||||
| 8797 | return std::nullopt; | ||||||||
| 8798 | } | ||||||||
| 8799 | |||||||||
| 8800 | static unsigned | ||||||||
| 8801 | findMatchingInlineAsmOperand(unsigned OperandNo, | ||||||||
| 8802 | const std::vector<SDValue> &AsmNodeOperands) { | ||||||||
| 8803 | // Scan until we find the definition we already emitted of this operand. | ||||||||
| 8804 | unsigned CurOp = InlineAsm::Op_FirstOperand; | ||||||||
| 8805 | for (; OperandNo; --OperandNo) { | ||||||||
| 8806 | // Advance to the next operand. | ||||||||
| 8807 | unsigned OpFlag = | ||||||||
| 8808 | cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); | ||||||||
| 8809 | assert((InlineAsm::isRegDefKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8810 | InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8811 | InlineAsm::isMemKind(OpFlag)) &&(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8812 | "Skipped past definitions?")(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind (OpFlag)) && "Skipped past definitions?") ? void (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 8813 | CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1; | ||||||||
| 8814 | } | ||||||||
| 8815 | return CurOp; | ||||||||
| 8816 | } | ||||||||
| 8817 | |||||||||
| 8818 | namespace { | ||||||||
| 8819 | |||||||||
| 8820 | class ExtraFlags { | ||||||||
| 8821 | unsigned Flags = 0; | ||||||||
| 8822 | |||||||||
| 8823 | public: | ||||||||
| 8824 | explicit ExtraFlags(const CallBase &Call) { | ||||||||
| 8825 | const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand()); | ||||||||
| 8826 | if (IA->hasSideEffects()) | ||||||||
| 8827 | Flags |= InlineAsm::Extra_HasSideEffects; | ||||||||
| 8828 | if (IA->isAlignStack()) | ||||||||
| 8829 | Flags |= InlineAsm::Extra_IsAlignStack; | ||||||||
| 8830 | if (Call.isConvergent()) | ||||||||
| 8831 | Flags |= InlineAsm::Extra_IsConvergent; | ||||||||
| 8832 | Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect; | ||||||||
| 8833 | } | ||||||||
| 8834 | |||||||||
| 8835 | void update(const TargetLowering::AsmOperandInfo &OpInfo) { | ||||||||
| 8836 | // Ideally, we would only check against memory constraints. However, the | ||||||||
| 8837 | // meaning of an Other constraint can be target-specific and we can't easily | ||||||||
| 8838 | // reason about it. Therefore, be conservative and set MayLoad/MayStore | ||||||||
| 8839 | // for Other constraints as well. | ||||||||
| 8840 | if (OpInfo.ConstraintType == TargetLowering::C_Memory || | ||||||||
| 8841 | OpInfo.ConstraintType == TargetLowering::C_Other) { | ||||||||
| 8842 | if (OpInfo.Type == InlineAsm::isInput) | ||||||||
| 8843 | Flags |= InlineAsm::Extra_MayLoad; | ||||||||
| 8844 | else if (OpInfo.Type == InlineAsm::isOutput) | ||||||||
| 8845 | Flags |= InlineAsm::Extra_MayStore; | ||||||||
| 8846 | else if (OpInfo.Type == InlineAsm::isClobber) | ||||||||
| 8847 | Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore); | ||||||||
| 8848 | } | ||||||||
| 8849 | } | ||||||||
| 8850 | |||||||||
| 8851 | unsigned get() const { return Flags; } | ||||||||
| 8852 | }; | ||||||||
| 8853 | |||||||||
| 8854 | } // end anonymous namespace | ||||||||
| 8855 | |||||||||
| 8856 | static bool isFunction(SDValue Op) { | ||||||||
| 8857 | if (Op && Op.getOpcode() == ISD::GlobalAddress) { | ||||||||
| 8858 | if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) { | ||||||||
| 8859 | auto Fn = dyn_cast_or_null<Function>(GA->getGlobal()); | ||||||||
| 8860 | |||||||||
| 8861 | // In normal "call dllimport func" instruction (non-inlineasm) it force | ||||||||
| 8862 | // indirect access by specifing call opcode. And usually specially print | ||||||||
| 8863 | // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can | ||||||||
| 8864 | // not do in this way now. (In fact, this is similar with "Data Access" | ||||||||
| 8865 | // action). So here we ignore dllimport function. | ||||||||
| 8866 | if (Fn && !Fn->hasDLLImportStorageClass()) | ||||||||
| 8867 | return true; | ||||||||
| 8868 | } | ||||||||
| 8869 | } | ||||||||
| 8870 | return false; | ||||||||
| 8871 | } | ||||||||
| 8872 | |||||||||
| 8873 | /// visitInlineAsm - Handle a call to an InlineAsm object. | ||||||||
| 8874 | void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call, | ||||||||
| 8875 | const BasicBlock *EHPadBB) { | ||||||||
| 8876 | const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand()); | ||||||||
| |||||||||
| 8877 | |||||||||
| 8878 | /// ConstraintOperands - Information about all of the constraints. | ||||||||
| 8879 | SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands; | ||||||||
| 8880 | |||||||||
| 8881 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 8882 | TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints( | ||||||||
| 8883 | DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call); | ||||||||
| 8884 | |||||||||
| 8885 | // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack, | ||||||||
| 8886 | // AsmDialect, MayLoad, MayStore). | ||||||||
| 8887 | bool HasSideEffect = IA->hasSideEffects(); | ||||||||
| 8888 | ExtraFlags ExtraInfo(Call); | ||||||||
| 8889 | |||||||||
| 8890 | for (auto &T : TargetConstraints) { | ||||||||
| 8891 | ConstraintOperands.push_back(SDISelAsmOperandInfo(T)); | ||||||||
| 8892 | SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); | ||||||||
| 8893 | |||||||||
| 8894 | if (OpInfo.CallOperandVal) | ||||||||
| 8895 | OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); | ||||||||
| 8896 | |||||||||
| 8897 | if (!HasSideEffect) | ||||||||
| 8898 | HasSideEffect = OpInfo.hasMemory(TLI); | ||||||||
| 8899 | |||||||||
| 8900 | // Determine if this InlineAsm MayLoad or MayStore based on the constraints. | ||||||||
| 8901 | // FIXME: Could we compute this on OpInfo rather than T? | ||||||||
| 8902 | |||||||||
| 8903 | // Compute the constraint code and ConstraintType to use. | ||||||||
| 8904 | TLI.ComputeConstraintToUse(T, SDValue()); | ||||||||
| 8905 | |||||||||
| 8906 | if (T.ConstraintType == TargetLowering::C_Immediate && | ||||||||
| 8907 | OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand)) | ||||||||
| 8908 | // We've delayed emitting a diagnostic like the "n" constraint because | ||||||||
| 8909 | // inlining could cause an integer showing up. | ||||||||
| 8910 | return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) + | ||||||||
| 8911 | "' expects an integer constant " | ||||||||
| 8912 | "expression"); | ||||||||
| 8913 | |||||||||
| 8914 | ExtraInfo.update(T); | ||||||||
| 8915 | } | ||||||||
| 8916 | |||||||||
| 8917 | // We won't need to flush pending loads if this asm doesn't touch | ||||||||
| 8918 | // memory and is nonvolatile. | ||||||||
| 8919 | SDValue Glue, Chain = (HasSideEffect
| ||||||||
| 8920 | |||||||||
| 8921 | bool EmitEHLabels = isa<InvokeInst>(Call); | ||||||||
| 8922 | if (EmitEHLabels
| ||||||||
| 8923 | assert(EHPadBB && "InvokeInst must have an EHPadBB")(static_cast <bool> (EHPadBB && "InvokeInst must have an EHPadBB" ) ? void (0) : __assert_fail ("EHPadBB && \"InvokeInst must have an EHPadBB\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8923 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 8924 | } | ||||||||
| 8925 | bool IsCallBr = isa<CallBrInst>(Call); | ||||||||
| 8926 | |||||||||
| 8927 | if (IsCallBr
| ||||||||
| 8928 | // If this is a callbr or invoke we need to flush pending exports since | ||||||||
| 8929 | // inlineasm_br and invoke are terminators. | ||||||||
| 8930 | // We need to do this before nodes are glued to the inlineasm_br node. | ||||||||
| 8931 | Chain = getControlRoot(); | ||||||||
| 8932 | } | ||||||||
| 8933 | |||||||||
| 8934 | MCSymbol *BeginLabel = nullptr; | ||||||||
| 8935 | if (EmitEHLabels
| ||||||||
| 8936 | Chain = lowerStartEH(Chain, EHPadBB, BeginLabel); | ||||||||
| 8937 | } | ||||||||
| 8938 | |||||||||
| 8939 | int OpNo = -1; | ||||||||
| 8940 | SmallVector<StringRef> AsmStrs; | ||||||||
| 8941 | IA->collectAsmStrs(AsmStrs); | ||||||||
| 8942 | |||||||||
| 8943 | // Second pass over the constraints: compute which constraint option to use. | ||||||||
| 8944 | for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { | ||||||||
| 8945 | if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput) | ||||||||
| 8946 | OpNo++; | ||||||||
| 8947 | |||||||||
| 8948 | // If this is an output operand with a matching input operand, look up the | ||||||||
| 8949 | // matching input. If their types mismatch, e.g. one is an integer, the | ||||||||
| 8950 | // other is floating point, or their sizes are different, flag it as an | ||||||||
| 8951 | // error. | ||||||||
| 8952 | if (OpInfo.hasMatchingInput()) { | ||||||||
| 8953 | SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput]; | ||||||||
| 8954 | patchMatchingInput(OpInfo, Input, DAG); | ||||||||
| 8955 | } | ||||||||
| 8956 | |||||||||
| 8957 | // Compute the constraint code and ConstraintType to use. | ||||||||
| 8958 | TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); | ||||||||
| 8959 | |||||||||
| 8960 | if ((OpInfo.ConstraintType == TargetLowering::C_Memory && | ||||||||
| 8961 | OpInfo.Type == InlineAsm::isClobber) || | ||||||||
| 8962 | OpInfo.ConstraintType == TargetLowering::C_Address) | ||||||||
| 8963 | continue; | ||||||||
| 8964 | |||||||||
| 8965 | // In Linux PIC model, there are 4 cases about value/label addressing: | ||||||||
| 8966 | // | ||||||||
| 8967 | // 1: Function call or Label jmp inside the module. | ||||||||
| 8968 | // 2: Data access (such as global variable, static variable) inside module. | ||||||||
| 8969 | // 3: Function call or Label jmp outside the module. | ||||||||
| 8970 | // 4: Data access (such as global variable) outside the module. | ||||||||
| 8971 | // | ||||||||
| 8972 | // Due to current llvm inline asm architecture designed to not "recognize" | ||||||||
| 8973 | // the asm code, there are quite troubles for us to treat mem addressing | ||||||||
| 8974 | // differently for same value/adress used in different instuctions. | ||||||||
| 8975 | // For example, in pic model, call a func may in plt way or direclty | ||||||||
| 8976 | // pc-related, but lea/mov a function adress may use got. | ||||||||
| 8977 | // | ||||||||
| 8978 | // Here we try to "recognize" function call for the case 1 and case 3 in | ||||||||
| 8979 | // inline asm. And try to adjust the constraint for them. | ||||||||
| 8980 | // | ||||||||
| 8981 | // TODO: Due to current inline asm didn't encourage to jmp to the outsider | ||||||||
| 8982 | // label, so here we don't handle jmp function label now, but we need to | ||||||||
| 8983 | // enhance it (especilly in PIC model) if we meet meaningful requirements. | ||||||||
| 8984 | if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) && | ||||||||
| 8985 | TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) && | ||||||||
| 8986 | TM.getCodeModel() != CodeModel::Large) { | ||||||||
| 8987 | OpInfo.isIndirect = false; | ||||||||
| 8988 | OpInfo.ConstraintType = TargetLowering::C_Address; | ||||||||
| 8989 | } | ||||||||
| 8990 | |||||||||
| 8991 | // If this is a memory input, and if the operand is not indirect, do what we | ||||||||
| 8992 | // need to provide an address for the memory input. | ||||||||
| 8993 | if (OpInfo.ConstraintType == TargetLowering::C_Memory && | ||||||||
| 8994 | !OpInfo.isIndirect) { | ||||||||
| 8995 | assert((OpInfo.isMultipleAlternative ||(static_cast <bool> ((OpInfo.isMultipleAlternative || ( OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!" ) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8997 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8996 | (OpInfo.Type == InlineAsm::isInput)) &&(static_cast <bool> ((OpInfo.isMultipleAlternative || ( OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!" ) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8997 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 8997 | "Can only indirectify direct input operands!")(static_cast <bool> ((OpInfo.isMultipleAlternative || ( OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!" ) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8997 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 8998 | |||||||||
| 8999 | // Memory operands really want the address of the value. | ||||||||
| 9000 | Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG); | ||||||||
| 9001 | |||||||||
| 9002 | // There is no longer a Value* corresponding to this operand. | ||||||||
| 9003 | OpInfo.CallOperandVal = nullptr; | ||||||||
| 9004 | |||||||||
| 9005 | // It is now an indirect operand. | ||||||||
| 9006 | OpInfo.isIndirect = true; | ||||||||
| 9007 | } | ||||||||
| 9008 | |||||||||
| 9009 | } | ||||||||
| 9010 | |||||||||
| 9011 | // AsmNodeOperands - The operands for the ISD::INLINEASM node. | ||||||||
| 9012 | std::vector<SDValue> AsmNodeOperands; | ||||||||
| 9013 | AsmNodeOperands.push_back(SDValue()); // reserve space for input chain | ||||||||
| 9014 | AsmNodeOperands.push_back(DAG.getTargetExternalSymbol( | ||||||||
| 9015 | IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout()))); | ||||||||
| 9016 | |||||||||
| 9017 | // If we have a !srcloc metadata node associated with it, we want to attach | ||||||||
| 9018 | // this to the ultimately generated inline asm machineinstr. To do this, we | ||||||||
| 9019 | // pass in the third operand as this (potentially null) inline asm MDNode. | ||||||||
| 9020 | const MDNode *SrcLoc = Call.getMetadata("srcloc"); | ||||||||
| 9021 | AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); | ||||||||
| 9022 | |||||||||
| 9023 | // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore | ||||||||
| 9024 | // bits as operand 3. | ||||||||
| 9025 | AsmNodeOperands.push_back(DAG.getTargetConstant( | ||||||||
| 9026 | ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 9027 | |||||||||
| 9028 | // Third pass: Loop over operands to prepare DAG-level operands.. As part of | ||||||||
| 9029 | // this, assign virtual and physical registers for inputs and otput. | ||||||||
| 9030 | for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { | ||||||||
| 9031 | // Assign Registers. | ||||||||
| 9032 | SDISelAsmOperandInfo &RefOpInfo = | ||||||||
| 9033 | OpInfo.isMatchingInputConstraint() | ||||||||
| 9034 | ? ConstraintOperands[OpInfo.getMatchedOperand()] | ||||||||
| 9035 | : OpInfo; | ||||||||
| 9036 | const auto RegError = | ||||||||
| 9037 | getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo); | ||||||||
| 9038 | if (RegError) { | ||||||||
| 9039 | const MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 9040 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | ||||||||
| 9041 | const char *RegName = TRI.getName(*RegError); | ||||||||
| 9042 | emitInlineAsmError(Call, "register '" + Twine(RegName) + | ||||||||
| 9043 | "' allocated for constraint '" + | ||||||||
| 9044 | Twine(OpInfo.ConstraintCode) + | ||||||||
| 9045 | "' does not match required type"); | ||||||||
| 9046 | return; | ||||||||
| 9047 | } | ||||||||
| 9048 | |||||||||
| 9049 | auto DetectWriteToReservedRegister = [&]() { | ||||||||
| 9050 | const MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 9051 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | ||||||||
| 9052 | for (unsigned Reg : OpInfo.AssignedRegs.Regs) { | ||||||||
| 9053 | if (Register::isPhysicalRegister(Reg) && | ||||||||
| 9054 | TRI.isInlineAsmReadOnlyReg(MF, Reg)) { | ||||||||
| 9055 | const char *RegName = TRI.getName(Reg); | ||||||||
| 9056 | emitInlineAsmError(Call, "write to reserved register '" + | ||||||||
| 9057 | Twine(RegName) + "'"); | ||||||||
| 9058 | return true; | ||||||||
| 9059 | } | ||||||||
| 9060 | } | ||||||||
| 9061 | return false; | ||||||||
| 9062 | }; | ||||||||
| 9063 | assert((OpInfo.ConstraintType != TargetLowering::C_Address ||(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering ::C_Address || (OpInfo.Type == InlineAsm::isInput && ! OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed." ) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9064 | (OpInfo.Type == InlineAsm::isInput &&(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering ::C_Address || (OpInfo.Type == InlineAsm::isInput && ! OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed." ) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9065 | !OpInfo.isMatchingInputConstraint())) &&(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering ::C_Address || (OpInfo.Type == InlineAsm::isInput && ! OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed." ) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9066 | "Only address as input operand is allowed.")(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering ::C_Address || (OpInfo.Type == InlineAsm::isInput && ! OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed." ) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9067 | |||||||||
| 9068 | switch (OpInfo.Type) { | ||||||||
| 9069 | case InlineAsm::isOutput: | ||||||||
| 9070 | if (OpInfo.ConstraintType == TargetLowering::C_Memory) { | ||||||||
| 9071 | unsigned ConstraintID = | ||||||||
| 9072 | TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); | ||||||||
| 9073 | assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9074 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9074 | "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9074 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9075 | |||||||||
| 9076 | // Add information to the INLINEASM node to know about this output. | ||||||||
| 9077 | unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); | ||||||||
| 9078 | OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID); | ||||||||
| 9079 | AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(), | ||||||||
| 9080 | MVT::i32)); | ||||||||
| 9081 | AsmNodeOperands.push_back(OpInfo.CallOperand); | ||||||||
| 9082 | } else { | ||||||||
| 9083 | // Otherwise, this outputs to a register (directly for C_Register / | ||||||||
| 9084 | // C_RegisterClass, and a target-defined fashion for | ||||||||
| 9085 | // C_Immediate/C_Other). Find a register that we can use. | ||||||||
| 9086 | if (OpInfo.AssignedRegs.Regs.empty()) { | ||||||||
| 9087 | emitInlineAsmError( | ||||||||
| 9088 | Call, "couldn't allocate output register for constraint '" + | ||||||||
| 9089 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||
| 9090 | return; | ||||||||
| 9091 | } | ||||||||
| 9092 | |||||||||
| 9093 | if (DetectWriteToReservedRegister()) | ||||||||
| 9094 | return; | ||||||||
| 9095 | |||||||||
| 9096 | // Add information to the INLINEASM node to know that this register is | ||||||||
| 9097 | // set. | ||||||||
| 9098 | OpInfo.AssignedRegs.AddInlineAsmOperands( | ||||||||
| 9099 | OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber | ||||||||
| 9100 | : InlineAsm::Kind_RegDef, | ||||||||
| 9101 | false, 0, getCurSDLoc(), DAG, AsmNodeOperands); | ||||||||
| 9102 | } | ||||||||
| 9103 | break; | ||||||||
| 9104 | |||||||||
| 9105 | case InlineAsm::isInput: | ||||||||
| 9106 | case InlineAsm::isLabel: { | ||||||||
| 9107 | SDValue InOperandVal = OpInfo.CallOperand; | ||||||||
| 9108 | |||||||||
| 9109 | if (OpInfo.isMatchingInputConstraint()) { | ||||||||
| 9110 | // If this is required to match an output register we have already set, | ||||||||
| 9111 | // just use its register. | ||||||||
| 9112 | auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(), | ||||||||
| 9113 | AsmNodeOperands); | ||||||||
| 9114 | unsigned OpFlag = | ||||||||
| 9115 | cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); | ||||||||
| 9116 | if (InlineAsm::isRegDefKind(OpFlag) || | ||||||||
| 9117 | InlineAsm::isRegDefEarlyClobberKind(OpFlag)) { | ||||||||
| 9118 | // Add (OpFlag&0xffff)>>3 registers to MatchedRegs. | ||||||||
| 9119 | if (OpInfo.isIndirect) { | ||||||||
| 9120 | // This happens on gcc/testsuite/gcc.dg/pr8788-1.c | ||||||||
| 9121 | emitInlineAsmError(Call, "inline asm not supported yet: " | ||||||||
| 9122 | "don't know how to handle tied " | ||||||||
| 9123 | "indirect register inputs"); | ||||||||
| 9124 | return; | ||||||||
| 9125 | } | ||||||||
| 9126 | |||||||||
| 9127 | SmallVector<unsigned, 4> Regs; | ||||||||
| 9128 | MachineFunction &MF = DAG.getMachineFunction(); | ||||||||
| 9129 | MachineRegisterInfo &MRI = MF.getRegInfo(); | ||||||||
| 9130 | const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); | ||||||||
| 9131 | auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]); | ||||||||
| 9132 | Register TiedReg = R->getReg(); | ||||||||
| 9133 | MVT RegVT = R->getSimpleValueType(0); | ||||||||
| 9134 | const TargetRegisterClass *RC = | ||||||||
| 9135 | TiedReg.isVirtual() ? MRI.getRegClass(TiedReg) | ||||||||
| 9136 | : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT) | ||||||||
| 9137 | : TRI.getMinimalPhysRegClass(TiedReg); | ||||||||
| 9138 | unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag); | ||||||||
| 9139 | for (unsigned i = 0; i != NumRegs; ++i) | ||||||||
| 9140 | Regs.push_back(MRI.createVirtualRegister(RC)); | ||||||||
| 9141 | |||||||||
| 9142 | RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType()); | ||||||||
| 9143 | |||||||||
| 9144 | SDLoc dl = getCurSDLoc(); | ||||||||
| 9145 | // Use the produced MatchedRegs object to | ||||||||
| 9146 | MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call); | ||||||||
| 9147 | MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, | ||||||||
| 9148 | true, OpInfo.getMatchedOperand(), dl, | ||||||||
| 9149 | DAG, AsmNodeOperands); | ||||||||
| 9150 | break; | ||||||||
| 9151 | } | ||||||||
| 9152 | |||||||||
| 9153 | assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!")(static_cast <bool> (InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!") ? void (0) : __assert_fail ( "InlineAsm::isMemKind(OpFlag) && \"Unknown matching constraint!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9153 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9154 | assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&(static_cast <bool> (InlineAsm::getNumOperandRegisters( OpFlag) == 1 && "Unexpected number of operands") ? void (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9155 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9155 | "Unexpected number of operands")(static_cast <bool> (InlineAsm::getNumOperandRegisters( OpFlag) == 1 && "Unexpected number of operands") ? void (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9155 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9156 | // Add information to the INLINEASM node to know about this input. | ||||||||
| 9157 | // See InlineAsm.h isUseOperandTiedToDef. | ||||||||
| 9158 | OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag); | ||||||||
| 9159 | OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, | ||||||||
| 9160 | OpInfo.getMatchedOperand()); | ||||||||
| 9161 | AsmNodeOperands.push_back(DAG.getTargetConstant( | ||||||||
| 9162 | OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 9163 | AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); | ||||||||
| 9164 | break; | ||||||||
| 9165 | } | ||||||||
| 9166 | |||||||||
| 9167 | // Treat indirect 'X' constraint as memory. | ||||||||
| 9168 | if (OpInfo.ConstraintType == TargetLowering::C_Other && | ||||||||
| 9169 | OpInfo.isIndirect) | ||||||||
| 9170 | OpInfo.ConstraintType = TargetLowering::C_Memory; | ||||||||
| 9171 | |||||||||
| 9172 | if (OpInfo.ConstraintType == TargetLowering::C_Immediate || | ||||||||
| 9173 | OpInfo.ConstraintType == TargetLowering::C_Other) { | ||||||||
| 9174 | std::vector<SDValue> Ops; | ||||||||
| 9175 | TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode, | ||||||||
| 9176 | Ops, DAG); | ||||||||
| 9177 | if (Ops.empty()) { | ||||||||
| 9178 | if (OpInfo.ConstraintType == TargetLowering::C_Immediate) | ||||||||
| 9179 | if (isa<ConstantSDNode>(InOperandVal)) { | ||||||||
| 9180 | emitInlineAsmError(Call, "value out of range for constraint '" + | ||||||||
| 9181 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||
| 9182 | return; | ||||||||
| 9183 | } | ||||||||
| 9184 | |||||||||
| 9185 | emitInlineAsmError(Call, | ||||||||
| 9186 | "invalid operand for inline asm constraint '" + | ||||||||
| 9187 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||
| 9188 | return; | ||||||||
| 9189 | } | ||||||||
| 9190 | |||||||||
| 9191 | // Add information to the INLINEASM node to know about this input. | ||||||||
| 9192 | unsigned ResOpType = | ||||||||
| 9193 | InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size()); | ||||||||
| 9194 | AsmNodeOperands.push_back(DAG.getTargetConstant( | ||||||||
| 9195 | ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); | ||||||||
| 9196 | llvm::append_range(AsmNodeOperands, Ops); | ||||||||
| 9197 | break; | ||||||||
| 9198 | } | ||||||||
| 9199 | |||||||||
| 9200 | if (OpInfo.ConstraintType == TargetLowering::C_Memory) { | ||||||||
| 9201 | assert((OpInfo.isIndirect ||(static_cast <bool> ((OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && "Operand must be indirect to be a mem!" ) ? void (0) : __assert_fail ("(OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && \"Operand must be indirect to be a mem!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9203 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9202 | OpInfo.ConstraintType != TargetLowering::C_Memory) &&(static_cast <bool> ((OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && "Operand must be indirect to be a mem!" ) ? void (0) : __assert_fail ("(OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && \"Operand must be indirect to be a mem!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9203 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9203 | "Operand must be indirect to be a mem!")(static_cast <bool> ((OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && "Operand must be indirect to be a mem!" ) ? void (0) : __assert_fail ("(OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && \"Operand must be indirect to be a mem!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9203 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9204 | assert(InOperandVal.getValueType() ==(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9206 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9205 | TLI.getPointerTy(DAG.getDataLayout()) &&(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9206 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9206 | "Memory operands expect pointer values")(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9206 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9207 | |||||||||
| 9208 | unsigned ConstraintID = | ||||||||
| 9209 | TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); | ||||||||
| 9210 | assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9211 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9211 | "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9211 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9212 | |||||||||
| 9213 | // Add information to the INLINEASM node to know about this input. | ||||||||
| 9214 | unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); | ||||||||
| 9215 | ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID); | ||||||||
| 9216 | AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, | ||||||||
| 9217 | getCurSDLoc(), | ||||||||
| 9218 | MVT::i32)); | ||||||||
| 9219 | AsmNodeOperands.push_back(InOperandVal); | ||||||||
| 9220 | break; | ||||||||
| 9221 | } | ||||||||
| 9222 | |||||||||
| 9223 | if (OpInfo.ConstraintType == TargetLowering::C_Address) { | ||||||||
| 9224 | assert(InOperandVal.getValueType() ==(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Address operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Address operands expect pointer values\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9226 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9225 | TLI.getPointerTy(DAG.getDataLayout()) &&(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Address operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Address operands expect pointer values\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9226 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9226 | "Address operands expect pointer values")(static_cast <bool> (InOperandVal.getValueType() == TLI .getPointerTy(DAG.getDataLayout()) && "Address operands expect pointer values" ) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Address operands expect pointer values\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9226 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9227 | |||||||||
| 9228 | unsigned ConstraintID = | ||||||||
| 9229 | TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); | ||||||||
| 9230 | assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9231 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9231 | "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id." ) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9231 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9232 | |||||||||
| 9233 | unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); | ||||||||
| 9234 | |||||||||
| 9235 | SDValue AsmOp = InOperandVal; | ||||||||
| 9236 | if (isFunction(InOperandVal)) { | ||||||||
| 9237 | auto *GA = cast<GlobalAddressSDNode>(InOperandVal); | ||||||||
| 9238 | ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Func, 1); | ||||||||
| 9239 | AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(), | ||||||||
| 9240 | InOperandVal.getValueType(), | ||||||||
| 9241 | GA->getOffset()); | ||||||||
| 9242 | } | ||||||||
| 9243 | |||||||||
| 9244 | // Add information to the INLINEASM node to know about this input. | ||||||||
| 9245 | ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID); | ||||||||
| 9246 | |||||||||
| 9247 | AsmNodeOperands.push_back( | ||||||||
| 9248 | DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32)); | ||||||||
| 9249 | |||||||||
| 9250 | AsmNodeOperands.push_back(AsmOp); | ||||||||
| 9251 | break; | ||||||||
| 9252 | } | ||||||||
| 9253 | |||||||||
| 9254 | assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering ::C_RegisterClass || OpInfo.ConstraintType == TargetLowering:: C_Register) && "Unknown constraint type!") ? void (0) : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9256 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9255 | OpInfo.ConstraintType == TargetLowering::C_Register) &&(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering ::C_RegisterClass || OpInfo.ConstraintType == TargetLowering:: C_Register) && "Unknown constraint type!") ? void (0) : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9256 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9256 | "Unknown constraint type!")(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering ::C_RegisterClass || OpInfo.ConstraintType == TargetLowering:: C_Register) && "Unknown constraint type!") ? void (0) : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9256 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9257 | |||||||||
| 9258 | // TODO: Support this. | ||||||||
| 9259 | if (OpInfo.isIndirect) { | ||||||||
| 9260 | emitInlineAsmError( | ||||||||
| 9261 | Call, "Don't know how to handle indirect register inputs yet " | ||||||||
| 9262 | "for constraint '" + | ||||||||
| 9263 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||
| 9264 | return; | ||||||||
| 9265 | } | ||||||||
| 9266 | |||||||||
| 9267 | // Copy the input into the appropriate registers. | ||||||||
| 9268 | if (OpInfo.AssignedRegs.Regs.empty()) { | ||||||||
| 9269 | emitInlineAsmError(Call, | ||||||||
| 9270 | "couldn't allocate input reg for constraint '" + | ||||||||
| 9271 | Twine(OpInfo.ConstraintCode) + "'"); | ||||||||
| 9272 | return; | ||||||||
| 9273 | } | ||||||||
| 9274 | |||||||||
| 9275 | if (DetectWriteToReservedRegister()) | ||||||||
| 9276 | return; | ||||||||
| 9277 | |||||||||
| 9278 | SDLoc dl = getCurSDLoc(); | ||||||||
| 9279 | |||||||||
| 9280 | OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, | ||||||||
| 9281 | &Call); | ||||||||
| 9282 | |||||||||
| 9283 | OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0, | ||||||||
| 9284 | dl, DAG, AsmNodeOperands); | ||||||||
| 9285 | break; | ||||||||
| 9286 | } | ||||||||
| 9287 | case InlineAsm::isClobber: | ||||||||
| 9288 | // Add the clobbered value to the operand list, so that the register | ||||||||
| 9289 | // allocator is aware that the physreg got clobbered. | ||||||||
| 9290 | if (!OpInfo.AssignedRegs.Regs.empty()) | ||||||||
| 9291 | OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber, | ||||||||
| 9292 | false, 0, getCurSDLoc(), DAG, | ||||||||
| 9293 | AsmNodeOperands); | ||||||||
| 9294 | break; | ||||||||
| 9295 | } | ||||||||
| 9296 | } | ||||||||
| 9297 | |||||||||
| 9298 | // Finish up input operands. Set the input chain and add the flag last. | ||||||||
| 9299 | AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; | ||||||||
| 9300 | if (Glue.getNode()) AsmNodeOperands.push_back(Glue); | ||||||||
| 9301 | |||||||||
| 9302 | unsigned ISDOpc = IsCallBr
| ||||||||
| 9303 | Chain = DAG.getNode(ISDOpc, getCurSDLoc(), | ||||||||
| 9304 | DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); | ||||||||
| 9305 | Glue = Chain.getValue(1); | ||||||||
| 9306 | |||||||||
| 9307 | // Do additional work to generate outputs. | ||||||||
| 9308 | |||||||||
| 9309 | SmallVector<EVT, 1> ResultVTs; | ||||||||
| 9310 | SmallVector<SDValue, 1> ResultValues; | ||||||||
| 9311 | SmallVector<SDValue, 8> OutChains; | ||||||||
| 9312 | |||||||||
| 9313 | llvm::Type *CallResultType = Call.getType(); | ||||||||
| 9314 | ArrayRef<Type *> ResultTypes; | ||||||||
| 9315 | if (StructType *StructResult
| ||||||||
| 9316 | ResultTypes = StructResult->elements(); | ||||||||
| 9317 | else if (!CallResultType->isVoidTy()) | ||||||||
| 9318 | ResultTypes = ArrayRef(CallResultType); | ||||||||
| 9319 | |||||||||
| 9320 | auto CurResultType = ResultTypes.begin(); | ||||||||
| 9321 | auto handleRegAssign = [&](SDValue V) { | ||||||||
| 9322 | assert(CurResultType != ResultTypes.end() && "Unexpected value")(static_cast <bool> (CurResultType != ResultTypes.end() && "Unexpected value") ? void (0) : __assert_fail ("CurResultType != ResultTypes.end() && \"Unexpected value\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9322 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9323 | assert((*CurResultType)->isSized() && "Unexpected unsized type")(static_cast <bool> ((*CurResultType)->isSized() && "Unexpected unsized type") ? void (0) : __assert_fail ("(*CurResultType)->isSized() && \"Unexpected unsized type\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9323 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9324 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType); | ||||||||
| 9325 | ++CurResultType; | ||||||||
| 9326 | // If the type of the inline asm call site return value is different but has | ||||||||
| 9327 | // same size as the type of the asm output bitcast it. One example of this | ||||||||
| 9328 | // is for vectors with different width / number of elements. This can | ||||||||
| 9329 | // happen for register classes that can contain multiple different value | ||||||||
| 9330 | // types. The preg or vreg allocated may not have the same VT as was | ||||||||
| 9331 | // expected. | ||||||||
| 9332 | // | ||||||||
| 9333 | // This can also happen for a return value that disagrees with the register | ||||||||
| 9334 | // class it is put in, eg. a double in a general-purpose register on a | ||||||||
| 9335 | // 32-bit machine. | ||||||||
| 9336 | if (ResultVT != V.getValueType() && | ||||||||
| 9337 | ResultVT.getSizeInBits() == V.getValueSizeInBits()) | ||||||||
| 9338 | V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V); | ||||||||
| 9339 | else if (ResultVT != V.getValueType() && ResultVT.isInteger() && | ||||||||
| 9340 | V.getValueType().isInteger()) { | ||||||||
| 9341 | // If a result value was tied to an input value, the computed result | ||||||||
| 9342 | // may have a wider width than the expected result. Extract the | ||||||||
| 9343 | // relevant portion. | ||||||||
| 9344 | V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V); | ||||||||
| 9345 | } | ||||||||
| 9346 | assert(ResultVT == V.getValueType() && "Asm result value mismatch!")(static_cast <bool> (ResultVT == V.getValueType() && "Asm result value mismatch!") ? void (0) : __assert_fail ("ResultVT == V.getValueType() && \"Asm result value mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9346 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9347 | ResultVTs.push_back(ResultVT); | ||||||||
| 9348 | ResultValues.push_back(V); | ||||||||
| 9349 | }; | ||||||||
| 9350 | |||||||||
| 9351 | // Deal with output operands. | ||||||||
| 9352 | for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { | ||||||||
| 9353 | if (OpInfo.Type == InlineAsm::isOutput) { | ||||||||
| 9354 | SDValue Val; | ||||||||
| 9355 | // Skip trivial output operands. | ||||||||
| 9356 | if (OpInfo.AssignedRegs.Regs.empty()) | ||||||||
| 9357 | continue; | ||||||||
| 9358 | |||||||||
| 9359 | switch (OpInfo.ConstraintType) { | ||||||||
| 9360 | case TargetLowering::C_Register: | ||||||||
| 9361 | case TargetLowering::C_RegisterClass: | ||||||||
| 9362 | Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), | ||||||||
| 9363 | Chain, &Glue, &Call); | ||||||||
| 9364 | break; | ||||||||
| 9365 | case TargetLowering::C_Immediate: | ||||||||
| 9366 | case TargetLowering::C_Other: | ||||||||
| 9367 | Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(), | ||||||||
| 9368 | OpInfo, DAG); | ||||||||
| 9369 | break; | ||||||||
| 9370 | case TargetLowering::C_Memory: | ||||||||
| 9371 | break; // Already handled. | ||||||||
| 9372 | case TargetLowering::C_Address: | ||||||||
| 9373 | break; // Silence warning. | ||||||||
| 9374 | case TargetLowering::C_Unknown: | ||||||||
| 9375 | assert(false && "Unexpected unknown constraint")(static_cast <bool> (false && "Unexpected unknown constraint" ) ? void (0) : __assert_fail ("false && \"Unexpected unknown constraint\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9375 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9376 | } | ||||||||
| 9377 | |||||||||
| 9378 | // Indirect output manifest as stores. Record output chains. | ||||||||
| 9379 | if (OpInfo.isIndirect) { | ||||||||
| 9380 | const Value *Ptr = OpInfo.CallOperandVal; | ||||||||
| 9381 | assert(Ptr && "Expected value CallOperandVal for indirect asm operand")(static_cast <bool> (Ptr && "Expected value CallOperandVal for indirect asm operand" ) ? void (0) : __assert_fail ("Ptr && \"Expected value CallOperandVal for indirect asm operand\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9381 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9382 | SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr), | ||||||||
| 9383 | MachinePointerInfo(Ptr)); | ||||||||
| 9384 | OutChains.push_back(Store); | ||||||||
| 9385 | } else { | ||||||||
| 9386 | // generate CopyFromRegs to associated registers. | ||||||||
| 9387 | assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() && "Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9387 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9388 | if (Val.getOpcode() == ISD::MERGE_VALUES) { | ||||||||
| 9389 | for (const SDValue &V : Val->op_values()) | ||||||||
| 9390 | handleRegAssign(V); | ||||||||
| 9391 | } else | ||||||||
| 9392 | handleRegAssign(Val); | ||||||||
| 9393 | } | ||||||||
| 9394 | } | ||||||||
| 9395 | } | ||||||||
| 9396 | |||||||||
| 9397 | // Set results. | ||||||||
| 9398 | if (!ResultValues.empty()) { | ||||||||
| 9399 | assert(CurResultType == ResultTypes.end() &&(static_cast <bool> (CurResultType == ResultTypes.end() && "Mismatch in number of ResultTypes") ? void (0) : __assert_fail ("CurResultType == ResultTypes.end() && \"Mismatch in number of ResultTypes\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9400 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9400 | "Mismatch in number of ResultTypes")(static_cast <bool> (CurResultType == ResultTypes.end() && "Mismatch in number of ResultTypes") ? void (0) : __assert_fail ("CurResultType == ResultTypes.end() && \"Mismatch in number of ResultTypes\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9400 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9401 | assert(ResultValues.size() == ResultTypes.size() &&(static_cast <bool> (ResultValues.size() == ResultTypes .size() && "Mismatch in number of output operands in asm result" ) ? void (0) : __assert_fail ("ResultValues.size() == ResultTypes.size() && \"Mismatch in number of output operands in asm result\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9402 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9402 | "Mismatch in number of output operands in asm result")(static_cast <bool> (ResultValues.size() == ResultTypes .size() && "Mismatch in number of output operands in asm result" ) ? void (0) : __assert_fail ("ResultValues.size() == ResultTypes.size() && \"Mismatch in number of output operands in asm result\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9402 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9403 | |||||||||
| 9404 | SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||
| 9405 | DAG.getVTList(ResultVTs), ResultValues); | ||||||||
| 9406 | setValue(&Call, V); | ||||||||
| 9407 | } | ||||||||
| 9408 | |||||||||
| 9409 | // Collect store chains. | ||||||||
| 9410 | if (!OutChains.empty()) | ||||||||
| 9411 | Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains); | ||||||||
| 9412 | |||||||||
| 9413 | if (EmitEHLabels) { | ||||||||
| 9414 | Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel); | ||||||||
| 9415 | } | ||||||||
| 9416 | |||||||||
| 9417 | // Only Update Root if inline assembly has a memory effect. | ||||||||
| 9418 | if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr || | ||||||||
| 9419 | EmitEHLabels) | ||||||||
| 9420 | DAG.setRoot(Chain); | ||||||||
| 9421 | } | ||||||||
| 9422 | |||||||||
| 9423 | void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call, | ||||||||
| 9424 | const Twine &Message) { | ||||||||
| 9425 | LLVMContext &Ctx = *DAG.getContext(); | ||||||||
| 9426 | Ctx.emitError(&Call, Message); | ||||||||
| 9427 | |||||||||
| 9428 | // Make sure we leave the DAG in a valid state | ||||||||
| 9429 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 9430 | SmallVector<EVT, 1> ValueVTs; | ||||||||
| 9431 | ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs); | ||||||||
| 9432 | |||||||||
| 9433 | if (ValueVTs.empty()) | ||||||||
| 9434 | return; | ||||||||
| 9435 | |||||||||
| 9436 | SmallVector<SDValue, 1> Ops; | ||||||||
| 9437 | for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i) | ||||||||
| 9438 | Ops.push_back(DAG.getUNDEF(ValueVTs[i])); | ||||||||
| 9439 | |||||||||
| 9440 | setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc())); | ||||||||
| 9441 | } | ||||||||
| 9442 | |||||||||
| 9443 | void SelectionDAGBuilder::visitVAStart(const CallInst &I) { | ||||||||
| 9444 | DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), | ||||||||
| 9445 | MVT::Other, getRoot(), | ||||||||
| 9446 | getValue(I.getArgOperand(0)), | ||||||||
| 9447 | DAG.getSrcValue(I.getArgOperand(0)))); | ||||||||
| 9448 | } | ||||||||
| 9449 | |||||||||
| 9450 | void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { | ||||||||
| 9451 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 9452 | const DataLayout &DL = DAG.getDataLayout(); | ||||||||
| 9453 | SDValue V = DAG.getVAArg( | ||||||||
| 9454 | TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(), | ||||||||
| 9455 | getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)), | ||||||||
| 9456 | DL.getABITypeAlign(I.getType()).value()); | ||||||||
| 9457 | DAG.setRoot(V.getValue(1)); | ||||||||
| 9458 | |||||||||
| 9459 | if (I.getType()->isPointerTy()) | ||||||||
| 9460 | V = DAG.getPtrExtOrTrunc( | ||||||||
| 9461 | V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType())); | ||||||||
| 9462 | setValue(&I, V); | ||||||||
| 9463 | } | ||||||||
| 9464 | |||||||||
| 9465 | void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { | ||||||||
| 9466 | DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), | ||||||||
| 9467 | MVT::Other, getRoot(), | ||||||||
| 9468 | getValue(I.getArgOperand(0)), | ||||||||
| 9469 | DAG.getSrcValue(I.getArgOperand(0)))); | ||||||||
| 9470 | } | ||||||||
| 9471 | |||||||||
| 9472 | void SelectionDAGBuilder::visitVACopy(const CallInst &I) { | ||||||||
| 9473 | DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), | ||||||||
| 9474 | MVT::Other, getRoot(), | ||||||||
| 9475 | getValue(I.getArgOperand(0)), | ||||||||
| 9476 | getValue(I.getArgOperand(1)), | ||||||||
| 9477 | DAG.getSrcValue(I.getArgOperand(0)), | ||||||||
| 9478 | DAG.getSrcValue(I.getArgOperand(1)))); | ||||||||
| 9479 | } | ||||||||
| 9480 | |||||||||
| 9481 | SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, | ||||||||
| 9482 | const Instruction &I, | ||||||||
| 9483 | SDValue Op) { | ||||||||
| 9484 | const MDNode *Range = I.getMetadata(LLVMContext::MD_range); | ||||||||
| 9485 | if (!Range) | ||||||||
| 9486 | return Op; | ||||||||
| 9487 | |||||||||
| 9488 | ConstantRange CR = getConstantRangeFromMetadata(*Range); | ||||||||
| 9489 | if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped()) | ||||||||
| 9490 | return Op; | ||||||||
| 9491 | |||||||||
| 9492 | APInt Lo = CR.getUnsignedMin(); | ||||||||
| 9493 | if (!Lo.isMinValue()) | ||||||||
| 9494 | return Op; | ||||||||
| 9495 | |||||||||
| 9496 | APInt Hi = CR.getUnsignedMax(); | ||||||||
| 9497 | unsigned Bits = std::max(Hi.getActiveBits(), | ||||||||
| 9498 | static_cast<unsigned>(IntegerType::MIN_INT_BITS)); | ||||||||
| 9499 | |||||||||
| 9500 | EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits); | ||||||||
| 9501 | |||||||||
| 9502 | SDLoc SL = getCurSDLoc(); | ||||||||
| 9503 | |||||||||
| 9504 | SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op, | ||||||||
| 9505 | DAG.getValueType(SmallVT)); | ||||||||
| 9506 | unsigned NumVals = Op.getNode()->getNumValues(); | ||||||||
| 9507 | if (NumVals == 1) | ||||||||
| 9508 | return ZExt; | ||||||||
| 9509 | |||||||||
| 9510 | SmallVector<SDValue, 4> Ops; | ||||||||
| 9511 | |||||||||
| 9512 | Ops.push_back(ZExt); | ||||||||
| 9513 | for (unsigned I = 1; I != NumVals; ++I) | ||||||||
| 9514 | Ops.push_back(Op.getValue(I)); | ||||||||
| 9515 | |||||||||
| 9516 | return DAG.getMergeValues(Ops, SL); | ||||||||
| 9517 | } | ||||||||
| 9518 | |||||||||
| 9519 | /// Populate a CallLowerinInfo (into \p CLI) based on the properties of | ||||||||
| 9520 | /// the call being lowered. | ||||||||
| 9521 | /// | ||||||||
| 9522 | /// This is a helper for lowering intrinsics that follow a target calling | ||||||||
| 9523 | /// convention or require stack pointer adjustment. Only a subset of the | ||||||||
| 9524 | /// intrinsic's operands need to participate in the calling convention. | ||||||||
| 9525 | void SelectionDAGBuilder::populateCallLoweringInfo( | ||||||||
| 9526 | TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, | ||||||||
| 9527 | unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, | ||||||||
| 9528 | bool IsPatchPoint) { | ||||||||
| 9529 | TargetLowering::ArgListTy Args; | ||||||||
| 9530 | Args.reserve(NumArgs); | ||||||||
| 9531 | |||||||||
| 9532 | // Populate the argument list. | ||||||||
| 9533 | // Attributes for args start at offset 1, after the return attribute. | ||||||||
| 9534 | for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; | ||||||||
| 9535 | ArgI != ArgE; ++ArgI) { | ||||||||
| 9536 | const Value *V = Call->getOperand(ArgI); | ||||||||
| 9537 | |||||||||
| 9538 | assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.")(static_cast <bool> (!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.") ? void (0) : __assert_fail ("!V->getType()->isEmptyTy() && \"Empty type passed to intrinsic.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9538 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9539 | |||||||||
| 9540 | TargetLowering::ArgListEntry Entry; | ||||||||
| 9541 | Entry.Node = getValue(V); | ||||||||
| 9542 | Entry.Ty = V->getType(); | ||||||||
| 9543 | Entry.setAttributes(Call, ArgI); | ||||||||
| 9544 | Args.push_back(Entry); | ||||||||
| 9545 | } | ||||||||
| 9546 | |||||||||
| 9547 | CLI.setDebugLoc(getCurSDLoc()) | ||||||||
| 9548 | .setChain(getRoot()) | ||||||||
| 9549 | .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args)) | ||||||||
| 9550 | .setDiscardResult(Call->use_empty()) | ||||||||
| 9551 | .setIsPatchPoint(IsPatchPoint) | ||||||||
| 9552 | .setIsPreallocated( | ||||||||
| 9553 | Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0); | ||||||||
| 9554 | } | ||||||||
| 9555 | |||||||||
| 9556 | /// Add a stack map intrinsic call's live variable operands to a stackmap | ||||||||
| 9557 | /// or patchpoint target node's operand list. | ||||||||
| 9558 | /// | ||||||||
| 9559 | /// Constants are converted to TargetConstants purely as an optimization to | ||||||||
| 9560 | /// avoid constant materialization and register allocation. | ||||||||
| 9561 | /// | ||||||||
| 9562 | /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not | ||||||||
| 9563 | /// generate addess computation nodes, and so FinalizeISel can convert the | ||||||||
| 9564 | /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids | ||||||||
| 9565 | /// address materialization and register allocation, but may also be required | ||||||||
| 9566 | /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an | ||||||||
| 9567 | /// alloca in the entry block, then the runtime may assume that the alloca's | ||||||||
| 9568 | /// StackMap location can be read immediately after compilation and that the | ||||||||
| 9569 | /// location is valid at any point during execution (this is similar to the | ||||||||
| 9570 | /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were | ||||||||
| 9571 | /// only available in a register, then the runtime would need to trap when | ||||||||
| 9572 | /// execution reaches the StackMap in order to read the alloca's location. | ||||||||
| 9573 | static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, | ||||||||
| 9574 | const SDLoc &DL, SmallVectorImpl<SDValue> &Ops, | ||||||||
| 9575 | SelectionDAGBuilder &Builder) { | ||||||||
| 9576 | SelectionDAG &DAG = Builder.DAG; | ||||||||
| 9577 | for (unsigned I = StartIdx; I < Call.arg_size(); I++) { | ||||||||
| 9578 | SDValue Op = Builder.getValue(Call.getArgOperand(I)); | ||||||||
| 9579 | |||||||||
| 9580 | // Things on the stack are pointer-typed, meaning that they are already | ||||||||
| 9581 | // legal and can be emitted directly to target nodes. | ||||||||
| 9582 | if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) { | ||||||||
| 9583 | Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType())); | ||||||||
| 9584 | } else { | ||||||||
| 9585 | // Otherwise emit a target independent node to be legalised. | ||||||||
| 9586 | Ops.push_back(Builder.getValue(Call.getArgOperand(I))); | ||||||||
| 9587 | } | ||||||||
| 9588 | } | ||||||||
| 9589 | } | ||||||||
| 9590 | |||||||||
| 9591 | /// Lower llvm.experimental.stackmap. | ||||||||
| 9592 | void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { | ||||||||
| 9593 | // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>, | ||||||||
| 9594 | // [live variables...]) | ||||||||
| 9595 | |||||||||
| 9596 | assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.")(static_cast <bool> (CI.getType()->isVoidTy() && "Stackmap cannot return a value.") ? void (0) : __assert_fail ("CI.getType()->isVoidTy() && \"Stackmap cannot return a value.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9596 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9597 | |||||||||
| 9598 | SDValue Chain, InGlue, Callee; | ||||||||
| 9599 | SmallVector<SDValue, 32> Ops; | ||||||||
| 9600 | |||||||||
| 9601 | SDLoc DL = getCurSDLoc(); | ||||||||
| 9602 | Callee = getValue(CI.getCalledOperand()); | ||||||||
| 9603 | |||||||||
| 9604 | // The stackmap intrinsic only records the live variables (the arguments | ||||||||
| 9605 | // passed to it) and emits NOPS (if requested). Unlike the patchpoint | ||||||||
| 9606 | // intrinsic, this won't be lowered to a function call. This means we don't | ||||||||
| 9607 | // have to worry about calling conventions and target specific lowering code. | ||||||||
| 9608 | // Instead we perform the call lowering right here. | ||||||||
| 9609 | // | ||||||||
| 9610 | // chain, flag = CALLSEQ_START(chain, 0, 0) | ||||||||
| 9611 | // chain, flag = STACKMAP(id, nbytes, ..., chain, flag) | ||||||||
| 9612 | // chain, flag = CALLSEQ_END(chain, 0, 0, flag) | ||||||||
| 9613 | // | ||||||||
| 9614 | Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL); | ||||||||
| 9615 | InGlue = Chain.getValue(1); | ||||||||
| 9616 | |||||||||
| 9617 | // Add the STACKMAP operands, starting with DAG house-keeping. | ||||||||
| 9618 | Ops.push_back(Chain); | ||||||||
| 9619 | Ops.push_back(InGlue); | ||||||||
| 9620 | |||||||||
| 9621 | // Add the <id>, <numShadowBytes> operands. | ||||||||
| 9622 | // | ||||||||
| 9623 | // These do not require legalisation, and can be emitted directly to target | ||||||||
| 9624 | // constant nodes. | ||||||||
| 9625 | SDValue ID = getValue(CI.getArgOperand(0)); | ||||||||
| 9626 | assert(ID.getValueType() == MVT::i64)(static_cast <bool> (ID.getValueType() == MVT::i64) ? void (0) : __assert_fail ("ID.getValueType() == MVT::i64", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 9626, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9627 | SDValue IDConst = DAG.getTargetConstant( | ||||||||
| 9628 | cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType()); | ||||||||
| 9629 | Ops.push_back(IDConst); | ||||||||
| 9630 | |||||||||
| 9631 | SDValue Shad = getValue(CI.getArgOperand(1)); | ||||||||
| 9632 | assert(Shad.getValueType() == MVT::i32)(static_cast <bool> (Shad.getValueType() == MVT::i32) ? void (0) : __assert_fail ("Shad.getValueType() == MVT::i32", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9632 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9633 | SDValue ShadConst = DAG.getTargetConstant( | ||||||||
| 9634 | cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType()); | ||||||||
| 9635 | Ops.push_back(ShadConst); | ||||||||
| 9636 | |||||||||
| 9637 | // Add the live variables. | ||||||||
| 9638 | addStackMapLiveVars(CI, 2, DL, Ops, *this); | ||||||||
| 9639 | |||||||||
| 9640 | // Create the STACKMAP node. | ||||||||
| 9641 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||
| 9642 | Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops); | ||||||||
| 9643 | InGlue = Chain.getValue(1); | ||||||||
| 9644 | |||||||||
| 9645 | Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL); | ||||||||
| 9646 | |||||||||
| 9647 | // Stackmaps don't generate values, so nothing goes into the NodeMap. | ||||||||
| 9648 | |||||||||
| 9649 | // Set the root to the target-lowered call chain. | ||||||||
| 9650 | DAG.setRoot(Chain); | ||||||||
| 9651 | |||||||||
| 9652 | // Inform the Frame Information that we have a stackmap in this function. | ||||||||
| 9653 | FuncInfo.MF->getFrameInfo().setHasStackMap(); | ||||||||
| 9654 | } | ||||||||
| 9655 | |||||||||
| 9656 | /// Lower llvm.experimental.patchpoint directly to its target opcode. | ||||||||
| 9657 | void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, | ||||||||
| 9658 | const BasicBlock *EHPadBB) { | ||||||||
| 9659 | // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>, | ||||||||
| 9660 | // i32 <numBytes>, | ||||||||
| 9661 | // i8* <target>, | ||||||||
| 9662 | // i32 <numArgs>, | ||||||||
| 9663 | // [Args...], | ||||||||
| 9664 | // [live variables...]) | ||||||||
| 9665 | |||||||||
| 9666 | CallingConv::ID CC = CB.getCallingConv(); | ||||||||
| 9667 | bool IsAnyRegCC = CC == CallingConv::AnyReg; | ||||||||
| 9668 | bool HasDef = !CB.getType()->isVoidTy(); | ||||||||
| 9669 | SDLoc dl = getCurSDLoc(); | ||||||||
| 9670 | SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos)); | ||||||||
| 9671 | |||||||||
| 9672 | // Handle immediate and symbolic callees. | ||||||||
| 9673 | if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee)) | ||||||||
| 9674 | Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl, | ||||||||
| 9675 | /*isTarget=*/true); | ||||||||
| 9676 | else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee)) | ||||||||
| 9677 | Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(), | ||||||||
| 9678 | SDLoc(SymbolicCallee), | ||||||||
| 9679 | SymbolicCallee->getValueType(0)); | ||||||||
| 9680 | |||||||||
| 9681 | // Get the real number of arguments participating in the call <numArgs> | ||||||||
| 9682 | SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos)); | ||||||||
| 9683 | unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue(); | ||||||||
| 9684 | |||||||||
| 9685 | // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> | ||||||||
| 9686 | // Intrinsics include all meta-operands up to but not including CC. | ||||||||
| 9687 | unsigned NumMetaOpers = PatchPointOpers::CCPos; | ||||||||
| 9688 | assert(CB.arg_size() >= NumMetaOpers + NumArgs &&(static_cast <bool> (CB.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic" ) ? void (0) : __assert_fail ("CB.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9689 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9689 | "Not enough arguments provided to the patchpoint intrinsic")(static_cast <bool> (CB.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic" ) ? void (0) : __assert_fail ("CB.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9689 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9690 | |||||||||
| 9691 | // For AnyRegCC the arguments are lowered later on manually. | ||||||||
| 9692 | unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs; | ||||||||
| 9693 | Type *ReturnTy = | ||||||||
| 9694 | IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType(); | ||||||||
| 9695 | |||||||||
| 9696 | TargetLowering::CallLoweringInfo CLI(DAG); | ||||||||
| 9697 | populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee, | ||||||||
| 9698 | ReturnTy, true); | ||||||||
| 9699 | std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB); | ||||||||
| 9700 | |||||||||
| 9701 | SDNode *CallEnd = Result.second.getNode(); | ||||||||
| 9702 | if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) | ||||||||
| 9703 | CallEnd = CallEnd->getOperand(0).getNode(); | ||||||||
| 9704 | |||||||||
| 9705 | /// Get a call instruction from the call sequence chain. | ||||||||
| 9706 | /// Tail calls are not allowed. | ||||||||
| 9707 | assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node.") ? void (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9708 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 9708 | "Expected a callseq node.")(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node.") ? void (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9708 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9709 | SDNode *Call = CallEnd->getOperand(0).getNode(); | ||||||||
| 9710 | bool HasGlue = Call->getGluedNode(); | ||||||||
| 9711 | |||||||||
| 9712 | // Replace the target specific call node with the patchable intrinsic. | ||||||||
| 9713 | SmallVector<SDValue, 8> Ops; | ||||||||
| 9714 | |||||||||
| 9715 | // Push the chain. | ||||||||
| 9716 | Ops.push_back(*(Call->op_begin())); | ||||||||
| 9717 | |||||||||
| 9718 | // Optionally, push the glue (if any). | ||||||||
| 9719 | if (HasGlue) | ||||||||
| 9720 | Ops.push_back(*(Call->op_end() - 1)); | ||||||||
| 9721 | |||||||||
| 9722 | // Push the register mask info. | ||||||||
| 9723 | if (HasGlue) | ||||||||
| 9724 | Ops.push_back(*(Call->op_end() - 2)); | ||||||||
| 9725 | else | ||||||||
| 9726 | Ops.push_back(*(Call->op_end() - 1)); | ||||||||
| 9727 | |||||||||
| 9728 | // Add the <id> and <numBytes> constants. | ||||||||
| 9729 | SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos)); | ||||||||
| 9730 | Ops.push_back(DAG.getTargetConstant( | ||||||||
| 9731 | cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64)); | ||||||||
| 9732 | SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos)); | ||||||||
| 9733 | Ops.push_back(DAG.getTargetConstant( | ||||||||
| 9734 | cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl, | ||||||||
| 9735 | MVT::i32)); | ||||||||
| 9736 | |||||||||
| 9737 | // Add the callee. | ||||||||
| 9738 | Ops.push_back(Callee); | ||||||||
| 9739 | |||||||||
| 9740 | // Adjust <numArgs> to account for any arguments that have been passed on the | ||||||||
| 9741 | // stack instead. | ||||||||
| 9742 | // Call Node: Chain, Target, {Args}, RegMask, [Glue] | ||||||||
| 9743 | unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3); | ||||||||
| 9744 | NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs; | ||||||||
| 9745 | Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32)); | ||||||||
| 9746 | |||||||||
| 9747 | // Add the calling convention | ||||||||
| 9748 | Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32)); | ||||||||
| 9749 | |||||||||
| 9750 | // Add the arguments we omitted previously. The register allocator should | ||||||||
| 9751 | // place these in any free register. | ||||||||
| 9752 | if (IsAnyRegCC) | ||||||||
| 9753 | for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) | ||||||||
| 9754 | Ops.push_back(getValue(CB.getArgOperand(i))); | ||||||||
| 9755 | |||||||||
| 9756 | // Push the arguments from the call instruction. | ||||||||
| 9757 | SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1; | ||||||||
| 9758 | Ops.append(Call->op_begin() + 2, e); | ||||||||
| 9759 | |||||||||
| 9760 | // Push live variables for the stack map. | ||||||||
| 9761 | addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this); | ||||||||
| 9762 | |||||||||
| 9763 | SDVTList NodeTys; | ||||||||
| 9764 | if (IsAnyRegCC && HasDef) { | ||||||||
| 9765 | // Create the return types based on the intrinsic definition | ||||||||
| 9766 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 9767 | SmallVector<EVT, 3> ValueVTs; | ||||||||
| 9768 | ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs); | ||||||||
| 9769 | assert(ValueVTs.size() == 1 && "Expected only one return value type.")(static_cast <bool> (ValueVTs.size() == 1 && "Expected only one return value type." ) ? void (0) : __assert_fail ("ValueVTs.size() == 1 && \"Expected only one return value type.\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9769 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 9770 | |||||||||
| 9771 | // There is always a chain and a glue type at the end | ||||||||
| 9772 | ValueVTs.push_back(MVT::Other); | ||||||||
| 9773 | ValueVTs.push_back(MVT::Glue); | ||||||||
| 9774 | NodeTys = DAG.getVTList(ValueVTs); | ||||||||
| 9775 | } else | ||||||||
| 9776 | NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | ||||||||
| 9777 | |||||||||
| 9778 | // Replace the target specific call node with a PATCHPOINT node. | ||||||||
| 9779 | SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops); | ||||||||
| 9780 | |||||||||
| 9781 | // Update the NodeMap. | ||||||||
| 9782 | if (HasDef) { | ||||||||
| 9783 | if (IsAnyRegCC) | ||||||||
| 9784 | setValue(&CB, SDValue(PPV.getNode(), 0)); | ||||||||
| 9785 | else | ||||||||
| 9786 | setValue(&CB, Result.first); | ||||||||
| 9787 | } | ||||||||
| 9788 | |||||||||
| 9789 | // Fixup the consumers of the intrinsic. The chain and glue may be used in the | ||||||||
| 9790 | // call sequence. Furthermore the location of the chain and glue can change | ||||||||
| 9791 | // when the AnyReg calling convention is used and the intrinsic returns a | ||||||||
| 9792 | // value. | ||||||||
| 9793 | if (IsAnyRegCC && HasDef) { | ||||||||
| 9794 | SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)}; | ||||||||
| 9795 | SDValue To[] = {PPV.getValue(1), PPV.getValue(2)}; | ||||||||
| 9796 | DAG.ReplaceAllUsesOfValuesWith(From, To, 2); | ||||||||
| 9797 | } else | ||||||||
| 9798 | DAG.ReplaceAllUsesWith(Call, PPV.getNode()); | ||||||||
| 9799 | DAG.DeleteNode(Call); | ||||||||
| 9800 | |||||||||
| 9801 | // Inform the Frame Information that we have a patchpoint in this function. | ||||||||
| 9802 | FuncInfo.MF->getFrameInfo().setHasPatchPoint(); | ||||||||
| 9803 | } | ||||||||
| 9804 | |||||||||
| 9805 | void SelectionDAGBuilder::visitVectorReduce(const CallInst &I, | ||||||||
| 9806 | unsigned Intrinsic) { | ||||||||
| 9807 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 9808 | SDValue Op1 = getValue(I.getArgOperand(0)); | ||||||||
| 9809 | SDValue Op2; | ||||||||
| 9810 | if (I.arg_size() > 1) | ||||||||
| 9811 | Op2 = getValue(I.getArgOperand(1)); | ||||||||
| 9812 | SDLoc dl = getCurSDLoc(); | ||||||||
| 9813 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 9814 | SDValue Res; | ||||||||
| 9815 | SDNodeFlags SDFlags; | ||||||||
| 9816 | if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) | ||||||||
| 9817 | SDFlags.copyFMF(*FPMO); | ||||||||
| 9818 | |||||||||
| 9819 | switch (Intrinsic) { | ||||||||
| 9820 | case Intrinsic::vector_reduce_fadd: | ||||||||
| 9821 | if (SDFlags.hasAllowReassociation()) | ||||||||
| 9822 | Res = DAG.getNode(ISD::FADD, dl, VT, Op1, | ||||||||
| 9823 | DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags), | ||||||||
| 9824 | SDFlags); | ||||||||
| 9825 | else | ||||||||
| 9826 | Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags); | ||||||||
| 9827 | break; | ||||||||
| 9828 | case Intrinsic::vector_reduce_fmul: | ||||||||
| 9829 | if (SDFlags.hasAllowReassociation()) | ||||||||
| 9830 | Res = DAG.getNode(ISD::FMUL, dl, VT, Op1, | ||||||||
| 9831 | DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags), | ||||||||
| 9832 | SDFlags); | ||||||||
| 9833 | else | ||||||||
| 9834 | Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags); | ||||||||
| 9835 | break; | ||||||||
| 9836 | case Intrinsic::vector_reduce_add: | ||||||||
| 9837 | Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1); | ||||||||
| 9838 | break; | ||||||||
| 9839 | case Intrinsic::vector_reduce_mul: | ||||||||
| 9840 | Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1); | ||||||||
| 9841 | break; | ||||||||
| 9842 | case Intrinsic::vector_reduce_and: | ||||||||
| 9843 | Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1); | ||||||||
| 9844 | break; | ||||||||
| 9845 | case Intrinsic::vector_reduce_or: | ||||||||
| 9846 | Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1); | ||||||||
| 9847 | break; | ||||||||
| 9848 | case Intrinsic::vector_reduce_xor: | ||||||||
| 9849 | Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1); | ||||||||
| 9850 | break; | ||||||||
| 9851 | case Intrinsic::vector_reduce_smax: | ||||||||
| 9852 | Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1); | ||||||||
| 9853 | break; | ||||||||
| 9854 | case Intrinsic::vector_reduce_smin: | ||||||||
| 9855 | Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1); | ||||||||
| 9856 | break; | ||||||||
| 9857 | case Intrinsic::vector_reduce_umax: | ||||||||
| 9858 | Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1); | ||||||||
| 9859 | break; | ||||||||
| 9860 | case Intrinsic::vector_reduce_umin: | ||||||||
| 9861 | Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1); | ||||||||
| 9862 | break; | ||||||||
| 9863 | case Intrinsic::vector_reduce_fmax: | ||||||||
| 9864 | Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags); | ||||||||
| 9865 | break; | ||||||||
| 9866 | case Intrinsic::vector_reduce_fmin: | ||||||||
| 9867 | Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags); | ||||||||
| 9868 | break; | ||||||||
| 9869 | default: | ||||||||
| 9870 | llvm_unreachable("Unhandled vector reduce intrinsic")::llvm::llvm_unreachable_internal("Unhandled vector reduce intrinsic" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9870 ); | ||||||||
| 9871 | } | ||||||||
| 9872 | setValue(&I, Res); | ||||||||
| 9873 | } | ||||||||
| 9874 | |||||||||
| 9875 | /// Returns an AttributeList representing the attributes applied to the return | ||||||||
| 9876 | /// value of the given call. | ||||||||
| 9877 | static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) { | ||||||||
| 9878 | SmallVector<Attribute::AttrKind, 2> Attrs; | ||||||||
| 9879 | if (CLI.RetSExt) | ||||||||
| 9880 | Attrs.push_back(Attribute::SExt); | ||||||||
| 9881 | if (CLI.RetZExt) | ||||||||
| 9882 | Attrs.push_back(Attribute::ZExt); | ||||||||
| 9883 | if (CLI.IsInReg) | ||||||||
| 9884 | Attrs.push_back(Attribute::InReg); | ||||||||
| 9885 | |||||||||
| 9886 | return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex, | ||||||||
| 9887 | Attrs); | ||||||||
| 9888 | } | ||||||||
| 9889 | |||||||||
| 9890 | /// TargetLowering::LowerCallTo - This is the default LowerCallTo | ||||||||
| 9891 | /// implementation, which just calls LowerCall. | ||||||||
| 9892 | /// FIXME: When all targets are | ||||||||
| 9893 | /// migrated to using LowerCall, this hook should be integrated into SDISel. | ||||||||
| 9894 | std::pair<SDValue, SDValue> | ||||||||
| 9895 | TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { | ||||||||
| 9896 | // Handle the incoming return values from the call. | ||||||||
| 9897 | CLI.Ins.clear(); | ||||||||
| 9898 | Type *OrigRetTy = CLI.RetTy; | ||||||||
| 9899 | SmallVector<EVT, 4> RetTys; | ||||||||
| 9900 | SmallVector<uint64_t, 4> Offsets; | ||||||||
| 9901 | auto &DL = CLI.DAG.getDataLayout(); | ||||||||
| 9902 | ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); | ||||||||
| 9903 | |||||||||
| 9904 | if (CLI.IsPostTypeLegalization) { | ||||||||
| 9905 | // If we are lowering a libcall after legalization, split the return type. | ||||||||
| 9906 | SmallVector<EVT, 4> OldRetTys; | ||||||||
| 9907 | SmallVector<uint64_t, 4> OldOffsets; | ||||||||
| 9908 | RetTys.swap(OldRetTys); | ||||||||
| 9909 | Offsets.swap(OldOffsets); | ||||||||
| 9910 | |||||||||
| 9911 | for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) { | ||||||||
| 9912 | EVT RetVT = OldRetTys[i]; | ||||||||
| 9913 | uint64_t Offset = OldOffsets[i]; | ||||||||
| 9914 | MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT); | ||||||||
| 9915 | unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT); | ||||||||
| 9916 | unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8; | ||||||||
| 9917 | RetTys.append(NumRegs, RegisterVT); | ||||||||
| 9918 | for (unsigned j = 0; j != NumRegs; ++j) | ||||||||
| 9919 | Offsets.push_back(Offset + j * RegisterVTByteSZ); | ||||||||
| 9920 | } | ||||||||
| 9921 | } | ||||||||
| 9922 | |||||||||
| 9923 | SmallVector<ISD::OutputArg, 4> Outs; | ||||||||
| 9924 | GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL); | ||||||||
| 9925 | |||||||||
| 9926 | bool CanLowerReturn = | ||||||||
| 9927 | this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(), | ||||||||
| 9928 | CLI.IsVarArg, Outs, CLI.RetTy->getContext()); | ||||||||
| 9929 | |||||||||
| 9930 | SDValue DemoteStackSlot; | ||||||||
| 9931 | int DemoteStackIdx = -100; | ||||||||
| 9932 | if (!CanLowerReturn) { | ||||||||
| 9933 | // FIXME: equivalent assert? | ||||||||
| 9934 | // assert(!CS.hasInAllocaArgument() && | ||||||||
| 9935 | // "sret demotion is incompatible with inalloca"); | ||||||||
| 9936 | uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy); | ||||||||
| 9937 | Align Alignment = DL.getPrefTypeAlign(CLI.RetTy); | ||||||||
| 9938 | MachineFunction &MF = CLI.DAG.getMachineFunction(); | ||||||||
| 9939 | DemoteStackIdx = | ||||||||
| 9940 | MF.getFrameInfo().CreateStackObject(TySize, Alignment, false); | ||||||||
| 9941 | Type *StackSlotPtrType = PointerType::get(CLI.RetTy, | ||||||||
| 9942 | DL.getAllocaAddrSpace()); | ||||||||
| 9943 | |||||||||
| 9944 | DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL)); | ||||||||
| 9945 | ArgListEntry Entry; | ||||||||
| 9946 | Entry.Node = DemoteStackSlot; | ||||||||
| 9947 | Entry.Ty = StackSlotPtrType; | ||||||||
| 9948 | Entry.IsSExt = false; | ||||||||
| 9949 | Entry.IsZExt = false; | ||||||||
| 9950 | Entry.IsInReg = false; | ||||||||
| 9951 | Entry.IsSRet = true; | ||||||||
| 9952 | Entry.IsNest = false; | ||||||||
| 9953 | Entry.IsByVal = false; | ||||||||
| 9954 | Entry.IsByRef = false; | ||||||||
| 9955 | Entry.IsReturned = false; | ||||||||
| 9956 | Entry.IsSwiftSelf = false; | ||||||||
| 9957 | Entry.IsSwiftAsync = false; | ||||||||
| 9958 | Entry.IsSwiftError = false; | ||||||||
| 9959 | Entry.IsCFGuardTarget = false; | ||||||||
| 9960 | Entry.Alignment = Alignment; | ||||||||
| 9961 | CLI.getArgs().insert(CLI.getArgs().begin(), Entry); | ||||||||
| 9962 | CLI.NumFixedArgs += 1; | ||||||||
| 9963 | CLI.getArgs()[0].IndirectType = CLI.RetTy; | ||||||||
| 9964 | CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext()); | ||||||||
| 9965 | |||||||||
| 9966 | // sret demotion isn't compatible with tail-calls, since the sret argument | ||||||||
| 9967 | // points into the callers stack frame. | ||||||||
| 9968 | CLI.IsTailCall = false; | ||||||||
| 9969 | } else { | ||||||||
| 9970 | bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( | ||||||||
| 9971 | CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL); | ||||||||
| 9972 | for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { | ||||||||
| 9973 | ISD::ArgFlagsTy Flags; | ||||||||
| 9974 | if (NeedsRegBlock) { | ||||||||
| 9975 | Flags.setInConsecutiveRegs(); | ||||||||
| 9976 | if (I == RetTys.size() - 1) | ||||||||
| 9977 | Flags.setInConsecutiveRegsLast(); | ||||||||
| 9978 | } | ||||||||
| 9979 | EVT VT = RetTys[I]; | ||||||||
| 9980 | MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), | ||||||||
| 9981 | CLI.CallConv, VT); | ||||||||
| 9982 | unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(), | ||||||||
| 9983 | CLI.CallConv, VT); | ||||||||
| 9984 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||
| 9985 | ISD::InputArg MyFlags; | ||||||||
| 9986 | MyFlags.Flags = Flags; | ||||||||
| 9987 | MyFlags.VT = RegisterVT; | ||||||||
| 9988 | MyFlags.ArgVT = VT; | ||||||||
| 9989 | MyFlags.Used = CLI.IsReturnValueUsed; | ||||||||
| 9990 | if (CLI.RetTy->isPointerTy()) { | ||||||||
| 9991 | MyFlags.Flags.setPointer(); | ||||||||
| 9992 | MyFlags.Flags.setPointerAddrSpace( | ||||||||
| 9993 | cast<PointerType>(CLI.RetTy)->getAddressSpace()); | ||||||||
| 9994 | } | ||||||||
| 9995 | if (CLI.RetSExt) | ||||||||
| 9996 | MyFlags.Flags.setSExt(); | ||||||||
| 9997 | if (CLI.RetZExt) | ||||||||
| 9998 | MyFlags.Flags.setZExt(); | ||||||||
| 9999 | if (CLI.IsInReg) | ||||||||
| 10000 | MyFlags.Flags.setInReg(); | ||||||||
| 10001 | CLI.Ins.push_back(MyFlags); | ||||||||
| 10002 | } | ||||||||
| 10003 | } | ||||||||
| 10004 | } | ||||||||
| 10005 | |||||||||
| 10006 | // We push in swifterror return as the last element of CLI.Ins. | ||||||||
| 10007 | ArgListTy &Args = CLI.getArgs(); | ||||||||
| 10008 | if (supportSwiftError()) { | ||||||||
| 10009 | for (const ArgListEntry &Arg : Args) { | ||||||||
| 10010 | if (Arg.IsSwiftError) { | ||||||||
| 10011 | ISD::InputArg MyFlags; | ||||||||
| 10012 | MyFlags.VT = getPointerTy(DL); | ||||||||
| 10013 | MyFlags.ArgVT = EVT(getPointerTy(DL)); | ||||||||
| 10014 | MyFlags.Flags.setSwiftError(); | ||||||||
| 10015 | CLI.Ins.push_back(MyFlags); | ||||||||
| 10016 | } | ||||||||
| 10017 | } | ||||||||
| 10018 | } | ||||||||
| 10019 | |||||||||
| 10020 | // Handle all of the outgoing arguments. | ||||||||
| 10021 | CLI.Outs.clear(); | ||||||||
| 10022 | CLI.OutVals.clear(); | ||||||||
| 10023 | for (unsigned i = 0, e = Args.size(); i != e; ++i) { | ||||||||
| 10024 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 10025 | ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs); | ||||||||
| 10026 | // FIXME: Split arguments if CLI.IsPostTypeLegalization | ||||||||
| 10027 | Type *FinalType = Args[i].Ty; | ||||||||
| 10028 | if (Args[i].IsByVal) | ||||||||
| 10029 | FinalType = Args[i].IndirectType; | ||||||||
| 10030 | bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( | ||||||||
| 10031 | FinalType, CLI.CallConv, CLI.IsVarArg, DL); | ||||||||
| 10032 | for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; | ||||||||
| 10033 | ++Value) { | ||||||||
| 10034 | EVT VT = ValueVTs[Value]; | ||||||||
| 10035 | Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext()); | ||||||||
| 10036 | SDValue Op = SDValue(Args[i].Node.getNode(), | ||||||||
| 10037 | Args[i].Node.getResNo() + Value); | ||||||||
| 10038 | ISD::ArgFlagsTy Flags; | ||||||||
| 10039 | |||||||||
| 10040 | // Certain targets (such as MIPS), may have a different ABI alignment | ||||||||
| 10041 | // for a type depending on the context. Give the target a chance to | ||||||||
| 10042 | // specify the alignment it wants. | ||||||||
| 10043 | const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL)); | ||||||||
| 10044 | Flags.setOrigAlign(OriginalAlignment); | ||||||||
| 10045 | |||||||||
| 10046 | if (Args[i].Ty->isPointerTy()) { | ||||||||
| 10047 | Flags.setPointer(); | ||||||||
| 10048 | Flags.setPointerAddrSpace( | ||||||||
| 10049 | cast<PointerType>(Args[i].Ty)->getAddressSpace()); | ||||||||
| 10050 | } | ||||||||
| 10051 | if (Args[i].IsZExt) | ||||||||
| 10052 | Flags.setZExt(); | ||||||||
| 10053 | if (Args[i].IsSExt) | ||||||||
| 10054 | Flags.setSExt(); | ||||||||
| 10055 | if (Args[i].IsInReg) { | ||||||||
| 10056 | // If we are using vectorcall calling convention, a structure that is | ||||||||
| 10057 | // passed InReg - is surely an HVA | ||||||||
| 10058 | if (CLI.CallConv == CallingConv::X86_VectorCall && | ||||||||
| 10059 | isa<StructType>(FinalType)) { | ||||||||
| 10060 | // The first value of a structure is marked | ||||||||
| 10061 | if (0 == Value) | ||||||||
| 10062 | Flags.setHvaStart(); | ||||||||
| 10063 | Flags.setHva(); | ||||||||
| 10064 | } | ||||||||
| 10065 | // Set InReg Flag | ||||||||
| 10066 | Flags.setInReg(); | ||||||||
| 10067 | } | ||||||||
| 10068 | if (Args[i].IsSRet) | ||||||||
| 10069 | Flags.setSRet(); | ||||||||
| 10070 | if (Args[i].IsSwiftSelf) | ||||||||
| 10071 | Flags.setSwiftSelf(); | ||||||||
| 10072 | if (Args[i].IsSwiftAsync) | ||||||||
| 10073 | Flags.setSwiftAsync(); | ||||||||
| 10074 | if (Args[i].IsSwiftError) | ||||||||
| 10075 | Flags.setSwiftError(); | ||||||||
| 10076 | if (Args[i].IsCFGuardTarget) | ||||||||
| 10077 | Flags.setCFGuardTarget(); | ||||||||
| 10078 | if (Args[i].IsByVal) | ||||||||
| 10079 | Flags.setByVal(); | ||||||||
| 10080 | if (Args[i].IsByRef) | ||||||||
| 10081 | Flags.setByRef(); | ||||||||
| 10082 | if (Args[i].IsPreallocated) { | ||||||||
| 10083 | Flags.setPreallocated(); | ||||||||
| 10084 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||
| 10085 | // preallocated. This way we can know how many bytes we should've | ||||||||
| 10086 | // allocated and how many bytes a callee cleanup function will pop. If | ||||||||
| 10087 | // we port preallocated to more targets, we'll have to add custom | ||||||||
| 10088 | // preallocated handling in the various CC lowering callbacks. | ||||||||
| 10089 | Flags.setByVal(); | ||||||||
| 10090 | } | ||||||||
| 10091 | if (Args[i].IsInAlloca) { | ||||||||
| 10092 | Flags.setInAlloca(); | ||||||||
| 10093 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||
| 10094 | // inalloca. This way we can know how many bytes we should've allocated | ||||||||
| 10095 | // and how many bytes a callee cleanup function will pop. If we port | ||||||||
| 10096 | // inalloca to more targets, we'll have to add custom inalloca handling | ||||||||
| 10097 | // in the various CC lowering callbacks. | ||||||||
| 10098 | Flags.setByVal(); | ||||||||
| 10099 | } | ||||||||
| 10100 | Align MemAlign; | ||||||||
| 10101 | if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) { | ||||||||
| 10102 | unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType); | ||||||||
| 10103 | Flags.setByValSize(FrameSize); | ||||||||
| 10104 | |||||||||
| 10105 | // info is not there but there are cases it cannot get right. | ||||||||
| 10106 | if (auto MA = Args[i].Alignment) | ||||||||
| 10107 | MemAlign = *MA; | ||||||||
| 10108 | else | ||||||||
| 10109 | MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL)); | ||||||||
| 10110 | } else if (auto MA = Args[i].Alignment) { | ||||||||
| 10111 | MemAlign = *MA; | ||||||||
| 10112 | } else { | ||||||||
| 10113 | MemAlign = OriginalAlignment; | ||||||||
| 10114 | } | ||||||||
| 10115 | Flags.setMemAlign(MemAlign); | ||||||||
| 10116 | if (Args[i].IsNest) | ||||||||
| 10117 | Flags.setNest(); | ||||||||
| 10118 | if (NeedsRegBlock) | ||||||||
| 10119 | Flags.setInConsecutiveRegs(); | ||||||||
| 10120 | |||||||||
| 10121 | MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), | ||||||||
| 10122 | CLI.CallConv, VT); | ||||||||
| 10123 | unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(), | ||||||||
| 10124 | CLI.CallConv, VT); | ||||||||
| 10125 | SmallVector<SDValue, 4> Parts(NumParts); | ||||||||
| 10126 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND; | ||||||||
| 10127 | |||||||||
| 10128 | if (Args[i].IsSExt) | ||||||||
| 10129 | ExtendKind = ISD::SIGN_EXTEND; | ||||||||
| 10130 | else if (Args[i].IsZExt) | ||||||||
| 10131 | ExtendKind = ISD::ZERO_EXTEND; | ||||||||
| 10132 | |||||||||
| 10133 | // Conservatively only handle 'returned' on non-vectors that can be lowered, | ||||||||
| 10134 | // for now. | ||||||||
| 10135 | if (Args[i].IsReturned && !Op.getValueType().isVector() && | ||||||||
| 10136 | CanLowerReturn) { | ||||||||
| 10137 | assert((CLI.RetTy == Args[i].Ty ||(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10138 | (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10139 | CLI.RetTy->getPointerAddressSpace() ==(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10140 | Args[i].Ty->getPointerAddressSpace())) &&(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10141 | RetTys.size() == NumValues && "unexpected use of 'returned'")(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy ->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace ())) && RetTys.size() == NumValues && "unexpected use of 'returned'" ) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10142 | // Before passing 'returned' to the target lowering code, ensure that | ||||||||
| 10143 | // either the register MVT and the actual EVT are the same size or that | ||||||||
| 10144 | // the return value and argument are extended in the same way; in these | ||||||||
| 10145 | // cases it's safe to pass the argument register value unchanged as the | ||||||||
| 10146 | // return register value (although it's at the target's option whether | ||||||||
| 10147 | // to do so) | ||||||||
| 10148 | // TODO: allow code generation to take advantage of partially preserved | ||||||||
| 10149 | // registers rather than clobbering the entire register when the | ||||||||
| 10150 | // parameter extension method is not compatible with the return | ||||||||
| 10151 | // extension method | ||||||||
| 10152 | if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) || | ||||||||
| 10153 | (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt && | ||||||||
| 10154 | CLI.RetZExt == Args[i].IsZExt)) | ||||||||
| 10155 | Flags.setReturned(); | ||||||||
| 10156 | } | ||||||||
| 10157 | |||||||||
| 10158 | getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB, | ||||||||
| 10159 | CLI.CallConv, ExtendKind); | ||||||||
| 10160 | |||||||||
| 10161 | for (unsigned j = 0; j != NumParts; ++j) { | ||||||||
| 10162 | // if it isn't first piece, alignment must be 1 | ||||||||
| 10163 | // For scalable vectors the scalable part is currently handled | ||||||||
| 10164 | // by individual targets, so we just use the known minimum size here. | ||||||||
| 10165 | ISD::OutputArg MyFlags( | ||||||||
| 10166 | Flags, Parts[j].getValueType().getSimpleVT(), VT, | ||||||||
| 10167 | i < CLI.NumFixedArgs, i, | ||||||||
| 10168 | j * Parts[j].getValueType().getStoreSize().getKnownMinValue()); | ||||||||
| 10169 | if (NumParts > 1 && j == 0) | ||||||||
| 10170 | MyFlags.Flags.setSplit(); | ||||||||
| 10171 | else if (j != 0) { | ||||||||
| 10172 | MyFlags.Flags.setOrigAlign(Align(1)); | ||||||||
| 10173 | if (j == NumParts - 1) | ||||||||
| 10174 | MyFlags.Flags.setSplitEnd(); | ||||||||
| 10175 | } | ||||||||
| 10176 | |||||||||
| 10177 | CLI.Outs.push_back(MyFlags); | ||||||||
| 10178 | CLI.OutVals.push_back(Parts[j]); | ||||||||
| 10179 | } | ||||||||
| 10180 | |||||||||
| 10181 | if (NeedsRegBlock && Value == NumValues - 1) | ||||||||
| 10182 | CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast(); | ||||||||
| 10183 | } | ||||||||
| 10184 | } | ||||||||
| 10185 | |||||||||
| 10186 | SmallVector<SDValue, 4> InVals; | ||||||||
| 10187 | CLI.Chain = LowerCall(CLI, InVals); | ||||||||
| 10188 | |||||||||
| 10189 | // Update CLI.InVals to use outside of this function. | ||||||||
| 10190 | CLI.InVals = InVals; | ||||||||
| 10191 | |||||||||
| 10192 | // Verify that the target's LowerCall behaved as expected. | ||||||||
| 10193 | assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&(static_cast <bool> (CLI.Chain.getNode() && CLI .Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!" ) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10194 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10194 | "LowerCall didn't return a valid chain!")(static_cast <bool> (CLI.Chain.getNode() && CLI .Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!" ) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10194 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10195 | assert((!CLI.IsTailCall || InVals.empty()) &&(static_cast <bool> ((!CLI.IsTailCall || InVals.empty() ) && "LowerCall emitted a return value for a tail call!" ) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10196 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10196 | "LowerCall emitted a return value for a tail call!")(static_cast <bool> ((!CLI.IsTailCall || InVals.empty() ) && "LowerCall emitted a return value for a tail call!" ) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10196 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10197 | assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&(static_cast <bool> ((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10198 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10198 | "LowerCall didn't emit the correct number of values!")(static_cast <bool> ((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10198 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10199 | |||||||||
| 10200 | // For a tail call, the return value is merely live-out and there aren't | ||||||||
| 10201 | // any nodes in the DAG representing it. Return a special value to | ||||||||
| 10202 | // indicate that a tail call has been emitted and no more Instructions | ||||||||
| 10203 | // should be processed in the current block. | ||||||||
| 10204 | if (CLI.IsTailCall) { | ||||||||
| 10205 | CLI.DAG.setRoot(CLI.Chain); | ||||||||
| 10206 | return std::make_pair(SDValue(), SDValue()); | ||||||||
| 10207 | } | ||||||||
| 10208 | |||||||||
| 10209 | #ifndef NDEBUG | ||||||||
| 10210 | for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) { | ||||||||
| 10211 | assert(InVals[i].getNode() && "LowerCall emitted a null value!")(static_cast <bool> (InVals[i].getNode() && "LowerCall emitted a null value!" ) ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerCall emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10211 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10212 | assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType () && "LowerCall emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10213 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10213 | "LowerCall emitted a value with the wrong type!")(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType () && "LowerCall emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10213 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10214 | } | ||||||||
| 10215 | #endif | ||||||||
| 10216 | |||||||||
| 10217 | SmallVector<SDValue, 4> ReturnValues; | ||||||||
| 10218 | if (!CanLowerReturn) { | ||||||||
| 10219 | // The instruction result is the result of loading from the | ||||||||
| 10220 | // hidden sret parameter. | ||||||||
| 10221 | SmallVector<EVT, 1> PVTs; | ||||||||
| 10222 | Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace()); | ||||||||
| 10223 | |||||||||
| 10224 | ComputeValueVTs(*this, DL, PtrRetTy, PVTs); | ||||||||
| 10225 | assert(PVTs.size() == 1 && "Pointers should fit in one register")(static_cast <bool> (PVTs.size() == 1 && "Pointers should fit in one register" ) ? void (0) : __assert_fail ("PVTs.size() == 1 && \"Pointers should fit in one register\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10225 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10226 | EVT PtrVT = PVTs[0]; | ||||||||
| 10227 | |||||||||
| 10228 | unsigned NumValues = RetTys.size(); | ||||||||
| 10229 | ReturnValues.resize(NumValues); | ||||||||
| 10230 | SmallVector<SDValue, 4> Chains(NumValues); | ||||||||
| 10231 | |||||||||
| 10232 | // An aggregate return value cannot wrap around the address space, so | ||||||||
| 10233 | // offsets to its parts don't wrap either. | ||||||||
| 10234 | SDNodeFlags Flags; | ||||||||
| 10235 | Flags.setNoUnsignedWrap(true); | ||||||||
| 10236 | |||||||||
| 10237 | MachineFunction &MF = CLI.DAG.getMachineFunction(); | ||||||||
| 10238 | Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx); | ||||||||
| 10239 | for (unsigned i = 0; i < NumValues; ++i) { | ||||||||
| 10240 | SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot, | ||||||||
| 10241 | CLI.DAG.getConstant(Offsets[i], CLI.DL, | ||||||||
| 10242 | PtrVT), Flags); | ||||||||
| 10243 | SDValue L = CLI.DAG.getLoad( | ||||||||
| 10244 | RetTys[i], CLI.DL, CLI.Chain, Add, | ||||||||
| 10245 | MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(), | ||||||||
| 10246 | DemoteStackIdx, Offsets[i]), | ||||||||
| 10247 | HiddenSRetAlign); | ||||||||
| 10248 | ReturnValues[i] = L; | ||||||||
| 10249 | Chains[i] = L.getValue(1); | ||||||||
| 10250 | } | ||||||||
| 10251 | |||||||||
| 10252 | CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains); | ||||||||
| 10253 | } else { | ||||||||
| 10254 | // Collect the legal value parts into potentially illegal values | ||||||||
| 10255 | // that correspond to the original function's return values. | ||||||||
| 10256 | std::optional<ISD::NodeType> AssertOp; | ||||||||
| 10257 | if (CLI.RetSExt) | ||||||||
| 10258 | AssertOp = ISD::AssertSext; | ||||||||
| 10259 | else if (CLI.RetZExt) | ||||||||
| 10260 | AssertOp = ISD::AssertZext; | ||||||||
| 10261 | unsigned CurReg = 0; | ||||||||
| 10262 | for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { | ||||||||
| 10263 | EVT VT = RetTys[I]; | ||||||||
| 10264 | MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), | ||||||||
| 10265 | CLI.CallConv, VT); | ||||||||
| 10266 | unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(), | ||||||||
| 10267 | CLI.CallConv, VT); | ||||||||
| 10268 | |||||||||
| 10269 | ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg], | ||||||||
| 10270 | NumRegs, RegisterVT, VT, nullptr, | ||||||||
| 10271 | CLI.CallConv, AssertOp)); | ||||||||
| 10272 | CurReg += NumRegs; | ||||||||
| 10273 | } | ||||||||
| 10274 | |||||||||
| 10275 | // For a function returning void, there is no return value. We can't create | ||||||||
| 10276 | // such a node, so we just return a null return value in that case. In | ||||||||
| 10277 | // that case, nothing will actually look at the value. | ||||||||
| 10278 | if (ReturnValues.empty()) | ||||||||
| 10279 | return std::make_pair(SDValue(), CLI.Chain); | ||||||||
| 10280 | } | ||||||||
| 10281 | |||||||||
| 10282 | SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL, | ||||||||
| 10283 | CLI.DAG.getVTList(RetTys), ReturnValues); | ||||||||
| 10284 | return std::make_pair(Res, CLI.Chain); | ||||||||
| 10285 | } | ||||||||
| 10286 | |||||||||
| 10287 | /// Places new result values for the node in Results (their number | ||||||||
| 10288 | /// and types must exactly match those of the original return values of | ||||||||
| 10289 | /// the node), or leaves Results empty, which indicates that the node is not | ||||||||
| 10290 | /// to be custom lowered after all. | ||||||||
| 10291 | void TargetLowering::LowerOperationWrapper(SDNode *N, | ||||||||
| 10292 | SmallVectorImpl<SDValue> &Results, | ||||||||
| 10293 | SelectionDAG &DAG) const { | ||||||||
| 10294 | SDValue Res = LowerOperation(SDValue(N, 0), DAG); | ||||||||
| 10295 | |||||||||
| 10296 | if (!Res.getNode()) | ||||||||
| 10297 | return; | ||||||||
| 10298 | |||||||||
| 10299 | // If the original node has one result, take the return value from | ||||||||
| 10300 | // LowerOperation as is. It might not be result number 0. | ||||||||
| 10301 | if (N->getNumValues() == 1) { | ||||||||
| 10302 | Results.push_back(Res); | ||||||||
| 10303 | return; | ||||||||
| 10304 | } | ||||||||
| 10305 | |||||||||
| 10306 | // If the original node has multiple results, then the return node should | ||||||||
| 10307 | // have the same number of results. | ||||||||
| 10308 | assert((N->getNumValues() == Res->getNumValues()) &&(static_cast <bool> ((N->getNumValues() == Res->getNumValues ()) && "Lowering returned the wrong number of results!" ) ? void (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10309 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10309 | "Lowering returned the wrong number of results!")(static_cast <bool> ((N->getNumValues() == Res->getNumValues ()) && "Lowering returned the wrong number of results!" ) ? void (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10309 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10310 | |||||||||
| 10311 | // Places new result values base on N result number. | ||||||||
| 10312 | for (unsigned I = 0, E = N->getNumValues(); I != E; ++I) | ||||||||
| 10313 | Results.push_back(Res.getValue(I)); | ||||||||
| 10314 | } | ||||||||
| 10315 | |||||||||
| 10316 | SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { | ||||||||
| 10317 | llvm_unreachable("LowerOperation not implemented for this target!")::llvm::llvm_unreachable_internal("LowerOperation not implemented for this target!" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10317 ); | ||||||||
| 10318 | } | ||||||||
| 10319 | |||||||||
| 10320 | void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, | ||||||||
| 10321 | unsigned Reg, | ||||||||
| 10322 | ISD::NodeType ExtendType) { | ||||||||
| 10323 | SDValue Op = getNonRegisterValue(V); | ||||||||
| 10324 | assert((Op.getOpcode() != ISD::CopyFromReg ||(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!") ? void (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10326 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10325 | cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!") ? void (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10326 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10326 | "Copy from a reg to the same reg!")(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!") ? void (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10326 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10327 | assert(!Register::isPhysicalRegister(Reg) && "Is a physreg")(static_cast <bool> (!Register::isPhysicalRegister(Reg) && "Is a physreg") ? void (0) : __assert_fail ("!Register::isPhysicalRegister(Reg) && \"Is a physreg\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10327 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10328 | |||||||||
| 10329 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 10330 | // If this is an InlineAsm we have to match the registers required, not the | ||||||||
| 10331 | // notional registers required by the type. | ||||||||
| 10332 | |||||||||
| 10333 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(), | ||||||||
| 10334 | std::nullopt); // This is not an ABI copy. | ||||||||
| 10335 | SDValue Chain = DAG.getEntryNode(); | ||||||||
| 10336 | |||||||||
| 10337 | if (ExtendType == ISD::ANY_EXTEND) { | ||||||||
| 10338 | auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V); | ||||||||
| 10339 | if (PreferredExtendIt != FuncInfo.PreferredExtendType.end()) | ||||||||
| 10340 | ExtendType = PreferredExtendIt->second; | ||||||||
| 10341 | } | ||||||||
| 10342 | RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType); | ||||||||
| 10343 | PendingExports.push_back(Chain); | ||||||||
| 10344 | } | ||||||||
| 10345 | |||||||||
| 10346 | #include "llvm/CodeGen/SelectionDAGISel.h" | ||||||||
| 10347 | |||||||||
| 10348 | /// isOnlyUsedInEntryBlock - If the specified argument is only used in the | ||||||||
| 10349 | /// entry block, return true. This includes arguments used by switches, since | ||||||||
| 10350 | /// the switch may expand into multiple basic blocks. | ||||||||
| 10351 | static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { | ||||||||
| 10352 | // With FastISel active, we may be splitting blocks, so force creation | ||||||||
| 10353 | // of virtual registers for all non-dead arguments. | ||||||||
| 10354 | if (FastISel) | ||||||||
| 10355 | return A->use_empty(); | ||||||||
| 10356 | |||||||||
| 10357 | const BasicBlock &Entry = A->getParent()->front(); | ||||||||
| 10358 | for (const User *U : A->users()) | ||||||||
| 10359 | if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U)) | ||||||||
| 10360 | return false; // Use not in entry block. | ||||||||
| 10361 | |||||||||
| 10362 | return true; | ||||||||
| 10363 | } | ||||||||
| 10364 | |||||||||
| 10365 | using ArgCopyElisionMapTy = | ||||||||
| 10366 | DenseMap<const Argument *, | ||||||||
| 10367 | std::pair<const AllocaInst *, const StoreInst *>>; | ||||||||
| 10368 | |||||||||
| 10369 | /// Scan the entry block of the function in FuncInfo for arguments that look | ||||||||
| 10370 | /// like copies into a local alloca. Record any copied arguments in | ||||||||
| 10371 | /// ArgCopyElisionCandidates. | ||||||||
| 10372 | static void | ||||||||
| 10373 | findArgumentCopyElisionCandidates(const DataLayout &DL, | ||||||||
| 10374 | FunctionLoweringInfo *FuncInfo, | ||||||||
| 10375 | ArgCopyElisionMapTy &ArgCopyElisionCandidates) { | ||||||||
| 10376 | // Record the state of every static alloca used in the entry block. Argument | ||||||||
| 10377 | // allocas are all used in the entry block, so we need approximately as many | ||||||||
| 10378 | // entries as we have arguments. | ||||||||
| 10379 | enum StaticAllocaInfo { Unknown, Clobbered, Elidable }; | ||||||||
| 10380 | SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas; | ||||||||
| 10381 | unsigned NumArgs = FuncInfo->Fn->arg_size(); | ||||||||
| 10382 | StaticAllocas.reserve(NumArgs * 2); | ||||||||
| 10383 | |||||||||
| 10384 | auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * { | ||||||||
| 10385 | if (!V) | ||||||||
| 10386 | return nullptr; | ||||||||
| 10387 | V = V->stripPointerCasts(); | ||||||||
| 10388 | const auto *AI = dyn_cast<AllocaInst>(V); | ||||||||
| 10389 | if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI)) | ||||||||
| 10390 | return nullptr; | ||||||||
| 10391 | auto Iter = StaticAllocas.insert({AI, Unknown}); | ||||||||
| 10392 | return &Iter.first->second; | ||||||||
| 10393 | }; | ||||||||
| 10394 | |||||||||
| 10395 | // Look for stores of arguments to static allocas. Look through bitcasts and | ||||||||
| 10396 | // GEPs to handle type coercions, as long as the alloca is fully initialized | ||||||||
| 10397 | // by the store. Any non-store use of an alloca escapes it and any subsequent | ||||||||
| 10398 | // unanalyzed store might write it. | ||||||||
| 10399 | // FIXME: Handle structs initialized with multiple stores. | ||||||||
| 10400 | for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) { | ||||||||
| 10401 | // Look for stores, and handle non-store uses conservatively. | ||||||||
| 10402 | const auto *SI = dyn_cast<StoreInst>(&I); | ||||||||
| 10403 | if (!SI) { | ||||||||
| 10404 | // We will look through cast uses, so ignore them completely. | ||||||||
| 10405 | if (I.isCast()) | ||||||||
| 10406 | continue; | ||||||||
| 10407 | // Ignore debug info and pseudo op intrinsics, they don't escape or store | ||||||||
| 10408 | // to allocas. | ||||||||
| 10409 | if (I.isDebugOrPseudoInst()) | ||||||||
| 10410 | continue; | ||||||||
| 10411 | // This is an unknown instruction. Assume it escapes or writes to all | ||||||||
| 10412 | // static alloca operands. | ||||||||
| 10413 | for (const Use &U : I.operands()) { | ||||||||
| 10414 | if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U)) | ||||||||
| 10415 | *Info = StaticAllocaInfo::Clobbered; | ||||||||
| 10416 | } | ||||||||
| 10417 | continue; | ||||||||
| 10418 | } | ||||||||
| 10419 | |||||||||
| 10420 | // If the stored value is a static alloca, mark it as escaped. | ||||||||
| 10421 | if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand())) | ||||||||
| 10422 | *Info = StaticAllocaInfo::Clobbered; | ||||||||
| 10423 | |||||||||
| 10424 | // Check if the destination is a static alloca. | ||||||||
| 10425 | const Value *Dst = SI->getPointerOperand()->stripPointerCasts(); | ||||||||
| 10426 | StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst); | ||||||||
| 10427 | if (!Info) | ||||||||
| 10428 | continue; | ||||||||
| 10429 | const AllocaInst *AI = cast<AllocaInst>(Dst); | ||||||||
| 10430 | |||||||||
| 10431 | // Skip allocas that have been initialized or clobbered. | ||||||||
| 10432 | if (*Info != StaticAllocaInfo::Unknown) | ||||||||
| 10433 | continue; | ||||||||
| 10434 | |||||||||
| 10435 | // Check if the stored value is an argument, and that this store fully | ||||||||
| 10436 | // initializes the alloca. | ||||||||
| 10437 | // If the argument type has padding bits we can't directly forward a pointer | ||||||||
| 10438 | // as the upper bits may contain garbage. | ||||||||
| 10439 | // Don't elide copies from the same argument twice. | ||||||||
| 10440 | const Value *Val = SI->getValueOperand()->stripPointerCasts(); | ||||||||
| 10441 | const auto *Arg = dyn_cast<Argument>(Val); | ||||||||
| 10442 | if (!Arg || Arg->hasPassPointeeByValueCopyAttr() || | ||||||||
| 10443 | Arg->getType()->isEmptyTy() || | ||||||||
| 10444 | DL.getTypeStoreSize(Arg->getType()) != | ||||||||
| 10445 | DL.getTypeAllocSize(AI->getAllocatedType()) || | ||||||||
| 10446 | !DL.typeSizeEqualsStoreSize(Arg->getType()) || | ||||||||
| 10447 | ArgCopyElisionCandidates.count(Arg)) { | ||||||||
| 10448 | *Info = StaticAllocaInfo::Clobbered; | ||||||||
| 10449 | continue; | ||||||||
| 10450 | } | ||||||||
| 10451 | |||||||||
| 10452 | LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Found argument copy elision candidate: " << *AI << '\n'; } } while (false) | ||||||||
| 10453 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Found argument copy elision candidate: " << *AI << '\n'; } } while (false); | ||||||||
| 10454 | |||||||||
| 10455 | // Mark this alloca and store for argument copy elision. | ||||||||
| 10456 | *Info = StaticAllocaInfo::Elidable; | ||||||||
| 10457 | ArgCopyElisionCandidates.insert({Arg, {AI, SI}}); | ||||||||
| 10458 | |||||||||
| 10459 | // Stop scanning if we've seen all arguments. This will happen early in -O0 | ||||||||
| 10460 | // builds, which is useful, because -O0 builds have large entry blocks and | ||||||||
| 10461 | // many allocas. | ||||||||
| 10462 | if (ArgCopyElisionCandidates.size() == NumArgs) | ||||||||
| 10463 | break; | ||||||||
| 10464 | } | ||||||||
| 10465 | } | ||||||||
| 10466 | |||||||||
| 10467 | /// Try to elide argument copies from memory into a local alloca. Succeeds if | ||||||||
| 10468 | /// ArgVal is a load from a suitable fixed stack object. | ||||||||
| 10469 | static void tryToElideArgumentCopy( | ||||||||
| 10470 | FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains, | ||||||||
| 10471 | DenseMap<int, int> &ArgCopyElisionFrameIndexMap, | ||||||||
| 10472 | SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs, | ||||||||
| 10473 | ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, | ||||||||
| 10474 | SDValue ArgVal, bool &ArgHasUses) { | ||||||||
| 10475 | // Check if this is a load from a fixed stack object. | ||||||||
| 10476 | auto *LNode = dyn_cast<LoadSDNode>(ArgVal); | ||||||||
| 10477 | if (!LNode) | ||||||||
| 10478 | return; | ||||||||
| 10479 | auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()); | ||||||||
| 10480 | if (!FINode) | ||||||||
| 10481 | return; | ||||||||
| 10482 | |||||||||
| 10483 | // Check that the fixed stack object is the right size and alignment. | ||||||||
| 10484 | // Look at the alignment that the user wrote on the alloca instead of looking | ||||||||
| 10485 | // at the stack object. | ||||||||
| 10486 | auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg); | ||||||||
| 10487 | assert(ArgCopyIter != ArgCopyElisionCandidates.end())(static_cast <bool> (ArgCopyIter != ArgCopyElisionCandidates .end()) ? void (0) : __assert_fail ("ArgCopyIter != ArgCopyElisionCandidates.end()" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10487 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10488 | const AllocaInst *AI = ArgCopyIter->second.first; | ||||||||
| 10489 | int FixedIndex = FINode->getIndex(); | ||||||||
| 10490 | int &AllocaIndex = FuncInfo.StaticAllocaMap[AI]; | ||||||||
| 10491 | int OldIndex = AllocaIndex; | ||||||||
| 10492 | MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo(); | ||||||||
| 10493 | if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) { | ||||||||
| 10494 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack " "object size\n"; } } while (false) | ||||||||
| 10495 | dbgs() << " argument copy elision failed due to bad fixed stack "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack " "object size\n"; } } while (false) | ||||||||
| 10496 | "object size\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack " "object size\n"; } } while (false); | ||||||||
| 10497 | return; | ||||||||
| 10498 | } | ||||||||
| 10499 | Align RequiredAlignment = AI->getAlign(); | ||||||||
| 10500 | if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) { | ||||||||
| 10501 | LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false) | ||||||||
| 10502 | "greater than stack argument alignment ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false) | ||||||||
| 10503 | << DebugStr(RequiredAlignment) << " vs "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false) | ||||||||
| 10504 | << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " argument copy elision failed: alignment of alloca " "greater than stack argument alignment (" << DebugStr( RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign (FixedIndex)) << ")\n"; } } while (false); | ||||||||
| 10505 | return; | ||||||||
| 10506 | } | ||||||||
| 10507 | |||||||||
| 10508 | // Perform the elision. Delete the old stack object and replace its only use | ||||||||
| 10509 | // in the variable info map. Mark the stack object as mutable. | ||||||||
| 10510 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||
| 10511 | dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||
| 10512 | << " Replacing frame index " << OldIndex << " with " << FixedIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||
| 10513 | << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false) | ||||||||
| 10514 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' << " Replacing frame index " << OldIndex << " with " << FixedIndex << '\n'; }; } } while (false); | ||||||||
| 10515 | MFI.RemoveStackObject(OldIndex); | ||||||||
| 10516 | MFI.setIsImmutableObjectIndex(FixedIndex, false); | ||||||||
| 10517 | AllocaIndex = FixedIndex; | ||||||||
| 10518 | ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex}); | ||||||||
| 10519 | Chains.push_back(ArgVal.getValue(1)); | ||||||||
| 10520 | |||||||||
| 10521 | // Avoid emitting code for the store implementing the copy. | ||||||||
| 10522 | const StoreInst *SI = ArgCopyIter->second.second; | ||||||||
| 10523 | ElidedArgCopyInstrs.insert(SI); | ||||||||
| 10524 | |||||||||
| 10525 | // Check for uses of the argument again so that we can avoid exporting ArgVal | ||||||||
| 10526 | // if it is't used by anything other than the store. | ||||||||
| 10527 | for (const Value *U : Arg.users()) { | ||||||||
| 10528 | if (U != SI) { | ||||||||
| 10529 | ArgHasUses = true; | ||||||||
| 10530 | break; | ||||||||
| 10531 | } | ||||||||
| 10532 | } | ||||||||
| 10533 | } | ||||||||
| 10534 | |||||||||
| 10535 | void SelectionDAGISel::LowerArguments(const Function &F) { | ||||||||
| 10536 | SelectionDAG &DAG = SDB->DAG; | ||||||||
| 10537 | SDLoc dl = SDB->getCurSDLoc(); | ||||||||
| 10538 | const DataLayout &DL = DAG.getDataLayout(); | ||||||||
| 10539 | SmallVector<ISD::InputArg, 16> Ins; | ||||||||
| 10540 | |||||||||
| 10541 | // In Naked functions we aren't going to save any registers. | ||||||||
| 10542 | if (F.hasFnAttribute(Attribute::Naked)) | ||||||||
| 10543 | return; | ||||||||
| 10544 | |||||||||
| 10545 | if (!FuncInfo->CanLowerReturn) { | ||||||||
| 10546 | // Put in an sret pointer parameter before all the other parameters. | ||||||||
| 10547 | SmallVector<EVT, 1> ValueVTs; | ||||||||
| 10548 | ComputeValueVTs(*TLI, DAG.getDataLayout(), | ||||||||
| 10549 | F.getReturnType()->getPointerTo( | ||||||||
| 10550 | DAG.getDataLayout().getAllocaAddrSpace()), | ||||||||
| 10551 | ValueVTs); | ||||||||
| 10552 | |||||||||
| 10553 | // NOTE: Assuming that a pointer will never break down to more than one VT | ||||||||
| 10554 | // or one register. | ||||||||
| 10555 | ISD::ArgFlagsTy Flags; | ||||||||
| 10556 | Flags.setSRet(); | ||||||||
| 10557 | MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]); | ||||||||
| 10558 | ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true, | ||||||||
| 10559 | ISD::InputArg::NoArgIndex, 0); | ||||||||
| 10560 | Ins.push_back(RetArg); | ||||||||
| 10561 | } | ||||||||
| 10562 | |||||||||
| 10563 | // Look for stores of arguments to static allocas. Mark such arguments with a | ||||||||
| 10564 | // flag to ask the target to give us the memory location of that argument if | ||||||||
| 10565 | // available. | ||||||||
| 10566 | ArgCopyElisionMapTy ArgCopyElisionCandidates; | ||||||||
| 10567 | findArgumentCopyElisionCandidates(DL, FuncInfo.get(), | ||||||||
| 10568 | ArgCopyElisionCandidates); | ||||||||
| 10569 | |||||||||
| 10570 | // Set up the incoming argument description vector. | ||||||||
| 10571 | for (const Argument &Arg : F.args()) { | ||||||||
| 10572 | unsigned ArgNo = Arg.getArgNo(); | ||||||||
| 10573 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 10574 | ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); | ||||||||
| 10575 | bool isArgValueUsed = !Arg.use_empty(); | ||||||||
| 10576 | unsigned PartBase = 0; | ||||||||
| 10577 | Type *FinalType = Arg.getType(); | ||||||||
| 10578 | if (Arg.hasAttribute(Attribute::ByVal)) | ||||||||
| 10579 | FinalType = Arg.getParamByValType(); | ||||||||
| 10580 | bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( | ||||||||
| 10581 | FinalType, F.getCallingConv(), F.isVarArg(), DL); | ||||||||
| 10582 | for (unsigned Value = 0, NumValues = ValueVTs.size(); | ||||||||
| 10583 | Value != NumValues; ++Value) { | ||||||||
| 10584 | EVT VT = ValueVTs[Value]; | ||||||||
| 10585 | Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); | ||||||||
| 10586 | ISD::ArgFlagsTy Flags; | ||||||||
| 10587 | |||||||||
| 10588 | |||||||||
| 10589 | if (Arg.getType()->isPointerTy()) { | ||||||||
| 10590 | Flags.setPointer(); | ||||||||
| 10591 | Flags.setPointerAddrSpace( | ||||||||
| 10592 | cast<PointerType>(Arg.getType())->getAddressSpace()); | ||||||||
| 10593 | } | ||||||||
| 10594 | if (Arg.hasAttribute(Attribute::ZExt)) | ||||||||
| 10595 | Flags.setZExt(); | ||||||||
| 10596 | if (Arg.hasAttribute(Attribute::SExt)) | ||||||||
| 10597 | Flags.setSExt(); | ||||||||
| 10598 | if (Arg.hasAttribute(Attribute::InReg)) { | ||||||||
| 10599 | // If we are using vectorcall calling convention, a structure that is | ||||||||
| 10600 | // passed InReg - is surely an HVA | ||||||||
| 10601 | if (F.getCallingConv() == CallingConv::X86_VectorCall && | ||||||||
| 10602 | isa<StructType>(Arg.getType())) { | ||||||||
| 10603 | // The first value of a structure is marked | ||||||||
| 10604 | if (0 == Value) | ||||||||
| 10605 | Flags.setHvaStart(); | ||||||||
| 10606 | Flags.setHva(); | ||||||||
| 10607 | } | ||||||||
| 10608 | // Set InReg Flag | ||||||||
| 10609 | Flags.setInReg(); | ||||||||
| 10610 | } | ||||||||
| 10611 | if (Arg.hasAttribute(Attribute::StructRet)) | ||||||||
| 10612 | Flags.setSRet(); | ||||||||
| 10613 | if (Arg.hasAttribute(Attribute::SwiftSelf)) | ||||||||
| 10614 | Flags.setSwiftSelf(); | ||||||||
| 10615 | if (Arg.hasAttribute(Attribute::SwiftAsync)) | ||||||||
| 10616 | Flags.setSwiftAsync(); | ||||||||
| 10617 | if (Arg.hasAttribute(Attribute::SwiftError)) | ||||||||
| 10618 | Flags.setSwiftError(); | ||||||||
| 10619 | if (Arg.hasAttribute(Attribute::ByVal)) | ||||||||
| 10620 | Flags.setByVal(); | ||||||||
| 10621 | if (Arg.hasAttribute(Attribute::ByRef)) | ||||||||
| 10622 | Flags.setByRef(); | ||||||||
| 10623 | if (Arg.hasAttribute(Attribute::InAlloca)) { | ||||||||
| 10624 | Flags.setInAlloca(); | ||||||||
| 10625 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||
| 10626 | // inalloca. This way we can know how many bytes we should've allocated | ||||||||
| 10627 | // and how many bytes a callee cleanup function will pop. If we port | ||||||||
| 10628 | // inalloca to more targets, we'll have to add custom inalloca handling | ||||||||
| 10629 | // in the various CC lowering callbacks. | ||||||||
| 10630 | Flags.setByVal(); | ||||||||
| 10631 | } | ||||||||
| 10632 | if (Arg.hasAttribute(Attribute::Preallocated)) { | ||||||||
| 10633 | Flags.setPreallocated(); | ||||||||
| 10634 | // Set the byval flag for CCAssignFn callbacks that don't know about | ||||||||
| 10635 | // preallocated. This way we can know how many bytes we should've | ||||||||
| 10636 | // allocated and how many bytes a callee cleanup function will pop. If | ||||||||
| 10637 | // we port preallocated to more targets, we'll have to add custom | ||||||||
| 10638 | // preallocated handling in the various CC lowering callbacks. | ||||||||
| 10639 | Flags.setByVal(); | ||||||||
| 10640 | } | ||||||||
| 10641 | |||||||||
| 10642 | // Certain targets (such as MIPS), may have a different ABI alignment | ||||||||
| 10643 | // for a type depending on the context. Give the target a chance to | ||||||||
| 10644 | // specify the alignment it wants. | ||||||||
| 10645 | const Align OriginalAlignment( | ||||||||
| 10646 | TLI->getABIAlignmentForCallingConv(ArgTy, DL)); | ||||||||
| 10647 | Flags.setOrigAlign(OriginalAlignment); | ||||||||
| 10648 | |||||||||
| 10649 | Align MemAlign; | ||||||||
| 10650 | Type *ArgMemTy = nullptr; | ||||||||
| 10651 | if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() || | ||||||||
| 10652 | Flags.isByRef()) { | ||||||||
| 10653 | if (!ArgMemTy) | ||||||||
| 10654 | ArgMemTy = Arg.getPointeeInMemoryValueType(); | ||||||||
| 10655 | |||||||||
| 10656 | uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy); | ||||||||
| 10657 | |||||||||
| 10658 | // For in-memory arguments, size and alignment should be passed from FE. | ||||||||
| 10659 | // BE will guess if this info is not there but there are cases it cannot | ||||||||
| 10660 | // get right. | ||||||||
| 10661 | if (auto ParamAlign = Arg.getParamStackAlign()) | ||||||||
| 10662 | MemAlign = *ParamAlign; | ||||||||
| 10663 | else if ((ParamAlign = Arg.getParamAlign())) | ||||||||
| 10664 | MemAlign = *ParamAlign; | ||||||||
| 10665 | else | ||||||||
| 10666 | MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL)); | ||||||||
| 10667 | if (Flags.isByRef()) | ||||||||
| 10668 | Flags.setByRefSize(MemSize); | ||||||||
| 10669 | else | ||||||||
| 10670 | Flags.setByValSize(MemSize); | ||||||||
| 10671 | } else if (auto ParamAlign = Arg.getParamStackAlign()) { | ||||||||
| 10672 | MemAlign = *ParamAlign; | ||||||||
| 10673 | } else { | ||||||||
| 10674 | MemAlign = OriginalAlignment; | ||||||||
| 10675 | } | ||||||||
| 10676 | Flags.setMemAlign(MemAlign); | ||||||||
| 10677 | |||||||||
| 10678 | if (Arg.hasAttribute(Attribute::Nest)) | ||||||||
| 10679 | Flags.setNest(); | ||||||||
| 10680 | if (NeedsRegBlock) | ||||||||
| 10681 | Flags.setInConsecutiveRegs(); | ||||||||
| 10682 | if (ArgCopyElisionCandidates.count(&Arg)) | ||||||||
| 10683 | Flags.setCopyElisionCandidate(); | ||||||||
| 10684 | if (Arg.hasAttribute(Attribute::Returned)) | ||||||||
| 10685 | Flags.setReturned(); | ||||||||
| 10686 | |||||||||
| 10687 | MVT RegisterVT = TLI->getRegisterTypeForCallingConv( | ||||||||
| 10688 | *CurDAG->getContext(), F.getCallingConv(), VT); | ||||||||
| 10689 | unsigned NumRegs = TLI->getNumRegistersForCallingConv( | ||||||||
| 10690 | *CurDAG->getContext(), F.getCallingConv(), VT); | ||||||||
| 10691 | for (unsigned i = 0; i != NumRegs; ++i) { | ||||||||
| 10692 | // For scalable vectors, use the minimum size; individual targets | ||||||||
| 10693 | // are responsible for handling scalable vector arguments and | ||||||||
| 10694 | // return values. | ||||||||
| 10695 | ISD::InputArg MyFlags( | ||||||||
| 10696 | Flags, RegisterVT, VT, isArgValueUsed, ArgNo, | ||||||||
| 10697 | PartBase + i * RegisterVT.getStoreSize().getKnownMinValue()); | ||||||||
| 10698 | if (NumRegs > 1 && i == 0) | ||||||||
| 10699 | MyFlags.Flags.setSplit(); | ||||||||
| 10700 | // if it isn't first piece, alignment must be 1 | ||||||||
| 10701 | else if (i > 0) { | ||||||||
| 10702 | MyFlags.Flags.setOrigAlign(Align(1)); | ||||||||
| 10703 | if (i == NumRegs - 1) | ||||||||
| 10704 | MyFlags.Flags.setSplitEnd(); | ||||||||
| 10705 | } | ||||||||
| 10706 | Ins.push_back(MyFlags); | ||||||||
| 10707 | } | ||||||||
| 10708 | if (NeedsRegBlock && Value == NumValues - 1) | ||||||||
| 10709 | Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast(); | ||||||||
| 10710 | PartBase += VT.getStoreSize().getKnownMinValue(); | ||||||||
| 10711 | } | ||||||||
| 10712 | } | ||||||||
| 10713 | |||||||||
| 10714 | // Call the target to set up the argument values. | ||||||||
| 10715 | SmallVector<SDValue, 8> InVals; | ||||||||
| 10716 | SDValue NewRoot = TLI->LowerFormalArguments( | ||||||||
| 10717 | DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals); | ||||||||
| 10718 | |||||||||
| 10719 | // Verify that the target's LowerFormalArguments behaved as expected. | ||||||||
| 10720 | assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&(static_cast <bool> (NewRoot.getNode() && NewRoot .getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!" ) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10721 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10721 | "LowerFormalArguments didn't return a valid chain!")(static_cast <bool> (NewRoot.getNode() && NewRoot .getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!" ) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10721 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10722 | assert(InVals.size() == Ins.size() &&(static_cast <bool> (InVals.size() == Ins.size() && "LowerFormalArguments didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10723 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10723 | "LowerFormalArguments didn't emit the correct number of values!")(static_cast <bool> (InVals.size() == Ins.size() && "LowerFormalArguments didn't emit the correct number of values!" ) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10723 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10724 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false) | ||||||||
| 10725 | for (unsigned i = 0, e = Ins.size(); i != e; ++i) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false) | ||||||||
| 10726 | assert(InVals[i].getNode() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false) | ||||||||
| 10727 | "LowerFormalArguments emitted a null value!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false) | ||||||||
| 10728 | assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false) | ||||||||
| 10729 | "LowerFormalArguments emitted a value with the wrong type!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false) | ||||||||
| 10730 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false) | ||||||||
| 10731 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++ i) { (static_cast <bool> (InVals[i].getNode() && "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727 , __extension__ __PRETTY_FUNCTION__)); (static_cast <bool> (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!" ) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729 , __extension__ __PRETTY_FUNCTION__)); } }; } } while (false); | ||||||||
| 10732 | |||||||||
| 10733 | // Update the DAG with the new chain value resulting from argument lowering. | ||||||||
| 10734 | DAG.setRoot(NewRoot); | ||||||||
| 10735 | |||||||||
| 10736 | // Set up the argument values. | ||||||||
| 10737 | unsigned i = 0; | ||||||||
| 10738 | if (!FuncInfo->CanLowerReturn) { | ||||||||
| 10739 | // Create a virtual register for the sret pointer, and put in a copy | ||||||||
| 10740 | // from the sret argument into it. | ||||||||
| 10741 | SmallVector<EVT, 1> ValueVTs; | ||||||||
| 10742 | ComputeValueVTs(*TLI, DAG.getDataLayout(), | ||||||||
| 10743 | F.getReturnType()->getPointerTo( | ||||||||
| 10744 | DAG.getDataLayout().getAllocaAddrSpace()), | ||||||||
| 10745 | ValueVTs); | ||||||||
| 10746 | MVT VT = ValueVTs[0].getSimpleVT(); | ||||||||
| 10747 | MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT); | ||||||||
| 10748 | std::optional<ISD::NodeType> AssertOp; | ||||||||
| 10749 | SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, | ||||||||
| 10750 | nullptr, F.getCallingConv(), AssertOp); | ||||||||
| 10751 | |||||||||
| 10752 | MachineFunction& MF = SDB->DAG.getMachineFunction(); | ||||||||
| 10753 | MachineRegisterInfo& RegInfo = MF.getRegInfo(); | ||||||||
| 10754 | Register SRetReg = | ||||||||
| 10755 | RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT)); | ||||||||
| 10756 | FuncInfo->DemoteRegister = SRetReg; | ||||||||
| 10757 | NewRoot = | ||||||||
| 10758 | SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue); | ||||||||
| 10759 | DAG.setRoot(NewRoot); | ||||||||
| 10760 | |||||||||
| 10761 | // i indexes lowered arguments. Bump it past the hidden sret argument. | ||||||||
| 10762 | ++i; | ||||||||
| 10763 | } | ||||||||
| 10764 | |||||||||
| 10765 | SmallVector<SDValue, 4> Chains; | ||||||||
| 10766 | DenseMap<int, int> ArgCopyElisionFrameIndexMap; | ||||||||
| 10767 | for (const Argument &Arg : F.args()) { | ||||||||
| 10768 | SmallVector<SDValue, 4> ArgValues; | ||||||||
| 10769 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 10770 | ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); | ||||||||
| 10771 | unsigned NumValues = ValueVTs.size(); | ||||||||
| 10772 | if (NumValues == 0) | ||||||||
| 10773 | continue; | ||||||||
| 10774 | |||||||||
| 10775 | bool ArgHasUses = !Arg.use_empty(); | ||||||||
| 10776 | |||||||||
| 10777 | // Elide the copying store if the target loaded this argument from a | ||||||||
| 10778 | // suitable fixed stack object. | ||||||||
| 10779 | if (Ins[i].Flags.isCopyElisionCandidate()) { | ||||||||
| 10780 | tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap, | ||||||||
| 10781 | ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg, | ||||||||
| 10782 | InVals[i], ArgHasUses); | ||||||||
| 10783 | } | ||||||||
| 10784 | |||||||||
| 10785 | // If this argument is unused then remember its value. It is used to generate | ||||||||
| 10786 | // debugging information. | ||||||||
| 10787 | bool isSwiftErrorArg = | ||||||||
| 10788 | TLI->supportSwiftError() && | ||||||||
| 10789 | Arg.hasAttribute(Attribute::SwiftError); | ||||||||
| 10790 | if (!ArgHasUses && !isSwiftErrorArg) { | ||||||||
| 10791 | SDB->setUnusedArgValue(&Arg, InVals[i]); | ||||||||
| 10792 | |||||||||
| 10793 | // Also remember any frame index for use in FastISel. | ||||||||
| 10794 | if (FrameIndexSDNode *FI = | ||||||||
| 10795 | dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) | ||||||||
| 10796 | FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); | ||||||||
| 10797 | } | ||||||||
| 10798 | |||||||||
| 10799 | for (unsigned Val = 0; Val != NumValues; ++Val) { | ||||||||
| 10800 | EVT VT = ValueVTs[Val]; | ||||||||
| 10801 | MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(), | ||||||||
| 10802 | F.getCallingConv(), VT); | ||||||||
| 10803 | unsigned NumParts = TLI->getNumRegistersForCallingConv( | ||||||||
| 10804 | *CurDAG->getContext(), F.getCallingConv(), VT); | ||||||||
| 10805 | |||||||||
| 10806 | // Even an apparent 'unused' swifterror argument needs to be returned. So | ||||||||
| 10807 | // we do generate a copy for it that can be used on return from the | ||||||||
| 10808 | // function. | ||||||||
| 10809 | if (ArgHasUses || isSwiftErrorArg) { | ||||||||
| 10810 | std::optional<ISD::NodeType> AssertOp; | ||||||||
| 10811 | if (Arg.hasAttribute(Attribute::SExt)) | ||||||||
| 10812 | AssertOp = ISD::AssertSext; | ||||||||
| 10813 | else if (Arg.hasAttribute(Attribute::ZExt)) | ||||||||
| 10814 | AssertOp = ISD::AssertZext; | ||||||||
| 10815 | |||||||||
| 10816 | ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts, | ||||||||
| 10817 | PartVT, VT, nullptr, | ||||||||
| 10818 | F.getCallingConv(), AssertOp)); | ||||||||
| 10819 | } | ||||||||
| 10820 | |||||||||
| 10821 | i += NumParts; | ||||||||
| 10822 | } | ||||||||
| 10823 | |||||||||
| 10824 | // We don't need to do anything else for unused arguments. | ||||||||
| 10825 | if (ArgValues.empty()) | ||||||||
| 10826 | continue; | ||||||||
| 10827 | |||||||||
| 10828 | // Note down frame index. | ||||||||
| 10829 | if (FrameIndexSDNode *FI = | ||||||||
| 10830 | dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) | ||||||||
| 10831 | FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); | ||||||||
| 10832 | |||||||||
| 10833 | SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues), | ||||||||
| 10834 | SDB->getCurSDLoc()); | ||||||||
| 10835 | |||||||||
| 10836 | SDB->setValue(&Arg, Res); | ||||||||
| 10837 | if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { | ||||||||
| 10838 | // We want to associate the argument with the frame index, among | ||||||||
| 10839 | // involved operands, that correspond to the lowest address. The | ||||||||
| 10840 | // getCopyFromParts function, called earlier, is swapping the order of | ||||||||
| 10841 | // the operands to BUILD_PAIR depending on endianness. The result of | ||||||||
| 10842 | // that swapping is that the least significant bits of the argument will | ||||||||
| 10843 | // be in the first operand of the BUILD_PAIR node, and the most | ||||||||
| 10844 | // significant bits will be in the second operand. | ||||||||
| 10845 | unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0; | ||||||||
| 10846 | if (LoadSDNode *LNode = | ||||||||
| 10847 | dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode())) | ||||||||
| 10848 | if (FrameIndexSDNode *FI = | ||||||||
| 10849 | dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) | ||||||||
| 10850 | FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); | ||||||||
| 10851 | } | ||||||||
| 10852 | |||||||||
| 10853 | // Analyses past this point are naive and don't expect an assertion. | ||||||||
| 10854 | if (Res.getOpcode() == ISD::AssertZext) | ||||||||
| 10855 | Res = Res.getOperand(0); | ||||||||
| 10856 | |||||||||
| 10857 | // Update the SwiftErrorVRegDefMap. | ||||||||
| 10858 | if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) { | ||||||||
| 10859 | unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); | ||||||||
| 10860 | if (Register::isVirtualRegister(Reg)) | ||||||||
| 10861 | SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(), | ||||||||
| 10862 | Reg); | ||||||||
| 10863 | } | ||||||||
| 10864 | |||||||||
| 10865 | // If this argument is live outside of the entry block, insert a copy from | ||||||||
| 10866 | // wherever we got it to the vreg that other BB's will reference it as. | ||||||||
| 10867 | if (Res.getOpcode() == ISD::CopyFromReg) { | ||||||||
| 10868 | // If we can, though, try to skip creating an unnecessary vreg. | ||||||||
| 10869 | // FIXME: This isn't very clean... it would be nice to make this more | ||||||||
| 10870 | // general. | ||||||||
| 10871 | unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); | ||||||||
| 10872 | if (Register::isVirtualRegister(Reg)) { | ||||||||
| 10873 | FuncInfo->ValueMap[&Arg] = Reg; | ||||||||
| 10874 | continue; | ||||||||
| 10875 | } | ||||||||
| 10876 | } | ||||||||
| 10877 | if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) { | ||||||||
| 10878 | FuncInfo->InitializeRegForValue(&Arg); | ||||||||
| 10879 | SDB->CopyToExportRegsIfNeeded(&Arg); | ||||||||
| 10880 | } | ||||||||
| 10881 | } | ||||||||
| 10882 | |||||||||
| 10883 | if (!Chains.empty()) { | ||||||||
| 10884 | Chains.push_back(NewRoot); | ||||||||
| 10885 | NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); | ||||||||
| 10886 | } | ||||||||
| 10887 | |||||||||
| 10888 | DAG.setRoot(NewRoot); | ||||||||
| 10889 | |||||||||
| 10890 | assert(i == InVals.size() && "Argument register count mismatch!")(static_cast <bool> (i == InVals.size() && "Argument register count mismatch!" ) ? void (0) : __assert_fail ("i == InVals.size() && \"Argument register count mismatch!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10890 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10891 | |||||||||
| 10892 | // If any argument copy elisions occurred and we have debug info, update the | ||||||||
| 10893 | // stale frame indices used in the dbg.declare variable info table. | ||||||||
| 10894 | MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo(); | ||||||||
| 10895 | if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) { | ||||||||
| 10896 | for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) { | ||||||||
| 10897 | auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot); | ||||||||
| 10898 | if (I != ArgCopyElisionFrameIndexMap.end()) | ||||||||
| 10899 | VI.Slot = I->second; | ||||||||
| 10900 | } | ||||||||
| 10901 | } | ||||||||
| 10902 | |||||||||
| 10903 | // Finally, if the target has anything special to do, allow it to do so. | ||||||||
| 10904 | emitFunctionEntryCode(); | ||||||||
| 10905 | } | ||||||||
| 10906 | |||||||||
| 10907 | /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to | ||||||||
| 10908 | /// ensure constants are generated when needed. Remember the virtual registers | ||||||||
| 10909 | /// that need to be added to the Machine PHI nodes as input. We cannot just | ||||||||
| 10910 | /// directly add them, because expansion might result in multiple MBB's for one | ||||||||
| 10911 | /// BB. As such, the start of the BB might correspond to a different MBB than | ||||||||
| 10912 | /// the end. | ||||||||
| 10913 | void | ||||||||
| 10914 | SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { | ||||||||
| 10915 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 10916 | |||||||||
| 10917 | SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; | ||||||||
| 10918 | |||||||||
| 10919 | // Check PHI nodes in successors that expect a value to be available from this | ||||||||
| 10920 | // block. | ||||||||
| 10921 | for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) { | ||||||||
| 10922 | if (!isa<PHINode>(SuccBB->begin())) continue; | ||||||||
| 10923 | MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; | ||||||||
| 10924 | |||||||||
| 10925 | // If this terminator has multiple identical successors (common for | ||||||||
| 10926 | // switches), only handle each succ once. | ||||||||
| 10927 | if (!SuccsHandled.insert(SuccMBB).second) | ||||||||
| 10928 | continue; | ||||||||
| 10929 | |||||||||
| 10930 | MachineBasicBlock::iterator MBBI = SuccMBB->begin(); | ||||||||
| 10931 | |||||||||
| 10932 | // At this point we know that there is a 1-1 correspondence between LLVM PHI | ||||||||
| 10933 | // nodes and Machine PHI nodes, but the incoming operands have not been | ||||||||
| 10934 | // emitted yet. | ||||||||
| 10935 | for (const PHINode &PN : SuccBB->phis()) { | ||||||||
| 10936 | // Ignore dead phi's. | ||||||||
| 10937 | if (PN.use_empty()) | ||||||||
| 10938 | continue; | ||||||||
| 10939 | |||||||||
| 10940 | // Skip empty types | ||||||||
| 10941 | if (PN.getType()->isEmptyTy()) | ||||||||
| 10942 | continue; | ||||||||
| 10943 | |||||||||
| 10944 | unsigned Reg; | ||||||||
| 10945 | const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB); | ||||||||
| 10946 | |||||||||
| 10947 | if (const auto *C = dyn_cast<Constant>(PHIOp)) { | ||||||||
| 10948 | unsigned &RegOut = ConstantsOut[C]; | ||||||||
| 10949 | if (RegOut == 0) { | ||||||||
| 10950 | RegOut = FuncInfo.CreateRegs(C); | ||||||||
| 10951 | // We need to zero/sign extend ConstantInt phi operands to match | ||||||||
| 10952 | // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo. | ||||||||
| 10953 | ISD::NodeType ExtendType = ISD::ANY_EXTEND; | ||||||||
| 10954 | if (auto *CI = dyn_cast<ConstantInt>(C)) | ||||||||
| 10955 | ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND | ||||||||
| 10956 | : ISD::ZERO_EXTEND; | ||||||||
| 10957 | CopyValueToVirtualRegister(C, RegOut, ExtendType); | ||||||||
| 10958 | } | ||||||||
| 10959 | Reg = RegOut; | ||||||||
| 10960 | } else { | ||||||||
| 10961 | DenseMap<const Value *, Register>::iterator I = | ||||||||
| 10962 | FuncInfo.ValueMap.find(PHIOp); | ||||||||
| 10963 | if (I != FuncInfo.ValueMap.end()) | ||||||||
| 10964 | Reg = I->second; | ||||||||
| 10965 | else { | ||||||||
| 10966 | assert(isa<AllocaInst>(PHIOp) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp) ) && "Didn't codegen value into a register!??") ? void (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10968 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10967 | FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp) ) && "Didn't codegen value into a register!??") ? void (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10968 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 10968 | "Didn't codegen value into a register!??")(static_cast <bool> (isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp) ) && "Didn't codegen value into a register!??") ? void (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10968 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 10969 | Reg = FuncInfo.CreateRegs(PHIOp); | ||||||||
| 10970 | CopyValueToVirtualRegister(PHIOp, Reg); | ||||||||
| 10971 | } | ||||||||
| 10972 | } | ||||||||
| 10973 | |||||||||
| 10974 | // Remember that this register needs to added to the machine PHI node as | ||||||||
| 10975 | // the input for this MBB. | ||||||||
| 10976 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 10977 | ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs); | ||||||||
| 10978 | for (EVT VT : ValueVTs) { | ||||||||
| 10979 | const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT); | ||||||||
| 10980 | for (unsigned i = 0; i != NumRegisters; ++i) | ||||||||
| 10981 | FuncInfo.PHINodesToUpdate.push_back( | ||||||||
| 10982 | std::make_pair(&*MBBI++, Reg + i)); | ||||||||
| 10983 | Reg += NumRegisters; | ||||||||
| 10984 | } | ||||||||
| 10985 | } | ||||||||
| 10986 | } | ||||||||
| 10987 | |||||||||
| 10988 | ConstantsOut.clear(); | ||||||||
| 10989 | } | ||||||||
| 10990 | |||||||||
| 10991 | MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) { | ||||||||
| 10992 | MachineFunction::iterator I(MBB); | ||||||||
| 10993 | if (++I == FuncInfo.MF->end()) | ||||||||
| 10994 | return nullptr; | ||||||||
| 10995 | return &*I; | ||||||||
| 10996 | } | ||||||||
| 10997 | |||||||||
| 10998 | /// During lowering new call nodes can be created (such as memset, etc.). | ||||||||
| 10999 | /// Those will become new roots of the current DAG, but complications arise | ||||||||
| 11000 | /// when they are tail calls. In such cases, the call lowering will update | ||||||||
| 11001 | /// the root, but the builder still needs to know that a tail call has been | ||||||||
| 11002 | /// lowered in order to avoid generating an additional return. | ||||||||
| 11003 | void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) { | ||||||||
| 11004 | // If the node is null, we do have a tail call. | ||||||||
| 11005 | if (MaybeTC.getNode() != nullptr) | ||||||||
| 11006 | DAG.setRoot(MaybeTC); | ||||||||
| 11007 | else | ||||||||
| 11008 | HasTailCall = true; | ||||||||
| 11009 | } | ||||||||
| 11010 | |||||||||
| 11011 | void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, | ||||||||
| 11012 | MachineBasicBlock *SwitchMBB, | ||||||||
| 11013 | MachineBasicBlock *DefaultMBB) { | ||||||||
| 11014 | MachineFunction *CurMF = FuncInfo.MF; | ||||||||
| 11015 | MachineBasicBlock *NextMBB = nullptr; | ||||||||
| 11016 | MachineFunction::iterator BBI(W.MBB); | ||||||||
| 11017 | if (++BBI != FuncInfo.MF->end()) | ||||||||
| 11018 | NextMBB = &*BBI; | ||||||||
| 11019 | |||||||||
| 11020 | unsigned Size = W.LastCluster - W.FirstCluster + 1; | ||||||||
| 11021 | |||||||||
| 11022 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||
| 11023 | |||||||||
| 11024 | if (Size == 2 && W.MBB == SwitchMBB) { | ||||||||
| 11025 | // If any two of the cases has the same destination, and if one value | ||||||||
| 11026 | // is the same as the other, but has one bit unset that the other has set, | ||||||||
| 11027 | // use bit manipulation to do two compares at once. For example: | ||||||||
| 11028 | // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)" | ||||||||
| 11029 | // TODO: This could be extended to merge any 2 cases in switches with 3 | ||||||||
| 11030 | // cases. | ||||||||
| 11031 | // TODO: Handle cases where W.CaseBB != SwitchBB. | ||||||||
| 11032 | CaseCluster &Small = *W.FirstCluster; | ||||||||
| 11033 | CaseCluster &Big = *W.LastCluster; | ||||||||
| 11034 | |||||||||
| 11035 | if (Small.Low == Small.High && Big.Low == Big.High && | ||||||||
| 11036 | Small.MBB == Big.MBB) { | ||||||||
| 11037 | const APInt &SmallValue = Small.Low->getValue(); | ||||||||
| 11038 | const APInt &BigValue = Big.Low->getValue(); | ||||||||
| 11039 | |||||||||
| 11040 | // Check that there is only one bit different. | ||||||||
| 11041 | APInt CommonBit = BigValue ^ SmallValue; | ||||||||
| 11042 | if (CommonBit.isPowerOf2()) { | ||||||||
| 11043 | SDValue CondLHS = getValue(Cond); | ||||||||
| 11044 | EVT VT = CondLHS.getValueType(); | ||||||||
| 11045 | SDLoc DL = getCurSDLoc(); | ||||||||
| 11046 | |||||||||
| 11047 | SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS, | ||||||||
| 11048 | DAG.getConstant(CommonBit, DL, VT)); | ||||||||
| 11049 | SDValue Cond = DAG.getSetCC( | ||||||||
| 11050 | DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT), | ||||||||
| 11051 | ISD::SETEQ); | ||||||||
| 11052 | |||||||||
| 11053 | // Update successor info. | ||||||||
| 11054 | // Both Small and Big will jump to Small.BB, so we sum up the | ||||||||
| 11055 | // probabilities. | ||||||||
| 11056 | addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob); | ||||||||
| 11057 | if (BPI) | ||||||||
| 11058 | addSuccessorWithProb( | ||||||||
| 11059 | SwitchMBB, DefaultMBB, | ||||||||
| 11060 | // The default destination is the first successor in IR. | ||||||||
| 11061 | BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0)); | ||||||||
| 11062 | else | ||||||||
| 11063 | addSuccessorWithProb(SwitchMBB, DefaultMBB); | ||||||||
| 11064 | |||||||||
| 11065 | // Insert the true branch. | ||||||||
| 11066 | SDValue BrCond = | ||||||||
| 11067 | DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond, | ||||||||
| 11068 | DAG.getBasicBlock(Small.MBB)); | ||||||||
| 11069 | // Insert the false branch. | ||||||||
| 11070 | BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond, | ||||||||
| 11071 | DAG.getBasicBlock(DefaultMBB)); | ||||||||
| 11072 | |||||||||
| 11073 | DAG.setRoot(BrCond); | ||||||||
| 11074 | return; | ||||||||
| 11075 | } | ||||||||
| 11076 | } | ||||||||
| 11077 | } | ||||||||
| 11078 | |||||||||
| 11079 | if (TM.getOptLevel() != CodeGenOpt::None) { | ||||||||
| 11080 | // Here, we order cases by probability so the most likely case will be | ||||||||
| 11081 | // checked first. However, two clusters can have the same probability in | ||||||||
| 11082 | // which case their relative ordering is non-deterministic. So we use Low | ||||||||
| 11083 | // as a tie-breaker as clusters are guaranteed to never overlap. | ||||||||
| 11084 | llvm::sort(W.FirstCluster, W.LastCluster + 1, | ||||||||
| 11085 | [](const CaseCluster &a, const CaseCluster &b) { | ||||||||
| 11086 | return a.Prob != b.Prob ? | ||||||||
| 11087 | a.Prob > b.Prob : | ||||||||
| 11088 | a.Low->getValue().slt(b.Low->getValue()); | ||||||||
| 11089 | }); | ||||||||
| 11090 | |||||||||
| 11091 | // Rearrange the case blocks so that the last one falls through if possible | ||||||||
| 11092 | // without changing the order of probabilities. | ||||||||
| 11093 | for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) { | ||||||||
| 11094 | --I; | ||||||||
| 11095 | if (I->Prob > W.LastCluster->Prob) | ||||||||
| 11096 | break; | ||||||||
| 11097 | if (I->Kind == CC_Range && I->MBB == NextMBB) { | ||||||||
| 11098 | std::swap(*I, *W.LastCluster); | ||||||||
| 11099 | break; | ||||||||
| 11100 | } | ||||||||
| 11101 | } | ||||||||
| 11102 | } | ||||||||
| 11103 | |||||||||
| 11104 | // Compute total probability. | ||||||||
| 11105 | BranchProbability DefaultProb = W.DefaultProb; | ||||||||
| 11106 | BranchProbability UnhandledProbs = DefaultProb; | ||||||||
| 11107 | for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I) | ||||||||
| 11108 | UnhandledProbs += I->Prob; | ||||||||
| 11109 | |||||||||
| 11110 | MachineBasicBlock *CurMBB = W.MBB; | ||||||||
| 11111 | for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) { | ||||||||
| 11112 | bool FallthroughUnreachable = false; | ||||||||
| 11113 | MachineBasicBlock *Fallthrough; | ||||||||
| 11114 | if (I == W.LastCluster) { | ||||||||
| 11115 | // For the last cluster, fall through to the default destination. | ||||||||
| 11116 | Fallthrough = DefaultMBB; | ||||||||
| 11117 | FallthroughUnreachable = isa<UnreachableInst>( | ||||||||
| 11118 | DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg()); | ||||||||
| 11119 | } else { | ||||||||
| 11120 | Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); | ||||||||
| 11121 | CurMF->insert(BBI, Fallthrough); | ||||||||
| 11122 | // Put Cond in a virtual register to make it available from the new blocks. | ||||||||
| 11123 | ExportFromCurrentBlock(Cond); | ||||||||
| 11124 | } | ||||||||
| 11125 | UnhandledProbs -= I->Prob; | ||||||||
| 11126 | |||||||||
| 11127 | switch (I->Kind) { | ||||||||
| 11128 | case CC_JumpTable: { | ||||||||
| 11129 | // FIXME: Optimize away range check based on pivot comparisons. | ||||||||
| 11130 | JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first; | ||||||||
| 11131 | SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second; | ||||||||
| 11132 | |||||||||
| 11133 | // The jump block hasn't been inserted yet; insert it here. | ||||||||
| 11134 | MachineBasicBlock *JumpMBB = JT->MBB; | ||||||||
| 11135 | CurMF->insert(BBI, JumpMBB); | ||||||||
| 11136 | |||||||||
| 11137 | auto JumpProb = I->Prob; | ||||||||
| 11138 | auto FallthroughProb = UnhandledProbs; | ||||||||
| 11139 | |||||||||
| 11140 | // If the default statement is a target of the jump table, we evenly | ||||||||
| 11141 | // distribute the default probability to successors of CurMBB. Also | ||||||||
| 11142 | // update the probability on the edge from JumpMBB to Fallthrough. | ||||||||
| 11143 | for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(), | ||||||||
| 11144 | SE = JumpMBB->succ_end(); | ||||||||
| 11145 | SI != SE; ++SI) { | ||||||||
| 11146 | if (*SI == DefaultMBB) { | ||||||||
| 11147 | JumpProb += DefaultProb / 2; | ||||||||
| 11148 | FallthroughProb -= DefaultProb / 2; | ||||||||
| 11149 | JumpMBB->setSuccProbability(SI, DefaultProb / 2); | ||||||||
| 11150 | JumpMBB->normalizeSuccProbs(); | ||||||||
| 11151 | break; | ||||||||
| 11152 | } | ||||||||
| 11153 | } | ||||||||
| 11154 | |||||||||
| 11155 | if (FallthroughUnreachable) | ||||||||
| 11156 | JTH->FallthroughUnreachable = true; | ||||||||
| 11157 | |||||||||
| 11158 | if (!JTH->FallthroughUnreachable) | ||||||||
| 11159 | addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); | ||||||||
| 11160 | addSuccessorWithProb(CurMBB, JumpMBB, JumpProb); | ||||||||
| 11161 | CurMBB->normalizeSuccProbs(); | ||||||||
| 11162 | |||||||||
| 11163 | // The jump table header will be inserted in our current block, do the | ||||||||
| 11164 | // range check, and fall through to our fallthrough block. | ||||||||
| 11165 | JTH->HeaderBB = CurMBB; | ||||||||
| 11166 | JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader. | ||||||||
| 11167 | |||||||||
| 11168 | // If we're in the right place, emit the jump table header right now. | ||||||||
| 11169 | if (CurMBB == SwitchMBB) { | ||||||||
| 11170 | visitJumpTableHeader(*JT, *JTH, SwitchMBB); | ||||||||
| 11171 | JTH->Emitted = true; | ||||||||
| 11172 | } | ||||||||
| 11173 | break; | ||||||||
| 11174 | } | ||||||||
| 11175 | case CC_BitTests: { | ||||||||
| 11176 | // FIXME: Optimize away range check based on pivot comparisons. | ||||||||
| 11177 | BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex]; | ||||||||
| 11178 | |||||||||
| 11179 | // The bit test blocks haven't been inserted yet; insert them here. | ||||||||
| 11180 | for (BitTestCase &BTC : BTB->Cases) | ||||||||
| 11181 | CurMF->insert(BBI, BTC.ThisBB); | ||||||||
| 11182 | |||||||||
| 11183 | // Fill in fields of the BitTestBlock. | ||||||||
| 11184 | BTB->Parent = CurMBB; | ||||||||
| 11185 | BTB->Default = Fallthrough; | ||||||||
| 11186 | |||||||||
| 11187 | BTB->DefaultProb = UnhandledProbs; | ||||||||
| 11188 | // If the cases in bit test don't form a contiguous range, we evenly | ||||||||
| 11189 | // distribute the probability on the edge to Fallthrough to two | ||||||||
| 11190 | // successors of CurMBB. | ||||||||
| 11191 | if (!BTB->ContiguousRange) { | ||||||||
| 11192 | BTB->Prob += DefaultProb / 2; | ||||||||
| 11193 | BTB->DefaultProb -= DefaultProb / 2; | ||||||||
| 11194 | } | ||||||||
| 11195 | |||||||||
| 11196 | if (FallthroughUnreachable) | ||||||||
| 11197 | BTB->FallthroughUnreachable = true; | ||||||||
| 11198 | |||||||||
| 11199 | // If we're in the right place, emit the bit test header right now. | ||||||||
| 11200 | if (CurMBB == SwitchMBB) { | ||||||||
| 11201 | visitBitTestHeader(*BTB, SwitchMBB); | ||||||||
| 11202 | BTB->Emitted = true; | ||||||||
| 11203 | } | ||||||||
| 11204 | break; | ||||||||
| 11205 | } | ||||||||
| 11206 | case CC_Range: { | ||||||||
| 11207 | const Value *RHS, *LHS, *MHS; | ||||||||
| 11208 | ISD::CondCode CC; | ||||||||
| 11209 | if (I->Low == I->High) { | ||||||||
| 11210 | // Check Cond == I->Low. | ||||||||
| 11211 | CC = ISD::SETEQ; | ||||||||
| 11212 | LHS = Cond; | ||||||||
| 11213 | RHS=I->Low; | ||||||||
| 11214 | MHS = nullptr; | ||||||||
| 11215 | } else { | ||||||||
| 11216 | // Check I->Low <= Cond <= I->High. | ||||||||
| 11217 | CC = ISD::SETLE; | ||||||||
| 11218 | LHS = I->Low; | ||||||||
| 11219 | MHS = Cond; | ||||||||
| 11220 | RHS = I->High; | ||||||||
| 11221 | } | ||||||||
| 11222 | |||||||||
| 11223 | // If Fallthrough is unreachable, fold away the comparison. | ||||||||
| 11224 | if (FallthroughUnreachable) | ||||||||
| 11225 | CC = ISD::SETTRUE; | ||||||||
| 11226 | |||||||||
| 11227 | // The false probability is the sum of all unhandled cases. | ||||||||
| 11228 | CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, | ||||||||
| 11229 | getCurSDLoc(), I->Prob, UnhandledProbs); | ||||||||
| 11230 | |||||||||
| 11231 | if (CurMBB == SwitchMBB) | ||||||||
| 11232 | visitSwitchCase(CB, SwitchMBB); | ||||||||
| 11233 | else | ||||||||
| 11234 | SL->SwitchCases.push_back(CB); | ||||||||
| 11235 | |||||||||
| 11236 | break; | ||||||||
| 11237 | } | ||||||||
| 11238 | } | ||||||||
| 11239 | CurMBB = Fallthrough; | ||||||||
| 11240 | } | ||||||||
| 11241 | } | ||||||||
| 11242 | |||||||||
| 11243 | unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC, | ||||||||
| 11244 | CaseClusterIt First, | ||||||||
| 11245 | CaseClusterIt Last) { | ||||||||
| 11246 | return std::count_if(First, Last + 1, [&](const CaseCluster &X) { | ||||||||
| 11247 | if (X.Prob != CC.Prob) | ||||||||
| 11248 | return X.Prob > CC.Prob; | ||||||||
| 11249 | |||||||||
| 11250 | // Ties are broken by comparing the case value. | ||||||||
| 11251 | return X.Low->getValue().slt(CC.Low->getValue()); | ||||||||
| 11252 | }); | ||||||||
| 11253 | } | ||||||||
| 11254 | |||||||||
| 11255 | void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList, | ||||||||
| 11256 | const SwitchWorkListItem &W, | ||||||||
| 11257 | Value *Cond, | ||||||||
| 11258 | MachineBasicBlock *SwitchMBB) { | ||||||||
| 11259 | assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&(static_cast <bool> (W.FirstCluster->Low->getValue ().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?" ) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11260 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 11260 | "Clusters not sorted?")(static_cast <bool> (W.FirstCluster->Low->getValue ().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?" ) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11260 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11261 | |||||||||
| 11262 | assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!")(static_cast <bool> (W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!") ? void (0) : __assert_fail ("W.LastCluster - W.FirstCluster + 1 >= 2 && \"Too small to split!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11262 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11263 | |||||||||
| 11264 | // Balance the tree based on branch probabilities to create a near-optimal (in | ||||||||
| 11265 | // terms of search time given key frequency) binary search tree. See e.g. Kurt | ||||||||
| 11266 | // Mehlhorn "Nearly Optimal Binary Search Trees" (1975). | ||||||||
| 11267 | CaseClusterIt LastLeft = W.FirstCluster; | ||||||||
| 11268 | CaseClusterIt FirstRight = W.LastCluster; | ||||||||
| 11269 | auto LeftProb = LastLeft->Prob + W.DefaultProb / 2; | ||||||||
| 11270 | auto RightProb = FirstRight->Prob + W.DefaultProb / 2; | ||||||||
| 11271 | |||||||||
| 11272 | // Move LastLeft and FirstRight towards each other from opposite directions to | ||||||||
| 11273 | // find a partitioning of the clusters which balances the probability on both | ||||||||
| 11274 | // sides. If LeftProb and RightProb are equal, alternate which side is | ||||||||
| 11275 | // taken to ensure 0-probability nodes are distributed evenly. | ||||||||
| 11276 | unsigned I = 0; | ||||||||
| 11277 | while (LastLeft + 1 < FirstRight) { | ||||||||
| 11278 | if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1))) | ||||||||
| 11279 | LeftProb += (++LastLeft)->Prob; | ||||||||
| 11280 | else | ||||||||
| 11281 | RightProb += (--FirstRight)->Prob; | ||||||||
| 11282 | I++; | ||||||||
| 11283 | } | ||||||||
| 11284 | |||||||||
| 11285 | while (true) { | ||||||||
| 11286 | // Our binary search tree differs from a typical BST in that ours can have up | ||||||||
| 11287 | // to three values in each leaf. The pivot selection above doesn't take that | ||||||||
| 11288 | // into account, which means the tree might require more nodes and be less | ||||||||
| 11289 | // efficient. We compensate for this here. | ||||||||
| 11290 | |||||||||
| 11291 | unsigned NumLeft = LastLeft - W.FirstCluster + 1; | ||||||||
| 11292 | unsigned NumRight = W.LastCluster - FirstRight + 1; | ||||||||
| 11293 | |||||||||
| 11294 | if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) { | ||||||||
| 11295 | // If one side has less than 3 clusters, and the other has more than 3, | ||||||||
| 11296 | // consider taking a cluster from the other side. | ||||||||
| 11297 | |||||||||
| 11298 | if (NumLeft < NumRight) { | ||||||||
| 11299 | // Consider moving the first cluster on the right to the left side. | ||||||||
| 11300 | CaseCluster &CC = *FirstRight; | ||||||||
| 11301 | unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); | ||||||||
| 11302 | unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); | ||||||||
| 11303 | if (LeftSideRank <= RightSideRank) { | ||||||||
| 11304 | // Moving the cluster to the left does not demote it. | ||||||||
| 11305 | ++LastLeft; | ||||||||
| 11306 | ++FirstRight; | ||||||||
| 11307 | continue; | ||||||||
| 11308 | } | ||||||||
| 11309 | } else { | ||||||||
| 11310 | assert(NumRight < NumLeft)(static_cast <bool> (NumRight < NumLeft) ? void (0) : __assert_fail ("NumRight < NumLeft", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11310, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11311 | // Consider moving the last element on the left to the right side. | ||||||||
| 11312 | CaseCluster &CC = *LastLeft; | ||||||||
| 11313 | unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); | ||||||||
| 11314 | unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); | ||||||||
| 11315 | if (RightSideRank <= LeftSideRank) { | ||||||||
| 11316 | // Moving the cluster to the right does not demot it. | ||||||||
| 11317 | --LastLeft; | ||||||||
| 11318 | --FirstRight; | ||||||||
| 11319 | continue; | ||||||||
| 11320 | } | ||||||||
| 11321 | } | ||||||||
| 11322 | } | ||||||||
| 11323 | break; | ||||||||
| 11324 | } | ||||||||
| 11325 | |||||||||
| 11326 | assert(LastLeft + 1 == FirstRight)(static_cast <bool> (LastLeft + 1 == FirstRight) ? void (0) : __assert_fail ("LastLeft + 1 == FirstRight", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11326, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11327 | assert(LastLeft >= W.FirstCluster)(static_cast <bool> (LastLeft >= W.FirstCluster) ? void (0) : __assert_fail ("LastLeft >= W.FirstCluster", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11327, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11328 | assert(FirstRight <= W.LastCluster)(static_cast <bool> (FirstRight <= W.LastCluster) ? void (0) : __assert_fail ("FirstRight <= W.LastCluster", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11328, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11329 | |||||||||
| 11330 | // Use the first element on the right as pivot since we will make less-than | ||||||||
| 11331 | // comparisons against it. | ||||||||
| 11332 | CaseClusterIt PivotCluster = FirstRight; | ||||||||
| 11333 | assert(PivotCluster > W.FirstCluster)(static_cast <bool> (PivotCluster > W.FirstCluster) ? void (0) : __assert_fail ("PivotCluster > W.FirstCluster" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11333 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11334 | assert(PivotCluster <= W.LastCluster)(static_cast <bool> (PivotCluster <= W.LastCluster) ? void (0) : __assert_fail ("PivotCluster <= W.LastCluster" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11334 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11335 | |||||||||
| 11336 | CaseClusterIt FirstLeft = W.FirstCluster; | ||||||||
| 11337 | CaseClusterIt LastRight = W.LastCluster; | ||||||||
| 11338 | |||||||||
| 11339 | const ConstantInt *Pivot = PivotCluster->Low; | ||||||||
| 11340 | |||||||||
| 11341 | // New blocks will be inserted immediately after the current one. | ||||||||
| 11342 | MachineFunction::iterator BBI(W.MBB); | ||||||||
| 11343 | ++BBI; | ||||||||
| 11344 | |||||||||
| 11345 | // We will branch to the LHS if Value < Pivot. If LHS is a single cluster, | ||||||||
| 11346 | // we can branch to its destination directly if it's squeezed exactly in | ||||||||
| 11347 | // between the known lower bound and Pivot - 1. | ||||||||
| 11348 | MachineBasicBlock *LeftMBB; | ||||||||
| 11349 | if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range && | ||||||||
| 11350 | FirstLeft->Low == W.GE && | ||||||||
| 11351 | (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) { | ||||||||
| 11352 | LeftMBB = FirstLeft->MBB; | ||||||||
| 11353 | } else { | ||||||||
| 11354 | LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); | ||||||||
| 11355 | FuncInfo.MF->insert(BBI, LeftMBB); | ||||||||
| 11356 | WorkList.push_back( | ||||||||
| 11357 | {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2}); | ||||||||
| 11358 | // Put Cond in a virtual register to make it available from the new blocks. | ||||||||
| 11359 | ExportFromCurrentBlock(Cond); | ||||||||
| 11360 | } | ||||||||
| 11361 | |||||||||
| 11362 | // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a | ||||||||
| 11363 | // single cluster, RHS.Low == Pivot, and we can branch to its destination | ||||||||
| 11364 | // directly if RHS.High equals the current upper bound. | ||||||||
| 11365 | MachineBasicBlock *RightMBB; | ||||||||
| 11366 | if (FirstRight == LastRight && FirstRight->Kind == CC_Range && | ||||||||
| 11367 | W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) { | ||||||||
| 11368 | RightMBB = FirstRight->MBB; | ||||||||
| 11369 | } else { | ||||||||
| 11370 | RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); | ||||||||
| 11371 | FuncInfo.MF->insert(BBI, RightMBB); | ||||||||
| 11372 | WorkList.push_back( | ||||||||
| 11373 | {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2}); | ||||||||
| 11374 | // Put Cond in a virtual register to make it available from the new blocks. | ||||||||
| 11375 | ExportFromCurrentBlock(Cond); | ||||||||
| 11376 | } | ||||||||
| 11377 | |||||||||
| 11378 | // Create the CaseBlock record that will be used to lower the branch. | ||||||||
| 11379 | CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB, | ||||||||
| 11380 | getCurSDLoc(), LeftProb, RightProb); | ||||||||
| 11381 | |||||||||
| 11382 | if (W.MBB == SwitchMBB) | ||||||||
| 11383 | visitSwitchCase(CB, SwitchMBB); | ||||||||
| 11384 | else | ||||||||
| 11385 | SL->SwitchCases.push_back(CB); | ||||||||
| 11386 | } | ||||||||
| 11387 | |||||||||
| 11388 | // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb | ||||||||
| 11389 | // from the swith statement. | ||||||||
| 11390 | static BranchProbability scaleCaseProbality(BranchProbability CaseProb, | ||||||||
| 11391 | BranchProbability PeeledCaseProb) { | ||||||||
| 11392 | if (PeeledCaseProb == BranchProbability::getOne()) | ||||||||
| 11393 | return BranchProbability::getZero(); | ||||||||
| 11394 | BranchProbability SwitchProb = PeeledCaseProb.getCompl(); | ||||||||
| 11395 | |||||||||
| 11396 | uint32_t Numerator = CaseProb.getNumerator(); | ||||||||
| 11397 | uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator()); | ||||||||
| 11398 | return BranchProbability(Numerator, std::max(Numerator, Denominator)); | ||||||||
| 11399 | } | ||||||||
| 11400 | |||||||||
| 11401 | // Try to peel the top probability case if it exceeds the threshold. | ||||||||
| 11402 | // Return current MachineBasicBlock for the switch statement if the peeling | ||||||||
| 11403 | // does not occur. | ||||||||
| 11404 | // If the peeling is performed, return the newly created MachineBasicBlock | ||||||||
| 11405 | // for the peeled switch statement. Also update Clusters to remove the peeled | ||||||||
| 11406 | // case. PeeledCaseProb is the BranchProbability for the peeled case. | ||||||||
| 11407 | MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster( | ||||||||
| 11408 | const SwitchInst &SI, CaseClusterVector &Clusters, | ||||||||
| 11409 | BranchProbability &PeeledCaseProb) { | ||||||||
| 11410 | MachineBasicBlock *SwitchMBB = FuncInfo.MBB; | ||||||||
| 11411 | // Don't perform if there is only one cluster or optimizing for size. | ||||||||
| 11412 | if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 || | ||||||||
| 11413 | TM.getOptLevel() == CodeGenOpt::None || | ||||||||
| 11414 | SwitchMBB->getParent()->getFunction().hasMinSize()) | ||||||||
| 11415 | return SwitchMBB; | ||||||||
| 11416 | |||||||||
| 11417 | BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100); | ||||||||
| 11418 | unsigned PeeledCaseIndex = 0; | ||||||||
| 11419 | bool SwitchPeeled = false; | ||||||||
| 11420 | for (unsigned Index = 0; Index < Clusters.size(); ++Index) { | ||||||||
| 11421 | CaseCluster &CC = Clusters[Index]; | ||||||||
| 11422 | if (CC.Prob < TopCaseProb) | ||||||||
| 11423 | continue; | ||||||||
| 11424 | TopCaseProb = CC.Prob; | ||||||||
| 11425 | PeeledCaseIndex = Index; | ||||||||
| 11426 | SwitchPeeled = true; | ||||||||
| 11427 | } | ||||||||
| 11428 | if (!SwitchPeeled) | ||||||||
| 11429 | return SwitchMBB; | ||||||||
| 11430 | |||||||||
| 11431 | LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: " << TopCaseProb << "\n"; } } while (false) | ||||||||
| 11432 | << TopCaseProb << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: " << TopCaseProb << "\n"; } } while (false); | ||||||||
| 11433 | |||||||||
| 11434 | // Record the MBB for the peeled switch statement. | ||||||||
| 11435 | MachineFunction::iterator BBI(SwitchMBB); | ||||||||
| 11436 | ++BBI; | ||||||||
| 11437 | MachineBasicBlock *PeeledSwitchMBB = | ||||||||
| 11438 | FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock()); | ||||||||
| 11439 | FuncInfo.MF->insert(BBI, PeeledSwitchMBB); | ||||||||
| 11440 | |||||||||
| 11441 | ExportFromCurrentBlock(SI.getCondition()); | ||||||||
| 11442 | auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex; | ||||||||
| 11443 | SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt, | ||||||||
| 11444 | nullptr, nullptr, TopCaseProb.getCompl()}; | ||||||||
| 11445 | lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB); | ||||||||
| 11446 | |||||||||
| 11447 | Clusters.erase(PeeledCaseIt); | ||||||||
| 11448 | for (CaseCluster &CC : Clusters) { | ||||||||
| 11449 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: " << CC.Prob << "\n"; } } while (false) | ||||||||
| 11450 | dbgs() << "Scale the probablity for one cluster, before scaling: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: " << CC.Prob << "\n"; } } while (false) | ||||||||
| 11451 | << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: " << CC.Prob << "\n"; } } while (false); | ||||||||
| 11452 | CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb); | ||||||||
| 11453 | LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "After scaling: " << CC.Prob << "\n"; } } while (false); | ||||||||
| 11454 | } | ||||||||
| 11455 | PeeledCaseProb = TopCaseProb; | ||||||||
| 11456 | return PeeledSwitchMBB; | ||||||||
| 11457 | } | ||||||||
| 11458 | |||||||||
| 11459 | void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { | ||||||||
| 11460 | // Extract cases from the switch. | ||||||||
| 11461 | BranchProbabilityInfo *BPI = FuncInfo.BPI; | ||||||||
| 11462 | CaseClusterVector Clusters; | ||||||||
| 11463 | Clusters.reserve(SI.getNumCases()); | ||||||||
| 11464 | for (auto I : SI.cases()) { | ||||||||
| 11465 | MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()]; | ||||||||
| 11466 | const ConstantInt *CaseVal = I.getCaseValue(); | ||||||||
| 11467 | BranchProbability Prob = | ||||||||
| 11468 | BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex()) | ||||||||
| 11469 | : BranchProbability(1, SI.getNumCases() + 1); | ||||||||
| 11470 | Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob)); | ||||||||
| 11471 | } | ||||||||
| 11472 | |||||||||
| 11473 | MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()]; | ||||||||
| 11474 | |||||||||
| 11475 | // Cluster adjacent cases with the same destination. We do this at all | ||||||||
| 11476 | // optimization levels because it's cheap to do and will make codegen faster | ||||||||
| 11477 | // if there are many clusters. | ||||||||
| 11478 | sortAndRangeify(Clusters); | ||||||||
| 11479 | |||||||||
| 11480 | // The branch probablity of the peeled case. | ||||||||
| 11481 | BranchProbability PeeledCaseProb = BranchProbability::getZero(); | ||||||||
| 11482 | MachineBasicBlock *PeeledSwitchMBB = | ||||||||
| 11483 | peelDominantCaseCluster(SI, Clusters, PeeledCaseProb); | ||||||||
| 11484 | |||||||||
| 11485 | // If there is only the default destination, jump there directly. | ||||||||
| 11486 | MachineBasicBlock *SwitchMBB = FuncInfo.MBB; | ||||||||
| 11487 | if (Clusters.empty()) { | ||||||||
| 11488 | assert(PeeledSwitchMBB == SwitchMBB)(static_cast <bool> (PeeledSwitchMBB == SwitchMBB) ? void (0) : __assert_fail ("PeeledSwitchMBB == SwitchMBB", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11488, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11489 | SwitchMBB->addSuccessor(DefaultMBB); | ||||||||
| 11490 | if (DefaultMBB != NextBlock(SwitchMBB)) { | ||||||||
| 11491 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, | ||||||||
| 11492 | getControlRoot(), DAG.getBasicBlock(DefaultMBB))); | ||||||||
| 11493 | } | ||||||||
| 11494 | return; | ||||||||
| 11495 | } | ||||||||
| 11496 | |||||||||
| 11497 | SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI()); | ||||||||
| 11498 | SL->findBitTestClusters(Clusters, &SI); | ||||||||
| 11499 | |||||||||
| 11500 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11501 | dbgs() << "Case clusters: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11502 | for (const CaseCluster &C : Clusters) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11503 | if (C.Kind == CC_JumpTable)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11504 | dbgs() << "JT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11505 | if (C.Kind == CC_BitTests)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11506 | dbgs() << "BT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11507 | |||||||||
| 11508 | C.Low->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11509 | if (C.Low != C.High) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11510 | dbgs() << '-';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11511 | C.High->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11512 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11513 | dbgs() << ' ';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11514 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11515 | dbgs() << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ) | ||||||||
| 11516 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() << "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low ->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs () << '-'; C.High->getValue().print(dbgs(), true); } dbgs() << ' '; } dbgs() << '\n'; }; } } while (false ); | ||||||||
| 11517 | |||||||||
| 11518 | assert(!Clusters.empty())(static_cast <bool> (!Clusters.empty()) ? void (0) : __assert_fail ("!Clusters.empty()", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp" , 11518, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11519 | SwitchWorkList WorkList; | ||||||||
| 11520 | CaseClusterIt First = Clusters.begin(); | ||||||||
| 11521 | CaseClusterIt Last = Clusters.end() - 1; | ||||||||
| 11522 | auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB); | ||||||||
| 11523 | // Scale the branchprobability for DefaultMBB if the peel occurs and | ||||||||
| 11524 | // DefaultMBB is not replaced. | ||||||||
| 11525 | if (PeeledCaseProb != BranchProbability::getZero() && | ||||||||
| 11526 | DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()]) | ||||||||
| 11527 | DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb); | ||||||||
| 11528 | WorkList.push_back( | ||||||||
| 11529 | {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb}); | ||||||||
| 11530 | |||||||||
| 11531 | while (!WorkList.empty()) { | ||||||||
| 11532 | SwitchWorkListItem W = WorkList.pop_back_val(); | ||||||||
| 11533 | unsigned NumClusters = W.LastCluster - W.FirstCluster + 1; | ||||||||
| 11534 | |||||||||
| 11535 | if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None && | ||||||||
| 11536 | !DefaultMBB->getParent()->getFunction().hasMinSize()) { | ||||||||
| 11537 | // For optimized builds, lower large range as a balanced binary tree. | ||||||||
| 11538 | splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB); | ||||||||
| 11539 | continue; | ||||||||
| 11540 | } | ||||||||
| 11541 | |||||||||
| 11542 | lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB); | ||||||||
| 11543 | } | ||||||||
| 11544 | } | ||||||||
| 11545 | |||||||||
| 11546 | void SelectionDAGBuilder::visitStepVector(const CallInst &I) { | ||||||||
| 11547 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 11548 | auto DL = getCurSDLoc(); | ||||||||
| 11549 | EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 11550 | setValue(&I, DAG.getStepVector(DL, ResultVT)); | ||||||||
| 11551 | } | ||||||||
| 11552 | |||||||||
| 11553 | void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) { | ||||||||
| 11554 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 11555 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 11556 | |||||||||
| 11557 | SDLoc DL = getCurSDLoc(); | ||||||||
| 11558 | SDValue V = getValue(I.getOperand(0)); | ||||||||
| 11559 | assert(VT == V.getValueType() && "Malformed vector.reverse!")(static_cast <bool> (VT == V.getValueType() && "Malformed vector.reverse!" ) ? void (0) : __assert_fail ("VT == V.getValueType() && \"Malformed vector.reverse!\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11559 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11560 | |||||||||
| 11561 | if (VT.isScalableVector()) { | ||||||||
| 11562 | setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V)); | ||||||||
| 11563 | return; | ||||||||
| 11564 | } | ||||||||
| 11565 | |||||||||
| 11566 | // Use VECTOR_SHUFFLE for the fixed-length vector | ||||||||
| 11567 | // to maintain existing behavior. | ||||||||
| 11568 | SmallVector<int, 8> Mask; | ||||||||
| 11569 | unsigned NumElts = VT.getVectorMinNumElements(); | ||||||||
| 11570 | for (unsigned i = 0; i != NumElts; ++i) | ||||||||
| 11571 | Mask.push_back(NumElts - 1 - i); | ||||||||
| 11572 | |||||||||
| 11573 | setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask)); | ||||||||
| 11574 | } | ||||||||
| 11575 | |||||||||
| 11576 | void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) { | ||||||||
| 11577 | auto DL = getCurSDLoc(); | ||||||||
| 11578 | SDValue InVec = getValue(I.getOperand(0)); | ||||||||
| 11579 | EVT OutVT = | ||||||||
| 11580 | InVec.getValueType().getHalfNumVectorElementsVT(*DAG.getContext()); | ||||||||
| 11581 | |||||||||
| 11582 | unsigned OutNumElts = OutVT.getVectorMinNumElements(); | ||||||||
| 11583 | |||||||||
| 11584 | // ISD Node needs the input vectors split into two equal parts | ||||||||
| 11585 | SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, | ||||||||
| 11586 | DAG.getVectorIdxConstant(0, DL)); | ||||||||
| 11587 | SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, | ||||||||
| 11588 | DAG.getVectorIdxConstant(OutNumElts, DL)); | ||||||||
| 11589 | |||||||||
| 11590 | // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing | ||||||||
| 11591 | // legalisation and combines. | ||||||||
| 11592 | if (OutVT.isFixedLengthVector()) { | ||||||||
| 11593 | SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi, | ||||||||
| 11594 | createStrideMask(0, 2, OutNumElts)); | ||||||||
| 11595 | SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi, | ||||||||
| 11596 | createStrideMask(1, 2, OutNumElts)); | ||||||||
| 11597 | SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc()); | ||||||||
| 11598 | setValue(&I, Res); | ||||||||
| 11599 | return; | ||||||||
| 11600 | } | ||||||||
| 11601 | |||||||||
| 11602 | SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, | ||||||||
| 11603 | DAG.getVTList(OutVT, OutVT), Lo, Hi); | ||||||||
| 11604 | setValue(&I, Res); | ||||||||
| 11605 | } | ||||||||
| 11606 | |||||||||
| 11607 | void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) { | ||||||||
| 11608 | auto DL = getCurSDLoc(); | ||||||||
| 11609 | EVT InVT = getValue(I.getOperand(0)).getValueType(); | ||||||||
| 11610 | SDValue InVec0 = getValue(I.getOperand(0)); | ||||||||
| 11611 | SDValue InVec1 = getValue(I.getOperand(1)); | ||||||||
| 11612 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 11613 | EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 11614 | |||||||||
| 11615 | // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing | ||||||||
| 11616 | // legalisation and combines. | ||||||||
| 11617 | if (OutVT.isFixedLengthVector()) { | ||||||||
| 11618 | unsigned NumElts = InVT.getVectorMinNumElements(); | ||||||||
| 11619 | SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1); | ||||||||
| 11620 | setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT), | ||||||||
| 11621 | createInterleaveMask(NumElts, 2))); | ||||||||
| 11622 | return; | ||||||||
| 11623 | } | ||||||||
| 11624 | |||||||||
| 11625 | SDValue Res = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, | ||||||||
| 11626 | DAG.getVTList(InVT, InVT), InVec0, InVec1); | ||||||||
| 11627 | Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0), | ||||||||
| 11628 | Res.getValue(1)); | ||||||||
| 11629 | setValue(&I, Res); | ||||||||
| 11630 | } | ||||||||
| 11631 | |||||||||
| 11632 | void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) { | ||||||||
| 11633 | SmallVector<EVT, 4> ValueVTs; | ||||||||
| 11634 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), | ||||||||
| 11635 | ValueVTs); | ||||||||
| 11636 | unsigned NumValues = ValueVTs.size(); | ||||||||
| 11637 | if (NumValues == 0) return; | ||||||||
| 11638 | |||||||||
| 11639 | SmallVector<SDValue, 4> Values(NumValues); | ||||||||
| 11640 | SDValue Op = getValue(I.getOperand(0)); | ||||||||
| 11641 | |||||||||
| 11642 | for (unsigned i = 0; i != NumValues; ++i) | ||||||||
| 11643 | Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i], | ||||||||
| 11644 | SDValue(Op.getNode(), Op.getResNo() + i)); | ||||||||
| 11645 | |||||||||
| 11646 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||
| 11647 | DAG.getVTList(ValueVTs), Values)); | ||||||||
| 11648 | } | ||||||||
| 11649 | |||||||||
| 11650 | void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) { | ||||||||
| 11651 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 11652 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); | ||||||||
| 11653 | |||||||||
| 11654 | SDLoc DL = getCurSDLoc(); | ||||||||
| 11655 | SDValue V1 = getValue(I.getOperand(0)); | ||||||||
| 11656 | SDValue V2 = getValue(I.getOperand(1)); | ||||||||
| 11657 | int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue(); | ||||||||
| 11658 | |||||||||
| 11659 | // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node. | ||||||||
| 11660 | if (VT.isScalableVector()) { | ||||||||
| 11661 | MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout()); | ||||||||
| 11662 | setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2, | ||||||||
| 11663 | DAG.getConstant(Imm, DL, IdxVT))); | ||||||||
| 11664 | return; | ||||||||
| 11665 | } | ||||||||
| 11666 | |||||||||
| 11667 | unsigned NumElts = VT.getVectorNumElements(); | ||||||||
| 11668 | |||||||||
| 11669 | uint64_t Idx = (NumElts + Imm) % NumElts; | ||||||||
| 11670 | |||||||||
| 11671 | // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors. | ||||||||
| 11672 | SmallVector<int, 8> Mask; | ||||||||
| 11673 | for (unsigned i = 0; i < NumElts; ++i) | ||||||||
| 11674 | Mask.push_back(Idx + i); | ||||||||
| 11675 | setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask)); | ||||||||
| 11676 | } | ||||||||
| 11677 | |||||||||
| 11678 | // Consider the following MIR after SelectionDAG, which produces output in | ||||||||
| 11679 | // phyregs in the first case or virtregs in the second case. | ||||||||
| 11680 | // | ||||||||
| 11681 | // INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx | ||||||||
| 11682 | // %5:gr32 = COPY $ebx | ||||||||
| 11683 | // %6:gr32 = COPY $edx | ||||||||
| 11684 | // %1:gr32 = COPY %6:gr32 | ||||||||
| 11685 | // %0:gr32 = COPY %5:gr32 | ||||||||
| 11686 | // | ||||||||
| 11687 | // INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32 | ||||||||
| 11688 | // %1:gr32 = COPY %6:gr32 | ||||||||
| 11689 | // %0:gr32 = COPY %5:gr32 | ||||||||
| 11690 | // | ||||||||
| 11691 | // Given %0, we'd like to return $ebx in the first case and %5 in the second. | ||||||||
| 11692 | // Given %1, we'd like to return $edx in the first case and %6 in the second. | ||||||||
| 11693 | // | ||||||||
| 11694 | // If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap | ||||||||
| 11695 | // to a single virtreg (such as %0). The remaining outputs monotonically | ||||||||
| 11696 | // increase in virtreg number from there. If a callbr has no outputs, then it | ||||||||
| 11697 | // should not have a corresponding callbr landingpad; in fact, the callbr | ||||||||
| 11698 | // landingpad would not even be able to refer to such a callbr. | ||||||||
| 11699 | static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) { | ||||||||
| 11700 | MachineInstr *MI = MRI.def_begin(Reg)->getParent(); | ||||||||
| 11701 | // There is definitely at least one copy. | ||||||||
| 11702 | assert(MI->getOpcode() == TargetOpcode::COPY &&(static_cast <bool> (MI->getOpcode() == TargetOpcode ::COPY && "start of copy chain MUST be COPY") ? void ( 0) : __assert_fail ("MI->getOpcode() == TargetOpcode::COPY && \"start of copy chain MUST be COPY\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11703 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 11703 | "start of copy chain MUST be COPY")(static_cast <bool> (MI->getOpcode() == TargetOpcode ::COPY && "start of copy chain MUST be COPY") ? void ( 0) : __assert_fail ("MI->getOpcode() == TargetOpcode::COPY && \"start of copy chain MUST be COPY\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11703 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11704 | Reg = MI->getOperand(1).getReg(); | ||||||||
| 11705 | MI = MRI.def_begin(Reg)->getParent(); | ||||||||
| 11706 | // There may be an optional second copy. | ||||||||
| 11707 | if (MI->getOpcode() == TargetOpcode::COPY) { | ||||||||
| 11708 | assert(Reg.isVirtual() && "expected COPY of virtual register")(static_cast <bool> (Reg.isVirtual() && "expected COPY of virtual register" ) ? void (0) : __assert_fail ("Reg.isVirtual() && \"expected COPY of virtual register\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11708 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11709 | Reg = MI->getOperand(1).getReg(); | ||||||||
| 11710 | assert(Reg.isPhysical() && "expected COPY of physical register")(static_cast <bool> (Reg.isPhysical() && "expected COPY of physical register" ) ? void (0) : __assert_fail ("Reg.isPhysical() && \"expected COPY of physical register\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11710 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11711 | MI = MRI.def_begin(Reg)->getParent(); | ||||||||
| 11712 | } | ||||||||
| 11713 | // The start of the chain must be an INLINEASM_BR. | ||||||||
| 11714 | assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&(static_cast <bool> (MI->getOpcode() == TargetOpcode ::INLINEASM_BR && "end of copy chain MUST be INLINEASM_BR" ) ? void (0) : __assert_fail ("MI->getOpcode() == TargetOpcode::INLINEASM_BR && \"end of copy chain MUST be INLINEASM_BR\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11715 , __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 11715 | "end of copy chain MUST be INLINEASM_BR")(static_cast <bool> (MI->getOpcode() == TargetOpcode ::INLINEASM_BR && "end of copy chain MUST be INLINEASM_BR" ) ? void (0) : __assert_fail ("MI->getOpcode() == TargetOpcode::INLINEASM_BR && \"end of copy chain MUST be INLINEASM_BR\"" , "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11715 , __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 11716 | return Reg; | ||||||||
| 11717 | } | ||||||||
| 11718 | |||||||||
| 11719 | // We must do this walk rather than the simpler | ||||||||
| 11720 | // setValue(&I, getCopyFromRegs(CBR, CBR->getType())); | ||||||||
| 11721 | // otherwise we will end up with copies of virtregs only valid along direct | ||||||||
| 11722 | // edges. | ||||||||
| 11723 | void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) { | ||||||||
| 11724 | SmallVector<EVT, 8> ResultVTs; | ||||||||
| 11725 | SmallVector<SDValue, 8> ResultValues; | ||||||||
| 11726 | const auto *CBR = | ||||||||
| 11727 | cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator()); | ||||||||
| 11728 | |||||||||
| 11729 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); | ||||||||
| 11730 | const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo(); | ||||||||
| 11731 | MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); | ||||||||
| 11732 | |||||||||
| 11733 | unsigned InitialDef = FuncInfo.ValueMap[CBR]; | ||||||||
| 11734 | SDValue Chain = DAG.getRoot(); | ||||||||
| 11735 | |||||||||
| 11736 | // Re-parse the asm constraints string. | ||||||||
| 11737 | TargetLowering::AsmOperandInfoVector TargetConstraints = | ||||||||
| 11738 | TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR); | ||||||||
| 11739 | for (auto &T : TargetConstraints) { | ||||||||
| 11740 | SDISelAsmOperandInfo OpInfo(T); | ||||||||
| 11741 | if (OpInfo.Type != InlineAsm::isOutput) | ||||||||
| 11742 | continue; | ||||||||
| 11743 | |||||||||
| 11744 | // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the | ||||||||
| 11745 | // individual constraint. | ||||||||
| 11746 | TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); | ||||||||
| 11747 | |||||||||
| 11748 | switch (OpInfo.ConstraintType) { | ||||||||
| 11749 | case TargetLowering::C_Register: | ||||||||
| 11750 | case TargetLowering::C_RegisterClass: { | ||||||||
| 11751 | // Fill in OpInfo.AssignedRegs.Regs. | ||||||||
| 11752 | getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo); | ||||||||
| 11753 | |||||||||
| 11754 | // getRegistersForValue may produce 1 to many registers based on whether | ||||||||
| 11755 | // the OpInfo.ConstraintVT is legal on the target or not. | ||||||||
| 11756 | for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) { | ||||||||
| 11757 | Register OriginalDef = FollowCopyChain(MRI, InitialDef++); | ||||||||
| 11758 | if (Register::isPhysicalRegister(OriginalDef)) | ||||||||
| 11759 | FuncInfo.MBB->addLiveIn(OriginalDef); | ||||||||
| 11760 | // Update the assigned registers to use the original defs. | ||||||||
| 11761 | OpInfo.AssignedRegs.Regs[i] = OriginalDef; | ||||||||
| 11762 | } | ||||||||
| 11763 | |||||||||
| 11764 | SDValue V = OpInfo.AssignedRegs.getCopyFromRegs( | ||||||||
| 11765 | DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR); | ||||||||
| 11766 | ResultValues.push_back(V); | ||||||||
| 11767 | ResultVTs.push_back(OpInfo.ConstraintVT); | ||||||||
| 11768 | break; | ||||||||
| 11769 | } | ||||||||
| 11770 | case TargetLowering::C_Other: { | ||||||||
| 11771 | SDValue Flag; | ||||||||
| 11772 | SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(), | ||||||||
| 11773 | OpInfo, DAG); | ||||||||
| 11774 | ++InitialDef; | ||||||||
| 11775 | ResultValues.push_back(V); | ||||||||
| 11776 | ResultVTs.push_back(OpInfo.ConstraintVT); | ||||||||
| 11777 | break; | ||||||||
| 11778 | } | ||||||||
| 11779 | default: | ||||||||
| 11780 | break; | ||||||||
| 11781 | } | ||||||||
| 11782 | } | ||||||||
| 11783 | SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), | ||||||||
| 11784 | DAG.getVTList(ResultVTs), ResultValues); | ||||||||
| 11785 | setValue(&I, V); | ||||||||
| 11786 | } |
| 1 | //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // This file declares the SDNode class and derived classes, which are used to | |||
| 10 | // represent the nodes and operations present in a SelectionDAG. These nodes | |||
| 11 | // and operations are machine code level operations, with some similarities to | |||
| 12 | // the GCC RTL representation. | |||
| 13 | // | |||
| 14 | // Clients should include the SelectionDAG.h file instead of this file directly. | |||
| 15 | // | |||
| 16 | //===----------------------------------------------------------------------===// | |||
| 17 | ||||
| 18 | #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
| 19 | #define LLVM_CODEGEN_SELECTIONDAGNODES_H | |||
| 20 | ||||
| 21 | #include "llvm/ADT/APFloat.h" | |||
| 22 | #include "llvm/ADT/ArrayRef.h" | |||
| 23 | #include "llvm/ADT/BitVector.h" | |||
| 24 | #include "llvm/ADT/FoldingSet.h" | |||
| 25 | #include "llvm/ADT/GraphTraits.h" | |||
| 26 | #include "llvm/ADT/SmallPtrSet.h" | |||
| 27 | #include "llvm/ADT/SmallVector.h" | |||
| 28 | #include "llvm/ADT/ilist_node.h" | |||
| 29 | #include "llvm/ADT/iterator.h" | |||
| 30 | #include "llvm/ADT/iterator_range.h" | |||
| 31 | #include "llvm/CodeGen/ISDOpcodes.h" | |||
| 32 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
| 33 | #include "llvm/CodeGen/MachineValueType.h" | |||
| 34 | #include "llvm/CodeGen/Register.h" | |||
| 35 | #include "llvm/CodeGen/ValueTypes.h" | |||
| 36 | #include "llvm/IR/Constants.h" | |||
| 37 | #include "llvm/IR/DebugLoc.h" | |||
| 38 | #include "llvm/IR/Instruction.h" | |||
| 39 | #include "llvm/IR/Instructions.h" | |||
| 40 | #include "llvm/IR/Metadata.h" | |||
| 41 | #include "llvm/IR/Operator.h" | |||
| 42 | #include "llvm/Support/AlignOf.h" | |||
| 43 | #include "llvm/Support/AtomicOrdering.h" | |||
| 44 | #include "llvm/Support/Casting.h" | |||
| 45 | #include "llvm/Support/ErrorHandling.h" | |||
| 46 | #include "llvm/Support/TypeSize.h" | |||
| 47 | #include <algorithm> | |||
| 48 | #include <cassert> | |||
| 49 | #include <climits> | |||
| 50 | #include <cstddef> | |||
| 51 | #include <cstdint> | |||
| 52 | #include <cstring> | |||
| 53 | #include <iterator> | |||
| 54 | #include <string> | |||
| 55 | #include <tuple> | |||
| 56 | #include <utility> | |||
| 57 | ||||
| 58 | namespace llvm { | |||
| 59 | ||||
| 60 | class APInt; | |||
| 61 | class Constant; | |||
| 62 | class GlobalValue; | |||
| 63 | class MachineBasicBlock; | |||
| 64 | class MachineConstantPoolValue; | |||
| 65 | class MCSymbol; | |||
| 66 | class raw_ostream; | |||
| 67 | class SDNode; | |||
| 68 | class SelectionDAG; | |||
| 69 | class Type; | |||
| 70 | class Value; | |||
| 71 | ||||
| 72 | void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, | |||
| 73 | bool force = false); | |||
| 74 | ||||
| 75 | /// This represents a list of ValueType's that has been intern'd by | |||
| 76 | /// a SelectionDAG. Instances of this simple value class are returned by | |||
| 77 | /// SelectionDAG::getVTList(...). | |||
| 78 | /// | |||
| 79 | struct SDVTList { | |||
| 80 | const EVT *VTs; | |||
| 81 | unsigned int NumVTs; | |||
| 82 | }; | |||
| 83 | ||||
| 84 | namespace ISD { | |||
| 85 | ||||
| 86 | /// Node predicates | |||
| 87 | ||||
| 88 | /// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the | |||
| 89 | /// same constant or undefined, return true and return the constant value in | |||
| 90 | /// \p SplatValue. | |||
| 91 | bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); | |||
| 92 | ||||
| 93 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
| 94 | /// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to | |||
| 95 | /// true, it only checks BUILD_VECTOR. | |||
| 96 | bool isConstantSplatVectorAllOnes(const SDNode *N, | |||
| 97 | bool BuildVectorOnly = false); | |||
| 98 | ||||
| 99 | /// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where | |||
| 100 | /// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it | |||
| 101 | /// only checks BUILD_VECTOR. | |||
| 102 | bool isConstantSplatVectorAllZeros(const SDNode *N, | |||
| 103 | bool BuildVectorOnly = false); | |||
| 104 | ||||
| 105 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
| 106 | /// elements are ~0 or undef. | |||
| 107 | bool isBuildVectorAllOnes(const SDNode *N); | |||
| 108 | ||||
| 109 | /// Return true if the specified node is a BUILD_VECTOR where all of the | |||
| 110 | /// elements are 0 or undef. | |||
| 111 | bool isBuildVectorAllZeros(const SDNode *N); | |||
| 112 | ||||
| 113 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
| 114 | /// ConstantSDNode or undef. | |||
| 115 | bool isBuildVectorOfConstantSDNodes(const SDNode *N); | |||
| 116 | ||||
| 117 | /// Return true if the specified node is a BUILD_VECTOR node of all | |||
| 118 | /// ConstantFPSDNode or undef. | |||
| 119 | bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); | |||
| 120 | ||||
| 121 | /// Returns true if the specified node is a vector where all elements can | |||
| 122 | /// be truncated to the specified element size without a loss in meaning. | |||
| 123 | bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed); | |||
| 124 | ||||
| 125 | /// Return true if the node has at least one operand and all operands of the | |||
| 126 | /// specified node are ISD::UNDEF. | |||
| 127 | bool allOperandsUndef(const SDNode *N); | |||
| 128 | ||||
| 129 | /// Return true if the specified node is FREEZE(UNDEF). | |||
| 130 | bool isFreezeUndef(const SDNode *N); | |||
| 131 | ||||
| 132 | } // end namespace ISD | |||
| 133 | ||||
| 134 | //===----------------------------------------------------------------------===// | |||
| 135 | /// Unlike LLVM values, Selection DAG nodes may return multiple | |||
| 136 | /// values as the result of a computation. Many nodes return multiple values, | |||
| 137 | /// from loads (which define a token and a return value) to ADDC (which returns | |||
| 138 | /// a result and a carry value), to calls (which may return an arbitrary number | |||
| 139 | /// of values). | |||
| 140 | /// | |||
| 141 | /// As such, each use of a SelectionDAG computation must indicate the node that | |||
| 142 | /// computes it as well as which return value to use from that node. This pair | |||
| 143 | /// of information is represented with the SDValue value type. | |||
| 144 | /// | |||
| 145 | class SDValue { | |||
| 146 | friend struct DenseMapInfo<SDValue>; | |||
| 147 | ||||
| 148 | SDNode *Node = nullptr; // The node defining the value we are using. | |||
| 149 | unsigned ResNo = 0; // Which return value of the node we are using. | |||
| 150 | ||||
| 151 | public: | |||
| 152 | SDValue() = default; | |||
| 153 | SDValue(SDNode *node, unsigned resno); | |||
| 154 | ||||
| 155 | /// get the index which selects a specific result in the SDNode | |||
| 156 | unsigned getResNo() const { return ResNo; } | |||
| 157 | ||||
| 158 | /// get the SDNode which holds the desired result | |||
| 159 | SDNode *getNode() const { return Node; } | |||
| 160 | ||||
| 161 | /// set the SDNode | |||
| 162 | void setNode(SDNode *N) { Node = N; } | |||
| 163 | ||||
| 164 | inline SDNode *operator->() const { return Node; } | |||
| 165 | ||||
| 166 | bool operator==(const SDValue &O) const { | |||
| 167 | return Node == O.Node && ResNo == O.ResNo; | |||
| 168 | } | |||
| 169 | bool operator!=(const SDValue &O) const { | |||
| 170 | return !operator==(O); | |||
| 171 | } | |||
| 172 | bool operator<(const SDValue &O) const { | |||
| 173 | return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); | |||
| 174 | } | |||
| 175 | explicit operator bool() const { | |||
| 176 | return Node != nullptr; | |||
| 177 | } | |||
| 178 | ||||
| 179 | SDValue getValue(unsigned R) const { | |||
| 180 | return SDValue(Node, R); | |||
| 181 | } | |||
| 182 | ||||
| 183 | /// Return true if this node is an operand of N. | |||
| 184 | bool isOperandOf(const SDNode *N) const; | |||
| 185 | ||||
| 186 | /// Return the ValueType of the referenced return value. | |||
| 187 | inline EVT getValueType() const; | |||
| 188 | ||||
| 189 | /// Return the simple ValueType of the referenced return value. | |||
| 190 | MVT getSimpleValueType() const { | |||
| 191 | return getValueType().getSimpleVT(); | |||
| 192 | } | |||
| 193 | ||||
| 194 | /// Returns the size of the value in bits. | |||
| 195 | /// | |||
| 196 | /// If the value type is a scalable vector type, the scalable property will | |||
| 197 | /// be set and the runtime size will be a positive integer multiple of the | |||
| 198 | /// base size. | |||
| 199 | TypeSize getValueSizeInBits() const { | |||
| 200 | return getValueType().getSizeInBits(); | |||
| 201 | } | |||
| 202 | ||||
| 203 | uint64_t getScalarValueSizeInBits() const { | |||
| 204 | return getValueType().getScalarType().getFixedSizeInBits(); | |||
| 205 | } | |||
| 206 | ||||
| 207 | // Forwarding methods - These forward to the corresponding methods in SDNode. | |||
| 208 | inline unsigned getOpcode() const; | |||
| 209 | inline unsigned getNumOperands() const; | |||
| 210 | inline const SDValue &getOperand(unsigned i) const; | |||
| 211 | inline uint64_t getConstantOperandVal(unsigned i) const; | |||
| 212 | inline const APInt &getConstantOperandAPInt(unsigned i) const; | |||
| 213 | inline bool isTargetMemoryOpcode() const; | |||
| 214 | inline bool isTargetOpcode() const; | |||
| 215 | inline bool isMachineOpcode() const; | |||
| 216 | inline bool isUndef() const; | |||
| 217 | inline unsigned getMachineOpcode() const; | |||
| 218 | inline const DebugLoc &getDebugLoc() const; | |||
| 219 | inline void dump() const; | |||
| 220 | inline void dump(const SelectionDAG *G) const; | |||
| 221 | inline void dumpr() const; | |||
| 222 | inline void dumpr(const SelectionDAG *G) const; | |||
| 223 | ||||
| 224 | /// Return true if this operand (which must be a chain) reaches the | |||
| 225 | /// specified operand without crossing any side-effecting instructions. | |||
| 226 | /// In practice, this looks through token factors and non-volatile loads. | |||
| 227 | /// In order to remain efficient, this only | |||
| 228 | /// looks a couple of nodes in, it does not do an exhaustive search. | |||
| 229 | bool reachesChainWithoutSideEffects(SDValue Dest, | |||
| 230 | unsigned Depth = 2) const; | |||
| 231 | ||||
| 232 | /// Return true if there are no nodes using value ResNo of Node. | |||
| 233 | inline bool use_empty() const; | |||
| 234 | ||||
| 235 | /// Return true if there is exactly one node using value ResNo of Node. | |||
| 236 | inline bool hasOneUse() const; | |||
| 237 | }; | |||
| 238 | ||||
| 239 | template<> struct DenseMapInfo<SDValue> { | |||
| 240 | static inline SDValue getEmptyKey() { | |||
| 241 | SDValue V; | |||
| 242 | V.ResNo = -1U; | |||
| 243 | return V; | |||
| 244 | } | |||
| 245 | ||||
| 246 | static inline SDValue getTombstoneKey() { | |||
| 247 | SDValue V; | |||
| 248 | V.ResNo = -2U; | |||
| 249 | return V; | |||
| 250 | } | |||
| 251 | ||||
| 252 | static unsigned getHashValue(const SDValue &Val) { | |||
| 253 | return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ | |||
| 254 | (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo(); | |||
| 255 | } | |||
| 256 | ||||
| 257 | static bool isEqual(const SDValue &LHS, const SDValue &RHS) { | |||
| 258 | return LHS == RHS; | |||
| 259 | } | |||
| 260 | }; | |||
| 261 | ||||
| 262 | /// Allow casting operators to work directly on | |||
| 263 | /// SDValues as if they were SDNode*'s. | |||
| 264 | template<> struct simplify_type<SDValue> { | |||
| 265 | using SimpleType = SDNode *; | |||
| 266 | ||||
| 267 | static SimpleType getSimplifiedValue(SDValue &Val) { | |||
| 268 | return Val.getNode(); | |||
| 269 | } | |||
| 270 | }; | |||
| 271 | template<> struct simplify_type<const SDValue> { | |||
| 272 | using SimpleType = /*const*/ SDNode *; | |||
| 273 | ||||
| 274 | static SimpleType getSimplifiedValue(const SDValue &Val) { | |||
| 275 | return Val.getNode(); | |||
| 276 | } | |||
| 277 | }; | |||
| 278 | ||||
| 279 | /// Represents a use of a SDNode. This class holds an SDValue, | |||
| 280 | /// which records the SDNode being used and the result number, a | |||
| 281 | /// pointer to the SDNode using the value, and Next and Prev pointers, | |||
| 282 | /// which link together all the uses of an SDNode. | |||
| 283 | /// | |||
| 284 | class SDUse { | |||
| 285 | /// Val - The value being used. | |||
| 286 | SDValue Val; | |||
| 287 | /// User - The user of this value. | |||
| 288 | SDNode *User = nullptr; | |||
| 289 | /// Prev, Next - Pointers to the uses list of the SDNode referred by | |||
| 290 | /// this operand. | |||
| 291 | SDUse **Prev = nullptr; | |||
| 292 | SDUse *Next = nullptr; | |||
| 293 | ||||
| 294 | public: | |||
| 295 | SDUse() = default; | |||
| 296 | SDUse(const SDUse &U) = delete; | |||
| 297 | SDUse &operator=(const SDUse &) = delete; | |||
| 298 | ||||
| 299 | /// Normally SDUse will just implicitly convert to an SDValue that it holds. | |||
| 300 | operator const SDValue&() const { return Val; } | |||
| 301 | ||||
| 302 | /// If implicit conversion to SDValue doesn't work, the get() method returns | |||
| 303 | /// the SDValue. | |||
| 304 | const SDValue &get() const { return Val; } | |||
| 305 | ||||
| 306 | /// This returns the SDNode that contains this Use. | |||
| 307 | SDNode *getUser() { return User; } | |||
| 308 | const SDNode *getUser() const { return User; } | |||
| 309 | ||||
| 310 | /// Get the next SDUse in the use list. | |||
| 311 | SDUse *getNext() const { return Next; } | |||
| 312 | ||||
| 313 | /// Convenience function for get().getNode(). | |||
| 314 | SDNode *getNode() const { return Val.getNode(); } | |||
| 315 | /// Convenience function for get().getResNo(). | |||
| 316 | unsigned getResNo() const { return Val.getResNo(); } | |||
| 317 | /// Convenience function for get().getValueType(). | |||
| 318 | EVT getValueType() const { return Val.getValueType(); } | |||
| 319 | ||||
| 320 | /// Convenience function for get().operator== | |||
| 321 | bool operator==(const SDValue &V) const { | |||
| 322 | return Val == V; | |||
| 323 | } | |||
| 324 | ||||
| 325 | /// Convenience function for get().operator!= | |||
| 326 | bool operator!=(const SDValue &V) const { | |||
| 327 | return Val != V; | |||
| 328 | } | |||
| 329 | ||||
| 330 | /// Convenience function for get().operator< | |||
| 331 | bool operator<(const SDValue &V) const { | |||
| 332 | return Val < V; | |||
| 333 | } | |||
| 334 | ||||
| 335 | private: | |||
| 336 | friend class SelectionDAG; | |||
| 337 | friend class SDNode; | |||
| 338 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
| 339 | friend class HandleSDNode; | |||
| 340 | ||||
| 341 | void setUser(SDNode *p) { User = p; } | |||
| 342 | ||||
| 343 | /// Remove this use from its existing use list, assign it the | |||
| 344 | /// given value, and add it to the new value's node's use list. | |||
| 345 | inline void set(const SDValue &V); | |||
| 346 | /// Like set, but only supports initializing a newly-allocated | |||
| 347 | /// SDUse with a non-null value. | |||
| 348 | inline void setInitial(const SDValue &V); | |||
| 349 | /// Like set, but only sets the Node portion of the value, | |||
| 350 | /// leaving the ResNo portion unmodified. | |||
| 351 | inline void setNode(SDNode *N); | |||
| 352 | ||||
| 353 | void addToList(SDUse **List) { | |||
| 354 | Next = *List; | |||
| 355 | if (Next) Next->Prev = &Next; | |||
| 356 | Prev = List; | |||
| 357 | *List = this; | |||
| 358 | } | |||
| 359 | ||||
| 360 | void removeFromList() { | |||
| 361 | *Prev = Next; | |||
| 362 | if (Next) Next->Prev = Prev; | |||
| 363 | } | |||
| 364 | }; | |||
| 365 | ||||
| 366 | /// simplify_type specializations - Allow casting operators to work directly on | |||
| 367 | /// SDValues as if they were SDNode*'s. | |||
| 368 | template<> struct simplify_type<SDUse> { | |||
| 369 | using SimpleType = SDNode *; | |||
| 370 | ||||
| 371 | static SimpleType getSimplifiedValue(SDUse &Val) { | |||
| 372 | return Val.getNode(); | |||
| 373 | } | |||
| 374 | }; | |||
| 375 | ||||
| 376 | /// These are IR-level optimization flags that may be propagated to SDNodes. | |||
| 377 | /// TODO: This data structure should be shared by the IR optimizer and the | |||
| 378 | /// the backend. | |||
| 379 | struct SDNodeFlags { | |||
| 380 | private: | |||
| 381 | bool NoUnsignedWrap : 1; | |||
| 382 | bool NoSignedWrap : 1; | |||
| 383 | bool Exact : 1; | |||
| 384 | bool NoNaNs : 1; | |||
| 385 | bool NoInfs : 1; | |||
| 386 | bool NoSignedZeros : 1; | |||
| 387 | bool AllowReciprocal : 1; | |||
| 388 | bool AllowContract : 1; | |||
| 389 | bool ApproximateFuncs : 1; | |||
| 390 | bool AllowReassociation : 1; | |||
| 391 | ||||
| 392 | // We assume instructions do not raise floating-point exceptions by default, | |||
| 393 | // and only those marked explicitly may do so. We could choose to represent | |||
| 394 | // this via a positive "FPExcept" flags like on the MI level, but having a | |||
| 395 | // negative "NoFPExcept" flag here (that defaults to true) makes the flag | |||
| 396 | // intersection logic more straightforward. | |||
| 397 | bool NoFPExcept : 1; | |||
| 398 | ||||
| 399 | public: | |||
| 400 | /// Default constructor turns off all optimization flags. | |||
| 401 | SDNodeFlags() | |||
| 402 | : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false), | |||
| 403 | NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), | |||
| 404 | AllowContract(false), ApproximateFuncs(false), | |||
| 405 | AllowReassociation(false), NoFPExcept(false) {} | |||
| 406 | ||||
| 407 | /// Propagate the fast-math-flags from an IR FPMathOperator. | |||
| 408 | void copyFMF(const FPMathOperator &FPMO) { | |||
| 409 | setNoNaNs(FPMO.hasNoNaNs()); | |||
| 410 | setNoInfs(FPMO.hasNoInfs()); | |||
| 411 | setNoSignedZeros(FPMO.hasNoSignedZeros()); | |||
| 412 | setAllowReciprocal(FPMO.hasAllowReciprocal()); | |||
| 413 | setAllowContract(FPMO.hasAllowContract()); | |||
| 414 | setApproximateFuncs(FPMO.hasApproxFunc()); | |||
| 415 | setAllowReassociation(FPMO.hasAllowReassoc()); | |||
| 416 | } | |||
| 417 | ||||
| 418 | // These are mutators for each flag. | |||
| 419 | void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } | |||
| 420 | void setNoSignedWrap(bool b) { NoSignedWrap = b; } | |||
| 421 | void setExact(bool b) { Exact = b; } | |||
| 422 | void setNoNaNs(bool b) { NoNaNs = b; } | |||
| 423 | void setNoInfs(bool b) { NoInfs = b; } | |||
| 424 | void setNoSignedZeros(bool b) { NoSignedZeros = b; } | |||
| 425 | void setAllowReciprocal(bool b) { AllowReciprocal = b; } | |||
| 426 | void setAllowContract(bool b) { AllowContract = b; } | |||
| 427 | void setApproximateFuncs(bool b) { ApproximateFuncs = b; } | |||
| 428 | void setAllowReassociation(bool b) { AllowReassociation = b; } | |||
| 429 | void setNoFPExcept(bool b) { NoFPExcept = b; } | |||
| 430 | ||||
| 431 | // These are accessors for each flag. | |||
| 432 | bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } | |||
| 433 | bool hasNoSignedWrap() const { return NoSignedWrap; } | |||
| 434 | bool hasExact() const { return Exact; } | |||
| 435 | bool hasNoNaNs() const { return NoNaNs; } | |||
| 436 | bool hasNoInfs() const { return NoInfs; } | |||
| 437 | bool hasNoSignedZeros() const { return NoSignedZeros; } | |||
| 438 | bool hasAllowReciprocal() const { return AllowReciprocal; } | |||
| 439 | bool hasAllowContract() const { return AllowContract; } | |||
| 440 | bool hasApproximateFuncs() const { return ApproximateFuncs; } | |||
| 441 | bool hasAllowReassociation() const { return AllowReassociation; } | |||
| 442 | bool hasNoFPExcept() const { return NoFPExcept; } | |||
| 443 | ||||
| 444 | /// Clear any flags in this flag set that aren't also set in Flags. All | |||
| 445 | /// flags will be cleared if Flags are undefined. | |||
| 446 | void intersectWith(const SDNodeFlags Flags) { | |||
| 447 | NoUnsignedWrap &= Flags.NoUnsignedWrap; | |||
| 448 | NoSignedWrap &= Flags.NoSignedWrap; | |||
| 449 | Exact &= Flags.Exact; | |||
| 450 | NoNaNs &= Flags.NoNaNs; | |||
| 451 | NoInfs &= Flags.NoInfs; | |||
| 452 | NoSignedZeros &= Flags.NoSignedZeros; | |||
| 453 | AllowReciprocal &= Flags.AllowReciprocal; | |||
| 454 | AllowContract &= Flags.AllowContract; | |||
| 455 | ApproximateFuncs &= Flags.ApproximateFuncs; | |||
| 456 | AllowReassociation &= Flags.AllowReassociation; | |||
| 457 | NoFPExcept &= Flags.NoFPExcept; | |||
| 458 | } | |||
| 459 | }; | |||
| 460 | ||||
| 461 | /// Represents one node in the SelectionDAG. | |||
| 462 | /// | |||
| 463 | class SDNode : public FoldingSetNode, public ilist_node<SDNode> { | |||
| 464 | private: | |||
| 465 | /// The operation that this node performs. | |||
| 466 | int32_t NodeType; | |||
| 467 | ||||
| 468 | public: | |||
| 469 | /// Unique and persistent id per SDNode in the DAG. Used for debug printing. | |||
| 470 | /// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` | |||
| 471 | /// intentionally because it adds unneeded complexity without noticeable | |||
| 472 | /// benefits (see discussion with @thakis in D120714). | |||
| 473 | uint16_t PersistentId = 0xffff; | |||
| 474 | ||||
| 475 | protected: | |||
| 476 | // We define a set of mini-helper classes to help us interpret the bits in our | |||
| 477 | // SubclassData. These are designed to fit within a uint16_t so they pack | |||
| 478 | // with PersistentId. | |||
| 479 | ||||
| 480 | #if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1)) | |||
| 481 | // Except for GCC; by default, AIX compilers store bit-fields in 4-byte words | |||
| 482 | // and give the `pack` pragma push semantics. | |||
| 483 | #define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2) | |||
| 484 | #define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop) | |||
| 485 | #else | |||
| 486 | #define BEGIN_TWO_BYTE_PACK() | |||
| 487 | #define END_TWO_BYTE_PACK() | |||
| 488 | #endif | |||
| 489 | ||||
| 490 | BEGIN_TWO_BYTE_PACK() | |||
| 491 | class SDNodeBitfields { | |||
| 492 | friend class SDNode; | |||
| 493 | friend class MemIntrinsicSDNode; | |||
| 494 | friend class MemSDNode; | |||
| 495 | friend class SelectionDAG; | |||
| 496 | ||||
| 497 | uint16_t HasDebugValue : 1; | |||
| 498 | uint16_t IsMemIntrinsic : 1; | |||
| 499 | uint16_t IsDivergent : 1; | |||
| 500 | }; | |||
| 501 | enum { NumSDNodeBits = 3 }; | |||
| 502 | ||||
| 503 | class ConstantSDNodeBitfields { | |||
| 504 | friend class ConstantSDNode; | |||
| 505 | ||||
| 506 | uint16_t : NumSDNodeBits; | |||
| 507 | ||||
| 508 | uint16_t IsOpaque : 1; | |||
| 509 | }; | |||
| 510 | ||||
| 511 | class MemSDNodeBitfields { | |||
| 512 | friend class MemSDNode; | |||
| 513 | friend class MemIntrinsicSDNode; | |||
| 514 | friend class AtomicSDNode; | |||
| 515 | ||||
| 516 | uint16_t : NumSDNodeBits; | |||
| 517 | ||||
| 518 | uint16_t IsVolatile : 1; | |||
| 519 | uint16_t IsNonTemporal : 1; | |||
| 520 | uint16_t IsDereferenceable : 1; | |||
| 521 | uint16_t IsInvariant : 1; | |||
| 522 | }; | |||
| 523 | enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; | |||
| 524 | ||||
| 525 | class LSBaseSDNodeBitfields { | |||
| 526 | friend class LSBaseSDNode; | |||
| 527 | friend class VPBaseLoadStoreSDNode; | |||
| 528 | friend class MaskedLoadStoreSDNode; | |||
| 529 | friend class MaskedGatherScatterSDNode; | |||
| 530 | friend class VPGatherScatterSDNode; | |||
| 531 | ||||
| 532 | uint16_t : NumMemSDNodeBits; | |||
| 533 | ||||
| 534 | // This storage is shared between disparate class hierarchies to hold an | |||
| 535 | // enumeration specific to the class hierarchy in use. | |||
| 536 | // LSBaseSDNode => enum ISD::MemIndexedMode | |||
| 537 | // VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode | |||
| 538 | // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode | |||
| 539 | // VPGatherScatterSDNode => enum ISD::MemIndexType | |||
| 540 | // MaskedGatherScatterSDNode => enum ISD::MemIndexType | |||
| 541 | uint16_t AddressingMode : 3; | |||
| 542 | }; | |||
| 543 | enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; | |||
| 544 | ||||
| 545 | class LoadSDNodeBitfields { | |||
| 546 | friend class LoadSDNode; | |||
| 547 | friend class VPLoadSDNode; | |||
| 548 | friend class VPStridedLoadSDNode; | |||
| 549 | friend class MaskedLoadSDNode; | |||
| 550 | friend class MaskedGatherSDNode; | |||
| 551 | friend class VPGatherSDNode; | |||
| 552 | ||||
| 553 | uint16_t : NumLSBaseSDNodeBits; | |||
| 554 | ||||
| 555 | uint16_t ExtTy : 2; // enum ISD::LoadExtType | |||
| 556 | uint16_t IsExpanding : 1; | |||
| 557 | }; | |||
| 558 | ||||
| 559 | class StoreSDNodeBitfields { | |||
| 560 | friend class StoreSDNode; | |||
| 561 | friend class VPStoreSDNode; | |||
| 562 | friend class VPStridedStoreSDNode; | |||
| 563 | friend class MaskedStoreSDNode; | |||
| 564 | friend class MaskedScatterSDNode; | |||
| 565 | friend class VPScatterSDNode; | |||
| 566 | ||||
| 567 | uint16_t : NumLSBaseSDNodeBits; | |||
| 568 | ||||
| 569 | uint16_t IsTruncating : 1; | |||
| 570 | uint16_t IsCompressing : 1; | |||
| 571 | }; | |||
| 572 | ||||
| 573 | union { | |||
| 574 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
| 575 | SDNodeBitfields SDNodeBits; | |||
| 576 | ConstantSDNodeBitfields ConstantSDNodeBits; | |||
| 577 | MemSDNodeBitfields MemSDNodeBits; | |||
| 578 | LSBaseSDNodeBitfields LSBaseSDNodeBits; | |||
| 579 | LoadSDNodeBitfields LoadSDNodeBits; | |||
| 580 | StoreSDNodeBitfields StoreSDNodeBits; | |||
| 581 | }; | |||
| 582 | END_TWO_BYTE_PACK() | |||
| 583 | #undef BEGIN_TWO_BYTE_PACK | |||
| 584 | #undef END_TWO_BYTE_PACK | |||
| 585 | ||||
| 586 | // RawSDNodeBits must cover the entirety of the union. This means that all of | |||
| 587 | // the union's members must have size <= RawSDNodeBits. We write the RHS as | |||
| 588 | // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter. | |||
| 589 | static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide"); | |||
| 590 | static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide"); | |||
| 591 | static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide"); | |||
| 592 | static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide"); | |||
| 593 | static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide"); | |||
| 594 | static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide"); | |||
| 595 | ||||
| 596 | private: | |||
| 597 | friend class SelectionDAG; | |||
| 598 | // TODO: unfriend HandleSDNode once we fix its operand handling. | |||
| 599 | friend class HandleSDNode; | |||
| 600 | ||||
| 601 | /// Unique id per SDNode in the DAG. | |||
| 602 | int NodeId = -1; | |||
| 603 | ||||
| 604 | /// The values that are used by this operation. | |||
| 605 | SDUse *OperandList = nullptr; | |||
| 606 | ||||
| 607 | /// The types of the values this node defines. SDNode's may | |||
| 608 | /// define multiple values simultaneously. | |||
| 609 | const EVT *ValueList; | |||
| 610 | ||||
| 611 | /// List of uses for this SDNode. | |||
| 612 | SDUse *UseList = nullptr; | |||
| 613 | ||||
| 614 | /// The number of entries in the Operand/Value list. | |||
| 615 | unsigned short NumOperands = 0; | |||
| 616 | unsigned short NumValues; | |||
| 617 | ||||
| 618 | // The ordering of the SDNodes. It roughly corresponds to the ordering of the | |||
| 619 | // original LLVM instructions. | |||
| 620 | // This is used for turning off scheduling, because we'll forgo | |||
| 621 | // the normal scheduling algorithms and output the instructions according to | |||
| 622 | // this ordering. | |||
| 623 | unsigned IROrder; | |||
| 624 | ||||
| 625 | /// Source line information. | |||
| 626 | DebugLoc debugLoc; | |||
| 627 | ||||
| 628 | /// Return a pointer to the specified value type. | |||
| 629 | static const EVT *getValueTypeList(EVT VT); | |||
| 630 | ||||
| 631 | SDNodeFlags Flags; | |||
| 632 | ||||
| 633 | uint32_t CFIType = 0; | |||
| 634 | ||||
| 635 | public: | |||
| 636 | //===--------------------------------------------------------------------===// | |||
| 637 | // Accessors | |||
| 638 | // | |||
| 639 | ||||
| 640 | /// Return the SelectionDAG opcode value for this node. For | |||
| 641 | /// pre-isel nodes (those for which isMachineOpcode returns false), these | |||
| 642 | /// are the opcode values in the ISD and <target>ISD namespaces. For | |||
| 643 | /// post-isel opcodes, see getMachineOpcode. | |||
| 644 | unsigned getOpcode() const { return (unsigned)NodeType; } | |||
| 645 | ||||
| 646 | /// Test if this node has a target-specific opcode (in the | |||
| 647 | /// \<target\>ISD namespace). | |||
| 648 | bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } | |||
| 649 | ||||
| 650 | /// Test if this node has a target-specific opcode that may raise | |||
| 651 | /// FP exceptions (in the \<target\>ISD namespace and greater than | |||
| 652 | /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory | |||
| 653 | /// opcode are currently automatically considered to possibly raise | |||
| 654 | /// FP exceptions as well. | |||
| 655 | bool isTargetStrictFPOpcode() const { | |||
| 656 | return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE; | |||
| 657 | } | |||
| 658 | ||||
| 659 | /// Test if this node has a target-specific | |||
| 660 | /// memory-referencing opcode (in the \<target\>ISD namespace and | |||
| 661 | /// greater than FIRST_TARGET_MEMORY_OPCODE). | |||
| 662 | bool isTargetMemoryOpcode() const { | |||
| 663 | return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; | |||
| 664 | } | |||
| 665 | ||||
| 666 | /// Return true if the type of the node type undefined. | |||
| 667 | bool isUndef() const { return NodeType == ISD::UNDEF; } | |||
| 668 | ||||
| 669 | /// Test if this node is a memory intrinsic (with valid pointer information). | |||
| 670 | /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for | |||
| 671 | /// non-memory intrinsics (with chains) that are not really instances of | |||
| 672 | /// MemSDNode. For such nodes, we need some extra state to determine the | |||
| 673 | /// proper classof relationship. | |||
| 674 | bool isMemIntrinsic() const { | |||
| 675 | return (NodeType == ISD::INTRINSIC_W_CHAIN || | |||
| 676 | NodeType == ISD::INTRINSIC_VOID) && | |||
| 677 | SDNodeBits.IsMemIntrinsic; | |||
| 678 | } | |||
| 679 | ||||
| 680 | /// Test if this node is a strict floating point pseudo-op. | |||
| 681 | bool isStrictFPOpcode() { | |||
| 682 | switch (NodeType) { | |||
| 683 | default: | |||
| 684 | return false; | |||
| 685 | case ISD::STRICT_FP16_TO_FP: | |||
| 686 | case ISD::STRICT_FP_TO_FP16: | |||
| 687 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | |||
| 688 | case ISD::STRICT_##DAGN: | |||
| 689 | #include "llvm/IR/ConstrainedOps.def" | |||
| 690 | return true; | |||
| 691 | } | |||
| 692 | } | |||
| 693 | ||||
| 694 | /// Test if this node is a vector predication operation. | |||
| 695 | bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); } | |||
| 696 | ||||
| 697 | /// Test if this node has a post-isel opcode, directly | |||
| 698 | /// corresponding to a MachineInstr opcode. | |||
| 699 | bool isMachineOpcode() const { return NodeType < 0; } | |||
| 700 | ||||
| 701 | /// This may only be called if isMachineOpcode returns | |||
| 702 | /// true. It returns the MachineInstr opcode value that the node's opcode | |||
| 703 | /// corresponds to. | |||
| 704 | unsigned getMachineOpcode() const { | |||
| 705 | assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!" ) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 705, __extension__ __PRETTY_FUNCTION__)); | |||
| 706 | return ~NodeType; | |||
| 707 | } | |||
| 708 | ||||
| 709 | bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } | |||
| 710 | void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } | |||
| 711 | ||||
| 712 | bool isDivergent() const { return SDNodeBits.IsDivergent; } | |||
| 713 | ||||
| 714 | /// Return true if there are no uses of this node. | |||
| 715 | bool use_empty() const { return UseList == nullptr; } | |||
| 716 | ||||
| 717 | /// Return true if there is exactly one use of this node. | |||
| 718 | bool hasOneUse() const { return hasSingleElement(uses()); } | |||
| 719 | ||||
| 720 | /// Return the number of uses of this node. This method takes | |||
| 721 | /// time proportional to the number of uses. | |||
| 722 | size_t use_size() const { return std::distance(use_begin(), use_end()); } | |||
| 723 | ||||
| 724 | /// Return the unique node id. | |||
| 725 | int getNodeId() const { return NodeId; } | |||
| 726 | ||||
| 727 | /// Set unique node id. | |||
| 728 | void setNodeId(int Id) { NodeId = Id; } | |||
| 729 | ||||
| 730 | /// Return the node ordering. | |||
| 731 | unsigned getIROrder() const { return IROrder; } | |||
| 732 | ||||
| 733 | /// Set the node ordering. | |||
| 734 | void setIROrder(unsigned Order) { IROrder = Order; } | |||
| 735 | ||||
| 736 | /// Return the source location info. | |||
| 737 | const DebugLoc &getDebugLoc() const { return debugLoc; } | |||
| 738 | ||||
| 739 | /// Set source location info. Try to avoid this, putting | |||
| 740 | /// it in the constructor is preferable. | |||
| 741 | void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } | |||
| 742 | ||||
| 743 | /// This class provides iterator support for SDUse | |||
| 744 | /// operands that use a specific SDNode. | |||
| 745 | class use_iterator { | |||
| 746 | friend class SDNode; | |||
| 747 | ||||
| 748 | SDUse *Op = nullptr; | |||
| 749 | ||||
| 750 | explicit use_iterator(SDUse *op) : Op(op) {} | |||
| 751 | ||||
| 752 | public: | |||
| 753 | using iterator_category = std::forward_iterator_tag; | |||
| 754 | using value_type = SDUse; | |||
| 755 | using difference_type = std::ptrdiff_t; | |||
| 756 | using pointer = value_type *; | |||
| 757 | using reference = value_type &; | |||
| 758 | ||||
| 759 | use_iterator() = default; | |||
| 760 | use_iterator(const use_iterator &I) = default; | |||
| 761 | use_iterator &operator=(const use_iterator &) = default; | |||
| 762 | ||||
| 763 | bool operator==(const use_iterator &x) const { return Op == x.Op; } | |||
| 764 | bool operator!=(const use_iterator &x) const { | |||
| 765 | return !operator==(x); | |||
| 766 | } | |||
| 767 | ||||
| 768 | /// Return true if this iterator is at the end of uses list. | |||
| 769 | bool atEnd() const { return Op == nullptr; } | |||
| 770 | ||||
| 771 | // Iterator traversal: forward iteration only. | |||
| 772 | use_iterator &operator++() { // Preincrement | |||
| 773 | assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 773, __extension__ __PRETTY_FUNCTION__)); | |||
| 774 | Op = Op->getNext(); | |||
| 775 | return *this; | |||
| 776 | } | |||
| 777 | ||||
| 778 | use_iterator operator++(int) { // Postincrement | |||
| 779 | use_iterator tmp = *this; ++*this; return tmp; | |||
| 780 | } | |||
| 781 | ||||
| 782 | /// Retrieve a pointer to the current user node. | |||
| 783 | SDNode *operator*() const { | |||
| 784 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 784, __extension__ __PRETTY_FUNCTION__)); | |||
| 785 | return Op->getUser(); | |||
| 786 | } | |||
| 787 | ||||
| 788 | SDNode *operator->() const { return operator*(); } | |||
| 789 | ||||
| 790 | SDUse &getUse() const { return *Op; } | |||
| 791 | ||||
| 792 | /// Retrieve the operand # of this use in its user. | |||
| 793 | unsigned getOperandNo() const { | |||
| 794 | assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!" ) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 794, __extension__ __PRETTY_FUNCTION__)); | |||
| 795 | return (unsigned)(Op - Op->getUser()->OperandList); | |||
| 796 | } | |||
| 797 | }; | |||
| 798 | ||||
| 799 | /// Provide iteration support to walk over all uses of an SDNode. | |||
| 800 | use_iterator use_begin() const { | |||
| 801 | return use_iterator(UseList); | |||
| 802 | } | |||
| 803 | ||||
| 804 | static use_iterator use_end() { return use_iterator(nullptr); } | |||
| 805 | ||||
| 806 | inline iterator_range<use_iterator> uses() { | |||
| 807 | return make_range(use_begin(), use_end()); | |||
| 808 | } | |||
| 809 | inline iterator_range<use_iterator> uses() const { | |||
| 810 | return make_range(use_begin(), use_end()); | |||
| 811 | } | |||
| 812 | ||||
| 813 | /// Return true if there are exactly NUSES uses of the indicated value. | |||
| 814 | /// This method ignores uses of other values defined by this operation. | |||
| 815 | bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; | |||
| 816 | ||||
| 817 | /// Return true if there are any use of the indicated value. | |||
| 818 | /// This method ignores uses of other values defined by this operation. | |||
| 819 | bool hasAnyUseOfValue(unsigned Value) const; | |||
| 820 | ||||
| 821 | /// Return true if this node is the only use of N. | |||
| 822 | bool isOnlyUserOf(const SDNode *N) const; | |||
| 823 | ||||
| 824 | /// Return true if this node is an operand of N. | |||
| 825 | bool isOperandOf(const SDNode *N) const; | |||
| 826 | ||||
| 827 | /// Return true if this node is a predecessor of N. | |||
| 828 | /// NOTE: Implemented on top of hasPredecessor and every bit as | |||
| 829 | /// expensive. Use carefully. | |||
| 830 | bool isPredecessorOf(const SDNode *N) const { | |||
| 831 | return N->hasPredecessor(this); | |||
| 832 | } | |||
| 833 | ||||
| 834 | /// Return true if N is a predecessor of this node. | |||
| 835 | /// N is either an operand of this node, or can be reached by recursively | |||
| 836 | /// traversing up the operands. | |||
| 837 | /// NOTE: This is an expensive method. Use it carefully. | |||
| 838 | bool hasPredecessor(const SDNode *N) const; | |||
| 839 | ||||
| 840 | /// Returns true if N is a predecessor of any node in Worklist. This | |||
| 841 | /// helper keeps Visited and Worklist sets externally to allow unions | |||
| 842 | /// searches to be performed in parallel, caching of results across | |||
| 843 | /// queries and incremental addition to Worklist. Stops early if N is | |||
| 844 | /// found but will resume. Remember to clear Visited and Worklists | |||
| 845 | /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before | |||
| 846 | /// giving up. The TopologicalPrune flag signals that positive NodeIds are | |||
| 847 | /// topologically ordered (Operands have strictly smaller node id) and search | |||
| 848 | /// can be pruned leveraging this. | |||
| 849 | static bool hasPredecessorHelper(const SDNode *N, | |||
| 850 | SmallPtrSetImpl<const SDNode *> &Visited, | |||
| 851 | SmallVectorImpl<const SDNode *> &Worklist, | |||
| 852 | unsigned int MaxSteps = 0, | |||
| 853 | bool TopologicalPrune = false) { | |||
| 854 | SmallVector<const SDNode *, 8> DeferredNodes; | |||
| 855 | if (Visited.count(N)) | |||
| 856 | return true; | |||
| 857 | ||||
| 858 | // Node Id's are assigned in three places: As a topological | |||
| 859 | // ordering (> 0), during legalization (results in values set to | |||
| 860 | // 0), new nodes (set to -1). If N has a topolgical id then we | |||
| 861 | // know that all nodes with ids smaller than it cannot be | |||
| 862 | // successors and we need not check them. Filter out all node | |||
| 863 | // that can't be matches. We add them to the worklist before exit | |||
| 864 | // in case of multiple calls. Note that during selection the topological id | |||
| 865 | // may be violated if a node's predecessor is selected before it. We mark | |||
| 866 | // this at selection negating the id of unselected successors and | |||
| 867 | // restricting topological pruning to positive ids. | |||
| 868 | ||||
| 869 | int NId = N->getNodeId(); | |||
| 870 | // If we Invalidated the Id, reconstruct original NId. | |||
| 871 | if (NId < -1) | |||
| 872 | NId = -(NId + 1); | |||
| 873 | ||||
| 874 | bool Found = false; | |||
| 875 | while (!Worklist.empty()) { | |||
| 876 | const SDNode *M = Worklist.pop_back_val(); | |||
| 877 | int MId = M->getNodeId(); | |||
| 878 | if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) && | |||
| 879 | (MId > 0) && (MId < NId)) { | |||
| 880 | DeferredNodes.push_back(M); | |||
| 881 | continue; | |||
| 882 | } | |||
| 883 | for (const SDValue &OpV : M->op_values()) { | |||
| 884 | SDNode *Op = OpV.getNode(); | |||
| 885 | if (Visited.insert(Op).second) | |||
| 886 | Worklist.push_back(Op); | |||
| 887 | if (Op == N) | |||
| 888 | Found = true; | |||
| 889 | } | |||
| 890 | if (Found) | |||
| 891 | break; | |||
| 892 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
| 893 | break; | |||
| 894 | } | |||
| 895 | // Push deferred nodes back on worklist. | |||
| 896 | Worklist.append(DeferredNodes.begin(), DeferredNodes.end()); | |||
| 897 | // If we bailed early, conservatively return found. | |||
| 898 | if (MaxSteps != 0 && Visited.size() >= MaxSteps) | |||
| 899 | return true; | |||
| 900 | return Found; | |||
| 901 | } | |||
| 902 | ||||
| 903 | /// Return true if all the users of N are contained in Nodes. | |||
| 904 | /// NOTE: Requires at least one match, but doesn't require them all. | |||
| 905 | static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N); | |||
| 906 | ||||
| 907 | /// Return the number of values used by this operation. | |||
| 908 | unsigned getNumOperands() const { return NumOperands; } | |||
| 909 | ||||
| 910 | /// Return the maximum number of operands that a SDNode can hold. | |||
| 911 | static constexpr size_t getMaxNumOperands() { | |||
| 912 | return std::numeric_limits<decltype(SDNode::NumOperands)>::max(); | |||
| 913 | } | |||
| 914 | ||||
| 915 | /// Helper method returns the integer value of a ConstantSDNode operand. | |||
| 916 | inline uint64_t getConstantOperandVal(unsigned Num) const; | |||
| 917 | ||||
| 918 | /// Helper method returns the APInt of a ConstantSDNode operand. | |||
| 919 | inline const APInt &getConstantOperandAPInt(unsigned Num) const; | |||
| 920 | ||||
| 921 | const SDValue &getOperand(unsigned Num) const { | |||
| 922 | assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!" ) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 922, __extension__ __PRETTY_FUNCTION__)); | |||
| 923 | return OperandList[Num]; | |||
| 924 | } | |||
| 925 | ||||
| 926 | using op_iterator = SDUse *; | |||
| 927 | ||||
| 928 | op_iterator op_begin() const { return OperandList; } | |||
| 929 | op_iterator op_end() const { return OperandList+NumOperands; } | |||
| 930 | ArrayRef<SDUse> ops() const { return ArrayRef(op_begin(), op_end()); } | |||
| 931 | ||||
| 932 | /// Iterator for directly iterating over the operand SDValue's. | |||
| 933 | struct value_op_iterator | |||
| 934 | : iterator_adaptor_base<value_op_iterator, op_iterator, | |||
| 935 | std::random_access_iterator_tag, SDValue, | |||
| 936 | ptrdiff_t, value_op_iterator *, | |||
| 937 | value_op_iterator *> { | |||
| 938 | explicit value_op_iterator(SDUse *U = nullptr) | |||
| 939 | : iterator_adaptor_base(U) {} | |||
| 940 | ||||
| 941 | const SDValue &operator*() const { return I->get(); } | |||
| 942 | }; | |||
| 943 | ||||
| 944 | iterator_range<value_op_iterator> op_values() const { | |||
| 945 | return make_range(value_op_iterator(op_begin()), | |||
| 946 | value_op_iterator(op_end())); | |||
| 947 | } | |||
| 948 | ||||
| 949 | SDVTList getVTList() const { | |||
| 950 | SDVTList X = { ValueList, NumValues }; | |||
| 951 | return X; | |||
| 952 | } | |||
| 953 | ||||
| 954 | /// If this node has a glue operand, return the node | |||
| 955 | /// to which the glue operand points. Otherwise return NULL. | |||
| 956 | SDNode *getGluedNode() const { | |||
| 957 | if (getNumOperands() != 0 && | |||
| 958 | getOperand(getNumOperands()-1).getValueType() == MVT::Glue) | |||
| 959 | return getOperand(getNumOperands()-1).getNode(); | |||
| 960 | return nullptr; | |||
| 961 | } | |||
| 962 | ||||
| 963 | /// If this node has a glue value with a user, return | |||
| 964 | /// the user (there is at most one). Otherwise return NULL. | |||
| 965 | SDNode *getGluedUser() const { | |||
| 966 | for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) | |||
| 967 | if (UI.getUse().get().getValueType() == MVT::Glue) | |||
| 968 | return *UI; | |||
| 969 | return nullptr; | |||
| 970 | } | |||
| 971 | ||||
| 972 | SDNodeFlags getFlags() const { return Flags; } | |||
| 973 | void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; } | |||
| 974 | ||||
| 975 | /// Clear any flags in this node that aren't also set in Flags. | |||
| 976 | /// If Flags is not in a defined state then this has no effect. | |||
| 977 | void intersectFlagsWith(const SDNodeFlags Flags); | |||
| 978 | ||||
| 979 | void setCFIType(uint32_t Type) { CFIType = Type; } | |||
| 980 | uint32_t getCFIType() const { return CFIType; } | |||
| 981 | ||||
| 982 | /// Return the number of values defined/returned by this operator. | |||
| 983 | unsigned getNumValues() const { return NumValues; } | |||
| 984 | ||||
| 985 | /// Return the type of a specified result. | |||
| 986 | EVT getValueType(unsigned ResNo) const { | |||
| 987 | assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!" ) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 987, __extension__ __PRETTY_FUNCTION__)); | |||
| 988 | return ValueList[ResNo]; | |||
| 989 | } | |||
| 990 | ||||
| 991 | /// Return the type of a specified result as a simple type. | |||
| 992 | MVT getSimpleValueType(unsigned ResNo) const { | |||
| 993 | return getValueType(ResNo).getSimpleVT(); | |||
| 994 | } | |||
| 995 | ||||
| 996 | /// Returns MVT::getSizeInBits(getValueType(ResNo)). | |||
| 997 | /// | |||
| 998 | /// If the value type is a scalable vector type, the scalable property will | |||
| 999 | /// be set and the runtime size will be a positive integer multiple of the | |||
| 1000 | /// base size. | |||
| 1001 | TypeSize getValueSizeInBits(unsigned ResNo) const { | |||
| 1002 | return getValueType(ResNo).getSizeInBits(); | |||
| 1003 | } | |||
| 1004 | ||||
| 1005 | using value_iterator = const EVT *; | |||
| 1006 | ||||
| 1007 | value_iterator value_begin() const { return ValueList; } | |||
| 1008 | value_iterator value_end() const { return ValueList+NumValues; } | |||
| 1009 | iterator_range<value_iterator> values() const { | |||
| 1010 | return llvm::make_range(value_begin(), value_end()); | |||
| 1011 | } | |||
| 1012 | ||||
| 1013 | /// Return the opcode of this operation for printing. | |||
| 1014 | std::string getOperationName(const SelectionDAG *G = nullptr) const; | |||
| 1015 | static const char* getIndexedModeName(ISD::MemIndexedMode AM); | |||
| 1016 | void print_types(raw_ostream &OS, const SelectionDAG *G) const; | |||
| 1017 | void print_details(raw_ostream &OS, const SelectionDAG *G) const; | |||
| 1018 | void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
| 1019 | void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; | |||
| 1020 | ||||
| 1021 | /// Print a SelectionDAG node and all children down to | |||
| 1022 | /// the leaves. The given SelectionDAG allows target-specific nodes | |||
| 1023 | /// to be printed in human-readable form. Unlike printr, this will | |||
| 1024 | /// print the whole DAG, including children that appear multiple | |||
| 1025 | /// times. | |||
| 1026 | /// | |||
| 1027 | void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; | |||
| 1028 | ||||
| 1029 | /// Print a SelectionDAG node and children up to | |||
| 1030 | /// depth "depth." The given SelectionDAG allows target-specific | |||
| 1031 | /// nodes to be printed in human-readable form. Unlike printr, this | |||
| 1032 | /// will print children that appear multiple times wherever they are | |||
| 1033 | /// used. | |||
| 1034 | /// | |||
| 1035 | void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, | |||
| 1036 | unsigned depth = 100) const; | |||
| 1037 | ||||
| 1038 | /// Dump this node, for debugging. | |||
| 1039 | void dump() const; | |||
| 1040 | ||||
| 1041 | /// Dump (recursively) this node and its use-def subgraph. | |||
| 1042 | void dumpr() const; | |||
| 1043 | ||||
| 1044 | /// Dump this node, for debugging. | |||
| 1045 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
| 1046 | /// in human-readable form. | |||
| 1047 | void dump(const SelectionDAG *G) const; | |||
| 1048 | ||||
| 1049 | /// Dump (recursively) this node and its use-def subgraph. | |||
| 1050 | /// The given SelectionDAG allows target-specific nodes to be printed | |||
| 1051 | /// in human-readable form. | |||
| 1052 | void dumpr(const SelectionDAG *G) const; | |||
| 1053 | ||||
| 1054 | /// printrFull to dbgs(). The given SelectionDAG allows | |||
| 1055 | /// target-specific nodes to be printed in human-readable form. | |||
| 1056 | /// Unlike dumpr, this will print the whole DAG, including children | |||
| 1057 | /// that appear multiple times. | |||
| 1058 | void dumprFull(const SelectionDAG *G = nullptr) const; | |||
| 1059 | ||||
| 1060 | /// printrWithDepth to dbgs(). The given | |||
| 1061 | /// SelectionDAG allows target-specific nodes to be printed in | |||
| 1062 | /// human-readable form. Unlike dumpr, this will print children | |||
| 1063 | /// that appear multiple times wherever they are used. | |||
| 1064 | /// | |||
| 1065 | void dumprWithDepth(const SelectionDAG *G = nullptr, | |||
| 1066 | unsigned depth = 100) const; | |||
| 1067 | ||||
| 1068 | /// Gather unique data for the node. | |||
| 1069 | void Profile(FoldingSetNodeID &ID) const; | |||
| 1070 | ||||
| 1071 | /// This method should only be used by the SDUse class. | |||
| 1072 | void addUse(SDUse &U) { U.addToList(&UseList); } | |||
| 1073 | ||||
| 1074 | protected: | |||
| 1075 | static SDVTList getSDVTList(EVT VT) { | |||
| 1076 | SDVTList Ret = { getValueTypeList(VT), 1 }; | |||
| 1077 | return Ret; | |||
| 1078 | } | |||
| 1079 | ||||
| 1080 | /// Create an SDNode. | |||
| 1081 | /// | |||
| 1082 | /// SDNodes are created without any operands, and never own the operand | |||
| 1083 | /// storage. To add operands, see SelectionDAG::createOperands. | |||
| 1084 | SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) | |||
| 1085 | : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs), | |||
| 1086 | IROrder(Order), debugLoc(std::move(dl)) { | |||
| 1087 | memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits)); | |||
| 1088 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() && "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1088, __extension__ __PRETTY_FUNCTION__)); | |||
| 1089 | assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1090, __extension__ __PRETTY_FUNCTION__)) | |||
| 1090 | "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!") ? void (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1090, __extension__ __PRETTY_FUNCTION__)); | |||
| 1091 | } | |||
| 1092 | ||||
| 1093 | /// Release the operands and set this node to have zero operands. | |||
| 1094 | void DropOperands(); | |||
| 1095 | }; | |||
| 1096 | ||||
| 1097 | /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed | |||
| 1098 | /// into SDNode creation functions. | |||
| 1099 | /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted | |||
| 1100 | /// from the original Instruction, and IROrder is the ordinal position of | |||
| 1101 | /// the instruction. | |||
| 1102 | /// When an SDNode is created after the DAG is being built, both DebugLoc and | |||
| 1103 | /// the IROrder are propagated from the original SDNode. | |||
| 1104 | /// So SDLoc class provides two constructors besides the default one, one to | |||
| 1105 | /// be used by the DAGBuilder, the other to be used by others. | |||
| 1106 | class SDLoc { | |||
| 1107 | private: | |||
| 1108 | DebugLoc DL; | |||
| 1109 | int IROrder = 0; | |||
| 1110 | ||||
| 1111 | public: | |||
| 1112 | SDLoc() = default; | |||
| 1113 | SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {} | |||
| 1114 | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} | |||
| 1115 | SDLoc(const Instruction *I, int Order) : IROrder(Order) { | |||
| 1116 | assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder" ) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1116, __extension__ __PRETTY_FUNCTION__)); | |||
| 1117 | if (I) | |||
| 1118 | DL = I->getDebugLoc(); | |||
| 1119 | } | |||
| 1120 | ||||
| 1121 | unsigned getIROrder() const { return IROrder; } | |||
| 1122 | const DebugLoc &getDebugLoc() const { return DL; } | |||
| 1123 | }; | |||
| 1124 | ||||
| 1125 | // Define inline functions from the SDValue class. | |||
| 1126 | ||||
| 1127 | inline SDValue::SDValue(SDNode *node, unsigned resno) | |||
| 1128 | : Node(node), ResNo(resno) { | |||
| 1129 | // Explicitly check for !ResNo to avoid use-after-free, because there are | |||
| 1130 | // callers that use SDValue(N, 0) with a deleted N to indicate successful | |||
| 1131 | // combines. | |||
| 1132 | assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1133, __extension__ __PRETTY_FUNCTION__)) | |||
| 1133 | "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node ->getNumValues()) && "Invalid result number for the given node!" ) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1133, __extension__ __PRETTY_FUNCTION__)); | |||
| 1134 | assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps." ) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1134, __extension__ __PRETTY_FUNCTION__)); | |||
| 1135 | } | |||
| 1136 | ||||
| 1137 | inline unsigned SDValue::getOpcode() const { | |||
| 1138 | return Node->getOpcode(); | |||
| ||||
| 1139 | } | |||
| 1140 | ||||
| 1141 | inline EVT SDValue::getValueType() const { | |||
| 1142 | return Node->getValueType(ResNo); | |||
| 1143 | } | |||
| 1144 | ||||
| 1145 | inline unsigned SDValue::getNumOperands() const { | |||
| 1146 | return Node->getNumOperands(); | |||
| 1147 | } | |||
| 1148 | ||||
| 1149 | inline const SDValue &SDValue::getOperand(unsigned i) const { | |||
| 1150 | return Node->getOperand(i); | |||
| 1151 | } | |||
| 1152 | ||||
| 1153 | inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { | |||
| 1154 | return Node->getConstantOperandVal(i); | |||
| 1155 | } | |||
| 1156 | ||||
| 1157 | inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { | |||
| 1158 | return Node->getConstantOperandAPInt(i); | |||
| 1159 | } | |||
| 1160 | ||||
| 1161 | inline bool SDValue::isTargetOpcode() const { | |||
| 1162 | return Node->isTargetOpcode(); | |||
| 1163 | } | |||
| 1164 | ||||
| 1165 | inline bool SDValue::isTargetMemoryOpcode() const { | |||
| 1166 | return Node->isTargetMemoryOpcode(); | |||
| 1167 | } | |||
| 1168 | ||||
| 1169 | inline bool SDValue::isMachineOpcode() const { | |||
| 1170 | return Node->isMachineOpcode(); | |||
| 1171 | } | |||
| 1172 | ||||
| 1173 | inline unsigned SDValue::getMachineOpcode() const { | |||
| 1174 | return Node->getMachineOpcode(); | |||
| 1175 | } | |||
| 1176 | ||||
| 1177 | inline bool SDValue::isUndef() const { | |||
| 1178 | return Node->isUndef(); | |||
| 1179 | } | |||
| 1180 | ||||
| 1181 | inline bool SDValue::use_empty() const { | |||
| 1182 | return !Node->hasAnyUseOfValue(ResNo); | |||
| 1183 | } | |||
| 1184 | ||||
| 1185 | inline bool SDValue::hasOneUse() const { | |||
| 1186 | return Node->hasNUsesOfValue(1, ResNo); | |||
| 1187 | } | |||
| 1188 | ||||
| 1189 | inline const DebugLoc &SDValue::getDebugLoc() const { | |||
| 1190 | return Node->getDebugLoc(); | |||
| 1191 | } | |||
| 1192 | ||||
| 1193 | inline void SDValue::dump() const { | |||
| 1194 | return Node->dump(); | |||
| 1195 | } | |||
| 1196 | ||||
| 1197 | inline void SDValue::dump(const SelectionDAG *G) const { | |||
| 1198 | return Node->dump(G); | |||
| 1199 | } | |||
| 1200 | ||||
| 1201 | inline void SDValue::dumpr() const { | |||
| 1202 | return Node->dumpr(); | |||
| 1203 | } | |||
| 1204 | ||||
| 1205 | inline void SDValue::dumpr(const SelectionDAG *G) const { | |||
| 1206 | return Node->dumpr(G); | |||
| 1207 | } | |||
| 1208 | ||||
| 1209 | // Define inline functions from the SDUse class. | |||
| 1210 | ||||
| 1211 | inline void SDUse::set(const SDValue &V) { | |||
| 1212 | if (Val.getNode()) removeFromList(); | |||
| 1213 | Val = V; | |||
| 1214 | if (V.getNode()) | |||
| 1215 | V->addUse(*this); | |||
| 1216 | } | |||
| 1217 | ||||
| 1218 | inline void SDUse::setInitial(const SDValue &V) { | |||
| 1219 | Val = V; | |||
| 1220 | V->addUse(*this); | |||
| 1221 | } | |||
| 1222 | ||||
| 1223 | inline void SDUse::setNode(SDNode *N) { | |||
| 1224 | if (Val.getNode()) removeFromList(); | |||
| 1225 | Val.setNode(N); | |||
| 1226 | if (N) N->addUse(*this); | |||
| 1227 | } | |||
| 1228 | ||||
| 1229 | /// This class is used to form a handle around another node that | |||
| 1230 | /// is persistent and is updated across invocations of replaceAllUsesWith on its | |||
| 1231 | /// operand. This node should be directly created by end-users and not added to | |||
| 1232 | /// the AllNodes list. | |||
| 1233 | class HandleSDNode : public SDNode { | |||
| 1234 | SDUse Op; | |||
| 1235 | ||||
| 1236 | public: | |||
| 1237 | explicit HandleSDNode(SDValue X) | |||
| 1238 | : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { | |||
| 1239 | // HandleSDNodes are never inserted into the DAG, so they won't be | |||
| 1240 | // auto-numbered. Use ID 65535 as a sentinel. | |||
| 1241 | PersistentId = 0xffff; | |||
| 1242 | ||||
| 1243 | // Manually set up the operand list. This node type is special in that it's | |||
| 1244 | // always stack allocated and SelectionDAG does not manage its operands. | |||
| 1245 | // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not | |||
| 1246 | // be so special. | |||
| 1247 | Op.setUser(this); | |||
| 1248 | Op.setInitial(X); | |||
| 1249 | NumOperands = 1; | |||
| 1250 | OperandList = &Op; | |||
| 1251 | } | |||
| 1252 | ~HandleSDNode(); | |||
| 1253 | ||||
| 1254 | const SDValue &getValue() const { return Op; } | |||
| 1255 | }; | |||
| 1256 | ||||
| 1257 | class AddrSpaceCastSDNode : public SDNode { | |||
| 1258 | private: | |||
| 1259 | unsigned SrcAddrSpace; | |||
| 1260 | unsigned DestAddrSpace; | |||
| 1261 | ||||
| 1262 | public: | |||
| 1263 | AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, | |||
| 1264 | unsigned SrcAS, unsigned DestAS); | |||
| 1265 | ||||
| 1266 | unsigned getSrcAddressSpace() const { return SrcAddrSpace; } | |||
| 1267 | unsigned getDestAddressSpace() const { return DestAddrSpace; } | |||
| 1268 | ||||
| 1269 | static bool classof(const SDNode *N) { | |||
| 1270 | return N->getOpcode() == ISD::ADDRSPACECAST; | |||
| 1271 | } | |||
| 1272 | }; | |||
| 1273 | ||||
| 1274 | /// This is an abstract virtual class for memory operations. | |||
| 1275 | class MemSDNode : public SDNode { | |||
| 1276 | private: | |||
| 1277 | // VT of in-memory value. | |||
| 1278 | EVT MemoryVT; | |||
| 1279 | ||||
| 1280 | protected: | |||
| 1281 | /// Memory reference information. | |||
| 1282 | MachineMemOperand *MMO; | |||
| 1283 | ||||
| 1284 | public: | |||
| 1285 | MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 1286 | EVT memvt, MachineMemOperand *MMO); | |||
| 1287 | ||||
| 1288 | bool readMem() const { return MMO->isLoad(); } | |||
| 1289 | bool writeMem() const { return MMO->isStore(); } | |||
| 1290 | ||||
| 1291 | /// Returns alignment and volatility of the memory access | |||
| 1292 | Align getOriginalAlign() const { return MMO->getBaseAlign(); } | |||
| 1293 | Align getAlign() const { return MMO->getAlign(); } | |||
| 1294 | ||||
| 1295 | /// Return the SubclassData value, without HasDebugValue. This contains an | |||
| 1296 | /// encoding of the volatile flag, as well as bits used by subclasses. This | |||
| 1297 | /// function should only be used to compute a FoldingSetNodeID value. | |||
| 1298 | /// The HasDebugValue bit is masked out because CSE map needs to match | |||
| 1299 | /// nodes with debug info with nodes without debug info. Same is about | |||
| 1300 | /// isDivergent bit. | |||
| 1301 | unsigned getRawSubclassData() const { | |||
| 1302 | uint16_t Data; | |||
| 1303 | union { | |||
| 1304 | char RawSDNodeBits[sizeof(uint16_t)]; | |||
| 1305 | SDNodeBitfields SDNodeBits; | |||
| 1306 | }; | |||
| 1307 | memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits)); | |||
| 1308 | SDNodeBits.HasDebugValue = 0; | |||
| 1309 | SDNodeBits.IsDivergent = false; | |||
| 1310 | memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits)); | |||
| 1311 | return Data; | |||
| 1312 | } | |||
| 1313 | ||||
| 1314 | bool isVolatile() const { return MemSDNodeBits.IsVolatile; } | |||
| 1315 | bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } | |||
| 1316 | bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } | |||
| 1317 | bool isInvariant() const { return MemSDNodeBits.IsInvariant; } | |||
| 1318 | ||||
| 1319 | // Returns the offset from the location of the access. | |||
| 1320 | int64_t getSrcValueOffset() const { return MMO->getOffset(); } | |||
| 1321 | ||||
| 1322 | /// Returns the AA info that describes the dereference. | |||
| 1323 | AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } | |||
| 1324 | ||||
| 1325 | /// Returns the Ranges that describes the dereference. | |||
| 1326 | const MDNode *getRanges() const { return MMO->getRanges(); } | |||
| 1327 | ||||
| 1328 | /// Returns the synchronization scope ID for this memory operation. | |||
| 1329 | SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } | |||
| 1330 | ||||
| 1331 | /// Return the atomic ordering requirements for this memory operation. For | |||
| 1332 | /// cmpxchg atomic operations, return the atomic ordering requirements when | |||
| 1333 | /// store occurs. | |||
| 1334 | AtomicOrdering getSuccessOrdering() const { | |||
| 1335 | return MMO->getSuccessOrdering(); | |||
| 1336 | } | |||
| 1337 | ||||
| 1338 | /// Return a single atomic ordering that is at least as strong as both the | |||
| 1339 | /// success and failure orderings for an atomic operation. (For operations | |||
| 1340 | /// other than cmpxchg, this is equivalent to getSuccessOrdering().) | |||
| 1341 | AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); } | |||
| 1342 | ||||
| 1343 | /// Return true if the memory operation ordering is Unordered or higher. | |||
| 1344 | bool isAtomic() const { return MMO->isAtomic(); } | |||
| 1345 | ||||
| 1346 | /// Returns true if the memory operation doesn't imply any ordering | |||
| 1347 | /// constraints on surrounding memory operations beyond the normal memory | |||
| 1348 | /// aliasing rules. | |||
| 1349 | bool isUnordered() const { return MMO->isUnordered(); } | |||
| 1350 | ||||
| 1351 | /// Returns true if the memory operation is neither atomic or volatile. | |||
| 1352 | bool isSimple() const { return !isAtomic() && !isVolatile(); } | |||
| 1353 | ||||
| 1354 | /// Return the type of the in-memory value. | |||
| 1355 | EVT getMemoryVT() const { return MemoryVT; } | |||
| 1356 | ||||
| 1357 | /// Return a MachineMemOperand object describing the memory | |||
| 1358 | /// reference performed by operation. | |||
| 1359 | MachineMemOperand *getMemOperand() const { return MMO; } | |||
| 1360 | ||||
| 1361 | const MachinePointerInfo &getPointerInfo() const { | |||
| 1362 | return MMO->getPointerInfo(); | |||
| 1363 | } | |||
| 1364 | ||||
| 1365 | /// Return the address space for the associated pointer | |||
| 1366 | unsigned getAddressSpace() const { | |||
| 1367 | return getPointerInfo().getAddrSpace(); | |||
| 1368 | } | |||
| 1369 | ||||
| 1370 | /// Update this MemSDNode's MachineMemOperand information | |||
| 1371 | /// to reflect the alignment of NewMMO, if it has a greater alignment. | |||
| 1372 | /// This must only be used when the new alignment applies to all users of | |||
| 1373 | /// this MachineMemOperand. | |||
| 1374 | void refineAlignment(const MachineMemOperand *NewMMO) { | |||
| 1375 | MMO->refineAlignment(NewMMO); | |||
| 1376 | } | |||
| 1377 | ||||
| 1378 | const SDValue &getChain() const { return getOperand(0); } | |||
| 1379 | ||||
| 1380 | const SDValue &getBasePtr() const { | |||
| 1381 | switch (getOpcode()) { | |||
| 1382 | case ISD::STORE: | |||
| 1383 | case ISD::VP_STORE: | |||
| 1384 | case ISD::MSTORE: | |||
| 1385 | case ISD::VP_SCATTER: | |||
| 1386 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: | |||
| 1387 | return getOperand(2); | |||
| 1388 | case ISD::MGATHER: | |||
| 1389 | case ISD::MSCATTER: | |||
| 1390 | return getOperand(3); | |||
| 1391 | default: | |||
| 1392 | return getOperand(1); | |||
| 1393 | } | |||
| 1394 | } | |||
| 1395 | ||||
| 1396 | // Methods to support isa and dyn_cast | |||
| 1397 | static bool classof(const SDNode *N) { | |||
| 1398 | // For some targets, we lower some target intrinsics to a MemIntrinsicNode | |||
| 1399 | // with either an intrinsic or a target opcode. | |||
| 1400 | switch (N->getOpcode()) { | |||
| 1401 | case ISD::LOAD: | |||
| 1402 | case ISD::STORE: | |||
| 1403 | case ISD::PREFETCH: | |||
| 1404 | case ISD::ATOMIC_CMP_SWAP: | |||
| 1405 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: | |||
| 1406 | case ISD::ATOMIC_SWAP: | |||
| 1407 | case ISD::ATOMIC_LOAD_ADD: | |||
| 1408 | case ISD::ATOMIC_LOAD_SUB: | |||
| 1409 | case ISD::ATOMIC_LOAD_AND: | |||
| 1410 | case ISD::ATOMIC_LOAD_CLR: | |||
| 1411 | case ISD::ATOMIC_LOAD_OR: | |||
| 1412 | case ISD::ATOMIC_LOAD_XOR: | |||
| 1413 | case ISD::ATOMIC_LOAD_NAND: | |||
| 1414 | case ISD::ATOMIC_LOAD_MIN: | |||
| 1415 | case ISD::ATOMIC_LOAD_MAX: | |||
| 1416 | case ISD::ATOMIC_LOAD_UMIN: | |||
| 1417 | case ISD::ATOMIC_LOAD_UMAX: | |||
| 1418 | case ISD::ATOMIC_LOAD_FADD: | |||
| 1419 | case ISD::ATOMIC_LOAD_FSUB: | |||
| 1420 | case ISD::ATOMIC_LOAD_FMAX: | |||
| 1421 | case ISD::ATOMIC_LOAD_FMIN: | |||
| 1422 | case ISD::ATOMIC_LOAD_UINC_WRAP: | |||
| 1423 | case ISD::ATOMIC_LOAD_UDEC_WRAP: | |||
| 1424 | case ISD::ATOMIC_LOAD: | |||
| 1425 | case ISD::ATOMIC_STORE: | |||
| 1426 | case ISD::MLOAD: | |||
| 1427 | case ISD::MSTORE: | |||
| 1428 | case ISD::MGATHER: | |||
| 1429 | case ISD::MSCATTER: | |||
| 1430 | case ISD::VP_LOAD: | |||
| 1431 | case ISD::VP_STORE: | |||
| 1432 | case ISD::VP_GATHER: | |||
| 1433 | case ISD::VP_SCATTER: | |||
| 1434 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: | |||
| 1435 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: | |||
| 1436 | return true; | |||
| 1437 | default: | |||
| 1438 | return N->isMemIntrinsic() || N->isTargetMemoryOpcode(); | |||
| 1439 | } | |||
| 1440 | } | |||
| 1441 | }; | |||
| 1442 | ||||
| 1443 | /// This is an SDNode representing atomic operations. | |||
| 1444 | class AtomicSDNode : public MemSDNode { | |||
| 1445 | public: | |||
| 1446 | AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, | |||
| 1447 | EVT MemVT, MachineMemOperand *MMO) | |||
| 1448 | : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { | |||
| 1449 | assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1450, __extension__ __PRETTY_FUNCTION__)) | |||
| 1450 | MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?" ) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1450, __extension__ __PRETTY_FUNCTION__)); | |||
| 1451 | } | |||
| 1452 | ||||
| 1453 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 1454 | const SDValue &getVal() const { return getOperand(2); } | |||
| 1455 | ||||
| 1456 | /// Returns true if this SDNode represents cmpxchg atomic operation, false | |||
| 1457 | /// otherwise. | |||
| 1458 | bool isCompareAndSwap() const { | |||
| 1459 | unsigned Op = getOpcode(); | |||
| 1460 | return Op == ISD::ATOMIC_CMP_SWAP || | |||
| 1461 | Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; | |||
| 1462 | } | |||
| 1463 | ||||
| 1464 | /// For cmpxchg atomic operations, return the atomic ordering requirements | |||
| 1465 | /// when store does not occur. | |||
| 1466 | AtomicOrdering getFailureOrdering() const { | |||
| 1467 | assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation" ) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1467, __extension__ __PRETTY_FUNCTION__)); | |||
| 1468 | return MMO->getFailureOrdering(); | |||
| 1469 | } | |||
| 1470 | ||||
| 1471 | // Methods to support isa and dyn_cast | |||
| 1472 | static bool classof(const SDNode *N) { | |||
| 1473 | return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || | |||
| 1474 | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || | |||
| 1475 | N->getOpcode() == ISD::ATOMIC_SWAP || | |||
| 1476 | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || | |||
| 1477 | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || | |||
| 1478 | N->getOpcode() == ISD::ATOMIC_LOAD_AND || | |||
| 1479 | N->getOpcode() == ISD::ATOMIC_LOAD_CLR || | |||
| 1480 | N->getOpcode() == ISD::ATOMIC_LOAD_OR || | |||
| 1481 | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || | |||
| 1482 | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || | |||
| 1483 | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || | |||
| 1484 | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || | |||
| 1485 | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || | |||
| 1486 | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || | |||
| 1487 | N->getOpcode() == ISD::ATOMIC_LOAD_FADD || | |||
| 1488 | N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || | |||
| 1489 | N->getOpcode() == ISD::ATOMIC_LOAD_FMAX || | |||
| 1490 | N->getOpcode() == ISD::ATOMIC_LOAD_FMIN || | |||
| 1491 | N->getOpcode() == ISD::ATOMIC_LOAD_UINC_WRAP || | |||
| 1492 | N->getOpcode() == ISD::ATOMIC_LOAD_UDEC_WRAP || | |||
| 1493 | N->getOpcode() == ISD::ATOMIC_LOAD || | |||
| 1494 | N->getOpcode() == ISD::ATOMIC_STORE; | |||
| 1495 | } | |||
| 1496 | }; | |||
| 1497 | ||||
| 1498 | /// This SDNode is used for target intrinsics that touch | |||
| 1499 | /// memory and need an associated MachineMemOperand. Its opcode may be | |||
| 1500 | /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode | |||
| 1501 | /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. | |||
| 1502 | class MemIntrinsicSDNode : public MemSDNode { | |||
| 1503 | public: | |||
| 1504 | MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, | |||
| 1505 | SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) | |||
| 1506 | : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { | |||
| 1507 | SDNodeBits.IsMemIntrinsic = true; | |||
| 1508 | } | |||
| 1509 | ||||
| 1510 | // Methods to support isa and dyn_cast | |||
| 1511 | static bool classof(const SDNode *N) { | |||
| 1512 | // We lower some target intrinsics to their target opcode | |||
| 1513 | // early a node with a target opcode can be of this class | |||
| 1514 | return N->isMemIntrinsic() || | |||
| 1515 | N->getOpcode() == ISD::PREFETCH || | |||
| 1516 | N->isTargetMemoryOpcode(); | |||
| 1517 | } | |||
| 1518 | }; | |||
| 1519 | ||||
| 1520 | /// This SDNode is used to implement the code generator | |||
| 1521 | /// support for the llvm IR shufflevector instruction. It combines elements | |||
| 1522 | /// from two input vectors into a new input vector, with the selection and | |||
| 1523 | /// ordering of elements determined by an array of integers, referred to as | |||
| 1524 | /// the shuffle mask. For input vectors of width N, mask indices of 0..N-1 | |||
| 1525 | /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS. | |||
| 1526 | /// An index of -1 is treated as undef, such that the code generator may put | |||
| 1527 | /// any value in the corresponding element of the result. | |||
| 1528 | class ShuffleVectorSDNode : public SDNode { | |||
| 1529 | // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and | |||
| 1530 | // is freed when the SelectionDAG object is destroyed. | |||
| 1531 | const int *Mask; | |||
| 1532 | ||||
| 1533 | protected: | |||
| 1534 | friend class SelectionDAG; | |||
| 1535 | ||||
| 1536 | ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M) | |||
| 1537 | : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} | |||
| 1538 | ||||
| 1539 | public: | |||
| 1540 | ArrayRef<int> getMask() const { | |||
| 1541 | EVT VT = getValueType(0); | |||
| 1542 | return ArrayRef(Mask, VT.getVectorNumElements()); | |||
| 1543 | } | |||
| 1544 | ||||
| 1545 | int getMaskElt(unsigned Idx) const { | |||
| 1546 | assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements () && "Idx out of range!") ? void (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1546, __extension__ __PRETTY_FUNCTION__)); | |||
| 1547 | return Mask[Idx]; | |||
| 1548 | } | |||
| 1549 | ||||
| 1550 | bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } | |||
| 1551 | ||||
| 1552 | int getSplatIndex() const { | |||
| 1553 | assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!" ) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1553, __extension__ __PRETTY_FUNCTION__)); | |||
| 1554 | EVT VT = getValueType(0); | |||
| 1555 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) | |||
| 1556 | if (Mask[i] >= 0) | |||
| 1557 | return Mask[i]; | |||
| 1558 | ||||
| 1559 | // We can choose any index value here and be correct because all elements | |||
| 1560 | // are undefined. Return 0 for better potential for callers to simplify. | |||
| 1561 | return 0; | |||
| 1562 | } | |||
| 1563 | ||||
| 1564 | static bool isSplatMask(const int *Mask, EVT VT); | |||
| 1565 | ||||
| 1566 | /// Change values in a shuffle permute mask assuming | |||
| 1567 | /// the two vector operands have swapped position. | |||
| 1568 | static void commuteMask(MutableArrayRef<int> Mask) { | |||
| 1569 | unsigned NumElems = Mask.size(); | |||
| 1570 | for (unsigned i = 0; i != NumElems; ++i) { | |||
| 1571 | int idx = Mask[i]; | |||
| 1572 | if (idx < 0) | |||
| 1573 | continue; | |||
| 1574 | else if (idx < (int)NumElems) | |||
| 1575 | Mask[i] = idx + NumElems; | |||
| 1576 | else | |||
| 1577 | Mask[i] = idx - NumElems; | |||
| 1578 | } | |||
| 1579 | } | |||
| 1580 | ||||
| 1581 | static bool classof(const SDNode *N) { | |||
| 1582 | return N->getOpcode() == ISD::VECTOR_SHUFFLE; | |||
| 1583 | } | |||
| 1584 | }; | |||
| 1585 | ||||
| 1586 | class ConstantSDNode : public SDNode { | |||
| 1587 | friend class SelectionDAG; | |||
| 1588 | ||||
| 1589 | const ConstantInt *Value; | |||
| 1590 | ||||
| 1591 | ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT) | |||
| 1592 | : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(), | |||
| 1593 | getSDVTList(VT)), | |||
| 1594 | Value(val) { | |||
| 1595 | ConstantSDNodeBits.IsOpaque = isOpaque; | |||
| 1596 | } | |||
| 1597 | ||||
| 1598 | public: | |||
| 1599 | const ConstantInt *getConstantIntValue() const { return Value; } | |||
| 1600 | const APInt &getAPIntValue() const { return Value->getValue(); } | |||
| 1601 | uint64_t getZExtValue() const { return Value->getZExtValue(); } | |||
| 1602 | int64_t getSExtValue() const { return Value->getSExtValue(); } | |||
| 1603 | uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) { | |||
| 1604 | return Value->getLimitedValue(Limit); | |||
| 1605 | } | |||
| 1606 | MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); } | |||
| 1607 | Align getAlignValue() const { return Value->getAlignValue(); } | |||
| 1608 | ||||
| 1609 | bool isOne() const { return Value->isOne(); } | |||
| 1610 | bool isZero() const { return Value->isZero(); } | |||
| 1611 | LLVM_DEPRECATED("use isZero instead", "isZero")__attribute__((deprecated("use isZero instead", "isZero"))) | |||
| 1612 | bool isNullValue() const { return isZero(); } | |||
| 1613 | bool isAllOnes() const { return Value->isMinusOne(); } | |||
| 1614 | LLVM_DEPRECATED("use isAllOnes instead", "isAllOnes")__attribute__((deprecated("use isAllOnes instead", "isAllOnes" ))) | |||
| 1615 | bool isAllOnesValue() const { return isAllOnes(); } | |||
| 1616 | bool isMaxSignedValue() const { return Value->isMaxValue(true); } | |||
| 1617 | bool isMinSignedValue() const { return Value->isMinValue(true); } | |||
| 1618 | ||||
| 1619 | bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } | |||
| 1620 | ||||
| 1621 | static bool classof(const SDNode *N) { | |||
| 1622 | return N->getOpcode() == ISD::Constant || | |||
| 1623 | N->getOpcode() == ISD::TargetConstant; | |||
| 1624 | } | |||
| 1625 | }; | |||
| 1626 | ||||
| 1627 | uint64_t SDNode::getConstantOperandVal(unsigned Num) const { | |||
| 1628 | return cast<ConstantSDNode>(getOperand(Num))->getZExtValue(); | |||
| 1629 | } | |||
| 1630 | ||||
| 1631 | const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { | |||
| 1632 | return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue(); | |||
| 1633 | } | |||
| 1634 | ||||
| 1635 | class ConstantFPSDNode : public SDNode { | |||
| 1636 | friend class SelectionDAG; | |||
| 1637 | ||||
| 1638 | const ConstantFP *Value; | |||
| 1639 | ||||
| 1640 | ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) | |||
| 1641 | : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, | |||
| 1642 | DebugLoc(), getSDVTList(VT)), | |||
| 1643 | Value(val) {} | |||
| 1644 | ||||
| 1645 | public: | |||
| 1646 | const APFloat& getValueAPF() const { return Value->getValueAPF(); } | |||
| 1647 | const ConstantFP *getConstantFPValue() const { return Value; } | |||
| 1648 | ||||
| 1649 | /// Return true if the value is positive or negative zero. | |||
| 1650 | bool isZero() const { return Value->isZero(); } | |||
| 1651 | ||||
| 1652 | /// Return true if the value is a NaN. | |||
| 1653 | bool isNaN() const { return Value->isNaN(); } | |||
| 1654 | ||||
| 1655 | /// Return true if the value is an infinity | |||
| 1656 | bool isInfinity() const { return Value->isInfinity(); } | |||
| 1657 | ||||
| 1658 | /// Return true if the value is negative. | |||
| 1659 | bool isNegative() const { return Value->isNegative(); } | |||
| 1660 | ||||
| 1661 | /// We don't rely on operator== working on double values, as | |||
| 1662 | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. | |||
| 1663 | /// As such, this method can be used to do an exact bit-for-bit comparison of | |||
| 1664 | /// two floating point values. | |||
| 1665 | ||||
| 1666 | /// We leave the version with the double argument here because it's just so | |||
| 1667 | /// convenient to write "2.0" and the like. Without this function we'd | |||
| 1668 | /// have to duplicate its logic everywhere it's called. | |||
| 1669 | bool isExactlyValue(double V) const { | |||
| 1670 | return Value->getValueAPF().isExactlyValue(V); | |||
| 1671 | } | |||
| 1672 | bool isExactlyValue(const APFloat& V) const; | |||
| 1673 | ||||
| 1674 | static bool isValueValidForType(EVT VT, const APFloat& Val); | |||
| 1675 | ||||
| 1676 | static bool classof(const SDNode *N) { | |||
| 1677 | return N->getOpcode() == ISD::ConstantFP || | |||
| 1678 | N->getOpcode() == ISD::TargetConstantFP; | |||
| 1679 | } | |||
| 1680 | }; | |||
| 1681 | ||||
| 1682 | /// Returns true if \p V is a constant integer zero. | |||
| 1683 | bool isNullConstant(SDValue V); | |||
| 1684 | ||||
| 1685 | /// Returns true if \p V is an FP constant with a value of positive zero. | |||
| 1686 | bool isNullFPConstant(SDValue V); | |||
| 1687 | ||||
| 1688 | /// Returns true if \p V is an integer constant with all bits set. | |||
| 1689 | bool isAllOnesConstant(SDValue V); | |||
| 1690 | ||||
| 1691 | /// Returns true if \p V is a constant integer one. | |||
| 1692 | bool isOneConstant(SDValue V); | |||
| 1693 | ||||
| 1694 | /// Returns true if \p V is a constant min signed integer value. | |||
| 1695 | bool isMinSignedConstant(SDValue V); | |||
| 1696 | ||||
| 1697 | /// Returns true if \p V is a neutral element of Opc with Flags. | |||
| 1698 | /// When OperandNo is 0, it checks that V is a left identity. Otherwise, it | |||
| 1699 | /// checks that V is a right identity. | |||
| 1700 | bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, | |||
| 1701 | unsigned OperandNo); | |||
| 1702 | ||||
| 1703 | /// Return the non-bitcasted source operand of \p V if it exists. | |||
| 1704 | /// If \p V is not a bitcasted value, it is returned as-is. | |||
| 1705 | SDValue peekThroughBitcasts(SDValue V); | |||
| 1706 | ||||
| 1707 | /// Return the non-bitcasted and one-use source operand of \p V if it exists. | |||
| 1708 | /// If \p V is not a bitcasted one-use value, it is returned as-is. | |||
| 1709 | SDValue peekThroughOneUseBitcasts(SDValue V); | |||
| 1710 | ||||
| 1711 | /// Return the non-extracted vector source operand of \p V if it exists. | |||
| 1712 | /// If \p V is not an extracted subvector, it is returned as-is. | |||
| 1713 | SDValue peekThroughExtractSubvectors(SDValue V); | |||
| 1714 | ||||
| 1715 | /// Return the non-truncated source operand of \p V if it exists. | |||
| 1716 | /// If \p V is not a truncation, it is returned as-is. | |||
| 1717 | SDValue peekThroughTruncates(SDValue V); | |||
| 1718 | ||||
| 1719 | /// Returns true if \p V is a bitwise not operation. Assumes that an all ones | |||
| 1720 | /// constant is canonicalized to be operand 1. | |||
| 1721 | bool isBitwiseNot(SDValue V, bool AllowUndefs = false); | |||
| 1722 | ||||
| 1723 | /// If \p V is a bitwise not, returns the inverted operand. Otherwise returns | |||
| 1724 | /// an empty SDValue. Only bits set in \p Mask are required to be inverted, | |||
| 1725 | /// other bits may be arbitrary. | |||
| 1726 | SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs); | |||
| 1727 | ||||
| 1728 | /// Returns the SDNode if it is a constant splat BuildVector or constant int. | |||
| 1729 | ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, | |||
| 1730 | bool AllowTruncation = false); | |||
| 1731 | ||||
| 1732 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
| 1733 | /// constant int. | |||
| 1734 | ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, | |||
| 1735 | bool AllowUndefs = false, | |||
| 1736 | bool AllowTruncation = false); | |||
| 1737 | ||||
| 1738 | /// Returns the SDNode if it is a constant splat BuildVector or constant float. | |||
| 1739 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); | |||
| 1740 | ||||
| 1741 | /// Returns the SDNode if it is a demanded constant splat BuildVector or | |||
| 1742 | /// constant float. | |||
| 1743 | ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, | |||
| 1744 | bool AllowUndefs = false); | |||
| 1745 | ||||
| 1746 | /// Return true if the value is a constant 0 integer or a splatted vector of | |||
| 1747 | /// a constant 0 integer (with no undefs by default). | |||
| 1748 | /// Build vector implicit truncation is not an issue for null values. | |||
| 1749 | bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); | |||
| 1750 | ||||
| 1751 | /// Return true if the value is a constant 1 integer or a splatted vector of a | |||
| 1752 | /// constant 1 integer (with no undefs). | |||
| 1753 | /// Build vector implicit truncation is allowed, but the truncated bits need to | |||
| 1754 | /// be zero. | |||
| 1755 | bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false); | |||
| 1756 | ||||
| 1757 | /// Return true if the value is a constant -1 integer or a splatted vector of a | |||
| 1758 | /// constant -1 integer (with no undefs). | |||
| 1759 | /// Does not permit build vector implicit truncation. | |||
| 1760 | bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false); | |||
| 1761 | ||||
| 1762 | /// Return true if \p V is either a integer or FP constant. | |||
| 1763 | inline bool isIntOrFPConstant(SDValue V) { | |||
| 1764 | return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V); | |||
| 1765 | } | |||
| 1766 | ||||
| 1767 | class GlobalAddressSDNode : public SDNode { | |||
| 1768 | friend class SelectionDAG; | |||
| 1769 | ||||
| 1770 | const GlobalValue *TheGlobal; | |||
| 1771 | int64_t Offset; | |||
| 1772 | unsigned TargetFlags; | |||
| 1773 | ||||
| 1774 | GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, | |||
| 1775 | const GlobalValue *GA, EVT VT, int64_t o, | |||
| 1776 | unsigned TF); | |||
| 1777 | ||||
| 1778 | public: | |||
| 1779 | const GlobalValue *getGlobal() const { return TheGlobal; } | |||
| 1780 | int64_t getOffset() const { return Offset; } | |||
| 1781 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1782 | // Return the address space this GlobalAddress belongs to. | |||
| 1783 | unsigned getAddressSpace() const; | |||
| 1784 | ||||
| 1785 | static bool classof(const SDNode *N) { | |||
| 1786 | return N->getOpcode() == ISD::GlobalAddress || | |||
| 1787 | N->getOpcode() == ISD::TargetGlobalAddress || | |||
| 1788 | N->getOpcode() == ISD::GlobalTLSAddress || | |||
| 1789 | N->getOpcode() == ISD::TargetGlobalTLSAddress; | |||
| 1790 | } | |||
| 1791 | }; | |||
| 1792 | ||||
| 1793 | class FrameIndexSDNode : public SDNode { | |||
| 1794 | friend class SelectionDAG; | |||
| 1795 | ||||
| 1796 | int FI; | |||
| 1797 | ||||
| 1798 | FrameIndexSDNode(int fi, EVT VT, bool isTarg) | |||
| 1799 | : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, | |||
| 1800 | 0, DebugLoc(), getSDVTList(VT)), FI(fi) { | |||
| 1801 | } | |||
| 1802 | ||||
| 1803 | public: | |||
| 1804 | int getIndex() const { return FI; } | |||
| 1805 | ||||
| 1806 | static bool classof(const SDNode *N) { | |||
| 1807 | return N->getOpcode() == ISD::FrameIndex || | |||
| 1808 | N->getOpcode() == ISD::TargetFrameIndex; | |||
| 1809 | } | |||
| 1810 | }; | |||
| 1811 | ||||
| 1812 | /// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate | |||
| 1813 | /// the offet and size that are started/ended in the underlying FrameIndex. | |||
| 1814 | class LifetimeSDNode : public SDNode { | |||
| 1815 | friend class SelectionDAG; | |||
| 1816 | int64_t Size; | |||
| 1817 | int64_t Offset; // -1 if offset is unknown. | |||
| 1818 | ||||
| 1819 | LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, | |||
| 1820 | SDVTList VTs, int64_t Size, int64_t Offset) | |||
| 1821 | : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} | |||
| 1822 | public: | |||
| 1823 | int64_t getFrameIndex() const { | |||
| 1824 | return cast<FrameIndexSDNode>(getOperand(1))->getIndex(); | |||
| 1825 | } | |||
| 1826 | ||||
| 1827 | bool hasOffset() const { return Offset >= 0; } | |||
| 1828 | int64_t getOffset() const { | |||
| 1829 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1829, __extension__ __PRETTY_FUNCTION__)); | |||
| 1830 | return Offset; | |||
| 1831 | } | |||
| 1832 | int64_t getSize() const { | |||
| 1833 | assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown" ) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1833, __extension__ __PRETTY_FUNCTION__)); | |||
| 1834 | return Size; | |||
| 1835 | } | |||
| 1836 | ||||
| 1837 | // Methods to support isa and dyn_cast | |||
| 1838 | static bool classof(const SDNode *N) { | |||
| 1839 | return N->getOpcode() == ISD::LIFETIME_START || | |||
| 1840 | N->getOpcode() == ISD::LIFETIME_END; | |||
| 1841 | } | |||
| 1842 | }; | |||
| 1843 | ||||
| 1844 | /// This SDNode is used for PSEUDO_PROBE values, which are the function guid and | |||
| 1845 | /// the index of the basic block being probed. A pseudo probe serves as a place | |||
| 1846 | /// holder and will be removed at the end of compilation. It does not have any | |||
| 1847 | /// operand because we do not want the instruction selection to deal with any. | |||
| 1848 | class PseudoProbeSDNode : public SDNode { | |||
| 1849 | friend class SelectionDAG; | |||
| 1850 | uint64_t Guid; | |||
| 1851 | uint64_t Index; | |||
| 1852 | uint32_t Attributes; | |||
| 1853 | ||||
| 1854 | PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl, | |||
| 1855 | SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr) | |||
| 1856 | : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index), | |||
| 1857 | Attributes(Attr) {} | |||
| 1858 | ||||
| 1859 | public: | |||
| 1860 | uint64_t getGuid() const { return Guid; } | |||
| 1861 | uint64_t getIndex() const { return Index; } | |||
| 1862 | uint32_t getAttributes() const { return Attributes; } | |||
| 1863 | ||||
| 1864 | // Methods to support isa and dyn_cast | |||
| 1865 | static bool classof(const SDNode *N) { | |||
| 1866 | return N->getOpcode() == ISD::PSEUDO_PROBE; | |||
| 1867 | } | |||
| 1868 | }; | |||
| 1869 | ||||
| 1870 | class JumpTableSDNode : public SDNode { | |||
| 1871 | friend class SelectionDAG; | |||
| 1872 | ||||
| 1873 | int JTI; | |||
| 1874 | unsigned TargetFlags; | |||
| 1875 | ||||
| 1876 | JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF) | |||
| 1877 | : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, | |||
| 1878 | 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { | |||
| 1879 | } | |||
| 1880 | ||||
| 1881 | public: | |||
| 1882 | int getIndex() const { return JTI; } | |||
| 1883 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1884 | ||||
| 1885 | static bool classof(const SDNode *N) { | |||
| 1886 | return N->getOpcode() == ISD::JumpTable || | |||
| 1887 | N->getOpcode() == ISD::TargetJumpTable; | |||
| 1888 | } | |||
| 1889 | }; | |||
| 1890 | ||||
| 1891 | class ConstantPoolSDNode : public SDNode { | |||
| 1892 | friend class SelectionDAG; | |||
| 1893 | ||||
| 1894 | union { | |||
| 1895 | const Constant *ConstVal; | |||
| 1896 | MachineConstantPoolValue *MachineCPVal; | |||
| 1897 | } Val; | |||
| 1898 | int Offset; // It's a MachineConstantPoolValue if top bit is set. | |||
| 1899 | Align Alignment; // Minimum alignment requirement of CP. | |||
| 1900 | unsigned TargetFlags; | |||
| 1901 | ||||
| 1902 | ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, | |||
| 1903 | Align Alignment, unsigned TF) | |||
| 1904 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
| 1905 | DebugLoc(), getSDVTList(VT)), | |||
| 1906 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
| 1907 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1907, __extension__ __PRETTY_FUNCTION__)); | |||
| 1908 | Val.ConstVal = c; | |||
| 1909 | } | |||
| 1910 | ||||
| 1911 | ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o, | |||
| 1912 | Align Alignment, unsigned TF) | |||
| 1913 | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, | |||
| 1914 | DebugLoc(), getSDVTList(VT)), | |||
| 1915 | Offset(o), Alignment(Alignment), TargetFlags(TF) { | |||
| 1916 | assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large" ) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1916, __extension__ __PRETTY_FUNCTION__)); | |||
| 1917 | Val.MachineCPVal = v; | |||
| 1918 | Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1); | |||
| 1919 | } | |||
| 1920 | ||||
| 1921 | public: | |||
| 1922 | bool isMachineConstantPoolEntry() const { | |||
| 1923 | return Offset < 0; | |||
| 1924 | } | |||
| 1925 | ||||
| 1926 | const Constant *getConstVal() const { | |||
| 1927 | assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1927, __extension__ __PRETTY_FUNCTION__)); | |||
| 1928 | return Val.ConstVal; | |||
| 1929 | } | |||
| 1930 | ||||
| 1931 | MachineConstantPoolValue *getMachineCPVal() const { | |||
| 1932 | assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() && "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1932, __extension__ __PRETTY_FUNCTION__)); | |||
| 1933 | return Val.MachineCPVal; | |||
| 1934 | } | |||
| 1935 | ||||
| 1936 | int getOffset() const { | |||
| 1937 | return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1)); | |||
| 1938 | } | |||
| 1939 | ||||
| 1940 | // Return the alignment of this constant pool object, which is either 0 (for | |||
| 1941 | // default alignment) or the desired value. | |||
| 1942 | Align getAlign() const { return Alignment; } | |||
| 1943 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1944 | ||||
| 1945 | Type *getType() const; | |||
| 1946 | ||||
| 1947 | static bool classof(const SDNode *N) { | |||
| 1948 | return N->getOpcode() == ISD::ConstantPool || | |||
| 1949 | N->getOpcode() == ISD::TargetConstantPool; | |||
| 1950 | } | |||
| 1951 | }; | |||
| 1952 | ||||
| 1953 | /// Completely target-dependent object reference. | |||
| 1954 | class TargetIndexSDNode : public SDNode { | |||
| 1955 | friend class SelectionDAG; | |||
| 1956 | ||||
| 1957 | unsigned TargetFlags; | |||
| 1958 | int Index; | |||
| 1959 | int64_t Offset; | |||
| 1960 | ||||
| 1961 | public: | |||
| 1962 | TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF) | |||
| 1963 | : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), | |||
| 1964 | TargetFlags(TF), Index(Idx), Offset(Ofs) {} | |||
| 1965 | ||||
| 1966 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 1967 | int getIndex() const { return Index; } | |||
| 1968 | int64_t getOffset() const { return Offset; } | |||
| 1969 | ||||
| 1970 | static bool classof(const SDNode *N) { | |||
| 1971 | return N->getOpcode() == ISD::TargetIndex; | |||
| 1972 | } | |||
| 1973 | }; | |||
| 1974 | ||||
| 1975 | class BasicBlockSDNode : public SDNode { | |||
| 1976 | friend class SelectionDAG; | |||
| 1977 | ||||
| 1978 | MachineBasicBlock *MBB; | |||
| 1979 | ||||
| 1980 | /// Debug info is meaningful and potentially useful here, but we create | |||
| 1981 | /// blocks out of order when they're jumped to, which makes it a bit | |||
| 1982 | /// harder. Let's see if we need it first. | |||
| 1983 | explicit BasicBlockSDNode(MachineBasicBlock *mbb) | |||
| 1984 | : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) | |||
| 1985 | {} | |||
| 1986 | ||||
| 1987 | public: | |||
| 1988 | MachineBasicBlock *getBasicBlock() const { return MBB; } | |||
| 1989 | ||||
| 1990 | static bool classof(const SDNode *N) { | |||
| 1991 | return N->getOpcode() == ISD::BasicBlock; | |||
| 1992 | } | |||
| 1993 | }; | |||
| 1994 | ||||
| 1995 | /// A "pseudo-class" with methods for operating on BUILD_VECTORs. | |||
| 1996 | class BuildVectorSDNode : public SDNode { | |||
| 1997 | public: | |||
| 1998 | // These are constructed as SDNodes and then cast to BuildVectorSDNodes. | |||
| 1999 | explicit BuildVectorSDNode() = delete; | |||
| 2000 | ||||
| 2001 | /// Check if this is a constant splat, and if so, find the | |||
| 2002 | /// smallest element size that splats the vector. If MinSplatBits is | |||
| 2003 | /// nonzero, the element size must be at least that large. Note that the | |||
| 2004 | /// splat element may be the entire vector (i.e., a one element vector). | |||
| 2005 | /// Returns the splat element value in SplatValue. Any undefined bits in | |||
| 2006 | /// that value are zero, and the corresponding bits in the SplatUndef mask | |||
| 2007 | /// are set. The SplatBitSize value is set to the splat element size in | |||
| 2008 | /// bits. HasAnyUndefs is set to true if any bits in the vector are | |||
| 2009 | /// undefined. isBigEndian describes the endianness of the target. | |||
| 2010 | bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, | |||
| 2011 | unsigned &SplatBitSize, bool &HasAnyUndefs, | |||
| 2012 | unsigned MinSplatBits = 0, | |||
| 2013 | bool isBigEndian = false) const; | |||
| 2014 | ||||
| 2015 | /// Returns the demanded splatted value or a null value if this is not a | |||
| 2016 | /// splat. | |||
| 2017 | /// | |||
| 2018 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
| 2019 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2020 | /// the vector width and set the bits where elements are undef. | |||
| 2021 | SDValue getSplatValue(const APInt &DemandedElts, | |||
| 2022 | BitVector *UndefElements = nullptr) const; | |||
| 2023 | ||||
| 2024 | /// Returns the splatted value or a null value if this is not a splat. | |||
| 2025 | /// | |||
| 2026 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2027 | /// the vector width and set the bits where elements are undef. | |||
| 2028 | SDValue getSplatValue(BitVector *UndefElements = nullptr) const; | |||
| 2029 | ||||
| 2030 | /// Find the shortest repeating sequence of values in the build vector. | |||
| 2031 | /// | |||
| 2032 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
| 2033 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
| 2034 | /// | |||
| 2035 | /// Currently this must be a power-of-2 build vector. | |||
| 2036 | /// The DemandedElts mask indicates the elements that must be present, | |||
| 2037 | /// undemanded elements in Sequence may be null (SDValue()). If passed a | |||
| 2038 | /// non-null UndefElements bitvector, it will resize it to match the original | |||
| 2039 | /// vector width and set the bits where elements are undef. If result is | |||
| 2040 | /// false, Sequence will be empty. | |||
| 2041 | bool getRepeatedSequence(const APInt &DemandedElts, | |||
| 2042 | SmallVectorImpl<SDValue> &Sequence, | |||
| 2043 | BitVector *UndefElements = nullptr) const; | |||
| 2044 | ||||
| 2045 | /// Find the shortest repeating sequence of values in the build vector. | |||
| 2046 | /// | |||
| 2047 | /// e.g. { u, X, u, X, u, u, X, u } -> { X } | |||
| 2048 | /// { X, Y, u, Y, u, u, X, u } -> { X, Y } | |||
| 2049 | /// | |||
| 2050 | /// Currently this must be a power-of-2 build vector. | |||
| 2051 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2052 | /// the original vector width and set the bits where elements are undef. | |||
| 2053 | /// If result is false, Sequence will be empty. | |||
| 2054 | bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, | |||
| 2055 | BitVector *UndefElements = nullptr) const; | |||
| 2056 | ||||
| 2057 | /// Returns the demanded splatted constant or null if this is not a constant | |||
| 2058 | /// splat. | |||
| 2059 | /// | |||
| 2060 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
| 2061 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2062 | /// the vector width and set the bits where elements are undef. | |||
| 2063 | ConstantSDNode * | |||
| 2064 | getConstantSplatNode(const APInt &DemandedElts, | |||
| 2065 | BitVector *UndefElements = nullptr) const; | |||
| 2066 | ||||
| 2067 | /// Returns the splatted constant or null if this is not a constant | |||
| 2068 | /// splat. | |||
| 2069 | /// | |||
| 2070 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2071 | /// the vector width and set the bits where elements are undef. | |||
| 2072 | ConstantSDNode * | |||
| 2073 | getConstantSplatNode(BitVector *UndefElements = nullptr) const; | |||
| 2074 | ||||
| 2075 | /// Returns the demanded splatted constant FP or null if this is not a | |||
| 2076 | /// constant FP splat. | |||
| 2077 | /// | |||
| 2078 | /// The DemandedElts mask indicates the elements that must be in the splat. | |||
| 2079 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2080 | /// the vector width and set the bits where elements are undef. | |||
| 2081 | ConstantFPSDNode * | |||
| 2082 | getConstantFPSplatNode(const APInt &DemandedElts, | |||
| 2083 | BitVector *UndefElements = nullptr) const; | |||
| 2084 | ||||
| 2085 | /// Returns the splatted constant FP or null if this is not a constant | |||
| 2086 | /// FP splat. | |||
| 2087 | /// | |||
| 2088 | /// If passed a non-null UndefElements bitvector, it will resize it to match | |||
| 2089 | /// the vector width and set the bits where elements are undef. | |||
| 2090 | ConstantFPSDNode * | |||
| 2091 | getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; | |||
| 2092 | ||||
| 2093 | /// If this is a constant FP splat and the splatted constant FP is an | |||
| 2094 | /// exact power or 2, return the log base 2 integer value. Otherwise, | |||
| 2095 | /// return -1. | |||
| 2096 | /// | |||
| 2097 | /// The BitWidth specifies the necessary bit precision. | |||
| 2098 | int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, | |||
| 2099 | uint32_t BitWidth) const; | |||
| 2100 | ||||
| 2101 | /// Extract the raw bit data from a build vector of Undef, Constant or | |||
| 2102 | /// ConstantFP node elements. Each raw bit element will be \p | |||
| 2103 | /// DstEltSizeInBits wide, undef elements are treated as zero, and entirely | |||
| 2104 | /// undefined elements are flagged in \p UndefElements. | |||
| 2105 | bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, | |||
| 2106 | SmallVectorImpl<APInt> &RawBitElements, | |||
| 2107 | BitVector &UndefElements) const; | |||
| 2108 | ||||
| 2109 | bool isConstant() const; | |||
| 2110 | ||||
| 2111 | /// If this BuildVector is constant and represents the numerical series | |||
| 2112 | /// "<a, a+n, a+2n, a+3n, ...>" where a is integer and n is a non-zero integer, | |||
| 2113 | /// the value "<a,n>" is returned. | |||
| 2114 | std::optional<std::pair<APInt, APInt>> isConstantSequence() const; | |||
| 2115 | ||||
| 2116 | /// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements. | |||
| 2117 | /// Undef elements are treated as zero, and entirely undefined elements are | |||
| 2118 | /// flagged in \p DstUndefElements. | |||
| 2119 | static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, | |||
| 2120 | SmallVectorImpl<APInt> &DstBitElements, | |||
| 2121 | ArrayRef<APInt> SrcBitElements, | |||
| 2122 | BitVector &DstUndefElements, | |||
| 2123 | const BitVector &SrcUndefElements); | |||
| 2124 | ||||
| 2125 | static bool classof(const SDNode *N) { | |||
| 2126 | return N->getOpcode() == ISD::BUILD_VECTOR; | |||
| 2127 | } | |||
| 2128 | }; | |||
| 2129 | ||||
| 2130 | /// An SDNode that holds an arbitrary LLVM IR Value. This is | |||
| 2131 | /// used when the SelectionDAG needs to make a simple reference to something | |||
| 2132 | /// in the LLVM IR representation. | |||
| 2133 | /// | |||
| 2134 | class SrcValueSDNode : public SDNode { | |||
| 2135 | friend class SelectionDAG; | |||
| 2136 | ||||
| 2137 | const Value *V; | |||
| 2138 | ||||
| 2139 | /// Create a SrcValue for a general value. | |||
| 2140 | explicit SrcValueSDNode(const Value *v) | |||
| 2141 | : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} | |||
| 2142 | ||||
| 2143 | public: | |||
| 2144 | /// Return the contained Value. | |||
| 2145 | const Value *getValue() const { return V; } | |||
| 2146 | ||||
| 2147 | static bool classof(const SDNode *N) { | |||
| 2148 | return N->getOpcode() == ISD::SRCVALUE; | |||
| 2149 | } | |||
| 2150 | }; | |||
| 2151 | ||||
| 2152 | class MDNodeSDNode : public SDNode { | |||
| 2153 | friend class SelectionDAG; | |||
| 2154 | ||||
| 2155 | const MDNode *MD; | |||
| 2156 | ||||
| 2157 | explicit MDNodeSDNode(const MDNode *md) | |||
| 2158 | : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) | |||
| 2159 | {} | |||
| 2160 | ||||
| 2161 | public: | |||
| 2162 | const MDNode *getMD() const { return MD; } | |||
| 2163 | ||||
| 2164 | static bool classof(const SDNode *N) { | |||
| 2165 | return N->getOpcode() == ISD::MDNODE_SDNODE; | |||
| 2166 | } | |||
| 2167 | }; | |||
| 2168 | ||||
| 2169 | class RegisterSDNode : public SDNode { | |||
| 2170 | friend class SelectionDAG; | |||
| 2171 | ||||
| 2172 | Register Reg; | |||
| 2173 | ||||
| 2174 | RegisterSDNode(Register reg, EVT VT) | |||
| 2175 | : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {} | |||
| 2176 | ||||
| 2177 | public: | |||
| 2178 | Register getReg() const { return Reg; } | |||
| 2179 | ||||
| 2180 | static bool classof(const SDNode *N) { | |||
| 2181 | return N->getOpcode() == ISD::Register; | |||
| 2182 | } | |||
| 2183 | }; | |||
| 2184 | ||||
| 2185 | class RegisterMaskSDNode : public SDNode { | |||
| 2186 | friend class SelectionDAG; | |||
| 2187 | ||||
| 2188 | // The memory for RegMask is not owned by the node. | |||
| 2189 | const uint32_t *RegMask; | |||
| 2190 | ||||
| 2191 | RegisterMaskSDNode(const uint32_t *mask) | |||
| 2192 | : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)), | |||
| 2193 | RegMask(mask) {} | |||
| 2194 | ||||
| 2195 | public: | |||
| 2196 | const uint32_t *getRegMask() const { return RegMask; } | |||
| 2197 | ||||
| 2198 | static bool classof(const SDNode *N) { | |||
| 2199 | return N->getOpcode() == ISD::RegisterMask; | |||
| 2200 | } | |||
| 2201 | }; | |||
| 2202 | ||||
| 2203 | class BlockAddressSDNode : public SDNode { | |||
| 2204 | friend class SelectionDAG; | |||
| 2205 | ||||
| 2206 | const BlockAddress *BA; | |||
| 2207 | int64_t Offset; | |||
| 2208 | unsigned TargetFlags; | |||
| 2209 | ||||
| 2210 | BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, | |||
| 2211 | int64_t o, unsigned Flags) | |||
| 2212 | : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), | |||
| 2213 | BA(ba), Offset(o), TargetFlags(Flags) {} | |||
| 2214 | ||||
| 2215 | public: | |||
| 2216 | const BlockAddress *getBlockAddress() const { return BA; } | |||
| 2217 | int64_t getOffset() const { return Offset; } | |||
| 2218 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 2219 | ||||
| 2220 | static bool classof(const SDNode *N) { | |||
| 2221 | return N->getOpcode() == ISD::BlockAddress || | |||
| 2222 | N->getOpcode() == ISD::TargetBlockAddress; | |||
| 2223 | } | |||
| 2224 | }; | |||
| 2225 | ||||
| 2226 | class LabelSDNode : public SDNode { | |||
| 2227 | friend class SelectionDAG; | |||
| 2228 | ||||
| 2229 | MCSymbol *Label; | |||
| 2230 | ||||
| 2231 | LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L) | |||
| 2232 | : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) { | |||
| 2233 | assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) && "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2233, __extension__ __PRETTY_FUNCTION__)); | |||
| 2234 | } | |||
| 2235 | ||||
| 2236 | public: | |||
| 2237 | MCSymbol *getLabel() const { return Label; } | |||
| 2238 | ||||
| 2239 | static bool classof(const SDNode *N) { | |||
| 2240 | return N->getOpcode() == ISD::EH_LABEL || | |||
| 2241 | N->getOpcode() == ISD::ANNOTATION_LABEL; | |||
| 2242 | } | |||
| 2243 | }; | |||
| 2244 | ||||
| 2245 | class ExternalSymbolSDNode : public SDNode { | |||
| 2246 | friend class SelectionDAG; | |||
| 2247 | ||||
| 2248 | const char *Symbol; | |||
| 2249 | unsigned TargetFlags; | |||
| 2250 | ||||
| 2251 | ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT) | |||
| 2252 | : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0, | |||
| 2253 | DebugLoc(), getSDVTList(VT)), | |||
| 2254 | Symbol(Sym), TargetFlags(TF) {} | |||
| 2255 | ||||
| 2256 | public: | |||
| 2257 | const char *getSymbol() const { return Symbol; } | |||
| 2258 | unsigned getTargetFlags() const { return TargetFlags; } | |||
| 2259 | ||||
| 2260 | static bool classof(const SDNode *N) { | |||
| 2261 | return N->getOpcode() == ISD::ExternalSymbol || | |||
| 2262 | N->getOpcode() == ISD::TargetExternalSymbol; | |||
| 2263 | } | |||
| 2264 | }; | |||
| 2265 | ||||
| 2266 | class MCSymbolSDNode : public SDNode { | |||
| 2267 | friend class SelectionDAG; | |||
| 2268 | ||||
| 2269 | MCSymbol *Symbol; | |||
| 2270 | ||||
| 2271 | MCSymbolSDNode(MCSymbol *Symbol, EVT VT) | |||
| 2272 | : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {} | |||
| 2273 | ||||
| 2274 | public: | |||
| 2275 | MCSymbol *getMCSymbol() const { return Symbol; } | |||
| 2276 | ||||
| 2277 | static bool classof(const SDNode *N) { | |||
| 2278 | return N->getOpcode() == ISD::MCSymbol; | |||
| 2279 | } | |||
| 2280 | }; | |||
| 2281 | ||||
| 2282 | class CondCodeSDNode : public SDNode { | |||
| 2283 | friend class SelectionDAG; | |||
| 2284 | ||||
| 2285 | ISD::CondCode Condition; | |||
| 2286 | ||||
| 2287 | explicit CondCodeSDNode(ISD::CondCode Cond) | |||
| 2288 | : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
| 2289 | Condition(Cond) {} | |||
| 2290 | ||||
| 2291 | public: | |||
| 2292 | ISD::CondCode get() const { return Condition; } | |||
| 2293 | ||||
| 2294 | static bool classof(const SDNode *N) { | |||
| 2295 | return N->getOpcode() == ISD::CONDCODE; | |||
| 2296 | } | |||
| 2297 | }; | |||
| 2298 | ||||
| 2299 | /// This class is used to represent EVT's, which are used | |||
| 2300 | /// to parameterize some operations. | |||
| 2301 | class VTSDNode : public SDNode { | |||
| 2302 | friend class SelectionDAG; | |||
| 2303 | ||||
| 2304 | EVT ValueType; | |||
| 2305 | ||||
| 2306 | explicit VTSDNode(EVT VT) | |||
| 2307 | : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)), | |||
| 2308 | ValueType(VT) {} | |||
| 2309 | ||||
| 2310 | public: | |||
| 2311 | EVT getVT() const { return ValueType; } | |||
| 2312 | ||||
| 2313 | static bool classof(const SDNode *N) { | |||
| 2314 | return N->getOpcode() == ISD::VALUETYPE; | |||
| 2315 | } | |||
| 2316 | }; | |||
| 2317 | ||||
| 2318 | /// Base class for LoadSDNode and StoreSDNode | |||
| 2319 | class LSBaseSDNode : public MemSDNode { | |||
| 2320 | public: | |||
| 2321 | LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, | |||
| 2322 | SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, | |||
| 2323 | MachineMemOperand *MMO) | |||
| 2324 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
| 2325 | LSBaseSDNodeBits.AddressingMode = AM; | |||
| 2326 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2326, __extension__ __PRETTY_FUNCTION__)); | |||
| 2327 | } | |||
| 2328 | ||||
| 2329 | const SDValue &getOffset() const { | |||
| 2330 | return getOperand(getOpcode() == ISD::LOAD ? 2 : 3); | |||
| 2331 | } | |||
| 2332 | ||||
| 2333 | /// Return the addressing mode for this load or store: | |||
| 2334 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
| 2335 | ISD::MemIndexedMode getAddressingMode() const { | |||
| 2336 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
| 2337 | } | |||
| 2338 | ||||
| 2339 | /// Return true if this is a pre/post inc/dec load/store. | |||
| 2340 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
| 2341 | ||||
| 2342 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
| 2343 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
| 2344 | ||||
| 2345 | static bool classof(const SDNode *N) { | |||
| 2346 | return N->getOpcode() == ISD::LOAD || | |||
| 2347 | N->getOpcode() == ISD::STORE; | |||
| 2348 | } | |||
| 2349 | }; | |||
| 2350 | ||||
| 2351 | /// This class is used to represent ISD::LOAD nodes. | |||
| 2352 | class LoadSDNode : public LSBaseSDNode { | |||
| 2353 | friend class SelectionDAG; | |||
| 2354 | ||||
| 2355 | LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2356 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, | |||
| 2357 | MachineMemOperand *MMO) | |||
| 2358 | : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2359 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2360 | assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!" ) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2360, __extension__ __PRETTY_FUNCTION__)); | |||
| 2361 | assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!" ) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2361, __extension__ __PRETTY_FUNCTION__)); | |||
| 2362 | } | |||
| 2363 | ||||
| 2364 | public: | |||
| 2365 | /// Return whether this is a plain node, | |||
| 2366 | /// or one of the varieties of value-extending loads. | |||
| 2367 | ISD::LoadExtType getExtensionType() const { | |||
| 2368 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
| 2369 | } | |||
| 2370 | ||||
| 2371 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 2372 | const SDValue &getOffset() const { return getOperand(2); } | |||
| 2373 | ||||
| 2374 | static bool classof(const SDNode *N) { | |||
| 2375 | return N->getOpcode() == ISD::LOAD; | |||
| 2376 | } | |||
| 2377 | }; | |||
| 2378 | ||||
| 2379 | /// This class is used to represent ISD::STORE nodes. | |||
| 2380 | class StoreSDNode : public LSBaseSDNode { | |||
| 2381 | friend class SelectionDAG; | |||
| 2382 | ||||
| 2383 | StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2384 | ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, | |||
| 2385 | MachineMemOperand *MMO) | |||
| 2386 | : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2387 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
| 2388 | assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!" ) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2388, __extension__ __PRETTY_FUNCTION__)); | |||
| 2389 | assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!" ) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2389, __extension__ __PRETTY_FUNCTION__)); | |||
| 2390 | } | |||
| 2391 | ||||
| 2392 | public: | |||
| 2393 | /// Return true if the op does a truncation before store. | |||
| 2394 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2395 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2396 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2397 | void setTruncatingStore(bool Truncating) { | |||
| 2398 | StoreSDNodeBits.IsTruncating = Truncating; | |||
| 2399 | } | |||
| 2400 | ||||
| 2401 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2402 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
| 2403 | const SDValue &getOffset() const { return getOperand(3); } | |||
| 2404 | ||||
| 2405 | static bool classof(const SDNode *N) { | |||
| 2406 | return N->getOpcode() == ISD::STORE; | |||
| 2407 | } | |||
| 2408 | }; | |||
| 2409 | ||||
| 2410 | /// This base class is used to represent VP_LOAD, VP_STORE, | |||
| 2411 | /// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes | |||
| 2412 | class VPBaseLoadStoreSDNode : public MemSDNode { | |||
| 2413 | public: | |||
| 2414 | friend class SelectionDAG; | |||
| 2415 | ||||
| 2416 | VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
| 2417 | const DebugLoc &DL, SDVTList VTs, | |||
| 2418 | ISD::MemIndexedMode AM, EVT MemVT, | |||
| 2419 | MachineMemOperand *MMO) | |||
| 2420 | : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) { | |||
| 2421 | LSBaseSDNodeBits.AddressingMode = AM; | |||
| 2422 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2422, __extension__ __PRETTY_FUNCTION__)); | |||
| 2423 | } | |||
| 2424 | ||||
| 2425 | // VPStridedStoreSDNode (Chain, Data, Ptr, Offset, Stride, Mask, EVL) | |||
| 2426 | // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL) | |||
| 2427 | // VPStridedLoadSDNode (Chain, Ptr, Offset, Stride, Mask, EVL) | |||
| 2428 | // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL) | |||
| 2429 | // Mask is a vector of i1 elements; | |||
| 2430 | // the type of EVL is TLI.getVPExplicitVectorLengthTy(). | |||
| 2431 | const SDValue &getOffset() const { | |||
| 2432 | return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || | |||
| 2433 | getOpcode() == ISD::VP_LOAD) | |||
| 2434 | ? 2 | |||
| 2435 | : 3); | |||
| 2436 | } | |||
| 2437 | const SDValue &getBasePtr() const { | |||
| 2438 | return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || | |||
| 2439 | getOpcode() == ISD::VP_LOAD) | |||
| 2440 | ? 1 | |||
| 2441 | : 2); | |||
| 2442 | } | |||
| 2443 | const SDValue &getMask() const { | |||
| 2444 | switch (getOpcode()) { | |||
| 2445 | default: | |||
| 2446 | llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2446); | |||
| 2447 | case ISD::VP_LOAD: | |||
| 2448 | return getOperand(3); | |||
| 2449 | case ISD::VP_STORE: | |||
| 2450 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: | |||
| 2451 | return getOperand(4); | |||
| 2452 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: | |||
| 2453 | return getOperand(5); | |||
| 2454 | } | |||
| 2455 | } | |||
| 2456 | const SDValue &getVectorLength() const { | |||
| 2457 | switch (getOpcode()) { | |||
| 2458 | default: | |||
| 2459 | llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h" , 2459); | |||
| 2460 | case ISD::VP_LOAD: | |||
| 2461 | return getOperand(4); | |||
| 2462 | case ISD::VP_STORE: | |||
| 2463 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: | |||
| 2464 | return getOperand(5); | |||
| 2465 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: | |||
| 2466 | return getOperand(6); | |||
| 2467 | } | |||
| 2468 | } | |||
| 2469 | ||||
| 2470 | /// Return the addressing mode for this load or store: | |||
| 2471 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
| 2472 | ISD::MemIndexedMode getAddressingMode() const { | |||
| 2473 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
| 2474 | } | |||
| 2475 | ||||
| 2476 | /// Return true if this is a pre/post inc/dec load/store. | |||
| 2477 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
| 2478 | ||||
| 2479 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
| 2480 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
| 2481 | ||||
| 2482 | static bool classof(const SDNode *N) { | |||
| 2483 | return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD || | |||
| 2484 | N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE || | |||
| 2485 | N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE; | |||
| 2486 | } | |||
| 2487 | }; | |||
| 2488 | ||||
| 2489 | /// This class is used to represent a VP_LOAD node | |||
| 2490 | class VPLoadSDNode : public VPBaseLoadStoreSDNode { | |||
| 2491 | public: | |||
| 2492 | friend class SelectionDAG; | |||
| 2493 | ||||
| 2494 | VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2495 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding, | |||
| 2496 | EVT MemVT, MachineMemOperand *MMO) | |||
| 2497 | : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2498 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2499 | LoadSDNodeBits.IsExpanding = isExpanding; | |||
| 2500 | } | |||
| 2501 | ||||
| 2502 | ISD::LoadExtType getExtensionType() const { | |||
| 2503 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
| 2504 | } | |||
| 2505 | ||||
| 2506 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 2507 | const SDValue &getOffset() const { return getOperand(2); } | |||
| 2508 | const SDValue &getMask() const { return getOperand(3); } | |||
| 2509 | const SDValue &getVectorLength() const { return getOperand(4); } | |||
| 2510 | ||||
| 2511 | static bool classof(const SDNode *N) { | |||
| 2512 | return N->getOpcode() == ISD::VP_LOAD; | |||
| 2513 | } | |||
| 2514 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } | |||
| 2515 | }; | |||
| 2516 | ||||
| 2517 | /// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node. | |||
| 2518 | class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode { | |||
| 2519 | public: | |||
| 2520 | friend class SelectionDAG; | |||
| 2521 | ||||
| 2522 | VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs, | |||
| 2523 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, | |||
| 2524 | bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) | |||
| 2525 | : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs, | |||
| 2526 | AM, MemVT, MMO) { | |||
| 2527 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2528 | LoadSDNodeBits.IsExpanding = IsExpanding; | |||
| 2529 | } | |||
| 2530 | ||||
| 2531 | ISD::LoadExtType getExtensionType() const { | |||
| 2532 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
| 2533 | } | |||
| 2534 | ||||
| 2535 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 2536 | const SDValue &getOffset() const { return getOperand(2); } | |||
| 2537 | const SDValue &getStride() const { return getOperand(3); } | |||
| 2538 | const SDValue &getMask() const { return getOperand(4); } | |||
| 2539 | const SDValue &getVectorLength() const { return getOperand(5); } | |||
| 2540 | ||||
| 2541 | static bool classof(const SDNode *N) { | |||
| 2542 | return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD; | |||
| 2543 | } | |||
| 2544 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } | |||
| 2545 | }; | |||
| 2546 | ||||
| 2547 | /// This class is used to represent a VP_STORE node | |||
| 2548 | class VPStoreSDNode : public VPBaseLoadStoreSDNode { | |||
| 2549 | public: | |||
| 2550 | friend class SelectionDAG; | |||
| 2551 | ||||
| 2552 | VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2553 | ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, | |||
| 2554 | EVT MemVT, MachineMemOperand *MMO) | |||
| 2555 | : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2556 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
| 2557 | StoreSDNodeBits.IsCompressing = isCompressing; | |||
| 2558 | } | |||
| 2559 | ||||
| 2560 | /// Return true if this is a truncating store. | |||
| 2561 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2562 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2563 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2564 | ||||
| 2565 | /// Returns true if the op does a compression to the vector before storing. | |||
| 2566 | /// The node contiguously stores the active elements (integers or floats) | |||
| 2567 | /// in src (those with their respective bit set in writemask k) to unaligned | |||
| 2568 | /// memory at base_addr. | |||
| 2569 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } | |||
| 2570 | ||||
| 2571 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2572 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
| 2573 | const SDValue &getOffset() const { return getOperand(3); } | |||
| 2574 | const SDValue &getMask() const { return getOperand(4); } | |||
| 2575 | const SDValue &getVectorLength() const { return getOperand(5); } | |||
| 2576 | ||||
| 2577 | static bool classof(const SDNode *N) { | |||
| 2578 | return N->getOpcode() == ISD::VP_STORE; | |||
| 2579 | } | |||
| 2580 | }; | |||
| 2581 | ||||
| 2582 | /// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node. | |||
| 2583 | class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode { | |||
| 2584 | public: | |||
| 2585 | friend class SelectionDAG; | |||
| 2586 | ||||
| 2587 | VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs, | |||
| 2588 | ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing, | |||
| 2589 | EVT MemVT, MachineMemOperand *MMO) | |||
| 2590 | : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL, | |||
| 2591 | VTs, AM, MemVT, MMO) { | |||
| 2592 | StoreSDNodeBits.IsTruncating = IsTrunc; | |||
| 2593 | StoreSDNodeBits.IsCompressing = IsCompressing; | |||
| 2594 | } | |||
| 2595 | ||||
| 2596 | /// Return true if this is a truncating store. | |||
| 2597 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2598 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2599 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2600 | ||||
| 2601 | /// Returns true if the op does a compression to the vector before storing. | |||
| 2602 | /// The node contiguously stores the active elements (integers or floats) | |||
| 2603 | /// in src (those with their respective bit set in writemask k) to unaligned | |||
| 2604 | /// memory at base_addr. | |||
| 2605 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } | |||
| 2606 | ||||
| 2607 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2608 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
| 2609 | const SDValue &getOffset() const { return getOperand(3); } | |||
| 2610 | const SDValue &getStride() const { return getOperand(4); } | |||
| 2611 | const SDValue &getMask() const { return getOperand(5); } | |||
| 2612 | const SDValue &getVectorLength() const { return getOperand(6); } | |||
| 2613 | ||||
| 2614 | static bool classof(const SDNode *N) { | |||
| 2615 | return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE; | |||
| 2616 | } | |||
| 2617 | }; | |||
| 2618 | ||||
| 2619 | /// This base class is used to represent MLOAD and MSTORE nodes | |||
| 2620 | class MaskedLoadStoreSDNode : public MemSDNode { | |||
| 2621 | public: | |||
| 2622 | friend class SelectionDAG; | |||
| 2623 | ||||
| 2624 | MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
| 2625 | const DebugLoc &dl, SDVTList VTs, | |||
| 2626 | ISD::MemIndexedMode AM, EVT MemVT, | |||
| 2627 | MachineMemOperand *MMO) | |||
| 2628 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
| 2629 | LSBaseSDNodeBits.AddressingMode = AM; | |||
| 2630 | assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM && "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2630, __extension__ __PRETTY_FUNCTION__)); | |||
| 2631 | } | |||
| 2632 | ||||
| 2633 | // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru) | |||
| 2634 | // MaskedStoreSDNode (Chain, data, ptr, offset, mask) | |||
| 2635 | // Mask is a vector of i1 elements | |||
| 2636 | const SDValue &getOffset() const { | |||
| 2637 | return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3); | |||
| 2638 | } | |||
| 2639 | const SDValue &getMask() const { | |||
| 2640 | return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4); | |||
| 2641 | } | |||
| 2642 | ||||
| 2643 | /// Return the addressing mode for this load or store: | |||
| 2644 | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. | |||
| 2645 | ISD::MemIndexedMode getAddressingMode() const { | |||
| 2646 | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); | |||
| 2647 | } | |||
| 2648 | ||||
| 2649 | /// Return true if this is a pre/post inc/dec load/store. | |||
| 2650 | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } | |||
| 2651 | ||||
| 2652 | /// Return true if this is NOT a pre/post inc/dec load/store. | |||
| 2653 | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } | |||
| 2654 | ||||
| 2655 | static bool classof(const SDNode *N) { | |||
| 2656 | return N->getOpcode() == ISD::MLOAD || | |||
| 2657 | N->getOpcode() == ISD::MSTORE; | |||
| 2658 | } | |||
| 2659 | }; | |||
| 2660 | ||||
| 2661 | /// This class is used to represent an MLOAD node | |||
| 2662 | class MaskedLoadSDNode : public MaskedLoadStoreSDNode { | |||
| 2663 | public: | |||
| 2664 | friend class SelectionDAG; | |||
| 2665 | ||||
| 2666 | MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2667 | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, | |||
| 2668 | bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) | |||
| 2669 | : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2670 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2671 | LoadSDNodeBits.IsExpanding = IsExpanding; | |||
| 2672 | } | |||
| 2673 | ||||
| 2674 | ISD::LoadExtType getExtensionType() const { | |||
| 2675 | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); | |||
| 2676 | } | |||
| 2677 | ||||
| 2678 | const SDValue &getBasePtr() const { return getOperand(1); } | |||
| 2679 | const SDValue &getOffset() const { return getOperand(2); } | |||
| 2680 | const SDValue &getMask() const { return getOperand(3); } | |||
| 2681 | const SDValue &getPassThru() const { return getOperand(4); } | |||
| 2682 | ||||
| 2683 | static bool classof(const SDNode *N) { | |||
| 2684 | return N->getOpcode() == ISD::MLOAD; | |||
| 2685 | } | |||
| 2686 | ||||
| 2687 | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } | |||
| 2688 | }; | |||
| 2689 | ||||
| 2690 | /// This class is used to represent an MSTORE node | |||
| 2691 | class MaskedStoreSDNode : public MaskedLoadStoreSDNode { | |||
| 2692 | public: | |||
| 2693 | friend class SelectionDAG; | |||
| 2694 | ||||
| 2695 | MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2696 | ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, | |||
| 2697 | EVT MemVT, MachineMemOperand *MMO) | |||
| 2698 | : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) { | |||
| 2699 | StoreSDNodeBits.IsTruncating = isTrunc; | |||
| 2700 | StoreSDNodeBits.IsCompressing = isCompressing; | |||
| 2701 | } | |||
| 2702 | ||||
| 2703 | /// Return true if the op does a truncation before store. | |||
| 2704 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2705 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2706 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2707 | ||||
| 2708 | /// Returns true if the op does a compression to the vector before storing. | |||
| 2709 | /// The node contiguously stores the active elements (integers or floats) | |||
| 2710 | /// in src (those with their respective bit set in writemask k) to unaligned | |||
| 2711 | /// memory at base_addr. | |||
| 2712 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } | |||
| 2713 | ||||
| 2714 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2715 | const SDValue &getBasePtr() const { return getOperand(2); } | |||
| 2716 | const SDValue &getOffset() const { return getOperand(3); } | |||
| 2717 | const SDValue &getMask() const { return getOperand(4); } | |||
| 2718 | ||||
| 2719 | static bool classof(const SDNode *N) { | |||
| 2720 | return N->getOpcode() == ISD::MSTORE; | |||
| 2721 | } | |||
| 2722 | }; | |||
| 2723 | ||||
| 2724 | /// This is a base class used to represent | |||
| 2725 | /// VP_GATHER and VP_SCATTER nodes | |||
| 2726 | /// | |||
| 2727 | class VPGatherScatterSDNode : public MemSDNode { | |||
| 2728 | public: | |||
| 2729 | friend class SelectionDAG; | |||
| 2730 | ||||
| 2731 | VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
| 2732 | const DebugLoc &dl, SDVTList VTs, EVT MemVT, | |||
| 2733 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) | |||
| 2734 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
| 2735 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
| 2736 | assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType && "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2736, __extension__ __PRETTY_FUNCTION__)); | |||
| 2737 | } | |||
| 2738 | ||||
| 2739 | /// How is Index applied to BasePtr when computing addresses. | |||
| 2740 | ISD::MemIndexType getIndexType() const { | |||
| 2741 | return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); | |||
| 2742 | } | |||
| 2743 | bool isIndexScaled() const { | |||
| 2744 | return !cast<ConstantSDNode>(getScale())->isOne(); | |||
| 2745 | } | |||
| 2746 | bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); } | |||
| 2747 | ||||
| 2748 | // In the both nodes address is Op1, mask is Op2: | |||
| 2749 | // VPGatherSDNode (Chain, base, index, scale, mask, vlen) | |||
| 2750 | // VPScatterSDNode (Chain, value, base, index, scale, mask, vlen) | |||
| 2751 | // Mask is a vector of i1 elements | |||
| 2752 | const SDValue &getBasePtr() const { | |||
| 2753 | return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2); | |||
| 2754 | } | |||
| 2755 | const SDValue &getIndex() const { | |||
| 2756 | return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3); | |||
| 2757 | } | |||
| 2758 | const SDValue &getScale() const { | |||
| 2759 | return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4); | |||
| 2760 | } | |||
| 2761 | const SDValue &getMask() const { | |||
| 2762 | return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5); | |||
| 2763 | } | |||
| 2764 | const SDValue &getVectorLength() const { | |||
| 2765 | return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6); | |||
| 2766 | } | |||
| 2767 | ||||
| 2768 | static bool classof(const SDNode *N) { | |||
| 2769 | return N->getOpcode() == ISD::VP_GATHER || | |||
| 2770 | N->getOpcode() == ISD::VP_SCATTER; | |||
| 2771 | } | |||
| 2772 | }; | |||
| 2773 | ||||
| 2774 | /// This class is used to represent an VP_GATHER node | |||
| 2775 | /// | |||
| 2776 | class VPGatherSDNode : public VPGatherScatterSDNode { | |||
| 2777 | public: | |||
| 2778 | friend class SelectionDAG; | |||
| 2779 | ||||
| 2780 | VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, | |||
| 2781 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) | |||
| 2782 | : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO, | |||
| 2783 | IndexType) {} | |||
| 2784 | ||||
| 2785 | static bool classof(const SDNode *N) { | |||
| 2786 | return N->getOpcode() == ISD::VP_GATHER; | |||
| 2787 | } | |||
| 2788 | }; | |||
| 2789 | ||||
| 2790 | /// This class is used to represent an VP_SCATTER node | |||
| 2791 | /// | |||
| 2792 | class VPScatterSDNode : public VPGatherScatterSDNode { | |||
| 2793 | public: | |||
| 2794 | friend class SelectionDAG; | |||
| 2795 | ||||
| 2796 | VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, | |||
| 2797 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) | |||
| 2798 | : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO, | |||
| 2799 | IndexType) {} | |||
| 2800 | ||||
| 2801 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2802 | ||||
| 2803 | static bool classof(const SDNode *N) { | |||
| 2804 | return N->getOpcode() == ISD::VP_SCATTER; | |||
| 2805 | } | |||
| 2806 | }; | |||
| 2807 | ||||
| 2808 | /// This is a base class used to represent | |||
| 2809 | /// MGATHER and MSCATTER nodes | |||
| 2810 | /// | |||
| 2811 | class MaskedGatherScatterSDNode : public MemSDNode { | |||
| 2812 | public: | |||
| 2813 | friend class SelectionDAG; | |||
| 2814 | ||||
| 2815 | MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, | |||
| 2816 | const DebugLoc &dl, SDVTList VTs, EVT MemVT, | |||
| 2817 | MachineMemOperand *MMO, ISD::MemIndexType IndexType) | |||
| 2818 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { | |||
| 2819 | LSBaseSDNodeBits.AddressingMode = IndexType; | |||
| 2820 | assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType && "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2820, __extension__ __PRETTY_FUNCTION__)); | |||
| 2821 | } | |||
| 2822 | ||||
| 2823 | /// How is Index applied to BasePtr when computing addresses. | |||
| 2824 | ISD::MemIndexType getIndexType() const { | |||
| 2825 | return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode); | |||
| 2826 | } | |||
| 2827 | bool isIndexScaled() const { | |||
| 2828 | return !cast<ConstantSDNode>(getScale())->isOne(); | |||
| 2829 | } | |||
| 2830 | bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); } | |||
| 2831 | ||||
| 2832 | // In the both nodes address is Op1, mask is Op2: | |||
| 2833 | // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale) | |||
| 2834 | // MaskedScatterSDNode (Chain, value, mask, base, index, scale) | |||
| 2835 | // Mask is a vector of i1 elements | |||
| 2836 | const SDValue &getBasePtr() const { return getOperand(3); } | |||
| 2837 | const SDValue &getIndex() const { return getOperand(4); } | |||
| 2838 | const SDValue &getMask() const { return getOperand(2); } | |||
| 2839 | const SDValue &getScale() const { return getOperand(5); } | |||
| 2840 | ||||
| 2841 | static bool classof(const SDNode *N) { | |||
| 2842 | return N->getOpcode() == ISD::MGATHER || | |||
| 2843 | N->getOpcode() == ISD::MSCATTER; | |||
| 2844 | } | |||
| 2845 | }; | |||
| 2846 | ||||
| 2847 | /// This class is used to represent an MGATHER node | |||
| 2848 | /// | |||
| 2849 | class MaskedGatherSDNode : public MaskedGatherScatterSDNode { | |||
| 2850 | public: | |||
| 2851 | friend class SelectionDAG; | |||
| 2852 | ||||
| 2853 | MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2854 | EVT MemVT, MachineMemOperand *MMO, | |||
| 2855 | ISD::MemIndexType IndexType, ISD::LoadExtType ETy) | |||
| 2856 | : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO, | |||
| 2857 | IndexType) { | |||
| 2858 | LoadSDNodeBits.ExtTy = ETy; | |||
| 2859 | } | |||
| 2860 | ||||
| 2861 | const SDValue &getPassThru() const { return getOperand(1); } | |||
| 2862 | ||||
| 2863 | ISD::LoadExtType getExtensionType() const { | |||
| 2864 | return ISD::LoadExtType(LoadSDNodeBits.ExtTy); | |||
| 2865 | } | |||
| 2866 | ||||
| 2867 | static bool classof(const SDNode *N) { | |||
| 2868 | return N->getOpcode() == ISD::MGATHER; | |||
| 2869 | } | |||
| 2870 | }; | |||
| 2871 | ||||
| 2872 | /// This class is used to represent an MSCATTER node | |||
| 2873 | /// | |||
| 2874 | class MaskedScatterSDNode : public MaskedGatherScatterSDNode { | |||
| 2875 | public: | |||
| 2876 | friend class SelectionDAG; | |||
| 2877 | ||||
| 2878 | MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, | |||
| 2879 | EVT MemVT, MachineMemOperand *MMO, | |||
| 2880 | ISD::MemIndexType IndexType, bool IsTrunc) | |||
| 2881 | : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO, | |||
| 2882 | IndexType) { | |||
| 2883 | StoreSDNodeBits.IsTruncating = IsTrunc; | |||
| 2884 | } | |||
| 2885 | ||||
| 2886 | /// Return true if the op does a truncation before store. | |||
| 2887 | /// For integers this is the same as doing a TRUNCATE and storing the result. | |||
| 2888 | /// For floats, it is the same as doing an FP_ROUND and storing the result. | |||
| 2889 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } | |||
| 2890 | ||||
| 2891 | const SDValue &getValue() const { return getOperand(1); } | |||
| 2892 | ||||
| 2893 | static bool classof(const SDNode *N) { | |||
| 2894 | return N->getOpcode() == ISD::MSCATTER; | |||
| 2895 | } | |||
| 2896 | }; | |||
| 2897 | ||||
| 2898 | /// An SDNode that represents everything that will be needed | |||
| 2899 | /// to construct a MachineInstr. These nodes are created during the | |||
| 2900 | /// instruction selection proper phase. | |||
| 2901 | /// | |||
| 2902 | /// Note that the only supported way to set the `memoperands` is by calling the | |||
| 2903 | /// `SelectionDAG::setNodeMemRefs` function as the memory management happens | |||
| 2904 | /// inside the DAG rather than in the node. | |||
| 2905 | class MachineSDNode : public SDNode { | |||
| 2906 | private: | |||
| 2907 | friend class SelectionDAG; | |||
| 2908 | ||||
| 2909 | MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) | |||
| 2910 | : SDNode(Opc, Order, DL, VTs) {} | |||
| 2911 | ||||
| 2912 | // We use a pointer union between a single `MachineMemOperand` pointer and | |||
| 2913 | // a pointer to an array of `MachineMemOperand` pointers. This is null when | |||
| 2914 | // the number of these is zero, the single pointer variant used when the | |||
| 2915 | // number is one, and the array is used for larger numbers. | |||
| 2916 | // | |||
| 2917 | // The array is allocated via the `SelectionDAG`'s allocator and so will | |||
| 2918 | // always live until the DAG is cleaned up and doesn't require ownership here. | |||
| 2919 | // | |||
| 2920 | // We can't use something simpler like `TinyPtrVector` here because `SDNode` | |||
| 2921 | // subclasses aren't managed in a conforming C++ manner. See the comments on | |||
| 2922 | // `SelectionDAG::MorphNodeTo` which details what all goes on, but the | |||
| 2923 | // constraint here is that these don't manage memory with their constructor or | |||
| 2924 | // destructor and can be initialized to a good state even if they start off | |||
| 2925 | // uninitialized. | |||
| 2926 | PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {}; | |||
| 2927 | ||||
| 2928 | // Note that this could be folded into the above `MemRefs` member if doing so | |||
| 2929 | // is advantageous at some point. We don't need to store this in most cases. | |||
| 2930 | // However, at the moment this doesn't appear to make the allocation any | |||
| 2931 | // smaller and makes the code somewhat simpler to read. | |||
| 2932 | int NumMemRefs = 0; | |||
| 2933 | ||||
| 2934 | public: | |||
| 2935 | using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator; | |||
| 2936 | ||||
| 2937 | ArrayRef<MachineMemOperand *> memoperands() const { | |||
| 2938 | // Special case the common cases. | |||
| 2939 | if (NumMemRefs == 0) | |||
| 2940 | return {}; | |||
| 2941 | if (NumMemRefs == 1) | |||
| 2942 | return ArrayRef(MemRefs.getAddrOfPtr1(), 1); | |||
| 2943 | ||||
| 2944 | // Otherwise we have an actual array. | |||
| 2945 | return ArrayRef(cast<MachineMemOperand **>(MemRefs), NumMemRefs); | |||
| 2946 | } | |||
| 2947 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } | |||
| 2948 | mmo_iterator memoperands_end() const { return memoperands().end(); } | |||
| 2949 | bool memoperands_empty() const { return memoperands().empty(); } | |||
| 2950 | ||||
| 2951 | /// Clear out the memory reference descriptor list. | |||
| 2952 | void clearMemRefs() { | |||
| 2953 | MemRefs = nullptr; | |||
| 2954 | NumMemRefs = 0; | |||
| 2955 | } | |||
| 2956 | ||||
| 2957 | static bool classof(const SDNode *N) { | |||
| 2958 | return N->isMachineOpcode(); | |||
| 2959 | } | |||
| 2960 | }; | |||
| 2961 | ||||
| 2962 | /// An SDNode that records if a register contains a value that is guaranteed to | |||
| 2963 | /// be aligned accordingly. | |||
| 2964 | class AssertAlignSDNode : public SDNode { | |||
| 2965 | Align Alignment; | |||
| 2966 | ||||
| 2967 | public: | |||
| 2968 | AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A) | |||
| 2969 | : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {} | |||
| 2970 | ||||
| 2971 | Align getAlign() const { return Alignment; } | |||
| 2972 | ||||
| 2973 | static bool classof(const SDNode *N) { | |||
| 2974 | return N->getOpcode() == ISD::AssertAlign; | |||
| 2975 | } | |||
| 2976 | }; | |||
| 2977 | ||||
| 2978 | class SDNodeIterator { | |||
| 2979 | const SDNode *Node; | |||
| 2980 | unsigned Operand; | |||
| 2981 | ||||
| 2982 | SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {} | |||
| 2983 | ||||
| 2984 | public: | |||
| 2985 | using iterator_category = std::forward_iterator_tag; | |||
| 2986 | using value_type = SDNode; | |||
| 2987 | using difference_type = std::ptrdiff_t; | |||
| 2988 | using pointer = value_type *; | |||
| 2989 | using reference = value_type &; | |||
| 2990 | ||||
| 2991 | bool operator==(const SDNodeIterator& x) const { | |||
| 2992 | return Operand == x.Operand; | |||
| 2993 | } | |||
| 2994 | bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } | |||
| 2995 | ||||
| 2996 | pointer operator*() const { | |||
| 2997 | return Node->getOperand(Operand).getNode(); | |||
| 2998 | } | |||
| 2999 | pointer operator->() const { return operator*(); } | |||
| 3000 | ||||
| 3001 | SDNodeIterator& operator++() { // Preincrement | |||
| 3002 | ++Operand; | |||
| 3003 | return *this; | |||
| 3004 | } | |||
| 3005 | SDNodeIterator operator++(int) { // Postincrement | |||
| 3006 | SDNodeIterator tmp = *this; ++*this; return tmp; | |||
| 3007 | } | |||
| 3008 | size_t operator-(SDNodeIterator Other) const { | |||
| 3009 | assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 3010, __extension__ __PRETTY_FUNCTION__)) | |||
| 3010 | "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!" ) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\"" , "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 3010, __extension__ __PRETTY_FUNCTION__)); | |||
| 3011 | return Operand - Other.Operand; | |||
| 3012 | } | |||
| 3013 | ||||
| 3014 | static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); } | |||
| 3015 | static SDNodeIterator end (const SDNode *N) { | |||
| 3016 | return SDNodeIterator(N, N->getNumOperands()); | |||
| 3017 | } | |||
| 3018 | ||||
| 3019 | unsigned getOperand() const { return Operand; } | |||
| 3020 | const SDNode *getNode() const { return Node; } | |||
| 3021 | }; | |||
| 3022 | ||||
| 3023 | template <> struct GraphTraits<SDNode*> { | |||
| 3024 | using NodeRef = SDNode *; | |||
| 3025 | using ChildIteratorType = SDNodeIterator; | |||
| 3026 | ||||
| 3027 | static NodeRef getEntryNode(SDNode *N) { return N; } | |||
| 3028 | ||||
| 3029 | static ChildIteratorType child_begin(NodeRef N) { | |||
| 3030 | return SDNodeIterator::begin(N); | |||
| 3031 | } | |||
| 3032 | ||||
| 3033 | static ChildIteratorType child_end(NodeRef N) { | |||
| 3034 | return SDNodeIterator::end(N); | |||
| 3035 | } | |||
| 3036 | }; | |||
| 3037 | ||||
| 3038 | /// A representation of the largest SDNode, for use in sizeof(). | |||
| 3039 | /// | |||
| 3040 | /// This needs to be a union because the largest node differs on 32 bit systems | |||
| 3041 | /// with 4 and 8 byte pointer alignment, respectively. | |||
| 3042 | using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, | |||
| 3043 | BlockAddressSDNode, | |||
| 3044 | GlobalAddressSDNode, | |||
| 3045 | PseudoProbeSDNode>; | |||
| 3046 | ||||
| 3047 | /// The SDNode class with the greatest alignment requirement. | |||
| 3048 | using MostAlignedSDNode = GlobalAddressSDNode; | |||
| 3049 | ||||
| 3050 | namespace ISD { | |||
| 3051 | ||||
| 3052 | /// Returns true if the specified node is a non-extending and unindexed load. | |||
| 3053 | inline bool isNormalLoad(const SDNode *N) { | |||
| 3054 | const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N); | |||
| 3055 | return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && | |||
| 3056 | Ld->getAddressingMode() == ISD::UNINDEXED; | |||
| 3057 | } | |||
| 3058 | ||||
| 3059 | /// Returns true if the specified node is a non-extending load. | |||
| 3060 | inline bool isNON_EXTLoad(const SDNode *N) { | |||
| 3061 | return isa<LoadSDNode>(N) && | |||
| 3062 | cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; | |||
| 3063 | } | |||
| 3064 | ||||
| 3065 | /// Returns true if the specified node is a EXTLOAD. | |||
| 3066 | inline bool isEXTLoad(const SDNode *N) { | |||
| 3067 | return isa<LoadSDNode>(N) && | |||
| 3068 | cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; | |||
| 3069 | } | |||
| 3070 | ||||
| 3071 | /// Returns true if the specified node is a SEXTLOAD. | |||
| 3072 | inline bool isSEXTLoad(const SDNode *N) { | |||
| 3073 | return isa<LoadSDNode>(N) && | |||
| 3074 | cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; | |||
| 3075 | } | |||
| 3076 | ||||
| 3077 | /// Returns true if the specified node is a ZEXTLOAD. | |||
| 3078 | inline bool isZEXTLoad(const SDNode *N) { | |||
| 3079 | return isa<LoadSDNode>(N) && | |||
| 3080 | cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; | |||
| 3081 | } | |||
| 3082 | ||||
| 3083 | /// Returns true if the specified node is an unindexed load. | |||
| 3084 | inline bool isUNINDEXEDLoad(const SDNode *N) { | |||
| 3085 | return isa<LoadSDNode>(N) && | |||
| 3086 | cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
| 3087 | } | |||
| 3088 | ||||
| 3089 | /// Returns true if the specified node is a non-truncating | |||
| 3090 | /// and unindexed store. | |||
| 3091 | inline bool isNormalStore(const SDNode *N) { | |||
| 3092 | const StoreSDNode *St = dyn_cast<StoreSDNode>(N); | |||
| 3093 | return St && !St->isTruncatingStore() && | |||
| 3094 | St->getAddressingMode() == ISD::UNINDEXED; | |||
| 3095 | } | |||
| 3096 | ||||
| 3097 | /// Returns true if the specified node is an unindexed store. | |||
| 3098 | inline bool isUNINDEXEDStore(const SDNode *N) { | |||
| 3099 | return isa<StoreSDNode>(N) && | |||
| 3100 | cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; | |||
| 3101 | } | |||
| 3102 | ||||
| 3103 | /// Attempt to match a unary predicate against a scalar/splat constant or | |||
| 3104 | /// every element of a constant BUILD_VECTOR. | |||
| 3105 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
| 3106 | bool matchUnaryPredicate(SDValue Op, | |||
| 3107 | std::function<bool(ConstantSDNode *)> Match, | |||
| 3108 | bool AllowUndefs = false); | |||
| 3109 | ||||
| 3110 | /// Attempt to match a binary predicate against a pair of scalar/splat | |||
| 3111 | /// constants or every element of a pair of constant BUILD_VECTORs. | |||
| 3112 | /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. | |||
| 3113 | /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match. | |||
| 3114 | bool matchBinaryPredicate( | |||
| 3115 | SDValue LHS, SDValue RHS, | |||
| 3116 | std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, | |||
| 3117 | bool AllowUndefs = false, bool AllowTypeMismatch = false); | |||
| 3118 | ||||
| 3119 | /// Returns true if the specified value is the overflow result from one | |||
| 3120 | /// of the overflow intrinsic nodes. | |||
| 3121 | inline bool isOverflowIntrOpRes(SDValue Op) { | |||
| 3122 | unsigned Opc = Op.getOpcode(); | |||
| 3123 | return (Op.getResNo() == 1 && | |||
| 3124 | (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || | |||
| 3125 | Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)); | |||
| 3126 | } | |||
| 3127 | ||||
| 3128 | } // end namespace ISD | |||
| 3129 | ||||
| 3130 | } // end namespace llvm | |||
| 3131 | ||||
| 3132 | #endif // LLVM_CODEGEN_SELECTIONDAGNODES_H |