Bug Summary

File:llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Warning:line 777, column 32
Division by zero

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAGBuilder.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include -D NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-04-040900-46481-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/None.h"
19#include "llvm/ADT/Optional.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/Triple.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/BlockFrequencyInfo.h"
28#include "llvm/Analysis/BranchProbabilityInfo.h"
29#include "llvm/Analysis/ConstantFolding.h"
30#include "llvm/Analysis/EHPersonalities.h"
31#include "llvm/Analysis/Loads.h"
32#include "llvm/Analysis/MemoryLocation.h"
33#include "llvm/Analysis/ProfileSummaryInfo.h"
34#include "llvm/Analysis/TargetLibraryInfo.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/Analysis/VectorUtils.h"
37#include "llvm/CodeGen/Analysis.h"
38#include "llvm/CodeGen/FunctionLoweringInfo.h"
39#include "llvm/CodeGen/GCMetadata.h"
40#include "llvm/CodeGen/MachineBasicBlock.h"
41#include "llvm/CodeGen/MachineFrameInfo.h"
42#include "llvm/CodeGen/MachineFunction.h"
43#include "llvm/CodeGen/MachineInstr.h"
44#include "llvm/CodeGen/MachineInstrBuilder.h"
45#include "llvm/CodeGen/MachineJumpTableInfo.h"
46#include "llvm/CodeGen/MachineMemOperand.h"
47#include "llvm/CodeGen/MachineModuleInfo.h"
48#include "llvm/CodeGen/MachineOperand.h"
49#include "llvm/CodeGen/MachineRegisterInfo.h"
50#include "llvm/CodeGen/RuntimeLibcalls.h"
51#include "llvm/CodeGen/SelectionDAG.h"
52#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
53#include "llvm/CodeGen/StackMaps.h"
54#include "llvm/CodeGen/SwiftErrorValueTracking.h"
55#include "llvm/CodeGen/TargetFrameLowering.h"
56#include "llvm/CodeGen/TargetInstrInfo.h"
57#include "llvm/CodeGen/TargetOpcodes.h"
58#include "llvm/CodeGen/TargetRegisterInfo.h"
59#include "llvm/CodeGen/TargetSubtargetInfo.h"
60#include "llvm/CodeGen/WinEHFuncInfo.h"
61#include "llvm/IR/Argument.h"
62#include "llvm/IR/Attributes.h"
63#include "llvm/IR/BasicBlock.h"
64#include "llvm/IR/CFG.h"
65#include "llvm/IR/CallingConv.h"
66#include "llvm/IR/Constant.h"
67#include "llvm/IR/ConstantRange.h"
68#include "llvm/IR/Constants.h"
69#include "llvm/IR/DataLayout.h"
70#include "llvm/IR/DebugInfoMetadata.h"
71#include "llvm/IR/DerivedTypes.h"
72#include "llvm/IR/DiagnosticInfo.h"
73#include "llvm/IR/Function.h"
74#include "llvm/IR/GetElementPtrTypeIterator.h"
75#include "llvm/IR/InlineAsm.h"
76#include "llvm/IR/InstrTypes.h"
77#include "llvm/IR/Instructions.h"
78#include "llvm/IR/IntrinsicInst.h"
79#include "llvm/IR/Intrinsics.h"
80#include "llvm/IR/IntrinsicsAArch64.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/Metadata.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Operator.h"
86#include "llvm/IR/PatternMatch.h"
87#include "llvm/IR/Statepoint.h"
88#include "llvm/IR/Type.h"
89#include "llvm/IR/User.h"
90#include "llvm/IR/Value.h"
91#include "llvm/MC/MCContext.h"
92#include "llvm/MC/MCSymbol.h"
93#include "llvm/Support/AtomicOrdering.h"
94#include "llvm/Support/Casting.h"
95#include "llvm/Support/CommandLine.h"
96#include "llvm/Support/Compiler.h"
97#include "llvm/Support/Debug.h"
98#include "llvm/Support/MathExtras.h"
99#include "llvm/Support/raw_ostream.h"
100#include "llvm/Target/TargetIntrinsicInfo.h"
101#include "llvm/Target/TargetMachine.h"
102#include "llvm/Target/TargetOptions.h"
103#include "llvm/Transforms/Utils/Local.h"
104#include <cstddef>
105#include <cstring>
106#include <iterator>
107#include <limits>
108#include <numeric>
109#include <tuple>
110
111using namespace llvm;
112using namespace PatternMatch;
113using namespace SwitchCG;
114
115#define DEBUG_TYPE"isel" "isel"
116
117/// LimitFloatPrecision - Generate low-precision inline sequences for
118/// some float libcalls (6, 8 or 12 bits).
119static unsigned LimitFloatPrecision;
120
121static cl::opt<bool>
122 InsertAssertAlign("insert-assert-align", cl::init(true),
123 cl::desc("Insert the experimental `assertalign` node."),
124 cl::ReallyHidden);
125
126static cl::opt<unsigned, true>
127 LimitFPPrecision("limit-float-precision",
128 cl::desc("Generate low-precision inline sequences "
129 "for some float libcalls"),
130 cl::location(LimitFloatPrecision), cl::Hidden,
131 cl::init(0));
132
133static cl::opt<unsigned> SwitchPeelThreshold(
134 "switch-peel-threshold", cl::Hidden, cl::init(66),
135 cl::desc("Set the case probability threshold for peeling the case from a "
136 "switch statement. A value greater than 100 will void this "
137 "optimization"));
138
139// Limit the width of DAG chains. This is important in general to prevent
140// DAG-based analysis from blowing up. For example, alias analysis and
141// load clustering may not complete in reasonable time. It is difficult to
142// recognize and avoid this situation within each individual analysis, and
143// future analyses are likely to have the same behavior. Limiting DAG width is
144// the safe approach and will be especially important with global DAGs.
145//
146// MaxParallelChains default is arbitrarily high to avoid affecting
147// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
148// sequence over this should have been converted to llvm.memcpy by the
149// frontend. It is easy to induce this behavior with .ll code such as:
150// %buffer = alloca [4096 x i8]
151// %data = load [4096 x i8]* %argPtr
152// store [4096 x i8] %data, [4096 x i8]* %buffer
153static const unsigned MaxParallelChains = 64;
154
155static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
156 const SDValue *Parts, unsigned NumParts,
157 MVT PartVT, EVT ValueVT, const Value *V,
158 Optional<CallingConv::ID> CC);
159
160/// getCopyFromParts - Create a value that contains the specified legal parts
161/// combined into the value they represent. If the parts combine to a type
162/// larger than ValueVT then AssertOp can be used to specify whether the extra
163/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
164/// (ISD::AssertSext).
165static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
166 const SDValue *Parts, unsigned NumParts,
167 MVT PartVT, EVT ValueVT, const Value *V,
168 Optional<CallingConv::ID> CC = None,
169 Optional<ISD::NodeType> AssertOp = None) {
170 // Let the target assemble the parts if it wants to
171 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
172 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
173 PartVT, ValueVT, CC))
174 return Val;
175
176 if (ValueVT.isVector())
177 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
178 CC);
179
180 assert(NumParts > 0 && "No parts to assemble!")(static_cast<void> (0));
181 SDValue Val = Parts[0];
182
183 if (NumParts > 1) {
184 // Assemble the value from multiple parts.
185 if (ValueVT.isInteger()) {
186 unsigned PartBits = PartVT.getSizeInBits();
187 unsigned ValueBits = ValueVT.getSizeInBits();
188
189 // Assemble the power of 2 part.
190 unsigned RoundParts =
191 (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
192 unsigned RoundBits = PartBits * RoundParts;
193 EVT RoundVT = RoundBits == ValueBits ?
194 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
195 SDValue Lo, Hi;
196
197 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
198
199 if (RoundParts > 2) {
200 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
201 PartVT, HalfVT, V);
202 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
203 RoundParts / 2, PartVT, HalfVT, V);
204 } else {
205 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
206 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
207 }
208
209 if (DAG.getDataLayout().isBigEndian())
210 std::swap(Lo, Hi);
211
212 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
213
214 if (RoundParts < NumParts) {
215 // Assemble the trailing non-power-of-2 part.
216 unsigned OddParts = NumParts - RoundParts;
217 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
218 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
219 OddVT, V, CC);
220
221 // Combine the round and odd parts.
222 Lo = Val;
223 if (DAG.getDataLayout().isBigEndian())
224 std::swap(Lo, Hi);
225 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
226 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
227 Hi =
228 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
229 DAG.getConstant(Lo.getValueSizeInBits(), DL,
230 TLI.getPointerTy(DAG.getDataLayout())));
231 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
232 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
233 }
234 } else if (PartVT.isFloatingPoint()) {
235 // FP split into multiple FP parts (for ppcf128)
236 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast<void> (0))
237 "Unexpected split")(static_cast<void> (0));
238 SDValue Lo, Hi;
239 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
240 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
241 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
242 std::swap(Lo, Hi);
243 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
244 } else {
245 // FP split into integer parts (soft fp)
246 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast<void> (0))
247 !PartVT.isVector() && "Unexpected split")(static_cast<void> (0));
248 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
249 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
250 }
251 }
252
253 // There is now one part, held in Val. Correct it to match ValueVT.
254 // PartEVT is the type of the register class that holds the value.
255 // ValueVT is the type of the inline asm operation.
256 EVT PartEVT = Val.getValueType();
257
258 if (PartEVT == ValueVT)
259 return Val;
260
261 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
262 ValueVT.bitsLT(PartEVT)) {
263 // For an FP value in an integer part, we need to truncate to the right
264 // width first.
265 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
266 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
267 }
268
269 // Handle types that have the same size.
270 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
271 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
272
273 // Handle types with different sizes.
274 if (PartEVT.isInteger() && ValueVT.isInteger()) {
275 if (ValueVT.bitsLT(PartEVT)) {
276 // For a truncate, see if we have any information to
277 // indicate whether the truncated bits will always be
278 // zero or sign-extension.
279 if (AssertOp.hasValue())
280 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
281 DAG.getValueType(ValueVT));
282 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
283 }
284 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
285 }
286
287 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
288 // FP_ROUND's are always exact here.
289 if (ValueVT.bitsLT(Val.getValueType()))
290 return DAG.getNode(
291 ISD::FP_ROUND, DL, ValueVT, Val,
292 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
293
294 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
295 }
296
297 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
298 // then truncating.
299 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
300 ValueVT.bitsLT(PartEVT)) {
301 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
302 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
303 }
304
305 report_fatal_error("Unknown mismatch in getCopyFromParts!");
306}
307
308static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
309 const Twine &ErrMsg) {
310 const Instruction *I = dyn_cast_or_null<Instruction>(V);
311 if (!V)
312 return Ctx.emitError(ErrMsg);
313
314 const char *AsmError = ", possible invalid constraint for vector type";
315 if (const CallInst *CI = dyn_cast<CallInst>(I))
316 if (CI->isInlineAsm())
317 return Ctx.emitError(I, ErrMsg + AsmError);
318
319 return Ctx.emitError(I, ErrMsg);
320}
321
322/// getCopyFromPartsVector - Create a value that contains the specified legal
323/// parts combined into the value they represent. If the parts combine to a
324/// type larger than ValueVT then AssertOp can be used to specify whether the
325/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
326/// ValueVT (ISD::AssertSext).
327static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
328 const SDValue *Parts, unsigned NumParts,
329 MVT PartVT, EVT ValueVT, const Value *V,
330 Optional<CallingConv::ID> CallConv) {
331 assert(ValueVT.isVector() && "Not a vector value")(static_cast<void> (0));
332 assert(NumParts > 0 && "No parts to assemble!")(static_cast<void> (0));
333 const bool IsABIRegCopy = CallConv.hasValue();
334
335 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
336 SDValue Val = Parts[0];
337
338 // Handle a multi-element vector.
339 if (NumParts > 1) {
340 EVT IntermediateVT;
341 MVT RegisterVT;
342 unsigned NumIntermediates;
343 unsigned NumRegs;
344
345 if (IsABIRegCopy) {
346 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
347 *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
348 NumIntermediates, RegisterVT);
349 } else {
350 NumRegs =
351 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
352 NumIntermediates, RegisterVT);
353 }
354
355 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast<void> (0));
356 NumParts = NumRegs; // Silence a compiler warning.
357 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast<void> (0));
358 assert(RegisterVT.getSizeInBits() ==(static_cast<void> (0))
359 Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast<void> (0))
360 "Part type sizes don't match!")(static_cast<void> (0));
361
362 // Assemble the parts into intermediate operands.
363 SmallVector<SDValue, 8> Ops(NumIntermediates);
364 if (NumIntermediates == NumParts) {
365 // If the register was not expanded, truncate or copy the value,
366 // as appropriate.
367 for (unsigned i = 0; i != NumParts; ++i)
368 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
369 PartVT, IntermediateVT, V, CallConv);
370 } else if (NumParts > 0) {
371 // If the intermediate type was expanded, build the intermediate
372 // operands from the parts.
373 assert(NumParts % NumIntermediates == 0 &&(static_cast<void> (0))
374 "Must expand into a divisible number of parts!")(static_cast<void> (0));
375 unsigned Factor = NumParts / NumIntermediates;
376 for (unsigned i = 0; i != NumIntermediates; ++i)
377 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
378 PartVT, IntermediateVT, V, CallConv);
379 }
380
381 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
382 // intermediate operands.
383 EVT BuiltVectorTy =
384 IntermediateVT.isVector()
385 ? EVT::getVectorVT(
386 *DAG.getContext(), IntermediateVT.getScalarType(),
387 IntermediateVT.getVectorElementCount() * NumParts)
388 : EVT::getVectorVT(*DAG.getContext(),
389 IntermediateVT.getScalarType(),
390 NumIntermediates);
391 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
392 : ISD::BUILD_VECTOR,
393 DL, BuiltVectorTy, Ops);
394 }
395
396 // There is now one part, held in Val. Correct it to match ValueVT.
397 EVT PartEVT = Val.getValueType();
398
399 if (PartEVT == ValueVT)
400 return Val;
401
402 if (PartEVT.isVector()) {
403 // Vector/Vector bitcast.
404 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
405 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
406
407 // If the element type of the source/dest vectors are the same, but the
408 // parts vector has more elements than the value vector, then we have a
409 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
410 // elements we want.
411 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
412 assert((PartEVT.getVectorElementCount().getKnownMinValue() >(static_cast<void> (0))
413 ValueVT.getVectorElementCount().getKnownMinValue()) &&(static_cast<void> (0))
414 (PartEVT.getVectorElementCount().isScalable() ==(static_cast<void> (0))
415 ValueVT.getVectorElementCount().isScalable()) &&(static_cast<void> (0))
416 "Cannot narrow, it would be a lossy transformation")(static_cast<void> (0));
417 PartEVT =
418 EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),
419 ValueVT.getVectorElementCount());
420 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
421 DAG.getVectorIdxConstant(0, DL));
422 if (PartEVT == ValueVT)
423 return Val;
424 }
425
426 // Promoted vector extract
427 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
428 }
429
430 // Trivial bitcast if the types are the same size and the destination
431 // vector type is legal.
432 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
433 TLI.isTypeLegal(ValueVT))
434 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
435
436 if (ValueVT.getVectorNumElements() != 1) {
437 // Certain ABIs require that vectors are passed as integers. For vectors
438 // are the same size, this is an obvious bitcast.
439 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
440 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
441 } else if (ValueVT.bitsLT(PartEVT)) {
442 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
443 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
444 // Drop the extra bits.
445 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
446 return DAG.getBitcast(ValueVT, Val);
447 }
448
449 diagnosePossiblyInvalidConstraint(
450 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
451 return DAG.getUNDEF(ValueVT);
452 }
453
454 // Handle cases such as i8 -> <1 x i1>
455 EVT ValueSVT = ValueVT.getVectorElementType();
456 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
457 if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())
458 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
459 else
460 Val = ValueVT.isFloatingPoint()
461 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
462 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
463 }
464
465 return DAG.getBuildVector(ValueVT, DL, Val);
466}
467
468static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
469 SDValue Val, SDValue *Parts, unsigned NumParts,
470 MVT PartVT, const Value *V,
471 Optional<CallingConv::ID> CallConv);
472
473/// getCopyToParts - Create a series of nodes that contain the specified value
474/// split into legal parts. If the parts contain more bits than Val, then, for
475/// integers, ExtendKind can be used to specify how to generate the extra bits.
476static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
477 SDValue *Parts, unsigned NumParts, MVT PartVT,
478 const Value *V,
479 Optional<CallingConv::ID> CallConv = None,
480 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
481 // Let the target split the parts if it wants to
482 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
483 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
24
Assuming the condition is false
25
Taking false branch
484 CallConv))
485 return;
486 EVT ValueVT = Val.getValueType();
487
488 // Handle the vector case separately.
489 if (ValueVT.isVector())
26
Assuming the condition is true
27
Taking true branch
490 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
28
Calling 'getCopyToPartsVector'
491 CallConv);
492
493 unsigned PartBits = PartVT.getSizeInBits();
494 unsigned OrigNumParts = NumParts;
495 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast<void> (0))
496 "Copying to an illegal type!")(static_cast<void> (0));
497
498 if (NumParts == 0)
499 return;
500
501 assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast<void> (0));
502 EVT PartEVT = PartVT;
503 if (PartEVT == ValueVT) {
504 assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast<void> (0));
505 Parts[0] = Val;
506 return;
507 }
508
509 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
510 // If the parts cover more bits than the value has, promote the value.
511 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
512 assert(NumParts == 1 && "Do not know what to promote to!")(static_cast<void> (0));
513 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
514 } else {
515 if (ValueVT.isFloatingPoint()) {
516 // FP values need to be bitcast, then extended if they are being put
517 // into a larger container.
518 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
519 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
520 }
521 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast<void> (0))
522 ValueVT.isInteger() &&(static_cast<void> (0))
523 "Unknown mismatch!")(static_cast<void> (0));
524 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
525 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
526 if (PartVT == MVT::x86mmx)
527 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
528 }
529 } else if (PartBits == ValueVT.getSizeInBits()) {
530 // Different types of the same size.
531 assert(NumParts == 1 && PartEVT != ValueVT)(static_cast<void> (0));
532 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
533 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
534 // If the parts cover less bits than value has, truncate the value.
535 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast<void> (0))
536 ValueVT.isInteger() &&(static_cast<void> (0))
537 "Unknown mismatch!")(static_cast<void> (0));
538 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
539 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
540 if (PartVT == MVT::x86mmx)
541 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
542 }
543
544 // The value may have changed - recompute ValueVT.
545 ValueVT = Val.getValueType();
546 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast<void> (0))
547 "Failed to tile the value with PartVT!")(static_cast<void> (0));
548
549 if (NumParts == 1) {
550 if (PartEVT != ValueVT) {
551 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
552 "scalar-to-vector conversion failed");
553 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
554 }
555
556 Parts[0] = Val;
557 return;
558 }
559
560 // Expand the value into multiple parts.
561 if (NumParts & (NumParts - 1)) {
562 // The number of parts is not a power of 2. Split off and copy the tail.
563 assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast<void> (0))
564 "Do not know what to expand to!")(static_cast<void> (0));
565 unsigned RoundParts = 1 << Log2_32(NumParts);
566 unsigned RoundBits = RoundParts * PartBits;
567 unsigned OddParts = NumParts - RoundParts;
568 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
569 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));
570
571 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
572 CallConv);
573
574 if (DAG.getDataLayout().isBigEndian())
575 // The odd parts were reversed by getCopyToParts - unreverse them.
576 std::reverse(Parts + RoundParts, Parts + NumParts);
577
578 NumParts = RoundParts;
579 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
580 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
581 }
582
583 // The number of parts is a power of 2. Repeatedly bisect the value using
584 // EXTRACT_ELEMENT.
585 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
586 EVT::getIntegerVT(*DAG.getContext(),
587 ValueVT.getSizeInBits()),
588 Val);
589
590 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
591 for (unsigned i = 0; i < NumParts; i += StepSize) {
592 unsigned ThisBits = StepSize * PartBits / 2;
593 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
594 SDValue &Part0 = Parts[i];
595 SDValue &Part1 = Parts[i+StepSize/2];
596
597 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
598 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
599 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
600 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
601
602 if (ThisBits == PartBits && ThisVT != PartVT) {
603 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
604 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
605 }
606 }
607 }
608
609 if (DAG.getDataLayout().isBigEndian())
610 std::reverse(Parts, Parts + OrigNumParts);
611}
612
613static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
614 const SDLoc &DL, EVT PartVT) {
615 if (!PartVT.isVector())
616 return SDValue();
617
618 EVT ValueVT = Val.getValueType();
619 ElementCount PartNumElts = PartVT.getVectorElementCount();
620 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
621
622 // We only support widening vectors with equivalent element types and
623 // fixed/scalable properties. If a target needs to widen a fixed-length type
624 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
625 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
626 PartNumElts.isScalable() != ValueNumElts.isScalable() ||
627 PartVT.getVectorElementType() != ValueVT.getVectorElementType())
628 return SDValue();
629
630 // Widening a scalable vector to another scalable vector is done by inserting
631 // the vector into a larger undef one.
632 if (PartNumElts.isScalable())
633 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
634 Val, DAG.getVectorIdxConstant(0, DL));
635
636 EVT ElementVT = PartVT.getVectorElementType();
637 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
638 // undef elements.
639 SmallVector<SDValue, 16> Ops;
640 DAG.ExtractVectorElements(Val, Ops);
641 SDValue EltUndef = DAG.getUNDEF(ElementVT);
642 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
643
644 // FIXME: Use CONCAT for 2x -> 4x.
645 return DAG.getBuildVector(PartVT, DL, Ops);
646}
647
648/// getCopyToPartsVector - Create a series of nodes that contain the specified
649/// value split into legal parts.
650static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
651 SDValue Val, SDValue *Parts, unsigned NumParts,
652 MVT PartVT, const Value *V,
653 Optional<CallingConv::ID> CallConv) {
654 EVT ValueVT = Val.getValueType();
655 assert(ValueVT.isVector() && "Not a vector")(static_cast<void> (0));
656 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
657 const bool IsABIRegCopy = CallConv.hasValue();
658
659 if (NumParts == 1) {
29
Assuming 'NumParts' is not equal to 1
30
Taking false branch
660 EVT PartEVT = PartVT;
661 if (PartEVT == ValueVT) {
662 // Nothing to do.
663 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
664 // Bitconvert vector->vector case.
665 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
666 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
667 Val = Widened;
668 } else if (PartVT.isVector() &&
669 PartEVT.getVectorElementType().bitsGE(
670 ValueVT.getVectorElementType()) &&
671 PartEVT.getVectorElementCount() ==
672 ValueVT.getVectorElementCount()) {
673
674 // Promoted vector extract
675 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
676 } else {
677 if (ValueVT.getVectorElementCount().isScalar()) {
678 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
679 DAG.getVectorIdxConstant(0, DL));
680 } else {
681 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
682 assert(PartVT.getFixedSizeInBits() > ValueSize &&(static_cast<void> (0))
683 "lossy conversion of vector to scalar type")(static_cast<void> (0));
684 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
685 Val = DAG.getBitcast(IntermediateType, Val);
686 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
687 }
688 }
689
690 assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast<void> (0));
691 Parts[0] = Val;
692 return;
693 }
694
695 // Handle a multi-element vector.
696 EVT IntermediateVT;
697 MVT RegisterVT;
698 unsigned NumIntermediates;
699 unsigned NumRegs;
700 if (IsABIRegCopy
30.1
'IsABIRegCopy' is false
) {
31
Taking false branch
701 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
702 *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
703 NumIntermediates, RegisterVT);
704 } else {
705 NumRegs =
706 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
32
Value assigned to 'NumIntermediates'
707 NumIntermediates, RegisterVT);
708 }
709
710 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast<void> (0));
711 NumParts = NumRegs; // Silence a compiler warning.
712 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast<void> (0));
713
714 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&(static_cast<void> (0))
715 "Mixing scalable and fixed vectors when copying in parts")(static_cast<void> (0));
716
717 Optional<ElementCount> DestEltCnt;
718
719 if (IntermediateVT.isVector())
33
Assuming the condition is true
34
Taking true branch
720 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
721 else
722 DestEltCnt = ElementCount::getFixed(NumIntermediates);
723
724 EVT BuiltVectorTy = EVT::getVectorVT(
725 *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());
726
727 if (ValueVT == BuiltVectorTy) {
35
Taking true branch
728 // Nothing to do.
729 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
730 // Bitconvert vector->vector case.
731 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
732 } else {
733 if (BuiltVectorTy.getVectorElementType().bitsGT(
734 ValueVT.getVectorElementType())) {
735 // Integer promotion.
736 ValueVT = EVT::getVectorVT(*DAG.getContext(),
737 BuiltVectorTy.getVectorElementType(),
738 ValueVT.getVectorElementCount());
739 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
740 }
741
742 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
743 Val = Widened;
744 }
745 }
746
747 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")(static_cast<void> (0));
748
749 // Split the vector into intermediate operands.
750 SmallVector<SDValue, 8> Ops(NumIntermediates);
751 for (unsigned i = 0; i != NumIntermediates; ++i) {
36
Assuming 'i' is equal to 'NumIntermediates'
37
Loop condition is false. Execution continues on line 766
752 if (IntermediateVT.isVector()) {
753 // This does something sensible for scalable vectors - see the
754 // definition of EXTRACT_SUBVECTOR for further details.
755 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
756 Ops[i] =
757 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
758 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
759 } else {
760 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
761 DAG.getVectorIdxConstant(i, DL));
762 }
763 }
764
765 // Split the intermediate operands into legal parts.
766 if (NumParts == NumIntermediates) {
38
Assuming 'NumParts' is not equal to 'NumIntermediates'
39
Taking false branch
767 // If the register was not expanded, promote or copy the value,
768 // as appropriate.
769 for (unsigned i = 0; i != NumParts; ++i)
770 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
771 } else if (NumParts
39.1
'NumParts' is > 0
> 0) {
40
Taking true branch
772 // If the intermediate type was expanded, split each the value into
773 // legal parts.
774 assert(NumIntermediates != 0 && "division by zero")(static_cast<void> (0));
775 assert(NumParts % NumIntermediates == 0 &&(static_cast<void> (0))
776 "Must expand into a divisible number of parts!")(static_cast<void> (0));
777 unsigned Factor = NumParts / NumIntermediates;
41
Division by zero
778 for (unsigned i = 0; i != NumIntermediates; ++i)
779 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
780 CallConv);
781 }
782}
783
784RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
785 EVT valuevt, Optional<CallingConv::ID> CC)
786 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
787 RegCount(1, regs.size()), CallConv(CC) {}
788
789RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
790 const DataLayout &DL, unsigned Reg, Type *Ty,
791 Optional<CallingConv::ID> CC) {
792 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
793
794 CallConv = CC;
795
796 for (EVT ValueVT : ValueVTs) {
797 unsigned NumRegs =
798 isABIMangled()
799 ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
800 : TLI.getNumRegisters(Context, ValueVT);
801 MVT RegisterVT =
802 isABIMangled()
803 ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
804 : TLI.getRegisterType(Context, ValueVT);
805 for (unsigned i = 0; i != NumRegs; ++i)
806 Regs.push_back(Reg + i);
807 RegVTs.push_back(RegisterVT);
808 RegCount.push_back(NumRegs);
809 Reg += NumRegs;
810 }
811}
812
813SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
814 FunctionLoweringInfo &FuncInfo,
815 const SDLoc &dl, SDValue &Chain,
816 SDValue *Flag, const Value *V) const {
817 // A Value with type {} or [0 x %t] needs no registers.
818 if (ValueVTs.empty())
819 return SDValue();
820
821 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
822
823 // Assemble the legal parts into the final values.
824 SmallVector<SDValue, 4> Values(ValueVTs.size());
825 SmallVector<SDValue, 8> Parts;
826 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
827 // Copy the legal parts from the registers.
828 EVT ValueVT = ValueVTs[Value];
829 unsigned NumRegs = RegCount[Value];
830 MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
831 *DAG.getContext(),
832 CallConv.getValue(), RegVTs[Value])
833 : RegVTs[Value];
834
835 Parts.resize(NumRegs);
836 for (unsigned i = 0; i != NumRegs; ++i) {
837 SDValue P;
838 if (!Flag) {
839 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
840 } else {
841 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
842 *Flag = P.getValue(2);
843 }
844
845 Chain = P.getValue(1);
846 Parts[i] = P;
847
848 // If the source register was virtual and if we know something about it,
849 // add an assert node.
850 if (!Register::isVirtualRegister(Regs[Part + i]) ||
851 !RegisterVT.isInteger())
852 continue;
853
854 const FunctionLoweringInfo::LiveOutInfo *LOI =
855 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
856 if (!LOI)
857 continue;
858
859 unsigned RegSize = RegisterVT.getScalarSizeInBits();
860 unsigned NumSignBits = LOI->NumSignBits;
861 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
862
863 if (NumZeroBits == RegSize) {
864 // The current value is a zero.
865 // Explicitly express that as it would be easier for
866 // optimizations to kick in.
867 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
868 continue;
869 }
870
871 // FIXME: We capture more information than the dag can represent. For
872 // now, just use the tightest assertzext/assertsext possible.
873 bool isSExt;
874 EVT FromVT(MVT::Other);
875 if (NumZeroBits) {
876 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
877 isSExt = false;
878 } else if (NumSignBits > 1) {
879 FromVT =
880 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
881 isSExt = true;
882 } else {
883 continue;
884 }
885 // Add an assertion node.
886 assert(FromVT != MVT::Other)(static_cast<void> (0));
887 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
888 RegisterVT, P, DAG.getValueType(FromVT));
889 }
890
891 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
892 RegisterVT, ValueVT, V, CallConv);
893 Part += NumRegs;
894 Parts.clear();
895 }
896
897 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
898}
899
900void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
901 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
902 const Value *V,
903 ISD::NodeType PreferredExtendType) const {
904 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
905 ISD::NodeType ExtendKind = PreferredExtendType;
906
907 // Get the list of the values's legal parts.
908 unsigned NumRegs = Regs.size();
909 SmallVector<SDValue, 8> Parts(NumRegs);
910 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
17
Assuming 'Value' is not equal to 'e'
18
Loop condition is true. Entering loop body
911 unsigned NumParts = RegCount[Value];
912
913 MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
19
Assuming the condition is false
20
'?' condition is false
914 *DAG.getContext(),
915 CallConv.getValue(), RegVTs[Value])
916 : RegVTs[Value];
917
918 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
21
Assuming 'ExtendKind' is not equal to ANY_EXTEND
22
Taking false branch
919 ExtendKind = ISD::ZERO_EXTEND;
920
921 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
23
Calling 'getCopyToParts'
922 NumParts, RegisterVT, V, CallConv, ExtendKind);
923 Part += NumParts;
924 }
925
926 // Copy the parts into the registers.
927 SmallVector<SDValue, 8> Chains(NumRegs);
928 for (unsigned i = 0; i != NumRegs; ++i) {
929 SDValue Part;
930 if (!Flag) {
931 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
932 } else {
933 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
934 *Flag = Part.getValue(1);
935 }
936
937 Chains[i] = Part.getValue(0);
938 }
939
940 if (NumRegs == 1 || Flag)
941 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
942 // flagged to it. That is the CopyToReg nodes and the user are considered
943 // a single scheduling unit. If we create a TokenFactor and return it as
944 // chain, then the TokenFactor is both a predecessor (operand) of the
945 // user as well as a successor (the TF operands are flagged to the user).
946 // c1, f1 = CopyToReg
947 // c2, f2 = CopyToReg
948 // c3 = TokenFactor c1, c2
949 // ...
950 // = op c3, ..., f2
951 Chain = Chains[NumRegs-1];
952 else
953 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
954}
955
956void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
957 unsigned MatchingIdx, const SDLoc &dl,
958 SelectionDAG &DAG,
959 std::vector<SDValue> &Ops) const {
960 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
961
962 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
963 if (HasMatching)
964 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
965 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
966 // Put the register class of the virtual registers in the flag word. That
967 // way, later passes can recompute register class constraints for inline
968 // assembly as well as normal instructions.
969 // Don't do this for tied operands that can use the regclass information
970 // from the def.
971 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
972 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
973 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
974 }
975
976 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
977 Ops.push_back(Res);
978
979 if (Code == InlineAsm::Kind_Clobber) {
980 // Clobbers should always have a 1:1 mapping with registers, and may
981 // reference registers that have illegal (e.g. vector) types. Hence, we
982 // shouldn't try to apply any sort of splitting logic to them.
983 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast<void> (0))
984 "No 1:1 mapping from clobbers to regs?")(static_cast<void> (0));
985 Register SP = TLI.getStackPointerRegisterToSaveRestore();
986 (void)SP;
987 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
988 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
989 assert((static_cast<void> (0))
990 (Regs[I] != SP ||(static_cast<void> (0))
991 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast<void> (0))
992 "If we clobbered the stack pointer, MFI should know about it.")(static_cast<void> (0));
993 }
994 return;
995 }
996
997 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
998 MVT RegisterVT = RegVTs[Value];
999 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1000 RegisterVT);
1001 for (unsigned i = 0; i != NumRegs; ++i) {
1002 assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast<void> (0));
1003 unsigned TheReg = Regs[Reg++];
1004 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1005 }
1006 }
1007}
1008
1009SmallVector<std::pair<unsigned, TypeSize>, 4>
1010RegsForValue::getRegsAndSizes() const {
1011 SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;
1012 unsigned I = 0;
1013 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1014 unsigned RegCount = std::get<0>(CountAndVT);
1015 MVT RegisterVT = std::get<1>(CountAndVT);
1016 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1017 for (unsigned E = I + RegCount; I != E; ++I)
1018 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1019 }
1020 return OutVec;
1021}
1022
1023void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
1024 const TargetLibraryInfo *li) {
1025 AA = aa;
1026 GFI = gfi;
1027 LibInfo = li;
1028 DL = &DAG.getDataLayout();
1029 Context = DAG.getContext();
1030 LPadToCallSiteMap.clear();
1031 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1032}
1033
1034void SelectionDAGBuilder::clear() {
1035 NodeMap.clear();
1036 UnusedArgNodeMap.clear();
1037 PendingLoads.clear();
1038 PendingExports.clear();
1039 PendingConstrainedFP.clear();
1040 PendingConstrainedFPStrict.clear();
1041 CurInst = nullptr;
1042 HasTailCall = false;
1043 SDNodeOrder = LowestSDNodeOrder;
1044 StatepointLowering.clear();
1045}
1046
1047void SelectionDAGBuilder::clearDanglingDebugInfo() {
1048 DanglingDebugInfoMap.clear();
1049}
1050
1051// Update DAG root to include dependencies on Pending chains.
1052SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1053 SDValue Root = DAG.getRoot();
1054
1055 if (Pending.empty())
1056 return Root;
1057
1058 // Add current root to PendingChains, unless we already indirectly
1059 // depend on it.
1060 if (Root.getOpcode() != ISD::EntryToken) {
1061 unsigned i = 0, e = Pending.size();
1062 for (; i != e; ++i) {
1063 assert(Pending[i].getNode()->getNumOperands() > 1)(static_cast<void> (0));
1064 if (Pending[i].getNode()->getOperand(0) == Root)
1065 break; // Don't add the root if we already indirectly depend on it.
1066 }
1067
1068 if (i == e)
1069 Pending.push_back(Root);
1070 }
1071
1072 if (Pending.size() == 1)
1073 Root = Pending[0];
1074 else
1075 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1076
1077 DAG.setRoot(Root);
1078 Pending.clear();
1079 return Root;
1080}
1081
1082SDValue SelectionDAGBuilder::getMemoryRoot() {
1083 return updateRoot(PendingLoads);
1084}
1085
1086SDValue SelectionDAGBuilder::getRoot() {
1087 // Chain up all pending constrained intrinsics together with all
1088 // pending loads, by simply appending them to PendingLoads and
1089 // then calling getMemoryRoot().
1090 PendingLoads.reserve(PendingLoads.size() +
1091 PendingConstrainedFP.size() +
1092 PendingConstrainedFPStrict.size());
1093 PendingLoads.append(PendingConstrainedFP.begin(),
1094 PendingConstrainedFP.end());
1095 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1096 PendingConstrainedFPStrict.end());
1097 PendingConstrainedFP.clear();
1098 PendingConstrainedFPStrict.clear();
1099 return getMemoryRoot();
1100}
1101
1102SDValue SelectionDAGBuilder::getControlRoot() {
1103 // We need to emit pending fpexcept.strict constrained intrinsics,
1104 // so append them to the PendingExports list.
1105 PendingExports.append(PendingConstrainedFPStrict.begin(),
1106 PendingConstrainedFPStrict.end());
1107 PendingConstrainedFPStrict.clear();
1108 return updateRoot(PendingExports);
1109}
1110
1111void SelectionDAGBuilder::visit(const Instruction &I) {
1112 // Set up outgoing PHI node register values before emitting the terminator.
1113 if (I.isTerminator()) {
1114 HandlePHINodesInSuccessorBlocks(I.getParent());
1115 }
1116
1117 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1118 if (!isa<DbgInfoIntrinsic>(I))
1119 ++SDNodeOrder;
1120
1121 CurInst = &I;
1122
1123 visit(I.getOpcode(), I);
1124
1125 if (!I.isTerminator() && !HasTailCall &&
1126 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1127 CopyToExportRegsIfNeeded(&I);
1128
1129 CurInst = nullptr;
1130}
1131
1132void SelectionDAGBuilder::visitPHI(const PHINode &) {
1133 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")__builtin_unreachable();
1134}
1135
1136void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1137 // Note: this doesn't use InstVisitor, because it has to work with
1138 // ConstantExpr's in addition to instructions.
1139 switch (Opcode) {
1140 default: llvm_unreachable("Unknown instruction type encountered!")__builtin_unreachable();
1141 // Build the switch statement using the Instruction.def file.
1142#define HANDLE_INST(NUM, OPCODE, CLASS) \
1143 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1144#include "llvm/IR/Instruction.def"
1145 }
1146}
1147
1148void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,
1149 DebugLoc DL, unsigned Order) {
1150 // We treat variadic dbg_values differently at this stage.
1151 if (DI->hasArgList()) {
1152 // For variadic dbg_values we will now insert an undef.
1153 // FIXME: We can potentially recover these!
1154 SmallVector<SDDbgOperand, 2> Locs;
1155 for (const Value *V : DI->getValues()) {
1156 auto Undef = UndefValue::get(V->getType());
1157 Locs.push_back(SDDbgOperand::fromConst(Undef));
1158 }
1159 SDDbgValue *SDV = DAG.getDbgValueList(
1160 DI->getVariable(), DI->getExpression(), Locs, {},
1161 /*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);
1162 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1163 } else {
1164 // TODO: Dangling debug info will eventually either be resolved or produce
1165 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1166 // between the original dbg.value location and its resolved DBG_VALUE,
1167 // which we should ideally fill with an extra Undef DBG_VALUE.
1168 assert(DI->getNumVariableLocationOps() == 1 &&(static_cast<void> (0))
1169 "DbgValueInst without an ArgList should have a single location "(static_cast<void> (0))
1170 "operand.")(static_cast<void> (0));
1171 DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);
1172 }
1173}
1174
1175void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1176 const DIExpression *Expr) {
1177 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1178 const DbgValueInst *DI = DDI.getDI();
1179 DIVariable *DanglingVariable = DI->getVariable();
1180 DIExpression *DanglingExpr = DI->getExpression();
1181 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1182 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { } while (false);
1183 return true;
1184 }
1185 return false;
1186 };
1187
1188 for (auto &DDIMI : DanglingDebugInfoMap) {
1189 DanglingDebugInfoVector &DDIV = DDIMI.second;
1190
1191 // If debug info is to be dropped, run it through final checks to see
1192 // whether it can be salvaged.
1193 for (auto &DDI : DDIV)
1194 if (isMatchingDbgValue(DDI))
1195 salvageUnresolvedDbgValue(DDI);
1196
1197 erase_if(DDIV, isMatchingDbgValue);
1198 }
1199}
1200
1201// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1202// generate the debug data structures now that we've seen its definition.
1203void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1204 SDValue Val) {
1205 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1206 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1207 return;
1208
1209 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1210 for (auto &DDI : DDIV) {
1211 const DbgValueInst *DI = DDI.getDI();
1212 assert(!DI->hasArgList() && "Not implemented for variadic dbg_values")(static_cast<void> (0));
1213 assert(DI && "Ill-formed DanglingDebugInfo")(static_cast<void> (0));
1214 DebugLoc dl = DDI.getdl();
1215 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1216 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1217 DILocalVariable *Variable = DI->getVariable();
1218 DIExpression *Expr = DI->getExpression();
1219 assert(Variable->isValidLocationForIntrinsic(dl) &&(static_cast<void> (0))
1220 "Expected inlined-at fields to agree")(static_cast<void> (0));
1221 SDDbgValue *SDV;
1222 if (Val.getNode()) {
1223 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1224 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1225 // we couldn't resolve it directly when examining the DbgValue intrinsic
1226 // in the first place we should not be more successful here). Unless we
1227 // have some test case that prove this to be correct we should avoid
1228 // calling EmitFuncArgumentDbgValue here.
1229 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1230 LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { } while (false)
1231 << DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { } while (false);
1232 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { } while (false);
1233 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1234 // inserted after the definition of Val when emitting the instructions
1235 // after ISel. An alternative could be to teach
1236 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1237 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { } while (false)
1238 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { } while (false)
1239 << ValSDNodeOrder << "\n")do { } while (false);
1240 SDV = getDbgValue(Val, Variable, Expr, dl,
1241 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1242 DAG.AddDbgValue(SDV, false);
1243 } else
1244 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { } while (false)
1245 << "in EmitFuncArgumentDbgValue\n")do { } while (false);
1246 } else {
1247 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { } while (false);
1248 auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1249 auto SDV =
1250 DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1251 DAG.AddDbgValue(SDV, false);
1252 }
1253 }
1254 DDIV.clear();
1255}
1256
1257void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
1258 // TODO: For the variadic implementation, instead of only checking the fail
1259 // state of `handleDebugValue`, we need know specifically which values were
1260 // invalid, so that we attempt to salvage only those values when processing
1261 // a DIArgList.
1262 assert(!DDI.getDI()->hasArgList() &&(static_cast<void> (0))
1263 "Not implemented for variadic dbg_values")(static_cast<void> (0));
1264 Value *V = DDI.getDI()->getValue(0);
1265 DILocalVariable *Var = DDI.getDI()->getVariable();
1266 DIExpression *Expr = DDI.getDI()->getExpression();
1267 DebugLoc DL = DDI.getdl();
1268 DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1269 unsigned SDOrder = DDI.getSDNodeOrder();
1270 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1271 // that DW_OP_stack_value is desired.
1272 assert(isa<DbgValueInst>(DDI.getDI()))(static_cast<void> (0));
1273 bool StackValue = true;
1274
1275 // Can this Value can be encoded without any further work?
1276 if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))
1277 return;
1278
1279 // Attempt to salvage back through as many instructions as possible. Bail if
1280 // a non-instruction is seen, such as a constant expression or global
1281 // variable. FIXME: Further work could recover those too.
1282 while (isa<Instruction>(V)) {
1283 Instruction &VAsInst = *cast<Instruction>(V);
1284 // Temporary "0", awaiting real implementation.
1285 SmallVector<uint64_t, 16> Ops;
1286 SmallVector<Value *, 4> AdditionalValues;
1287 V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1288 AdditionalValues);
1289 // If we cannot salvage any further, and haven't yet found a suitable debug
1290 // expression, bail out.
1291 if (!V)
1292 break;
1293
1294 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1295 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1296 // here for variadic dbg_values, remove that condition.
1297 if (!AdditionalValues.empty())
1298 break;
1299
1300 // New value and expr now represent this debuginfo.
1301 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1302
1303 // Some kind of simplification occurred: check whether the operand of the
1304 // salvaged debug expression can be encoded in this DAG.
1305 if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,
1306 /*IsVariadic=*/false)) {
1307 LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "do { } while (false)
1308 << DDI.getDI() << "\nBy stripping back to:\n " << V)do { } while (false);
1309 return;
1310 }
1311 }
1312
1313 // This was the final opportunity to salvage this debug information, and it
1314 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1315 // any earlier variable location.
1316 auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1317 auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1318 DAG.AddDbgValue(SDV, false);
1319
1320 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()do { } while (false)
1321 << "\n")do { } while (false);
1322 LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)do { } while (false)
1323 << "\n")do { } while (false);
1324}
1325
1326bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1327 DILocalVariable *Var,
1328 DIExpression *Expr, DebugLoc dl,
1329 DebugLoc InstDL, unsigned Order,
1330 bool IsVariadic) {
1331 if (Values.empty())
1332 return true;
1333 SmallVector<SDDbgOperand> LocationOps;
1334 SmallVector<SDNode *> Dependencies;
1335 for (const Value *V : Values) {
1336 // Constant value.
1337 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1338 isa<ConstantPointerNull>(V)) {
1339 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1340 continue;
1341 }
1342
1343 // If the Value is a frame index, we can create a FrameIndex debug value
1344 // without relying on the DAG at all.
1345 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1346 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1347 if (SI != FuncInfo.StaticAllocaMap.end()) {
1348 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1349 continue;
1350 }
1351 }
1352
1353 // Do not use getValue() in here; we don't want to generate code at
1354 // this point if it hasn't been done yet.
1355 SDValue N = NodeMap[V];
1356 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1357 N = UnusedArgNodeMap[V];
1358 if (N.getNode()) {
1359 // Only emit func arg dbg value for non-variadic dbg.values for now.
1360 if (!IsVariadic && EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
1361 return true;
1362 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1363 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1364 // describe stack slot locations.
1365 //
1366 // Consider "int x = 0; int *px = &x;". There are two kinds of
1367 // interesting debug values here after optimization:
1368 //
1369 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1370 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1371 //
1372 // Both describe the direct values of their associated variables.
1373 Dependencies.push_back(N.getNode());
1374 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1375 continue;
1376 }
1377 LocationOps.emplace_back(
1378 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1379 continue;
1380 }
1381
1382 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1383 // Special rules apply for the first dbg.values of parameter variables in a
1384 // function. Identify them by the fact they reference Argument Values, that
1385 // they're parameters, and they are parameters of the current function. We
1386 // need to let them dangle until they get an SDNode.
1387 bool IsParamOfFunc =
1388 isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();
1389 if (IsParamOfFunc)
1390 return false;
1391
1392 // The value is not used in this block yet (or it would have an SDNode).
1393 // We still want the value to appear for the user if possible -- if it has
1394 // an associated VReg, we can refer to that instead.
1395 auto VMI = FuncInfo.ValueMap.find(V);
1396 if (VMI != FuncInfo.ValueMap.end()) {
1397 unsigned Reg = VMI->second;
1398 // If this is a PHI node, it may be split up into several MI PHI nodes
1399 // (in FunctionLoweringInfo::set).
1400 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1401 V->getType(), None);
1402 if (RFV.occupiesMultipleRegs()) {
1403 // FIXME: We could potentially support variadic dbg_values here.
1404 if (IsVariadic)
1405 return false;
1406 unsigned Offset = 0;
1407 unsigned BitsToDescribe = 0;
1408 if (auto VarSize = Var->getSizeInBits())
1409 BitsToDescribe = *VarSize;
1410 if (auto Fragment = Expr->getFragmentInfo())
1411 BitsToDescribe = Fragment->SizeInBits;
1412 for (auto RegAndSize : RFV.getRegsAndSizes()) {
1413 // Bail out if all bits are described already.
1414 if (Offset >= BitsToDescribe)
1415 break;
1416 // TODO: handle scalable vectors.
1417 unsigned RegisterSize = RegAndSize.second;
1418 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1419 ? BitsToDescribe - Offset
1420 : RegisterSize;
1421 auto FragmentExpr = DIExpression::createFragmentExpression(
1422 Expr, Offset, FragmentSize);
1423 if (!FragmentExpr)
1424 continue;
1425 SDDbgValue *SDV = DAG.getVRegDbgValue(
1426 Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);
1427 DAG.AddDbgValue(SDV, false);
1428 Offset += RegisterSize;
1429 }
1430 return true;
1431 }
1432 // We can use simple vreg locations for variadic dbg_values as well.
1433 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1434 continue;
1435 }
1436 // We failed to create a SDDbgOperand for V.
1437 return false;
1438 }
1439
1440 // We have created a SDDbgOperand for each Value in Values.
1441 // Should use Order instead of SDNodeOrder?
1442 assert(!LocationOps.empty())(static_cast<void> (0));
1443 SDDbgValue *SDV =
1444 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1445 /*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);
1446 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1447 return true;
1448}
1449
1450void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1451 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1452 for (auto &Pair : DanglingDebugInfoMap)
1453 for (auto &DDI : Pair.second)
1454 salvageUnresolvedDbgValue(DDI);
1455 clearDanglingDebugInfo();
1456}
1457
1458/// getCopyFromRegs - If there was virtual register allocated for the value V
1459/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1460SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1461 DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);
1462 SDValue Result;
1463
1464 if (It != FuncInfo.ValueMap.end()) {
1465 Register InReg = It->second;
1466
1467 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1468 DAG.getDataLayout(), InReg, Ty,
1469 None); // This is not an ABI copy.
1470 SDValue Chain = DAG.getEntryNode();
1471 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1472 V);
1473 resolveDanglingDebugInfo(V, Result);
1474 }
1475
1476 return Result;
1477}
1478
1479/// getValue - Return an SDValue for the given Value.
1480SDValue SelectionDAGBuilder::getValue(const Value *V) {
1481 // If we already have an SDValue for this value, use it. It's important
1482 // to do this first, so that we don't create a CopyFromReg if we already
1483 // have a regular SDValue.
1484 SDValue &N = NodeMap[V];
1485 if (N.getNode()) return N;
1486
1487 // If there's a virtual register allocated and initialized for this
1488 // value, use it.
1489 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1490 return copyFromReg;
1491
1492 // Otherwise create a new SDValue and remember it.
1493 SDValue Val = getValueImpl(V);
1494 NodeMap[V] = Val;
1495 resolveDanglingDebugInfo(V, Val);
1496 return Val;
1497}
1498
1499/// getNonRegisterValue - Return an SDValue for the given Value, but
1500/// don't look in FuncInfo.ValueMap for a virtual register.
1501SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1502 // If we already have an SDValue for this value, use it.
1503 SDValue &N = NodeMap[V];
1504 if (N.getNode()) {
1505 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1506 // Remove the debug location from the node as the node is about to be used
1507 // in a location which may differ from the original debug location. This
1508 // is relevant to Constant and ConstantFP nodes because they can appear
1509 // as constant expressions inside PHI nodes.
1510 N->setDebugLoc(DebugLoc());
1511 }
1512 return N;
1513 }
1514
1515 // Otherwise create a new SDValue and remember it.
1516 SDValue Val = getValueImpl(V);
1517 NodeMap[V] = Val;
1518 resolveDanglingDebugInfo(V, Val);
1519 return Val;
1520}
1521
1522/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1523/// Create an SDValue for the given value.
1524SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1525 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1526
1527 if (const Constant *C = dyn_cast<Constant>(V)) {
1528 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1529
1530 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1531 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1532
1533 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1534 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1535
1536 if (isa<ConstantPointerNull>(C)) {
1537 unsigned AS = V->getType()->getPointerAddressSpace();
1538 return DAG.getConstant(0, getCurSDLoc(),
1539 TLI.getPointerTy(DAG.getDataLayout(), AS));
1540 }
1541
1542 if (match(C, m_VScale(DAG.getDataLayout())))
1543 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1544
1545 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1546 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1547
1548 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1549 return DAG.getUNDEF(VT);
1550
1551 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1552 visit(CE->getOpcode(), *CE);
1553 SDValue N1 = NodeMap[V];
1554 assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast<void> (0));
1555 return N1;
1556 }
1557
1558 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1559 SmallVector<SDValue, 4> Constants;
1560 for (const Use &U : C->operands()) {
1561 SDNode *Val = getValue(U).getNode();
1562 // If the operand is an empty aggregate, there are no values.
1563 if (!Val) continue;
1564 // Add each leaf value from the operand to the Constants list
1565 // to form a flattened list of all the values.
1566 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1567 Constants.push_back(SDValue(Val, i));
1568 }
1569
1570 return DAG.getMergeValues(Constants, getCurSDLoc());
1571 }
1572
1573 if (const ConstantDataSequential *CDS =
1574 dyn_cast<ConstantDataSequential>(C)) {
1575 SmallVector<SDValue, 4> Ops;
1576 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1577 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1578 // Add each leaf value from the operand to the Constants list
1579 // to form a flattened list of all the values.
1580 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1581 Ops.push_back(SDValue(Val, i));
1582 }
1583
1584 if (isa<ArrayType>(CDS->getType()))
1585 return DAG.getMergeValues(Ops, getCurSDLoc());
1586 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1587 }
1588
1589 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1590 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast<void> (0))
1591 "Unknown struct or array constant!")(static_cast<void> (0));
1592
1593 SmallVector<EVT, 4> ValueVTs;
1594 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1595 unsigned NumElts = ValueVTs.size();
1596 if (NumElts == 0)
1597 return SDValue(); // empty struct
1598 SmallVector<SDValue, 4> Constants(NumElts);
1599 for (unsigned i = 0; i != NumElts; ++i) {
1600 EVT EltVT = ValueVTs[i];
1601 if (isa<UndefValue>(C))
1602 Constants[i] = DAG.getUNDEF(EltVT);
1603 else if (EltVT.isFloatingPoint())
1604 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1605 else
1606 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1607 }
1608
1609 return DAG.getMergeValues(Constants, getCurSDLoc());
1610 }
1611
1612 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1613 return DAG.getBlockAddress(BA, VT);
1614
1615 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1616 return getValue(Equiv->getGlobalValue());
1617
1618 VectorType *VecTy = cast<VectorType>(V->getType());
1619
1620 // Now that we know the number and type of the elements, get that number of
1621 // elements into the Ops array based on what kind of constant it is.
1622 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1623 SmallVector<SDValue, 16> Ops;
1624 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1625 for (unsigned i = 0; i != NumElements; ++i)
1626 Ops.push_back(getValue(CV->getOperand(i)));
1627
1628 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1629 } else if (isa<ConstantAggregateZero>(C)) {
1630 EVT EltVT =
1631 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1632
1633 SDValue Op;
1634 if (EltVT.isFloatingPoint())
1635 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1636 else
1637 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1638
1639 if (isa<ScalableVectorType>(VecTy))
1640 return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);
1641 else {
1642 SmallVector<SDValue, 16> Ops;
1643 Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);
1644 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1645 }
1646 }
1647 llvm_unreachable("Unknown vector constant")__builtin_unreachable();
1648 }
1649
1650 // If this is a static alloca, generate it as the frameindex instead of
1651 // computation.
1652 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1653 DenseMap<const AllocaInst*, int>::iterator SI =
1654 FuncInfo.StaticAllocaMap.find(AI);
1655 if (SI != FuncInfo.StaticAllocaMap.end())
1656 return DAG.getFrameIndex(SI->second,
1657 TLI.getFrameIndexTy(DAG.getDataLayout()));
1658 }
1659
1660 // If this is an instruction which fast-isel has deferred, select it now.
1661 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1662 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1663
1664 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1665 Inst->getType(), None);
1666 SDValue Chain = DAG.getEntryNode();
1667 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1668 }
1669
1670 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) {
1671 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1672 }
1673 llvm_unreachable("Can't get register for value!")__builtin_unreachable();
1674}
1675
1676void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1677 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1678 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1679 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1680 bool IsSEH = isAsynchronousEHPersonality(Pers);
1681 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1682 if (!IsSEH)
1683 CatchPadMBB->setIsEHScopeEntry();
1684 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1685 if (IsMSVCCXX || IsCoreCLR)
1686 CatchPadMBB->setIsEHFuncletEntry();
1687}
1688
1689void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1690 // Update machine-CFG edge.
1691 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1692 FuncInfo.MBB->addSuccessor(TargetMBB);
1693 TargetMBB->setIsEHCatchretTarget(true);
1694 DAG.getMachineFunction().setHasEHCatchret(true);
1695
1696 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1697 bool IsSEH = isAsynchronousEHPersonality(Pers);
1698 if (IsSEH) {
1699 // If this is not a fall-through branch or optimizations are switched off,
1700 // emit the branch.
1701 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1702 TM.getOptLevel() == CodeGenOpt::None)
1703 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1704 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1705 return;
1706 }
1707
1708 // Figure out the funclet membership for the catchret's successor.
1709 // This will be used by the FuncletLayout pass to determine how to order the
1710 // BB's.
1711 // A 'catchret' returns to the outer scope's color.
1712 Value *ParentPad = I.getCatchSwitchParentPad();
1713 const BasicBlock *SuccessorColor;
1714 if (isa<ConstantTokenNone>(ParentPad))
1715 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1716 else
1717 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1718 assert(SuccessorColor && "No parent funclet for catchret!")(static_cast<void> (0));
1719 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1720 assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast<void> (0));
1721
1722 // Create the terminator node.
1723 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1724 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1725 DAG.getBasicBlock(SuccessorColorMBB));
1726 DAG.setRoot(Ret);
1727}
1728
1729void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1730 // Don't emit any special code for the cleanuppad instruction. It just marks
1731 // the start of an EH scope/funclet.
1732 FuncInfo.MBB->setIsEHScopeEntry();
1733 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1734 if (Pers != EHPersonality::Wasm_CXX) {
1735 FuncInfo.MBB->setIsEHFuncletEntry();
1736 FuncInfo.MBB->setIsCleanupFuncletEntry();
1737 }
1738}
1739
1740// In wasm EH, even though a catchpad may not catch an exception if a tag does
1741// not match, it is OK to add only the first unwind destination catchpad to the
1742// successors, because there will be at least one invoke instruction within the
1743// catch scope that points to the next unwind destination, if one exists, so
1744// CFGSort cannot mess up with BB sorting order.
1745// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1746// call within them, and catchpads only consisting of 'catch (...)' have a
1747// '__cxa_end_catch' call within them, both of which generate invokes in case
1748// the next unwind destination exists, i.e., the next unwind destination is not
1749// the caller.)
1750//
1751// Having at most one EH pad successor is also simpler and helps later
1752// transformations.
1753//
1754// For example,
1755// current:
1756// invoke void @foo to ... unwind label %catch.dispatch
1757// catch.dispatch:
1758// %0 = catchswitch within ... [label %catch.start] unwind label %next
1759// catch.start:
1760// ...
1761// ... in this BB or some other child BB dominated by this BB there will be an
1762// invoke that points to 'next' BB as an unwind destination
1763//
1764// next: ; We don't need to add this to 'current' BB's successor
1765// ...
1766static void findWasmUnwindDestinations(
1767 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1768 BranchProbability Prob,
1769 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1770 &UnwindDests) {
1771 while (EHPadBB) {
1772 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1773 if (isa<CleanupPadInst>(Pad)) {
1774 // Stop on cleanup pads.
1775 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1776 UnwindDests.back().first->setIsEHScopeEntry();
1777 break;
1778 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1779 // Add the catchpad handlers to the possible destinations. We don't
1780 // continue to the unwind destination of the catchswitch for wasm.
1781 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1782 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1783 UnwindDests.back().first->setIsEHScopeEntry();
1784 }
1785 break;
1786 } else {
1787 continue;
1788 }
1789 }
1790}
1791
1792/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1793/// many places it could ultimately go. In the IR, we have a single unwind
1794/// destination, but in the machine CFG, we enumerate all the possible blocks.
1795/// This function skips over imaginary basic blocks that hold catchswitch
1796/// instructions, and finds all the "real" machine
1797/// basic block destinations. As those destinations may not be successors of
1798/// EHPadBB, here we also calculate the edge probability to those destinations.
1799/// The passed-in Prob is the edge probability to EHPadBB.
1800static void findUnwindDestinations(
1801 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1802 BranchProbability Prob,
1803 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1804 &UnwindDests) {
1805 EHPersonality Personality =
1806 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1807 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1808 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1809 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1810 bool IsSEH = isAsynchronousEHPersonality(Personality);
1811
1812 if (IsWasmCXX) {
1813 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1814 assert(UnwindDests.size() <= 1 &&(static_cast<void> (0))
1815 "There should be at most one unwind destination for wasm")(static_cast<void> (0));
1816 return;
1817 }
1818
1819 while (EHPadBB) {
1820 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1821 BasicBlock *NewEHPadBB = nullptr;
1822 if (isa<LandingPadInst>(Pad)) {
1823 // Stop on landingpads. They are not funclets.
1824 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1825 break;
1826 } else if (isa<CleanupPadInst>(Pad)) {
1827 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1828 // personalities.
1829 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1830 UnwindDests.back().first->setIsEHScopeEntry();
1831 UnwindDests.back().first->setIsEHFuncletEntry();
1832 break;
1833 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1834 // Add the catchpad handlers to the possible destinations.
1835 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1836 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1837 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1838 if (IsMSVCCXX || IsCoreCLR)
1839 UnwindDests.back().first->setIsEHFuncletEntry();
1840 if (!IsSEH)
1841 UnwindDests.back().first->setIsEHScopeEntry();
1842 }
1843 NewEHPadBB = CatchSwitch->getUnwindDest();
1844 } else {
1845 continue;
1846 }
1847
1848 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1849 if (BPI && NewEHPadBB)
1850 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1851 EHPadBB = NewEHPadBB;
1852 }
1853}
1854
1855void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1856 // Update successor info.
1857 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1858 auto UnwindDest = I.getUnwindDest();
1859 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1860 BranchProbability UnwindDestProb =
1861 (BPI && UnwindDest)
1862 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1863 : BranchProbability::getZero();
1864 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1865 for (auto &UnwindDest : UnwindDests) {
1866 UnwindDest.first->setIsEHPad();
1867 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1868 }
1869 FuncInfo.MBB->normalizeSuccProbs();
1870
1871 // Create the terminator node.
1872 SDValue Ret =
1873 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1874 DAG.setRoot(Ret);
1875}
1876
1877void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1878 report_fatal_error("visitCatchSwitch not yet implemented!");
1879}
1880
1881void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1882 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1883 auto &DL = DAG.getDataLayout();
1884 SDValue Chain = getControlRoot();
1885 SmallVector<ISD::OutputArg, 8> Outs;
1886 SmallVector<SDValue, 8> OutVals;
1887
1888 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1889 // lower
1890 //
1891 // %val = call <ty> @llvm.experimental.deoptimize()
1892 // ret <ty> %val
1893 //
1894 // differently.
1895 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1896 LowerDeoptimizingReturn();
1897 return;
1898 }
1899
1900 if (!FuncInfo.CanLowerReturn) {
1901 unsigned DemoteReg = FuncInfo.DemoteRegister;
1902 const Function *F = I.getParent()->getParent();
1903
1904 // Emit a store of the return value through the virtual register.
1905 // Leave Outs empty so that LowerReturn won't try to load return
1906 // registers the usual way.
1907 SmallVector<EVT, 1> PtrValueVTs;
1908 ComputeValueVTs(TLI, DL,
1909 F->getReturnType()->getPointerTo(
1910 DAG.getDataLayout().getAllocaAddrSpace()),
1911 PtrValueVTs);
1912
1913 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1914 DemoteReg, PtrValueVTs[0]);
1915 SDValue RetOp = getValue(I.getOperand(0));
1916
1917 SmallVector<EVT, 4> ValueVTs, MemVTs;
1918 SmallVector<uint64_t, 4> Offsets;
1919 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1920 &Offsets);
1921 unsigned NumValues = ValueVTs.size();
1922
1923 SmallVector<SDValue, 4> Chains(NumValues);
1924 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
1925 for (unsigned i = 0; i != NumValues; ++i) {
1926 // An aggregate return value cannot wrap around the address space, so
1927 // offsets to its parts don't wrap either.
1928 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
1929 TypeSize::Fixed(Offsets[i]));
1930
1931 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
1932 if (MemVTs[i] != ValueVTs[i])
1933 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1934 Chains[i] = DAG.getStore(
1935 Chain, getCurSDLoc(), Val,
1936 // FIXME: better loc info would be nice.
1937 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
1938 commonAlignment(BaseAlign, Offsets[i]));
1939 }
1940
1941 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1942 MVT::Other, Chains);
1943 } else if (I.getNumOperands() != 0) {
1944 SmallVector<EVT, 4> ValueVTs;
1945 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1946 unsigned NumValues = ValueVTs.size();
1947 if (NumValues) {
1948 SDValue RetOp = getValue(I.getOperand(0));
1949
1950 const Function *F = I.getParent()->getParent();
1951
1952 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1953 I.getOperand(0)->getType(), F->getCallingConv(),
1954 /*IsVarArg*/ false, DL);
1955
1956 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1957 if (F->getAttributes().hasRetAttr(Attribute::SExt))
1958 ExtendKind = ISD::SIGN_EXTEND;
1959 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
1960 ExtendKind = ISD::ZERO_EXTEND;
1961
1962 LLVMContext &Context = F->getContext();
1963 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
1964
1965 for (unsigned j = 0; j != NumValues; ++j) {
1966 EVT VT = ValueVTs[j];
1967
1968 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1969 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1970
1971 CallingConv::ID CC = F->getCallingConv();
1972
1973 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1974 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1975 SmallVector<SDValue, 4> Parts(NumParts);
1976 getCopyToParts(DAG, getCurSDLoc(),
1977 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1978 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1979
1980 // 'inreg' on function refers to return value
1981 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1982 if (RetInReg)
1983 Flags.setInReg();
1984
1985 if (I.getOperand(0)->getType()->isPointerTy()) {
1986 Flags.setPointer();
1987 Flags.setPointerAddrSpace(
1988 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
1989 }
1990
1991 if (NeedsRegBlock) {
1992 Flags.setInConsecutiveRegs();
1993 if (j == NumValues - 1)
1994 Flags.setInConsecutiveRegsLast();
1995 }
1996
1997 // Propagate extension type if any
1998 if (ExtendKind == ISD::SIGN_EXTEND)
1999 Flags.setSExt();
2000 else if (ExtendKind == ISD::ZERO_EXTEND)
2001 Flags.setZExt();
2002
2003 for (unsigned i = 0; i < NumParts; ++i) {
2004 Outs.push_back(ISD::OutputArg(Flags,
2005 Parts[i].getValueType().getSimpleVT(),
2006 VT, /*isfixed=*/true, 0, 0));
2007 OutVals.push_back(Parts[i]);
2008 }
2009 }
2010 }
2011 }
2012
2013 // Push in swifterror virtual register as the last element of Outs. This makes
2014 // sure swifterror virtual register will be returned in the swifterror
2015 // physical register.
2016 const Function *F = I.getParent()->getParent();
2017 if (TLI.supportSwiftError() &&
2018 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2019 assert(SwiftError.getFunctionArg() && "Need a swift error argument")(static_cast<void> (0));
2020 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2021 Flags.setSwiftError();
2022 Outs.push_back(ISD::OutputArg(
2023 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2024 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2025 // Create SDNode for the swifterror virtual register.
2026 OutVals.push_back(
2027 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2028 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2029 EVT(TLI.getPointerTy(DL))));
2030 }
2031
2032 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2033 CallingConv::ID CallConv =
2034 DAG.getMachineFunction().getFunction().getCallingConv();
2035 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2036 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2037
2038 // Verify that the target's LowerReturn behaved as expected.
2039 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast<void> (0))
2040 "LowerReturn didn't return a valid chain!")(static_cast<void> (0));
2041
2042 // Update the DAG with the new chain value resulting from return lowering.
2043 DAG.setRoot(Chain);
2044}
2045
2046/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2047/// created for it, emit nodes to copy the value into the virtual
2048/// registers.
2049void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2050 // Skip empty types
2051 if (V->getType()->isEmptyTy())
2052 return;
2053
2054 DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);
2055 if (VMI != FuncInfo.ValueMap.end()) {
2056 assert(!V->use_empty() && "Unused value assigned virtual registers!")(static_cast<void> (0));
2057 CopyValueToVirtualRegister(V, VMI->second);
2058 }
2059}
2060
2061/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2062/// the current basic block, add it to ValueMap now so that we'll get a
2063/// CopyTo/FromReg.
2064void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2065 // No need to export constants.
2066 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2067
2068 // Already exported?
2069 if (FuncInfo.isExportedInst(V)) return;
2070
2071 unsigned Reg = FuncInfo.InitializeRegForValue(V);
2072 CopyValueToVirtualRegister(V, Reg);
2073}
2074
2075bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2076 const BasicBlock *FromBB) {
2077 // The operands of the setcc have to be in this block. We don't know
2078 // how to export them from some other block.
2079 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2080 // Can export from current BB.
2081 if (VI->getParent() == FromBB)
2082 return true;
2083
2084 // Is already exported, noop.
2085 return FuncInfo.isExportedInst(V);
2086 }
2087
2088 // If this is an argument, we can export it if the BB is the entry block or
2089 // if it is already exported.
2090 if (isa<Argument>(V)) {
2091 if (FromBB->isEntryBlock())
2092 return true;
2093
2094 // Otherwise, can only export this if it is already exported.
2095 return FuncInfo.isExportedInst(V);
2096 }
2097
2098 // Otherwise, constants can always be exported.
2099 return true;
2100}
2101
2102/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2103BranchProbability
2104SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2105 const MachineBasicBlock *Dst) const {
2106 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2107 const BasicBlock *SrcBB = Src->getBasicBlock();
2108 const BasicBlock *DstBB = Dst->getBasicBlock();
2109 if (!BPI) {
2110 // If BPI is not available, set the default probability as 1 / N, where N is
2111 // the number of successors.
2112 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2113 return BranchProbability(1, SuccSize);
2114 }
2115 return BPI->getEdgeProbability(SrcBB, DstBB);
2116}
2117
2118void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2119 MachineBasicBlock *Dst,
2120 BranchProbability Prob) {
2121 if (!FuncInfo.BPI)
2122 Src->addSuccessorWithoutProb(Dst);
2123 else {
2124 if (Prob.isUnknown())
2125 Prob = getEdgeProbability(Src, Dst);
2126 Src->addSuccessor(Dst, Prob);
2127 }
2128}
2129
2130static bool InBlock(const Value *V, const BasicBlock *BB) {
2131 if (const Instruction *I = dyn_cast<Instruction>(V))
2132 return I->getParent() == BB;
2133 return true;
2134}
2135
2136/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2137/// This function emits a branch and is used at the leaves of an OR or an
2138/// AND operator tree.
2139void
2140SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2141 MachineBasicBlock *TBB,
2142 MachineBasicBlock *FBB,
2143 MachineBasicBlock *CurBB,
2144 MachineBasicBlock *SwitchBB,
2145 BranchProbability TProb,
2146 BranchProbability FProb,
2147 bool InvertCond) {
2148 const BasicBlock *BB = CurBB->getBasicBlock();
2149
2150 // If the leaf of the tree is a comparison, merge the condition into
2151 // the caseblock.
2152 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2153 // The operands of the cmp have to be in this block. We don't know
2154 // how to export them from some other block. If this is the first block
2155 // of the sequence, no exporting is needed.
2156 if (CurBB == SwitchBB ||
2157 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2158 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2159 ISD::CondCode Condition;
2160 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2161 ICmpInst::Predicate Pred =
2162 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2163 Condition = getICmpCondCode(Pred);
2164 } else {
2165 const FCmpInst *FC = cast<FCmpInst>(Cond);
2166 FCmpInst::Predicate Pred =
2167 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2168 Condition = getFCmpCondCode(Pred);
2169 if (TM.Options.NoNaNsFPMath)
2170 Condition = getFCmpCodeWithoutNaN(Condition);
2171 }
2172
2173 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2174 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2175 SL->SwitchCases.push_back(CB);
2176 return;
2177 }
2178 }
2179
2180 // Create a CaseBlock record representing this branch.
2181 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2182 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2183 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2184 SL->SwitchCases.push_back(CB);
2185}
2186
2187void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2188 MachineBasicBlock *TBB,
2189 MachineBasicBlock *FBB,
2190 MachineBasicBlock *CurBB,
2191 MachineBasicBlock *SwitchBB,
2192 Instruction::BinaryOps Opc,
2193 BranchProbability TProb,
2194 BranchProbability FProb,
2195 bool InvertCond) {
2196 // Skip over not part of the tree and remember to invert op and operands at
2197 // next level.
2198 Value *NotCond;
2199 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2200 InBlock(NotCond, CurBB->getBasicBlock())) {
2201 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2202 !InvertCond);
2203 return;
2204 }
2205
2206 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2207 const Value *BOpOp0, *BOpOp1;
2208 // Compute the effective opcode for Cond, taking into account whether it needs
2209 // to be inverted, e.g.
2210 // and (not (or A, B)), C
2211 // gets lowered as
2212 // and (and (not A, not B), C)
2213 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2214 if (BOp) {
2215 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2216 ? Instruction::And
2217 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2218 ? Instruction::Or
2219 : (Instruction::BinaryOps)0);
2220 if (InvertCond) {
2221 if (BOpc == Instruction::And)
2222 BOpc = Instruction::Or;
2223 else if (BOpc == Instruction::Or)
2224 BOpc = Instruction::And;
2225 }
2226 }
2227
2228 // If this node is not part of the or/and tree, emit it as a branch.
2229 // Note that all nodes in the tree should have same opcode.
2230 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2231 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2232 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2233 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2234 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2235 TProb, FProb, InvertCond);
2236 return;
2237 }
2238
2239 // Create TmpBB after CurBB.
2240 MachineFunction::iterator BBI(CurBB);
2241 MachineFunction &MF = DAG.getMachineFunction();
2242 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
2243 CurBB->getParent()->insert(++BBI, TmpBB);
2244
2245 if (Opc == Instruction::Or) {
2246 // Codegen X | Y as:
2247 // BB1:
2248 // jmp_if_X TBB
2249 // jmp TmpBB
2250 // TmpBB:
2251 // jmp_if_Y TBB
2252 // jmp FBB
2253 //
2254
2255 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2256 // The requirement is that
2257 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2258 // = TrueProb for original BB.
2259 // Assuming the original probabilities are A and B, one choice is to set
2260 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2261 // A/(1+B) and 2B/(1+B). This choice assumes that
2262 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2263 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2264 // TmpBB, but the math is more complicated.
2265
2266 auto NewTrueProb = TProb / 2;
2267 auto NewFalseProb = TProb / 2 + FProb;
2268 // Emit the LHS condition.
2269 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2270 NewFalseProb, InvertCond);
2271
2272 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2273 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2274 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2275 // Emit the RHS condition into TmpBB.
2276 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2277 Probs[1], InvertCond);
2278 } else {
2279 assert(Opc == Instruction::And && "Unknown merge op!")(static_cast<void> (0));
2280 // Codegen X & Y as:
2281 // BB1:
2282 // jmp_if_X TmpBB
2283 // jmp FBB
2284 // TmpBB:
2285 // jmp_if_Y TBB
2286 // jmp FBB
2287 //
2288 // This requires creation of TmpBB after CurBB.
2289
2290 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2291 // The requirement is that
2292 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2293 // = FalseProb for original BB.
2294 // Assuming the original probabilities are A and B, one choice is to set
2295 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2296 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2297 // TrueProb for BB1 * FalseProb for TmpBB.
2298
2299 auto NewTrueProb = TProb + FProb / 2;
2300 auto NewFalseProb = FProb / 2;
2301 // Emit the LHS condition.
2302 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2303 NewFalseProb, InvertCond);
2304
2305 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2306 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2307 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2308 // Emit the RHS condition into TmpBB.
2309 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2310 Probs[1], InvertCond);
2311 }
2312}
2313
2314/// If the set of cases should be emitted as a series of branches, return true.
2315/// If we should emit this as a bunch of and/or'd together conditions, return
2316/// false.
2317bool
2318SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2319 if (Cases.size() != 2) return true;
2320
2321 // If this is two comparisons of the same values or'd or and'd together, they
2322 // will get folded into a single comparison, so don't emit two blocks.
2323 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2324 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2325 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2326 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2327 return false;
2328 }
2329
2330 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2331 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2332 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2333 Cases[0].CC == Cases[1].CC &&
2334 isa<Constant>(Cases[0].CmpRHS) &&
2335 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2336 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2337 return false;
2338 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2339 return false;
2340 }
2341
2342 return true;
2343}
2344
2345void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2346 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2347
2348 // Update machine-CFG edges.
2349 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2350
2351 if (I.isUnconditional()) {
2352 // Update machine-CFG edges.
2353 BrMBB->addSuccessor(Succ0MBB);
2354
2355 // If this is not a fall-through branch or optimizations are switched off,
2356 // emit the branch.
2357 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2358 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2359 MVT::Other, getControlRoot(),
2360 DAG.getBasicBlock(Succ0MBB)));
2361
2362 return;
2363 }
2364
2365 // If this condition is one of the special cases we handle, do special stuff
2366 // now.
2367 const Value *CondVal = I.getCondition();
2368 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2369
2370 // If this is a series of conditions that are or'd or and'd together, emit
2371 // this as a sequence of branches instead of setcc's with and/or operations.
2372 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2373 // unpredictable branches, and vector extracts because those jumps are likely
2374 // expensive for any target), this should improve performance.
2375 // For example, instead of something like:
2376 // cmp A, B
2377 // C = seteq
2378 // cmp D, E
2379 // F = setle
2380 // or C, F
2381 // jnz foo
2382 // Emit:
2383 // cmp A, B
2384 // je foo
2385 // cmp D, E
2386 // jle foo
2387 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2388 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2389 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2390 Value *Vec;
2391 const Value *BOp0, *BOp1;
2392 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2393 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2394 Opcode = Instruction::And;
2395 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2396 Opcode = Instruction::Or;
2397
2398 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2399 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2400 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2401 getEdgeProbability(BrMBB, Succ0MBB),
2402 getEdgeProbability(BrMBB, Succ1MBB),
2403 /*InvertCond=*/false);
2404 // If the compares in later blocks need to use values not currently
2405 // exported from this block, export them now. This block should always
2406 // be the first entry.
2407 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast<void> (0));
2408
2409 // Allow some cases to be rejected.
2410 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2411 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2412 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2413 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2414 }
2415
2416 // Emit the branch for this block.
2417 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2418 SL->SwitchCases.erase(SL->SwitchCases.begin());
2419 return;
2420 }
2421
2422 // Okay, we decided not to do this, remove any inserted MBB's and clear
2423 // SwitchCases.
2424 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2425 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2426
2427 SL->SwitchCases.clear();
2428 }
2429 }
2430
2431 // Create a CaseBlock record representing this branch.
2432 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2433 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2434
2435 // Use visitSwitchCase to actually insert the fast branch sequence for this
2436 // cond branch.
2437 visitSwitchCase(CB, BrMBB);
2438}
2439
2440/// visitSwitchCase - Emits the necessary code to represent a single node in
2441/// the binary search tree resulting from lowering a switch instruction.
2442void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2443 MachineBasicBlock *SwitchBB) {
2444 SDValue Cond;
2445 SDValue CondLHS = getValue(CB.CmpLHS);
2446 SDLoc dl = CB.DL;
2447
2448 if (CB.CC == ISD::SETTRUE) {
2449 // Branch or fall through to TrueBB.
2450 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2451 SwitchBB->normalizeSuccProbs();
2452 if (CB.TrueBB != NextBlock(SwitchBB)) {
2453 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2454 DAG.getBasicBlock(CB.TrueBB)));
2455 }
2456 return;
2457 }
2458
2459 auto &TLI = DAG.getTargetLoweringInfo();
2460 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2461
2462 // Build the setcc now.
2463 if (!CB.CmpMHS) {
2464 // Fold "(X == true)" to X and "(X == false)" to !X to
2465 // handle common cases produced by branch lowering.
2466 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2467 CB.CC == ISD::SETEQ)
2468 Cond = CondLHS;
2469 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2470 CB.CC == ISD::SETEQ) {
2471 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2472 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2473 } else {
2474 SDValue CondRHS = getValue(CB.CmpRHS);
2475
2476 // If a pointer's DAG type is larger than its memory type then the DAG
2477 // values are zero-extended. This breaks signed comparisons so truncate
2478 // back to the underlying type before doing the compare.
2479 if (CondLHS.getValueType() != MemVT) {
2480 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2481 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2482 }
2483 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2484 }
2485 } else {
2486 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast<void> (0));
2487
2488 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2489 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2490
2491 SDValue CmpOp = getValue(CB.CmpMHS);
2492 EVT VT = CmpOp.getValueType();
2493
2494 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2495 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2496 ISD::SETLE);
2497 } else {
2498 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2499 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2500 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2501 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2502 }
2503 }
2504
2505 // Update successor info
2506 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2507 // TrueBB and FalseBB are always different unless the incoming IR is
2508 // degenerate. This only happens when running llc on weird IR.
2509 if (CB.TrueBB != CB.FalseBB)
2510 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2511 SwitchBB->normalizeSuccProbs();
2512
2513 // If the lhs block is the next block, invert the condition so that we can
2514 // fall through to the lhs instead of the rhs block.
2515 if (CB.TrueBB == NextBlock(SwitchBB)) {
2516 std::swap(CB.TrueBB, CB.FalseBB);
2517 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2518 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2519 }
2520
2521 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2522 MVT::Other, getControlRoot(), Cond,
2523 DAG.getBasicBlock(CB.TrueBB));
2524
2525 // Insert the false branch. Do this even if it's a fall through branch,
2526 // this makes it easier to do DAG optimizations which require inverting
2527 // the branch condition.
2528 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2529 DAG.getBasicBlock(CB.FalseBB));
2530
2531 DAG.setRoot(BrCond);
2532}
2533
2534/// visitJumpTable - Emit JumpTable node in the current MBB
2535void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
2536 // Emit the code for the jump table
2537 assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast<void> (0));
2538 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2539 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2540 JT.Reg, PTy);
2541 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2542 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2543 MVT::Other, Index.getValue(1),
2544 Table, Index);
2545 DAG.setRoot(BrJumpTable);
2546}
2547
2548/// visitJumpTableHeader - This function emits necessary code to produce index
2549/// in the JumpTable from switch case.
2550void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
2551 JumpTableHeader &JTH,
2552 MachineBasicBlock *SwitchBB) {
2553 SDLoc dl = getCurSDLoc();
2554
2555 // Subtract the lowest switch case value from the value being switched on.
2556 SDValue SwitchOp = getValue(JTH.SValue);
2557 EVT VT = SwitchOp.getValueType();
2558 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2559 DAG.getConstant(JTH.First, dl, VT));
2560
2561 // The SDNode we just created, which holds the value being switched on minus
2562 // the smallest case value, needs to be copied to a virtual register so it
2563 // can be used as an index into the jump table in a subsequent basic block.
2564 // This value may be smaller or larger than the target's pointer type, and
2565 // therefore require extension or truncating.
2566 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2567 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2568
2569 unsigned JumpTableReg =
2570 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2571 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2572 JumpTableReg, SwitchOp);
2573 JT.Reg = JumpTableReg;
2574
2575 if (!JTH.OmitRangeCheck) {
2576 // Emit the range check for the jump table, and branch to the default block
2577 // for the switch statement if the value being switched on exceeds the
2578 // largest case in the switch.
2579 SDValue CMP = DAG.getSetCC(
2580 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2581 Sub.getValueType()),
2582 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2583
2584 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2585 MVT::Other, CopyTo, CMP,
2586 DAG.getBasicBlock(JT.Default));
2587
2588 // Avoid emitting unnecessary branches to the next block.
2589 if (JT.MBB != NextBlock(SwitchBB))
2590 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2591 DAG.getBasicBlock(JT.MBB));
2592
2593 DAG.setRoot(BrCond);
2594 } else {
2595 // Avoid emitting unnecessary branches to the next block.
2596 if (JT.MBB != NextBlock(SwitchBB))
2597 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2598 DAG.getBasicBlock(JT.MBB)));
2599 else
2600 DAG.setRoot(CopyTo);
2601 }
2602}
2603
2604/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2605/// variable if there exists one.
2606static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2607 SDValue &Chain) {
2608 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2609 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2610 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2611 MachineFunction &MF = DAG.getMachineFunction();
2612 Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2613 MachineSDNode *Node =
2614 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2615 if (Global) {
2616 MachinePointerInfo MPInfo(Global);
2617 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2618 MachineMemOperand::MODereferenceable;
2619 MachineMemOperand *MemRef = MF.getMachineMemOperand(
2620 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2621 DAG.setNodeMemRefs(Node, {MemRef});
2622 }
2623 if (PtrTy != PtrMemTy)
2624 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2625 return SDValue(Node, 0);
2626}
2627
2628/// Codegen a new tail for a stack protector check ParentMBB which has had its
2629/// tail spliced into a stack protector check success bb.
2630///
2631/// For a high level explanation of how this fits into the stack protector
2632/// generation see the comment on the declaration of class
2633/// StackProtectorDescriptor.
2634void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2635 MachineBasicBlock *ParentBB) {
2636
2637 // First create the loads to the guard/stack slot for the comparison.
2638 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2639 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2640 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2641
2642 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2643 int FI = MFI.getStackProtectorIndex();
2644
2645 SDValue Guard;
2646 SDLoc dl = getCurSDLoc();
2647 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2648 const Module &M = *ParentBB->getParent()->getFunction().getParent();
2649 Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
2650
2651 // Generate code to load the content of the guard slot.
2652 SDValue GuardVal = DAG.getLoad(
2653 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2654 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2655 MachineMemOperand::MOVolatile);
2656
2657 if (TLI.useStackGuardXorFP())
2658 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2659
2660 // Retrieve guard check function, nullptr if instrumentation is inlined.
2661 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2662 // The target provides a guard check function to validate the guard value.
2663 // Generate a call to that function with the content of the guard slot as
2664 // argument.
2665 FunctionType *FnTy = GuardCheckFn->getFunctionType();
2666 assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast<void> (0));
2667
2668 TargetLowering::ArgListTy Args;
2669 TargetLowering::ArgListEntry Entry;
2670 Entry.Node = GuardVal;
2671 Entry.Ty = FnTy->getParamType(0);
2672 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2673 Entry.IsInReg = true;
2674 Args.push_back(Entry);
2675
2676 TargetLowering::CallLoweringInfo CLI(DAG);
2677 CLI.setDebugLoc(getCurSDLoc())
2678 .setChain(DAG.getEntryNode())
2679 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2680 getValue(GuardCheckFn), std::move(Args));
2681
2682 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2683 DAG.setRoot(Result.second);
2684 return;
2685 }
2686
2687 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2688 // Otherwise, emit a volatile load to retrieve the stack guard value.
2689 SDValue Chain = DAG.getEntryNode();
2690 if (TLI.useLoadStackGuardNode()) {
2691 Guard = getLoadStackGuard(DAG, dl, Chain);
2692 } else {
2693 const Value *IRGuard = TLI.getSDagStackGuard(M);
2694 SDValue GuardPtr = getValue(IRGuard);
2695
2696 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2697 MachinePointerInfo(IRGuard, 0), Align,
2698 MachineMemOperand::MOVolatile);
2699 }
2700
2701 // Perform the comparison via a getsetcc.
2702 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2703 *DAG.getContext(),
2704 Guard.getValueType()),
2705 Guard, GuardVal, ISD::SETNE);
2706
2707 // If the guard/stackslot do not equal, branch to failure MBB.
2708 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2709 MVT::Other, GuardVal.getOperand(0),
2710 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2711 // Otherwise branch to success MBB.
2712 SDValue Br = DAG.getNode(ISD::BR, dl,
2713 MVT::Other, BrCond,
2714 DAG.getBasicBlock(SPD.getSuccessMBB()));
2715
2716 DAG.setRoot(Br);
2717}
2718
2719/// Codegen the failure basic block for a stack protector check.
2720///
2721/// A failure stack protector machine basic block consists simply of a call to
2722/// __stack_chk_fail().
2723///
2724/// For a high level explanation of how this fits into the stack protector
2725/// generation see the comment on the declaration of class
2726/// StackProtectorDescriptor.
2727void
2728SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2729 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2730 TargetLowering::MakeLibCallOptions CallOptions;
2731 CallOptions.setDiscardResult(true);
2732 SDValue Chain =
2733 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2734 None, CallOptions, getCurSDLoc()).second;
2735 // On PS4, the "return address" must still be within the calling function,
2736 // even if it's at the very end, so emit an explicit TRAP here.
2737 // Passing 'true' for doesNotReturn above won't generate the trap for us.
2738 if (TM.getTargetTriple().isPS4CPU())
2739 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2740 // WebAssembly needs an unreachable instruction after a non-returning call,
2741 // because the function return type can be different from __stack_chk_fail's
2742 // return type (void).
2743 if (TM.getTargetTriple().isWasm())
2744 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2745
2746 DAG.setRoot(Chain);
2747}
2748
2749/// visitBitTestHeader - This function emits necessary code to produce value
2750/// suitable for "bit tests"
2751void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2752 MachineBasicBlock *SwitchBB) {
2753 SDLoc dl = getCurSDLoc();
2754
2755 // Subtract the minimum value.
2756 SDValue SwitchOp = getValue(B.SValue);
2757 EVT VT = SwitchOp.getValueType();
2758 SDValue RangeSub =
2759 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2760
2761 // Determine the type of the test operands.
2762 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2763 bool UsePtrType = false;
2764 if (!TLI.isTypeLegal(VT)) {
2765 UsePtrType = true;
2766 } else {
2767 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2768 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2769 // Switch table case range are encoded into series of masks.
2770 // Just use pointer type, it's guaranteed to fit.
2771 UsePtrType = true;
2772 break;
2773 }
2774 }
2775 SDValue Sub = RangeSub;
2776 if (UsePtrType) {
2777 VT = TLI.getPointerTy(DAG.getDataLayout());
2778 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2779 }
2780
2781 B.RegVT = VT.getSimpleVT();
2782 B.Reg = FuncInfo.CreateReg(B.RegVT);
2783 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2784
2785 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2786
2787 if (!B.OmitRangeCheck)
2788 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2789 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2790 SwitchBB->normalizeSuccProbs();
2791
2792 SDValue Root = CopyTo;
2793 if (!B.OmitRangeCheck) {
2794 // Conditional branch to the default block.
2795 SDValue RangeCmp = DAG.getSetCC(dl,
2796 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2797 RangeSub.getValueType()),
2798 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2799 ISD::SETUGT);
2800
2801 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2802 DAG.getBasicBlock(B.Default));
2803 }
2804
2805 // Avoid emitting unnecessary branches to the next block.
2806 if (MBB != NextBlock(SwitchBB))
2807 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2808
2809 DAG.setRoot(Root);
2810}
2811
2812/// visitBitTestCase - this function produces one "bit test"
2813void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2814 MachineBasicBlock* NextMBB,
2815 BranchProbability BranchProbToNext,
2816 unsigned Reg,
2817 BitTestCase &B,
2818 MachineBasicBlock *SwitchBB) {
2819 SDLoc dl = getCurSDLoc();
2820 MVT VT = BB.RegVT;
2821 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2822 SDValue Cmp;
2823 unsigned PopCount = countPopulation(B.Mask);
2824 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2825 if (PopCount == 1) {
2826 // Testing for a single bit; just compare the shift count with what it
2827 // would need to be to shift a 1 bit in that position.
2828 Cmp = DAG.getSetCC(
2829 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2830 ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2831 ISD::SETEQ);
2832 } else if (PopCount == BB.Range) {
2833 // There is only one zero bit in the range, test for it directly.
2834 Cmp = DAG.getSetCC(
2835 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2836 ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2837 ISD::SETNE);
2838 } else {
2839 // Make desired shift
2840 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2841 DAG.getConstant(1, dl, VT), ShiftOp);
2842
2843 // Emit bit tests and jumps
2844 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2845 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2846 Cmp = DAG.getSetCC(
2847 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2848 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2849 }
2850
2851 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2852 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2853 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2854 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2855 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2856 // one as they are relative probabilities (and thus work more like weights),
2857 // and hence we need to normalize them to let the sum of them become one.
2858 SwitchBB->normalizeSuccProbs();
2859
2860 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2861 MVT::Other, getControlRoot(),
2862 Cmp, DAG.getBasicBlock(B.TargetBB));
2863
2864 // Avoid emitting unnecessary branches to the next block.
2865 if (NextMBB != NextBlock(SwitchBB))
2866 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2867 DAG.getBasicBlock(NextMBB));
2868
2869 DAG.setRoot(BrAnd);
2870}
2871
2872void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2873 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2874
2875 // Retrieve successors. Look through artificial IR level blocks like
2876 // catchswitch for successors.
2877 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2878 const BasicBlock *EHPadBB = I.getSuccessor(1);
2879
2880 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2881 // have to do anything here to lower funclet bundles.
2882 assert(!I.hasOperandBundlesOtherThan((static_cast<void> (0))
2883 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,(static_cast<void> (0))
2884 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,(static_cast<void> (0))
2885 LLVMContext::OB_cfguardtarget,(static_cast<void> (0))
2886 LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast<void> (0))
2887 "Cannot lower invokes with arbitrary operand bundles yet!")(static_cast<void> (0));
2888
2889 const Value *Callee(I.getCalledOperand());
2890 const Function *Fn = dyn_cast<Function>(Callee);
2891 if (isa<InlineAsm>(Callee))
2892 visitInlineAsm(I, EHPadBB);
2893 else if (Fn && Fn->isIntrinsic()) {
2894 switch (Fn->getIntrinsicID()) {
2895 default:
2896 llvm_unreachable("Cannot invoke this intrinsic")__builtin_unreachable();
2897 case Intrinsic::donothing:
2898 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2899 case Intrinsic::seh_try_begin:
2900 case Intrinsic::seh_scope_begin:
2901 case Intrinsic::seh_try_end:
2902 case Intrinsic::seh_scope_end:
2903 break;
2904 case Intrinsic::experimental_patchpoint_void:
2905 case Intrinsic::experimental_patchpoint_i64:
2906 visitPatchpoint(I, EHPadBB);
2907 break;
2908 case Intrinsic::experimental_gc_statepoint:
2909 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
2910 break;
2911 case Intrinsic::wasm_rethrow: {
2912 // This is usually done in visitTargetIntrinsic, but this intrinsic is
2913 // special because it can be invoked, so we manually lower it to a DAG
2914 // node here.
2915 SmallVector<SDValue, 8> Ops;
2916 Ops.push_back(getRoot()); // inchain
2917 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2918 Ops.push_back(
2919 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
2920 TLI.getPointerTy(DAG.getDataLayout())));
2921 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2922 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
2923 break;
2924 }
2925 }
2926 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2927 // Currently we do not lower any intrinsic calls with deopt operand bundles.
2928 // Eventually we will support lowering the @llvm.experimental.deoptimize
2929 // intrinsic, and right now there are no plans to support other intrinsics
2930 // with deopt state.
2931 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2932 } else {
2933 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
2934 }
2935
2936 // If the value of the invoke is used outside of its defining block, make it
2937 // available as a virtual register.
2938 // We already took care of the exported value for the statepoint instruction
2939 // during call to the LowerStatepoint.
2940 if (!isa<GCStatepointInst>(I)) {
2941 CopyToExportRegsIfNeeded(&I);
2942 }
2943
2944 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2945 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2946 BranchProbability EHPadBBProb =
2947 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2948 : BranchProbability::getZero();
2949 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2950
2951 // Update successor info.
2952 addSuccessorWithProb(InvokeMBB, Return);
2953 for (auto &UnwindDest : UnwindDests) {
2954 UnwindDest.first->setIsEHPad();
2955 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2956 }
2957 InvokeMBB->normalizeSuccProbs();
2958
2959 // Drop into normal successor.
2960 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2961 DAG.getBasicBlock(Return)));
2962}
2963
2964void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2965 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2966
2967 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2968 // have to do anything here to lower funclet bundles.
2969 assert(!I.hasOperandBundlesOtherThan((static_cast<void> (0))
2970 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast<void> (0))
2971 "Cannot lower callbrs with arbitrary operand bundles yet!")(static_cast<void> (0));
2972
2973 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")(static_cast<void> (0));
2974 visitInlineAsm(I);
2975 CopyToExportRegsIfNeeded(&I);
2976
2977 // Retrieve successors.
2978 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2979
2980 // Update successor info.
2981 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
2982 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
2983 MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
2984 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
2985 Target->setIsInlineAsmBrIndirectTarget();
2986 }
2987 CallBrMBB->normalizeSuccProbs();
2988
2989 // Drop into default successor.
2990 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2991 MVT::Other, getControlRoot(),
2992 DAG.getBasicBlock(Return)));
2993}
2994
2995void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2996 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")__builtin_unreachable();
2997}
2998
2999void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3000 assert(FuncInfo.MBB->isEHPad() &&(static_cast<void> (0))
3001 "Call to landingpad not in landing pad!")(static_cast<void> (0));
3002
3003 // If there aren't registers to copy the values into (e.g., during SjLj
3004 // exceptions), then don't bother to create these DAG nodes.
3005 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3006 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3007 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3008 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3009 return;
3010
3011 // If landingpad's return type is token type, we don't create DAG nodes
3012 // for its exception pointer and selector value. The extraction of exception
3013 // pointer or selector value from token type landingpads is not currently
3014 // supported.
3015 if (LP.getType()->isTokenTy())
3016 return;
3017
3018 SmallVector<EVT, 2> ValueVTs;
3019 SDLoc dl = getCurSDLoc();
3020 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3021 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast<void> (0));
3022
3023 // Get the two live-in registers as SDValues. The physregs have already been
3024 // copied into virtual registers.
3025 SDValue Ops[2];
3026 if (FuncInfo.ExceptionPointerVirtReg) {
3027 Ops[0] = DAG.getZExtOrTrunc(
3028 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3029 FuncInfo.ExceptionPointerVirtReg,
3030 TLI.getPointerTy(DAG.getDataLayout())),
3031 dl, ValueVTs[0]);
3032 } else {
3033 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3034 }
3035 Ops[1] = DAG.getZExtOrTrunc(
3036 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3037 FuncInfo.ExceptionSelectorVirtReg,
3038 TLI.getPointerTy(DAG.getDataLayout())),
3039 dl, ValueVTs[1]);
3040
3041 // Merge into one.
3042 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3043 DAG.getVTList(ValueVTs), Ops);
3044 setValue(&LP, Res);
3045}
3046
3047void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3048 MachineBasicBlock *Last) {
3049 // Update JTCases.
3050 for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)
3051 if (SL->JTCases[i].first.HeaderBB == First)
3052 SL->JTCases[i].first.HeaderBB = Last;
3053
3054 // Update BitTestCases.
3055 for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
3056 if (SL->BitTestCases[i].Parent == First)
3057 SL->BitTestCases[i].Parent = Last;
3058}
3059
3060void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3061 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3062
3063 // Update machine-CFG edges with unique successors.
3064 SmallSet<BasicBlock*, 32> Done;
3065 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3066 BasicBlock *BB = I.getSuccessor(i);
3067 bool Inserted = Done.insert(BB).second;
3068 if (!Inserted)
3069 continue;
3070
3071 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3072 addSuccessorWithProb(IndirectBrMBB, Succ);
3073 }
3074 IndirectBrMBB->normalizeSuccProbs();
3075
3076 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
3077 MVT::Other, getControlRoot(),
3078 getValue(I.getAddress())));
3079}
3080
3081void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3082 if (!DAG.getTarget().Options.TrapUnreachable)
3083 return;
3084
3085 // We may be able to ignore unreachable behind a noreturn call.
3086 if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
3087 const BasicBlock &BB = *I.getParent();
3088 if (&I != &BB.front()) {
3089 BasicBlock::const_iterator PredI =
3090 std::prev(BasicBlock::const_iterator(&I));
3091 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3092 if (Call->doesNotReturn())
3093 return;
3094 }
3095 }
3096 }
3097
3098 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3099}
3100
3101void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3102 SDNodeFlags Flags;
3103
3104 SDValue Op = getValue(I.getOperand(0));
3105 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3106 Op, Flags);
3107 setValue(&I, UnNodeValue);
3108}
3109
3110void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3111 SDNodeFlags Flags;
3112 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3113 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3114 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3115 }
3116 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3117 Flags.setExact(ExactOp->isExact());
3118 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3119 Flags.copyFMF(*FPOp);
3120
3121 SDValue Op1 = getValue(I.getOperand(0));
3122 SDValue Op2 = getValue(I.getOperand(1));
3123 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3124 Op1, Op2, Flags);
3125 setValue(&I, BinNodeValue);
3126}
3127
3128void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3129 SDValue Op1 = getValue(I.getOperand(0));
3130 SDValue Op2 = getValue(I.getOperand(1));
3131
3132 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3133 Op1.getValueType(), DAG.getDataLayout());
3134
3135 // Coerce the shift amount to the right type if we can.
3136 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3137 unsigned ShiftSize = ShiftTy.getSizeInBits();
3138 unsigned Op2Size = Op2.getValueSizeInBits();
3139 SDLoc DL = getCurSDLoc();
3140
3141 // If the operand is smaller than the shift count type, promote it.
3142 if (ShiftSize > Op2Size)
3143 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
3144
3145 // If the operand is larger than the shift count type but the shift
3146 // count type has enough bits to represent any shift value, truncate
3147 // it now. This is a common case and it exposes the truncate to
3148 // optimization early.
3149 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
3150 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
3151 // Otherwise we'll need to temporarily settle for some other convenient
3152 // type. Type legalization will make adjustments once the shiftee is split.
3153 else
3154 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
3155 }
3156
3157 bool nuw = false;
3158 bool nsw = false;
3159 bool exact = false;
3160
3161 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3162
3163 if (const OverflowingBinaryOperator *OFBinOp =
3164 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3165 nuw = OFBinOp->hasNoUnsignedWrap();
3166 nsw = OFBinOp->hasNoSignedWrap();
3167 }
3168 if (const PossiblyExactOperator *ExactOp =
3169 dyn_cast<const PossiblyExactOperator>(&I))
3170 exact = ExactOp->isExact();
3171 }
3172 SDNodeFlags Flags;
3173 Flags.setExact(exact);
3174 Flags.setNoSignedWrap(nsw);
3175 Flags.setNoUnsignedWrap(nuw);
3176 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3177 Flags);
3178 setValue(&I, Res);
3179}
3180
3181void SelectionDAGBuilder::visitSDiv(const User &I) {
3182 SDValue Op1 = getValue(I.getOperand(0));
3183 SDValue Op2 = getValue(I.getOperand(1));
3184
3185 SDNodeFlags Flags;
3186 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3187 cast<PossiblyExactOperator>(&I)->isExact());
3188 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3189 Op2, Flags));
3190}
3191
3192void SelectionDAGBuilder::visitICmp(const User &I) {
3193 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
3194 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3195 predicate = IC->getPredicate();
3196 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3197 predicate = ICmpInst::Predicate(IC->getPredicate());
3198 SDValue Op1 = getValue(I.getOperand(0));
3199 SDValue Op2 = getValue(I.getOperand(1));
3200 ISD::CondCode Opcode = getICmpCondCode(predicate);
3201
3202 auto &TLI = DAG.getTargetLoweringInfo();
3203 EVT MemVT =
3204 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3205
3206 // If a pointer's DAG type is larger than its memory type then the DAG values
3207 // are zero-extended. This breaks signed comparisons so truncate back to the
3208 // underlying type before doing the compare.
3209 if (Op1.getValueType() != MemVT) {
3210 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3211 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3212 }
3213
3214 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3215 I.getType());
3216 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3217}
3218
3219void SelectionDAGBuilder::visitFCmp(const User &I) {
3220 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
3221 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3222 predicate = FC->getPredicate();
3223 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3224 predicate = FCmpInst::Predicate(FC->getPredicate());
3225 SDValue Op1 = getValue(I.getOperand(0));
3226 SDValue Op2 = getValue(I.getOperand(1));
3227
3228 ISD::CondCode Condition = getFCmpCondCode(predicate);
3229 auto *FPMO = cast<FPMathOperator>(&I);
3230 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3231 Condition = getFCmpCodeWithoutNaN(Condition);
3232
3233 SDNodeFlags Flags;
3234 Flags.copyFMF(*FPMO);
3235 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3236
3237 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3238 I.getType());
3239 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3240}
3241
3242// Check if the condition of the select has one use or two users that are both
3243// selects with the same condition.
3244static bool hasOnlySelectUsers(const Value *Cond) {
3245 return llvm::all_of(Cond->users(), [](const Value *V) {
3246 return isa<SelectInst>(V);
3247 });
3248}
3249
3250void SelectionDAGBuilder::visitSelect(const User &I) {
3251 SmallVector<EVT, 4> ValueVTs;
3252 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3253 ValueVTs);
3254 unsigned NumValues = ValueVTs.size();
3255 if (NumValues == 0) return;
3256
3257 SmallVector<SDValue, 4> Values(NumValues);
3258 SDValue Cond = getValue(I.getOperand(0));
3259 SDValue LHSVal = getValue(I.getOperand(1));
3260 SDValue RHSVal = getValue(I.getOperand(2));
3261 SmallVector<SDValue, 1> BaseOps(1, Cond);
3262 ISD::NodeType OpCode =
3263 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3264
3265 bool IsUnaryAbs = false;
3266 bool Negate = false;
3267
3268 SDNodeFlags Flags;
3269 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3270 Flags.copyFMF(*FPOp);
3271
3272 // Min/max matching is only viable if all output VTs are the same.
3273 if (is_splat(ValueVTs)) {
3274 EVT VT = ValueVTs[0];
3275 LLVMContext &Ctx = *DAG.getContext();
3276 auto &TLI = DAG.getTargetLoweringInfo();
3277
3278 // We care about the legality of the operation after it has been type
3279 // legalized.
3280 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3281 VT = TLI.getTypeToTransformTo(Ctx, VT);
3282
3283 // If the vselect is legal, assume we want to leave this as a vector setcc +
3284 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3285 // min/max is legal on the scalar type.
3286 bool UseScalarMinMax = VT.isVector() &&
3287 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3288
3289 Value *LHS, *RHS;
3290 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3291 ISD::NodeType Opc = ISD::DELETED_NODE;
3292 switch (SPR.Flavor) {
3293 case SPF_UMAX: Opc = ISD::UMAX; break;
3294 case SPF_UMIN: Opc = ISD::UMIN; break;
3295 case SPF_SMAX: Opc = ISD::SMAX; break;
3296 case SPF_SMIN: Opc = ISD::SMIN; break;
3297 case SPF_FMINNUM:
3298 switch (SPR.NaNBehavior) {
3299 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")__builtin_unreachable();
3300 case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;
3301 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3302 case SPNB_RETURNS_ANY: {
3303 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3304 Opc = ISD::FMINNUM;
3305 else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3306 Opc = ISD::FMINIMUM;
3307 else if (UseScalarMinMax)
3308 Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3309 ISD::FMINNUM : ISD::FMINIMUM;
3310 break;
3311 }
3312 }
3313 break;
3314 case SPF_FMAXNUM:
3315 switch (SPR.NaNBehavior) {
3316 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")__builtin_unreachable();
3317 case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;
3318 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3319 case SPNB_RETURNS_ANY:
3320
3321 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3322 Opc = ISD::FMAXNUM;
3323 else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3324 Opc = ISD::FMAXIMUM;
3325 else if (UseScalarMinMax)
3326 Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3327 ISD::FMAXNUM : ISD::FMAXIMUM;
3328 break;
3329 }
3330 break;
3331 case SPF_NABS:
3332 Negate = true;
3333 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3334 case SPF_ABS:
3335 IsUnaryAbs = true;
3336 Opc = ISD::ABS;
3337 break;
3338 default: break;
3339 }
3340
3341 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3342 (TLI.isOperationLegalOrCustom(Opc, VT) ||
3343 (UseScalarMinMax &&
3344 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3345 // If the underlying comparison instruction is used by any other
3346 // instruction, the consumed instructions won't be destroyed, so it is
3347 // not profitable to convert to a min/max.
3348 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3349 OpCode = Opc;
3350 LHSVal = getValue(LHS);
3351 RHSVal = getValue(RHS);
3352 BaseOps.clear();
3353 }
3354
3355 if (IsUnaryAbs) {
3356 OpCode = Opc;
3357 LHSVal = getValue(LHS);
3358 BaseOps.clear();
3359 }
3360 }
3361
3362 if (IsUnaryAbs) {
3363 for (unsigned i = 0; i != NumValues; ++i) {
3364 SDLoc dl = getCurSDLoc();
3365 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3366 Values[i] =
3367 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3368 if (Negate)
3369 Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
3370 Values[i]);
3371 }
3372 } else {
3373 for (unsigned i = 0; i != NumValues; ++i) {
3374 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3375 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3376 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3377 Values[i] = DAG.getNode(
3378 OpCode, getCurSDLoc(),
3379 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3380 }
3381 }
3382
3383 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3384 DAG.getVTList(ValueVTs), Values));
3385}
3386
3387void SelectionDAGBuilder::visitTrunc(const User &I) {
3388 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3389 SDValue N = getValue(I.getOperand(0));
3390 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3391 I.getType());
3392 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3393}
3394
3395void SelectionDAGBuilder::visitZExt(const User &I) {
3396 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3397 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3398 SDValue N = getValue(I.getOperand(0));
3399 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3400 I.getType());
3401 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3402}
3403
3404void SelectionDAGBuilder::visitSExt(const User &I) {
3405 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3406 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3407 SDValue N = getValue(I.getOperand(0));
3408 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3409 I.getType());
3410 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3411}
3412
3413void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3414 // FPTrunc is never a no-op cast, no need to check
3415 SDValue N = getValue(I.getOperand(0));
3416 SDLoc dl = getCurSDLoc();
3417 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3418 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3419 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3420 DAG.getTargetConstant(
3421 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3422}
3423
3424void SelectionDAGBuilder::visitFPExt(const User &I) {
3425 // FPExt is never a no-op cast, no need to check
3426 SDValue N = getValue(I.getOperand(0));
3427 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3428 I.getType());
3429 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3430}
3431
3432void SelectionDAGBuilder::visitFPToUI(const User &I) {
3433 // FPToUI is never a no-op cast, no need to check
3434 SDValue N = getValue(I.getOperand(0));
3435 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3436 I.getType());
3437 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3438}
3439
3440void SelectionDAGBuilder::visitFPToSI(const User &I) {
3441 // FPToSI is never a no-op cast, no need to check
3442 SDValue N = getValue(I.getOperand(0));
3443 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3444 I.getType());
3445 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3446}
3447
3448void SelectionDAGBuilder::visitUIToFP(const User &I) {
3449 // UIToFP is never a no-op cast, no need to check
3450 SDValue N = getValue(I.getOperand(0));
3451 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3452 I.getType());
3453 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3454}
3455
3456void SelectionDAGBuilder::visitSIToFP(const User &I) {
3457 // SIToFP is never a no-op cast, no need to check
3458 SDValue N = getValue(I.getOperand(0));
3459 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3460 I.getType());
3461 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3462}
3463
3464void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3465 // What to do depends on the size of the integer and the size of the pointer.
3466 // We can either truncate, zero extend, or no-op, accordingly.
3467 SDValue N = getValue(I.getOperand(0));
3468 auto &TLI = DAG.getTargetLoweringInfo();
3469 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3470 I.getType());
3471 EVT PtrMemVT =
3472 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3473 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3474 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3475 setValue(&I, N);
3476}
3477
3478void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3479 // What to do depends on the size of the integer and the size of the pointer.
3480 // We can either truncate, zero extend, or no-op, accordingly.
3481 SDValue N = getValue(I.getOperand(0));
3482 auto &TLI = DAG.getTargetLoweringInfo();
3483 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3484 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3485 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3486 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3487 setValue(&I, N);
3488}
3489
3490void SelectionDAGBuilder::visitBitCast(const User &I) {
3491 SDValue N = getValue(I.getOperand(0));
3492 SDLoc dl = getCurSDLoc();
3493 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3494 I.getType());
3495
3496 // BitCast assures us that source and destination are the same size so this is
3497 // either a BITCAST or a no-op.
3498 if (DestVT != N.getValueType())
3499 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3500 DestVT, N)); // convert types.
3501 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3502 // might fold any kind of constant expression to an integer constant and that
3503 // is not what we are looking for. Only recognize a bitcast of a genuine
3504 // constant integer as an opaque constant.
3505 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3506 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3507 /*isOpaque*/true));
3508 else
3509 setValue(&I, N); // noop cast.
3510}
3511
3512void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3513 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3514 const Value *SV = I.getOperand(0);
3515 SDValue N = getValue(SV);
3516 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3517
3518 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3519 unsigned DestAS = I.getType()->getPointerAddressSpace();
3520
3521 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3522 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3523
3524 setValue(&I, N);
3525}
3526
3527void SelectionDAGBuilder::visitInsertElement(const User &I) {
3528 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3529 SDValue InVec = getValue(I.getOperand(0));
3530 SDValue InVal = getValue(I.getOperand(1));
3531 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3532 TLI.getVectorIdxTy(DAG.getDataLayout()));
3533 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3534 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3535 InVec, InVal, InIdx));
3536}
3537
3538void SelectionDAGBuilder::visitExtractElement(const User &I) {
3539 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3540 SDValue InVec = getValue(I.getOperand(0));
3541 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3542 TLI.getVectorIdxTy(DAG.getDataLayout()));
3543 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3544 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3545 InVec, InIdx));
3546}
3547
3548void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3549 SDValue Src1 = getValue(I.getOperand(0));
3550 SDValue Src2 = getValue(I.getOperand(1));
3551 ArrayRef<int> Mask;
3552 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3553 Mask = SVI->getShuffleMask();
3554 else
3555 Mask = cast<ConstantExpr>(I).getShuffleMask();
3556 SDLoc DL = getCurSDLoc();
3557 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3558 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3559 EVT SrcVT = Src1.getValueType();
3560
3561 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3562 VT.isScalableVector()) {
3563 // Canonical splat form of first element of first input vector.
3564 SDValue FirstElt =
3565 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
3566 DAG.getVectorIdxConstant(0, DL));
3567 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3568 return;
3569 }
3570
3571 // For now, we only handle splats for scalable vectors.
3572 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3573 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3574 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")(static_cast<void> (0));
3575
3576 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3577 unsigned MaskNumElts = Mask.size();
3578
3579 if (SrcNumElts == MaskNumElts) {
3580 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3581 return;
3582 }
3583
3584 // Normalize the shuffle vector since mask and vector length don't match.
3585 if (SrcNumElts < MaskNumElts) {
3586 // Mask is longer than the source vectors. We can use concatenate vector to
3587 // make the mask and vectors lengths match.
3588
3589 if (MaskNumElts % SrcNumElts == 0) {
3590 // Mask length is a multiple of the source vector length.
3591 // Check if the shuffle is some kind of concatenation of the input
3592 // vectors.
3593 unsigned NumConcat = MaskNumElts / SrcNumElts;
3594 bool IsConcat = true;
3595 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3596 for (unsigned i = 0; i != MaskNumElts; ++i) {
3597 int Idx = Mask[i];
3598 if (Idx < 0)
3599 continue;
3600 // Ensure the indices in each SrcVT sized piece are sequential and that
3601 // the same source is used for the whole piece.
3602 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3603 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3604 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3605 IsConcat = false;
3606 break;
3607 }
3608 // Remember which source this index came from.
3609 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3610 }
3611
3612 // The shuffle is concatenating multiple vectors together. Just emit
3613 // a CONCAT_VECTORS operation.
3614 if (IsConcat) {
3615 SmallVector<SDValue, 8> ConcatOps;
3616 for (auto Src : ConcatSrcs) {
3617 if (Src < 0)
3618 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3619 else if (Src == 0)
3620 ConcatOps.push_back(Src1);
3621 else
3622 ConcatOps.push_back(Src2);
3623 }
3624 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3625 return;
3626 }
3627 }
3628
3629 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3630 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3631 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3632 PaddedMaskNumElts);
3633
3634 // Pad both vectors with undefs to make them the same length as the mask.
3635 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3636
3637 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3638 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3639 MOps1[0] = Src1;
3640 MOps2[0] = Src2;
3641
3642 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3643 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3644
3645 // Readjust mask for new input vector length.
3646 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3647 for (unsigned i = 0; i != MaskNumElts; ++i) {
3648 int Idx = Mask[i];
3649 if (Idx >= (int)SrcNumElts)
3650 Idx -= SrcNumElts - PaddedMaskNumElts;
3651 MappedOps[i] = Idx;
3652 }
3653
3654 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3655
3656 // If the concatenated vector was padded, extract a subvector with the
3657 // correct number of elements.
3658 if (MaskNumElts != PaddedMaskNumElts)
3659 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3660 DAG.getVectorIdxConstant(0, DL));
3661
3662 setValue(&I, Result);
3663 return;
3664 }
3665
3666 if (SrcNumElts > MaskNumElts) {
3667 // Analyze the access pattern of the vector to see if we can extract
3668 // two subvectors and do the shuffle.
3669 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3670 bool CanExtract = true;
3671 for (int Idx : Mask) {
3672 unsigned Input = 0;
3673 if (Idx < 0)
3674 continue;
3675
3676 if (Idx >= (int)SrcNumElts) {
3677 Input = 1;
3678 Idx -= SrcNumElts;
3679 }
3680
3681 // If all the indices come from the same MaskNumElts sized portion of
3682 // the sources we can use extract. Also make sure the extract wouldn't
3683 // extract past the end of the source.
3684 int NewStartIdx = alignDown(Idx, MaskNumElts);
3685 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3686 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3687 CanExtract = false;
3688 // Make sure we always update StartIdx as we use it to track if all
3689 // elements are undef.
3690 StartIdx[Input] = NewStartIdx;
3691 }
3692
3693 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3694 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3695 return;
3696 }
3697 if (CanExtract) {
3698 // Extract appropriate subvector and generate a vector shuffle
3699 for (unsigned Input = 0; Input < 2; ++Input) {
3700 SDValue &Src = Input == 0 ? Src1 : Src2;
3701 if (StartIdx[Input] < 0)
3702 Src = DAG.getUNDEF(VT);
3703 else {
3704 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3705 DAG.getVectorIdxConstant(StartIdx[Input], DL));
3706 }
3707 }
3708
3709 // Calculate new mask.
3710 SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3711 for (int &Idx : MappedOps) {
3712 if (Idx >= (int)SrcNumElts)
3713 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3714 else if (Idx >= 0)
3715 Idx -= StartIdx[0];
3716 }
3717
3718 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3719 return;
3720 }
3721 }
3722
3723 // We can't use either concat vectors or extract subvectors so fall back to
3724 // replacing the shuffle with extract and build vector.
3725 // to insert and build vector.
3726 EVT EltVT = VT.getVectorElementType();
3727 SmallVector<SDValue,8> Ops;
3728 for (int Idx : Mask) {
3729 SDValue Res;
3730
3731 if (Idx < 0) {
3732 Res = DAG.getUNDEF(EltVT);
3733 } else {
3734 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3735 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3736
3737 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3738 DAG.getVectorIdxConstant(Idx, DL));
3739 }
3740
3741 Ops.push_back(Res);
3742 }
3743
3744 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3745}
3746
3747void SelectionDAGBuilder::visitInsertValue(const User &I) {
3748 ArrayRef<unsigned> Indices;
3749 if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3750 Indices = IV->getIndices();
3751 else
3752 Indices = cast<ConstantExpr>(&I)->getIndices();
3753
3754 const Value *Op0 = I.getOperand(0);
3755 const Value *Op1 = I.getOperand(1);
3756 Type *AggTy = I.getType();
3757 Type *ValTy = Op1->getType();
3758 bool IntoUndef = isa<UndefValue>(Op0);
3759 bool FromUndef = isa<UndefValue>(Op1);
3760
3761 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3762
3763 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3764 SmallVector<EVT, 4> AggValueVTs;
3765 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3766 SmallVector<EVT, 4> ValValueVTs;
3767 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3768
3769 unsigned NumAggValues = AggValueVTs.size();
3770 unsigned NumValValues = ValValueVTs.size();
3771 SmallVector<SDValue, 4> Values(NumAggValues);
3772
3773 // Ignore an insertvalue that produces an empty object
3774 if (!NumAggValues) {
3775 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3776 return;
3777 }
3778
3779 SDValue Agg = getValue(Op0);
3780 unsigned i = 0;
3781 // Copy the beginning value(s) from the original aggregate.
3782 for (; i != LinearIndex; ++i)
3783 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3784 SDValue(Agg.getNode(), Agg.getResNo() + i);
3785 // Copy values from the inserted value(s).
3786 if (NumValValues) {
3787 SDValue Val = getValue(Op1);
3788 for (; i != LinearIndex + NumValValues; ++i)
3789 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3790 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3791 }
3792 // Copy remaining value(s) from the original aggregate.
3793 for (; i != NumAggValues; ++i)
3794 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3795 SDValue(Agg.getNode(), Agg.getResNo() + i);
3796
3797 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3798 DAG.getVTList(AggValueVTs), Values));
3799}
3800
3801void SelectionDAGBuilder::visitExtractValue(const User &I) {
3802 ArrayRef<unsigned> Indices;
3803 if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3804 Indices = EV->getIndices();
3805 else
3806 Indices = cast<ConstantExpr>(&I)->getIndices();
3807
3808 const Value *Op0 = I.getOperand(0);
3809 Type *AggTy = Op0->getType();
3810 Type *ValTy = I.getType();
3811 bool OutOfUndef = isa<UndefValue>(Op0);
3812
3813 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3814
3815 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3816 SmallVector<EVT, 4> ValValueVTs;
3817 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3818
3819 unsigned NumValValues = ValValueVTs.size();
3820
3821 // Ignore a extractvalue that produces an empty object
3822 if (!NumValValues) {
3823 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3824 return;
3825 }
3826
3827 SmallVector<SDValue, 4> Values(NumValValues);
3828
3829 SDValue Agg = getValue(Op0);
3830 // Copy out the selected value(s).
3831 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3832 Values[i - LinearIndex] =
3833 OutOfUndef ?
3834 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3835 SDValue(Agg.getNode(), Agg.getResNo() + i);
3836
3837 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3838 DAG.getVTList(ValValueVTs), Values));
3839}
3840
3841void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3842 Value *Op0 = I.getOperand(0);
3843 // Note that the pointer operand may be a vector of pointers. Take the scalar
3844 // element which holds a pointer.
3845 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3846 SDValue N = getValue(Op0);
3847 SDLoc dl = getCurSDLoc();
3848 auto &TLI = DAG.getTargetLoweringInfo();
3849
3850 // Normalize Vector GEP - all scalar operands should be converted to the
3851 // splat vector.
3852 bool IsVectorGEP = I.getType()->isVectorTy();
3853 ElementCount VectorElementCount =
3854 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3855 : ElementCount::getFixed(0);
3856
3857 if (IsVectorGEP && !N.getValueType().isVector()) {
3858 LLVMContext &Context = *DAG.getContext();
3859 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3860 if (VectorElementCount.isScalable())
3861 N = DAG.getSplatVector(VT, dl, N);
3862 else
3863 N = DAG.getSplatBuildVector(VT, dl, N);
3864 }
3865
3866 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3867 GTI != E; ++GTI) {
3868 const Value *Idx = GTI.getOperand();
3869 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3870 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3871 if (Field) {
3872 // N = N + Offset
3873 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3874
3875 // In an inbounds GEP with an offset that is nonnegative even when
3876 // interpreted as signed, assume there is no unsigned overflow.
3877 SDNodeFlags Flags;
3878 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3879 Flags.setNoUnsignedWrap(true);
3880
3881 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3882 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3883 }
3884 } else {
3885 // IdxSize is the width of the arithmetic according to IR semantics.
3886 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3887 // (and fix up the result later).
3888 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3889 MVT IdxTy = MVT::getIntegerVT(IdxSize);
3890 TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
3891 // We intentionally mask away the high bits here; ElementSize may not
3892 // fit in IdxTy.
3893 APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
3894 bool ElementScalable = ElementSize.isScalable();
3895
3896 // If this is a scalar constant or a splat vector of constants,
3897 // handle it quickly.
3898 const auto *C = dyn_cast<Constant>(Idx);
3899 if (C && isa<VectorType>(C->getType()))
3900 C = C->getSplatValue();
3901
3902 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3903 if (CI && CI->isZero())
3904 continue;
3905 if (CI && !ElementScalable) {
3906 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3907 LLVMContext &Context = *DAG.getContext();
3908 SDValue OffsVal;
3909 if (IsVectorGEP)
3910 OffsVal = DAG.getConstant(
3911 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3912 else
3913 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3914
3915 // In an inbounds GEP with an offset that is nonnegative even when
3916 // interpreted as signed, assume there is no unsigned overflow.
3917 SDNodeFlags Flags;
3918 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3919 Flags.setNoUnsignedWrap(true);
3920
3921 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3922
3923 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3924 continue;
3925 }
3926
3927 // N = N + Idx * ElementMul;
3928 SDValue IdxN = getValue(Idx);
3929
3930 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
3931 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
3932 VectorElementCount);
3933 if (VectorElementCount.isScalable())
3934 IdxN = DAG.getSplatVector(VT, dl, IdxN);
3935 else
3936 IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3937 }
3938
3939 // If the index is smaller or larger than intptr_t, truncate or extend
3940 // it.
3941 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3942
3943 if (ElementScalable) {
3944 EVT VScaleTy = N.getValueType().getScalarType();
3945 SDValue VScale = DAG.getNode(
3946 ISD::VSCALE, dl, VScaleTy,
3947 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
3948 if (IsVectorGEP)
3949 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
3950 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
3951 } else {
3952 // If this is a multiply by a power of two, turn it into a shl
3953 // immediately. This is a very common case.
3954 if (ElementMul != 1) {
3955 if (ElementMul.isPowerOf2()) {
3956 unsigned Amt = ElementMul.logBase2();
3957 IdxN = DAG.getNode(ISD::SHL, dl,
3958 N.getValueType(), IdxN,
3959 DAG.getConstant(Amt, dl, IdxN.getValueType()));
3960 } else {
3961 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
3962 IdxN.getValueType());
3963 IdxN = DAG.getNode(ISD::MUL, dl,
3964 N.getValueType(), IdxN, Scale);
3965 }
3966 }
3967 }
3968
3969 N = DAG.getNode(ISD::ADD, dl,
3970 N.getValueType(), N, IdxN);
3971 }
3972 }
3973
3974 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3975 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3976 if (IsVectorGEP) {
3977 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
3978 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
3979 }
3980
3981 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3982 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3983
3984 setValue(&I, N);
3985}
3986
3987void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3988 // If this is a fixed sized alloca in the entry block of the function,
3989 // allocate it statically on the stack.
3990 if (FuncInfo.StaticAllocaMap.count(&I))
3991 return; // getValue will auto-populate this.
3992
3993 SDLoc dl = getCurSDLoc();
3994 Type *Ty = I.getAllocatedType();
3995 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3996 auto &DL = DAG.getDataLayout();
3997 uint64_t TySize = DL.getTypeAllocSize(Ty);
3998 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
3999
4000 SDValue AllocSize = getValue(I.getArraySize());
4001
4002 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
4003 if (AllocSize.getValueType() != IntPtr)
4004 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4005
4006 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
4007 AllocSize,
4008 DAG.getConstant(TySize, dl, IntPtr));
4009
4010 // Handle alignment. If the requested alignment is less than or equal to
4011 // the stack alignment, ignore it. If the size is greater than or equal to
4012 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4013 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4014 if (*Alignment <= StackAlign)
4015 Alignment = None;
4016
4017 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4018 // Round the size of the allocation up to the stack alignment size
4019 // by add SA-1 to the size. This doesn't overflow because we're computing
4020 // an address inside an alloca.
4021 SDNodeFlags Flags;
4022 Flags.setNoUnsignedWrap(true);
4023 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4024 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4025
4026 // Mask out the low bits for alignment purposes.
4027 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4028 DAG.getConstant(~StackAlignMask, dl, IntPtr));
4029
4030 SDValue Ops[] = {
4031 getRoot(), AllocSize,
4032 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4033 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4034 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4035 setValue(&I, DSA);
4036 DAG.setRoot(DSA.getValue(1));
4037
4038 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast<void> (0));
4039}
4040
4041void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4042 if (I.isAtomic())
4043 return visitAtomicLoad(I);
4044
4045 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4046 const Value *SV = I.getOperand(0);
4047 if (TLI.supportSwiftError()) {
4048 // Swifterror values can come from either a function parameter with
4049 // swifterror attribute or an alloca with swifterror attribute.
4050 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4051 if (Arg->hasSwiftErrorAttr())
4052 return visitLoadFromSwiftError(I);
4053 }
4054
4055 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4056 if (Alloca->isSwiftError())
4057 return visitLoadFromSwiftError(I);
4058 }
4059 }
4060
4061 SDValue Ptr = getValue(SV);
4062
4063 Type *Ty = I.getType();
4064 Align Alignment = I.getAlign();
4065
4066 AAMDNodes AAInfo;
4067 I.getAAMetadata(AAInfo);
4068 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4069
4070 SmallVector<EVT, 4> ValueVTs, MemVTs;
4071 SmallVector<uint64_t, 4> Offsets;
4072 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4073 unsigned NumValues = ValueVTs.size();
4074 if (NumValues == 0)
4075 return;
4076
4077 bool isVolatile = I.isVolatile();
4078
4079 SDValue Root;
4080 bool ConstantMemory = false;
4081 if (isVolatile)
4082 // Serialize volatile loads with other side effects.
4083 Root = getRoot();
4084 else if (NumValues > MaxParallelChains)
4085 Root = getMemoryRoot();
4086 else if (AA &&
4087 AA->pointsToConstantMemory(MemoryLocation(
4088 SV,
4089 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4090 AAInfo))) {
4091 // Do not serialize (non-volatile) loads of constant memory with anything.
4092 Root = DAG.getEntryNode();
4093 ConstantMemory = true;
4094 } else {
4095 // Do not serialize non-volatile loads against each other.
4096 Root = DAG.getRoot();
4097 }
4098
4099 SDLoc dl = getCurSDLoc();
4100
4101 if (isVolatile)
4102 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4103
4104 // An aggregate load cannot wrap around the address space, so offsets to its
4105 // parts don't wrap either.
4106 SDNodeFlags Flags;
4107 Flags.setNoUnsignedWrap(true);
4108
4109 SmallVector<SDValue, 4> Values(NumValues);
4110 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4111 EVT PtrVT = Ptr.getValueType();
4112
4113 MachineMemOperand::Flags MMOFlags
4114 = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4115
4116 unsigned ChainI = 0;
4117 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4118 // Serializing loads here may result in excessive register pressure, and
4119 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4120 // could recover a bit by hoisting nodes upward in the chain by recognizing
4121 // they are side-effect free or do not alias. The optimizer should really
4122 // avoid this case by converting large object/array copies to llvm.memcpy
4123 // (MaxParallelChains should always remain as failsafe).
4124 if (ChainI == MaxParallelChains) {
4125 assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast<void> (0));
4126 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4127 makeArrayRef(Chains.data(), ChainI));
4128 Root = Chain;
4129 ChainI = 0;
4130 }
4131 SDValue A = DAG.getNode(ISD::ADD, dl,
4132 PtrVT, Ptr,
4133 DAG.getConstant(Offsets[i], dl, PtrVT),
4134 Flags);
4135
4136 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4137 MachinePointerInfo(SV, Offsets[i]), Alignment,
4138 MMOFlags, AAInfo, Ranges);
4139 Chains[ChainI] = L.getValue(1);
4140
4141 if (MemVTs[i] != ValueVTs[i])
4142 L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4143
4144 Values[i] = L;
4145 }
4146
4147 if (!ConstantMemory) {
4148 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4149 makeArrayRef(Chains.data(), ChainI));
4150 if (isVolatile)
4151 DAG.setRoot(Chain);
4152 else
4153 PendingLoads.push_back(Chain);
4154 }
4155
4156 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4157 DAG.getVTList(ValueVTs), Values));
4158}
4159
4160void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4161 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast<void> (0))
4162 "call visitStoreToSwiftError when backend supports swifterror")(static_cast<void> (0));
4163
4164 SmallVector<EVT, 4> ValueVTs;
4165 SmallVector<uint64_t, 4> Offsets;
4166 const Value *SrcV = I.getOperand(0);
4167 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4168 SrcV->getType(), ValueVTs, &Offsets);
4169 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast<void> (0))
4170 "expect a single EVT for swifterror")(static_cast<void> (0));
4171
4172 SDValue Src = getValue(SrcV);
4173 // Create a virtual register, then update the virtual register.
4174 Register VReg =
4175 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4176 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4177 // Chain can be getRoot or getControlRoot.
4178 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4179 SDValue(Src.getNode(), Src.getResNo()));
4180 DAG.setRoot(CopyNode);
4181}
4182
4183void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4184 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast<void> (0))
4185 "call visitLoadFromSwiftError when backend supports swifterror")(static_cast<void> (0));
4186
4187 assert(!I.isVolatile() &&(static_cast<void> (0))
4188 !I.hasMetadata(LLVMContext::MD_nontemporal) &&(static_cast<void> (0))
4189 !I.hasMetadata(LLVMContext::MD_invariant_load) &&(static_cast<void> (0))
4190 "Support volatile, non temporal, invariant for load_from_swift_error")(static_cast<void> (0));
4191
4192 const Value *SV = I.getOperand(0);
4193 Type *Ty = I.getType();
4194 AAMDNodes AAInfo;
4195 I.getAAMetadata(AAInfo);
4196 assert((static_cast<void> (0))
4197 (!AA ||(static_cast<void> (0))
4198 !AA->pointsToConstantMemory(MemoryLocation((static_cast<void> (0))
4199 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(static_cast<void> (0))
4200 AAInfo))) &&(static_cast<void> (0))
4201 "load_from_swift_error should not be constant memory")(static_cast<void> (0));
4202
4203 SmallVector<EVT, 4> ValueVTs;
4204 SmallVector<uint64_t, 4> Offsets;
4205 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
4206 ValueVTs, &Offsets);
4207 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast<void> (0))
4208 "expect a single EVT for swifterror")(static_cast<void> (0));
4209
4210 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4211 SDValue L = DAG.getCopyFromReg(
4212 getRoot(), getCurSDLoc(),
4213 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4214
4215 setValue(&I, L);
4216}
4217
4218void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4219 if (I.isAtomic())
4220 return visitAtomicStore(I);
4221
4222 const Value *SrcV = I.getOperand(0);
4223 const Value *PtrV = I.getOperand(1);
4224
4225 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4226 if (TLI.supportSwiftError()) {
4227 // Swifterror values can come from either a function parameter with
4228 // swifterror attribute or an alloca with swifterror attribute.
4229 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4230 if (Arg->hasSwiftErrorAttr())
4231 return visitStoreToSwiftError(I);
4232 }
4233
4234 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4235 if (Alloca->isSwiftError())
4236 return visitStoreToSwiftError(I);
4237 }
4238 }
4239
4240 SmallVector<EVT, 4> ValueVTs, MemVTs;
4241 SmallVector<uint64_t, 4> Offsets;
4242 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4243 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4244 unsigned NumValues = ValueVTs.size();
4245 if (NumValues == 0)
4246 return;
4247
4248 // Get the lowered operands. Note that we do this after
4249 // checking if NumResults is zero, because with zero results
4250 // the operands won't have values in the map.
4251 SDValue Src = getValue(SrcV);
4252 SDValue Ptr = getValue(PtrV);
4253
4254 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4255 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4256 SDLoc dl = getCurSDLoc();
4257 Align Alignment = I.getAlign();
4258 AAMDNodes AAInfo;
4259 I.getAAMetadata(AAInfo);
4260
4261 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4262
4263 // An aggregate load cannot wrap around the address space, so offsets to its
4264 // parts don't wrap either.
4265 SDNodeFlags Flags;
4266 Flags.setNoUnsignedWrap(true);
4267
4268 unsigned ChainI = 0;
4269 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4270 // See visitLoad comments.
4271 if (ChainI == MaxParallelChains) {
4272 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4273 makeArrayRef(Chains.data(), ChainI));
4274 Root = Chain;
4275 ChainI = 0;
4276 }
4277 SDValue Add =
4278 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);
4279 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4280 if (MemVTs[i] != ValueVTs[i])
4281 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4282 SDValue St =
4283 DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4284 Alignment, MMOFlags, AAInfo);
4285 Chains[ChainI] = St;
4286 }
4287
4288 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4289 makeArrayRef(Chains.data(), ChainI));
4290 DAG.setRoot(StoreNode);
4291}
4292
4293void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4294 bool IsCompressing) {
4295 SDLoc sdl = getCurSDLoc();
4296
4297 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4298 MaybeAlign &Alignment) {
4299 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4300 Src0 = I.getArgOperand(0);
4301 Ptr = I.getArgOperand(1);
4302 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
4303 Mask = I.getArgOperand(3);
4304 };
4305 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4306 MaybeAlign &Alignment) {
4307 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4308 Src0 = I.getArgOperand(0);
4309 Ptr = I.getArgOperand(1);
4310 Mask = I.getArgOperand(2);
4311 Alignment = None;
4312 };
4313
4314 Value *PtrOperand, *MaskOperand, *Src0Operand;
4315 MaybeAlign Alignment;
4316 if (IsCompressing)
4317 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4318 else
4319 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4320
4321 SDValue Ptr = getValue(PtrOperand);
4322 SDValue Src0 = getValue(Src0Operand);
4323 SDValue Mask = getValue(MaskOperand);
4324 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4325
4326 EVT VT = Src0.getValueType();
4327 if (!Alignment)
4328 Alignment = DAG.getEVTAlign(VT);
4329
4330 AAMDNodes AAInfo;
4331 I.getAAMetadata(AAInfo);
4332
4333 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4334 MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
4335 // TODO: Make MachineMemOperands aware of scalable
4336 // vectors.
4337 VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);
4338 SDValue StoreNode =
4339 DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4340 ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4341 DAG.setRoot(StoreNode);
4342 setValue(&I, StoreNode);
4343}
4344
4345// Get a uniform base for the Gather/Scatter intrinsic.
4346// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4347// We try to represent it as a base pointer + vector of indices.
4348// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4349// The first operand of the GEP may be a single pointer or a vector of pointers
4350// Example:
4351// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4352// or
4353// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4354// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4355//
4356// When the first GEP operand is a single pointer - it is the uniform base we
4357// are looking for. If first operand of the GEP is a splat vector - we
4358// extract the splat value and use it as a uniform base.
4359// In all other cases the function returns 'false'.
4360static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4361 ISD::MemIndexType &IndexType, SDValue &Scale,
4362 SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {
4363 SelectionDAG& DAG = SDB->DAG;
4364 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4365 const DataLayout &DL = DAG.getDataLayout();
4366
4367 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type")(static_cast<void> (0));
4368
4369 // Handle splat constant pointer.
4370 if (auto *C = dyn_cast<Constant>(Ptr)) {
4371 C = C->getSplatValue();
4372 if (!C)
4373 return false;
4374
4375 Base = SDB->getValue(C);
4376
4377 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4378 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4379 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4380 IndexType = ISD::SIGNED_SCALED;
4381 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4382 return true;
4383 }
4384
4385 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4386 if (!GEP || GEP->getParent() != CurBB)
4387 return false;
4388
4389 if (GEP->getNumOperands() != 2)
4390 return false;
4391
4392 const Value *BasePtr = GEP->getPointerOperand();
4393 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4394
4395 // Make sure the base is scalar and the index is a vector.
4396 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4397 return false;
4398
4399 Base = SDB->getValue(BasePtr);
4400 Index = SDB->getValue(IndexVal);
4401 IndexType = ISD::SIGNED_SCALED;
4402 Scale = DAG.getTargetConstant(
4403 DL.getTypeAllocSize(GEP->getResultElementType()),
4404 SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4405 return true;
4406}
4407
4408void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4409 SDLoc sdl = getCurSDLoc();
4410
4411 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4412 const Value *Ptr = I.getArgOperand(1);
4413 SDValue Src0 = getValue(I.getArgOperand(0));
4414 SDValue Mask = getValue(I.getArgOperand(3));
4415 EVT VT = Src0.getValueType();
4416 Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4417 ->getMaybeAlignValue()
4418 .getValueOr(DAG.getEVTAlign(VT.getScalarType()));
4419 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4420
4421 AAMDNodes AAInfo;
4422 I.getAAMetadata(AAInfo);
4423
4424 SDValue Base;
4425 SDValue Index;
4426 ISD::MemIndexType IndexType;
4427 SDValue Scale;
4428 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4429 I.getParent());
4430
4431 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4432 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4433 MachinePointerInfo(AS), MachineMemOperand::MOStore,
4434 // TODO: Make MachineMemOperands aware of scalable
4435 // vectors.
4436 MemoryLocation::UnknownSize, Alignment, AAInfo);
4437 if (!UniformBase) {
4438 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4439 Index = getValue(Ptr);
4440 IndexType = ISD::SIGNED_UNSCALED;
4441 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4442 }
4443
4444 EVT IdxVT = Index.getValueType();
4445 EVT EltTy = IdxVT.getVectorElementType();
4446 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4447 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4448 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4449 }
4450
4451 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4452 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4453 Ops, MMO, IndexType, false);
4454 DAG.setRoot(Scatter);
4455 setValue(&I, Scatter);
4456}
4457
4458void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4459 SDLoc sdl = getCurSDLoc();
4460
4461 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4462 MaybeAlign &Alignment) {
4463 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4464 Ptr = I.getArgOperand(0);
4465 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
4466 Mask = I.getArgOperand(2);
4467 Src0 = I.getArgOperand(3);
4468 };
4469 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4470 MaybeAlign &Alignment) {
4471 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4472 Ptr = I.getArgOperand(0);
4473 Alignment = None;
4474 Mask = I.getArgOperand(1);
4475 Src0 = I.getArgOperand(2);
4476 };
4477
4478 Value *PtrOperand, *MaskOperand, *Src0Operand;
4479 MaybeAlign Alignment;
4480 if (IsExpanding)
4481 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4482 else
4483 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4484
4485 SDValue Ptr = getValue(PtrOperand);
4486 SDValue Src0 = getValue(Src0Operand);
4487 SDValue Mask = getValue(MaskOperand);
4488 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4489
4490 EVT VT = Src0.getValueType();
4491 if (!Alignment)
4492 Alignment = DAG.getEVTAlign(VT);
4493
4494 AAMDNodes AAInfo;
4495 I.getAAMetadata(AAInfo);
4496 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4497
4498 // Do not serialize masked loads of constant memory with anything.
4499 MemoryLocation ML;
4500 if (VT.isScalableVector())
4501 ML = MemoryLocation::getAfter(PtrOperand);
4502 else
4503 ML = MemoryLocation(PtrOperand, LocationSize::precise(
4504 DAG.getDataLayout().getTypeStoreSize(I.getType())),
4505 AAInfo);
4506 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4507
4508 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4509
4510 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4511 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
4512 // TODO: Make MachineMemOperands aware of scalable
4513 // vectors.
4514 VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);
4515
4516 SDValue Load =
4517 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4518 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4519 if (AddToChain)
4520 PendingLoads.push_back(Load.getValue(1));
4521 setValue(&I, Load);
4522}
4523
4524void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4525 SDLoc sdl = getCurSDLoc();
4526
4527 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4528 const Value *Ptr = I.getArgOperand(0);
4529 SDValue Src0 = getValue(I.getArgOperand(3));
4530 SDValue Mask = getValue(I.getArgOperand(2));
4531
4532 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4533 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4534 Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4535 ->getMaybeAlignValue()
4536 .getValueOr(DAG.getEVTAlign(VT.getScalarType()));
4537
4538 AAMDNodes AAInfo;
4539 I.getAAMetadata(AAInfo);
4540 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4541
4542 SDValue Root = DAG.getRoot();
4543 SDValue Base;
4544 SDValue Index;
4545 ISD::MemIndexType IndexType;
4546 SDValue Scale;
4547 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4548 I.getParent());
4549 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4550 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4551 MachinePointerInfo(AS), MachineMemOperand::MOLoad,
4552 // TODO: Make MachineMemOperands aware of scalable
4553 // vectors.
4554 MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);
4555
4556 if (!UniformBase) {
4557 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4558 Index = getValue(Ptr);
4559 IndexType = ISD::SIGNED_UNSCALED;
4560 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4561 }
4562
4563 EVT IdxVT = Index.getValueType();
4564 EVT EltTy = IdxVT.getVectorElementType();
4565 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4566 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4567 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4568 }
4569
4570 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4571 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4572 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4573
4574 PendingLoads.push_back(Gather.getValue(1));
4575 setValue(&I, Gather);
4576}
4577
4578void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4579 SDLoc dl = getCurSDLoc();
4580 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4581 AtomicOrdering FailureOrdering = I.getFailureOrdering();
4582 SyncScope::ID SSID = I.getSyncScopeID();
4583
4584 SDValue InChain = getRoot();
4585
4586 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4587 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4588
4589 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4590 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4591
4592 MachineFunction &MF = DAG.getMachineFunction();
4593 MachineMemOperand *MMO = MF.getMachineMemOperand(
4594 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4595 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
4596 FailureOrdering);
4597
4598 SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
4599 dl, MemVT, VTs, InChain,
4600 getValue(I.getPointerOperand()),
4601 getValue(I.getCompareOperand()),
4602 getValue(I.getNewValOperand()), MMO);
4603
4604 SDValue OutChain = L.getValue(2);
4605
4606 setValue(&I, L);
4607 DAG.setRoot(OutChain);
4608}
4609
4610void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4611 SDLoc dl = getCurSDLoc();
4612 ISD::NodeType NT;
4613 switch (I.getOperation()) {
4614 default: llvm_unreachable("Unknown atomicrmw operation")__builtin_unreachable();
4615 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4616 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4617 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4618 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4619 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4620 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4621 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4622 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4623 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4624 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4625 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4626 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4627 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4628 }
4629 AtomicOrdering Ordering = I.getOrdering();
4630 SyncScope::ID SSID = I.getSyncScopeID();
4631
4632 SDValue InChain = getRoot();
4633
4634 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4635 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4636 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4637
4638 MachineFunction &MF = DAG.getMachineFunction();
4639 MachineMemOperand *MMO = MF.getMachineMemOperand(
4640 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4641 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
4642
4643 SDValue L =
4644 DAG.getAtomic(NT, dl, MemVT, InChain,
4645 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4646 MMO);
4647
4648 SDValue OutChain = L.getValue(1);
4649
4650 setValue(&I, L);
4651 DAG.setRoot(OutChain);
4652}
4653
4654void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4655 SDLoc dl = getCurSDLoc();
4656 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4657 SDValue Ops[3];
4658 Ops[0] = getRoot();
4659 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
4660 TLI.getFenceOperandTy(DAG.getDataLayout()));
4661 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
4662 TLI.getFenceOperandTy(DAG.getDataLayout()));
4663 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4664}
4665
4666void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4667 SDLoc dl = getCurSDLoc();
4668 AtomicOrdering Order = I.getOrdering();
4669 SyncScope::ID SSID = I.getSyncScopeID();
4670
4671 SDValue InChain = getRoot();
4672
4673 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4674 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4675 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4676
4677 if (!TLI.supportsUnalignedAtomics() &&
4678 I.getAlignment() < MemVT.getSizeInBits() / 8)
4679 report_fatal_error("Cannot generate unaligned atomic load");
4680
4681 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4682
4683 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4684 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4685 I.getAlign(), AAMDNodes(), nullptr, SSID, Order);
4686
4687 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4688
4689 SDValue Ptr = getValue(I.getPointerOperand());
4690
4691 if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {
4692 // TODO: Once this is better exercised by tests, it should be merged with
4693 // the normal path for loads to prevent future divergence.
4694 SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);
4695 if (MemVT != VT)
4696 L = DAG.getPtrExtOrTrunc(L, dl, VT);
4697
4698 setValue(&I, L);
4699 SDValue OutChain = L.getValue(1);
4700 if (!I.isUnordered())
4701 DAG.setRoot(OutChain);
4702 else
4703 PendingLoads.push_back(OutChain);
4704 return;
4705 }
4706
4707 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4708 Ptr, MMO);
4709
4710 SDValue OutChain = L.getValue(1);
4711 if (MemVT != VT)
4712 L = DAG.getPtrExtOrTrunc(L, dl, VT);
4713
4714 setValue(&I, L);
4715 DAG.setRoot(OutChain);
4716}
4717
4718void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4719 SDLoc dl = getCurSDLoc();
4720
4721 AtomicOrdering Ordering = I.getOrdering();
4722 SyncScope::ID SSID = I.getSyncScopeID();
4723
4724 SDValue InChain = getRoot();
4725
4726 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4727 EVT MemVT =
4728 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4729
4730 if (I.getAlignment() < MemVT.getSizeInBits() / 8)
4731 report_fatal_error("Cannot generate unaligned atomic store");
4732
4733 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4734
4735 MachineFunction &MF = DAG.getMachineFunction();
4736 MachineMemOperand *MMO = MF.getMachineMemOperand(
4737 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4738 I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
4739
4740 SDValue Val = getValue(I.getValueOperand());
4741 if (Val.getValueType() != MemVT)
4742 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4743 SDValue Ptr = getValue(I.getPointerOperand());
4744
4745 if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {
4746 // TODO: Once this is better exercised by tests, it should be merged with
4747 // the normal path for stores to prevent future divergence.
4748 SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);
4749 DAG.setRoot(S);
4750 return;
4751 }
4752 SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4753 Ptr, Val, MMO);
4754
4755
4756 DAG.setRoot(OutChain);
4757}
4758
4759/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4760/// node.
4761void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4762 unsigned Intrinsic) {
4763 // Ignore the callsite's attributes. A specific call site may be marked with
4764 // readnone, but the lowering code will expect the chain based on the
4765 // definition.
4766 const Function *F = I.getCalledFunction();
4767 bool HasChain = !F->doesNotAccessMemory();
4768 bool OnlyLoad = HasChain && F->onlyReadsMemory();
4769
4770 // Build the operand list.
4771 SmallVector<SDValue, 8> Ops;
4772 if (HasChain) { // If this intrinsic has side-effects, chainify it.
4773 if (OnlyLoad) {
4774 // We don't need to serialize loads against other loads.
4775 Ops.push_back(DAG.getRoot());
4776 } else {
4777 Ops.push_back(getRoot());
4778 }
4779 }
4780
4781 // Info is set by getTgtMemInstrinsic
4782 TargetLowering::IntrinsicInfo Info;
4783 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4784 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4785 DAG.getMachineFunction(),
4786 Intrinsic);
4787
4788 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4789 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4790 Info.opc == ISD::INTRINSIC_W_CHAIN)
4791 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4792 TLI.getPointerTy(DAG.getDataLayout())));
4793
4794 // Add all operands of the call to the operand list.
4795 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4796 const Value *Arg = I.getArgOperand(i);
4797 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
4798 Ops.push_back(getValue(Arg));
4799 continue;
4800 }
4801
4802 // Use TargetConstant instead of a regular constant for immarg.
4803 EVT VT = TLI.getValueType(*DL, Arg->getType(), true);
4804 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
4805 assert(CI->getBitWidth() <= 64 &&(static_cast<void> (0))
4806 "large intrinsic immediates not handled")(static_cast<void> (0));
4807 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
4808 } else {
4809 Ops.push_back(
4810 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
4811 }
4812 }
4813
4814 SmallVector<EVT, 4> ValueVTs;
4815 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4816
4817 if (HasChain)
4818 ValueVTs.push_back(MVT::Other);
4819
4820 SDVTList VTs = DAG.getVTList(ValueVTs);
4821
4822 // Propagate fast-math-flags from IR to node(s).
4823 SDNodeFlags Flags;
4824 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
4825 Flags.copyFMF(*FPMO);
4826 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
4827
4828 // Create the node.
4829 SDValue Result;
4830 if (IsTgtIntrinsic) {
4831 // This is target intrinsic that touches memory
4832 AAMDNodes AAInfo;
4833 I.getAAMetadata(AAInfo);
4834 Result =
4835 DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
4836 MachinePointerInfo(Info.ptrVal, Info.offset),
4837 Info.align, Info.flags, Info.size, AAInfo);
4838 } else if (!HasChain) {
4839 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4840 } else if (!I.getType()->isVoidTy()) {
4841 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4842 } else {
4843 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4844 }
4845
4846 if (HasChain) {
4847 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4848 if (OnlyLoad)
4849 PendingLoads.push_back(Chain);
4850 else
4851 DAG.setRoot(Chain);
4852 }
4853
4854 if (!I.getType()->isVoidTy()) {
4855 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4856 EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4857 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4858 } else
4859 Result = lowerRangeToAssertZExt(DAG, I, Result);
4860
4861 MaybeAlign Alignment = I.getRetAlign();
4862 if (!Alignment)
4863 Alignment = F->getAttributes().getRetAlignment();
4864 // Insert `assertalign` node if there's an alignment.
4865 if (InsertAssertAlign && Alignment) {
4866 Result =
4867 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
4868 }
4869
4870 setValue(&I, Result);
4871 }
4872}
4873
4874/// GetSignificand - Get the significand and build it into a floating-point
4875/// number with exponent of 1:
4876///
4877/// Op = (Op & 0x007fffff) | 0x3f800000;
4878///
4879/// where Op is the hexadecimal representation of floating point value.
4880static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4881 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4882 DAG.getConstant(0x007fffff, dl, MVT::i32));
4883 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4884 DAG.getConstant(0x3f800000, dl, MVT::i32));
4885 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4886}
4887
4888/// GetExponent - Get the exponent:
4889///
4890/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4891///
4892/// where Op is the hexadecimal representation of floating point value.
4893static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4894 const TargetLowering &TLI, const SDLoc &dl) {
4895 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4896 DAG.getConstant(0x7f800000, dl, MVT::i32));
4897 SDValue t1 = DAG.getNode(
4898 ISD::SRL, dl, MVT::i32, t0,
4899 DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4900 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4901 DAG.getConstant(127, dl, MVT::i32));
4902 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4903}
4904
4905/// getF32Constant - Get 32-bit floating point constant.
4906static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4907 const SDLoc &dl) {
4908 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4909 MVT::f32);
4910}
4911
4912static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4913 SelectionDAG &DAG) {
4914 // TODO: What fast-math-flags should be set on the floating-point nodes?
4915
4916 // IntegerPartOfX = ((int32_t)(t0);
4917 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4918
4919 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4920 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4921 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4922
4923 // IntegerPartOfX <<= 23;
4924 IntegerPartOfX = DAG.getNode(
4925 ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4926 DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4927 DAG.getDataLayout())));
4928
4929 SDValue TwoToFractionalPartOfX;
4930 if (LimitFloatPrecision <= 6) {
4931 // For floating-point precision of 6:
4932 //
4933 // TwoToFractionalPartOfX =
4934 // 0.997535578f +
4935 // (0.735607626f + 0.252464424f * x) * x;
4936 //
4937 // error 0.0144103317, which is 6 bits
4938 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4939 getF32Constant(DAG, 0x3e814304, dl));
4940 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4941 getF32Constant(DAG, 0x3f3c50c8, dl));
4942 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4943 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4944 getF32Constant(DAG, 0x3f7f5e7e, dl));
4945 } else if (LimitFloatPrecision <= 12) {
4946 // For floating-point precision of 12:
4947 //
4948 // TwoToFractionalPartOfX =
4949 // 0.999892986f +
4950 // (0.696457318f +
4951 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4952 //
4953 // error 0.000107046256, which is 13 to 14 bits
4954 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4955 getF32Constant(DAG, 0x3da235e3, dl));
4956 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4957 getF32Constant(DAG, 0x3e65b8f3, dl));
4958 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4959 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4960 getF32Constant(DAG, 0x3f324b07, dl));
4961 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4962 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4963 getF32Constant(DAG, 0x3f7ff8fd, dl));
4964 } else { // LimitFloatPrecision <= 18
4965 // For floating-point precision of 18:
4966 //
4967 // TwoToFractionalPartOfX =
4968 // 0.999999982f +
4969 // (0.693148872f +
4970 // (0.240227044f +
4971 // (0.554906021e-1f +
4972 // (0.961591928e-2f +
4973 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4974 // error 2.47208000*10^(-7), which is better than 18 bits
4975 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4976 getF32Constant(DAG, 0x3924b03e, dl));
4977 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4978 getF32Constant(DAG, 0x3ab24b87, dl));
4979 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4980 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4981 getF32Constant(DAG, 0x3c1d8c17, dl));
4982 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4983 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4984 getF32Constant(DAG, 0x3d634a1d, dl));
4985 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4986 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4987 getF32Constant(DAG, 0x3e75fe14, dl));
4988 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4989 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4990 getF32Constant(DAG, 0x3f317234, dl));
4991 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4992 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4993 getF32Constant(DAG, 0x3f800000, dl));
4994 }
4995
4996 // Add the exponent into the result in integer domain.
4997 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4998 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4999 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5000}
5001
5002/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5003/// limited-precision mode.
5004static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5005 const TargetLowering &TLI, SDNodeFlags Flags) {
5006 if (Op.getValueType() == MVT::f32 &&
5007 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5008
5009 // Put the exponent in the right bit position for later addition to the
5010 // final result:
5011 //
5012 // t0 = Op * log2(e)
5013
5014 // TODO: What fast-math-flags should be set here?
5015 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5016 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5017 return getLimitedPrecisionExp2(t0, dl, DAG);
5018 }
5019
5020 // No special expansion.
5021 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5022}
5023
5024/// expandLog - Lower a log intrinsic. Handles the special sequences for
5025/// limited-precision mode.
5026static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5027 const TargetLowering &TLI, SDNodeFlags Flags) {
5028 // TODO: What fast-math-flags should be set on the floating-point nodes?
5029
5030 if (Op.getValueType() == MVT::f32 &&
5031 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5032 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5033
5034 // Scale the exponent by log(2).
5035 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5036 SDValue LogOfExponent =
5037 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5038 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5039
5040 // Get the significand and build it into a floating-point number with
5041 // exponent of 1.
5042 SDValue X = GetSignificand(DAG, Op1, dl);
5043
5044 SDValue LogOfMantissa;
5045 if (LimitFloatPrecision <= 6) {
5046 // For floating-point precision of 6:
5047 //
5048 // LogofMantissa =
5049 // -1.1609546f +
5050 // (1.4034025f - 0.23903021f * x) * x;
5051 //
5052 // error 0.0034276066, which is better than 8 bits
5053 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5054 getF32Constant(DAG, 0xbe74c456, dl));
5055 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5056 getF32Constant(DAG, 0x3fb3a2b1, dl));
5057 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5058 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5059 getF32Constant(DAG, 0x3f949a29, dl));
5060 } else if (LimitFloatPrecision <= 12) {
5061 // For floating-point precision of 12:
5062 //
5063 // LogOfMantissa =
5064 // -1.7417939f +
5065 // (2.8212026f +
5066 // (-1.4699568f +
5067 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5068 //
5069 // error 0.000061011436, which is 14 bits
5070 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5071 getF32Constant(DAG, 0xbd67b6d6, dl));
5072 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5073 getF32Constant(DAG, 0x3ee4f4b8, dl));
5074 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5075 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5076 getF32Constant(DAG, 0x3fbc278b, dl));
5077 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5078 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5079 getF32Constant(DAG, 0x40348e95, dl));
5080 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5081 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5082 getF32Constant(DAG, 0x3fdef31a, dl));
5083 } else { // LimitFloatPrecision <= 18
5084 // For floating-point precision of 18:
5085 //
5086 // LogOfMantissa =
5087 // -2.1072184f +
5088 // (4.2372794f +
5089 // (-3.7029485f +
5090 // (2.2781945f +
5091 // (-0.87823314f +
5092 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5093 //
5094 // error 0.0000023660568, which is better than 18 bits
5095 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5096 getF32Constant(DAG, 0xbc91e5ac, dl));
5097 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5098 getF32Constant(DAG, 0x3e4350aa, dl));
5099 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5100 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5101 getF32Constant(DAG, 0x3f60d3e3, dl));
5102 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5103 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5104 getF32Constant(DAG, 0x4011cdf0, dl));
5105 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5106 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5107 getF32Constant(DAG, 0x406cfd1c, dl));
5108 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5109 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5110 getF32Constant(DAG, 0x408797cb, dl));
5111 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5112 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5113 getF32Constant(DAG, 0x4006dcab, dl));
5114 }
5115
5116 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5117 }
5118
5119 // No special expansion.
5120 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5121}
5122
5123/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5124/// limited-precision mode.
5125static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5126 const TargetLowering &TLI, SDNodeFlags Flags) {
5127 // TODO: What fast-math-flags should be set on the floating-point nodes?
5128
5129 if (Op.getValueType() == MVT::f32 &&
5130 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5131 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5132
5133 // Get the exponent.
5134 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5135
5136 // Get the significand and build it into a floating-point number with
5137 // exponent of 1.
5138 SDValue X = GetSignificand(DAG, Op1, dl);
5139
5140 // Different possible minimax approximations of significand in
5141 // floating-point for various degrees of accuracy over [1,2].
5142 SDValue Log2ofMantissa;
5143 if (LimitFloatPrecision <= 6) {
5144 // For floating-point precision of 6:
5145 //
5146 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5147 //
5148 // error 0.0049451742, which is more than 7 bits
5149 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5150 getF32Constant(DAG, 0xbeb08fe0, dl));
5151 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5152 getF32Constant(DAG, 0x40019463, dl));
5153 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5154 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5155 getF32Constant(DAG, 0x3fd6633d, dl));
5156 } else if (LimitFloatPrecision <= 12) {
5157 // For floating-point precision of 12:
5158 //
5159 // Log2ofMantissa =
5160 // -2.51285454f +
5161 // (4.07009056f +
5162 // (-2.12067489f +
5163 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5164 //
5165 // error 0.0000876136000, which is better than 13 bits
5166 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5167 getF32Constant(DAG, 0xbda7262e, dl));
5168 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5169 getF32Constant(DAG, 0x3f25280b, dl));
5170 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5171 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5172 getF32Constant(DAG, 0x4007b923, dl));
5173 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5174 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5175 getF32Constant(DAG, 0x40823e2f, dl));
5176 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5177 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5178 getF32Constant(DAG, 0x4020d29c, dl));
5179 } else { // LimitFloatPrecision <= 18
5180 // For floating-point precision of 18:
5181 //
5182 // Log2ofMantissa =
5183 // -3.0400495f +
5184 // (6.1129976f +
5185 // (-5.3420409f +
5186 // (3.2865683f +
5187 // (-1.2669343f +
5188 // (0.27515199f -
5189 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5190 //
5191 // error 0.0000018516, which is better than 18 bits
5192 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5193 getF32Constant(DAG, 0xbcd2769e, dl));
5194 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5195 getF32Constant(DAG, 0x3e8ce0b9, dl));
5196 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5197 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5198 getF32Constant(DAG, 0x3fa22ae7, dl));
5199 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5200 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5201 getF32Constant(DAG, 0x40525723, dl));
5202 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5203 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5204 getF32Constant(DAG, 0x40aaf200, dl));
5205 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5206 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5207 getF32Constant(DAG, 0x40c39dad, dl));
5208 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5209 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5210 getF32Constant(DAG, 0x4042902c, dl));
5211 }
5212
5213 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5214 }
5215
5216 // No special expansion.
5217 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5218}
5219
5220/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5221/// limited-precision mode.
5222static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5223 const TargetLowering &TLI, SDNodeFlags Flags) {
5224 // TODO: What fast-math-flags should be set on the floating-point nodes?
5225
5226 if (Op.getValueType() == MVT::f32 &&
5227 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5228 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5229
5230 // Scale the exponent by log10(2) [0.30102999f].
5231 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5232 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5233 getF32Constant(DAG, 0x3e9a209a, dl));
5234
5235 // Get the significand and build it into a floating-point number with
5236 // exponent of 1.
5237 SDValue X = GetSignificand(DAG, Op1, dl);
5238
5239 SDValue Log10ofMantissa;
5240 if (LimitFloatPrecision <= 6) {
5241 // For floating-point precision of 6:
5242 //
5243 // Log10ofMantissa =
5244 // -0.50419619f +
5245 // (0.60948995f - 0.10380950f * x) * x;
5246 //
5247 // error 0.0014886165, which is 6 bits
5248 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5249 getF32Constant(DAG, 0xbdd49a13, dl));
5250 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5251 getF32Constant(DAG, 0x3f1c0789, dl));
5252 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5253 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5254 getF32Constant(DAG, 0x3f011300, dl));
5255 } else if (LimitFloatPrecision <= 12) {
5256 // For floating-point precision of 12:
5257 //
5258 // Log10ofMantissa =
5259 // -0.64831180f +
5260 // (0.91751397f +
5261 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5262 //
5263 // error 0.00019228036, which is better than 12 bits
5264 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5265 getF32Constant(DAG, 0x3d431f31, dl));
5266 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5267 getF32Constant(DAG, 0x3ea21fb2, dl));
5268 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5269 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5270 getF32Constant(DAG, 0x3f6ae232, dl));
5271 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5272 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5273 getF32Constant(DAG, 0x3f25f7c3, dl));
5274 } else { // LimitFloatPrecision <= 18
5275 // For floating-point precision of 18:
5276 //
5277 // Log10ofMantissa =
5278 // -0.84299375f +
5279 // (1.5327582f +
5280 // (-1.0688956f +
5281 // (0.49102474f +
5282 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5283 //
5284 // error 0.0000037995730, which is better than 18 bits
5285 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5286 getF32Constant(DAG, 0x3c5d51ce, dl));
5287 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5288 getF32Constant(DAG, 0x3e00685a, dl));
5289 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5290 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5291 getF32Constant(DAG, 0x3efb6798, dl));
5292 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5293 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5294 getF32Constant(DAG, 0x3f88d192, dl));
5295 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5296 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5297 getF32Constant(DAG, 0x3fc4316c, dl));
5298 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5299 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5300 getF32Constant(DAG, 0x3f57ce70, dl));
5301 }
5302
5303 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5304 }
5305
5306 // No special expansion.
5307 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5308}
5309
5310/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5311/// limited-precision mode.
5312static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5313 const TargetLowering &TLI, SDNodeFlags Flags) {
5314 if (Op.getValueType() == MVT::f32 &&
5315 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
5316 return getLimitedPrecisionExp2(Op, dl, DAG);
5317
5318 // No special expansion.
5319 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5320}
5321
5322/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5323/// limited-precision mode with x == 10.0f.
5324static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5325 SelectionDAG &DAG, const TargetLowering &TLI,
5326 SDNodeFlags Flags) {
5327 bool IsExp10 = false;
5328 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5329 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5330 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5331 APFloat Ten(10.0f);
5332 IsExp10 = LHSC->isExactlyValue(Ten);
5333 }
5334 }
5335
5336 // TODO: What fast-math-flags should be set on the FMUL node?
5337 if (IsExp10) {
5338 // Put the exponent in the right bit position for later addition to the
5339 // final result:
5340 //
5341 // #define LOG2OF10 3.3219281f
5342 // t0 = Op * LOG2OF10;
5343 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5344 getF32Constant(DAG, 0x40549a78, dl));
5345 return getLimitedPrecisionExp2(t0, dl, DAG);
5346 }
5347
5348 // No special expansion.
5349 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5350}
5351
5352/// ExpandPowI - Expand a llvm.powi intrinsic.
5353static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5354 SelectionDAG &DAG) {
5355 // If RHS is a constant, we can expand this out to a multiplication tree,
5356 // otherwise we end up lowering to a call to __powidf2 (for example). When
5357 // optimizing for size, we only want to do this if the expansion would produce
5358 // a small number of multiplies, otherwise we do the full expansion.
5359 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5360 // Get the exponent as a positive value.
5361 unsigned Val = RHSC->getSExtValue();
5362 if ((int)Val < 0) Val = -Val;
5363
5364 // powi(x, 0) -> 1.0
5365 if (Val == 0)
5366 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5367
5368 bool OptForSize = DAG.shouldOptForSize();
5369 if (!OptForSize ||
5370 // If optimizing for size, don't insert too many multiplies.
5371 // This inserts up to 5 multiplies.
5372 countPopulation(Val) + Log2_32(Val) < 7) {
5373 // We use the simple binary decomposition method to generate the multiply
5374 // sequence. There are more optimal ways to do this (for example,
5375 // powi(x,15) generates one more multiply than it should), but this has
5376 // the benefit of being both really simple and much better than a libcall.
5377 SDValue Res; // Logically starts equal to 1.0
5378 SDValue CurSquare = LHS;
5379 // TODO: Intrinsics should have fast-math-flags that propagate to these
5380 // nodes.
5381 while (Val) {
5382 if (Val & 1) {
5383 if (Res.getNode())
5384 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
5385 else
5386 Res = CurSquare; // 1.0*CurSquare.
5387 }
5388
5389 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5390 CurSquare, CurSquare);
5391 Val >>= 1;
5392 }
5393
5394 // If the original was negative, invert the result, producing 1/(x*x*x).
5395 if (RHSC->getSExtValue() < 0)
5396 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5397 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5398 return Res;
5399 }
5400 }
5401
5402 // Otherwise, expand to a libcall.
5403 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5404}
5405
5406static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5407 SDValue LHS, SDValue RHS, SDValue Scale,
5408 SelectionDAG &DAG, const TargetLowering &TLI) {
5409 EVT VT = LHS.getValueType();
5410 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5411 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5412 LLVMContext &Ctx = *DAG.getContext();
5413
5414 // If the type is legal but the operation isn't, this node might survive all
5415 // the way to operation legalization. If we end up there and we do not have
5416 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5417 // node.
5418
5419 // Coax the legalizer into expanding the node during type legalization instead
5420 // by bumping the size by one bit. This will force it to Promote, enabling the
5421 // early expansion and avoiding the need to expand later.
5422
5423 // We don't have to do this if Scale is 0; that can always be expanded, unless
5424 // it's a saturating signed operation. Those can experience true integer
5425 // division overflow, a case which we must avoid.
5426
5427 // FIXME: We wouldn't have to do this (or any of the early
5428 // expansion/promotion) if it was possible to expand a libcall of an
5429 // illegal type during operation legalization. But it's not, so things
5430 // get a bit hacky.
5431 unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();
5432 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5433 (TLI.isTypeLegal(VT) ||
5434 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5435 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
5436 Opcode, VT, ScaleInt);
5437 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5438 EVT PromVT;
5439 if (VT.isScalarInteger())
5440 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5441 else if (VT.isVector()) {
5442 PromVT = VT.getVectorElementType();
5443 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5444 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5445 } else
5446 llvm_unreachable("Wrong VT for DIVFIX?")__builtin_unreachable();
5447 if (Signed) {
5448 LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);
5449 RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);
5450 } else {
5451 LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);
5452 RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);
5453 }
5454 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5455 // For saturating operations, we need to shift up the LHS to get the
5456 // proper saturation width, and then shift down again afterwards.
5457 if (Saturating)
5458 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5459 DAG.getConstant(1, DL, ShiftTy));
5460 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5461 if (Saturating)
5462 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5463 DAG.getConstant(1, DL, ShiftTy));
5464 return DAG.getZExtOrTrunc(Res, DL, VT);
5465 }
5466 }
5467
5468 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5469}
5470
5471// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5472// bitcasted, or split argument. Returns a list of <Register, size in bits>
5473static void
5474getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5475 const SDValue &N) {
5476 switch (N.getOpcode()) {
5477 case ISD::CopyFromReg: {
5478 SDValue Op = N.getOperand(1);
5479 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5480 Op.getValueType().getSizeInBits());
5481 return;
5482 }
5483 case ISD::BITCAST:
5484 case ISD::AssertZext:
5485 case ISD::AssertSext:
5486 case ISD::TRUNCATE:
5487 getUnderlyingArgRegs(Regs, N.getOperand(0));
5488 return;
5489 case ISD::BUILD_PAIR:
5490 case ISD::BUILD_VECTOR:
5491 case ISD::CONCAT_VECTORS:
5492 for (SDValue Op : N->op_values())
5493 getUnderlyingArgRegs(Regs, Op);
5494 return;
5495 default:
5496 return;
5497 }
5498}
5499
5500/// If the DbgValueInst is a dbg_value of a function argument, create the
5501/// corresponding DBG_VALUE machine instruction for it now. At the end of
5502/// instruction selection, they will be inserted to the entry BB.
5503/// We don't currently support this for variadic dbg_values, as they shouldn't
5504/// appear for function arguments or in the prologue.
5505bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5506 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5507 DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
5508 const Argument *Arg = dyn_cast<Argument>(V);
5509 if (!Arg)
5510 return false;
5511
5512 MachineFunction &MF = DAG.getMachineFunction();
5513 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5514
5515 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5516 // we've been asked to pursue.
5517 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5518 bool Indirect) {
5519 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5520 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5521 // pointing at the VReg, which will be patched up later.
5522 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5523 auto MIB = BuildMI(MF, DL, Inst);
5524 MIB.addReg(Reg, RegState::Debug);
5525 MIB.addImm(0);
5526 MIB.addMetadata(Variable);
5527 auto *NewDIExpr = FragExpr;
5528 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5529 // the DIExpression.
5530 if (Indirect)
5531 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5532 MIB.addMetadata(NewDIExpr);
5533 return MIB;
5534 } else {
5535 // Create a completely standard DBG_VALUE.
5536 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5537 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5538 }
5539 };
5540
5541 if (!IsDbgDeclare) {
5542 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5543 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5544 // the entry block.
5545 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5546 if (!IsInEntryBlock)
5547 return false;
5548
5549 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5550 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5551 // variable that also is a param.
5552 //
5553 // Although, if we are at the top of the entry block already, we can still
5554 // emit using ArgDbgValue. This might catch some situations when the
5555 // dbg.value refers to an argument that isn't used in the entry block, so
5556 // any CopyToReg node would be optimized out and the only way to express
5557 // this DBG_VALUE is by using the physical reg (or FI) as done in this
5558 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
5559 // we should only emit as ArgDbgValue if the Variable is an argument to the
5560 // current function, and the dbg.value intrinsic is found in the entry
5561 // block.
5562 bool VariableIsFunctionInputArg = Variable->isParameter() &&
5563 !DL->getInlinedAt();
5564 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5565 if (!IsInPrologue && !VariableIsFunctionInputArg)
5566 return false;
5567
5568 // Here we assume that a function argument on IR level only can be used to
5569 // describe one input parameter on source level. If we for example have
5570 // source code like this
5571 //
5572 // struct A { long x, y; };
5573 // void foo(struct A a, long b) {
5574 // ...
5575 // b = a.x;
5576 // ...
5577 // }
5578 //
5579 // and IR like this
5580 //
5581 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
5582 // entry:
5583 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5584 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
5585 // call void @llvm.dbg.value(metadata i32 %b, "b",
5586 // ...
5587 // call void @llvm.dbg.value(metadata i32 %a1, "b"
5588 // ...
5589 //
5590 // then the last dbg.value is describing a parameter "b" using a value that
5591 // is an argument. But since we already has used %a1 to describe a parameter
5592 // we should not handle that last dbg.value here (that would result in an
5593 // incorrect hoisting of the DBG_VALUE to the function entry).
5594 // Notice that we allow one dbg.value per IR level argument, to accommodate
5595 // for the situation with fragments above.
5596 if (VariableIsFunctionInputArg) {
5597 unsigned ArgNo = Arg->getArgNo();
5598 if (ArgNo >= FuncInfo.DescribedArgs.size())
5599 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
5600 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
5601 return false;
5602 FuncInfo.DescribedArgs.set(ArgNo);
5603 }
5604 }
5605
5606 bool IsIndirect = false;
5607 Optional<MachineOperand> Op;
5608 // Some arguments' frame index is recorded during argument lowering.
5609 int FI = FuncInfo.getArgumentFrameIndex(Arg);
5610 if (FI != std::numeric_limits<int>::max())
5611 Op = MachineOperand::CreateFI(FI);
5612
5613 SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;
5614 if (!Op && N.getNode()) {
5615 getUnderlyingArgRegs(ArgRegsAndSizes, N);
5616 Register Reg;
5617 if (ArgRegsAndSizes.size() == 1)
5618 Reg = ArgRegsAndSizes.front().first;
5619
5620 if (Reg && Reg.isVirtual()) {
5621 MachineRegisterInfo &RegInfo = MF.getRegInfo();
5622 Register PR = RegInfo.getLiveInPhysReg(Reg);
5623 if (PR)
5624 Reg = PR;
5625 }
5626 if (Reg) {
5627 Op = MachineOperand::CreateReg(Reg, false);
5628 IsIndirect = IsDbgDeclare;
5629 }
5630 }
5631
5632 if (!Op && N.getNode()) {
5633 // Check if frame index is available.
5634 SDValue LCandidate = peekThroughBitcasts(N);
5635 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
5636 if (FrameIndexSDNode *FINode =
5637 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
5638 Op = MachineOperand::CreateFI(FINode->getIndex());
5639 }
5640
5641 if (!Op) {
5642 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
5643 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
5644 SplitRegs) {
5645 unsigned Offset = 0;
5646 for (auto RegAndSize : SplitRegs) {
5647 // If the expression is already a fragment, the current register
5648 // offset+size might extend beyond the fragment. In this case, only
5649 // the register bits that are inside the fragment are relevant.
5650 int RegFragmentSizeInBits = RegAndSize.second;
5651 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
5652 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
5653 // The register is entirely outside the expression fragment,
5654 // so is irrelevant for debug info.
5655 if (Offset >= ExprFragmentSizeInBits)
5656 break;
5657 // The register is partially outside the expression fragment, only
5658 // the low bits within the fragment are relevant for debug info.
5659 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
5660 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
5661 }
5662 }
5663
5664 auto FragmentExpr = DIExpression::createFragmentExpression(
5665 Expr, Offset, RegFragmentSizeInBits);
5666 Offset += RegAndSize.second;
5667 // If a valid fragment expression cannot be created, the variable's
5668 // correct value cannot be determined and so it is set as Undef.
5669 if (!FragmentExpr) {
5670 SDDbgValue *SDV = DAG.getConstantDbgValue(
5671 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
5672 DAG.AddDbgValue(SDV, false);
5673 continue;
5674 }
5675 MachineInstr *NewMI =
5676 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr, IsDbgDeclare);
5677 FuncInfo.ArgDbgValues.push_back(NewMI);
5678 }
5679 };
5680
5681 // Check if ValueMap has reg number.
5682 DenseMap<const Value *, Register>::const_iterator
5683 VMI = FuncInfo.ValueMap.find(V);
5684 if (VMI != FuncInfo.ValueMap.end()) {
5685 const auto &TLI = DAG.getTargetLoweringInfo();
5686 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
5687 V->getType(), None);
5688 if (RFV.occupiesMultipleRegs()) {
5689 splitMultiRegDbgValue(RFV.getRegsAndSizes());
5690 return true;
5691 }
5692
5693 Op = MachineOperand::CreateReg(VMI->second, false);
5694 IsIndirect = IsDbgDeclare;
5695 } else if (ArgRegsAndSizes.size() > 1) {
5696 // This was split due to the calling convention, and no virtual register
5697 // mapping exists for the value.
5698 splitMultiRegDbgValue(ArgRegsAndSizes);
5699 return true;
5700 }
5701 }
5702
5703 if (!Op)
5704 return false;
5705
5706 assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast<void> (0))
5707 "Expected inlined-at fields to agree")(static_cast<void> (0));
5708 MachineInstr *NewMI = nullptr;
5709
5710 if (Op->isReg())
5711 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
5712 else
5713 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
5714 Variable, Expr);
5715
5716 FuncInfo.ArgDbgValues.push_back(NewMI);
5717 return true;
5718}
5719
5720/// Return the appropriate SDDbgValue based on N.
5721SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5722 DILocalVariable *Variable,
5723 DIExpression *Expr,
5724 const DebugLoc &dl,
5725 unsigned DbgSDNodeOrder) {
5726 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
5727 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
5728 // stack slot locations.
5729 //
5730 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
5731 // debug values here after optimization:
5732 //
5733 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
5734 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
5735 //
5736 // Both describe the direct values of their associated variables.
5737 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5738 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5739 }
5740 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5741 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5742}
5743
5744static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
5745 switch (Intrinsic) {
5746 case Intrinsic::smul_fix:
5747 return ISD::SMULFIX;
5748 case Intrinsic::umul_fix:
5749 return ISD::UMULFIX;
5750 case Intrinsic::smul_fix_sat:
5751 return ISD::SMULFIXSAT;
5752 case Intrinsic::umul_fix_sat:
5753 return ISD::UMULFIXSAT;
5754 case Intrinsic::sdiv_fix:
5755 return ISD::SDIVFIX;
5756 case Intrinsic::udiv_fix:
5757 return ISD::UDIVFIX;
5758 case Intrinsic::sdiv_fix_sat:
5759 return ISD::SDIVFIXSAT;
5760 case Intrinsic::udiv_fix_sat:
5761 return ISD::UDIVFIXSAT;
5762 default:
5763 llvm_unreachable("Unhandled fixed point intrinsic")__builtin_unreachable();
5764 }
5765}
5766
5767void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
5768 const char *FunctionName) {
5769 assert(FunctionName && "FunctionName must not be nullptr")(static_cast<void> (0));
5770 SDValue Callee = DAG.getExternalSymbol(
5771 FunctionName,
5772 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
5773 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
5774}
5775
5776/// Given a @llvm.call.preallocated.setup, return the corresponding
5777/// preallocated call.
5778static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
5779 assert(cast<CallBase>(PreallocatedSetup)(static_cast<void> (0))
5780 ->getCalledFunction()(static_cast<void> (0))
5781 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&(static_cast<void> (0))
5782 "expected call_preallocated_setup Value")(static_cast<void> (0));
5783 for (auto *U : PreallocatedSetup->users()) {
5784 auto *UseCall = cast<CallBase>(U);
5785 const Function *Fn = UseCall->getCalledFunction();
5786 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
5787 return UseCall;
5788 }
5789 }
5790 llvm_unreachable("expected corresponding call to preallocated setup/arg")__builtin_unreachable();
5791}
5792
5793/// Lower the call to the specified intrinsic function.
5794void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
5795 unsigned Intrinsic) {
5796 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5797 SDLoc sdl = getCurSDLoc();
5798 DebugLoc dl = getCurDebugLoc();
5799 SDValue Res;
5800
5801 SDNodeFlags Flags;
5802 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
5803 Flags.copyFMF(*FPOp);
5804
5805 switch (Intrinsic) {
5806 default:
5807 // By default, turn this into a target intrinsic node.
5808 visitTargetIntrinsic(I, Intrinsic);
5809 return;
5810 case Intrinsic::vscale: {
5811 match(&I, m_VScale(DAG.getDataLayout()));
5812 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5813 setValue(&I,
5814 DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)));
5815 return;
5816 }
5817 case Intrinsic::vastart: visitVAStart(I); return;
5818 case Intrinsic::vaend: visitVAEnd(I); return;
5819 case Intrinsic::vacopy: visitVACopy(I); return;
5820 case Intrinsic::returnaddress:
5821 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5822 TLI.getPointerTy(DAG.getDataLayout()),
5823 getValue(I.getArgOperand(0))));
5824 return;
5825 case Intrinsic::addressofreturnaddress:
5826 setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5827 TLI.getPointerTy(DAG.getDataLayout())));
5828 return;
5829 case Intrinsic::sponentry:
5830 setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl,
5831 TLI.getFrameIndexTy(DAG.getDataLayout())));
5832 return;
5833 case Intrinsic::frameaddress:
5834 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5835 TLI.getFrameIndexTy(DAG.getDataLayout()),
5836 getValue(I.getArgOperand(0))));
5837 return;
5838 case Intrinsic::read_volatile_register:
5839 case Intrinsic::read_register: {
5840 Value *Reg = I.getArgOperand(0);
5841 SDValue Chain = getRoot();
5842 SDValue RegName =
5843 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5844 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5845 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5846 DAG.getVTList(VT, MVT::Other), Chain, RegName);
5847 setValue(&I, Res);
5848 DAG.setRoot(Res.getValue(1));
5849 return;
5850 }
5851 case Intrinsic::write_register: {
5852 Value *Reg = I.getArgOperand(0);
5853 Value *RegValue = I.getArgOperand(1);
5854 SDValue Chain = getRoot();
5855 SDValue RegName =
5856 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5857 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5858 RegName, getValue(RegValue)));
5859 return;
5860 }
5861 case Intrinsic::memcpy: {
5862 const auto &MCI = cast<MemCpyInst>(I);
5863 SDValue Op1 = getValue(I.getArgOperand(0));
5864 SDValue Op2 = getValue(I.getArgOperand(1));
5865 SDValue Op3 = getValue(I.getArgOperand(2));
5866 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5867 Align DstAlign = MCI.getDestAlign().valueOrOne();
5868 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5869 Align Alignment = commonAlignment(DstAlign, SrcAlign);
5870 bool isVol = MCI.isVolatile();
5871 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5872 // FIXME: Support passing different dest/src alignments to the memcpy DAG
5873 // node.
5874 SDValue Root = isVol ? getRoot() : getMemoryRoot();
5875 AAMDNodes AAInfo;
5876 I.getAAMetadata(AAInfo);
5877 SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5878 /* AlwaysInline */ false, isTC,
5879 MachinePointerInfo(I.getArgOperand(0)),
5880 MachinePointerInfo(I.getArgOperand(1)), AAInfo);
5881 updateDAGForMaybeTailCall(MC);
5882 return;
5883 }
5884 case Intrinsic::memcpy_inline: {
5885 const auto &MCI = cast<MemCpyInlineInst>(I);
5886 SDValue Dst = getValue(I.getArgOperand(0));
5887 SDValue Src = getValue(I.getArgOperand(1));
5888 SDValue Size = getValue(I.getArgOperand(2));
5889 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size")(static_cast<void> (0));
5890 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
5891 Align DstAlign = MCI.getDestAlign().valueOrOne();
5892 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5893 Align Alignment = commonAlignment(DstAlign, SrcAlign);
5894 bool isVol = MCI.isVolatile();
5895 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5896 // FIXME: Support passing different dest/src alignments to the memcpy DAG
5897 // node.
5898 AAMDNodes AAInfo;
5899 I.getAAMetadata(AAInfo);
5900 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
5901 /* AlwaysInline */ true, isTC,
5902 MachinePointerInfo(I.getArgOperand(0)),
5903 MachinePointerInfo(I.getArgOperand(1)), AAInfo);
5904 updateDAGForMaybeTailCall(MC);
5905 return;
5906 }
5907 case Intrinsic::memset: {
5908 const auto &MSI = cast<MemSetInst>(I);
5909 SDValue Op1 = getValue(I.getArgOperand(0));
5910 SDValue Op2 = getValue(I.getArgOperand(1));
5911 SDValue Op3 = getValue(I.getArgOperand(2));
5912 // @llvm.memset defines 0 and 1 to both mean no alignment.
5913 Align Alignment = MSI.getDestAlign().valueOrOne();
5914 bool isVol = MSI.isVolatile();
5915 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5916 SDValue Root = isVol ? getRoot() : getMemoryRoot();
5917 AAMDNodes AAInfo;
5918 I.getAAMetadata(AAInfo);
5919 SDValue MS = DAG.getMemset(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC,
5920 MachinePointerInfo(I.getArgOperand(0)), AAInfo);
5921 updateDAGForMaybeTailCall(MS);
5922 return;
5923 }
5924 case Intrinsic::memmove: {
5925 const auto &MMI = cast<MemMoveInst>(I);
5926 SDValue Op1 = getValue(I.getArgOperand(0));
5927 SDValue Op2 = getValue(I.getArgOperand(1));
5928 SDValue Op3 = getValue(I.getArgOperand(2));
5929 // @llvm.memmove defines 0 and 1 to both mean no alignment.
5930 Align DstAlign = MMI.getDestAlign().valueOrOne();
5931 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
5932 Align Alignment = commonAlignment(DstAlign, SrcAlign);
5933 bool isVol = MMI.isVolatile();
5934 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5935 // FIXME: Support passing different dest/src alignments to the memmove DAG
5936 // node.
5937 SDValue Root = isVol ? getRoot() : getMemoryRoot();
5938 AAMDNodes AAInfo;
5939 I.getAAMetadata(AAInfo);
5940 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5941 isTC, MachinePointerInfo(I.getArgOperand(0)),
5942 MachinePointerInfo(I.getArgOperand(1)), AAInfo);
5943 updateDAGForMaybeTailCall(MM);
5944 return;
5945 }
5946 case Intrinsic::memcpy_element_unordered_atomic: {
5947 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5948 SDValue Dst = getValue(MI.getRawDest());
5949 SDValue Src = getValue(MI.getRawSource());
5950 SDValue Length = getValue(MI.getLength());
5951
5952 unsigned DstAlign = MI.getDestAlignment();
5953 unsigned SrcAlign = MI.getSourceAlignment();
5954 Type *LengthTy = MI.getLength()->getType();
5955 unsigned ElemSz = MI.getElementSizeInBytes();
5956 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5957 SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,
5958 SrcAlign, Length, LengthTy, ElemSz, isTC,
5959 MachinePointerInfo(MI.getRawDest()),
5960 MachinePointerInfo(MI.getRawSource()));
5961 updateDAGForMaybeTailCall(MC);
5962 return;
5963 }
5964 case Intrinsic::memmove_element_unordered_atomic: {
5965 auto &MI = cast<AtomicMemMoveInst>(I);
5966 SDValue Dst = getValue(MI.getRawDest());
5967 SDValue Src = getValue(MI.getRawSource());
5968 SDValue Length = getValue(MI.getLength());
5969
5970 unsigned DstAlign = MI.getDestAlignment();
5971 unsigned SrcAlign = MI.getSourceAlignment();
5972 Type *LengthTy = MI.getLength()->getType();
5973 unsigned ElemSz = MI.getElementSizeInBytes();
5974 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5975 SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,
5976 SrcAlign, Length, LengthTy, ElemSz, isTC,
5977 MachinePointerInfo(MI.getRawDest()),
5978 MachinePointerInfo(MI.getRawSource()));
5979 updateDAGForMaybeTailCall(MC);
5980 return;
5981 }
5982 case Intrinsic::memset_element_unordered_atomic: {
5983 auto &MI = cast<AtomicMemSetInst>(I);
5984 SDValue Dst = getValue(MI.getRawDest());
5985 SDValue Val = getValue(MI.getValue());
5986 SDValue Length = getValue(MI.getLength());
5987
5988 unsigned DstAlign = MI.getDestAlignment();
5989 Type *LengthTy = MI.getLength()->getType();
5990 unsigned ElemSz = MI.getElementSizeInBytes();
5991 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5992 SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,
5993 LengthTy, ElemSz, isTC,
5994 MachinePointerInfo(MI.getRawDest()));
5995 updateDAGForMaybeTailCall(MC);
5996 return;
5997 }
5998 case Intrinsic::call_preallocated_setup: {
5999 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6000 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6001 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6002 getRoot(), SrcValue);
6003 setValue(&I, Res);
6004 DAG.setRoot(Res);
6005 return;
6006 }
6007 case Intrinsic::call_preallocated_arg: {
6008 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6009 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6010 SDValue Ops[3];
6011 Ops[0] = getRoot();
6012 Ops[1] = SrcValue;