Bug Summary

File:lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Warning:line 7225, column 14
Forming reference to null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAGBuilder.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn337657/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn337657/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn337657/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn337657/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-07-23-043044-26795-1 -x c++ /build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp -faddrsig
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements routines for translating from LLVM IR into SelectionDAG IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SelectionDAGBuilder.h"
15#include "SDNodeDbgValue.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/Optional.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/Triple.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/Analysis/AliasAnalysis.h"
31#include "llvm/Analysis/BranchProbabilityInfo.h"
32#include "llvm/Analysis/ConstantFolding.h"
33#include "llvm/Analysis/EHPersonalities.h"
34#include "llvm/Analysis/Loads.h"
35#include "llvm/Analysis/MemoryLocation.h"
36#include "llvm/Analysis/TargetLibraryInfo.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/Analysis/VectorUtils.h"
39#include "llvm/CodeGen/Analysis.h"
40#include "llvm/CodeGen/FunctionLoweringInfo.h"
41#include "llvm/CodeGen/GCMetadata.h"
42#include "llvm/CodeGen/ISDOpcodes.h"
43#include "llvm/CodeGen/MachineBasicBlock.h"
44#include "llvm/CodeGen/MachineFrameInfo.h"
45#include "llvm/CodeGen/MachineFunction.h"
46#include "llvm/CodeGen/MachineInstr.h"
47#include "llvm/CodeGen/MachineInstrBuilder.h"
48#include "llvm/CodeGen/MachineJumpTableInfo.h"
49#include "llvm/CodeGen/MachineMemOperand.h"
50#include "llvm/CodeGen/MachineModuleInfo.h"
51#include "llvm/CodeGen/MachineOperand.h"
52#include "llvm/CodeGen/MachineRegisterInfo.h"
53#include "llvm/CodeGen/RuntimeLibcalls.h"
54#include "llvm/CodeGen/SelectionDAG.h"
55#include "llvm/CodeGen/SelectionDAGNodes.h"
56#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
57#include "llvm/CodeGen/StackMaps.h"
58#include "llvm/CodeGen/TargetFrameLowering.h"
59#include "llvm/CodeGen/TargetInstrInfo.h"
60#include "llvm/CodeGen/TargetLowering.h"
61#include "llvm/CodeGen/TargetOpcodes.h"
62#include "llvm/CodeGen/TargetRegisterInfo.h"
63#include "llvm/CodeGen/TargetSubtargetInfo.h"
64#include "llvm/CodeGen/ValueTypes.h"
65#include "llvm/CodeGen/WinEHFuncInfo.h"
66#include "llvm/IR/Argument.h"
67#include "llvm/IR/Attributes.h"
68#include "llvm/IR/BasicBlock.h"
69#include "llvm/IR/CFG.h"
70#include "llvm/IR/CallSite.h"
71#include "llvm/IR/CallingConv.h"
72#include "llvm/IR/Constant.h"
73#include "llvm/IR/ConstantRange.h"
74#include "llvm/IR/Constants.h"
75#include "llvm/IR/DataLayout.h"
76#include "llvm/IR/DebugInfoMetadata.h"
77#include "llvm/IR/DebugLoc.h"
78#include "llvm/IR/DerivedTypes.h"
79#include "llvm/IR/Function.h"
80#include "llvm/IR/GetElementPtrTypeIterator.h"
81#include "llvm/IR/InlineAsm.h"
82#include "llvm/IR/InstrTypes.h"
83#include "llvm/IR/Instruction.h"
84#include "llvm/IR/Instructions.h"
85#include "llvm/IR/IntrinsicInst.h"
86#include "llvm/IR/Intrinsics.h"
87#include "llvm/IR/LLVMContext.h"
88#include "llvm/IR/Metadata.h"
89#include "llvm/IR/Module.h"
90#include "llvm/IR/Operator.h"
91#include "llvm/IR/Statepoint.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCContext.h"
96#include "llvm/MC/MCSymbol.h"
97#include "llvm/Support/AtomicOrdering.h"
98#include "llvm/Support/BranchProbability.h"
99#include "llvm/Support/Casting.h"
100#include "llvm/Support/CodeGen.h"
101#include "llvm/Support/CommandLine.h"
102#include "llvm/Support/Compiler.h"
103#include "llvm/Support/Debug.h"
104#include "llvm/Support/ErrorHandling.h"
105#include "llvm/Support/MachineValueType.h"
106#include "llvm/Support/MathExtras.h"
107#include "llvm/Support/raw_ostream.h"
108#include "llvm/Target/TargetIntrinsicInfo.h"
109#include "llvm/Target/TargetMachine.h"
110#include "llvm/Target/TargetOptions.h"
111#include <algorithm>
112#include <cassert>
113#include <cstddef>
114#include <cstdint>
115#include <cstring>
116#include <iterator>
117#include <limits>
118#include <numeric>
119#include <tuple>
120#include <utility>
121#include <vector>
122
123using namespace llvm;
124
125#define DEBUG_TYPE"isel" "isel"
126
127/// LimitFloatPrecision - Generate low-precision inline sequences for
128/// some float libcalls (6, 8 or 12 bits).
129static unsigned LimitFloatPrecision;
130
131static cl::opt<unsigned, true>
132 LimitFPPrecision("limit-float-precision",
133 cl::desc("Generate low-precision inline sequences "
134 "for some float libcalls"),
135 cl::location(LimitFloatPrecision), cl::Hidden,
136 cl::init(0));
137
138static cl::opt<unsigned> SwitchPeelThreshold(
139 "switch-peel-threshold", cl::Hidden, cl::init(66),
140 cl::desc("Set the case probability threshold for peeling the case from a "
141 "switch statement. A value greater than 100 will void this "
142 "optimization"));
143
144// Limit the width of DAG chains. This is important in general to prevent
145// DAG-based analysis from blowing up. For example, alias analysis and
146// load clustering may not complete in reasonable time. It is difficult to
147// recognize and avoid this situation within each individual analysis, and
148// future analyses are likely to have the same behavior. Limiting DAG width is
149// the safe approach and will be especially important with global DAGs.
150//
151// MaxParallelChains default is arbitrarily high to avoid affecting
152// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
153// sequence over this should have been converted to llvm.memcpy by the
154// frontend. It is easy to induce this behavior with .ll code such as:
155// %buffer = alloca [4096 x i8]
156// %data = load [4096 x i8]* %argPtr
157// store [4096 x i8] %data, [4096 x i8]* %buffer
158static const unsigned MaxParallelChains = 64;
159
160// True if the Value passed requires ABI mangling as it is a parameter to a
161// function or a return value from a function which is not an intrinsic.
162static bool isABIRegCopy(const Value *V) {
163 const bool IsRetInst = V && isa<ReturnInst>(V);
164 const bool IsCallInst = V && isa<CallInst>(V);
165 const bool IsInLineAsm =
166 IsCallInst && static_cast<const CallInst *>(V)->isInlineAsm();
167 const bool IsIndirectFunctionCall =
168 IsCallInst && !IsInLineAsm &&
169 !static_cast<const CallInst *>(V)->getCalledFunction();
170 // It is possible that the call instruction is an inline asm statement or an
171 // indirect function call in which case the return value of
172 // getCalledFunction() would be nullptr.
173 const bool IsInstrinsicCall =
174 IsCallInst && !IsInLineAsm && !IsIndirectFunctionCall &&
175 static_cast<const CallInst *>(V)->getCalledFunction()->getIntrinsicID() !=
176 Intrinsic::not_intrinsic;
177
178 return IsRetInst || (IsCallInst && (!IsInLineAsm && !IsInstrinsicCall));
179}
180
181static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
182 const SDValue *Parts, unsigned NumParts,
183 MVT PartVT, EVT ValueVT, const Value *V,
184 bool IsABIRegCopy);
185
186/// getCopyFromParts - Create a value that contains the specified legal parts
187/// combined into the value they represent. If the parts combine to a type
188/// larger than ValueVT then AssertOp can be used to specify whether the extra
189/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
190/// (ISD::AssertSext).
191static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
192 const SDValue *Parts, unsigned NumParts,
193 MVT PartVT, EVT ValueVT, const Value *V,
194 Optional<ISD::NodeType> AssertOp = None,
195 bool IsABIRegCopy = false) {
196 if (ValueVT.isVector())
197 return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
198 PartVT, ValueVT, V, IsABIRegCopy);
199
200 assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 200, __extension__ __PRETTY_FUNCTION__))
;
201 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
202 SDValue Val = Parts[0];
203
204 if (NumParts > 1) {
205 // Assemble the value from multiple parts.
206 if (ValueVT.isInteger()) {
207 unsigned PartBits = PartVT.getSizeInBits();
208 unsigned ValueBits = ValueVT.getSizeInBits();
209
210 // Assemble the power of 2 part.
211 unsigned RoundParts = NumParts & (NumParts - 1) ?
212 1 << Log2_32(NumParts) : NumParts;
213 unsigned RoundBits = PartBits * RoundParts;
214 EVT RoundVT = RoundBits == ValueBits ?
215 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
216 SDValue Lo, Hi;
217
218 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
219
220 if (RoundParts > 2) {
221 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
222 PartVT, HalfVT, V);
223 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
224 RoundParts / 2, PartVT, HalfVT, V);
225 } else {
226 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
227 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
228 }
229
230 if (DAG.getDataLayout().isBigEndian())
231 std::swap(Lo, Hi);
232
233 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
234
235 if (RoundParts < NumParts) {
236 // Assemble the trailing non-power-of-2 part.
237 unsigned OddParts = NumParts - RoundParts;
238 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
239 Hi = getCopyFromParts(DAG, DL,
240 Parts + RoundParts, OddParts, PartVT, OddVT, V);
241
242 // Combine the round and odd parts.
243 Lo = Val;
244 if (DAG.getDataLayout().isBigEndian())
245 std::swap(Lo, Hi);
246 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
247 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
248 Hi =
249 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
250 DAG.getConstant(Lo.getValueSizeInBits(), DL,
251 TLI.getPointerTy(DAG.getDataLayout())));
252 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
253 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
254 }
255 } else if (PartVT.isFloatingPoint()) {
256 // FP split into multiple FP parts (for ppcf128)
257 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 258, __extension__ __PRETTY_FUNCTION__))
258 "Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 258, __extension__ __PRETTY_FUNCTION__))
;
259 SDValue Lo, Hi;
260 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
261 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
262 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
263 std::swap(Lo, Hi);
264 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
265 } else {
266 // FP split into integer parts (soft fp)
267 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 268, __extension__ __PRETTY_FUNCTION__))
268 !PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 268, __extension__ __PRETTY_FUNCTION__))
;
269 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
270 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
271 }
272 }
273
274 // There is now one part, held in Val. Correct it to match ValueVT.
275 // PartEVT is the type of the register class that holds the value.
276 // ValueVT is the type of the inline asm operation.
277 EVT PartEVT = Val.getValueType();
278
279 if (PartEVT == ValueVT)
280 return Val;
281
282 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
283 ValueVT.bitsLT(PartEVT)) {
284 // For an FP value in an integer part, we need to truncate to the right
285 // width first.
286 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
287 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
288 }
289
290 // Handle types that have the same size.
291 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
292 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
293
294 // Handle types with different sizes.
295 if (PartEVT.isInteger() && ValueVT.isInteger()) {
296 if (ValueVT.bitsLT(PartEVT)) {
297 // For a truncate, see if we have any information to
298 // indicate whether the truncated bits will always be
299 // zero or sign-extension.
300 if (AssertOp.hasValue())
301 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
302 DAG.getValueType(ValueVT));
303 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
304 }
305 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
306 }
307
308 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
309 // FP_ROUND's are always exact here.
310 if (ValueVT.bitsLT(Val.getValueType()))
311 return DAG.getNode(
312 ISD::FP_ROUND, DL, ValueVT, Val,
313 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
314
315 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
316 }
317
318 llvm_unreachable("Unknown mismatch!")::llvm::llvm_unreachable_internal("Unknown mismatch!", "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 318)
;
319}
320
321static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
322 const Twine &ErrMsg) {
323 const Instruction *I = dyn_cast_or_null<Instruction>(V);
324 if (!V)
325 return Ctx.emitError(ErrMsg);
326
327 const char *AsmError = ", possible invalid constraint for vector type";
328 if (const CallInst *CI = dyn_cast<CallInst>(I))
329 if (isa<InlineAsm>(CI->getCalledValue()))
330 return Ctx.emitError(I, ErrMsg + AsmError);
331
332 return Ctx.emitError(I, ErrMsg);
333}
334
335/// getCopyFromPartsVector - Create a value that contains the specified legal
336/// parts combined into the value they represent. If the parts combine to a
337/// type larger than ValueVT then AssertOp can be used to specify whether the
338/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
339/// ValueVT (ISD::AssertSext).
340static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
341 const SDValue *Parts, unsigned NumParts,
342 MVT PartVT, EVT ValueVT, const Value *V,
343 bool IsABIRegCopy) {
344 assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 344, __extension__ __PRETTY_FUNCTION__))
;
345 assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 345, __extension__ __PRETTY_FUNCTION__))
;
346 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
347 SDValue Val = Parts[0];
348
349 // Handle a multi-element vector.
350 if (NumParts > 1) {
351 EVT IntermediateVT;
352 MVT RegisterVT;
353 unsigned NumIntermediates;
354 unsigned NumRegs;
355
356 if (IsABIRegCopy) {
357 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
358 *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates,
359 RegisterVT);
360 } else {
361 NumRegs =
362 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
363 NumIntermediates, RegisterVT);
364 }
365
366 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 366, __extension__ __PRETTY_FUNCTION__))
;
367 NumParts = NumRegs; // Silence a compiler warning.
368 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 368, __extension__ __PRETTY_FUNCTION__))
;
369 assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 371, __extension__ __PRETTY_FUNCTION__))
370 Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 371, __extension__ __PRETTY_FUNCTION__))
371 "Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 371, __extension__ __PRETTY_FUNCTION__))
;
372
373 // Assemble the parts into intermediate operands.
374 SmallVector<SDValue, 8> Ops(NumIntermediates);
375 if (NumIntermediates == NumParts) {
376 // If the register was not expanded, truncate or copy the value,
377 // as appropriate.
378 for (unsigned i = 0; i != NumParts; ++i)
379 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
380 PartVT, IntermediateVT, V);
381 } else if (NumParts > 0) {
382 // If the intermediate type was expanded, build the intermediate
383 // operands from the parts.
384 assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 385, __extension__ __PRETTY_FUNCTION__))
385 "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 385, __extension__ __PRETTY_FUNCTION__))
;
386 unsigned Factor = NumParts / NumIntermediates;
387 for (unsigned i = 0; i != NumIntermediates; ++i)
388 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
389 PartVT, IntermediateVT, V);
390 }
391
392 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
393 // intermediate operands.
394 EVT BuiltVectorTy =
395 EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
396 (IntermediateVT.isVector()
397 ? IntermediateVT.getVectorNumElements() * NumParts
398 : NumIntermediates));
399 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
400 : ISD::BUILD_VECTOR,
401 DL, BuiltVectorTy, Ops);
402 }
403
404 // There is now one part, held in Val. Correct it to match ValueVT.
405 EVT PartEVT = Val.getValueType();
406
407 if (PartEVT == ValueVT)
408 return Val;
409
410 if (PartEVT.isVector()) {
411 // If the element type of the source/dest vectors are the same, but the
412 // parts vector has more elements than the value vector, then we have a
413 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
414 // elements we want.
415 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
416 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&(static_cast <bool> (PartEVT.getVectorNumElements() >
ValueVT.getVectorNumElements() && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 417, __extension__ __PRETTY_FUNCTION__))
417 "Cannot narrow, it would be a lossy transformation")(static_cast <bool> (PartEVT.getVectorNumElements() >
ValueVT.getVectorNumElements() && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 417, __extension__ __PRETTY_FUNCTION__))
;
418 return DAG.getNode(
419 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
420 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
421 }
422
423 // Vector/Vector bitcast.
424 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
425 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
426
427 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&(static_cast <bool> (PartEVT.getVectorNumElements() == ValueVT
.getVectorNumElements() && "Cannot handle this kind of promotion"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 428, __extension__ __PRETTY_FUNCTION__))
428 "Cannot handle this kind of promotion")(static_cast <bool> (PartEVT.getVectorNumElements() == ValueVT
.getVectorNumElements() && "Cannot handle this kind of promotion"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 428, __extension__ __PRETTY_FUNCTION__))
;
429 // Promoted vector extract
430 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
431
432 }
433
434 // Trivial bitcast if the types are the same size and the destination
435 // vector type is legal.
436 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
437 TLI.isTypeLegal(ValueVT))
438 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
439
440 if (ValueVT.getVectorNumElements() != 1) {
441 // Certain ABIs require that vectors are passed as integers. For vectors
442 // are the same size, this is an obvious bitcast.
443 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
444 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
445 } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
446 // Bitcast Val back the original type and extract the corresponding
447 // vector we want.
448 unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
449 EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
450 ValueVT.getVectorElementType(), Elts);
451 Val = DAG.getBitcast(WiderVecType, Val);
452 return DAG.getNode(
453 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
454 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
455 }
456
457 diagnosePossiblyInvalidConstraint(
458 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
459 return DAG.getUNDEF(ValueVT);
460 }
461
462 // Handle cases such as i8 -> <1 x i1>
463 EVT ValueSVT = ValueVT.getVectorElementType();
464 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
465 Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
466 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
467
468 return DAG.getBuildVector(ValueVT, DL, Val);
469}
470
471static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
472 SDValue Val, SDValue *Parts, unsigned NumParts,
473 MVT PartVT, const Value *V, bool IsABIRegCopy);
474
475/// getCopyToParts - Create a series of nodes that contain the specified value
476/// split into legal parts. If the parts contain more bits than Val, then, for
477/// integers, ExtendKind can be used to specify how to generate the extra bits.
478static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
479 SDValue *Parts, unsigned NumParts, MVT PartVT,
480 const Value *V,
481 ISD::NodeType ExtendKind = ISD::ANY_EXTEND,
482 bool IsABIRegCopy = false) {
483 EVT ValueVT = Val.getValueType();
484
485 // Handle the vector case separately.
486 if (ValueVT.isVector())
487 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
488 IsABIRegCopy);
489
490 unsigned PartBits = PartVT.getSizeInBits();
491 unsigned OrigNumParts = NumParts;
492 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 493, __extension__ __PRETTY_FUNCTION__))
493 "Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 493, __extension__ __PRETTY_FUNCTION__))
;
494
495 if (NumParts == 0)
496 return;
497
498 assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere"
) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 498, __extension__ __PRETTY_FUNCTION__))
;
499 EVT PartEVT = PartVT;
500 if (PartEVT == ValueVT) {
501 assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __extension__ __PRETTY_FUNCTION__))
;
502 Parts[0] = Val;
503 return;
504 }
505
506 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
507 // If the parts cover more bits than the value has, promote the value.
508 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
509 assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 509, __extension__ __PRETTY_FUNCTION__))
;
510 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
511 } else {
512 if (ValueVT.isFloatingPoint()) {
513 // FP values need to be bitcast, then extended if they are being put
514 // into a larger container.
515 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
516 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
517 }
518 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __extension__ __PRETTY_FUNCTION__))
519 ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __extension__ __PRETTY_FUNCTION__))
520 "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __extension__ __PRETTY_FUNCTION__))
;
521 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
522 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
523 if (PartVT == MVT::x86mmx)
524 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
525 }
526 } else if (PartBits == ValueVT.getSizeInBits()) {
527 // Different types of the same size.
528 assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT !=
ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 528, __extension__ __PRETTY_FUNCTION__))
;
529 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
530 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
531 // If the parts cover less bits than value has, truncate the value.
532 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __extension__ __PRETTY_FUNCTION__))
533 ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __extension__ __PRETTY_FUNCTION__))
534 "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __extension__ __PRETTY_FUNCTION__))
;
535 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
536 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
537 if (PartVT == MVT::x86mmx)
538 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
539 }
540
541 // The value may have changed - recompute ValueVT.
542 ValueVT = Val.getValueType();
543 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 544, __extension__ __PRETTY_FUNCTION__))
544 "Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 544, __extension__ __PRETTY_FUNCTION__))
;
545
546 if (NumParts == 1) {
547 if (PartEVT != ValueVT) {
548 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
549 "scalar-to-vector conversion failed");
550 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
551 }
552
553 Parts[0] = Val;
554 return;
555 }
556
557 // Expand the value into multiple parts.
558 if (NumParts & (NumParts - 1)) {
559 // The number of parts is not a power of 2. Split off and copy the tail.
560 assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 561, __extension__ __PRETTY_FUNCTION__))
561 "Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 561, __extension__ __PRETTY_FUNCTION__))
;
562 unsigned RoundParts = 1 << Log2_32(NumParts);
563 unsigned RoundBits = RoundParts * PartBits;
564 unsigned OddParts = NumParts - RoundParts;
565 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
566 DAG.getIntPtrConstant(RoundBits, DL));
567 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
568
569 if (DAG.getDataLayout().isBigEndian())
570 // The odd parts were reversed by getCopyToParts - unreverse them.
571 std::reverse(Parts + RoundParts, Parts + NumParts);
572
573 NumParts = RoundParts;
574 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
575 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
576 }
577
578 // The number of parts is a power of 2. Repeatedly bisect the value using
579 // EXTRACT_ELEMENT.
580 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
581 EVT::getIntegerVT(*DAG.getContext(),
582 ValueVT.getSizeInBits()),
583 Val);
584
585 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
586 for (unsigned i = 0; i < NumParts; i += StepSize) {
587 unsigned ThisBits = StepSize * PartBits / 2;
588 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
589 SDValue &Part0 = Parts[i];
590 SDValue &Part1 = Parts[i+StepSize/2];
591
592 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
593 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
594 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
595 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
596
597 if (ThisBits == PartBits && ThisVT != PartVT) {
598 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
599 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
600 }
601 }
602 }
603
604 if (DAG.getDataLayout().isBigEndian())
605 std::reverse(Parts, Parts + OrigNumParts);
606}
607
608
609/// getCopyToPartsVector - Create a series of nodes that contain the specified
610/// value split into legal parts.
611static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
612 SDValue Val, SDValue *Parts, unsigned NumParts,
613 MVT PartVT, const Value *V,
614 bool IsABIRegCopy) {
615 EVT ValueVT = Val.getValueType();
616 assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 616, __extension__ __PRETTY_FUNCTION__))
;
617 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
618
619 if (NumParts == 1) {
620 EVT PartEVT = PartVT;
621 if (PartEVT == ValueVT) {
622 // Nothing to do.
623 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
624 // Bitconvert vector->vector case.
625 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
626 } else if (PartVT.isVector() &&
627 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
628 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
629 EVT ElementVT = PartVT.getVectorElementType();
630 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
631 // undef elements.
632 SmallVector<SDValue, 16> Ops;
633 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
634 Ops.push_back(DAG.getNode(
635 ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
636 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
637
638 for (unsigned i = ValueVT.getVectorNumElements(),
639 e = PartVT.getVectorNumElements(); i != e; ++i)
640 Ops.push_back(DAG.getUNDEF(ElementVT));
641
642 Val = DAG.getBuildVector(PartVT, DL, Ops);
643
644 // FIXME: Use CONCAT for 2x -> 4x.
645
646 //SDValue UndefElts = DAG.getUNDEF(VectorTy);
647 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
648 } else if (PartVT.isVector() &&
649 PartEVT.getVectorElementType().bitsGE(
650 ValueVT.getVectorElementType()) &&
651 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
652
653 // Promoted vector extract
654 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
655 } else {
656 if (ValueVT.getVectorNumElements() == 1) {
657 Val = DAG.getNode(
658 ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
659 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
660 } else {
661 assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&(static_cast <bool> (PartVT.getSizeInBits() > ValueVT
.getSizeInBits() && "lossy conversion of vector to scalar type"
) ? void (0) : __assert_fail ("PartVT.getSizeInBits() > ValueVT.getSizeInBits() && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 662, __extension__ __PRETTY_FUNCTION__))
662 "lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getSizeInBits() > ValueVT
.getSizeInBits() && "lossy conversion of vector to scalar type"
) ? void (0) : __assert_fail ("PartVT.getSizeInBits() > ValueVT.getSizeInBits() && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 662, __extension__ __PRETTY_FUNCTION__))
;
663 EVT IntermediateType =
664 EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
665 Val = DAG.getBitcast(IntermediateType, Val);
666 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
667 }
668 }
669
670 assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT &&
"Unexpected vector part value type") ? void (0) : __assert_fail
("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 670, __extension__ __PRETTY_FUNCTION__))
;
671 Parts[0] = Val;
672 return;
673 }
674
675 // Handle a multi-element vector.
676 EVT IntermediateVT;
677 MVT RegisterVT;
678 unsigned NumIntermediates;
679 unsigned NumRegs;
680 if (IsABIRegCopy) {
681 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
682 *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates,
683 RegisterVT);
684 } else {
685 NumRegs =
686 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
687 NumIntermediates, RegisterVT);
688 }
689 unsigned NumElements = ValueVT.getVectorNumElements();
690
691 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 691, __extension__ __PRETTY_FUNCTION__))
;
692 NumParts = NumRegs; // Silence a compiler warning.
693 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 693, __extension__ __PRETTY_FUNCTION__))
;
694
695 // Convert the vector to the appropiate type if necessary.
696 unsigned DestVectorNoElts =
697 NumIntermediates *
698 (IntermediateVT.isVector() ? IntermediateVT.getVectorNumElements() : 1);
699 EVT BuiltVectorTy = EVT::getVectorVT(
700 *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
701 if (Val.getValueType() != BuiltVectorTy)
702 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
703
704 // Split the vector into intermediate operands.
705 SmallVector<SDValue, 8> Ops(NumIntermediates);
706 for (unsigned i = 0; i != NumIntermediates; ++i) {
707 if (IntermediateVT.isVector())
708 Ops[i] =
709 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
710 DAG.getConstant(i * (NumElements / NumIntermediates), DL,
711 TLI.getVectorIdxTy(DAG.getDataLayout())));
712 else
713 Ops[i] = DAG.getNode(
714 ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
715 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
716 }
717
718 // Split the intermediate operands into legal parts.
719 if (NumParts == NumIntermediates) {
720 // If the register was not expanded, promote or copy the value,
721 // as appropriate.
722 for (unsigned i = 0; i != NumParts; ++i)
723 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
724 } else if (NumParts > 0) {
725 // If the intermediate type was expanded, split each the value into
726 // legal parts.
727 assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero"
) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 727, __extension__ __PRETTY_FUNCTION__))
;
728 assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 729, __extension__ __PRETTY_FUNCTION__))
729 "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 729, __extension__ __PRETTY_FUNCTION__))
;
730 unsigned Factor = NumParts / NumIntermediates;
731 for (unsigned i = 0; i != NumIntermediates; ++i)
732 getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
733 }
734}
735
736RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
737 EVT valuevt, bool IsABIMangledValue)
738 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
739 RegCount(1, regs.size()), IsABIMangled(IsABIMangledValue) {}
740
741RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
742 const DataLayout &DL, unsigned Reg, Type *Ty,
743 bool IsABIMangledValue) {
744 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
745
746 IsABIMangled = IsABIMangledValue;
747
748 for (EVT ValueVT : ValueVTs) {
749 unsigned NumRegs = IsABIMangledValue
750 ? TLI.getNumRegistersForCallingConv(Context, ValueVT)
751 : TLI.getNumRegisters(Context, ValueVT);
752 MVT RegisterVT = IsABIMangledValue
753 ? TLI.getRegisterTypeForCallingConv(Context, ValueVT)
754 : TLI.getRegisterType(Context, ValueVT);
755 for (unsigned i = 0; i != NumRegs; ++i)
756 Regs.push_back(Reg + i);
757 RegVTs.push_back(RegisterVT);
758 RegCount.push_back(NumRegs);
759 Reg += NumRegs;
760 }
761}
762
763SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
764 FunctionLoweringInfo &FuncInfo,
765 const SDLoc &dl, SDValue &Chain,
766 SDValue *Flag, const Value *V) const {
767 // A Value with type {} or [0 x %t] needs no registers.
768 if (ValueVTs.empty())
769 return SDValue();
770
771 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
772
773 // Assemble the legal parts into the final values.
774 SmallVector<SDValue, 4> Values(ValueVTs.size());
775 SmallVector<SDValue, 8> Parts;
776 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
777 // Copy the legal parts from the registers.
778 EVT ValueVT = ValueVTs[Value];
779 unsigned NumRegs = RegCount[Value];
780 MVT RegisterVT = IsABIMangled
781 ? TLI.getRegisterTypeForCallingConv(*DAG.getContext(), RegVTs[Value])
782 : RegVTs[Value];
783
784 Parts.resize(NumRegs);
785 for (unsigned i = 0; i != NumRegs; ++i) {
786 SDValue P;
787 if (!Flag) {
788 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
789 } else {
790 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
791 *Flag = P.getValue(2);
792 }
793
794 Chain = P.getValue(1);
795 Parts[i] = P;
796
797 // If the source register was virtual and if we know something about it,
798 // add an assert node.
799 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
800 !RegisterVT.isInteger() || RegisterVT.isVector())
801 continue;
802
803 const FunctionLoweringInfo::LiveOutInfo *LOI =
804 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
805 if (!LOI)
806 continue;
807
808 unsigned RegSize = RegisterVT.getSizeInBits();
809 unsigned NumSignBits = LOI->NumSignBits;
810 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
811
812 if (NumZeroBits == RegSize) {
813 // The current value is a zero.
814 // Explicitly express that as it would be easier for
815 // optimizations to kick in.
816 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
817 continue;
818 }
819
820 // FIXME: We capture more information than the dag can represent. For
821 // now, just use the tightest assertzext/assertsext possible.
822 bool isSExt;
823 EVT FromVT(MVT::Other);
824 if (NumZeroBits) {
825 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
826 isSExt = false;
827 } else if (NumSignBits > 1) {
828 FromVT =
829 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
830 isSExt = true;
831 } else {
832 continue;
833 }
834 // Add an assertion node.
835 assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) :
__assert_fail ("FromVT != MVT::Other", "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 835, __extension__ __PRETTY_FUNCTION__))
;
836 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
837 RegisterVT, P, DAG.getValueType(FromVT));
838 }
839
840 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
841 NumRegs, RegisterVT, ValueVT, V);
842 Part += NumRegs;
843 Parts.clear();
844 }
845
846 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
847}
848
849void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
850 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
851 const Value *V,
852 ISD::NodeType PreferredExtendType) const {
853 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
854 ISD::NodeType ExtendKind = PreferredExtendType;
855
856 // Get the list of the values's legal parts.
857 unsigned NumRegs = Regs.size();
858 SmallVector<SDValue, 8> Parts(NumRegs);
859 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
860 unsigned NumParts = RegCount[Value];
861
862 MVT RegisterVT = IsABIMangled
863 ? TLI.getRegisterTypeForCallingConv(*DAG.getContext(), RegVTs[Value])
864 : RegVTs[Value];
865
866 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
867 ExtendKind = ISD::ZERO_EXTEND;
868
869 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
870 &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
871 Part += NumParts;
872 }
873
874 // Copy the parts into the registers.
875 SmallVector<SDValue, 8> Chains(NumRegs);
876 for (unsigned i = 0; i != NumRegs; ++i) {
877 SDValue Part;
878 if (!Flag) {
879 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
880 } else {
881 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
882 *Flag = Part.getValue(1);
883 }
884
885 Chains[i] = Part.getValue(0);
886 }
887
888 if (NumRegs == 1 || Flag)
889 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
890 // flagged to it. That is the CopyToReg nodes and the user are considered
891 // a single scheduling unit. If we create a TokenFactor and return it as
892 // chain, then the TokenFactor is both a predecessor (operand) of the
893 // user as well as a successor (the TF operands are flagged to the user).
894 // c1, f1 = CopyToReg
895 // c2, f2 = CopyToReg
896 // c3 = TokenFactor c1, c2
897 // ...
898 // = op c3, ..., f2
899 Chain = Chains[NumRegs-1];
900 else
901 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
902}
903
904void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
905 unsigned MatchingIdx, const SDLoc &dl,
906 SelectionDAG &DAG,
907 std::vector<SDValue> &Ops) const {
908 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
909
910 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
911 if (HasMatching)
912 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
913 else if (!Regs.empty() &&
914 TargetRegisterInfo::isVirtualRegister(Regs.front())) {
915 // Put the register class of the virtual registers in the flag word. That
916 // way, later passes can recompute register class constraints for inline
917 // assembly as well as normal instructions.
918 // Don't do this for tied operands that can use the regclass information
919 // from the def.
920 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
921 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
922 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
923 }
924
925 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
926 Ops.push_back(Res);
927
928 if (Code == InlineAsm::Kind_Clobber) {
929 // Clobbers should always have a 1:1 mapping with registers, and may
930 // reference registers that have illegal (e.g. vector) types. Hence, we
931 // shouldn't try to apply any sort of splitting logic to them.
932 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 933, __extension__ __PRETTY_FUNCTION__))
933 "No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 933, __extension__ __PRETTY_FUNCTION__))
;
934 unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
935 (void)SP;
936 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
937 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
938 assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 941, __extension__ __PRETTY_FUNCTION__))
939 (Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 941, __extension__ __PRETTY_FUNCTION__))
940 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 941, __extension__ __PRETTY_FUNCTION__))
941 "If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 941, __extension__ __PRETTY_FUNCTION__))
;
942 }
943 return;
944 }
945
946 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
947 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
948 MVT RegisterVT = RegVTs[Value];
949 for (unsigned i = 0; i != NumRegs; ++i) {
950 assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected"
) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 950, __extension__ __PRETTY_FUNCTION__))
;
951 unsigned TheReg = Regs[Reg++];
952 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
953 }
954 }
955}
956
957SmallVector<std::pair<unsigned, unsigned>, 4>
958RegsForValue::getRegsAndSizes() const {
959 SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;
960 unsigned I = 0;
961 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
962 unsigned RegCount = std::get<0>(CountAndVT);
963 MVT RegisterVT = std::get<1>(CountAndVT);
964 unsigned RegisterSize = RegisterVT.getSizeInBits();
965 for (unsigned E = I + RegCount; I != E; ++I)
966 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
967 }
968 return OutVec;
969}
970
971void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
972 const TargetLibraryInfo *li) {
973 AA = aa;
974 GFI = gfi;
975 LibInfo = li;
976 DL = &DAG.getDataLayout();
977 Context = DAG.getContext();
978 LPadToCallSiteMap.clear();
979}
980
981void SelectionDAGBuilder::clear() {
982 NodeMap.clear();
983 UnusedArgNodeMap.clear();
984 PendingLoads.clear();
985 PendingExports.clear();
986 CurInst = nullptr;
987 HasTailCall = false;
988 SDNodeOrder = LowestSDNodeOrder;
989 StatepointLowering.clear();
990}
991
992void SelectionDAGBuilder::clearDanglingDebugInfo() {
993 DanglingDebugInfoMap.clear();
994}
995
996SDValue SelectionDAGBuilder::getRoot() {
997 if (PendingLoads.empty())
998 return DAG.getRoot();
999
1000 if (PendingLoads.size() == 1) {
1001 SDValue Root = PendingLoads[0];
1002 DAG.setRoot(Root);
1003 PendingLoads.clear();
1004 return Root;
1005 }
1006
1007 // Otherwise, we have to make a token factor node.
1008 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1009 PendingLoads);
1010 PendingLoads.clear();
1011 DAG.setRoot(Root);
1012 return Root;
1013}
1014
1015SDValue SelectionDAGBuilder::getControlRoot() {
1016 SDValue Root = DAG.getRoot();
1017
1018 if (PendingExports.empty())
1019 return Root;
1020
1021 // Turn all of the CopyToReg chains into one factored node.
1022 if (Root.getOpcode() != ISD::EntryToken) {
1023 unsigned i = 0, e = PendingExports.size();
1024 for (; i != e; ++i) {
1025 assert(PendingExports[i].getNode()->getNumOperands() > 1)(static_cast <bool> (PendingExports[i].getNode()->getNumOperands
() > 1) ? void (0) : __assert_fail ("PendingExports[i].getNode()->getNumOperands() > 1"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1025, __extension__ __PRETTY_FUNCTION__))
;
1026 if (PendingExports[i].getNode()->getOperand(0) == Root)
1027 break; // Don't add the root if we already indirectly depend on it.
1028 }
1029
1030 if (i == e)
1031 PendingExports.push_back(Root);
1032 }
1033
1034 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1035 PendingExports);
1036 PendingExports.clear();
1037 DAG.setRoot(Root);
1038 return Root;
1039}
1040
1041void SelectionDAGBuilder::visit(const Instruction &I) {
1042 // Set up outgoing PHI node register values before emitting the terminator.
1043 if (isa<TerminatorInst>(&I)) {
1044 HandlePHINodesInSuccessorBlocks(I.getParent());
1045 }
1046
1047 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1048 if (!isa<DbgInfoIntrinsic>(I))
1049 ++SDNodeOrder;
1050
1051 CurInst = &I;
1052
1053 visit(I.getOpcode(), I);
1054
1055 if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1056 // Propagate the fast-math-flags of this IR instruction to the DAG node that
1057 // maps to this instruction.
1058 // TODO: We could handle all flags (nsw, etc) here.
1059 // TODO: If an IR instruction maps to >1 node, only the final node will have
1060 // flags set.
1061 if (SDNode *Node = getNodeForIRValue(&I)) {
1062 SDNodeFlags IncomingFlags;
1063 IncomingFlags.copyFMF(*FPMO);
1064 if (!Node->getFlags().isDefined())
1065 Node->setFlags(IncomingFlags);
1066 else
1067 Node->intersectFlagsWith(IncomingFlags);
1068 }
1069 }
1070
1071 if (!isa<TerminatorInst>(&I) && !HasTailCall &&
1072 !isStatepoint(&I)) // statepoints handle their exports internally
1073 CopyToExportRegsIfNeeded(&I);
1074
1075 CurInst = nullptr;
1076}
1077
1078void SelectionDAGBuilder::visitPHI(const PHINode &) {
1079 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1079)
;
1080}
1081
1082void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1083 // Note: this doesn't use InstVisitor, because it has to work with
1084 // ConstantExpr's in addition to instructions.
1085 switch (Opcode) {
1086 default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1086)
;
1087 // Build the switch statement using the Instruction.def file.
1088#define HANDLE_INST(NUM, OPCODE, CLASS) \
1089 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1090#include "llvm/IR/Instruction.def"
1091 }
1092}
1093
1094void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1095 const DIExpression *Expr) {
1096 for (auto &DDIMI : DanglingDebugInfoMap)
1097 for (auto &DDI : DDIMI.second)
1098 if (DDI.getDI()) {
1099 const DbgValueInst *DI = DDI.getDI();
1100 DIVariable *DanglingVariable = DI->getVariable();
1101 DIExpression *DanglingExpr = DI->getExpression();
1102 if (DanglingVariable == Variable &&
1103 Expr->fragmentsOverlap(DanglingExpr)) {
1104 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< *DI << "\n"; } } while (false)
1105 << "Dropping dangling debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< *DI << "\n"; } } while (false)
;
1106 DDI = DanglingDebugInfo();
1107 }
1108 }
1109}
1110
1111// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1112// generate the debug data structures now that we've seen its definition.
1113void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1114 SDValue Val) {
1115 DanglingDebugInfoVector &DDIV = DanglingDebugInfoMap[V];
1116 for (auto &DDI : DDIV) {
1117 if (!DDI.getDI())
1118 continue;
1119 const DbgValueInst *DI = DDI.getDI();
1120 DebugLoc dl = DDI.getdl();
1121 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1122 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1123 DILocalVariable *Variable = DI->getVariable();
1124 DIExpression *Expr = DI->getExpression();
1125 assert(Variable->isValidLocationForIntrinsic(dl) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1126, __extension__ __PRETTY_FUNCTION__))
1126 "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1126, __extension__ __PRETTY_FUNCTION__))
;
1127 SDDbgValue *SDV;
1128 if (Val.getNode()) {
1129 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1130 LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false)
1131 << DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false)
;
1132 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " By mapping to:\n "; Val.dump
(); } } while (false)
;
1133 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1134 // inserted after the definition of Val when emitting the instructions
1135 // after ISel. An alternative could be to teach
1136 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1137 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)
1138 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)
1139 << ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)
;
1140 SDV = getDbgValue(Val, Variable, Expr, dl,
1141 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1142 DAG.AddDbgValue(SDV, Val.getNode(), false);
1143 } else
1144 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false)
1145 << "in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false)
;
1146 } else
1147 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false)
;
1148 }
1149 DanglingDebugInfoMap[V].clear();
1150}
1151
1152/// getCopyFromRegs - If there was virtual register allocated for the value V
1153/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1154SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1155 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1156 SDValue Result;
1157
1158 if (It != FuncInfo.ValueMap.end()) {
1159 unsigned InReg = It->second;
1160
1161 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1162 DAG.getDataLayout(), InReg, Ty, isABIRegCopy(V));
1163 SDValue Chain = DAG.getEntryNode();
1164 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1165 V);
1166 resolveDanglingDebugInfo(V, Result);
1167 }
1168
1169 return Result;
1170}
1171
1172/// getValue - Return an SDValue for the given Value.
1173SDValue SelectionDAGBuilder::getValue(const Value *V) {
1174 // If we already have an SDValue for this value, use it. It's important
1175 // to do this first, so that we don't create a CopyFromReg if we already
1176 // have a regular SDValue.
1177 SDValue &N = NodeMap[V];
1178 if (N.getNode()) return N;
1179
1180 // If there's a virtual register allocated and initialized for this
1181 // value, use it.
1182 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1183 return copyFromReg;
1184
1185 // Otherwise create a new SDValue and remember it.
1186 SDValue Val = getValueImpl(V);
1187 NodeMap[V] = Val;
1188 resolveDanglingDebugInfo(V, Val);
1189 return Val;
1190}
1191
1192// Return true if SDValue exists for the given Value
1193bool SelectionDAGBuilder::findValue(const Value *V) const {
1194 return (NodeMap.find(V) != NodeMap.end()) ||
1195 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1196}
1197
1198/// getNonRegisterValue - Return an SDValue for the given Value, but
1199/// don't look in FuncInfo.ValueMap for a virtual register.
1200SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1201 // If we already have an SDValue for this value, use it.
1202 SDValue &N = NodeMap[V];
1203 if (N.getNode()) {
1204 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1205 // Remove the debug location from the node as the node is about to be used
1206 // in a location which may differ from the original debug location. This
1207 // is relevant to Constant and ConstantFP nodes because they can appear
1208 // as constant expressions inside PHI nodes.
1209 N->setDebugLoc(DebugLoc());
1210 }
1211 return N;
1212 }
1213
1214 // Otherwise create a new SDValue and remember it.
1215 SDValue Val = getValueImpl(V);
1216 NodeMap[V] = Val;
1217 resolveDanglingDebugInfo(V, Val);
1218 return Val;
1219}
1220
1221/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1222/// Create an SDValue for the given value.
1223SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1224 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1225
1226 if (const Constant *C = dyn_cast<Constant>(V)) {
1227 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1228
1229 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1230 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1231
1232 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1233 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1234
1235 if (isa<ConstantPointerNull>(C)) {
1236 unsigned AS = V->getType()->getPointerAddressSpace();
1237 return DAG.getConstant(0, getCurSDLoc(),
1238 TLI.getPointerTy(DAG.getDataLayout(), AS));
1239 }
1240
1241 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1242 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1243
1244 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1245 return DAG.getUNDEF(VT);
1246
1247 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1248 visit(CE->getOpcode(), *CE);
1249 SDValue N1 = NodeMap[V];
1250 assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!"
) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1250, __extension__ __PRETTY_FUNCTION__))
;
1251 return N1;
1252 }
1253
1254 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1255 SmallVector<SDValue, 4> Constants;
1256 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1257 OI != OE; ++OI) {
1258 SDNode *Val = getValue(*OI).getNode();
1259 // If the operand is an empty aggregate, there are no values.
1260 if (!Val) continue;
1261 // Add each leaf value from the operand to the Constants list
1262 // to form a flattened list of all the values.
1263 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1264 Constants.push_back(SDValue(Val, i));
1265 }
1266
1267 return DAG.getMergeValues(Constants, getCurSDLoc());
1268 }
1269
1270 if (const ConstantDataSequential *CDS =
1271 dyn_cast<ConstantDataSequential>(C)) {
1272 SmallVector<SDValue, 4> Ops;
1273 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1274 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1275 // Add each leaf value from the operand to the Constants list
1276 // to form a flattened list of all the values.
1277 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1278 Ops.push_back(SDValue(Val, i));
1279 }
1280
1281 if (isa<ArrayType>(CDS->getType()))
1282 return DAG.getMergeValues(Ops, getCurSDLoc());
1283 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1284 }
1285
1286 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1287 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1288, __extension__ __PRETTY_FUNCTION__))
1288 "Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1288, __extension__ __PRETTY_FUNCTION__))
;
1289
1290 SmallVector<EVT, 4> ValueVTs;
1291 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1292 unsigned NumElts = ValueVTs.size();
1293 if (NumElts == 0)
1294 return SDValue(); // empty struct
1295 SmallVector<SDValue, 4> Constants(NumElts);
1296 for (unsigned i = 0; i != NumElts; ++i) {
1297 EVT EltVT = ValueVTs[i];
1298 if (isa<UndefValue>(C))
1299 Constants[i] = DAG.getUNDEF(EltVT);
1300 else if (EltVT.isFloatingPoint())
1301 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1302 else
1303 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1304 }
1305
1306 return DAG.getMergeValues(Constants, getCurSDLoc());
1307 }
1308
1309 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1310 return DAG.getBlockAddress(BA, VT);
1311
1312 VectorType *VecTy = cast<VectorType>(V->getType());
1313 unsigned NumElements = VecTy->getNumElements();
1314
1315 // Now that we know the number and type of the elements, get that number of
1316 // elements into the Ops array based on what kind of constant it is.
1317 SmallVector<SDValue, 16> Ops;
1318 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1319 for (unsigned i = 0; i != NumElements; ++i)
1320 Ops.push_back(getValue(CV->getOperand(i)));
1321 } else {
1322 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!")(static_cast <bool> (isa<ConstantAggregateZero>(C
) && "Unknown vector constant!") ? void (0) : __assert_fail
("isa<ConstantAggregateZero>(C) && \"Unknown vector constant!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1322, __extension__ __PRETTY_FUNCTION__))
;
1323 EVT EltVT =
1324 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1325
1326 SDValue Op;
1327 if (EltVT.isFloatingPoint())
1328 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1329 else
1330 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1331 Ops.assign(NumElements, Op);
1332 }
1333
1334 // Create a BUILD_VECTOR node.
1335 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1336 }
1337
1338 // If this is a static alloca, generate it as the frameindex instead of
1339 // computation.
1340 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1341 DenseMap<const AllocaInst*, int>::iterator SI =
1342 FuncInfo.StaticAllocaMap.find(AI);
1343 if (SI != FuncInfo.StaticAllocaMap.end())
1344 return DAG.getFrameIndex(SI->second,
1345 TLI.getFrameIndexTy(DAG.getDataLayout()));
1346 }
1347
1348 // If this is an instruction which fast-isel has deferred, select it now.
1349 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1350 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1351
1352 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1353 Inst->getType(), isABIRegCopy(V));
1354 SDValue Chain = DAG.getEntryNode();
1355 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1356 }
1357
1358 llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1358)
;
1359}
1360
1361void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1362 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1363 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1364 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1365 bool IsSEH = isAsynchronousEHPersonality(Pers);
1366 bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1367 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1368 if (!IsSEH)
1369 CatchPadMBB->setIsEHScopeEntry();
1370 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1371 if (IsMSVCCXX || IsCoreCLR)
1372 CatchPadMBB->setIsEHFuncletEntry();
1373 // Wasm does not need catchpads anymore
1374 if (!IsWasmCXX)
1375 DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1376 getControlRoot()));
1377}
1378
1379void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1380 // Update machine-CFG edge.
1381 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1382 FuncInfo.MBB->addSuccessor(TargetMBB);
1383
1384 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1385 bool IsSEH = isAsynchronousEHPersonality(Pers);
1386 if (IsSEH) {
1387 // If this is not a fall-through branch or optimizations are switched off,
1388 // emit the branch.
1389 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1390 TM.getOptLevel() == CodeGenOpt::None)
1391 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1392 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1393 return;
1394 }
1395
1396 // Figure out the funclet membership for the catchret's successor.
1397 // This will be used by the FuncletLayout pass to determine how to order the
1398 // BB's.
1399 // A 'catchret' returns to the outer scope's color.
1400 Value *ParentPad = I.getCatchSwitchParentPad();
1401 const BasicBlock *SuccessorColor;
1402 if (isa<ConstantTokenNone>(ParentPad))
1403 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1404 else
1405 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1406 assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!"
) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1406, __extension__ __PRETTY_FUNCTION__))
;
1407 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1408 assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!"
) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1408, __extension__ __PRETTY_FUNCTION__))
;
1409
1410 // Create the terminator node.
1411 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1412 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1413 DAG.getBasicBlock(SuccessorColorMBB));
1414 DAG.setRoot(Ret);
1415}
1416
1417void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1418 // Don't emit any special code for the cleanuppad instruction. It just marks
1419 // the start of an EH scope/funclet.
1420 FuncInfo.MBB->setIsEHScopeEntry();
1421 FuncInfo.MBB->setIsEHFuncletEntry();
1422 FuncInfo.MBB->setIsCleanupFuncletEntry();
1423}
1424
1425/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1426/// many places it could ultimately go. In the IR, we have a single unwind
1427/// destination, but in the machine CFG, we enumerate all the possible blocks.
1428/// This function skips over imaginary basic blocks that hold catchswitch
1429/// instructions, and finds all the "real" machine
1430/// basic block destinations. As those destinations may not be successors of
1431/// EHPadBB, here we also calculate the edge probability to those destinations.
1432/// The passed-in Prob is the edge probability to EHPadBB.
1433static void findUnwindDestinations(
1434 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1435 BranchProbability Prob,
1436 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1437 &UnwindDests) {
1438 EHPersonality Personality =
1439 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1440 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1441 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1442 bool IsSEH = isAsynchronousEHPersonality(Personality);
1443
1444 while (EHPadBB) {
1445 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1446 BasicBlock *NewEHPadBB = nullptr;
1447 if (isa<LandingPadInst>(Pad)) {
1448 // Stop on landingpads. They are not funclets.
1449 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1450 break;
1451 } else if (isa<CleanupPadInst>(Pad)) {
1452 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1453 // personalities.
1454 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1455 UnwindDests.back().first->setIsEHScopeEntry();
1456 UnwindDests.back().first->setIsEHFuncletEntry();
1457 break;
1458 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1459 // Add the catchpad handlers to the possible destinations.
1460 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1461 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1462 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1463 if (IsMSVCCXX || IsCoreCLR)
1464 UnwindDests.back().first->setIsEHFuncletEntry();
1465 if (!IsSEH)
1466 UnwindDests.back().first->setIsEHScopeEntry();
1467 }
1468 NewEHPadBB = CatchSwitch->getUnwindDest();
1469 } else {
1470 continue;
1471 }
1472
1473 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1474 if (BPI && NewEHPadBB)
1475 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1476 EHPadBB = NewEHPadBB;
1477 }
1478}
1479
1480void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1481 // Update successor info.
1482 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1483 auto UnwindDest = I.getUnwindDest();
1484 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1485 BranchProbability UnwindDestProb =
1486 (BPI && UnwindDest)
1487 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1488 : BranchProbability::getZero();
1489 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1490 for (auto &UnwindDest : UnwindDests) {
1491 UnwindDest.first->setIsEHPad();
1492 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1493 }
1494 FuncInfo.MBB->normalizeSuccProbs();
1495
1496 // Create the terminator node.
1497 SDValue Ret =
1498 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1499 DAG.setRoot(Ret);
1500}
1501
1502void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1503 report_fatal_error("visitCatchSwitch not yet implemented!");
1504}
1505
1506void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1507 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1508 auto &DL = DAG.getDataLayout();
1509 SDValue Chain = getControlRoot();
1510 SmallVector<ISD::OutputArg, 8> Outs;
1511 SmallVector<SDValue, 8> OutVals;
1512
1513 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1514 // lower
1515 //
1516 // %val = call <ty> @llvm.experimental.deoptimize()
1517 // ret <ty> %val
1518 //
1519 // differently.
1520 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1521 LowerDeoptimizingReturn();
1522 return;
1523 }
1524
1525 if (!FuncInfo.CanLowerReturn) {
1526 unsigned DemoteReg = FuncInfo.DemoteRegister;
1527 const Function *F = I.getParent()->getParent();
1528
1529 // Emit a store of the return value through the virtual register.
1530 // Leave Outs empty so that LowerReturn won't try to load return
1531 // registers the usual way.
1532 SmallVector<EVT, 1> PtrValueVTs;
1533 ComputeValueVTs(TLI, DL,
1534 F->getReturnType()->getPointerTo(
1535 DAG.getDataLayout().getAllocaAddrSpace()),
1536 PtrValueVTs);
1537
1538 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1539 DemoteReg, PtrValueVTs[0]);
1540 SDValue RetOp = getValue(I.getOperand(0));
1541
1542 SmallVector<EVT, 4> ValueVTs;
1543 SmallVector<uint64_t, 4> Offsets;
1544 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1545 unsigned NumValues = ValueVTs.size();
1546
1547 SmallVector<SDValue, 4> Chains(NumValues);
1548 for (unsigned i = 0; i != NumValues; ++i) {
1549 // An aggregate return value cannot wrap around the address space, so
1550 // offsets to its parts don't wrap either.
1551 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1552 Chains[i] = DAG.getStore(
1553 Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1554 // FIXME: better loc info would be nice.
1555 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1556 }
1557
1558 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1559 MVT::Other, Chains);
1560 } else if (I.getNumOperands() != 0) {
1561 SmallVector<EVT, 4> ValueVTs;
1562 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1563 unsigned NumValues = ValueVTs.size();
1564 if (NumValues) {
1565 SDValue RetOp = getValue(I.getOperand(0));
1566
1567 const Function *F = I.getParent()->getParent();
1568
1569 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1570 if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1571 Attribute::SExt))
1572 ExtendKind = ISD::SIGN_EXTEND;
1573 else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1574 Attribute::ZExt))
1575 ExtendKind = ISD::ZERO_EXTEND;
1576
1577 LLVMContext &Context = F->getContext();
1578 bool RetInReg = F->getAttributes().hasAttribute(
1579 AttributeList::ReturnIndex, Attribute::InReg);
1580
1581 for (unsigned j = 0; j != NumValues; ++j) {
1582 EVT VT = ValueVTs[j];
1583
1584 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1585 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1586
1587 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, VT);
1588 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, VT);
1589 SmallVector<SDValue, 4> Parts(NumParts);
1590 getCopyToParts(DAG, getCurSDLoc(),
1591 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1592 &Parts[0], NumParts, PartVT, &I, ExtendKind, true);
1593
1594 // 'inreg' on function refers to return value
1595 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1596 if (RetInReg)
1597 Flags.setInReg();
1598
1599 // Propagate extension type if any
1600 if (ExtendKind == ISD::SIGN_EXTEND)
1601 Flags.setSExt();
1602 else if (ExtendKind == ISD::ZERO_EXTEND)
1603 Flags.setZExt();
1604
1605 for (unsigned i = 0; i < NumParts; ++i) {
1606 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1607 VT, /*isfixed=*/true, 0, 0));
1608 OutVals.push_back(Parts[i]);
1609 }
1610 }
1611 }
1612 }
1613
1614 // Push in swifterror virtual register as the last element of Outs. This makes
1615 // sure swifterror virtual register will be returned in the swifterror
1616 // physical register.
1617 const Function *F = I.getParent()->getParent();
1618 if (TLI.supportSwiftError() &&
1619 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1620 assert(FuncInfo.SwiftErrorArg && "Need a swift error argument")(static_cast <bool> (FuncInfo.SwiftErrorArg && "Need a swift error argument"
) ? void (0) : __assert_fail ("FuncInfo.SwiftErrorArg && \"Need a swift error argument\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1620, __extension__ __PRETTY_FUNCTION__))
;
1621 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1622 Flags.setSwiftError();
1623 Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1624 EVT(TLI.getPointerTy(DL)) /*argvt*/,
1625 true /*isfixed*/, 1 /*origidx*/,
1626 0 /*partOffs*/));
1627 // Create SDNode for the swifterror virtual register.
1628 OutVals.push_back(
1629 DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1630 &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1631 EVT(TLI.getPointerTy(DL))));
1632 }
1633
1634 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1635 CallingConv::ID CallConv =
1636 DAG.getMachineFunction().getFunction().getCallingConv();
1637 Chain = DAG.getTargetLoweringInfo().LowerReturn(
1638 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1639
1640 // Verify that the target's LowerReturn behaved as expected.
1641 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1642, __extension__ __PRETTY_FUNCTION__))
1642 "LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1642, __extension__ __PRETTY_FUNCTION__))
;
1643
1644 // Update the DAG with the new chain value resulting from return lowering.
1645 DAG.setRoot(Chain);
1646}
1647
1648/// CopyToExportRegsIfNeeded - If the given value has virtual registers
1649/// created for it, emit nodes to copy the value into the virtual
1650/// registers.
1651void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1652 // Skip empty types
1653 if (V->getType()->isEmptyTy())
1654 return;
1655
1656 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1657 if (VMI != FuncInfo.ValueMap.end()) {
1658 assert(!V->use_empty() && "Unused value assigned virtual registers!")(static_cast <bool> (!V->use_empty() && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1658, __extension__ __PRETTY_FUNCTION__))
;
1659 CopyValueToVirtualRegister(V, VMI->second);
1660 }
1661}
1662
1663/// ExportFromCurrentBlock - If this condition isn't known to be exported from
1664/// the current basic block, add it to ValueMap now so that we'll get a
1665/// CopyTo/FromReg.
1666void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1667 // No need to export constants.
1668 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1669
1670 // Already exported?
1671 if (FuncInfo.isExportedInst(V)) return;
1672
1673 unsigned Reg = FuncInfo.InitializeRegForValue(V);
1674 CopyValueToVirtualRegister(V, Reg);
1675}
1676
1677bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1678 const BasicBlock *FromBB) {
1679 // The operands of the setcc have to be in this block. We don't know
1680 // how to export them from some other block.
1681 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1682 // Can export from current BB.
1683 if (VI->getParent() == FromBB)
1684 return true;
1685
1686 // Is already exported, noop.
1687 return FuncInfo.isExportedInst(V);
1688 }
1689
1690 // If this is an argument, we can export it if the BB is the entry block or
1691 // if it is already exported.
1692 if (isa<Argument>(V)) {
1693 if (FromBB == &FromBB->getParent()->getEntryBlock())
1694 return true;
1695
1696 // Otherwise, can only export this if it is already exported.
1697 return FuncInfo.isExportedInst(V);
1698 }
1699
1700 // Otherwise, constants can always be exported.
1701 return true;
1702}
1703
1704/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1705BranchProbability
1706SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1707 const MachineBasicBlock *Dst) const {
1708 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1709 const BasicBlock *SrcBB = Src->getBasicBlock();
1710 const BasicBlock *DstBB = Dst->getBasicBlock();
1711 if (!BPI) {
1712 // If BPI is not available, set the default probability as 1 / N, where N is
1713 // the number of successors.
1714 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1715 return BranchProbability(1, SuccSize);
1716 }
1717 return BPI->getEdgeProbability(SrcBB, DstBB);
1718}
1719
1720void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1721 MachineBasicBlock *Dst,
1722 BranchProbability Prob) {
1723 if (!FuncInfo.BPI)
1724 Src->addSuccessorWithoutProb(Dst);
1725 else {
1726 if (Prob.isUnknown())
1727 Prob = getEdgeProbability(Src, Dst);
1728 Src->addSuccessor(Dst, Prob);
1729 }
1730}
1731
1732static bool InBlock(const Value *V, const BasicBlock *BB) {
1733 if (const Instruction *I = dyn_cast<Instruction>(V))
1734 return I->getParent() == BB;
1735 return true;
1736}
1737
1738/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1739/// This function emits a branch and is used at the leaves of an OR or an
1740/// AND operator tree.
1741void
1742SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1743 MachineBasicBlock *TBB,
1744 MachineBasicBlock *FBB,
1745 MachineBasicBlock *CurBB,
1746 MachineBasicBlock *SwitchBB,
1747 BranchProbability TProb,
1748 BranchProbability FProb,
1749 bool InvertCond) {
1750 const BasicBlock *BB = CurBB->getBasicBlock();
1751
1752 // If the leaf of the tree is a comparison, merge the condition into
1753 // the caseblock.
1754 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1755 // The operands of the cmp have to be in this block. We don't know
1756 // how to export them from some other block. If this is the first block
1757 // of the sequence, no exporting is needed.
1758 if (CurBB == SwitchBB ||
1759 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1760 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1761 ISD::CondCode Condition;
1762 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1763 ICmpInst::Predicate Pred =
1764 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1765 Condition = getICmpCondCode(Pred);
1766 } else {
1767 const FCmpInst *FC = cast<FCmpInst>(Cond);
1768 FCmpInst::Predicate Pred =
1769 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1770 Condition = getFCmpCondCode(Pred);
1771 if (TM.Options.NoNaNsFPMath)
1772 Condition = getFCmpCodeWithoutNaN(Condition);
1773 }
1774
1775 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1776 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1777 SwitchCases.push_back(CB);
1778 return;
1779 }
1780 }
1781
1782 // Create a CaseBlock record representing this branch.
1783 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1784 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1785 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1786 SwitchCases.push_back(CB);
1787}
1788
1789/// FindMergedConditions - If Cond is an expression like
1790void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1791 MachineBasicBlock *TBB,
1792 MachineBasicBlock *FBB,
1793 MachineBasicBlock *CurBB,
1794 MachineBasicBlock *SwitchBB,
1795 Instruction::BinaryOps Opc,
1796 BranchProbability TProb,
1797 BranchProbability FProb,
1798 bool InvertCond) {
1799 // Skip over not part of the tree and remember to invert op and operands at
1800 // next level.
1801 if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1802 const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1803 if (InBlock(CondOp, CurBB->getBasicBlock())) {
1804 FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1805 !InvertCond);
1806 return;
1807 }
1808 }
1809
1810 const Instruction *BOp = dyn_cast<Instruction>(Cond);
1811 // Compute the effective opcode for Cond, taking into account whether it needs
1812 // to be inverted, e.g.
1813 // and (not (or A, B)), C
1814 // gets lowered as
1815 // and (and (not A, not B), C)
1816 unsigned BOpc = 0;
1817 if (BOp) {
1818 BOpc = BOp->getOpcode();
1819 if (InvertCond) {
1820 if (BOpc == Instruction::And)
1821 BOpc = Instruction::Or;
1822 else if (BOpc == Instruction::Or)
1823 BOpc = Instruction::And;
1824 }
1825 }
1826
1827 // If this node is not part of the or/and tree, emit it as a branch.
1828 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1829 BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
1830 BOp->getParent() != CurBB->getBasicBlock() ||
1831 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1832 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1833 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1834 TProb, FProb, InvertCond);
1835 return;
1836 }
1837
1838 // Create TmpBB after CurBB.
1839 MachineFunction::iterator BBI(CurBB);
1840 MachineFunction &MF = DAG.getMachineFunction();
1841 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1842 CurBB->getParent()->insert(++BBI, TmpBB);
1843
1844 if (Opc == Instruction::Or) {
1845 // Codegen X | Y as:
1846 // BB1:
1847 // jmp_if_X TBB
1848 // jmp TmpBB
1849 // TmpBB:
1850 // jmp_if_Y TBB
1851 // jmp FBB
1852 //
1853
1854 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1855 // The requirement is that
1856 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1857 // = TrueProb for original BB.
1858 // Assuming the original probabilities are A and B, one choice is to set
1859 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1860 // A/(1+B) and 2B/(1+B). This choice assumes that
1861 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1862 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1863 // TmpBB, but the math is more complicated.
1864
1865 auto NewTrueProb = TProb / 2;
1866 auto NewFalseProb = TProb / 2 + FProb;
1867 // Emit the LHS condition.
1868 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1869 NewTrueProb, NewFalseProb, InvertCond);
1870
1871 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1872 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1873 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1874 // Emit the RHS condition into TmpBB.
1875 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1876 Probs[0], Probs[1], InvertCond);
1877 } else {
1878 assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And &&
"Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1878, __extension__ __PRETTY_FUNCTION__))
;
1879 // Codegen X & Y as:
1880 // BB1:
1881 // jmp_if_X TmpBB
1882 // jmp FBB
1883 // TmpBB:
1884 // jmp_if_Y TBB
1885 // jmp FBB
1886 //
1887 // This requires creation of TmpBB after CurBB.
1888
1889 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1890 // The requirement is that
1891 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1892 // = FalseProb for original BB.
1893 // Assuming the original probabilities are A and B, one choice is to set
1894 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1895 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1896 // TrueProb for BB1 * FalseProb for TmpBB.
1897
1898 auto NewTrueProb = TProb + FProb / 2;
1899 auto NewFalseProb = FProb / 2;
1900 // Emit the LHS condition.
1901 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1902 NewTrueProb, NewFalseProb, InvertCond);
1903
1904 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1905 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1906 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1907 // Emit the RHS condition into TmpBB.
1908 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1909 Probs[0], Probs[1], InvertCond);
1910 }
1911}
1912
1913/// If the set of cases should be emitted as a series of branches, return true.
1914/// If we should emit this as a bunch of and/or'd together conditions, return
1915/// false.
1916bool
1917SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1918 if (Cases.size() != 2) return true;
1919
1920 // If this is two comparisons of the same values or'd or and'd together, they
1921 // will get folded into a single comparison, so don't emit two blocks.
1922 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1923 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1924 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1925 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1926 return false;
1927 }
1928
1929 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1930 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1931 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1932 Cases[0].CC == Cases[1].CC &&
1933 isa<Constant>(Cases[0].CmpRHS) &&
1934 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1935 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1936 return false;
1937 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1938 return false;
1939 }
1940
1941 return true;
1942}
1943
1944void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1945 MachineBasicBlock *BrMBB = FuncInfo.MBB;
1946
1947 // Update machine-CFG edges.
1948 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1949
1950 if (I.isUnconditional()) {
1951 // Update machine-CFG edges.
1952 BrMBB->addSuccessor(Succ0MBB);
1953
1954 // If this is not a fall-through branch or optimizations are switched off,
1955 // emit the branch.
1956 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1957 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1958 MVT::Other, getControlRoot(),
1959 DAG.getBasicBlock(Succ0MBB)));
1960
1961 return;
1962 }
1963
1964 // If this condition is one of the special cases we handle, do special stuff
1965 // now.
1966 const Value *CondVal = I.getCondition();
1967 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1968
1969 // If this is a series of conditions that are or'd or and'd together, emit
1970 // this as a sequence of branches instead of setcc's with and/or operations.
1971 // As long as jumps are not expensive, this should improve performance.
1972 // For example, instead of something like:
1973 // cmp A, B
1974 // C = seteq
1975 // cmp D, E
1976 // F = setle
1977 // or C, F
1978 // jnz foo
1979 // Emit:
1980 // cmp A, B
1981 // je foo
1982 // cmp D, E
1983 // jle foo
1984 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1985 Instruction::BinaryOps Opcode = BOp->getOpcode();
1986 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
1987 !I.getMetadata(LLVMContext::MD_unpredictable) &&
1988 (Opcode == Instruction::And || Opcode == Instruction::Or)) {
1989 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1990 Opcode,
1991 getEdgeProbability(BrMBB, Succ0MBB),
1992 getEdgeProbability(BrMBB, Succ1MBB),
1993 /*InvertCond=*/false);
1994 // If the compares in later blocks need to use values not currently
1995 // exported from this block, export them now. This block should always
1996 // be the first entry.
1997 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!") ? void (0) : __assert_fail ("SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1997, __extension__ __PRETTY_FUNCTION__))
;
1998
1999 // Allow some cases to be rejected.
2000 if (ShouldEmitAsBranches(SwitchCases)) {
2001 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2002 ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2003 ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2004 }
2005
2006 // Emit the branch for this block.
2007 visitSwitchCase(SwitchCases[0], BrMBB);
2008 SwitchCases.erase(SwitchCases.begin());
2009 return;
2010 }
2011
2012 // Okay, we decided not to do this, remove any inserted MBB's and clear
2013 // SwitchCases.
2014 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2015 FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2016
2017 SwitchCases.clear();
2018 }
2019 }
2020
2021 // Create a CaseBlock record representing this branch.
2022 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2023 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2024
2025 // Use visitSwitchCase to actually insert the fast branch sequence for this
2026 // cond branch.
2027 visitSwitchCase(CB, BrMBB);
2028}
2029
2030/// visitSwitchCase - Emits the necessary code to represent a single node in
2031/// the binary search tree resulting from lowering a switch instruction.
2032void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2033 MachineBasicBlock *SwitchBB) {
2034 SDValue Cond;
2035 SDValue CondLHS = getValue(CB.CmpLHS);
2036 SDLoc dl = CB.DL;
2037
2038 // Build the setcc now.
2039 if (!CB.CmpMHS) {
2040 // Fold "(X == true)" to X and "(X == false)" to !X to
2041 // handle common cases produced by branch lowering.
2042 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2043 CB.CC == ISD::SETEQ)
2044 Cond = CondLHS;
2045 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2046 CB.CC == ISD::SETEQ) {
2047 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2048 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2049 } else
2050 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
2051 } else {
2052 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2052, __extension__ __PRETTY_FUNCTION__))
;
2053
2054 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2055 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2056
2057 SDValue CmpOp = getValue(CB.CmpMHS);
2058 EVT VT = CmpOp.getValueType();
2059
2060 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2061 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2062 ISD::SETLE);
2063 } else {
2064 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2065 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2066 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2067 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2068 }
2069 }
2070
2071 // Update successor info
2072 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2073 // TrueBB and FalseBB are always different unless the incoming IR is
2074 // degenerate. This only happens when running llc on weird IR.
2075 if (CB.TrueBB != CB.FalseBB)
2076 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2077 SwitchBB->normalizeSuccProbs();
2078
2079 // If the lhs block is the next block, invert the condition so that we can
2080 // fall through to the lhs instead of the rhs block.
2081 if (CB.TrueBB == NextBlock(SwitchBB)) {
2082 std::swap(CB.TrueBB, CB.FalseBB);
2083 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2084 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2085 }
2086
2087 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2088 MVT::Other, getControlRoot(), Cond,
2089 DAG.getBasicBlock(CB.TrueBB));
2090
2091 // Insert the false branch. Do this even if it's a fall through branch,
2092 // this makes it easier to do DAG optimizations which require inverting
2093 // the branch condition.
2094 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2095 DAG.getBasicBlock(CB.FalseBB));
2096
2097 DAG.setRoot(BrCond);
2098}
2099
2100/// visitJumpTable - Emit JumpTable node in the current MBB
2101void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
2102 // Emit the code for the jump table
2103 assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!"
) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2103, __extension__ __PRETTY_FUNCTION__))
;
2104 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2105 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2106 JT.Reg, PTy);
2107 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2108 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2109 MVT::Other, Index.getValue(1),
2110 Table, Index);
2111 DAG.setRoot(BrJumpTable);
2112}
2113
2114/// visitJumpTableHeader - This function emits necessary code to produce index
2115/// in the JumpTable from switch case.
2116void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
2117 JumpTableHeader &JTH,
2118 MachineBasicBlock *SwitchBB) {
2119 SDLoc dl = getCurSDLoc();
2120
2121 // Subtract the lowest switch case value from the value being switched on and
2122 // conditional branch to default mbb if the result is greater than the
2123 // difference between smallest and largest cases.
2124 SDValue SwitchOp = getValue(JTH.SValue);
2125 EVT VT = SwitchOp.getValueType();
2126 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2127 DAG.getConstant(JTH.First, dl, VT));
2128
2129 // The SDNode we just created, which holds the value being switched on minus
2130 // the smallest case value, needs to be copied to a virtual register so it
2131 // can be used as an index into the jump table in a subsequent basic block.
2132 // This value may be smaller or larger than the target's pointer type, and
2133 // therefore require extension or truncating.
2134 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2135 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2136
2137 unsigned JumpTableReg =
2138 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2139 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2140 JumpTableReg, SwitchOp);
2141 JT.Reg = JumpTableReg;
2142
2143 // Emit the range check for the jump table, and branch to the default block
2144 // for the switch statement if the value being switched on exceeds the largest
2145 // case in the switch.
2146 SDValue CMP = DAG.getSetCC(
2147 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2148 Sub.getValueType()),
2149 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2150
2151 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2152 MVT::Other, CopyTo, CMP,
2153 DAG.getBasicBlock(JT.Default));
2154
2155 // Avoid emitting unnecessary branches to the next block.
2156 if (JT.MBB != NextBlock(SwitchBB))
2157 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2158 DAG.getBasicBlock(JT.MBB));
2159
2160 DAG.setRoot(BrCond);
2161}
2162
2163/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2164/// variable if there exists one.
2165static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2166 SDValue &Chain) {
2167 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2168 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2169 MachineFunction &MF = DAG.getMachineFunction();
2170 Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2171 MachineSDNode *Node =
2172 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2173 if (Global) {
2174 MachinePointerInfo MPInfo(Global);
2175 MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
2176 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2177 MachineMemOperand::MODereferenceable;
2178 *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
2179 DAG.getEVTAlignment(PtrTy));
2180 Node->setMemRefs(MemRefs, MemRefs + 1);
2181 }
2182 return SDValue(Node, 0);
2183}
2184
2185/// Codegen a new tail for a stack protector check ParentMBB which has had its
2186/// tail spliced into a stack protector check success bb.
2187///
2188/// For a high level explanation of how this fits into the stack protector
2189/// generation see the comment on the declaration of class
2190/// StackProtectorDescriptor.
2191void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2192 MachineBasicBlock *ParentBB) {
2193
2194 // First create the loads to the guard/stack slot for the comparison.
2195 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2196 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2197
2198 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2199 int FI = MFI.getStackProtectorIndex();
2200
2201 SDValue Guard;
2202 SDLoc dl = getCurSDLoc();
2203 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2204 const Module &M = *ParentBB->getParent()->getFunction().getParent();
2205 unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2206
2207 // Generate code to load the content of the guard slot.
2208 SDValue GuardVal = DAG.getLoad(
2209 PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2210 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2211 MachineMemOperand::MOVolatile);
2212
2213 if (TLI.useStackGuardXorFP())
2214 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2215
2216 // Retrieve guard check function, nullptr if instrumentation is inlined.
2217 if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2218 // The target provides a guard check function to validate the guard value.
2219 // Generate a call to that function with the content of the guard slot as
2220 // argument.
2221 auto *Fn = cast<Function>(GuardCheck);
2222 FunctionType *FnTy = Fn->getFunctionType();
2223 assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 &&
"Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2223, __extension__ __PRETTY_FUNCTION__))
;
2224
2225 TargetLowering::ArgListTy Args;
2226 TargetLowering::ArgListEntry Entry;
2227 Entry.Node = GuardVal;
2228 Entry.Ty = FnTy->getParamType(0);
2229 if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2230 Entry.IsInReg = true;
2231 Args.push_back(Entry);
2232
2233 TargetLowering::CallLoweringInfo CLI(DAG);
2234 CLI.setDebugLoc(getCurSDLoc())
2235 .setChain(DAG.getEntryNode())
2236 .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2237 getValue(GuardCheck), std::move(Args));
2238
2239 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2240 DAG.setRoot(Result.second);
2241 return;
2242 }
2243
2244 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2245 // Otherwise, emit a volatile load to retrieve the stack guard value.
2246 SDValue Chain = DAG.getEntryNode();
2247 if (TLI.useLoadStackGuardNode()) {
2248 Guard = getLoadStackGuard(DAG, dl, Chain);
2249 } else {
2250 const Value *IRGuard = TLI.getSDagStackGuard(M);
2251 SDValue GuardPtr = getValue(IRGuard);
2252
2253 Guard =
2254 DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2255 Align, MachineMemOperand::MOVolatile);
2256 }
2257
2258 // Perform the comparison via a subtract/getsetcc.
2259 EVT VT = Guard.getValueType();
2260 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2261
2262 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2263 *DAG.getContext(),
2264 Sub.getValueType()),
2265 Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2266
2267 // If the sub is not 0, then we know the guard/stackslot do not equal, so
2268 // branch to failure MBB.
2269 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2270 MVT::Other, GuardVal.getOperand(0),
2271 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2272 // Otherwise branch to success MBB.
2273 SDValue Br = DAG.getNode(ISD::BR, dl,
2274 MVT::Other, BrCond,
2275 DAG.getBasicBlock(SPD.getSuccessMBB()));
2276
2277 DAG.setRoot(Br);
2278}
2279
2280/// Codegen the failure basic block for a stack protector check.
2281///
2282/// A failure stack protector machine basic block consists simply of a call to
2283/// __stack_chk_fail().
2284///
2285/// For a high level explanation of how this fits into the stack protector
2286/// generation see the comment on the declaration of class
2287/// StackProtectorDescriptor.
2288void
2289SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2290 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2291 SDValue Chain =
2292 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2293 None, false, getCurSDLoc(), false, false).second;
2294 DAG.setRoot(Chain);
2295}
2296
2297/// visitBitTestHeader - This function emits necessary code to produce value
2298/// suitable for "bit tests"
2299void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2300 MachineBasicBlock *SwitchBB) {
2301 SDLoc dl = getCurSDLoc();
2302
2303 // Subtract the minimum value
2304 SDValue SwitchOp = getValue(B.SValue);
2305 EVT VT = SwitchOp.getValueType();
2306 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2307 DAG.getConstant(B.First, dl, VT));
2308
2309 // Check range
2310 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2311 SDValue RangeCmp = DAG.getSetCC(
2312 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2313 Sub.getValueType()),
2314 Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2315
2316 // Determine the type of the test operands.
2317 bool UsePtrType = false;
2318 if (!TLI.isTypeLegal(VT))
2319 UsePtrType = true;
2320 else {
2321 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2322 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2323 // Switch table case range are encoded into series of masks.
2324 // Just use pointer type, it's guaranteed to fit.
2325 UsePtrType = true;
2326 break;
2327 }
2328 }
2329 if (UsePtrType) {
2330 VT = TLI.getPointerTy(DAG.getDataLayout());
2331 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2332 }
2333
2334 B.RegVT = VT.getSimpleVT();
2335 B.Reg = FuncInfo.CreateReg(B.RegVT);
2336 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2337
2338 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2339
2340 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2341 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2342 SwitchBB->normalizeSuccProbs();
2343
2344 SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2345 MVT::Other, CopyTo, RangeCmp,
2346 DAG.getBasicBlock(B.Default));
2347
2348 // Avoid emitting unnecessary branches to the next block.
2349 if (MBB != NextBlock(SwitchBB))
2350 BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2351 DAG.getBasicBlock(MBB));
2352
2353 DAG.setRoot(BrRange);
2354}
2355
2356/// visitBitTestCase - this function produces one "bit test"
2357void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2358 MachineBasicBlock* NextMBB,
2359 BranchProbability BranchProbToNext,
2360 unsigned Reg,
2361 BitTestCase &B,
2362 MachineBasicBlock *SwitchBB) {
2363 SDLoc dl = getCurSDLoc();
2364 MVT VT = BB.RegVT;
2365 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2366 SDValue Cmp;
2367 unsigned PopCount = countPopulation(B.Mask);
2368 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2369 if (PopCount == 1) {
2370 // Testing for a single bit; just compare the shift count with what it
2371 // would need to be to shift a 1 bit in that position.
2372 Cmp = DAG.getSetCC(
2373 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2374 ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2375 ISD::SETEQ);
2376 } else if (PopCount == BB.Range) {
2377 // There is only one zero bit in the range, test for it directly.
2378 Cmp = DAG.getSetCC(
2379 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2380 ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2381 ISD::SETNE);
2382 } else {
2383 // Make desired shift
2384 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2385 DAG.getConstant(1, dl, VT), ShiftOp);
2386
2387 // Emit bit tests and jumps
2388 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2389 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2390 Cmp = DAG.getSetCC(
2391 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2392 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2393 }
2394
2395 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2396 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2397 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2398 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2399 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2400 // one as they are relative probabilities (and thus work more like weights),
2401 // and hence we need to normalize them to let the sum of them become one.
2402 SwitchBB->normalizeSuccProbs();
2403
2404 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2405 MVT::Other, getControlRoot(),
2406 Cmp, DAG.getBasicBlock(B.TargetBB));
2407
2408 // Avoid emitting unnecessary branches to the next block.
2409 if (NextMBB != NextBlock(SwitchBB))
2410 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2411 DAG.getBasicBlock(NextMBB));
2412
2413 DAG.setRoot(BrAnd);
2414}
2415
2416void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2417 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2418
2419 // Retrieve successors. Look through artificial IR level blocks like
2420 // catchswitch for successors.
2421 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2422 const BasicBlock *EHPadBB = I.getSuccessor(1);
2423
2424 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2425 // have to do anything here to lower funclet bundles.
2426 assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2428, __extension__ __PRETTY_FUNCTION__))
2427 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2428, __extension__ __PRETTY_FUNCTION__))
2428 "Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2428, __extension__ __PRETTY_FUNCTION__))
;
2429
2430 const Value *Callee(I.getCalledValue());
2431 const Function *Fn = dyn_cast<Function>(Callee);
2432 if (isa<InlineAsm>(Callee))
2433 visitInlineAsm(&I);
2434 else if (Fn && Fn->isIntrinsic()) {
2435 switch (Fn->getIntrinsicID()) {
2436 default:
2437 llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2437)
;
2438 case Intrinsic::donothing:
2439 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2440 break;
2441 case Intrinsic::experimental_patchpoint_void:
2442 case Intrinsic::experimental_patchpoint_i64:
2443 visitPatchpoint(&I, EHPadBB);
2444 break;
2445 case Intrinsic::experimental_gc_statepoint:
2446 LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2447 break;
2448 }
2449 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2450 // Currently we do not lower any intrinsic calls with deopt operand bundles.
2451 // Eventually we will support lowering the @llvm.experimental.deoptimize
2452 // intrinsic, and right now there are no plans to support other intrinsics
2453 // with deopt state.
2454 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2455 } else {
2456 LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2457 }
2458
2459 // If the value of the invoke is used outside of its defining block, make it
2460 // available as a virtual register.
2461 // We already took care of the exported value for the statepoint instruction
2462 // during call to the LowerStatepoint.
2463 if (!isStatepoint(I)) {
2464 CopyToExportRegsIfNeeded(&I);
2465 }
2466
2467 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2468 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2469 BranchProbability EHPadBBProb =
2470 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2471 : BranchProbability::getZero();
2472 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2473
2474 // Update successor info.
2475 addSuccessorWithProb(InvokeMBB, Return);
2476 for (auto &UnwindDest : UnwindDests) {
2477 UnwindDest.first->setIsEHPad();
2478 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2479 }
2480 InvokeMBB->normalizeSuccProbs();
2481
2482 // Drop into normal successor.
2483 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2484 MVT::Other, getControlRoot(),
2485 DAG.getBasicBlock(Return)));
2486}
2487
2488void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2489 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2489)
;
2490}
2491
2492void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2493 assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2494, __extension__ __PRETTY_FUNCTION__))
2494 "Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2494, __extension__ __PRETTY_FUNCTION__))
;
2495
2496 MachineBasicBlock *MBB = FuncInfo.MBB;
2497 addLandingPadInfo(LP, *MBB);
2498
2499 // If there aren't registers to copy the values into (e.g., during SjLj
2500 // exceptions), then don't bother to create these DAG nodes.
2501 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2502 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2503 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2504 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2505 return;
2506
2507 // If landingpad's return type is token type, we don't create DAG nodes
2508 // for its exception pointer and selector value. The extraction of exception
2509 // pointer or selector value from token type landingpads is not currently
2510 // supported.
2511 if (LP.getType()->isTokenTy())
2512 return;
2513
2514 SmallVector<EVT, 2> ValueVTs;
2515 SDLoc dl = getCurSDLoc();
2516 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2517 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2517, __extension__ __PRETTY_FUNCTION__))
;
2518
2519 // Get the two live-in registers as SDValues. The physregs have already been
2520 // copied into virtual registers.
2521 SDValue Ops[2];
2522 if (FuncInfo.ExceptionPointerVirtReg) {
2523 Ops[0] = DAG.getZExtOrTrunc(
2524 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2525 FuncInfo.ExceptionPointerVirtReg,
2526 TLI.getPointerTy(DAG.getDataLayout())),
2527 dl, ValueVTs[0]);
2528 } else {
2529 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2530 }
2531 Ops[1] = DAG.getZExtOrTrunc(
2532 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2533 FuncInfo.ExceptionSelectorVirtReg,
2534 TLI.getPointerTy(DAG.getDataLayout())),
2535 dl, ValueVTs[1]);
2536
2537 // Merge into one.
2538 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2539 DAG.getVTList(ValueVTs), Ops);
2540 setValue(&LP, Res);
2541}
2542
2543void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2544#ifndef NDEBUG
2545 for (const CaseCluster &CC : Clusters)
2546 assert(CC.Low == CC.High && "Input clusters must be single-case")(static_cast <bool> (CC.Low == CC.High && "Input clusters must be single-case"
) ? void (0) : __assert_fail ("CC.Low == CC.High && \"Input clusters must be single-case\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2546, __extension__ __PRETTY_FUNCTION__))
;
2547#endif
2548
2549 llvm::sort(Clusters.begin(), Clusters.end(),
2550 [](const CaseCluster &a, const CaseCluster &b) {
2551 return a.Low->getValue().slt(b.Low->getValue());
2552 });
2553
2554 // Merge adjacent clusters with the same destination.
2555 const unsigned N = Clusters.size();
2556 unsigned DstIndex = 0;
2557 for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2558 CaseCluster &CC = Clusters[SrcIndex];
2559 const ConstantInt *CaseVal = CC.Low;
2560 MachineBasicBlock *Succ = CC.MBB;
2561
2562 if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2563 (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2564 // If this case has the same successor and is a neighbour, merge it into
2565 // the previous cluster.
2566 Clusters[DstIndex - 1].High = CaseVal;
2567 Clusters[DstIndex - 1].Prob += CC.Prob;
2568 } else {
2569 std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2570 sizeof(Clusters[SrcIndex]));
2571 }
2572 }
2573 Clusters.resize(DstIndex);
2574}
2575
2576void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2577 MachineBasicBlock *Last) {
2578 // Update JTCases.
2579 for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2580 if (JTCases[i].first.HeaderBB == First)
2581 JTCases[i].first.HeaderBB = Last;
2582
2583 // Update BitTestCases.
2584 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2585 if (BitTestCases[i].Parent == First)
2586 BitTestCases[i].Parent = Last;
2587}
2588
2589void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2590 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2591
2592 // Update machine-CFG edges with unique successors.
2593 SmallSet<BasicBlock*, 32> Done;
2594 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2595 BasicBlock *BB = I.getSuccessor(i);
2596 bool Inserted = Done.insert(BB).second;
2597 if (!Inserted)
2598 continue;
2599
2600 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2601 addSuccessorWithProb(IndirectBrMBB, Succ);
2602 }
2603 IndirectBrMBB->normalizeSuccProbs();
2604
2605 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2606 MVT::Other, getControlRoot(),
2607 getValue(I.getAddress())));
2608}
2609
2610void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2611 if (!DAG.getTarget().Options.TrapUnreachable)
2612 return;
2613
2614 // We may be able to ignore unreachable behind a noreturn call.
2615 if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
2616 const BasicBlock &BB = *I.getParent();
2617 if (&I != &BB.front()) {
2618 BasicBlock::const_iterator PredI =
2619 std::prev(BasicBlock::const_iterator(&I));
2620 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2621 if (Call->doesNotReturn())
2622 return;
2623 }
2624 }
2625 }
2626
2627 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2628}
2629
2630void SelectionDAGBuilder::visitFSub(const User &I) {
2631 // -0.0 - X --> fneg
2632 Type *Ty = I.getType();
2633 if (isa<Constant>(I.getOperand(0)) &&
2634 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2635 SDValue Op2 = getValue(I.getOperand(1));
2636 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2637 Op2.getValueType(), Op2));
2638 return;
2639 }
2640
2641 visitBinary(I, ISD::FSUB);
2642}
2643
2644/// Checks if the given instruction performs a vector reduction, in which case
2645/// we have the freedom to alter the elements in the result as long as the
2646/// reduction of them stays unchanged.
2647static bool isVectorReductionOp(const User *I) {
2648 const Instruction *Inst = dyn_cast<Instruction>(I);
2649 if (!Inst || !Inst->getType()->isVectorTy())
2650 return false;
2651
2652 auto OpCode = Inst->getOpcode();
2653 switch (OpCode) {
2654 case Instruction::Add:
2655 case Instruction::Mul:
2656 case Instruction::And:
2657 case Instruction::Or:
2658 case Instruction::Xor:
2659 break;
2660 case Instruction::FAdd:
2661 case Instruction::FMul:
2662 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2663 if (FPOp->getFastMathFlags().isFast())
2664 break;
2665 LLVM_FALLTHROUGH[[clang::fallthrough]];
2666 default:
2667 return false;
2668 }
2669
2670 unsigned ElemNum = Inst->getType()->getVectorNumElements();
2671 // Ensure the reduction size is a power of 2.
2672 if (!isPowerOf2_32(ElemNum))
2673 return false;
2674
2675 unsigned ElemNumToReduce = ElemNum;
2676
2677 // Do DFS search on the def-use chain from the given instruction. We only
2678 // allow four kinds of operations during the search until we reach the
2679 // instruction that extracts the first element from the vector:
2680 //
2681 // 1. The reduction operation of the same opcode as the given instruction.
2682 //
2683 // 2. PHI node.
2684 //
2685 // 3. ShuffleVector instruction together with a reduction operation that
2686 // does a partial reduction.
2687 //
2688 // 4. ExtractElement that extracts the first element from the vector, and we
2689 // stop searching the def-use chain here.
2690 //
2691 // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2692 // from 1-3 to the stack to continue the DFS. The given instruction is not
2693 // a reduction operation if we meet any other instructions other than those
2694 // listed above.
2695
2696 SmallVector<const User *, 16> UsersToVisit{Inst};
2697 SmallPtrSet<const User *, 16> Visited;
2698 bool ReduxExtracted = false;
2699
2700 while (!UsersToVisit.empty()) {
2701 auto User = UsersToVisit.back();
2702 UsersToVisit.pop_back();
2703 if (!Visited.insert(User).second)
2704 continue;
2705
2706 for (const auto &U : User->users()) {
2707 auto Inst = dyn_cast<Instruction>(U);
2708 if (!Inst)
2709 return false;
2710
2711 if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2712 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2713 if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
2714 return false;
2715 UsersToVisit.push_back(U);
2716 } else if (const ShuffleVectorInst *ShufInst =
2717 dyn_cast<ShuffleVectorInst>(U)) {
2718 // Detect the following pattern: A ShuffleVector instruction together
2719 // with a reduction that do partial reduction on the first and second
2720 // ElemNumToReduce / 2 elements, and store the result in
2721 // ElemNumToReduce / 2 elements in another vector.
2722
2723 unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2724 if (ResultElements < ElemNum)
2725 return false;
2726
2727 if (ElemNumToReduce == 1)
2728 return false;
2729 if (!isa<UndefValue>(U->getOperand(1)))
2730 return false;
2731 for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2732 if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2733 return false;
2734 for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2735 if (ShufInst->getMaskValue(i) != -1)
2736 return false;
2737
2738 // There is only one user of this ShuffleVector instruction, which
2739 // must be a reduction operation.
2740 if (!U->hasOneUse())
2741 return false;
2742
2743 auto U2 = dyn_cast<Instruction>(*U->user_begin());
2744 if (!U2 || U2->getOpcode() != OpCode)
2745 return false;
2746
2747 // Check operands of the reduction operation.
2748 if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2749 (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2750 UsersToVisit.push_back(U2);
2751 ElemNumToReduce /= 2;
2752 } else
2753 return false;
2754 } else if (isa<ExtractElementInst>(U)) {
2755 // At this moment we should have reduced all elements in the vector.
2756 if (ElemNumToReduce != 1)
2757 return false;
2758
2759 const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2760 if (!Val || !Val->isZero())
2761 return false;
2762
2763 ReduxExtracted = true;
2764 } else
2765 return false;
2766 }
2767 }
2768 return ReduxExtracted;
2769}
2770
2771void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
2772 SDNodeFlags Flags;
2773 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
2774 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
2775 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
2776 }
2777 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
2778 Flags.setExact(ExactOp->isExact());
2779 }
2780 if (isVectorReductionOp(&I)) {
2781 Flags.setVectorReduction(true);
2782 LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Detected a reduction operation:"
<< I << "\n"; } } while (false)
;
2783 }
2784
2785 SDValue Op1 = getValue(I.getOperand(0));
2786 SDValue Op2 = getValue(I.getOperand(1));
2787 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
2788 Op1, Op2, Flags);
2789 setValue(&I, BinNodeValue);
2790}
2791
2792void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2793 SDValue Op1 = getValue(I.getOperand(0));
2794 SDValue Op2 = getValue(I.getOperand(1));
2795
2796 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2797 Op2.getValueType(), DAG.getDataLayout());
2798
2799 // Coerce the shift amount to the right type if we can.
2800 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2801 unsigned ShiftSize = ShiftTy.getSizeInBits();
2802 unsigned Op2Size = Op2.getValueSizeInBits();
2803 SDLoc DL = getCurSDLoc();
2804
2805 // If the operand is smaller than the shift count type, promote it.
2806 if (ShiftSize > Op2Size)
2807 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2808
2809 // If the operand is larger than the shift count type but the shift
2810 // count type has enough bits to represent any shift value, truncate
2811 // it now. This is a common case and it exposes the truncate to
2812 // optimization early.
2813 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2814 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2815 // Otherwise we'll need to temporarily settle for some other convenient
2816 // type. Type legalization will make adjustments once the shiftee is split.
2817 else
2818 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2819 }
2820
2821 bool nuw = false;
2822 bool nsw = false;
2823 bool exact = false;
2824
2825 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2826
2827 if (const OverflowingBinaryOperator *OFBinOp =
2828 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2829 nuw = OFBinOp->hasNoUnsignedWrap();
2830 nsw = OFBinOp->hasNoSignedWrap();
2831 }
2832 if (const PossiblyExactOperator *ExactOp =
2833 dyn_cast<const PossiblyExactOperator>(&I))
2834 exact = ExactOp->isExact();
2835 }
2836 SDNodeFlags Flags;
2837 Flags.setExact(exact);
2838 Flags.setNoSignedWrap(nsw);
2839 Flags.setNoUnsignedWrap(nuw);
2840 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2841 Flags);
2842 setValue(&I, Res);
2843}
2844
2845void SelectionDAGBuilder::visitSDiv(const User &I) {
2846 SDValue Op1 = getValue(I.getOperand(0));
2847 SDValue Op2 = getValue(I.getOperand(1));
2848
2849 SDNodeFlags Flags;
2850 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2851 cast<PossiblyExactOperator>(&I)->isExact());
2852 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2853 Op2, Flags));
2854}
2855
2856void SelectionDAGBuilder::visitICmp(const User &I) {
2857 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2858 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2859 predicate = IC->getPredicate();
2860 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2861 predicate = ICmpInst::Predicate(IC->getPredicate());
2862 SDValue Op1 = getValue(I.getOperand(0));
2863 SDValue Op2 = getValue(I.getOperand(1));
2864 ISD::CondCode Opcode = getICmpCondCode(predicate);
2865
2866 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2867 I.getType());
2868 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2869}
2870
2871void SelectionDAGBuilder::visitFCmp(const User &I) {
2872 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2873 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2874 predicate = FC->getPredicate();
2875 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2876 predicate = FCmpInst::Predicate(FC->getPredicate());
2877 SDValue Op1 = getValue(I.getOperand(0));
2878 SDValue Op2 = getValue(I.getOperand(1));
2879
2880 ISD::CondCode Condition = getFCmpCondCode(predicate);
2881 auto *FPMO = dyn_cast<FPMathOperator>(&I);
2882 if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
2883 Condition = getFCmpCodeWithoutNaN(Condition);
2884
2885 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2886 I.getType());
2887 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2888}
2889
2890// Check if the condition of the select has one use or two users that are both
2891// selects with the same condition.
2892static bool hasOnlySelectUsers(const Value *Cond) {
2893 return llvm::all_of(Cond->users(), [](const Value *V) {
2894 return isa<SelectInst>(V);
2895 });
2896}
2897
2898void SelectionDAGBuilder::visitSelect(const User &I) {
2899 SmallVector<EVT, 4> ValueVTs;
2900 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2901 ValueVTs);
2902 unsigned NumValues = ValueVTs.size();
2903 if (NumValues == 0) return;
2904
2905 SmallVector<SDValue, 4> Values(NumValues);
2906 SDValue Cond = getValue(I.getOperand(0));
2907 SDValue LHSVal = getValue(I.getOperand(1));
2908 SDValue RHSVal = getValue(I.getOperand(2));
2909 auto BaseOps = {Cond};
2910 ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2911 ISD::VSELECT : ISD::SELECT;
2912
2913 // Min/max matching is only viable if all output VTs are the same.
2914 if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2915 EVT VT = ValueVTs[0];
2916 LLVMContext &Ctx = *DAG.getContext();
2917 auto &TLI = DAG.getTargetLoweringInfo();
2918
2919 // We care about the legality of the operation after it has been type
2920 // legalized.
2921 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2922 VT != TLI.getTypeToTransformTo(Ctx, VT))
2923 VT = TLI.getTypeToTransformTo(Ctx, VT);
2924
2925 // If the vselect is legal, assume we want to leave this as a vector setcc +
2926 // vselect. Otherwise, if this is going to be scalarized, we want to see if
2927 // min/max is legal on the scalar type.
2928 bool UseScalarMinMax = VT.isVector() &&
2929 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2930
2931 Value *LHS, *RHS;
2932 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2933 ISD::NodeType Opc = ISD::DELETED_NODE;
2934 switch (SPR.Flavor) {
2935 case SPF_UMAX: Opc = ISD::UMAX; break;
2936 case SPF_UMIN: Opc = ISD::UMIN; break;
2937 case SPF_SMAX: Opc = ISD::SMAX; break;
2938 case SPF_SMIN: Opc = ISD::SMIN; break;
2939 case SPF_FMINNUM:
2940 switch (SPR.NaNBehavior) {
2941 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2941)
;
2942 case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2943 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2944 case SPNB_RETURNS_ANY: {
2945 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2946 Opc = ISD::FMINNUM;
2947 else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2948 Opc = ISD::FMINNAN;
2949 else if (UseScalarMinMax)
2950 Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2951 ISD::FMINNUM : ISD::FMINNAN;
2952 break;
2953 }
2954 }
2955 break;
2956 case SPF_FMAXNUM:
2957 switch (SPR.NaNBehavior) {
2958 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2958)
;
2959 case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2960 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2961 case SPNB_RETURNS_ANY:
2962
2963 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2964 Opc = ISD::FMAXNUM;
2965 else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2966 Opc = ISD::FMAXNAN;
2967 else if (UseScalarMinMax)
2968 Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
2969 ISD::FMAXNUM : ISD::FMAXNAN;
2970 break;
2971 }
2972 break;
2973 default: break;
2974 }
2975
2976 if (Opc != ISD::DELETED_NODE &&
2977 (TLI.isOperationLegalOrCustom(Opc, VT) ||
2978 (UseScalarMinMax &&
2979 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
2980 // If the underlying comparison instruction is used by any other
2981 // instruction, the consumed instructions won't be destroyed, so it is
2982 // not profitable to convert to a min/max.
2983 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
2984 OpCode = Opc;
2985 LHSVal = getValue(LHS);
2986 RHSVal = getValue(RHS);
2987 BaseOps = {};
2988 }
2989 }
2990
2991 for (unsigned i = 0; i != NumValues; ++i) {
2992 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2993 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2994 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2995 Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2996 LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2997 Ops);
2998 }
2999
3000 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3001 DAG.getVTList(ValueVTs), Values));
3002}
3003
3004void SelectionDAGBuilder::visitTrunc(const User &I) {
3005 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3006 SDValue N = getValue(I.getOperand(0));
3007 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3008 I.getType());
3009 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3010}
3011
3012void SelectionDAGBuilder::visitZExt(const User &I) {
3013 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3014 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3015 SDValue N = getValue(I.getOperand(0));
3016 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3017 I.getType());
3018 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3019}
3020
3021void SelectionDAGBuilder::visitSExt(const User &I) {
3022 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3023 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3024 SDValue N = getValue(I.getOperand(0));
3025 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3026 I.getType());
3027 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3028}
3029
3030void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3031 // FPTrunc is never a no-op cast, no need to check
3032 SDValue N = getValue(I.getOperand(0));
3033 SDLoc dl = getCurSDLoc();
3034 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3035 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3036 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3037 DAG.getTargetConstant(
3038 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3039}
3040
3041void SelectionDAGBuilder::visitFPExt(const User &I) {
3042 // FPExt is never a no-op cast, no need to check
3043 SDValue N = getValue(I.getOperand(0));
3044 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3045 I.getType());
3046 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3047}
3048
3049void SelectionDAGBuilder::visitFPToUI(const User &I) {
3050 // FPToUI is never a no-op cast, no need to check
3051 SDValue N = getValue(I.getOperand(0));
3052 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3053 I.getType());
3054 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3055}
3056
3057void SelectionDAGBuilder::visitFPToSI(const User &I) {
3058 // FPToSI is never a no-op cast, no need to check
3059 SDValue N = getValue(I.getOperand(0));
3060 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3061 I.getType());
3062 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3063}
3064
3065void SelectionDAGBuilder::visitUIToFP(const User &I) {
3066 // UIToFP is never a no-op cast, no need to check
3067 SDValue N = getValue(I.getOperand(0));
3068 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3069 I.getType());
3070 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3071}
3072
3073void SelectionDAGBuilder::visitSIToFP(const User &I) {
3074 // SIToFP is never a no-op cast, no need to check
3075 SDValue N = getValue(I.getOperand(0));
3076 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3077 I.getType());
3078 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3079}
3080
3081void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3082 // What to do depends on the size of the integer and the size of the pointer.
3083 // We can either truncate, zero extend, or no-op, accordingly.
3084 SDValue N = getValue(I.getOperand(0));
3085 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3086 I.getType());
3087 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3088}
3089
3090void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3091 // What to do depends on the size of the integer and the size of the pointer.
3092 // We can either truncate, zero extend, or no-op, accordingly.
3093 SDValue N = getValue(I.getOperand(0));
3094 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3095 I.getType());
3096 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3097}
3098
3099void SelectionDAGBuilder::visitBitCast(const User &I) {
3100 SDValue N = getValue(I.getOperand(0));
3101 SDLoc dl = getCurSDLoc();
3102 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3103 I.getType());
3104
3105 // BitCast assures us that source and destination are the same size so this is
3106 // either a BITCAST or a no-op.
3107 if (DestVT != N.getValueType())
3108 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3109 DestVT, N)); // convert types.
3110 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3111 // might fold any kind of constant expression to an integer constant and that
3112 // is not what we are looking for. Only recognize a bitcast of a genuine
3113 // constant integer as an opaque constant.
3114 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3115 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3116 /*isOpaque*/true));
3117 else
3118 setValue(&I, N); // noop cast.
3119}
3120
3121void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3122 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3123 const Value *SV = I.getOperand(0);
3124 SDValue N = getValue(SV);
3125 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3126
3127 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3128 unsigned DestAS = I.getType()->getPointerAddressSpace();
3129
3130 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3131 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3132
3133 setValue(&I, N);
3134}
3135
3136void SelectionDAGBuilder::visitInsertElement(const User &I) {
3137 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3138 SDValue InVec = getValue(I.getOperand(0));
3139 SDValue InVal = getValue(I.getOperand(1));
3140 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3141 TLI.getVectorIdxTy(DAG.getDataLayout()));
3142 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3143 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3144 InVec, InVal, InIdx));
3145}
3146
3147void SelectionDAGBuilder::visitExtractElement(const User &I) {
3148 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3149 SDValue InVec = getValue(I.getOperand(0));
3150 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3151 TLI.getVectorIdxTy(DAG.getDataLayout()));
3152 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3153 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3154 InVec, InIdx));
3155}
3156
3157void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3158 SDValue Src1 = getValue(I.getOperand(0));
3159 SDValue Src2 = getValue(I.getOperand(1));
3160 SDLoc DL = getCurSDLoc();
3161
3162 SmallVector<int, 8> Mask;
3163 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3164 unsigned MaskNumElts = Mask.size();
3165
3166 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3167 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3168 EVT SrcVT = Src1.getValueType();
3169 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3170
3171 if (SrcNumElts == MaskNumElts) {
3172 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3173 return;
3174 }
3175
3176 // Normalize the shuffle vector since mask and vector length don't match.
3177 if (SrcNumElts < MaskNumElts) {
3178 // Mask is longer than the source vectors. We can use concatenate vector to
3179 // make the mask and vectors lengths match.
3180
3181 if (MaskNumElts % SrcNumElts == 0) {
3182 // Mask length is a multiple of the source vector length.
3183 // Check if the shuffle is some kind of concatenation of the input
3184 // vectors.
3185 unsigned NumConcat = MaskNumElts / SrcNumElts;
3186 bool IsConcat = true;
3187 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3188 for (unsigned i = 0; i != MaskNumElts; ++i) {
3189 int Idx = Mask[i];
3190 if (Idx < 0)
3191 continue;
3192 // Ensure the indices in each SrcVT sized piece are sequential and that
3193 // the same source is used for the whole piece.
3194 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3195 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3196 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3197 IsConcat = false;
3198 break;
3199 }
3200 // Remember which source this index came from.
3201 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3202 }
3203
3204 // The shuffle is concatenating multiple vectors together. Just emit
3205 // a CONCAT_VECTORS operation.
3206 if (IsConcat) {
3207 SmallVector<SDValue, 8> ConcatOps;
3208 for (auto Src : ConcatSrcs) {
3209 if (Src < 0)
3210 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3211 else if (Src == 0)
3212 ConcatOps.push_back(Src1);
3213 else
3214 ConcatOps.push_back(Src2);
3215 }
3216 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3217 return;
3218 }
3219 }
3220
3221 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3222 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3223 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3224 PaddedMaskNumElts);
3225
3226 // Pad both vectors with undefs to make them the same length as the mask.
3227 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3228
3229 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3230 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3231 MOps1[0] = Src1;
3232 MOps2[0] = Src2;
3233
3234 Src1 = Src1.isUndef()
3235 ? DAG.getUNDEF(PaddedVT)
3236 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3237 Src2 = Src2.isUndef()
3238 ? DAG.getUNDEF(PaddedVT)
3239 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3240
3241 // Readjust mask for new input vector length.
3242 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3243 for (unsigned i = 0; i != MaskNumElts; ++i) {
3244 int Idx = Mask[i];
3245 if (Idx >= (int)SrcNumElts)
3246 Idx -= SrcNumElts - PaddedMaskNumElts;
3247 MappedOps[i] = Idx;
3248 }
3249
3250 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3251
3252 // If the concatenated vector was padded, extract a subvector with the
3253 // correct number of elements.
3254 if (MaskNumElts != PaddedMaskNumElts)
3255 Result = DAG.getNode(
3256 ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3257 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3258
3259 setValue(&I, Result);
3260 return;
3261 }
3262
3263 if (SrcNumElts > MaskNumElts) {
3264 // Analyze the access pattern of the vector to see if we can extract
3265 // two subvectors and do the shuffle.
3266 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3267 bool CanExtract = true;
3268 for (int Idx : Mask) {
3269 unsigned Input = 0;
3270 if (Idx < 0)
3271 continue;
3272
3273 if (Idx >= (int)SrcNumElts) {
3274 Input = 1;
3275 Idx -= SrcNumElts;
3276 }
3277
3278 // If all the indices come from the same MaskNumElts sized portion of
3279 // the sources we can use extract. Also make sure the extract wouldn't
3280 // extract past the end of the source.
3281 int NewStartIdx = alignDown(Idx, MaskNumElts);
3282 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3283 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3284 CanExtract = false;
3285 // Make sure we always update StartIdx as we use it to track if all
3286 // elements are undef.
3287 StartIdx[Input] = NewStartIdx;
3288 }
3289
3290 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3291 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3292 return;
3293 }
3294 if (CanExtract) {
3295 // Extract appropriate subvector and generate a vector shuffle
3296 for (unsigned Input = 0; Input < 2; ++Input) {
3297 SDValue &Src = Input == 0 ? Src1 : Src2;
3298 if (StartIdx[Input] < 0)
3299 Src = DAG.getUNDEF(VT);
3300 else {
3301 Src = DAG.getNode(
3302 ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3303 DAG.getConstant(StartIdx[Input], DL,
3304 TLI.getVectorIdxTy(DAG.getDataLayout())));
3305 }
3306 }
3307
3308 // Calculate new mask.
3309 SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3310 for (int &Idx : MappedOps) {
3311 if (Idx >= (int)SrcNumElts)
3312 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3313 else if (Idx >= 0)
3314 Idx -= StartIdx[0];
3315 }
3316
3317 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3318 return;
3319 }
3320 }
3321
3322 // We can't use either concat vectors or extract subvectors so fall back to
3323 // replacing the shuffle with extract and build vector.
3324 // to insert and build vector.
3325 EVT EltVT = VT.getVectorElementType();
3326 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3327 SmallVector<SDValue,8> Ops;
3328 for (int Idx : Mask) {
3329 SDValue Res;
3330
3331 if (Idx < 0) {
3332 Res = DAG.getUNDEF(EltVT);
3333 } else {
3334 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3335 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3336
3337 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3338 EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3339 }
3340
3341 Ops.push_back(Res);
3342 }
3343
3344 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3345}
3346
3347void SelectionDAGBuilder::visitInsertValue(const User &I) {
3348 ArrayRef<unsigned> Indices;
3349 if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3350 Indices = IV->getIndices();
3351 else
3352 Indices = cast<ConstantExpr>(&I)->getIndices();
3353
3354 const Value *Op0 = I.getOperand(0);
3355 const Value *Op1 = I.getOperand(1);
3356 Type *AggTy = I.getType();
3357 Type *ValTy = Op1->getType();
3358 bool IntoUndef = isa<UndefValue>(Op0);
3359 bool FromUndef = isa<UndefValue>(Op1);
3360
3361 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3362
3363 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3364 SmallVector<EVT, 4> AggValueVTs;
3365 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3366 SmallVector<EVT, 4> ValValueVTs;
3367 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3368
3369 unsigned NumAggValues = AggValueVTs.size();
3370 unsigned NumValValues = ValValueVTs.size();
3371 SmallVector<SDValue, 4> Values(NumAggValues);
3372
3373 // Ignore an insertvalue that produces an empty object
3374 if (!NumAggValues) {
3375 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3376 return;
3377 }
3378
3379 SDValue Agg = getValue(Op0);
3380 unsigned i = 0;
3381 // Copy the beginning value(s) from the original aggregate.
3382 for (; i != LinearIndex; ++i)
3383 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3384 SDValue(Agg.getNode(), Agg.getResNo() + i);
3385 // Copy values from the inserted value(s).
3386 if (NumValValues) {
3387 SDValue Val = getValue(Op1);
3388 for (; i != LinearIndex + NumValValues; ++i)
3389 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3390 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3391 }
3392 // Copy remaining value(s) from the original aggregate.
3393 for (; i != NumAggValues; ++i)
3394 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3395 SDValue(Agg.getNode(), Agg.getResNo() + i);
3396
3397 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3398 DAG.getVTList(AggValueVTs), Values));
3399}
3400
3401void SelectionDAGBuilder::visitExtractValue(const User &I) {
3402 ArrayRef<unsigned> Indices;
3403 if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3404 Indices = EV->getIndices();
3405 else
3406 Indices = cast<ConstantExpr>(&I)->getIndices();
3407
3408 const Value *Op0 = I.getOperand(0);
3409 Type *AggTy = Op0->getType();
3410 Type *ValTy = I.getType();
3411 bool OutOfUndef = isa<UndefValue>(Op0);
3412
3413 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3414
3415 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3416 SmallVector<EVT, 4> ValValueVTs;
3417 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3418
3419 unsigned NumValValues = ValValueVTs.size();
3420
3421 // Ignore a extractvalue that produces an empty object
3422 if (!NumValValues) {
3423 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3424 return;
3425 }
3426
3427 SmallVector<SDValue, 4> Values(NumValValues);
3428
3429 SDValue Agg = getValue(Op0);
3430 // Copy out the selected value(s).
3431 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3432 Values[i - LinearIndex] =
3433 OutOfUndef ?
3434 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3435 SDValue(Agg.getNode(), Agg.getResNo() + i);
3436
3437 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3438 DAG.getVTList(ValValueVTs), Values));
3439}
3440
3441void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3442 Value *Op0 = I.getOperand(0);
3443 // Note that the pointer operand may be a vector of pointers. Take the scalar
3444 // element which holds a pointer.
3445 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3446 SDValue N = getValue(Op0);
3447 SDLoc dl = getCurSDLoc();
3448
3449 // Normalize Vector GEP - all scalar operands should be converted to the
3450 // splat vector.
3451 unsigned VectorWidth = I.getType()->isVectorTy() ?
3452 cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3453
3454 if (VectorWidth && !N.getValueType().isVector()) {
3455 LLVMContext &Context = *DAG.getContext();
3456 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3457 N = DAG.getSplatBuildVector(VT, dl, N);
3458 }
3459
3460 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3461 GTI != E; ++GTI) {
3462 const Value *Idx = GTI.getOperand();
3463 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3464 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3465 if (Field) {
3466 // N = N + Offset
3467 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3468
3469 // In an inbounds GEP with an offset that is nonnegative even when
3470 // interpreted as signed, assume there is no unsigned overflow.
3471 SDNodeFlags Flags;
3472 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3473 Flags.setNoUnsignedWrap(true);
3474
3475 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3476 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3477 }
3478 } else {
3479 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3480 MVT IdxTy = MVT::getIntegerVT(IdxSize);
3481 APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3482
3483 // If this is a scalar constant or a splat vector of constants,
3484 // handle it quickly.
3485 const auto *CI = dyn_cast<ConstantInt>(Idx);
3486 if (!CI && isa<ConstantDataVector>(Idx) &&
3487 cast<ConstantDataVector>(Idx)->getSplatValue())
3488 CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3489
3490 if (CI) {
3491 if (CI->isZero())
3492 continue;
3493 APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3494 LLVMContext &Context = *DAG.getContext();
3495 SDValue OffsVal = VectorWidth ?
3496 DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3497 DAG.getConstant(Offs, dl, IdxTy);
3498
3499 // In an inbouds GEP with an offset that is nonnegative even when
3500 // interpreted as signed, assume there is no unsigned overflow.
3501 SDNodeFlags Flags;
3502 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3503 Flags.setNoUnsignedWrap(true);
3504
3505 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3506 continue;
3507 }
3508
3509 // N = N + Idx * ElementSize;
3510 SDValue IdxN = getValue(Idx);
3511
3512 if (!IdxN.getValueType().isVector() && VectorWidth) {
3513 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3514 IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3515 }
3516
3517 // If the index is smaller or larger than intptr_t, truncate or extend
3518 // it.
3519 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3520
3521 // If this is a multiply by a power of two, turn it into a shl
3522 // immediately. This is a very common case.
3523 if (ElementSize != 1) {
3524 if (ElementSize.isPowerOf2()) {
3525 unsigned Amt = ElementSize.logBase2();
3526 IdxN = DAG.getNode(ISD::SHL, dl,
3527 N.getValueType(), IdxN,
3528 DAG.getConstant(Amt, dl, IdxN.getValueType()));
3529 } else {
3530 SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3531 IdxN = DAG.getNode(ISD::MUL, dl,
3532 N.getValueType(), IdxN, Scale);
3533 }
3534 }
3535
3536 N = DAG.getNode(ISD::ADD, dl,
3537 N.getValueType(), N, IdxN);
3538 }
3539 }
3540
3541 setValue(&I, N);
3542}
3543
3544void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3545 // If this is a fixed sized alloca in the entry block of the function,
3546 // allocate it statically on the stack.
3547 if (FuncInfo.StaticAllocaMap.count(&I))
3548 return; // getValue will auto-populate this.
3549
3550 SDLoc dl = getCurSDLoc();
3551 Type *Ty = I.getAllocatedType();
3552 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3553 auto &DL = DAG.getDataLayout();
3554 uint64_t TySize = DL.getTypeAllocSize(Ty);
3555 unsigned Align =
3556 std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3557
3558 SDValue AllocSize = getValue(I.getArraySize());
3559
3560 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3561 if (AllocSize.getValueType() != IntPtr)
3562 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3563
3564 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3565 AllocSize,
3566 DAG.getConstant(TySize, dl, IntPtr));
3567
3568 // Handle alignment. If the requested alignment is less than or equal to
3569 // the stack alignment, ignore it. If the size is greater than or equal to
3570 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3571 unsigned StackAlign =
3572 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3573 if (Align <= StackAlign)
3574 Align = 0;
3575
3576 // Round the size of the allocation up to the stack alignment size
3577 // by add SA-1 to the size. This doesn't overflow because we're computing
3578 // an address inside an alloca.
3579 SDNodeFlags Flags;
3580 Flags.setNoUnsignedWrap(true);
3581 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3582 DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3583
3584 // Mask out the low bits for alignment purposes.
3585 AllocSize =
3586 DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3587 DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3588
3589 SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3590 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3591 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3592 setValue(&I, DSA);
3593 DAG.setRoot(DSA.getValue(1));
3594
3595 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects
()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3595, __extension__ __PRETTY_FUNCTION__))
;
3596}
3597
3598void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3599 if (I.isAtomic())
3600 return visitAtomicLoad(I);
3601
3602 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3603 const Value *SV = I.getOperand(0);
3604 if (TLI.supportSwiftError()) {
3605 // Swifterror values can come from either a function parameter with
3606 // swifterror attribute or an alloca with swifterror attribute.
3607 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3608 if (Arg->hasSwiftErrorAttr())
3609 return visitLoadFromSwiftError(I);
3610 }
3611
3612 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3613 if (Alloca->isSwiftError())
3614 return visitLoadFromSwiftError(I);
3615 }
3616 }
3617
3618 SDValue Ptr = getValue(SV);
3619
3620 Type *Ty = I.getType();
3621
3622 bool isVolatile = I.isVolatile();
3623 bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3624 bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3625 bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3626 unsigned Alignment = I.getAlignment();
3627
3628 AAMDNodes AAInfo;
3629 I.getAAMetadata(AAInfo);
3630 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3631
3632 SmallVector<EVT, 4> ValueVTs;
3633 SmallVector<uint64_t, 4> Offsets;
3634 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3635 unsigned NumValues = ValueVTs.size();
3636 if (NumValues == 0)
3637 return;
3638
3639 SDValue Root;
3640 bool ConstantMemory = false;
3641 if (isVolatile || NumValues > MaxParallelChains)
3642 // Serialize volatile loads with other side effects.
3643 Root = getRoot();
3644 else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3645 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3646 // Do not serialize (non-volatile) loads of constant memory with anything.
3647 Root = DAG.getEntryNode();
3648 ConstantMemory = true;
3649 } else {
3650 // Do not serialize non-volatile loads against each other.
3651 Root = DAG.getRoot();
3652 }
3653
3654 SDLoc dl = getCurSDLoc();
3655
3656 if (isVolatile)
3657 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3658
3659 // An aggregate load cannot wrap around the address space, so offsets to its
3660 // parts don't wrap either.
3661 SDNodeFlags Flags;
3662 Flags.setNoUnsignedWrap(true);
3663
3664 SmallVector<SDValue, 4> Values(NumValues);
3665 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3666 EVT PtrVT = Ptr.getValueType();
3667 unsigned ChainI = 0;
3668 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3669 // Serializing loads here may result in excessive register pressure, and
3670 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3671 // could recover a bit by hoisting nodes upward in the chain by recognizing
3672 // they are side-effect free or do not alias. The optimizer should really
3673 // avoid this case by converting large object/array copies to llvm.memcpy
3674 // (MaxParallelChains should always remain as failsafe).
3675 if (ChainI == MaxParallelChains) {
3676 assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first"
) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3676, __extension__ __PRETTY_FUNCTION__))
;
3677 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3678 makeArrayRef(Chains.data(), ChainI));
3679 Root = Chain;
3680 ChainI = 0;
3681 }
3682 SDValue A = DAG.getNode(ISD::ADD, dl,
3683 PtrVT, Ptr,
3684 DAG.getConstant(Offsets[i], dl, PtrVT),
3685 Flags);
3686 auto MMOFlags = MachineMemOperand::MONone;
3687 if (isVolatile)
3688 MMOFlags |= MachineMemOperand::MOVolatile;
3689 if (isNonTemporal)
3690 MMOFlags |= MachineMemOperand::MONonTemporal;
3691 if (isInvariant)
3692 MMOFlags |= MachineMemOperand::MOInvariant;
3693 if (isDereferenceable)
3694 MMOFlags |= MachineMemOperand::MODereferenceable;
3695 MMOFlags |= TLI.getMMOFlags(I);
3696
3697 SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3698 MachinePointerInfo(SV, Offsets[i]), Alignment,
3699 MMOFlags, AAInfo, Ranges);
3700
3701 Values[i] = L;
3702 Chains[ChainI] = L.getValue(1);
3703 }
3704
3705 if (!ConstantMemory) {
3706 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3707 makeArrayRef(Chains.data(), ChainI));
3708 if (isVolatile)
3709 DAG.setRoot(Chain);
3710 else
3711 PendingLoads.push_back(Chain);
3712 }
3713
3714 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3715 DAG.getVTList(ValueVTs), Values));
3716}
3717
3718void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3719 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3720, __extension__ __PRETTY_FUNCTION__))
3720 "call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3720, __extension__ __PRETTY_FUNCTION__))
;
3721
3722 SmallVector<EVT, 4> ValueVTs;
3723 SmallVector<uint64_t, 4> Offsets;
3724 const Value *SrcV = I.getOperand(0);
3725 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3726 SrcV->getType(), ValueVTs, &Offsets);
3727 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3728, __extension__ __PRETTY_FUNCTION__))
3728 "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3728, __extension__ __PRETTY_FUNCTION__))
;
3729
3730 SDValue Src = getValue(SrcV);
3731 // Create a virtual register, then update the virtual register.
3732 unsigned VReg; bool CreatedVReg;
3733 std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3734 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3735 // Chain can be getRoot or getControlRoot.
3736 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3737 SDValue(Src.getNode(), Src.getResNo()));
3738 DAG.setRoot(CopyNode);
3739 if (CreatedVReg)
3740 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3741}
3742
3743void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3744 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3745, __extension__ __PRETTY_FUNCTION__))
3745 "call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3745, __extension__ __PRETTY_FUNCTION__))
;
3746
3747 assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3750, __extension__ __PRETTY_FUNCTION__))
3748 I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3750, __extension__ __PRETTY_FUNCTION__))
3749 I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3750, __extension__ __PRETTY_FUNCTION__))
3750 "Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3750, __extension__ __PRETTY_FUNCTION__))
;
3751
3752 const Value *SV = I.getOperand(0);
3753 Type *Ty = I.getType();
3754 AAMDNodes AAInfo;
3755 I.getAAMetadata(AAInfo);
3756 assert((!AA || !AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty)
, AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3758, __extension__ __PRETTY_FUNCTION__))
3757 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty)
, AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3758, __extension__ __PRETTY_FUNCTION__))
3758 "load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty)
, AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3758, __extension__ __PRETTY_FUNCTION__))
;
3759
3760 SmallVector<EVT, 4> ValueVTs;
3761 SmallVector<uint64_t, 4> Offsets;
3762 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3763 ValueVTs, &Offsets);
3764 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3765, __extension__ __PRETTY_FUNCTION__))
3765 "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3765, __extension__ __PRETTY_FUNCTION__))
;
3766
3767 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3768 SDValue L = DAG.getCopyFromReg(
3769 getRoot(), getCurSDLoc(),
3770 FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3771 ValueVTs[0]);
3772
3773 setValue(&I, L);
3774}
3775
3776void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3777 if (I.isAtomic())
3778 return visitAtomicStore(I);
3779
3780 const Value *SrcV = I.getOperand(0);
3781 const Value *PtrV = I.getOperand(1);
3782
3783 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3784 if (TLI.supportSwiftError()) {
3785 // Swifterror values can come from either a function parameter with
3786 // swifterror attribute or an alloca with swifterror attribute.
3787 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3788 if (Arg->hasSwiftErrorAttr())
3789 return visitStoreToSwiftError(I);
3790 }
3791
3792 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3793 if (Alloca->isSwiftError())
3794 return visitStoreToSwiftError(I);
3795 }
3796 }
3797
3798 SmallVector<EVT, 4> ValueVTs;
3799 SmallVector<uint64_t, 4> Offsets;
3800 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3801 SrcV->getType(), ValueVTs, &Offsets);
3802 unsigned NumValues = ValueVTs.size();
3803 if (NumValues == 0)
3804 return;
3805
3806 // Get the lowered operands. Note that we do this after
3807 // checking if NumResults is zero, because with zero results
3808 // the operands won't have values in the map.
3809 SDValue Src = getValue(SrcV);
3810 SDValue Ptr = getValue(PtrV);
3811
3812 SDValue Root = getRoot();
3813 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3814 SDLoc dl = getCurSDLoc();
3815 EVT PtrVT = Ptr.getValueType();
3816 unsigned Alignment = I.getAlignment();
3817 AAMDNodes AAInfo;
3818 I.getAAMetadata(AAInfo);
3819
3820 auto MMOFlags = MachineMemOperand::MONone;
3821 if (I.isVolatile())
3822 MMOFlags |= MachineMemOperand::MOVolatile;
3823 if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3824 MMOFlags |= MachineMemOperand::MONonTemporal;
3825 MMOFlags |= TLI.getMMOFlags(I);
3826
3827 // An aggregate load cannot wrap around the address space, so offsets to its
3828 // parts don't wrap either.
3829 SDNodeFlags Flags;
3830 Flags.setNoUnsignedWrap(true);
3831
3832 unsigned ChainI = 0;
3833 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3834 // See visitLoad comments.
3835 if (ChainI == MaxParallelChains) {
3836 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3837 makeArrayRef(Chains.data(), ChainI));
3838 Root = Chain;
3839 ChainI = 0;
3840 }
3841 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3842 DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3843 SDValue St = DAG.getStore(
3844 Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3845 MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3846 Chains[ChainI] = St;
3847 }
3848
3849 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3850 makeArrayRef(Chains.data(), ChainI));
3851 DAG.setRoot(StoreNode);
3852}
3853
3854void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3855 bool IsCompressing) {
3856 SDLoc sdl = getCurSDLoc();
3857
3858 auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3859 unsigned& Alignment) {
3860 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3861 Src0 = I.getArgOperand(0);
3862 Ptr = I.getArgOperand(1);
3863 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3864 Mask = I.getArgOperand(3);
3865 };
3866 auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3867 unsigned& Alignment) {
3868 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3869 Src0 = I.getArgOperand(0);
3870 Ptr = I.getArgOperand(1);
3871 Mask = I.getArgOperand(2);
3872 Alignment = 0;
3873 };
3874
3875 Value *PtrOperand, *MaskOperand, *Src0Operand;
3876 unsigned Alignment;
3877 if (IsCompressing)
3878 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3879 else
3880 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3881
3882 SDValue Ptr = getValue(PtrOperand);
3883 SDValue Src0 = getValue(Src0Operand);
3884 SDValue Mask = getValue(MaskOperand);
3885
3886 EVT VT = Src0.getValueType();
3887 if (!Alignment)
3888 Alignment = DAG.getEVTAlignment(VT);
3889
3890 AAMDNodes AAInfo;
3891 I.getAAMetadata(AAInfo);
3892
3893 MachineMemOperand *MMO =
3894 DAG.getMachineFunction().
3895 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3896 MachineMemOperand::MOStore, VT.getStoreSize(),
3897 Alignment, AAInfo);
3898 SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3899 MMO, false /* Truncating */,
3900 IsCompressing);
3901 DAG.setRoot(StoreNode);
3902 setValue(&I, StoreNode);
3903}
3904
3905// Get a uniform base for the Gather/Scatter intrinsic.
3906// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3907// We try to represent it as a base pointer + vector of indices.
3908// Usually, the vector of pointers comes from a 'getelementptr' instruction.
3909// The first operand of the GEP may be a single pointer or a vector of pointers
3910// Example:
3911// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3912// or
3913// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3914// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3915//
3916// When the first GEP operand is a single pointer - it is the uniform base we
3917// are looking for. If first operand of the GEP is a splat vector - we
3918// extract the splat value and use it as a uniform base.
3919// In all other cases the function returns 'false'.
3920static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3921 SDValue &Scale, SelectionDAGBuilder* SDB) {
3922 SelectionDAG& DAG = SDB->DAG;
3923 LLVMContext &Context = *DAG.getContext();
3924
3925 assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
&& "Uexpected pointer type") ? void (0) : __assert_fail
("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3925, __extension__ __PRETTY_FUNCTION__))
;
3926 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3927 if (!GEP)
3928 return false;
3929
3930 const Value *GEPPtr = GEP->getPointerOperand();
3931 if (!GEPPtr->getType()->isVectorTy())
3932 Ptr = GEPPtr;
3933 else if (!(Ptr = getSplatValue(GEPPtr)))
3934 return false;
3935
3936 unsigned FinalIndex = GEP->getNumOperands() - 1;
3937 Value *IndexVal = GEP->getOperand(FinalIndex);
3938
3939 // Ensure all the other indices are 0.
3940 for (unsigned i = 1; i < FinalIndex; ++i) {
3941 auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
3942 if (!C || !C->isZero())
3943 return false;
3944 }
3945
3946 // The operands of the GEP may be defined in another basic block.
3947 // In this case we'll not find nodes for the operands.
3948 if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3949 return false;
3950
3951 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3952 const DataLayout &DL = DAG.getDataLayout();
3953 Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
3954 SDB->getCurSDLoc(), TLI.getPointerTy(DL));
3955 Base = SDB->getValue(Ptr);
3956 Index = SDB->getValue(IndexVal);
3957
3958 if (!Index.getValueType().isVector()) {
3959 unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3960 EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3961 Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3962 }
3963 return true;
3964}
3965
3966void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3967 SDLoc sdl = getCurSDLoc();
3968
3969 // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3970 const Value *Ptr = I.getArgOperand(1);
3971 SDValue Src0 = getValue(I.getArgOperand(0));
3972 SDValue Mask = getValue(I.getArgOperand(3));
3973 EVT VT = Src0.getValueType();
3974 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3975 if (!Alignment)
3976 Alignment = DAG.getEVTAlignment(VT);
3977 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3978
3979 AAMDNodes AAInfo;
3980 I.getAAMetadata(AAInfo);
3981
3982 SDValue Base;
3983 SDValue Index;
3984 SDValue Scale;
3985 const Value *BasePtr = Ptr;
3986 bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
3987
3988 const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3989 MachineMemOperand *MMO = DAG.getMachineFunction().
3990 getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3991 MachineMemOperand::MOStore, VT.getStoreSize(),
3992 Alignment, AAInfo);
3993 if (!UniformBase) {
3994 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3995 Index = getValue(Ptr);
3996 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3997 }
3998 SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
3999 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4000 Ops, MMO);
4001 DAG.setRoot(Scatter);
4002 setValue(&I, Scatter);
4003}
4004
4005void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4006 SDLoc sdl = getCurSDLoc();
4007
4008 auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4009 unsigned& Alignment) {
4010 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4011 Ptr = I.getArgOperand(0);
4012 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4013 Mask = I.getArgOperand(2);
4014 Src0 = I.getArgOperand(3);
4015 };
4016 auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4017 unsigned& Alignment) {
4018 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4019 Ptr = I.getArgOperand(0);
4020 Alignment = 0;
4021 Mask = I.getArgOperand(1);
4022 Src0 = I.getArgOperand(2);
4023 };
4024
4025 Value *PtrOperand, *MaskOperand, *Src0Operand;
4026 unsigned Alignment;
4027 if (IsExpanding)
4028 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4029 else
4030 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4031
4032 SDValue Ptr = getValue(PtrOperand);
4033 SDValue Src0 = getValue(Src0Operand);
4034 SDValue Mask = getValue(MaskOperand);
4035
4036 EVT VT = Src0.getValueType();
4037 if (!Alignment)
4038 Alignment = DAG.getEVTAlignment(VT);
4039
4040 AAMDNodes AAInfo;
4041 I.getAAMetadata(AAInfo);
4042 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4043
4044 // Do not serialize masked loads of constant memory with anything.
4045 bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
4046 PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
4047 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4048
4049 MachineMemOperand *MMO =
4050 DAG.getMachineFunction().
4051 getMachineMemOperand(MachinePointerInfo(PtrOperand),
4052 MachineMemOperand::MOLoad, VT.getStoreSize(),
4053 Alignment, AAInfo, Ranges);
4054
4055 SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4056 ISD::NON_EXTLOAD, IsExpanding);
4057 if (AddToChain) {
4058 SDValue OutChain = Load.getValue(1);
4059 DAG.setRoot(OutChain);
4060 }
4061 setValue(&I, Load);
4062}
4063
4064void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4065 SDLoc sdl = getCurSDLoc();
4066
4067 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4068 const Value *Ptr = I.getArgOperand(0);
4069 SDValue Src0 = getValue(I.getArgOperand(3));
4070 SDValue Mask = getValue(I.getArgOperand(2));
4071
4072 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4073 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4074 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4075 if (!Alignment)
4076 Alignment = DAG.getEVTAlignment(VT);
4077
4078 AAMDNodes AAInfo;
4079 I.getAAMetadata(AAInfo);
4080 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4081
4082 SDValue Root = DAG.getRoot();
4083 SDValue Base;
4084 SDValue Index;
4085 SDValue Scale;
4086 const Value *BasePtr = Ptr;
4087 bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4088 bool ConstantMemory = false;
4089 if (UniformBase &&
4090 AA && AA->pointsToConstantMemory(MemoryLocation(
4091 BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
4092 AAInfo))) {
4093 // Do not serialize (non-volatile) loads of constant memory with anything.
4094 Root = DAG.getEntryNode();
4095 ConstantMemory = true;
4096 }
4097
4098 MachineMemOperand *MMO =
4099 DAG.getMachineFunction().
4100 getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4101 MachineMemOperand::MOLoad, VT.getStoreSize(),
4102 Alignment, AAInfo, Ranges);
4103
4104 if (!UniformBase) {
4105 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4106 Index = getValue(Ptr);
4107 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4108 }
4109 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4110 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4111 Ops, MMO);
4112
4113 SDValue OutChain = Gather.getValue(1);
4114 if (!ConstantMemory)
4115 PendingLoads.push_back(OutChain);
4116 setValue(&I, Gather);
4117}
4118
4119void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4120 SDLoc dl = getCurSDLoc();
4121 AtomicOrdering SuccessOrder = I.getSuccessOrdering();
4122 AtomicOrdering FailureOrder = I.getFailureOrdering();
4123 SyncScope::ID SSID = I.getSyncScopeID();
4124
4125 SDValue InChain = getRoot();
4126
4127 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4128 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4129 SDValue L = DAG.getAtomicCmpSwap(
4130 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4131 getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4132 getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
4133 /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4134
4135 SDValue OutChain = L.getValue(2);
4136
4137 setValue(&I, L);
4138 DAG.setRoot(OutChain);
4139}
4140
4141void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4142 SDLoc dl = getCurSDLoc();
4143 ISD::NodeType NT;
4144 switch (I.getOperation()) {
4145 default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4145)
;
4146 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4147 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4148 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4149 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4150 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4151 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4152 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4153 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4154 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4155 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4156 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4157 }
4158 AtomicOrdering Order = I.getOrdering();
4159 SyncScope::ID SSID = I.getSyncScopeID();
4160
4161 SDValue InChain = getRoot();
4162
4163 SDValue L =
4164 DAG.getAtomic(NT, dl,
4165 getValue(I.getValOperand()).getSimpleValueType(),
4166 InChain,
4167 getValue(I.getPointerOperand()),
4168 getValue(I.getValOperand()),
4169 I.getPointerOperand(),
4170 /* Alignment=*/ 0, Order, SSID);
4171
4172 SDValue OutChain = L.getValue(1);
4173
4174 setValue(&I, L);
4175 DAG.setRoot(OutChain);
4176}
4177
4178void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4179 SDLoc dl = getCurSDLoc();
4180 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4181 SDValue Ops[3];
4182 Ops[0] = getRoot();
4183 Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4184 TLI.getFenceOperandTy(DAG.getDataLayout()));
4185 Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4186 TLI.getFenceOperandTy(DAG.getDataLayout()));
4187 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4188}
4189
4190void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4191 SDLoc dl = getCurSDLoc();
4192 AtomicOrdering Order = I.getOrdering();
4193 SyncScope::ID SSID = I.getSyncScopeID();
4194
4195 SDValue InChain = getRoot();
4196
4197 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4198 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4199
4200 if (!TLI.supportsUnalignedAtomics() &&
4201 I.getAlignment() < VT.getStoreSize())
4202 report_fatal_error("Cannot generate unaligned atomic load");
4203
4204 MachineMemOperand *MMO =
4205 DAG.getMachineFunction().
4206 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4207 MachineMemOperand::MOVolatile |
4208 MachineMemOperand::MOLoad,
4209 VT.getStoreSize(),
4210 I.getAlignment() ? I.getAlignment() :
4211 DAG.getEVTAlignment(VT),
4212 AAMDNodes(), nullptr, SSID, Order);
4213
4214 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4215 SDValue L =
4216 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4217 getValue(I.getPointerOperand()), MMO);
4218
4219 SDValue OutChain = L.getValue(1);
4220
4221 setValue(&I, L);
4222 DAG.setRoot(OutChain);
4223}
4224
4225void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4226 SDLoc dl = getCurSDLoc();
4227
4228 AtomicOrdering Order = I.getOrdering();
4229 SyncScope::ID SSID = I.getSyncScopeID();
4230
4231 SDValue InChain = getRoot();
4232
4233 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4234 EVT VT =
4235 TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4236
4237 if (I.getAlignment() < VT.getStoreSize())
4238 report_fatal_error("Cannot generate unaligned atomic store");
4239
4240 SDValue OutChain =
4241 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4242 InChain,
4243 getValue(I.getPointerOperand()),
4244 getValue(I.getValueOperand()),
4245 I.getPointerOperand(), I.getAlignment(),
4246 Order, SSID);
4247
4248 DAG.setRoot(OutChain);
4249}
4250
4251/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4252/// node.
4253void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4254 unsigned Intrinsic) {
4255 // Ignore the callsite's attributes. A specific call site may be marked with
4256 // readnone, but the lowering code will expect the chain based on the
4257 // definition.
4258 const Function *F = I.getCalledFunction();
4259 bool HasChain = !F->doesNotAccessMemory();
4260 bool OnlyLoad = HasChain && F->onlyReadsMemory();
4261
4262 // Build the operand list.
4263 SmallVector<SDValue, 8> Ops;
4264 if (HasChain) { // If this intrinsic has side-effects, chainify it.
4265 if (OnlyLoad) {
4266 // We don't need to serialize loads against other loads.
4267 Ops.push_back(DAG.getRoot());
4268 } else {
4269 Ops.push_back(getRoot());
4270 }
4271 }
4272
4273 // Info is set by getTgtMemInstrinsic
4274 TargetLowering::IntrinsicInfo Info;
4275 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4276 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4277 DAG.getMachineFunction(),
4278 Intrinsic);
4279
4280 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4281 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4282 Info.opc == ISD::INTRINSIC_W_CHAIN)
4283 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4284 TLI.getPointerTy(DAG.getDataLayout())));
4285
4286 // Add all operands of the call to the operand list.
4287 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4288 SDValue Op = getValue(I.getArgOperand(i));
4289 Ops.push_back(Op);
4290 }
4291
4292 SmallVector<EVT, 4> ValueVTs;
4293 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4294
4295 if (HasChain)
4296 ValueVTs.push_back(MVT::Other);
4297
4298 SDVTList VTs = DAG.getVTList(ValueVTs);
4299
4300 // Create the node.
4301 SDValue Result;
4302 if (IsTgtIntrinsic) {
4303 // This is target intrinsic that touches memory
4304 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4305 Ops, Info.memVT,
4306 MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4307 Info.flags, Info.size);
4308 } else if (!HasChain) {
4309 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4310 } else if (!I.getType()->isVoidTy()) {
4311 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4312 } else {
4313 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4314 }
4315
4316 if (HasChain) {
4317 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4318 if (OnlyLoad)
4319 PendingLoads.push_back(Chain);
4320 else
4321 DAG.setRoot(Chain);
4322 }
4323
4324 if (!I.getType()->isVoidTy()) {
4325 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4326 EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4327 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4328 } else
4329 Result = lowerRangeToAssertZExt(DAG, I, Result);
4330
4331 setValue(&I, Result);
4332 }
4333}
4334
4335/// GetSignificand - Get the significand and build it into a floating-point
4336/// number with exponent of 1:
4337///
4338/// Op = (Op & 0x007fffff) | 0x3f800000;
4339///
4340/// where Op is the hexadecimal representation of floating point value.
4341static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4342 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4343 DAG.getConstant(0x007fffff, dl, MVT::i32));
4344 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4345 DAG.getConstant(0x3f800000, dl, MVT::i32));
4346 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4347}
4348
4349/// GetExponent - Get the exponent:
4350///
4351/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4352///
4353/// where Op is the hexadecimal representation of floating point value.
4354static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4355 const TargetLowering &TLI, const SDLoc &dl) {
4356 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4357 DAG.getConstant(0x7f800000, dl, MVT::i32));
4358 SDValue t1 = DAG.getNode(
4359 ISD::SRL, dl, MVT::i32, t0,
4360 DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4361 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4362 DAG.getConstant(127, dl, MVT::i32));
4363 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4364}
4365
4366/// getF32Constant - Get 32-bit floating point constant.
4367static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4368 const SDLoc &dl) {
4369 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4370 MVT::f32);
4371}
4372
4373static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4374 SelectionDAG &DAG) {
4375 // TODO: What fast-math-flags should be set on the floating-point nodes?
4376
4377 // IntegerPartOfX = ((int32_t)(t0);
4378 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4379
4380 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4381 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4382 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4383
4384 // IntegerPartOfX <<= 23;
4385 IntegerPartOfX = DAG.getNode(
4386 ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4387 DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4388 DAG.getDataLayout())));
4389
4390 SDValue TwoToFractionalPartOfX;
4391 if (LimitFloatPrecision <= 6) {
4392 // For floating-point precision of 6:
4393 //
4394 // TwoToFractionalPartOfX =
4395 // 0.997535578f +
4396 // (0.735607626f + 0.252464424f * x) * x;
4397 //
4398 // error 0.0144103317, which is 6 bits
4399 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4400 getF32Constant(DAG, 0x3e814304, dl));
4401 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4402 getF32Constant(DAG, 0x3f3c50c8, dl));
4403 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4404 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4405 getF32Constant(DAG, 0x3f7f5e7e, dl));
4406 } else if (LimitFloatPrecision <= 12) {
4407 // For floating-point precision of 12:
4408 //
4409 // TwoToFractionalPartOfX =
4410 // 0.999892986f +
4411 // (0.696457318f +
4412 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4413 //
4414 // error 0.000107046256, which is 13 to 14 bits
4415 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4416 getF32Constant(DAG, 0x3da235e3, dl));
4417 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4418 getF32Constant(DAG, 0x3e65b8f3, dl));
4419 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4420 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4421 getF32Constant(DAG, 0x3f324b07, dl));
4422 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4423 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4424 getF32Constant(DAG, 0x3f7ff8fd, dl));
4425 } else { // LimitFloatPrecision <= 18
4426 // For floating-point precision of 18:
4427 //
4428 // TwoToFractionalPartOfX =
4429 // 0.999999982f +
4430 // (0.693148872f +
4431 // (0.240227044f +
4432 // (0.554906021e-1f +
4433 // (0.961591928e-2f +
4434 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4435 // error 2.47208000*10^(-7), which is better than 18 bits
4436 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4437 getF32Constant(DAG, 0x3924b03e, dl));
4438 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4439 getF32Constant(DAG, 0x3ab24b87, dl));
4440 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4441 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4442 getF32Constant(DAG, 0x3c1d8c17, dl));
4443 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4444 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4445 getF32Constant(DAG, 0x3d634a1d, dl));
4446 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4447 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4448 getF32Constant(DAG, 0x3e75fe14, dl));
4449 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4450 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4451 getF32Constant(DAG, 0x3f317234, dl));
4452 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4453 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4454 getF32Constant(DAG, 0x3f800000, dl));
4455 }
4456
4457 // Add the exponent into the result in integer domain.
4458 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4459 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4460 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4461}
4462
4463/// expandExp - Lower an exp intrinsic. Handles the special sequences for
4464/// limited-precision mode.
4465static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4466 const TargetLowering &TLI) {
4467 if (Op.getValueType() == MVT::f32 &&
4468 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4469
4470 // Put the exponent in the right bit position for later addition to the
4471 // final result:
4472 //
4473 // #define LOG2OFe 1.4426950f
4474 // t0 = Op * LOG2OFe
4475
4476 // TODO: What fast-math-flags should be set here?
4477 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4478 getF32Constant(DAG, 0x3fb8aa3b, dl));
4479 return getLimitedPrecisionExp2(t0, dl, DAG);
4480 }
4481
4482 // No special expansion.
4483 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4484}
4485
4486/// expandLog - Lower a log intrinsic. Handles the special sequences for
4487/// limited-precision mode.
4488static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4489 const TargetLowering &TLI) {
4490 // TODO: What fast-math-flags should be set on the floating-point nodes?
4491
4492 if (Op.getValueType() == MVT::f32 &&
4493 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4494 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4495
4496 // Scale the exponent by log(2) [0.69314718f].
4497 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4498 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4499 getF32Constant(DAG, 0x3f317218, dl));
4500
4501 // Get the significand and build it into a floating-point number with
4502 // exponent of 1.
4503 SDValue X = GetSignificand(DAG, Op1, dl);
4504
4505 SDValue LogOfMantissa;
4506 if (LimitFloatPrecision <= 6) {
4507 // For floating-point precision of 6:
4508 //
4509 // LogofMantissa =
4510 // -1.1609546f +
4511 // (1.4034025f - 0.23903021f * x) * x;
4512 //
4513 // error 0.0034276066, which is better than 8 bits
4514 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4515 getF32Constant(DAG, 0xbe74c456, dl));
4516 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4517 getF32Constant(DAG, 0x3fb3a2b1, dl));
4518 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4519 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4520 getF32Constant(DAG, 0x3f949a29, dl));
4521 } else if (LimitFloatPrecision <= 12) {
4522 // For floating-point precision of 12:
4523 //
4524 // LogOfMantissa =
4525 // -1.7417939f +
4526 // (2.8212026f +
4527 // (-1.4699568f +
4528 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4529 //
4530 // error 0.000061011436, which is 14 bits
4531 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4532 getF32Constant(DAG, 0xbd67b6d6, dl));
4533 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4534 getF32Constant(DAG, 0x3ee4f4b8, dl));
4535 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4536 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4537 getF32Constant(DAG, 0x3fbc278b, dl));
4538 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4539 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4540 getF32Constant(DAG, 0x40348e95, dl));
4541 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4542 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4543 getF32Constant(DAG, 0x3fdef31a, dl));
4544 } else { // LimitFloatPrecision <= 18
4545 // For floating-point precision of 18:
4546 //
4547 // LogOfMantissa =
4548 // -2.1072184f +
4549 // (4.2372794f +
4550 // (-3.7029485f +
4551 // (2.2781945f +
4552 // (-0.87823314f +
4553 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4554 //
4555 // error 0.0000023660568, which is better than 18 bits
4556 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4557 getF32Constant(DAG, 0xbc91e5ac, dl));
4558 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4559 getF32Constant(DAG, 0x3e4350aa, dl));
4560 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4561 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4562 getF32Constant(DAG, 0x3f60d3e3, dl));
4563 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4564 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4565 getF32Constant(DAG, 0x4011cdf0, dl));
4566 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4567 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4568 getF32Constant(DAG, 0x406cfd1c, dl));
4569 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4570 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4571 getF32Constant(DAG, 0x408797cb, dl));
4572 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4573 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4574 getF32Constant(DAG, 0x4006dcab, dl));
4575 }
4576
4577 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4578 }
4579
4580 // No special expansion.
4581 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4582}
4583
4584/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4585/// limited-precision mode.
4586static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4587 const TargetLowering &TLI) {
4588 // TODO: What fast-math-flags should be set on the floating-point nodes?
4589
4590 if (Op.getValueType() == MVT::f32 &&
4591 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4592 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4593
4594 // Get the exponent.
4595 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4596
4597 // Get the significand and build it into a floating-point number with
4598 // exponent of 1.
4599 SDValue X = GetSignificand(DAG, Op1, dl);
4600
4601 // Different possible minimax approximations of significand in
4602 // floating-point for various degrees of accuracy over [1,2].
4603 SDValue Log2ofMantissa;
4604 if (LimitFloatPrecision <= 6) {
4605 // For floating-point precision of 6:
4606 //
4607 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4608 //
4609 // error 0.0049451742, which is more than 7 bits
4610 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4611 getF32Constant(DAG, 0xbeb08fe0, dl));
4612 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4613 getF32Constant(DAG, 0x40019463, dl));
4614 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4615 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4616 getF32Constant(DAG, 0x3fd6633d, dl));
4617 } else if (LimitFloatPrecision <= 12) {
4618 // For floating-point precision of 12:
4619 //
4620 // Log2ofMantissa =
4621 // -2.51285454f +
4622 // (4.07009056f +
4623 // (-2.12067489f +
4624 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4625 //
4626 // error 0.0000876136000, which is better than 13 bits
4627 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4628 getF32Constant(DAG, 0xbda7262e, dl));
4629 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4630 getF32Constant(DAG, 0x3f25280b, dl));
4631 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4632 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4633 getF32Constant(DAG, 0x4007b923, dl));
4634 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4635 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4636 getF32Constant(DAG, 0x40823e2f, dl));
4637 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4638 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4639 getF32Constant(DAG, 0x4020d29c, dl));
4640 } else { // LimitFloatPrecision <= 18
4641 // For floating-point precision of 18:
4642 //
4643 // Log2ofMantissa =
4644 // -3.0400495f +
4645 // (6.1129976f +
4646 // (-5.3420409f +
4647 // (3.2865683f +
4648 // (-1.2669343f +
4649 // (0.27515199f -
4650 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4651 //
4652 // error 0.0000018516, which is better than 18 bits
4653 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4654 getF32Constant(DAG, 0xbcd2769e, dl));
4655 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4656 getF32Constant(DAG, 0x3e8ce0b9, dl));
4657 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4658 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4659 getF32Constant(DAG, 0x3fa22ae7, dl));
4660 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4661 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4662 getF32Constant(DAG, 0x40525723, dl));
4663 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4664 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4665 getF32Constant(DAG, 0x40aaf200, dl));
4666 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4667 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4668 getF32Constant(DAG, 0x40c39dad, dl));
4669 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4670 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4671 getF32Constant(DAG, 0x4042902c, dl));
4672 }
4673
4674 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4675 }
4676
4677 // No special expansion.
4678 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4679}
4680
4681/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4682/// limited-precision mode.
4683static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4684 const TargetLowering &TLI) {
4685 // TODO: What fast-math-flags should be set on the floating-point nodes?
4686
4687 if (Op.getValueType() == MVT::f32 &&
4688 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4689 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4690
4691 // Scale the exponent by log10(2) [0.30102999f].
4692 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4693 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4694 getF32Constant(DAG, 0x3e9a209a, dl));
4695
4696 // Get the significand and build it into a floating-point number with
4697 // exponent of 1.
4698 SDValue X = GetSignificand(DAG, Op1, dl);
4699
4700 SDValue Log10ofMantissa;
4701 if (LimitFloatPrecision <= 6) {
4702 // For floating-point precision of 6:
4703 //
4704 // Log10ofMantissa =
4705 // -0.50419619f +
4706 // (0.60948995f - 0.10380950f * x) * x;
4707 //
4708 // error 0.0014886165, which is 6 bits
4709 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4710 getF32Constant(DAG, 0xbdd49a13, dl));
4711 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4712 getF32Constant(DAG, 0x3f1c0789, dl));
4713 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4714 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4715 getF32Constant(DAG, 0x3f011300, dl));
4716 } else if (LimitFloatPrecision <= 12) {
4717 // For floating-point precision of 12:
4718 //
4719 // Log10ofMantissa =
4720 // -0.64831180f +
4721 // (0.91751397f +
4722 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4723 //
4724 // error 0.00019228036, which is better than 12 bits
4725 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4726 getF32Constant(DAG, 0x3d431f31, dl));
4727 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4728 getF32Constant(DAG, 0x3ea21fb2, dl));
4729 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4730 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4731 getF32Constant(DAG, 0x3f6ae232, dl));
4732 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4733 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4734 getF32Constant(DAG, 0x3f25f7c3, dl));
4735 } else { // LimitFloatPrecision <= 18
4736 // For floating-point precision of 18:
4737 //
4738 // Log10ofMantissa =
4739 // -0.84299375f +
4740 // (1.5327582f +
4741 // (-1.0688956f +
4742 // (0.49102474f +
4743 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4744 //
4745 // error 0.0000037995730, which is better than 18 bits
4746 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4747 getF32Constant(DAG, 0x3c5d51ce, dl));
4748 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4749 getF32Constant(DAG, 0x3e00685a, dl));
4750 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4751 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4752 getF32Constant(DAG, 0x3efb6798, dl));
4753 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4754 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4755 getF32Constant(DAG, 0x3f88d192, dl));
4756 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4757 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4758 getF32Constant(DAG, 0x3fc4316c, dl));
4759 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4760 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4761 getF32Constant(DAG, 0x3f57ce70, dl));
4762 }
4763
4764 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4765 }
4766
4767 // No special expansion.
4768 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4769}
4770
4771/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4772/// limited-precision mode.
4773static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4774 const TargetLowering &TLI) {
4775 if (Op.getValueType() == MVT::f32 &&
4776 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4777 return getLimitedPrecisionExp2(Op, dl, DAG);
4778
4779 // No special expansion.
4780 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4781}
4782
4783/// visitPow - Lower a pow intrinsic. Handles the special sequences for
4784/// limited-precision mode with x == 10.0f.
4785static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4786 SelectionDAG &DAG, const TargetLowering &TLI) {
4787 bool IsExp10 = false;
4788 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4789 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4790 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4791 APFloat Ten(10.0f);
4792 IsExp10 = LHSC->isExactlyValue(Ten);
4793 }
4794 }
4795
4796 // TODO: What fast-math-flags should be set on the FMUL node?
4797 if (IsExp10) {
4798 // Put the exponent in the right bit position for later addition to the
4799 // final result:
4800 //
4801 // #define LOG2OF10 3.3219281f
4802 // t0 = Op * LOG2OF10;
4803 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4804 getF32Constant(DAG, 0x40549a78, dl));
4805 return getLimitedPrecisionExp2(t0, dl, DAG);
4806 }
4807
4808 // No special expansion.
4809 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4810}
4811
4812/// ExpandPowI - Expand a llvm.powi intrinsic.
4813static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4814 SelectionDAG &DAG) {
4815 // If RHS is a constant, we can expand this out to a multiplication tree,
4816 // otherwise we end up lowering to a call to __powidf2 (for example). When
4817 // optimizing for size, we only want to do this if the expansion would produce
4818 // a small number of multiplies, otherwise we do the full expansion.
4819 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4820 // Get the exponent as a positive value.
4821 unsigned Val = RHSC->getSExtValue();
4822 if ((int)Val < 0) Val = -Val;
4823
4824 // powi(x, 0) -> 1.0
4825 if (Val == 0)
4826 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4827
4828 const Function &F = DAG.getMachineFunction().getFunction();
4829 if (!F.optForSize() ||
4830 // If optimizing for size, don't insert too many multiplies.
4831 // This inserts up to 5 multiplies.
4832 countPopulation(Val) + Log2_32(Val) < 7) {
4833 // We use the simple binary decomposition method to generate the multiply
4834 // sequence. There are more optimal ways to do this (for example,
4835 // powi(x,15) generates one more multiply than it should), but this has
4836 // the benefit of being both really simple and much better than a libcall.
4837 SDValue Res; // Logically starts equal to 1.0
4838 SDValue CurSquare = LHS;
4839 // TODO: Intrinsics should have fast-math-flags that propagate to these
4840 // nodes.
4841 while (Val) {
4842 if (Val & 1) {
4843 if (Res.getNode())
4844 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4845 else
4846 Res = CurSquare; // 1.0*CurSquare.
4847 }
4848
4849 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4850 CurSquare, CurSquare);
4851 Val >>= 1;
4852 }
4853
4854 // If the original was negative, invert the result, producing 1/(x*x*x).
4855 if (RHSC->getSExtValue() < 0)
4856 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4857 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4858 return Res;
4859 }
4860 }
4861
4862 // Otherwise, expand to a libcall.
4863 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4864}
4865
4866// getUnderlyingArgReg - Find underlying register used for a truncated or
4867// bitcasted argument.
4868static unsigned getUnderlyingArgReg(const SDValue &N) {
4869 switch (N.getOpcode()) {
4870 case ISD::CopyFromReg:
4871 return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4872 case ISD::BITCAST:
4873 case ISD::AssertZext:
4874 case ISD::AssertSext:
4875 case ISD::TRUNCATE:
4876 return getUnderlyingArgReg(N.getOperand(0));
4877 default:
4878 return 0;
4879 }
4880}
4881
4882/// If the DbgValueInst is a dbg_value of a function argument, create the
4883/// corresponding DBG_VALUE machine instruction for it now. At the end of
4884/// instruction selection, they will be inserted to the entry BB.
4885bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4886 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4887 DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
4888 const Argument *Arg = dyn_cast<Argument>(V);
4889 if (!Arg)
4890 return false;
4891
4892 MachineFunction &MF = DAG.getMachineFunction();
4893 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4894
4895 bool IsIndirect = false;
4896 Optional<MachineOperand> Op;
4897 // Some arguments' frame index is recorded during argument lowering.
4898 int FI = FuncInfo.getArgumentFrameIndex(Arg);
4899 if (FI != std::numeric_limits<int>::max())
4900 Op = MachineOperand::CreateFI(FI);
4901
4902 if (!Op && N.getNode()) {
4903 unsigned Reg = getUnderlyingArgReg(N);
4904 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4905 MachineRegisterInfo &RegInfo = MF.getRegInfo();
4906 unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4907 if (PR)
4908 Reg = PR;
4909 }
4910 if (Reg) {
4911 Op = MachineOperand::CreateReg(Reg, false);
4912 IsIndirect = IsDbgDeclare;
4913 }
4914 }
4915
4916 if (!Op && N.getNode())
4917 // Check if frame index is available.
4918 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4919 if (FrameIndexSDNode *FINode =
4920 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4921 Op = MachineOperand::CreateFI(FINode->getIndex());
4922
4923 if (!Op) {
4924 // Check if ValueMap has reg number.
4925 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4926 if (VMI != FuncInfo.ValueMap.end()) {
4927 const auto &TLI = DAG.getTargetLoweringInfo();
4928 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
4929 V->getType(), isABIRegCopy(V));
4930 if (RFV.occupiesMultipleRegs()) {
4931 unsigned Offset = 0;
4932 for (auto RegAndSize : RFV.getRegsAndSizes()) {
4933 Op = MachineOperand::CreateReg(RegAndSize.first, false);
4934 auto FragmentExpr = DIExpression::createFragmentExpression(
4935 Expr, Offset, RegAndSize.second);
4936 if (!FragmentExpr)
4937 continue;
4938 FuncInfo.ArgDbgValues.push_back(
4939 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,
4940 Op->getReg(), Variable, *FragmentExpr));
4941 Offset += RegAndSize.second;
4942 }
4943 return true;
4944 }
4945 Op = MachineOperand::CreateReg(VMI->second, false);
4946 IsIndirect = IsDbgDeclare;
4947 }
4948 }
4949
4950 if (!Op)
4951 return false;
4952
4953 assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4954, __extension__ __PRETTY_FUNCTION__))
4954 "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4954, __extension__ __PRETTY_FUNCTION__))
;
4955 IsIndirect = (Op->isReg()) ? IsIndirect : true;
4956 FuncInfo.ArgDbgValues.push_back(
4957 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4958 *Op, Variable, Expr));
4959
4960 return true;
4961}
4962
4963/// Return the appropriate SDDbgValue based on N.
4964SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4965 DILocalVariable *Variable,
4966 DIExpression *Expr,
4967 const DebugLoc &dl,
4968 unsigned DbgSDNodeOrder) {
4969 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
4970 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4971 // stack slot locations as such instead of as indirectly addressed
4972 // locations.
4973 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), dl,
4974 DbgSDNodeOrder);
4975 }
4976 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), false, dl,
4977 DbgSDNodeOrder);
4978}
4979
4980// VisualStudio defines setjmp as _setjmp
4981#if defined(_MSC_VER) && defined(setjmp) && \
4982 !defined(setjmp_undefined_for_msvc)
4983# pragma push_macro("setjmp")
4984# undef setjmp
4985# define setjmp_undefined_for_msvc
4986#endif
4987
4988/// Lower the call to the specified intrinsic function. If we want to emit this
4989/// as a call to a named external function, return the name. Otherwise, lower it
4990/// and return null.
4991const char *
4992SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4993 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4994 SDLoc sdl = getCurSDLoc();
4995 DebugLoc dl = getCurDebugLoc();
4996 SDValue Res;
4997
4998 switch (Intrinsic) {
4999 default:
5000 // By default, turn this into a target intrinsic node.
5001 visitTargetIntrinsic(I, Intrinsic);
5002 return nullptr;
5003 case Intrinsic::vastart: visitVAStart(I); return nullptr;
5004 case Intrinsic::vaend: visitVAEnd(I); return nullptr;
5005 case Intrinsic::vacopy: visitVACopy(I); return nullptr;
5006 case Intrinsic::returnaddress:
5007 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5008 TLI.getPointerTy(DAG.getDataLayout()),
5009 getValue(I.getArgOperand(0))));
5010 return nullptr;
5011 case Intrinsic::addressofreturnaddress:
5012 setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5013 TLI.getPointerTy(DAG.getDataLayout())));
5014 return nullptr;
5015 case Intrinsic::frameaddress:
5016 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5017 TLI.getPointerTy(DAG.getDataLayout()),
5018 getValue(I.getArgOperand(0))));
5019 return nullptr;
5020 case Intrinsic::read_register: {
5021 Value *Reg = I.getArgOperand(0);
5022 SDValue Chain = getRoot();
5023 SDValue RegName =
5024 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5025 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5026 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5027 DAG.getVTList(VT, MVT::Other), Chain, RegName);
5028 setValue(&I, Res);
5029 DAG.setRoot(Res.getValue(1));
5030 return nullptr;
5031 }
5032 case Intrinsic::write_register: {
5033 Value *Reg = I.getArgOperand(0);
5034 Value *RegValue = I.getArgOperand(1);
5035 SDValue Chain = getRoot();
5036 SDValue RegName =
5037 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5038 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5039 RegName, getValue(RegValue)));
5040 return nullptr;
5041 }
5042 case Intrinsic::setjmp:
5043 return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
5044 case Intrinsic::longjmp:
5045 return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
5046 case Intrinsic::memcpy: {
5047 const auto &MCI = cast<MemCpyInst>(I);
5048 SDValue Op1 = getValue(I.getArgOperand(0));
5049 SDValue Op2 = getValue(I.getArgOperand(1));
5050 SDValue Op3 = getValue(I.getArgOperand(2));
5051 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5052 unsigned DstAlign = std::max<unsigned>(MCI.getDestAlignment(), 1);
5053 unsigned SrcAlign = std::max<unsigned>(MCI.getSourceAlignment(), 1);
5054 unsigned Align = MinAlign(DstAlign, SrcAlign);
5055 bool isVol = MCI.isVolatile();
5056 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5057 // FIXME: Support passing different dest/src alignments to the memcpy DAG
5058 // node.
5059 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5060 false, isTC,
5061 MachinePointerInfo(I.getArgOperand(0)),
5062 MachinePointerInfo(I.getArgOperand(1)));
5063 updateDAGForMaybeTailCall(MC);
5064 return nullptr;
5065 }
5066 case Intrinsic::memset: {
5067 const auto &MSI = cast<MemSetInst>(I);
5068 SDValue Op1 = getValue(I.getArgOperand(0));
5069 SDValue Op2 = getValue(I.getArgOperand(1));
5070 SDValue Op3 = getValue(I.getArgOperand(2));
5071 // @llvm.memset defines 0 and 1 to both mean no alignment.
5072 unsigned Align = std::max<unsigned>(MSI.getDestAlignment(), 1);
5073 bool isVol = MSI.isVolatile();
5074 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5075 SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5076 isTC, MachinePointerInfo(I.getArgOperand(0)));
5077 updateDAGForMaybeTailCall(MS);
5078 return nullptr;
5079 }
5080 case Intrinsic::memmove: {
5081 const auto &MMI = cast<MemMoveInst>(I);
5082 SDValue Op1 = getValue(I.getArgOperand(0));
5083 SDValue Op2 = getValue(I.getArgOperand(1));
5084 SDValue Op3 = getValue(I.getArgOperand(2));
5085 // @llvm.memmove defines 0 and 1 to both mean no alignment.
5086 unsigned DstAlign = std::max<unsigned>(MMI.getDestAlignment(), 1);
5087 unsigned SrcAlign = std::max<unsigned>(MMI.getSourceAlignment(), 1);
5088 unsigned Align = MinAlign(DstAlign, SrcAlign);
5089 bool isVol = MMI.isVolatile();
5090 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5091 // FIXME: Support passing different dest/src alignments to the memmove DAG
5092 // node.
5093 SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5094 isTC, MachinePointerInfo(I.getArgOperand(0)),
5095 MachinePointerInfo(I.getArgOperand(1)));
5096 updateDAGForMaybeTailCall(MM);
5097 return nullptr;
5098 }
5099 case Intrinsic::memcpy_element_unordered_atomic: {
5100 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5101 SDValue Dst = getValue(MI.getRawDest());
5102 SDValue Src = getValue(MI.getRawSource());
5103 SDValue Length = getValue(MI.getLength());
5104
5105 unsigned DstAlign = MI.getDestAlignment();
5106 unsigned SrcAlign = MI.getSourceAlignment();
5107 Type *LengthTy = MI.getLength()->getType();
5108 unsigned ElemSz = MI.getElementSizeInBytes();
5109 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5110 SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,
5111 SrcAlign, Length, LengthTy, ElemSz, isTC,
5112 MachinePointerInfo(MI.getRawDest()),
5113 MachinePointerInfo(MI.getRawSource()));
5114 updateDAGForMaybeTailCall(MC);
5115 return nullptr;
5116 }
5117 case Intrinsic::memmove_element_unordered_atomic: {
5118 auto &MI = cast<AtomicMemMoveInst>(I);
5119 SDValue Dst = getValue(MI.getRawDest());
5120 SDValue Src = getValue(MI.getRawSource());
5121 SDValue Length = getValue(MI.getLength());
5122
5123 unsigned DstAlign = MI.getDestAlignment();
5124 unsigned SrcAlign = MI.getSourceAlignment();
5125 Type *LengthTy = MI.getLength()->getType();
5126 unsigned ElemSz = MI.getElementSizeInBytes();
5127 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5128 SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,
5129 SrcAlign, Length, LengthTy, ElemSz, isTC,
5130 MachinePointerInfo(MI.getRawDest()),
5131 MachinePointerInfo(MI.getRawSource()));
5132 updateDAGForMaybeTailCall(MC);
5133 return nullptr;
5134 }
5135 case Intrinsic::memset_element_unordered_atomic: {
5136 auto &MI = cast<AtomicMemSetInst>(I);
5137 SDValue Dst = getValue(MI.getRawDest());
5138 SDValue Val = getValue(MI.getValue());
5139 SDValue Length = getValue(MI.getLength());
5140
5141 unsigned DstAlign = MI.getDestAlignment();
5142 Type *LengthTy = MI.getLength()->getType();
5143 unsigned ElemSz = MI.getElementSizeInBytes();
5144 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5145 SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,
5146 LengthTy, ElemSz, isTC,
5147 MachinePointerInfo(MI.getRawDest()));
5148 updateDAGForMaybeTailCall(MC);
5149 return nullptr;
5150 }
5151 case Intrinsic::dbg_addr:
5152 case Intrinsic::dbg_declare: {
5153 const DbgInfoIntrinsic &DI = cast<DbgInfoIntrinsic>(I);
5154 DILocalVariable *Variable = DI.getVariable();
5155 DIExpression *Expression = DI.getExpression();
5156 dropDanglingDebugInfo(Variable, Expression);
5157 assert(Variable && "Missing variable")(static_cast <bool> (Variable && "Missing variable"
) ? void (0) : __assert_fail ("Variable && \"Missing variable\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5157, __extension__ __PRETTY_FUNCTION__))
;
5158
5159 // Check if address has undef value.
5160 const Value *Address = DI.getVariableLocation();
5161 if (!Address || isa<UndefValue>(Address) ||
5162 (Address->use_empty() && !isa<Argument>(Address))) {
5163 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
5164 return nullptr;
5165 }
5166
5167 bool isParameter = Variable->isParameter() || isa<Argument>(Address);
5168
5169 // Check if this variable can be described by a frame index, typically
5170 // either as a static alloca or a byval parameter.
5171 int FI = std::numeric_limits<int>::max();
5172 if (const auto *AI =
5173 dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
5174 if (AI->isStaticAlloca()) {
5175 auto I = FuncInfo.StaticAllocaMap.find(AI);
5176 if (I != FuncInfo.StaticAllocaMap.end())
5177 FI = I->second;
5178 }
5179 } else if (const auto *Arg = dyn_cast<Argument>(
5180 Address->stripInBoundsConstantOffsets())) {
5181 FI = FuncInfo.getArgumentFrameIndex(Arg);
5182 }
5183
5184 // llvm.dbg.addr is control dependent and always generates indirect
5185 // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in
5186 // the MachineFunction variable table.
5187 if (FI != std::numeric_limits<int>::max()) {
5188 if (Intrinsic == Intrinsic::dbg_addr) {
5189 SDDbgValue *SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
5190 FI, dl, SDNodeOrder);
5191 DAG.AddDbgValue(SDV, getRoot().getNode(), isParameter);
5192 }
5193 return nullptr;
5194 }
5195
5196 SDValue &N = NodeMap[Address];
5197 if (!N.getNode() && isa<Argument>(Address))
5198 // Check unused arguments map.
5199 N = UnusedArgNodeMap[Address];
5200 SDDbgValue *SDV;
5201 if (N.getNode()) {
5202 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
5203 Address = BCI->getOperand(0);
5204 // Parameters are handled specially.
5205 auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
5206 if (isParameter && FINode) {
5207 // Byval parameter. We have a frame index at this point.
5208 SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
5209 FINode->getIndex(), dl, SDNodeOrder);
5210 } else if (isa<Argument>(Address)) {
5211 // Address is an argument, so try to emit its dbg value using
5212 // virtual register info from the FuncInfo.ValueMap.
5213 EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N);
5214 return nullptr;
5215 } else {
5216 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
5217 true, dl, SDNodeOrder);
5218 }
5219 DAG.AddDbgValue(SDV, N.getNode(), isParameter);
5220 } else {
5221 // If Address is an argument then try to emit its dbg value using
5222 // virtual register info from the FuncInfo.ValueMap.
5223 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true,
5224 N)) {
5225 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
5226 }
5227 }
5228 return nullptr;
5229 }
5230 case Intrinsic::dbg_label: {
5231 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
5232 DILabel *Label = DI.getLabel();
5233 assert(Label && "Missing label")(static_cast <bool> (Label && "Missing label") ?
void (0) : __assert_fail ("Label && \"Missing label\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5233, __extension__ __PRETTY_FUNCTION__))
;
5234
5235 SDDbgLabel *SDV;
5236 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
5237 DAG.AddDbgLabel(SDV);
5238 return nullptr;
5239 }
5240 case Intrinsic::dbg_value: {
5241 const DbgValueInst &DI = cast<DbgValueInst>(I);
5242 assert(DI.getVariable() && "Missing variable")(static_cast <bool> (DI.getVariable() && "Missing variable"
) ? void (0) : __assert_fail ("DI.getVariable() && \"Missing variable\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5242, __extension__ __PRETTY_FUNCTION__))
;
5243
5244 DILocalVariable *Variable = DI.getVariable();
5245 DIExpression *Expression = DI.getExpression();
5246 dropDanglingDebugInfo(Variable, Expression);
5247 const Value *V = DI.getValue();
5248 if (!V)
5249 return nullptr;
5250
5251 SDDbgValue *SDV;
5252 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
5253 SDV = DAG.getConstantDbgValue(Variable, Expression, V, dl, SDNodeOrder);
5254 DAG.AddDbgValue(SDV, nullptr, false);
5255 return nullptr;
5256 }
5257
5258 // Do not use getValue() in here; we don't want to generate code at
5259 // this point if it hasn't been done yet.
5260 SDValue N = NodeMap[V];
5261 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
5262 N = UnusedArgNodeMap[V];
5263 if (N.getNode()) {
5264 if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, false, N))
5265 return nullptr;
5266 SDV = getDbgValue(N, Variable, Expression, dl, SDNodeOrder);
5267 DAG.AddDbgValue(SDV, N.getNode(), false);
5268 return nullptr;
5269 }
5270
5271 // PHI nodes have already been selected, so we should know which VReg that
5272 // is assigns to already.
5273 if (isa<PHINode>(V)) {
5274 auto VMI = FuncInfo.ValueMap.find(V);
5275 if (VMI != FuncInfo.ValueMap.end()) {
5276 unsigned Reg = VMI->second;
5277 // The PHI node may be split up into several MI PHI nodes (in
5278 // FunctionLoweringInfo::set).
5279 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
5280 V->getType(), false);
5281 if (RFV.occupiesMultipleRegs()) {
5282 unsigned Offset = 0;
5283 unsigned BitsToDescribe = 0;
5284 if (auto VarSize = Variable->getSizeInBits())
5285 BitsToDescribe = *VarSize;
5286 if (auto Fragment = Expression->getFragmentInfo())
5287 BitsToDescribe = Fragment->SizeInBits;
5288 for (auto RegAndSize : RFV.getRegsAndSizes()) {
5289 unsigned RegisterSize = RegAndSize.second;
5290 // Bail out if all bits are described already.
5291 if (Offset >= BitsToDescribe)
5292 break;
5293 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
5294 ? BitsToDescribe - Offset
5295 : RegisterSize;
5296 auto FragmentExpr = DIExpression::createFragmentExpression(
5297 Expression, Offset, FragmentSize);
5298 if (!FragmentExpr)
5299 continue;
5300 SDV = DAG.getVRegDbgValue(Variable, *FragmentExpr, RegAndSize.first,
5301 false, dl, SDNodeOrder);
5302 DAG.AddDbgValue(SDV, nullptr, false);
5303 Offset += RegisterSize;
5304 }
5305 } else {
5306 SDV = DAG.getVRegDbgValue(Variable, Expression, Reg, false, dl,
5307 SDNodeOrder);
5308 DAG.AddDbgValue(SDV, nullptr, false);
5309 }
5310 return nullptr;
5311 }
5312 }
5313
5314 // TODO: When we get here we will either drop the dbg.value completely, or
5315 // we try to move it forward by letting it dangle for awhile. So we should
5316 // probably add an extra DbgValue to the DAG here, with a reference to
5317 // "noreg", to indicate that we have lost the debug location for the
5318 // variable.
5319
5320 if (!V->use_empty() ) {
5321 // Do not call getValue(V) yet, as we don't want to generate code.
5322 // Remember it for later.
5323 DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
5324 DanglingDebugInfoMap[V].push_back(DDI);
5325 return nullptr;
5326 }
5327
5328 LLVM_DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug location info for:\n "
<< DI << "\n"; } } while (false)
;
5329 LLVM_DEBUG(dbgs() << " Last seen at:\n " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
V << "\n"; } } while (false)
;
5330 return nullptr;
5331 }
5332
5333 case Intrinsic::eh_typeid_for: {
5334 // Find the type id for the given typeinfo.
5335 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5336 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5337 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5338 setValue(&I, Res);
5339 return nullptr;
5340 }
5341
5342 case Intrinsic::eh_return_i32:
5343 case Intrinsic::eh_return_i64:
5344 DAG.getMachineFunction().setCallsEHReturn(true);
5345 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5346 MVT::Other,
5347 getControlRoot(),
5348 getValue(I.getArgOperand(0)),
5349 getValue(I.getArgOperand(1))));
5350 return nullptr;
5351 case Intrinsic::eh_unwind_init:
5352 DAG.getMachineFunction().setCallsUnwindInit(true);
5353 return nullptr;
5354 case Intrinsic::eh_dwarf_cfa:
5355 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5356 TLI.getPointerTy(DAG.getDataLayout()),
5357 getValue(I.getArgOperand(0))));
5358 return nullptr;
5359 case Intrinsic::eh_sjlj_callsite: {
5360 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5361 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5362 assert(CI && "Non-constant call site value in eh.sjlj.callsite!")(static_cast <bool> (CI && "Non-constant call site value in eh.sjlj.callsite!"
) ? void (0) : __assert_fail ("CI && \"Non-constant call site value in eh.sjlj.callsite!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5362, __extension__ __PRETTY_FUNCTION__))
;
5363 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!")(static_cast <bool> (MMI.getCurrentCallSite() == 0 &&
"Overlapping call sites!") ? void (0) : __assert_fail ("MMI.getCurrentCallSite() == 0 && \"Overlapping call sites!\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5363, __extension__ __PRETTY_FUNCTION__))
;
5364
5365 MMI.setCurrentCallSite(CI->getZExtValue());
5366 return nullptr;
5367 }
5368 case Intrinsic::eh_sjlj_functioncontext: {
5369 // Get and store the index of the function context.
5370 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5371 AllocaInst *FnCtx =
5372 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5373 int FI = FuncInfo.StaticAllocaMap[FnCtx];
5374 MFI.setFunctionContextIndex(FI);
5375 return nullptr;
5376 }
5377 case Intrinsic::eh_sjlj_setjmp: {
5378 SDValue Ops[2];
5379 Ops[0] = getRoot();
5380 Ops[1] = getValue(I.getArgOperand(0));
5381 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5382 DAG.getVTList(MVT::i32, MVT::Other), Ops);
5383 setValue(&I, Op.getValue(0));
5384 DAG.setRoot(Op.getValue(1));
5385 return nullptr;
5386 }
5387 case Intrinsic::eh_sjlj_longjmp:
5388 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5389 getRoot(), getValue(I.getArgOperand(0))));
5390 return nullptr;
5391 case Intrinsic::eh_sjlj_setup_dispatch:
5392 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5393 getRoot()));
5394 return nullptr;
5395 case Intrinsic::masked_gather:
5396 visitMaskedGather(I);
5397 return nullptr;
5398 case Intrinsic::masked_load:
5399 visitMaskedLoad(I);
5400 return nullptr;
5401 case Intrinsic::masked_scatter:
5402 visitMaskedScatter(I);
5403 return nullptr;
5404 case Intrinsic::masked_store:
5405 visitMaskedStore(I);
5406 return nullptr;
5407 case Intrinsic::masked_expandload:
5408 visitMaskedLoad(I, true /* IsExpanding */);
5409 return nullptr;
5410 case Intrinsic::masked_compressstore:
5411 visitMaskedStore(I, true /* IsCompressing */);
5412 return nullptr;
5413 case Intrinsic::x86_mmx_pslli_w:
5414 case Intrinsic::x86_mmx_pslli_d:
5415 case Intrinsic::x86_mmx_pslli_q:
5416 case Intrinsic::x86_mmx_psrli_w:
5417 case Intrinsic::x86_mmx_psrli_d:
5418 case Intrinsic::x86_mmx_psrli_q:
5419 case Intrinsic::x86_mmx_psrai_w:
5420 case Intrinsic::x86_mmx_psrai_d: {
5421 SDValue ShAmt = getValue(I.getArgOperand(1));
5422 if (isa<ConstantSDNode>(ShAmt)) {
5423 visitTargetIntrinsic(I, Intrinsic);
5424 return nullptr;
5425 }
5426 unsigned NewIntrinsic = 0;
5427 EVT ShAmtVT = MVT::v2i32;
5428 switch (Intrinsic) {
5429 case Intrinsic::x86_mmx_pslli_w:
5430 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5431 break;
5432 case Intrinsic::x86_mmx_pslli_d:
5433 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5434 break;
5435 case Intrinsic::x86_mmx_pslli_q:
5436 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5437 break;
5438 case Intrinsic::x86_mmx_psrli_w:
5439 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5440 break;
5441 case Intrinsic::x86_mmx_psrli_d:
5442 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5443 break;
5444 case Intrinsic::x86_mmx_psrli_q:
5445 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5446 break;
5447 case Intrinsic::x86_mmx_psrai_w:
5448 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5449 break;
5450 case Intrinsic::x86_mmx_psrai_d:
5451 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5452 break;
5453 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5453)
; // Can't reach here.
5454 }
5455
5456 // The vector shift intrinsics with scalars uses 32b shift amounts but
5457 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5458 // to be zero.
5459 // We must do this early because v2i32 is not a legal type.
5460 SDValue ShOps[2];
5461 ShOps[0] = ShAmt;
5462 ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5463 ShAmt = DAG.getBuildVector(ShAmtVT, sdl, ShOps);
5464 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5465 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5466 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5467 DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5468 getValue(I.getArgOperand(0)), ShAmt);
5469 setValue(&I, Res);
5470 return nullptr;
5471 }
5472 case Intrinsic::powi:
5473 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5474 getValue(I.getArgOperand(1)), DAG));
5475 return nullptr;
5476 case Intrinsic::log:
5477 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5478 return nullptr;
5479 case Intrinsic::log2:
5480 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5481 return nullptr;
5482 case Intrinsic::log10:
5483 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5484 return nullptr;
5485 case Intrinsic::exp:
5486 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5487 return nullptr;
5488 case Intrinsic::exp2:
5489 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5490 return nullptr;
5491 case Intrinsic::pow:
5492 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5493 getValue(I.getArgOperand(1)), DAG, TLI));
5494 return nullptr;
5495 case Intrinsic::sqrt:
5496 case Intrinsic::fabs:
5497 case Intrinsic::sin:
5498 case Intrinsic::cos:
5499 case Intrinsic::floor:
5500 case Intrinsic::ceil:
5501 case Intrinsic::trunc:
5502 case Intrinsic::rint:
5503 case Intrinsic::nearbyint:
5504 case Intrinsic::round:
5505 case Intrinsic::canonicalize: {
5506 unsigned Opcode;
5507 switch (Intrinsic) {
5508 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5508)
; // Can't reach here.
5509 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
5510 case Intrinsic::fabs: Opcode = ISD::FABS; break;
5511 case Intrinsic::sin: Opcode = ISD::FSIN; break;
5512 case Intrinsic::cos: Opcode = ISD::FCOS; break;
5513 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
5514 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
5515 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
5516 case Intrinsic::rint: Opcode = ISD::FRINT; break;
5517 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5518 case Intrinsic::round: Opcode = ISD::FROUND; break;
5519 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5520 }
5521
5522 setValue(&I, DAG.getNode(Opcode, sdl,
5523 getValue(I.getArgOperand(0)).getValueType(),
5524 getValue(I.getArgOperand(0))));
5525 return nullptr;
5526 }
5527 case Intrinsic::minnum: {
5528 auto VT = getValue(I.getArgOperand(0)).getValueType();
5529 unsigned Opc =
5530 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT)
5531 ? ISD::FMINNAN
5532 : ISD::FMINNUM;
5533 setValue(&I, DAG.getNode(Opc, sdl, VT,
5534 getValue(I.getArgOperand(0)),
5535 getValue(I.getArgOperand(1))));
5536 return nullptr;
5537 }
5538 case Intrinsic::maxnum: {
5539 auto VT = getValue(I.getArgOperand(0)).getValueType();
5540 unsigned Opc =
5541 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT)
5542 ? ISD::FMAXNAN
5543 : ISD::FMAXNUM;
5544 setValue(&I, DAG.getNode(Opc, sdl, VT,
5545 getValue(I.getArgOperand(0)),
5546 getValue(I.getArgOperand(1))));
5547 return nullptr;
5548 }
5549 case Intrinsic::copysign:
5550 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5551 getValue(I.getArgOperand(0)).getValueType(),
5552 getValue(I.getArgOperand(0)),
5553 getValue(I.getArgOperand(1))));
5554 return nullptr;
5555 case Intrinsic::fma:
5556 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5557 getValue(I.getArgOperand(0)).getValueType(),
5558 getValue(I.getArgOperand(0)),
5559 getValue(I.getArgOperand(1)),
5560 getValue(I.getArgOperand(2))));
5561 return nullptr;
5562 case Intrinsic::experimental_constrained_fadd:
5563 case Intrinsic::experimental_constrained_fsub:
5564 case Intrinsic::experimental_constrained_fmul:
5565 case Intrinsic::experimental_constrained_fdiv:
5566 case Intrinsic::experimental_constrained_frem:
5567 case Intrinsic::experimental_constrained_fma:
5568 case Intrinsic::experimental_constrained_sqrt:
5569 case Intrinsic::experimental_constrained_pow:
5570 case Intrinsic::experimental_constrained_powi:
5571 case Intrinsic::experimental_constrained_sin:
5572 case Intrinsic::experimental_constrained_cos:
5573 case Intrinsic::experimental_constrained_exp:
5574 case Intrinsic::experimental_constrained_exp2:
5575 case Intrinsic::experimental_constrained_log:
5576 case Intrinsic::experimental_constrained_log10:
5577 case Intrinsic::experimental_constrained_log2:
5578 case Intrinsic::experimental_constrained_rint:
5579 case Intrinsic::experimental_constrained_nearbyint:
5580 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
5581 return nullptr;
5582 case Intrinsic::fmuladd: {
5583 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5584 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5585 TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5586 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5587 getValue(I.getArgOperand(0)).getValueType(),
5588 getValue(I.getArgOperand(0)),
5589 getValue(I.getArgOperand(1)),
5590 getValue(I.getArgOperand(2))));
5591 } else {
5592 // TODO: Intrinsic calls should have fast-math-flags.
5593 SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5594 getValue(I.getArgOperand(0)).getValueType(),
5595 getValue(I.getArgOperand(0)),
5596 getValue(I.getArgOperand(1)));
5597 SDValue Add = DAG.getNode(ISD::FADD, sdl,
5598 getValue(I.getArgOperand(0)).getValueType(),
5599 Mul,
5600 getValue(I.getArgOperand(2)));
5601 setValue(&I, Add);
5602 }
5603 return nullptr;
5604 }
5605 case Intrinsic::convert_to_fp16:
5606 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5607 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5608 getValue(I.getArgOperand(0)),
5609 DAG.getTargetConstant(0, sdl,
5610 MVT::i32))));
5611 return nullptr;
5612 case Intrinsic::convert_from_fp16:
5613 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5614 TLI.getValueType(DAG.getDataLayout(), I.getType()),
5615 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5616 getValue(I.getArgOperand(0)))));
5617 return nullptr;
5618 case Intrinsic::pcmarker: {
5619 SDValue Tmp = getValue(I.getArgOperand(0));
5620 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5621 return nullptr;
5622 }
5623 case Intrinsic::readcyclecounter: {
5624 SDValue Op = getRoot();
5625 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5626 DAG.getVTList(MVT::i64, MVT::Other), Op);
5627 setValue(&I, Res);
5628 DAG.setRoot(Res.getValue(1));
5629 return nullptr;
5630 }
5631 case Intrinsic::bitreverse:
5632 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5633 getValue(I.getArgOperand(0)).getValueType(),
5634 getValue(I.getArgOperand(0))));
5635 return nullptr;
5636 case Intrinsic::bswap:
5637 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5638 getValue(I.getArgOperand(0)).getValueType(),
5639 getValue(I.getArgOperand(0))));
5640 return nullptr;
5641 case Intrinsic::cttz: {
5642 SDValue Arg = getValue(I.getArgOperand(0));
5643 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5644 EVT Ty = Arg.getValueType();
5645 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5646 sdl, Ty, Arg));
5647 return nullptr;
5648 }
5649 case Intrinsic::ctlz: {
5650 SDValue Arg = getValue(I.getArgOperand(0));
5651 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5652 EVT Ty = Arg.getValueType();
5653 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5654 sdl, Ty, Arg));
5655 return nullptr;
5656 }
5657 case Intrinsic::ctpop: {
5658 SDValue Arg = getValue(I.getArgOperand(0));
5659 EVT Ty = Arg.getValueType();
5660 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5661 return nullptr;
5662 }
5663 case Intrinsic::fshl:
5664 case Intrinsic::fshr: {
5665 bool IsFSHL = Intrinsic == Intrinsic::fshl;
5666 SDValue X = getValue(I.getArgOperand(0));
5667 SDValue Y = getValue(I.getArgOperand(1));
5668 SDValue Z = getValue(I.getArgOperand(2));
5669 EVT VT = X.getValueType();
5670
5671 // TODO: When X == Y, this is rotate. Create the node directly if legal.
5672
5673 // Get the shift amount and inverse shift amount, modulo the bit-width.
5674 SDValue BitWidthC = DAG.getConstant(VT.getScalarSizeInBits(), sdl, VT);
5675 SDValue ShAmt = DAG.getNode(ISD::UREM, sdl, VT, Z, BitWidthC);
5676 SDValue NegZ = DAG.getNode(ISD::SUB, sdl, VT, BitWidthC, Z);
5677 SDValue InvShAmt = DAG.getNode(ISD::UREM, sdl, VT, NegZ, BitWidthC);
5678
5679 // fshl: (X << (Z % BW)) | (Y >> ((BW - Z) % BW))
5680 // fshr: (X << ((BW - Z) % BW)) | (Y >> (Z % BW))
5681 SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : InvShAmt);
5682 SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, Y, IsFSHL ? InvShAmt : ShAmt);
5683 SDValue Res = DAG.getNode(ISD::OR, sdl, VT, ShX, ShY);
5684
5685 // If (Z % BW == 0), then (BW - Z) % BW is also zero, so the result would
5686 // be X | Y. If X == Y (rotate), that's fine. If not, we have to select.
5687 if (X != Y) {
5688 SDValue Zero = DAG.getConstant(0, sdl, VT);
5689 EVT CCVT = MVT::i1;
5690 if (VT.isVector())
5691 CCVT = EVT::getVectorVT(*Context, CCVT, VT.getVectorNumElements());
5692 // For fshl, 0 shift returns the 1st arg (X).
5693 // For fshr, 0 shift returns the 2nd arg (Y).
5694 SDValue IsZeroShift = DAG.getSetCC(sdl, CCVT, ShAmt, Zero, ISD::SETEQ);
5695 Res = DAG.getSelect(sdl, VT, IsZeroShift, IsFSHL ? X : Y, Res);
5696 }
5697 setValue(&I, Res);
5698 return nullptr;
5699 }
5700 case Intrinsic::stacksave: {
5701 SDValue Op = getRoot();
5702 Res = DAG.getNode(
5703 ISD::STACKSAVE, sdl,
5704 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5705 setValue(&I, Res);
5706 DAG.setRoot(Res.getValue(1));
5707 return nullptr;
5708 }
5709 case Intrinsic::stackrestore:
5710 Res = getValue(I.getArgOperand(0));
5711 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5712 return nullptr;
5713 case Intrinsic::get_dynamic_area_offset: {
5714 SDValue Op = getRoot();
5715 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5716 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5717 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5718 // target.
5719 if (PtrTy != ResTy)
5720 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5721 " intrinsic!");
5722 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5723 Op);
5724 DAG.setRoot(Op);
5725 setValue(&I, Res);
5726 return nullptr;
5727 }
5728 case Intrinsic::stackguard: {
5729 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5730 MachineFunction &MF = DAG.getMachineFunction();
5731 const Module &M = *MF.getFunction().getParent();
5732 SDValue Chain = getRoot();
5733 if (TLI.useLoadStackGuardNode()) {
5734 Res = getLoadStackGuard(DAG, sdl, Chain);
5735 } else {
5736 const Value *Global = TLI.getSDagStackGuard(M);
5737 unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5738 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5739 MachinePointerInfo(Global, 0), Align,
5740 MachineMemOperand::MOVolatile);
5741 }
5742 if (TLI.useStackGuardXorFP())
5743 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
5744 DAG.setRoot(Chain);
5745 setValue(&I, Res);
5746 return nullptr;
5747 }
5748 case Intrinsic::stackprotector: {
5749 // Emit code into the DAG to store the stack guard onto the stack.
5750 MachineFunction &MF = DAG.getMachineFunction();
5751 MachineFrameInfo &MFI = MF.getFrameInfo();
5752 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5753 SDValue Src, Chain = getRoot();
5754
5755 if (TLI.useLoadStackGuardNode())
5756 Src = getLoadStackGuard(DAG, sdl, Chain);
5757 else
5758 Src = getValue(I.getArgOperand(0)); // The guard's value.
5759
5760 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5761
5762 int FI = FuncInfo.StaticAllocaMap[Slot];
5763 MFI.setStackProtectorIndex(FI);
5764
5765 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5766
5767 // Store the stack protector onto the stack.
5768 Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5769 DAG.getMachineFunction(), FI),
5770 /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5771 setValue(&I, Res);
5772 DAG.setRoot(Res);
5773 return nullptr;
5774 }
5775 case Intrinsic::objectsize: {
5776 // If we don't know by now, we're never going to know.
5777 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5778
5779 assert(CI && "Non-constant type in __builtin_object_size?")(static_cast <bool> (CI && "Non-constant type in __builtin_object_size?"
) ? void (0) : __assert_fail ("CI && \"Non-constant type in __builtin_object_size?\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5779, __extension__ __PRETTY_FUNCTION__))
;
5780
5781 SDValue Arg = getValue(I.getCalledValue());
5782 EVT Ty = Arg.getValueType();
5783
5784 if (CI->isZero())
5785 Res = DAG.getConstant(-1ULL, sdl, Ty);
5786 else
5787 Res = DAG.getConstant(0, sdl, Ty);
5788
5789 setValue(&I, Res);
5790 return nullptr;
5791 }
5792 case Intrinsic::annotation:
5793 case Intrinsic::ptr_annotation:
5794 case Intrinsic::launder_invariant_group:
5795 case Intrinsic::strip_invariant_group:
5796 // Drop the intrinsic, but forward the value
5797 setValue(&I, getValue(I.getOperand(0)));
5798 return nullptr;
5799 case Intrinsic::assume:
5800 case Intrinsic::var_annotation:
5801 case Intrinsic::sideeffect:
5802 // Discard annotate attributes, assumptions, and artificial side-effects.
5803 return nullptr;
5804
5805 case Intrinsic::codeview_annotation: {
5806 // Emit a label associated with this metadata.
5807 MachineFunction &MF = DAG.getMachineFunction();
5808 MCSymbol *Label =
5809 MF.getMMI().getContext().createTempSymbol("annotation", true);
5810 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
5811 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
5812 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
5813 DAG.setRoot(Res);
5814 return nullptr;
5815 }
5816
5817 case Intrinsic::init_trampoline: {
5818 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5819
5820 SDValue Ops[6];
5821 Ops[0] = getRoot();
5822 Ops[1] = getValue(I.getArgOperand(0));
5823 Ops[2] = getValue(I.getArgOperand(1));
5824 Ops[3] = getValue(I.getArgOperand(2));
5825 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5826 Ops[5] = DAG.getSrcValue(F);
5827
5828 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5829
5830 DAG.setRoot(Res);
5831 return nullptr;
5832 }
5833 case Intrinsic::adjust_trampoline:
5834 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5835 TLI.getPointerTy(DAG.getDataLayout()),
5836 getValue(I.getArgOperand(0))));
5837 return nullptr;
5838 case Intrinsic::gcroot: {
5839 assert(DAG.getMachineFunction().getFunction().hasGC() &&(static_cast <bool> (DAG.getMachineFunction().getFunction
().hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5840, __extension__ __PRETTY_FUNCTION__))
5840 "only valid in functions with gc specified, enforced by Verifier")(static_cast <bool> (DAG.getMachineFunction().getFunction
().hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5840, __extension__ __PRETTY_FUNCTION__))
;
5841 assert(GFI && "implied by previous")(static_cast <bool> (GFI && "implied by previous"
) ? void (0) : __assert_fail ("GFI && \"implied by previous\""
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5841, __extension__ __PRETTY_FUNCTION__))
;
5842 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5843 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5844
5845 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5846 GFI->addStackRoot(FI->getIndex(), TypeMap);
5847 return nullptr;
5848 }
5849 case Intrinsic::gcread:
5850 case Intrinsic::gcwrite:
5851 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!")::llvm::llvm_unreachable_internal("GC failed to lower gcread/gcwrite intrinsics!"
, "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5851)
;
5852 case Intrinsic::flt_rounds:
5853 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
5854 return nullptr;
5855
5856 case Intrinsic::expect:
5857 // Just replace __builtin_expect(exp, c) with EXP.
5858 setValue(&I, getValue(I.getArgOperand(0)));
5859 return nullptr;
5860
5861 case Intrinsic::debugtrap:
5862 case Intrinsic::trap: {
5863 StringRef TrapFuncName =
5864 I.getAttributes()
5865 .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
5866 .getValueAsString();
5867 if (TrapFuncName.empty()) {
5868 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
5869 ISD::TRAP : ISD::DEBUGTRAP;
5870 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
5871 return nullptr;
5872 }
5873 TargetLowering::ArgListTy Args;
5874
5875 TargetLowering::CallLoweringInfo CLI(DAG);
5876 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5877 CallingConv::C, I.getType(),
5878 DAG.getExternalSymbol(TrapFuncName.data(),
5879 TLI.getPointerTy(DAG.getDataLayout())),
5880 std::move(Args));
5881
5882 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5883 DAG.setRoot(Result.second);
5884 return nullptr;
5885 }
5886
5887 case Intrinsic::uadd_with_overflow:
5888 case Intrinsic::sadd_with_overflow:
5889 case Intrinsic::usub_with_overflow:
5890 case Intrinsic::ssub_with_overflow:
5891 case Intrinsic::umul_with_overflow:
5892 case Intrinsic::smul_with_overflow: {
5893 ISD::NodeType Op;
5894 switch (Intrinsic) {
5895 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-7~svn337657/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5895)
; // Can't reach here.
5896 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
5897 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
5898 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
5899 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
5900 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
5901 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
5902 }
5903 SDValue Op1 = getValue(I.getArgOperand(0));
5904 SDValue Op2 = getValue(I.getArgOperand(1));
5905
5906 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
5907 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
5908 return nullptr;
5909 }
5910 case Intrinsic::prefetch: {
5911 SDValue Ops[5];
5912 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
5913 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
5914 Ops[0] = DAG.getRoot();
5915 Ops[1] = getValue(I.getArgOperand(0));
5916 Ops[2] = getValue(I.getArgOperand(1));
5917 Ops[3] = getValue(I.getArgOperand(2));
5918 Ops[4] = getValue(I.getArgOperand(3));
5919 SDValue Result = DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
5920 DAG.getVTList(MVT::Other), Ops,
5921 EVT::getIntegerVT(*Context, 8),
5922 MachinePointerInfo(I.getArgOperand(0)),
5923