Bug Summary

File:lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Warning:line 6743, column 14
Called C++ object pointer is null

Annotated Source Code

1//===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements routines for translating from LLVM IR into SelectionDAG IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SelectionDAGBuilder.h"
15#include "SDNodeDbgValue.h"
16#include "llvm/ADT/BitVector.h"
17#include "llvm/ADT/Optional.h"
18#include "llvm/ADT/SmallSet.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/Analysis/AliasAnalysis.h"
21#include "llvm/Analysis/BranchProbabilityInfo.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/Loads.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/Analysis/VectorUtils.h"
27#include "llvm/CodeGen/Analysis.h"
28#include "llvm/CodeGen/FastISel.h"
29#include "llvm/CodeGen/FunctionLoweringInfo.h"
30#include "llvm/CodeGen/GCMetadata.h"
31#include "llvm/CodeGen/GCStrategy.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineJumpTableInfo.h"
36#include "llvm/CodeGen/MachineModuleInfo.h"
37#include "llvm/CodeGen/MachineRegisterInfo.h"
38#include "llvm/CodeGen/SelectionDAG.h"
39#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
40#include "llvm/CodeGen/StackMaps.h"
41#include "llvm/CodeGen/WinEHFuncInfo.h"
42#include "llvm/IR/CallingConv.h"
43#include "llvm/IR/ConstantRange.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/DebugInfo.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/GetElementPtrTypeIterator.h"
50#include "llvm/IR/GlobalVariable.h"
51#include "llvm/IR/InlineAsm.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/IntrinsicInst.h"
54#include "llvm/IR/Intrinsics.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/Statepoint.h"
58#include "llvm/MC/MCSymbol.h"
59#include "llvm/Support/CommandLine.h"
60#include "llvm/Support/Debug.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/MathExtras.h"
63#include "llvm/Support/raw_ostream.h"
64#include "llvm/Target/TargetFrameLowering.h"
65#include "llvm/Target/TargetInstrInfo.h"
66#include "llvm/Target/TargetIntrinsicInfo.h"
67#include "llvm/Target/TargetLowering.h"
68#include "llvm/Target/TargetOptions.h"
69#include "llvm/Target/TargetSubtargetInfo.h"
70#include <algorithm>
71#include <utility>
72using namespace llvm;
73
74#define DEBUG_TYPE"isel" "isel"
75
76/// LimitFloatPrecision - Generate low-precision inline sequences for
77/// some float libcalls (6, 8 or 12 bits).
78static unsigned LimitFloatPrecision;
79
80static cl::opt<unsigned, true>
81LimitFPPrecision("limit-float-precision",
82 cl::desc("Generate low-precision inline sequences "
83 "for some float libcalls"),
84 cl::location(LimitFloatPrecision),
85 cl::init(0));
86
87static cl::opt<bool>
88EnableFMFInDAG("enable-fmf-dag", cl::init(true), cl::Hidden,
89 cl::desc("Enable fast-math-flags for DAG nodes"));
90
91/// Minimum jump table density for normal functions.
92static cl::opt<unsigned>
93JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
94 cl::desc("Minimum density for building a jump table in "
95 "a normal function"));
96
97/// Minimum jump table density for -Os or -Oz functions.
98static cl::opt<unsigned>
99OptsizeJumpTableDensity("optsize-jump-table-density", cl::init(40), cl::Hidden,
100 cl::desc("Minimum density for building a jump table in "
101 "an optsize function"));
102
103
104// Limit the width of DAG chains. This is important in general to prevent
105// DAG-based analysis from blowing up. For example, alias analysis and
106// load clustering may not complete in reasonable time. It is difficult to
107// recognize and avoid this situation within each individual analysis, and
108// future analyses are likely to have the same behavior. Limiting DAG width is
109// the safe approach and will be especially important with global DAGs.
110//
111// MaxParallelChains default is arbitrarily high to avoid affecting
112// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
113// sequence over this should have been converted to llvm.memcpy by the
114// frontend. It is easy to induce this behavior with .ll code such as:
115// %buffer = alloca [4096 x i8]
116// %data = load [4096 x i8]* %argPtr
117// store [4096 x i8] %data, [4096 x i8]* %buffer
118static const unsigned MaxParallelChains = 64;
119
120static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
121 const SDValue *Parts, unsigned NumParts,
122 MVT PartVT, EVT ValueVT, const Value *V);
123
124/// getCopyFromParts - Create a value that contains the specified legal parts
125/// combined into the value they represent. If the parts combine to a type
126/// larger than ValueVT then AssertOp can be used to specify whether the extra
127/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
128/// (ISD::AssertSext).
129static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
130 const SDValue *Parts, unsigned NumParts,
131 MVT PartVT, EVT ValueVT, const Value *V,
132 Optional<ISD::NodeType> AssertOp = None) {
133 if (ValueVT.isVector())
134 return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
135 PartVT, ValueVT, V);
136
137 assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 137, __PRETTY_FUNCTION__))
;
138 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
139 SDValue Val = Parts[0];
140
141 if (NumParts > 1) {
142 // Assemble the value from multiple parts.
143 if (ValueVT.isInteger()) {
144 unsigned PartBits = PartVT.getSizeInBits();
145 unsigned ValueBits = ValueVT.getSizeInBits();
146
147 // Assemble the power of 2 part.
148 unsigned RoundParts = NumParts & (NumParts - 1) ?
149 1 << Log2_32(NumParts) : NumParts;
150 unsigned RoundBits = PartBits * RoundParts;
151 EVT RoundVT = RoundBits == ValueBits ?
152 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
153 SDValue Lo, Hi;
154
155 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
156
157 if (RoundParts > 2) {
158 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
159 PartVT, HalfVT, V);
160 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
161 RoundParts / 2, PartVT, HalfVT, V);
162 } else {
163 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
164 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
165 }
166
167 if (DAG.getDataLayout().isBigEndian())
168 std::swap(Lo, Hi);
169
170 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
171
172 if (RoundParts < NumParts) {
173 // Assemble the trailing non-power-of-2 part.
174 unsigned OddParts = NumParts - RoundParts;
175 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
176 Hi = getCopyFromParts(DAG, DL,
177 Parts + RoundParts, OddParts, PartVT, OddVT, V);
178
179 // Combine the round and odd parts.
180 Lo = Val;
181 if (DAG.getDataLayout().isBigEndian())
182 std::swap(Lo, Hi);
183 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
184 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
185 Hi =
186 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
187 DAG.getConstant(Lo.getValueSizeInBits(), DL,
188 TLI.getPointerTy(DAG.getDataLayout())));
189 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
190 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
191 }
192 } else if (PartVT.isFloatingPoint()) {
193 // FP split into multiple FP parts (for ppcf128)
194 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 195, __PRETTY_FUNCTION__))
195 "Unexpected split")((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 195, __PRETTY_FUNCTION__))
;
196 SDValue Lo, Hi;
197 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
198 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
199 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
200 std::swap(Lo, Hi);
201 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
202 } else {
203 // FP split into integer parts (soft fp)
204 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 205, __PRETTY_FUNCTION__))
205 !PartVT.isVector() && "Unexpected split")((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 205, __PRETTY_FUNCTION__))
;
206 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
207 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
208 }
209 }
210
211 // There is now one part, held in Val. Correct it to match ValueVT.
212 // PartEVT is the type of the register class that holds the value.
213 // ValueVT is the type of the inline asm operation.
214 EVT PartEVT = Val.getValueType();
215
216 if (PartEVT == ValueVT)
217 return Val;
218
219 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
220 ValueVT.bitsLT(PartEVT)) {
221 // For an FP value in an integer part, we need to truncate to the right
222 // width first.
223 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
224 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
225 }
226
227 // Handle types that have the same size.
228 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
229 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
230
231 // Handle types with different sizes.
232 if (PartEVT.isInteger() && ValueVT.isInteger()) {
233 if (ValueVT.bitsLT(PartEVT)) {
234 // For a truncate, see if we have any information to
235 // indicate whether the truncated bits will always be
236 // zero or sign-extension.
237 if (AssertOp.hasValue())
238 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
239 DAG.getValueType(ValueVT));
240 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
241 }
242 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
243 }
244
245 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
246 // FP_ROUND's are always exact here.
247 if (ValueVT.bitsLT(Val.getValueType()))
248 return DAG.getNode(
249 ISD::FP_ROUND, DL, ValueVT, Val,
250 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
251
252 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
253 }
254
255 llvm_unreachable("Unknown mismatch!")::llvm::llvm_unreachable_internal("Unknown mismatch!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 255)
;
256}
257
258static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
259 const Twine &ErrMsg) {
260 const Instruction *I = dyn_cast_or_null<Instruction>(V);
261 if (!V)
262 return Ctx.emitError(ErrMsg);
263
264 const char *AsmError = ", possible invalid constraint for vector type";
265 if (const CallInst *CI = dyn_cast<CallInst>(I))
266 if (isa<InlineAsm>(CI->getCalledValue()))
267 return Ctx.emitError(I, ErrMsg + AsmError);
268
269 return Ctx.emitError(I, ErrMsg);
270}
271
272/// getCopyFromPartsVector - Create a value that contains the specified legal
273/// parts combined into the value they represent. If the parts combine to a
274/// type larger than ValueVT then AssertOp can be used to specify whether the
275/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
276/// ValueVT (ISD::AssertSext).
277static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
278 const SDValue *Parts, unsigned NumParts,
279 MVT PartVT, EVT ValueVT, const Value *V) {
280 assert(ValueVT.isVector() && "Not a vector value")((ValueVT.isVector() && "Not a vector value") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 280, __PRETTY_FUNCTION__))
;
281 assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 281, __PRETTY_FUNCTION__))
;
282 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
283 SDValue Val = Parts[0];
284
285 // Handle a multi-element vector.
286 if (NumParts > 1) {
287 EVT IntermediateVT;
288 MVT RegisterVT;
289 unsigned NumIntermediates;
290 unsigned NumRegs =
291 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
292 NumIntermediates, RegisterVT);
293 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 293, __PRETTY_FUNCTION__))
;
294 NumParts = NumRegs; // Silence a compiler warning.
295 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 295, __PRETTY_FUNCTION__))
;
296 assert(RegisterVT.getSizeInBits() ==((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 298, __PRETTY_FUNCTION__))
297 Parts[0].getSimpleValueType().getSizeInBits() &&((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 298, __PRETTY_FUNCTION__))
298 "Part type sizes don't match!")((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 298, __PRETTY_FUNCTION__))
;
299
300 // Assemble the parts into intermediate operands.
301 SmallVector<SDValue, 8> Ops(NumIntermediates);
302 if (NumIntermediates == NumParts) {
303 // If the register was not expanded, truncate or copy the value,
304 // as appropriate.
305 for (unsigned i = 0; i != NumParts; ++i)
306 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
307 PartVT, IntermediateVT, V);
308 } else if (NumParts > 0) {
309 // If the intermediate type was expanded, build the intermediate
310 // operands from the parts.
311 assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 312, __PRETTY_FUNCTION__))
312 "Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 312, __PRETTY_FUNCTION__))
;
313 unsigned Factor = NumParts / NumIntermediates;
314 for (unsigned i = 0; i != NumIntermediates; ++i)
315 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
316 PartVT, IntermediateVT, V);
317 }
318
319 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
320 // intermediate operands.
321 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
322 : ISD::BUILD_VECTOR,
323 DL, ValueVT, Ops);
324 }
325
326 // There is now one part, held in Val. Correct it to match ValueVT.
327 EVT PartEVT = Val.getValueType();
328
329 if (PartEVT == ValueVT)
330 return Val;
331
332 if (PartEVT.isVector()) {
333 // If the element type of the source/dest vectors are the same, but the
334 // parts vector has more elements than the value vector, then we have a
335 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
336 // elements we want.
337 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
338 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&((PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements
() && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 339, __PRETTY_FUNCTION__))
339 "Cannot narrow, it would be a lossy transformation")((PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements
() && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 339, __PRETTY_FUNCTION__))
;
340 return DAG.getNode(
341 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
342 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
343 }
344
345 // Vector/Vector bitcast.
346 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
347 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
348
349 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&((PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 350, __PRETTY_FUNCTION__))
350 "Cannot handle this kind of promotion")((PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 350, __PRETTY_FUNCTION__))
;
351 // Promoted vector extract
352 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
353
354 }
355
356 // Trivial bitcast if the types are the same size and the destination
357 // vector type is legal.
358 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
359 TLI.isTypeLegal(ValueVT))
360 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
361
362 // Handle cases such as i8 -> <1 x i1>
363 if (ValueVT.getVectorNumElements() != 1) {
364 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
365 "non-trivial scalar-to-vector conversion");
366 return DAG.getUNDEF(ValueVT);
367 }
368
369 if (ValueVT.getVectorNumElements() == 1 &&
370 ValueVT.getVectorElementType() != PartEVT)
371 Val = DAG.getAnyExtOrTrunc(Val, DL, ValueVT.getScalarType());
372
373 return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
374}
375
376static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
377 SDValue Val, SDValue *Parts, unsigned NumParts,
378 MVT PartVT, const Value *V);
379
380/// getCopyToParts - Create a series of nodes that contain the specified value
381/// split into legal parts. If the parts contain more bits than Val, then, for
382/// integers, ExtendKind can be used to specify how to generate the extra bits.
383static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
384 SDValue *Parts, unsigned NumParts, MVT PartVT,
385 const Value *V,
386 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
387 EVT ValueVT = Val.getValueType();
388
389 // Handle the vector case separately.
390 if (ValueVT.isVector())
391 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
392
393 unsigned PartBits = PartVT.getSizeInBits();
394 unsigned OrigNumParts = NumParts;
395 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 396, __PRETTY_FUNCTION__))
396 "Copying to an illegal type!")((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 396, __PRETTY_FUNCTION__))
;
397
398 if (NumParts == 0)
399 return;
400
401 assert(!ValueVT.isVector() && "Vector case handled elsewhere")((!ValueVT.isVector() && "Vector case handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 401, __PRETTY_FUNCTION__))
;
402 EVT PartEVT = PartVT;
403 if (PartEVT == ValueVT) {
404 assert(NumParts == 1 && "No-op copy with multiple parts!")((NumParts == 1 && "No-op copy with multiple parts!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 404, __PRETTY_FUNCTION__))
;
405 Parts[0] = Val;
406 return;
407 }
408
409 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
410 // If the parts cover more bits than the value has, promote the value.
411 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
412 assert(NumParts == 1 && "Do not know what to promote to!")((NumParts == 1 && "Do not know what to promote to!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 412, __PRETTY_FUNCTION__))
;
413 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
414 } else {
415 if (ValueVT.isFloatingPoint()) {
416 // FP values need to be bitcast, then extended if they are being put
417 // into a larger container.
418 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
419 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
420 }
421 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 423, __PRETTY_FUNCTION__))
422 ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 423, __PRETTY_FUNCTION__))
423 "Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 423, __PRETTY_FUNCTION__))
;
424 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
425 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
426 if (PartVT == MVT::x86mmx)
427 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
428 }
429 } else if (PartBits == ValueVT.getSizeInBits()) {
430 // Different types of the same size.
431 assert(NumParts == 1 && PartEVT != ValueVT)((NumParts == 1 && PartEVT != ValueVT) ? static_cast<
void> (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 431, __PRETTY_FUNCTION__))
;
432 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
433 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
434 // If the parts cover less bits than value has, truncate the value.
435 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 437, __PRETTY_FUNCTION__))
436 ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 437, __PRETTY_FUNCTION__))
437 "Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 437, __PRETTY_FUNCTION__))
;
438 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
439 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
440 if (PartVT == MVT::x86mmx)
441 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
442 }
443
444 // The value may have changed - recompute ValueVT.
445 ValueVT = Val.getValueType();
446 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 447, __PRETTY_FUNCTION__))
447 "Failed to tile the value with PartVT!")((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 447, __PRETTY_FUNCTION__))
;
448
449 if (NumParts == 1) {
450 if (PartEVT != ValueVT) {
451 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
452 "scalar-to-vector conversion failed");
453 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
454 }
455
456 Parts[0] = Val;
457 return;
458 }
459
460 // Expand the value into multiple parts.
461 if (NumParts & (NumParts - 1)) {
462 // The number of parts is not a power of 2. Split off and copy the tail.
463 assert(PartVT.isInteger() && ValueVT.isInteger() &&((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 464, __PRETTY_FUNCTION__))
464 "Do not know what to expand to!")((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 464, __PRETTY_FUNCTION__))
;
465 unsigned RoundParts = 1 << Log2_32(NumParts);
466 unsigned RoundBits = RoundParts * PartBits;
467 unsigned OddParts = NumParts - RoundParts;
468 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
469 DAG.getIntPtrConstant(RoundBits, DL));
470 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
471
472 if (DAG.getDataLayout().isBigEndian())
473 // The odd parts were reversed by getCopyToParts - unreverse them.
474 std::reverse(Parts + RoundParts, Parts + NumParts);
475
476 NumParts = RoundParts;
477 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
478 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
479 }
480
481 // The number of parts is a power of 2. Repeatedly bisect the value using
482 // EXTRACT_ELEMENT.
483 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
484 EVT::getIntegerVT(*DAG.getContext(),
485 ValueVT.getSizeInBits()),
486 Val);
487
488 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
489 for (unsigned i = 0; i < NumParts; i += StepSize) {
490 unsigned ThisBits = StepSize * PartBits / 2;
491 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
492 SDValue &Part0 = Parts[i];
493 SDValue &Part1 = Parts[i+StepSize/2];
494
495 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
496 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
497 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
498 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
499
500 if (ThisBits == PartBits && ThisVT != PartVT) {
501 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
502 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
503 }
504 }
505 }
506
507 if (DAG.getDataLayout().isBigEndian())
508 std::reverse(Parts, Parts + OrigNumParts);
509}
510
511
512/// getCopyToPartsVector - Create a series of nodes that contain the specified
513/// value split into legal parts.
514static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
515 SDValue Val, SDValue *Parts, unsigned NumParts,
516 MVT PartVT, const Value *V) {
517 EVT ValueVT = Val.getValueType();
518 assert(ValueVT.isVector() && "Not a vector")((ValueVT.isVector() && "Not a vector") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 518, __PRETTY_FUNCTION__))
;
519 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
520
521 if (NumParts == 1) {
522 EVT PartEVT = PartVT;
523 if (PartEVT == ValueVT) {
524 // Nothing to do.
525 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
526 // Bitconvert vector->vector case.
527 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
528 } else if (PartVT.isVector() &&
529 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
530 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
531 EVT ElementVT = PartVT.getVectorElementType();
532 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
533 // undef elements.
534 SmallVector<SDValue, 16> Ops;
535 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
536 Ops.push_back(DAG.getNode(
537 ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
538 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
539
540 for (unsigned i = ValueVT.getVectorNumElements(),
541 e = PartVT.getVectorNumElements(); i != e; ++i)
542 Ops.push_back(DAG.getUNDEF(ElementVT));
543
544 Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
545
546 // FIXME: Use CONCAT for 2x -> 4x.
547
548 //SDValue UndefElts = DAG.getUNDEF(VectorTy);
549 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
550 } else if (PartVT.isVector() &&
551 PartEVT.getVectorElementType().bitsGE(
552 ValueVT.getVectorElementType()) &&
553 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
554
555 // Promoted vector extract
556 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
557 } else{
558 // Vector -> scalar conversion.
559 assert(ValueVT.getVectorNumElements() == 1 &&((ValueVT.getVectorNumElements() == 1 && "Only trivial vector-to-scalar conversions should get here!"
) ? static_cast<void> (0) : __assert_fail ("ValueVT.getVectorNumElements() == 1 && \"Only trivial vector-to-scalar conversions should get here!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 560, __PRETTY_FUNCTION__))
560 "Only trivial vector-to-scalar conversions should get here!")((ValueVT.getVectorNumElements() == 1 && "Only trivial vector-to-scalar conversions should get here!"
) ? static_cast<void> (0) : __assert_fail ("ValueVT.getVectorNumElements() == 1 && \"Only trivial vector-to-scalar conversions should get here!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 560, __PRETTY_FUNCTION__))
;
561 Val = DAG.getNode(
562 ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
563 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
564
565 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
566 }
567
568 Parts[0] = Val;
569 return;
570 }
571
572 // Handle a multi-element vector.
573 EVT IntermediateVT;
574 MVT RegisterVT;
575 unsigned NumIntermediates;
576 unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
577 IntermediateVT,
578 NumIntermediates, RegisterVT);
579 unsigned NumElements = ValueVT.getVectorNumElements();
580
581 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 581, __PRETTY_FUNCTION__))
;
582 NumParts = NumRegs; // Silence a compiler warning.
583 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 583, __PRETTY_FUNCTION__))
;
584
585 // Split the vector into intermediate operands.
586 SmallVector<SDValue, 8> Ops(NumIntermediates);
587 for (unsigned i = 0; i != NumIntermediates; ++i) {
588 if (IntermediateVT.isVector())
589 Ops[i] =
590 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
591 DAG.getConstant(i * (NumElements / NumIntermediates), DL,
592 TLI.getVectorIdxTy(DAG.getDataLayout())));
593 else
594 Ops[i] = DAG.getNode(
595 ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
596 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
597 }
598
599 // Split the intermediate operands into legal parts.
600 if (NumParts == NumIntermediates) {
601 // If the register was not expanded, promote or copy the value,
602 // as appropriate.
603 for (unsigned i = 0; i != NumParts; ++i)
604 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
605 } else if (NumParts > 0) {
606 // If the intermediate type was expanded, split each the value into
607 // legal parts.
608 assert(NumIntermediates != 0 && "division by zero")((NumIntermediates != 0 && "division by zero") ? static_cast
<void> (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 608, __PRETTY_FUNCTION__))
;
609 assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 610, __PRETTY_FUNCTION__))
610 "Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 610, __PRETTY_FUNCTION__))
;
611 unsigned Factor = NumParts / NumIntermediates;
612 for (unsigned i = 0; i != NumIntermediates; ++i)
613 getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
614 }
615}
616
617RegsForValue::RegsForValue() {}
618
619RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
620 EVT valuevt)
621 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
622
623RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
624 const DataLayout &DL, unsigned Reg, Type *Ty) {
625 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
626
627 for (EVT ValueVT : ValueVTs) {
628 unsigned NumRegs = TLI.getNumRegisters(Context, ValueVT);
629 MVT RegisterVT = TLI.getRegisterType(Context, ValueVT);
630 for (unsigned i = 0; i != NumRegs; ++i)
631 Regs.push_back(Reg + i);
632 RegVTs.push_back(RegisterVT);
633 Reg += NumRegs;
634 }
635}
636
637/// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
638/// this value and returns the result as a ValueVT value. This uses
639/// Chain/Flag as the input and updates them for the output Chain/Flag.
640/// If the Flag pointer is NULL, no flag is used.
641SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
642 FunctionLoweringInfo &FuncInfo,
643 const SDLoc &dl, SDValue &Chain,
644 SDValue *Flag, const Value *V) const {
645 // A Value with type {} or [0 x %t] needs no registers.
646 if (ValueVTs.empty())
647 return SDValue();
648
649 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
650
651 // Assemble the legal parts into the final values.
652 SmallVector<SDValue, 4> Values(ValueVTs.size());
653 SmallVector<SDValue, 8> Parts;
654 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
655 // Copy the legal parts from the registers.
656 EVT ValueVT = ValueVTs[Value];
657 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
658 MVT RegisterVT = RegVTs[Value];
659
660 Parts.resize(NumRegs);
661 for (unsigned i = 0; i != NumRegs; ++i) {
662 SDValue P;
663 if (!Flag) {
664 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
665 } else {
666 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
667 *Flag = P.getValue(2);
668 }
669
670 Chain = P.getValue(1);
671 Parts[i] = P;
672
673 // If the source register was virtual and if we know something about it,
674 // add an assert node.
675 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
676 !RegisterVT.isInteger() || RegisterVT.isVector())
677 continue;
678
679 const FunctionLoweringInfo::LiveOutInfo *LOI =
680 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
681 if (!LOI)
682 continue;
683
684 unsigned RegSize = RegisterVT.getSizeInBits();
685 unsigned NumSignBits = LOI->NumSignBits;
686 unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
687
688 if (NumZeroBits == RegSize) {
689 // The current value is a zero.
690 // Explicitly express that as it would be easier for
691 // optimizations to kick in.
692 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
693 continue;
694 }
695
696 // FIXME: We capture more information than the dag can represent. For
697 // now, just use the tightest assertzext/assertsext possible.
698 bool isSExt = true;
699 EVT FromVT(MVT::Other);
700 if (NumSignBits == RegSize) {
701 isSExt = true; // ASSERT SEXT 1
702 FromVT = MVT::i1;
703 } else if (NumZeroBits >= RegSize - 1) {
704 isSExt = false; // ASSERT ZEXT 1
705 FromVT = MVT::i1;
706 } else if (NumSignBits > RegSize - 8) {
707 isSExt = true; // ASSERT SEXT 8
708 FromVT = MVT::i8;
709 } else if (NumZeroBits >= RegSize - 8) {
710 isSExt = false; // ASSERT ZEXT 8
711 FromVT = MVT::i8;
712 } else if (NumSignBits > RegSize - 16) {
713 isSExt = true; // ASSERT SEXT 16
714 FromVT = MVT::i16;
715 } else if (NumZeroBits >= RegSize - 16) {
716 isSExt = false; // ASSERT ZEXT 16
717 FromVT = MVT::i16;
718 } else if (NumSignBits > RegSize - 32) {
719 isSExt = true; // ASSERT SEXT 32
720 FromVT = MVT::i32;
721 } else if (NumZeroBits >= RegSize - 32) {
722 isSExt = false; // ASSERT ZEXT 32
723 FromVT = MVT::i32;
724 } else {
725 continue;
726 }
727 // Add an assertion node.
728 assert(FromVT != MVT::Other)((FromVT != MVT::Other) ? static_cast<void> (0) : __assert_fail
("FromVT != MVT::Other", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 728, __PRETTY_FUNCTION__))
;
729 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
730 RegisterVT, P, DAG.getValueType(FromVT));
731 }
732
733 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
734 NumRegs, RegisterVT, ValueVT, V);
735 Part += NumRegs;
736 Parts.clear();
737 }
738
739 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
740}
741
742/// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
743/// specified value into the registers specified by this object. This uses
744/// Chain/Flag as the input and updates them for the output Chain/Flag.
745/// If the Flag pointer is NULL, no flag is used.
746void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
747 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
748 const Value *V,
749 ISD::NodeType PreferredExtendType) const {
750 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
751 ISD::NodeType ExtendKind = PreferredExtendType;
752
753 // Get the list of the values's legal parts.
754 unsigned NumRegs = Regs.size();
755 SmallVector<SDValue, 8> Parts(NumRegs);
756 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
757 EVT ValueVT = ValueVTs[Value];
758 unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
759 MVT RegisterVT = RegVTs[Value];
760
761 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
762 ExtendKind = ISD::ZERO_EXTEND;
763
764 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
765 &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
766 Part += NumParts;
767 }
768
769 // Copy the parts into the registers.
770 SmallVector<SDValue, 8> Chains(NumRegs);
771 for (unsigned i = 0; i != NumRegs; ++i) {
772 SDValue Part;
773 if (!Flag) {
774 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
775 } else {
776 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
777 *Flag = Part.getValue(1);
778 }
779
780 Chains[i] = Part.getValue(0);
781 }
782
783 if (NumRegs == 1 || Flag)
784 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
785 // flagged to it. That is the CopyToReg nodes and the user are considered
786 // a single scheduling unit. If we create a TokenFactor and return it as
787 // chain, then the TokenFactor is both a predecessor (operand) of the
788 // user as well as a successor (the TF operands are flagged to the user).
789 // c1, f1 = CopyToReg
790 // c2, f2 = CopyToReg
791 // c3 = TokenFactor c1, c2
792 // ...
793 // = op c3, ..., f2
794 Chain = Chains[NumRegs-1];
795 else
796 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
797}
798
799/// AddInlineAsmOperands - Add this value to the specified inlineasm node
800/// operand list. This adds the code marker and includes the number of
801/// values added into it.
802void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
803 unsigned MatchingIdx, const SDLoc &dl,
804 SelectionDAG &DAG,
805 std::vector<SDValue> &Ops) const {
806 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
807
808 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
809 if (HasMatching)
810 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
811 else if (!Regs.empty() &&
812 TargetRegisterInfo::isVirtualRegister(Regs.front())) {
813 // Put the register class of the virtual registers in the flag word. That
814 // way, later passes can recompute register class constraints for inline
815 // assembly as well as normal instructions.
816 // Don't do this for tied operands that can use the regclass information
817 // from the def.
818 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
819 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
820 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
821 }
822
823 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
824 Ops.push_back(Res);
825
826 unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
827 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
828 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
829 MVT RegisterVT = RegVTs[Value];
830 for (unsigned i = 0; i != NumRegs; ++i) {
831 assert(Reg < Regs.size() && "Mismatch in # registers expected")((Reg < Regs.size() && "Mismatch in # registers expected"
) ? static_cast<void> (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 831, __PRETTY_FUNCTION__))
;
832 unsigned TheReg = Regs[Reg++];
833 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
834
835 if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
836 // If we clobbered the stack pointer, MFI should know about it.
837 assert(DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment())((DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) ? static_cast<void> (0) : __assert_fail ("DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 837, __PRETTY_FUNCTION__))
;
838 }
839 }
840 }
841}
842
843void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
844 const TargetLibraryInfo *li) {
845 AA = &aa;
846 GFI = gfi;
847 LibInfo = li;
848 DL = &DAG.getDataLayout();
849 Context = DAG.getContext();
850 LPadToCallSiteMap.clear();
851}
852
853/// clear - Clear out the current SelectionDAG and the associated
854/// state and prepare this SelectionDAGBuilder object to be used
855/// for a new block. This doesn't clear out information about
856/// additional blocks that are needed to complete switch lowering
857/// or PHI node updating; that information is cleared out as it is
858/// consumed.
859void SelectionDAGBuilder::clear() {
860 NodeMap.clear();
861 UnusedArgNodeMap.clear();
862 PendingLoads.clear();
863 PendingExports.clear();
864 CurInst = nullptr;
865 HasTailCall = false;
866 SDNodeOrder = LowestSDNodeOrder;
867 StatepointLowering.clear();
868}
869
870/// clearDanglingDebugInfo - Clear the dangling debug information
871/// map. This function is separated from the clear so that debug
872/// information that is dangling in a basic block can be properly
873/// resolved in a different basic block. This allows the
874/// SelectionDAG to resolve dangling debug information attached
875/// to PHI nodes.
876void SelectionDAGBuilder::clearDanglingDebugInfo() {
877 DanglingDebugInfoMap.clear();
878}
879
880/// getRoot - Return the current virtual root of the Selection DAG,
881/// flushing any PendingLoad items. This must be done before emitting
882/// a store or any other node that may need to be ordered after any
883/// prior load instructions.
884///
885SDValue SelectionDAGBuilder::getRoot() {
886 if (PendingLoads.empty())
887 return DAG.getRoot();
888
889 if (PendingLoads.size() == 1) {
890 SDValue Root = PendingLoads[0];
891 DAG.setRoot(Root);
892 PendingLoads.clear();
893 return Root;
894 }
895
896 // Otherwise, we have to make a token factor node.
897 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
898 PendingLoads);
899 PendingLoads.clear();
900 DAG.setRoot(Root);
901 return Root;
902}
903
904/// getControlRoot - Similar to getRoot, but instead of flushing all the
905/// PendingLoad items, flush all the PendingExports items. It is necessary
906/// to do this before emitting a terminator instruction.
907///
908SDValue SelectionDAGBuilder::getControlRoot() {
909 SDValue Root = DAG.getRoot();
910
911 if (PendingExports.empty())
912 return Root;
913
914 // Turn all of the CopyToReg chains into one factored node.
915 if (Root.getOpcode() != ISD::EntryToken) {
916 unsigned i = 0, e = PendingExports.size();
917 for (; i != e; ++i) {
918 assert(PendingExports[i].getNode()->getNumOperands() > 1)((PendingExports[i].getNode()->getNumOperands() > 1) ? static_cast
<void> (0) : __assert_fail ("PendingExports[i].getNode()->getNumOperands() > 1"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 918, __PRETTY_FUNCTION__))
;
919 if (PendingExports[i].getNode()->getOperand(0) == Root)
920 break; // Don't add the root if we already indirectly depend on it.
921 }
922
923 if (i == e)
924 PendingExports.push_back(Root);
925 }
926
927 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
928 PendingExports);
929 PendingExports.clear();
930 DAG.setRoot(Root);
931 return Root;
932}
933
934void SelectionDAGBuilder::visit(const Instruction &I) {
935 // Set up outgoing PHI node register values before emitting the terminator.
936 if (isa<TerminatorInst>(&I)) {
937 HandlePHINodesInSuccessorBlocks(I.getParent());
938 }
939
940 // Increase the SDNodeOrder if dealing with a non-debug instruction.
941 if (!isa<DbgInfoIntrinsic>(I))
942 ++SDNodeOrder;
943
944 CurInst = &I;
945
946 visit(I.getOpcode(), I);
947
948 if (!isa<TerminatorInst>(&I) && !HasTailCall &&
949 !isStatepoint(&I)) // statepoints handle their exports internally
950 CopyToExportRegsIfNeeded(&I);
951
952 CurInst = nullptr;
953}
954
955void SelectionDAGBuilder::visitPHI(const PHINode &) {
956 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 956)
;
957}
958
959void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
960 // Note: this doesn't use InstVisitor, because it has to work with
961 // ConstantExpr's in addition to instructions.
962 switch (Opcode) {
963 default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 963)
;
964 // Build the switch statement using the Instruction.def file.
965#define HANDLE_INST(NUM, OPCODE, CLASS) \
966 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
967#include "llvm/IR/Instruction.def"
968 }
969}
970
971// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
972// generate the debug data structures now that we've seen its definition.
973void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
974 SDValue Val) {
975 DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
976 if (DDI.getDI()) {
977 const DbgValueInst *DI = DDI.getDI();
978 DebugLoc dl = DDI.getdl();
979 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
980 DILocalVariable *Variable = DI->getVariable();
981 DIExpression *Expr = DI->getExpression();
982 assert(Variable->isValidLocationForIntrinsic(dl) &&((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 983, __PRETTY_FUNCTION__))
983 "Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 983, __PRETTY_FUNCTION__))
;
984 uint64_t Offset = DI->getOffset();
985 SDDbgValue *SDV;
986 if (Val.getNode()) {
987 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, false,
988 Val)) {
989 SDV = getDbgValue(Val, Variable, Expr, Offset, dl, DbgSDNodeOrder);
990 DAG.AddDbgValue(SDV, Val.getNode(), false);
991 }
992 } else
993 DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false)
;
994 DanglingDebugInfoMap[V] = DanglingDebugInfo();
995 }
996}
997
998/// getCopyFromRegs - If there was virtual register allocated for the value V
999/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1000SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1001 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1002 SDValue Result;
1003
1004 if (It != FuncInfo.ValueMap.end()) {
1005 unsigned InReg = It->second;
1006 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1007 DAG.getDataLayout(), InReg, Ty);
1008 SDValue Chain = DAG.getEntryNode();
1009 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1010 resolveDanglingDebugInfo(V, Result);
1011 }
1012
1013 return Result;
1014}
1015
1016/// getValue - Return an SDValue for the given Value.
1017SDValue SelectionDAGBuilder::getValue(const Value *V) {
1018 // If we already have an SDValue for this value, use it. It's important
1019 // to do this first, so that we don't create a CopyFromReg if we already
1020 // have a regular SDValue.
1021 SDValue &N = NodeMap[V];
1022 if (N.getNode()) return N;
1023
1024 // If there's a virtual register allocated and initialized for this
1025 // value, use it.
1026 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1027 return copyFromReg;
1028
1029 // Otherwise create a new SDValue and remember it.
1030 SDValue Val = getValueImpl(V);
1031 NodeMap[V] = Val;
1032 resolveDanglingDebugInfo(V, Val);
1033 return Val;
1034}
1035
1036// Return true if SDValue exists for the given Value
1037bool SelectionDAGBuilder::findValue(const Value *V) const {
1038 return (NodeMap.find(V) != NodeMap.end()) ||
1039 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1040}
1041
1042/// getNonRegisterValue - Return an SDValue for the given Value, but
1043/// don't look in FuncInfo.ValueMap for a virtual register.
1044SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1045 // If we already have an SDValue for this value, use it.
1046 SDValue &N = NodeMap[V];
1047 if (N.getNode()) {
1048 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1049 // Remove the debug location from the node as the node is about to be used
1050 // in a location which may differ from the original debug location. This
1051 // is relevant to Constant and ConstantFP nodes because they can appear
1052 // as constant expressions inside PHI nodes.
1053 N->setDebugLoc(DebugLoc());
1054 }
1055 return N;
1056 }
1057
1058 // Otherwise create a new SDValue and remember it.
1059 SDValue Val = getValueImpl(V);
1060 NodeMap[V] = Val;
1061 resolveDanglingDebugInfo(V, Val);
1062 return Val;
1063}
1064
1065/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1066/// Create an SDValue for the given value.
1067SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1068 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1069
1070 if (const Constant *C = dyn_cast<Constant>(V)) {
1071 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1072
1073 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1074 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1075
1076 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1077 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1078
1079 if (isa<ConstantPointerNull>(C)) {
1080 unsigned AS = V->getType()->getPointerAddressSpace();
1081 return DAG.getConstant(0, getCurSDLoc(),
1082 TLI.getPointerTy(DAG.getDataLayout(), AS));
1083 }
1084
1085 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1086 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1087
1088 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1089 return DAG.getUNDEF(VT);
1090
1091 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1092 visit(CE->getOpcode(), *CE);
1093 SDValue N1 = NodeMap[V];
1094 assert(N1.getNode() && "visit didn't populate the NodeMap!")((N1.getNode() && "visit didn't populate the NodeMap!"
) ? static_cast<void> (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1094, __PRETTY_FUNCTION__))
;
1095 return N1;
1096 }
1097
1098 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1099 SmallVector<SDValue, 4> Constants;
1100 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1101 OI != OE; ++OI) {
1102 SDNode *Val = getValue(*OI).getNode();
1103 // If the operand is an empty aggregate, there are no values.
1104 if (!Val) continue;
1105 // Add each leaf value from the operand to the Constants list
1106 // to form a flattened list of all the values.
1107 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1108 Constants.push_back(SDValue(Val, i));
1109 }
1110
1111 return DAG.getMergeValues(Constants, getCurSDLoc());
1112 }
1113
1114 if (const ConstantDataSequential *CDS =
1115 dyn_cast<ConstantDataSequential>(C)) {
1116 SmallVector<SDValue, 4> Ops;
1117 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1118 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1119 // Add each leaf value from the operand to the Constants list
1120 // to form a flattened list of all the values.
1121 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1122 Ops.push_back(SDValue(Val, i));
1123 }
1124
1125 if (isa<ArrayType>(CDS->getType()))
1126 return DAG.getMergeValues(Ops, getCurSDLoc());
1127 return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
1128 VT, Ops);
1129 }
1130
1131 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1132 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1133, __PRETTY_FUNCTION__))
1133 "Unknown struct or array constant!")(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1133, __PRETTY_FUNCTION__))
;
1134
1135 SmallVector<EVT, 4> ValueVTs;
1136 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1137 unsigned NumElts = ValueVTs.size();
1138 if (NumElts == 0)
1139 return SDValue(); // empty struct
1140 SmallVector<SDValue, 4> Constants(NumElts);
1141 for (unsigned i = 0; i != NumElts; ++i) {
1142 EVT EltVT = ValueVTs[i];
1143 if (isa<UndefValue>(C))
1144 Constants[i] = DAG.getUNDEF(EltVT);
1145 else if (EltVT.isFloatingPoint())
1146 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1147 else
1148 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1149 }
1150
1151 return DAG.getMergeValues(Constants, getCurSDLoc());
1152 }
1153
1154 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1155 return DAG.getBlockAddress(BA, VT);
1156
1157 VectorType *VecTy = cast<VectorType>(V->getType());
1158 unsigned NumElements = VecTy->getNumElements();
1159
1160 // Now that we know the number and type of the elements, get that number of
1161 // elements into the Ops array based on what kind of constant it is.
1162 SmallVector<SDValue, 16> Ops;
1163 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1164 for (unsigned i = 0; i != NumElements; ++i)
1165 Ops.push_back(getValue(CV->getOperand(i)));
1166 } else {
1167 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!")((isa<ConstantAggregateZero>(C) && "Unknown vector constant!"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantAggregateZero>(C) && \"Unknown vector constant!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1167, __PRETTY_FUNCTION__))
;
1168 EVT EltVT =
1169 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1170
1171 SDValue Op;
1172 if (EltVT.isFloatingPoint())
1173 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1174 else
1175 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1176 Ops.assign(NumElements, Op);
1177 }
1178
1179 // Create a BUILD_VECTOR node.
1180 return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
1181 }
1182
1183 // If this is a static alloca, generate it as the frameindex instead of
1184 // computation.
1185 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1186 DenseMap<const AllocaInst*, int>::iterator SI =
1187 FuncInfo.StaticAllocaMap.find(AI);
1188 if (SI != FuncInfo.StaticAllocaMap.end())
1189 return DAG.getFrameIndex(SI->second,
1190 TLI.getPointerTy(DAG.getDataLayout()));
1191 }
1192
1193 // If this is an instruction which fast-isel has deferred, select it now.
1194 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1195 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1196 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1197 Inst->getType());
1198 SDValue Chain = DAG.getEntryNode();
1199 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1200 }
1201
1202 llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1202)
;
1203}
1204
1205void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1206 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1207 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1208 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1209 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1210 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1211 if (IsMSVCCXX || IsCoreCLR)
1212 CatchPadMBB->setIsEHFuncletEntry();
1213
1214 DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other, getControlRoot()));
1215}
1216
1217void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1218 // Update machine-CFG edge.
1219 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1220 FuncInfo.MBB->addSuccessor(TargetMBB);
1221
1222 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1223 bool IsSEH = isAsynchronousEHPersonality(Pers);
1224 if (IsSEH) {
1225 // If this is not a fall-through branch or optimizations are switched off,
1226 // emit the branch.
1227 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1228 TM.getOptLevel() == CodeGenOpt::None)
1229 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1230 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1231 return;
1232 }
1233
1234 // Figure out the funclet membership for the catchret's successor.
1235 // This will be used by the FuncletLayout pass to determine how to order the
1236 // BB's.
1237 // A 'catchret' returns to the outer scope's color.
1238 Value *ParentPad = I.getCatchSwitchParentPad();
1239 const BasicBlock *SuccessorColor;
1240 if (isa<ConstantTokenNone>(ParentPad))
1241 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1242 else
1243 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1244 assert(SuccessorColor && "No parent funclet for catchret!")((SuccessorColor && "No parent funclet for catchret!"
) ? static_cast<void> (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1244, __PRETTY_FUNCTION__))
;
1245 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1246 assert(SuccessorColorMBB && "No MBB for SuccessorColor!")((SuccessorColorMBB && "No MBB for SuccessorColor!") ?
static_cast<void> (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1246, __PRETTY_FUNCTION__))
;
1247
1248 // Create the terminator node.
1249 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1250 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1251 DAG.getBasicBlock(SuccessorColorMBB));
1252 DAG.setRoot(Ret);
1253}
1254
1255void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1256 // Don't emit any special code for the cleanuppad instruction. It just marks
1257 // the start of a funclet.
1258 FuncInfo.MBB->setIsEHFuncletEntry();
1259 FuncInfo.MBB->setIsCleanupFuncletEntry();
1260}
1261
1262/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1263/// many places it could ultimately go. In the IR, we have a single unwind
1264/// destination, but in the machine CFG, we enumerate all the possible blocks.
1265/// This function skips over imaginary basic blocks that hold catchswitch
1266/// instructions, and finds all the "real" machine
1267/// basic block destinations. As those destinations may not be successors of
1268/// EHPadBB, here we also calculate the edge probability to those destinations.
1269/// The passed-in Prob is the edge probability to EHPadBB.
1270static void findUnwindDestinations(
1271 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1272 BranchProbability Prob,
1273 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1274 &UnwindDests) {
1275 EHPersonality Personality =
1276 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1277 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1278 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1279
1280 while (EHPadBB) {
1281 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1282 BasicBlock *NewEHPadBB = nullptr;
1283 if (isa<LandingPadInst>(Pad)) {
1284 // Stop on landingpads. They are not funclets.
1285 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1286 break;
1287 } else if (isa<CleanupPadInst>(Pad)) {
1288 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1289 // personalities.
1290 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1291 UnwindDests.back().first->setIsEHFuncletEntry();
1292 break;
1293 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1294 // Add the catchpad handlers to the possible destinations.
1295 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1296 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1297 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1298 if (IsMSVCCXX || IsCoreCLR)
1299 UnwindDests.back().first->setIsEHFuncletEntry();
1300 }
1301 NewEHPadBB = CatchSwitch->getUnwindDest();
1302 } else {
1303 continue;
1304 }
1305
1306 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1307 if (BPI && NewEHPadBB)
1308 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1309 EHPadBB = NewEHPadBB;
1310 }
1311}
1312
1313void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1314 // Update successor info.
1315 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1316 auto UnwindDest = I.getUnwindDest();
1317 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1318 BranchProbability UnwindDestProb =
1319 (BPI && UnwindDest)
1320 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1321 : BranchProbability::getZero();
1322 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1323 for (auto &UnwindDest : UnwindDests) {
1324 UnwindDest.first->setIsEHPad();
1325 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1326 }
1327 FuncInfo.MBB->normalizeSuccProbs();
1328
1329 // Create the terminator node.
1330 SDValue Ret =
1331 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1332 DAG.setRoot(Ret);
1333}
1334
1335void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1336 report_fatal_error("visitCatchSwitch not yet implemented!");
1337}
1338
1339void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1340 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1341 auto &DL = DAG.getDataLayout();
1342 SDValue Chain = getControlRoot();
1343 SmallVector<ISD::OutputArg, 8> Outs;
1344 SmallVector<SDValue, 8> OutVals;
1345
1346 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1347 // lower
1348 //
1349 // %val = call <ty> @llvm.experimental.deoptimize()
1350 // ret <ty> %val
1351 //
1352 // differently.
1353 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1354 LowerDeoptimizingReturn();
1355 return;
1356 }
1357
1358 if (!FuncInfo.CanLowerReturn) {
1359 unsigned DemoteReg = FuncInfo.DemoteRegister;
1360 const Function *F = I.getParent()->getParent();
1361
1362 // Emit a store of the return value through the virtual register.
1363 // Leave Outs empty so that LowerReturn won't try to load return
1364 // registers the usual way.
1365 SmallVector<EVT, 1> PtrValueVTs;
1366 ComputeValueVTs(TLI, DL, PointerType::getUnqual(F->getReturnType()),
1367 PtrValueVTs);
1368
1369 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1370 DemoteReg, PtrValueVTs[0]);
1371 SDValue RetOp = getValue(I.getOperand(0));
1372
1373 SmallVector<EVT, 4> ValueVTs;
1374 SmallVector<uint64_t, 4> Offsets;
1375 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1376 unsigned NumValues = ValueVTs.size();
1377
1378 // An aggregate return value cannot wrap around the address space, so
1379 // offsets to its parts don't wrap either.
1380 SDNodeFlags Flags;
1381 Flags.setNoUnsignedWrap(true);
1382
1383 SmallVector<SDValue, 4> Chains(NumValues);
1384 for (unsigned i = 0; i != NumValues; ++i) {
1385 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
1386 RetPtr.getValueType(), RetPtr,
1387 DAG.getIntPtrConstant(Offsets[i],
1388 getCurSDLoc()),
1389 &Flags);
1390 Chains[i] = DAG.getStore(Chain, getCurSDLoc(),
1391 SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1392 // FIXME: better loc info would be nice.
1393 Add, MachinePointerInfo());
1394 }
1395
1396 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1397 MVT::Other, Chains);
1398 } else if (I.getNumOperands() != 0) {
1399 SmallVector<EVT, 4> ValueVTs;
1400 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1401 unsigned NumValues = ValueVTs.size();
1402 if (NumValues) {
1403 SDValue RetOp = getValue(I.getOperand(0));
1404
1405 const Function *F = I.getParent()->getParent();
1406
1407 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1408 if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1409 Attribute::SExt))
1410 ExtendKind = ISD::SIGN_EXTEND;
1411 else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1412 Attribute::ZExt))
1413 ExtendKind = ISD::ZERO_EXTEND;
1414
1415 LLVMContext &Context = F->getContext();
1416 bool RetInReg = F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
1417 Attribute::InReg);
1418
1419 for (unsigned j = 0; j != NumValues; ++j) {
1420 EVT VT = ValueVTs[j];
1421
1422 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1423 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1424
1425 unsigned NumParts = TLI.getNumRegisters(Context, VT);
1426 MVT PartVT = TLI.getRegisterType(Context, VT);
1427 SmallVector<SDValue, 4> Parts(NumParts);
1428 getCopyToParts(DAG, getCurSDLoc(),
1429 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1430 &Parts[0], NumParts, PartVT, &I, ExtendKind);
1431
1432 // 'inreg' on function refers to return value
1433 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1434 if (RetInReg)
1435 Flags.setInReg();
1436
1437 // Propagate extension type if any
1438 if (ExtendKind == ISD::SIGN_EXTEND)
1439 Flags.setSExt();
1440 else if (ExtendKind == ISD::ZERO_EXTEND)
1441 Flags.setZExt();
1442
1443 for (unsigned i = 0; i < NumParts; ++i) {
1444 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1445 VT, /*isfixed=*/true, 0, 0));
1446 OutVals.push_back(Parts[i]);
1447 }
1448 }
1449 }
1450 }
1451
1452 // Push in swifterror virtual register as the last element of Outs. This makes
1453 // sure swifterror virtual register will be returned in the swifterror
1454 // physical register.
1455 const Function *F = I.getParent()->getParent();
1456 if (TLI.supportSwiftError() &&
1457 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1458 assert(FuncInfo.SwiftErrorArg && "Need a swift error argument")((FuncInfo.SwiftErrorArg && "Need a swift error argument"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.SwiftErrorArg && \"Need a swift error argument\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1458, __PRETTY_FUNCTION__))
;
1459 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1460 Flags.setSwiftError();
1461 Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1462 EVT(TLI.getPointerTy(DL)) /*argvt*/,
1463 true /*isfixed*/, 1 /*origidx*/,
1464 0 /*partOffs*/));
1465 // Create SDNode for the swifterror virtual register.
1466 OutVals.push_back(DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVReg(
1467 FuncInfo.MBB, FuncInfo.SwiftErrorArg),
1468 EVT(TLI.getPointerTy(DL))));
1469 }
1470
1471 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1472 CallingConv::ID CallConv =
1473 DAG.getMachineFunction().getFunction()->getCallingConv();
1474 Chain = DAG.getTargetLoweringInfo().LowerReturn(
1475 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1476
1477 // Verify that the target's LowerReturn behaved as expected.
1478 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1479, __PRETTY_FUNCTION__))
1479 "LowerReturn didn't return a valid chain!")((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1479, __PRETTY_FUNCTION__))
;
1480
1481 // Update the DAG with the new chain value resulting from return lowering.
1482 DAG.setRoot(Chain);
1483}
1484
1485/// CopyToExportRegsIfNeeded - If the given value has virtual registers
1486/// created for it, emit nodes to copy the value into the virtual
1487/// registers.
1488void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1489 // Skip empty types
1490 if (V->getType()->isEmptyTy())
1491 return;
1492
1493 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1494 if (VMI != FuncInfo.ValueMap.end()) {
1495 assert(!V->use_empty() && "Unused value assigned virtual registers!")((!V->use_empty() && "Unused value assigned virtual registers!"
) ? static_cast<void> (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1495, __PRETTY_FUNCTION__))
;
1496 CopyValueToVirtualRegister(V, VMI->second);
1497 }
1498}
1499
1500/// ExportFromCurrentBlock - If this condition isn't known to be exported from
1501/// the current basic block, add it to ValueMap now so that we'll get a
1502/// CopyTo/FromReg.
1503void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1504 // No need to export constants.
1505 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1506
1507 // Already exported?
1508 if (FuncInfo.isExportedInst(V)) return;
1509
1510 unsigned Reg = FuncInfo.InitializeRegForValue(V);
1511 CopyValueToVirtualRegister(V, Reg);
1512}
1513
1514bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1515 const BasicBlock *FromBB) {
1516 // The operands of the setcc have to be in this block. We don't know
1517 // how to export them from some other block.
1518 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1519 // Can export from current BB.
1520 if (VI->getParent() == FromBB)
1521 return true;
1522
1523 // Is already exported, noop.
1524 return FuncInfo.isExportedInst(V);
1525 }
1526
1527 // If this is an argument, we can export it if the BB is the entry block or
1528 // if it is already exported.
1529 if (isa<Argument>(V)) {
1530 if (FromBB == &FromBB->getParent()->getEntryBlock())
1531 return true;
1532
1533 // Otherwise, can only export this if it is already exported.
1534 return FuncInfo.isExportedInst(V);
1535 }
1536
1537 // Otherwise, constants can always be exported.
1538 return true;
1539}
1540
1541/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1542BranchProbability
1543SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1544 const MachineBasicBlock *Dst) const {
1545 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1546 const BasicBlock *SrcBB = Src->getBasicBlock();
1547 const BasicBlock *DstBB = Dst->getBasicBlock();
1548 if (!BPI) {
1549 // If BPI is not available, set the default probability as 1 / N, where N is
1550 // the number of successors.
1551 auto SuccSize = std::max<uint32_t>(
1552 std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
1553 return BranchProbability(1, SuccSize);
1554 }
1555 return BPI->getEdgeProbability(SrcBB, DstBB);
1556}
1557
1558void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1559 MachineBasicBlock *Dst,
1560 BranchProbability Prob) {
1561 if (!FuncInfo.BPI)
1562 Src->addSuccessorWithoutProb(Dst);
1563 else {
1564 if (Prob.isUnknown())
1565 Prob = getEdgeProbability(Src, Dst);
1566 Src->addSuccessor(Dst, Prob);
1567 }
1568}
1569
1570static bool InBlock(const Value *V, const BasicBlock *BB) {
1571 if (const Instruction *I = dyn_cast<Instruction>(V))
1572 return I->getParent() == BB;
1573 return true;
1574}
1575
1576/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1577/// This function emits a branch and is used at the leaves of an OR or an
1578/// AND operator tree.
1579///
1580void
1581SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1582 MachineBasicBlock *TBB,
1583 MachineBasicBlock *FBB,
1584 MachineBasicBlock *CurBB,
1585 MachineBasicBlock *SwitchBB,
1586 BranchProbability TProb,
1587 BranchProbability FProb,
1588 bool InvertCond) {
1589 const BasicBlock *BB = CurBB->getBasicBlock();
1590
1591 // If the leaf of the tree is a comparison, merge the condition into
1592 // the caseblock.
1593 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1594 // The operands of the cmp have to be in this block. We don't know
1595 // how to export them from some other block. If this is the first block
1596 // of the sequence, no exporting is needed.
1597 if (CurBB == SwitchBB ||
1598 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1599 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1600 ISD::CondCode Condition;
1601 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1602 ICmpInst::Predicate Pred =
1603 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1604 Condition = getICmpCondCode(Pred);
1605 } else {
1606 const FCmpInst *FC = cast<FCmpInst>(Cond);
1607 FCmpInst::Predicate Pred =
1608 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1609 Condition = getFCmpCondCode(Pred);
1610 if (TM.Options.NoNaNsFPMath)
1611 Condition = getFCmpCodeWithoutNaN(Condition);
1612 }
1613
1614 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1615 TBB, FBB, CurBB, TProb, FProb);
1616 SwitchCases.push_back(CB);
1617 return;
1618 }
1619 }
1620
1621 // Create a CaseBlock record representing this branch.
1622 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1623 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1624 nullptr, TBB, FBB, CurBB, TProb, FProb);
1625 SwitchCases.push_back(CB);
1626}
1627
1628/// FindMergedConditions - If Cond is an expression like
1629void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1630 MachineBasicBlock *TBB,
1631 MachineBasicBlock *FBB,
1632 MachineBasicBlock *CurBB,
1633 MachineBasicBlock *SwitchBB,
1634 Instruction::BinaryOps Opc,
1635 BranchProbability TProb,
1636 BranchProbability FProb,
1637 bool InvertCond) {
1638 // Skip over not part of the tree and remember to invert op and operands at
1639 // next level.
1640 if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1641 const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1642 if (InBlock(CondOp, CurBB->getBasicBlock())) {
1643 FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1644 !InvertCond);
1645 return;
1646 }
1647 }
1648
1649 const Instruction *BOp = dyn_cast<Instruction>(Cond);
1650 // Compute the effective opcode for Cond, taking into account whether it needs
1651 // to be inverted, e.g.
1652 // and (not (or A, B)), C
1653 // gets lowered as
1654 // and (and (not A, not B), C)
1655 unsigned BOpc = 0;
1656 if (BOp) {
1657 BOpc = BOp->getOpcode();
1658 if (InvertCond) {
1659 if (BOpc == Instruction::And)
1660 BOpc = Instruction::Or;
1661 else if (BOpc == Instruction::Or)
1662 BOpc = Instruction::And;
1663 }
1664 }
1665
1666 // If this node is not part of the or/and tree, emit it as a branch.
1667 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1668 BOpc != Opc || !BOp->hasOneUse() ||
1669 BOp->getParent() != CurBB->getBasicBlock() ||
1670 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1671 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1672 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1673 TProb, FProb, InvertCond);
1674 return;
1675 }
1676
1677 // Create TmpBB after CurBB.
1678 MachineFunction::iterator BBI(CurBB);
1679 MachineFunction &MF = DAG.getMachineFunction();
1680 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1681 CurBB->getParent()->insert(++BBI, TmpBB);
1682
1683 if (Opc == Instruction::Or) {
1684 // Codegen X | Y as:
1685 // BB1:
1686 // jmp_if_X TBB
1687 // jmp TmpBB
1688 // TmpBB:
1689 // jmp_if_Y TBB
1690 // jmp FBB
1691 //
1692
1693 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1694 // The requirement is that
1695 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1696 // = TrueProb for original BB.
1697 // Assuming the original probabilities are A and B, one choice is to set
1698 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1699 // A/(1+B) and 2B/(1+B). This choice assumes that
1700 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1701 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1702 // TmpBB, but the math is more complicated.
1703
1704 auto NewTrueProb = TProb / 2;
1705 auto NewFalseProb = TProb / 2 + FProb;
1706 // Emit the LHS condition.
1707 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1708 NewTrueProb, NewFalseProb, InvertCond);
1709
1710 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1711 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1712 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1713 // Emit the RHS condition into TmpBB.
1714 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1715 Probs[0], Probs[1], InvertCond);
1716 } else {
1717 assert(Opc == Instruction::And && "Unknown merge op!")((Opc == Instruction::And && "Unknown merge op!") ? static_cast
<void> (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1717, __PRETTY_FUNCTION__))
;
1718 // Codegen X & Y as:
1719 // BB1:
1720 // jmp_if_X TmpBB
1721 // jmp FBB
1722 // TmpBB:
1723 // jmp_if_Y TBB
1724 // jmp FBB
1725 //
1726 // This requires creation of TmpBB after CurBB.
1727
1728 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1729 // The requirement is that
1730 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1731 // = FalseProb for original BB.
1732 // Assuming the original probabilities are A and B, one choice is to set
1733 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1734 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1735 // TrueProb for BB1 * FalseProb for TmpBB.
1736
1737 auto NewTrueProb = TProb + FProb / 2;
1738 auto NewFalseProb = FProb / 2;
1739 // Emit the LHS condition.
1740 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1741 NewTrueProb, NewFalseProb, InvertCond);
1742
1743 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1744 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1745 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1746 // Emit the RHS condition into TmpBB.
1747 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1748 Probs[0], Probs[1], InvertCond);
1749 }
1750}
1751
1752/// If the set of cases should be emitted as a series of branches, return true.
1753/// If we should emit this as a bunch of and/or'd together conditions, return
1754/// false.
1755bool
1756SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1757 if (Cases.size() != 2) return true;
1758
1759 // If this is two comparisons of the same values or'd or and'd together, they
1760 // will get folded into a single comparison, so don't emit two blocks.
1761 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1762 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1763 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1764 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1765 return false;
1766 }
1767
1768 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1769 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1770 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1771 Cases[0].CC == Cases[1].CC &&
1772 isa<Constant>(Cases[0].CmpRHS) &&
1773 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1774 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1775 return false;
1776 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1777 return false;
1778 }
1779
1780 return true;
1781}
1782
1783void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1784 MachineBasicBlock *BrMBB = FuncInfo.MBB;
1785
1786 // Update machine-CFG edges.
1787 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1788
1789 if (I.isUnconditional()) {
1790 // Update machine-CFG edges.
1791 BrMBB->addSuccessor(Succ0MBB);
1792
1793 // If this is not a fall-through branch or optimizations are switched off,
1794 // emit the branch.
1795 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1796 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1797 MVT::Other, getControlRoot(),
1798 DAG.getBasicBlock(Succ0MBB)));
1799
1800 return;
1801 }
1802
1803 // If this condition is one of the special cases we handle, do special stuff
1804 // now.
1805 const Value *CondVal = I.getCondition();
1806 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1807
1808 // If this is a series of conditions that are or'd or and'd together, emit
1809 // this as a sequence of branches instead of setcc's with and/or operations.
1810 // As long as jumps are not expensive, this should improve performance.
1811 // For example, instead of something like:
1812 // cmp A, B
1813 // C = seteq
1814 // cmp D, E
1815 // F = setle
1816 // or C, F
1817 // jnz foo
1818 // Emit:
1819 // cmp A, B
1820 // je foo
1821 // cmp D, E
1822 // jle foo
1823 //
1824 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1825 Instruction::BinaryOps Opcode = BOp->getOpcode();
1826 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
1827 !I.getMetadata(LLVMContext::MD_unpredictable) &&
1828 (Opcode == Instruction::And || Opcode == Instruction::Or)) {
1829 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1830 Opcode,
1831 getEdgeProbability(BrMBB, Succ0MBB),
1832 getEdgeProbability(BrMBB, Succ1MBB),
1833 /*InvertCond=*/false);
1834 // If the compares in later blocks need to use values not currently
1835 // exported from this block, export them now. This block should always
1836 // be the first entry.
1837 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")((SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"
) ? static_cast<void> (0) : __assert_fail ("SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1837, __PRETTY_FUNCTION__))
;
1838
1839 // Allow some cases to be rejected.
1840 if (ShouldEmitAsBranches(SwitchCases)) {
1841 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
1842 ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
1843 ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
1844 }
1845
1846 // Emit the branch for this block.
1847 visitSwitchCase(SwitchCases[0], BrMBB);
1848 SwitchCases.erase(SwitchCases.begin());
1849 return;
1850 }
1851
1852 // Okay, we decided not to do this, remove any inserted MBB's and clear
1853 // SwitchCases.
1854 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
1855 FuncInfo.MF->erase(SwitchCases[i].ThisBB);
1856
1857 SwitchCases.clear();
1858 }
1859 }
1860
1861 // Create a CaseBlock record representing this branch.
1862 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
1863 nullptr, Succ0MBB, Succ1MBB, BrMBB);
1864
1865 // Use visitSwitchCase to actually insert the fast branch sequence for this
1866 // cond branch.
1867 visitSwitchCase(CB, BrMBB);
1868}
1869
1870/// visitSwitchCase - Emits the necessary code to represent a single node in
1871/// the binary search tree resulting from lowering a switch instruction.
1872void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
1873 MachineBasicBlock *SwitchBB) {
1874 SDValue Cond;
1875 SDValue CondLHS = getValue(CB.CmpLHS);
1876 SDLoc dl = getCurSDLoc();
1877
1878 // Build the setcc now.
1879 if (!CB.CmpMHS) {
1880 // Fold "(X == true)" to X and "(X == false)" to !X to
1881 // handle common cases produced by branch lowering.
1882 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
1883 CB.CC == ISD::SETEQ)
1884 Cond = CondLHS;
1885 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
1886 CB.CC == ISD::SETEQ) {
1887 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
1888 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
1889 } else
1890 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
1891 } else {
1892 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")((CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? static_cast<void> (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1892, __PRETTY_FUNCTION__))
;
1893
1894 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
1895 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
1896
1897 SDValue CmpOp = getValue(CB.CmpMHS);
1898 EVT VT = CmpOp.getValueType();
1899
1900 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
1901 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
1902 ISD::SETLE);
1903 } else {
1904 SDValue SUB = DAG.getNode(ISD::SUB, dl,
1905 VT, CmpOp, DAG.getConstant(Low, dl, VT));
1906 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
1907 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
1908 }
1909 }
1910
1911 // Update successor info
1912 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
1913 // TrueBB and FalseBB are always different unless the incoming IR is
1914 // degenerate. This only happens when running llc on weird IR.
1915 if (CB.TrueBB != CB.FalseBB)
1916 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
1917 SwitchBB->normalizeSuccProbs();
1918
1919 // If the lhs block is the next block, invert the condition so that we can
1920 // fall through to the lhs instead of the rhs block.
1921 if (CB.TrueBB == NextBlock(SwitchBB)) {
1922 std::swap(CB.TrueBB, CB.FalseBB);
1923 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
1924 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
1925 }
1926
1927 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1928 MVT::Other, getControlRoot(), Cond,
1929 DAG.getBasicBlock(CB.TrueBB));
1930
1931 // Insert the false branch. Do this even if it's a fall through branch,
1932 // this makes it easier to do DAG optimizations which require inverting
1933 // the branch condition.
1934 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1935 DAG.getBasicBlock(CB.FalseBB));
1936
1937 DAG.setRoot(BrCond);
1938}
1939
1940/// visitJumpTable - Emit JumpTable node in the current MBB
1941void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
1942 // Emit the code for the jump table
1943 assert(JT.Reg != -1U && "Should lower JT Header first!")((JT.Reg != -1U && "Should lower JT Header first!") ?
static_cast<void> (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1943, __PRETTY_FUNCTION__))
;
1944 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
1945 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
1946 JT.Reg, PTy);
1947 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
1948 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
1949 MVT::Other, Index.getValue(1),
1950 Table, Index);
1951 DAG.setRoot(BrJumpTable);
1952}
1953
1954/// visitJumpTableHeader - This function emits necessary code to produce index
1955/// in the JumpTable from switch case.
1956void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
1957 JumpTableHeader &JTH,
1958 MachineBasicBlock *SwitchBB) {
1959 SDLoc dl = getCurSDLoc();
1960
1961 // Subtract the lowest switch case value from the value being switched on and
1962 // conditional branch to default mbb if the result is greater than the
1963 // difference between smallest and largest cases.
1964 SDValue SwitchOp = getValue(JTH.SValue);
1965 EVT VT = SwitchOp.getValueType();
1966 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
1967 DAG.getConstant(JTH.First, dl, VT));
1968
1969 // The SDNode we just created, which holds the value being switched on minus
1970 // the smallest case value, needs to be copied to a virtual register so it
1971 // can be used as an index into the jump table in a subsequent basic block.
1972 // This value may be smaller or larger than the target's pointer type, and
1973 // therefore require extension or truncating.
1974 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1975 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
1976
1977 unsigned JumpTableReg =
1978 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
1979 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
1980 JumpTableReg, SwitchOp);
1981 JT.Reg = JumpTableReg;
1982
1983 // Emit the range check for the jump table, and branch to the default block
1984 // for the switch statement if the value being switched on exceeds the largest
1985 // case in the switch.
1986 SDValue CMP = DAG.getSetCC(
1987 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1988 Sub.getValueType()),
1989 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
1990
1991 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1992 MVT::Other, CopyTo, CMP,
1993 DAG.getBasicBlock(JT.Default));
1994
1995 // Avoid emitting unnecessary branches to the next block.
1996 if (JT.MBB != NextBlock(SwitchBB))
1997 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1998 DAG.getBasicBlock(JT.MBB));
1999
2000 DAG.setRoot(BrCond);
2001}
2002
2003/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2004/// variable if there exists one.
2005static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2006 SDValue &Chain) {
2007 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2008 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2009 MachineFunction &MF = DAG.getMachineFunction();
2010 Value *Global = TLI.getSDagStackGuard(*MF.getFunction()->getParent());
2011 MachineSDNode *Node =
2012 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2013 if (Global) {
2014 MachinePointerInfo MPInfo(Global);
2015 MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
2016 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2017 MachineMemOperand::MODereferenceable;
2018 *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
2019 DAG.getEVTAlignment(PtrTy));
2020 Node->setMemRefs(MemRefs, MemRefs + 1);
2021 }
2022 return SDValue(Node, 0);
2023}
2024
2025/// Codegen a new tail for a stack protector check ParentMBB which has had its
2026/// tail spliced into a stack protector check success bb.
2027///
2028/// For a high level explanation of how this fits into the stack protector
2029/// generation see the comment on the declaration of class
2030/// StackProtectorDescriptor.
2031void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2032 MachineBasicBlock *ParentBB) {
2033
2034 // First create the loads to the guard/stack slot for the comparison.
2035 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2036 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2037
2038 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2039 int FI = MFI.getStackProtectorIndex();
2040
2041 SDValue Guard;
2042 SDLoc dl = getCurSDLoc();
2043 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2044 const Module &M = *ParentBB->getParent()->getFunction()->getParent();
2045 unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2046
2047 // Generate code to load the content of the guard slot.
2048 SDValue StackSlot = DAG.getLoad(
2049 PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2050 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2051 MachineMemOperand::MOVolatile);
2052
2053 // Retrieve guard check function, nullptr if instrumentation is inlined.
2054 if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2055 // The target provides a guard check function to validate the guard value.
2056 // Generate a call to that function with the content of the guard slot as
2057 // argument.
2058 auto *Fn = cast<Function>(GuardCheck);
2059 FunctionType *FnTy = Fn->getFunctionType();
2060 assert(FnTy->getNumParams() == 1 && "Invalid function signature")((FnTy->getNumParams() == 1 && "Invalid function signature"
) ? static_cast<void> (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2060, __PRETTY_FUNCTION__))
;
2061
2062 TargetLowering::ArgListTy Args;
2063 TargetLowering::ArgListEntry Entry;
2064 Entry.Node = StackSlot;
2065 Entry.Ty = FnTy->getParamType(0);
2066 if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2067 Entry.isInReg = true;
2068 Args.push_back(Entry);
2069
2070 TargetLowering::CallLoweringInfo CLI(DAG);
2071 CLI.setDebugLoc(getCurSDLoc())
2072 .setChain(DAG.getEntryNode())
2073 .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2074 getValue(GuardCheck), std::move(Args));
2075
2076 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2077 DAG.setRoot(Result.second);
2078 return;
2079 }
2080
2081 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2082 // Otherwise, emit a volatile load to retrieve the stack guard value.
2083 SDValue Chain = DAG.getEntryNode();
2084 if (TLI.useLoadStackGuardNode()) {
2085 Guard = getLoadStackGuard(DAG, dl, Chain);
2086 } else {
2087 const Value *IRGuard = TLI.getSDagStackGuard(M);
2088 SDValue GuardPtr = getValue(IRGuard);
2089
2090 Guard =
2091 DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2092 Align, MachineMemOperand::MOVolatile);
2093 }
2094
2095 // Perform the comparison via a subtract/getsetcc.
2096 EVT VT = Guard.getValueType();
2097 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
2098
2099 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2100 *DAG.getContext(),
2101 Sub.getValueType()),
2102 Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2103
2104 // If the sub is not 0, then we know the guard/stackslot do not equal, so
2105 // branch to failure MBB.
2106 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2107 MVT::Other, StackSlot.getOperand(0),
2108 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2109 // Otherwise branch to success MBB.
2110 SDValue Br = DAG.getNode(ISD::BR, dl,
2111 MVT::Other, BrCond,
2112 DAG.getBasicBlock(SPD.getSuccessMBB()));
2113
2114 DAG.setRoot(Br);
2115}
2116
2117/// Codegen the failure basic block for a stack protector check.
2118///
2119/// A failure stack protector machine basic block consists simply of a call to
2120/// __stack_chk_fail().
2121///
2122/// For a high level explanation of how this fits into the stack protector
2123/// generation see the comment on the declaration of class
2124/// StackProtectorDescriptor.
2125void
2126SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2127 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2128 SDValue Chain =
2129 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2130 None, false, getCurSDLoc(), false, false).second;
2131 DAG.setRoot(Chain);
2132}
2133
2134/// visitBitTestHeader - This function emits necessary code to produce value
2135/// suitable for "bit tests"
2136void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2137 MachineBasicBlock *SwitchBB) {
2138 SDLoc dl = getCurSDLoc();
2139
2140 // Subtract the minimum value
2141 SDValue SwitchOp = getValue(B.SValue);
2142 EVT VT = SwitchOp.getValueType();
2143 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2144 DAG.getConstant(B.First, dl, VT));
2145
2146 // Check range
2147 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2148 SDValue RangeCmp = DAG.getSetCC(
2149 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2150 Sub.getValueType()),
2151 Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2152
2153 // Determine the type of the test operands.
2154 bool UsePtrType = false;
2155 if (!TLI.isTypeLegal(VT))
2156 UsePtrType = true;
2157 else {
2158 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2159 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2160 // Switch table case range are encoded into series of masks.
2161 // Just use pointer type, it's guaranteed to fit.
2162 UsePtrType = true;
2163 break;
2164 }
2165 }
2166 if (UsePtrType) {
2167 VT = TLI.getPointerTy(DAG.getDataLayout());
2168 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2169 }
2170
2171 B.RegVT = VT.getSimpleVT();
2172 B.Reg = FuncInfo.CreateReg(B.RegVT);
2173 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2174
2175 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2176
2177 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2178 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2179 SwitchBB->normalizeSuccProbs();
2180
2181 SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2182 MVT::Other, CopyTo, RangeCmp,
2183 DAG.getBasicBlock(B.Default));
2184
2185 // Avoid emitting unnecessary branches to the next block.
2186 if (MBB != NextBlock(SwitchBB))
2187 BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2188 DAG.getBasicBlock(MBB));
2189
2190 DAG.setRoot(BrRange);
2191}
2192
2193/// visitBitTestCase - this function produces one "bit test"
2194void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2195 MachineBasicBlock* NextMBB,
2196 BranchProbability BranchProbToNext,
2197 unsigned Reg,
2198 BitTestCase &B,
2199 MachineBasicBlock *SwitchBB) {
2200 SDLoc dl = getCurSDLoc();
2201 MVT VT = BB.RegVT;
2202 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2203 SDValue Cmp;
2204 unsigned PopCount = countPopulation(B.Mask);
2205 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2206 if (PopCount == 1) {
2207 // Testing for a single bit; just compare the shift count with what it
2208 // would need to be to shift a 1 bit in that position.
2209 Cmp = DAG.getSetCC(
2210 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2211 ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2212 ISD::SETEQ);
2213 } else if (PopCount == BB.Range) {
2214 // There is only one zero bit in the range, test for it directly.
2215 Cmp = DAG.getSetCC(
2216 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2217 ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2218 ISD::SETNE);
2219 } else {
2220 // Make desired shift
2221 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2222 DAG.getConstant(1, dl, VT), ShiftOp);
2223
2224 // Emit bit tests and jumps
2225 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2226 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2227 Cmp = DAG.getSetCC(
2228 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2229 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2230 }
2231
2232 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2233 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2234 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2235 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2236 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2237 // one as they are relative probabilities (and thus work more like weights),
2238 // and hence we need to normalize them to let the sum of them become one.
2239 SwitchBB->normalizeSuccProbs();
2240
2241 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2242 MVT::Other, getControlRoot(),
2243 Cmp, DAG.getBasicBlock(B.TargetBB));
2244
2245 // Avoid emitting unnecessary branches to the next block.
2246 if (NextMBB != NextBlock(SwitchBB))
2247 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2248 DAG.getBasicBlock(NextMBB));
2249
2250 DAG.setRoot(BrAnd);
2251}
2252
2253void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2254 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2255
2256 // Retrieve successors. Look through artificial IR level blocks like
2257 // catchswitch for successors.
2258 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2259 const BasicBlock *EHPadBB = I.getSuccessor(1);
2260
2261 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2262 // have to do anything here to lower funclet bundles.
2263 assert(!I.hasOperandBundlesOtherThan(((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2265, __PRETTY_FUNCTION__))
2264 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2265, __PRETTY_FUNCTION__))
2265 "Cannot lower invokes with arbitrary operand bundles yet!")((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2265, __PRETTY_FUNCTION__))
;
2266
2267 const Value *Callee(I.getCalledValue());
2268 const Function *Fn = dyn_cast<Function>(Callee);
2269 if (isa<InlineAsm>(Callee))
2270 visitInlineAsm(&I);
2271 else if (Fn && Fn->isIntrinsic()) {
2272 switch (Fn->getIntrinsicID()) {
2273 default:
2274 llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2274)
;
2275 case Intrinsic::donothing:
2276 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2277 break;
2278 case Intrinsic::experimental_patchpoint_void:
2279 case Intrinsic::experimental_patchpoint_i64:
2280 visitPatchpoint(&I, EHPadBB);
2281 break;
2282 case Intrinsic::experimental_gc_statepoint:
2283 LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2284 break;
2285 }
2286 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2287 // Currently we do not lower any intrinsic calls with deopt operand bundles.
2288 // Eventually we will support lowering the @llvm.experimental.deoptimize
2289 // intrinsic, and right now there are no plans to support other intrinsics
2290 // with deopt state.
2291 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2292 } else {
2293 LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2294 }
2295
2296 // If the value of the invoke is used outside of its defining block, make it
2297 // available as a virtual register.
2298 // We already took care of the exported value for the statepoint instruction
2299 // during call to the LowerStatepoint.
2300 if (!isStatepoint(I)) {
2301 CopyToExportRegsIfNeeded(&I);
2302 }
2303
2304 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2305 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2306 BranchProbability EHPadBBProb =
2307 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2308 : BranchProbability::getZero();
2309 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2310
2311 // Update successor info.
2312 addSuccessorWithProb(InvokeMBB, Return);
2313 for (auto &UnwindDest : UnwindDests) {
2314 UnwindDest.first->setIsEHPad();
2315 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2316 }
2317 InvokeMBB->normalizeSuccProbs();
2318
2319 // Drop into normal successor.
2320 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2321 MVT::Other, getControlRoot(),
2322 DAG.getBasicBlock(Return)));
2323}
2324
2325void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2326 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2326)
;
2327}
2328
2329void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2330 assert(FuncInfo.MBB->isEHPad() &&((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2331, __PRETTY_FUNCTION__))
2331 "Call to landingpad not in landing pad!")((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2331, __PRETTY_FUNCTION__))
;
2332
2333 MachineBasicBlock *MBB = FuncInfo.MBB;
2334 addLandingPadInfo(LP, *MBB);
2335
2336 // If there aren't registers to copy the values into (e.g., during SjLj
2337 // exceptions), then don't bother to create these DAG nodes.
2338 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2339 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2340 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2341 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2342 return;
2343
2344 // If landingpad's return type is token type, we don't create DAG nodes
2345 // for its exception pointer and selector value. The extraction of exception
2346 // pointer or selector value from token type landingpads is not currently
2347 // supported.
2348 if (LP.getType()->isTokenTy())
2349 return;
2350
2351 SmallVector<EVT, 2> ValueVTs;
2352 SDLoc dl = getCurSDLoc();
2353 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2354 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")((ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2354, __PRETTY_FUNCTION__))
;
2355
2356 // Get the two live-in registers as SDValues. The physregs have already been
2357 // copied into virtual registers.
2358 SDValue Ops[2];
2359 if (FuncInfo.ExceptionPointerVirtReg) {
2360 Ops[0] = DAG.getZExtOrTrunc(
2361 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2362 FuncInfo.ExceptionPointerVirtReg,
2363 TLI.getPointerTy(DAG.getDataLayout())),
2364 dl, ValueVTs[0]);
2365 } else {
2366 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2367 }
2368 Ops[1] = DAG.getZExtOrTrunc(
2369 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2370 FuncInfo.ExceptionSelectorVirtReg,
2371 TLI.getPointerTy(DAG.getDataLayout())),
2372 dl, ValueVTs[1]);
2373
2374 // Merge into one.
2375 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2376 DAG.getVTList(ValueVTs), Ops);
2377 setValue(&LP, Res);
2378}
2379
2380void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2381#ifndef NDEBUG
2382 for (const CaseCluster &CC : Clusters)
2383 assert(CC.Low == CC.High && "Input clusters must be single-case")((CC.Low == CC.High && "Input clusters must be single-case"
) ? static_cast<void> (0) : __assert_fail ("CC.Low == CC.High && \"Input clusters must be single-case\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2383, __PRETTY_FUNCTION__))
;
2384#endif
2385
2386 std::sort(Clusters.begin(), Clusters.end(),
2387 [](const CaseCluster &a, const CaseCluster &b) {
2388 return a.Low->getValue().slt(b.Low->getValue());
2389 });
2390
2391 // Merge adjacent clusters with the same destination.
2392 const unsigned N = Clusters.size();
2393 unsigned DstIndex = 0;
2394 for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2395 CaseCluster &CC = Clusters[SrcIndex];
2396 const ConstantInt *CaseVal = CC.Low;
2397 MachineBasicBlock *Succ = CC.MBB;
2398
2399 if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2400 (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2401 // If this case has the same successor and is a neighbour, merge it into
2402 // the previous cluster.
2403 Clusters[DstIndex - 1].High = CaseVal;
2404 Clusters[DstIndex - 1].Prob += CC.Prob;
2405 } else {
2406 std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2407 sizeof(Clusters[SrcIndex]));
2408 }
2409 }
2410 Clusters.resize(DstIndex);
2411}
2412
2413void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2414 MachineBasicBlock *Last) {
2415 // Update JTCases.
2416 for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2417 if (JTCases[i].first.HeaderBB == First)
2418 JTCases[i].first.HeaderBB = Last;
2419
2420 // Update BitTestCases.
2421 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2422 if (BitTestCases[i].Parent == First)
2423 BitTestCases[i].Parent = Last;
2424}
2425
2426void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2427 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2428
2429 // Update machine-CFG edges with unique successors.
2430 SmallSet<BasicBlock*, 32> Done;
2431 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2432 BasicBlock *BB = I.getSuccessor(i);
2433 bool Inserted = Done.insert(BB).second;
2434 if (!Inserted)
2435 continue;
2436
2437 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2438 addSuccessorWithProb(IndirectBrMBB, Succ);
2439 }
2440 IndirectBrMBB->normalizeSuccProbs();
2441
2442 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2443 MVT::Other, getControlRoot(),
2444 getValue(I.getAddress())));
2445}
2446
2447void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2448 if (DAG.getTarget().Options.TrapUnreachable)
2449 DAG.setRoot(
2450 DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2451}
2452
2453void SelectionDAGBuilder::visitFSub(const User &I) {
2454 // -0.0 - X --> fneg
2455 Type *Ty = I.getType();
2456 if (isa<Constant>(I.getOperand(0)) &&
2457 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2458 SDValue Op2 = getValue(I.getOperand(1));
2459 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2460 Op2.getValueType(), Op2));
2461 return;
2462 }
2463
2464 visitBinary(I, ISD::FSUB);
2465}
2466
2467/// Checks if the given instruction performs a vector reduction, in which case
2468/// we have the freedom to alter the elements in the result as long as the
2469/// reduction of them stays unchanged.
2470static bool isVectorReductionOp(const User *I) {
2471 const Instruction *Inst = dyn_cast<Instruction>(I);
2472 if (!Inst || !Inst->getType()->isVectorTy())
2473 return false;
2474
2475 auto OpCode = Inst->getOpcode();
2476 switch (OpCode) {
2477 case Instruction::Add:
2478 case Instruction::Mul:
2479 case Instruction::And:
2480 case Instruction::Or:
2481 case Instruction::Xor:
2482 break;
2483 case Instruction::FAdd:
2484 case Instruction::FMul:
2485 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2486 if (FPOp->getFastMathFlags().unsafeAlgebra())
2487 break;
2488 LLVM_FALLTHROUGH[[clang::fallthrough]];
2489 default:
2490 return false;
2491 }
2492
2493 unsigned ElemNum = Inst->getType()->getVectorNumElements();
2494 unsigned ElemNumToReduce = ElemNum;
2495
2496 // Do DFS search on the def-use chain from the given instruction. We only
2497 // allow four kinds of operations during the search until we reach the
2498 // instruction that extracts the first element from the vector:
2499 //
2500 // 1. The reduction operation of the same opcode as the given instruction.
2501 //
2502 // 2. PHI node.
2503 //
2504 // 3. ShuffleVector instruction together with a reduction operation that
2505 // does a partial reduction.
2506 //
2507 // 4. ExtractElement that extracts the first element from the vector, and we
2508 // stop searching the def-use chain here.
2509 //
2510 // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2511 // from 1-3 to the stack to continue the DFS. The given instruction is not
2512 // a reduction operation if we meet any other instructions other than those
2513 // listed above.
2514
2515 SmallVector<const User *, 16> UsersToVisit{Inst};
2516 SmallPtrSet<const User *, 16> Visited;
2517 bool ReduxExtracted = false;
2518
2519 while (!UsersToVisit.empty()) {
2520 auto User = UsersToVisit.back();
2521 UsersToVisit.pop_back();
2522 if (!Visited.insert(User).second)
2523 continue;
2524
2525 for (const auto &U : User->users()) {
2526 auto Inst = dyn_cast<Instruction>(U);
2527 if (!Inst)
2528 return false;
2529
2530 if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2531 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2532 if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().unsafeAlgebra())
2533 return false;
2534 UsersToVisit.push_back(U);
2535 } else if (const ShuffleVectorInst *ShufInst =
2536 dyn_cast<ShuffleVectorInst>(U)) {
2537 // Detect the following pattern: A ShuffleVector instruction together
2538 // with a reduction that do partial reduction on the first and second
2539 // ElemNumToReduce / 2 elements, and store the result in
2540 // ElemNumToReduce / 2 elements in another vector.
2541
2542 unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2543 if (ResultElements < ElemNum)
2544 return false;
2545
2546 if (ElemNumToReduce == 1)
2547 return false;
2548 if (!isa<UndefValue>(U->getOperand(1)))
2549 return false;
2550 for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2551 if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2552 return false;
2553 for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2554 if (ShufInst->getMaskValue(i) != -1)
2555 return false;
2556
2557 // There is only one user of this ShuffleVector instruction, which
2558 // must be a reduction operation.
2559 if (!U->hasOneUse())
2560 return false;
2561
2562 auto U2 = dyn_cast<Instruction>(*U->user_begin());
2563 if (!U2 || U2->getOpcode() != OpCode)
2564 return false;
2565
2566 // Check operands of the reduction operation.
2567 if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2568 (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2569 UsersToVisit.push_back(U2);
2570 ElemNumToReduce /= 2;
2571 } else
2572 return false;
2573 } else if (isa<ExtractElementInst>(U)) {
2574 // At this moment we should have reduced all elements in the vector.
2575 if (ElemNumToReduce != 1)
2576 return false;
2577
2578 const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2579 if (!Val || Val->getZExtValue() != 0)
2580 return false;
2581
2582 ReduxExtracted = true;
2583 } else
2584 return false;
2585 }
2586 }
2587 return ReduxExtracted;
2588}
2589
2590void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
2591 SDValue Op1 = getValue(I.getOperand(0));
2592 SDValue Op2 = getValue(I.getOperand(1));
2593
2594 bool nuw = false;
2595 bool nsw = false;
2596 bool exact = false;
2597 bool vec_redux = false;
2598 FastMathFlags FMF;
2599
2600 if (const OverflowingBinaryOperator *OFBinOp =
2601 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2602 nuw = OFBinOp->hasNoUnsignedWrap();
2603 nsw = OFBinOp->hasNoSignedWrap();
2604 }
2605 if (const PossiblyExactOperator *ExactOp =
2606 dyn_cast<const PossiblyExactOperator>(&I))
2607 exact = ExactOp->isExact();
2608 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
2609 FMF = FPOp->getFastMathFlags();
2610
2611 if (isVectorReductionOp(&I)) {
2612 vec_redux = true;
2613 DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Detected a reduction operation:"
<< I << "\n"; } } while (false)
;
2614 }
2615
2616 SDNodeFlags Flags;
2617 Flags.setExact(exact);
2618 Flags.setNoSignedWrap(nsw);
2619 Flags.setNoUnsignedWrap(nuw);
2620 Flags.setVectorReduction(vec_redux);
2621 if (EnableFMFInDAG) {
2622 Flags.setAllowReciprocal(FMF.allowReciprocal());
2623 Flags.setNoInfs(FMF.noInfs());
2624 Flags.setNoNaNs(FMF.noNaNs());
2625 Flags.setNoSignedZeros(FMF.noSignedZeros());
2626 Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
2627 }
2628 SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
2629 Op1, Op2, &Flags);
2630 setValue(&I, BinNodeValue);
2631}
2632
2633void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2634 SDValue Op1 = getValue(I.getOperand(0));
2635 SDValue Op2 = getValue(I.getOperand(1));
2636
2637 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2638 Op2.getValueType(), DAG.getDataLayout());
2639
2640 // Coerce the shift amount to the right type if we can.
2641 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2642 unsigned ShiftSize = ShiftTy.getSizeInBits();
2643 unsigned Op2Size = Op2.getValueSizeInBits();
2644 SDLoc DL = getCurSDLoc();
2645
2646 // If the operand is smaller than the shift count type, promote it.
2647 if (ShiftSize > Op2Size)
2648 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2649
2650 // If the operand is larger than the shift count type but the shift
2651 // count type has enough bits to represent any shift value, truncate
2652 // it now. This is a common case and it exposes the truncate to
2653 // optimization early.
2654 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2655 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2656 // Otherwise we'll need to temporarily settle for some other convenient
2657 // type. Type legalization will make adjustments once the shiftee is split.
2658 else
2659 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2660 }
2661
2662 bool nuw = false;
2663 bool nsw = false;
2664 bool exact = false;
2665
2666 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2667
2668 if (const OverflowingBinaryOperator *OFBinOp =
2669 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2670 nuw = OFBinOp->hasNoUnsignedWrap();
2671 nsw = OFBinOp->hasNoSignedWrap();
2672 }
2673 if (const PossiblyExactOperator *ExactOp =
2674 dyn_cast<const PossiblyExactOperator>(&I))
2675 exact = ExactOp->isExact();
2676 }
2677 SDNodeFlags Flags;
2678 Flags.setExact(exact);
2679 Flags.setNoSignedWrap(nsw);
2680 Flags.setNoUnsignedWrap(nuw);
2681 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2682 &Flags);
2683 setValue(&I, Res);
2684}
2685
2686void SelectionDAGBuilder::visitSDiv(const User &I) {
2687 SDValue Op1 = getValue(I.getOperand(0));
2688 SDValue Op2 = getValue(I.getOperand(1));
2689
2690 SDNodeFlags Flags;
2691 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2692 cast<PossiblyExactOperator>(&I)->isExact());
2693 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2694 Op2, &Flags));
2695}
2696
2697void SelectionDAGBuilder::visitICmp(const User &I) {
2698 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2699 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2700 predicate = IC->getPredicate();
2701 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2702 predicate = ICmpInst::Predicate(IC->getPredicate());
2703 SDValue Op1 = getValue(I.getOperand(0));
2704 SDValue Op2 = getValue(I.getOperand(1));
2705 ISD::CondCode Opcode = getICmpCondCode(predicate);
2706
2707 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2708 I.getType());
2709 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2710}
2711
2712void SelectionDAGBuilder::visitFCmp(const User &I) {
2713 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2714 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2715 predicate = FC->getPredicate();
2716 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2717 predicate = FCmpInst::Predicate(FC->getPredicate());
2718 SDValue Op1 = getValue(I.getOperand(0));
2719 SDValue Op2 = getValue(I.getOperand(1));
2720 ISD::CondCode Condition = getFCmpCondCode(predicate);
2721
2722 // FIXME: Fcmp instructions have fast-math-flags in IR, so we should use them.
2723 // FIXME: We should propagate the fast-math-flags to the DAG node itself for
2724 // further optimization, but currently FMF is only applicable to binary nodes.
2725 if (TM.Options.NoNaNsFPMath)
2726 Condition = getFCmpCodeWithoutNaN(Condition);
2727 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2728 I.getType());
2729 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2730}
2731
2732// Check if the condition of the select has one use or two users that are both
2733// selects with the same condition.
2734static bool hasOnlySelectUsers(const Value *Cond) {
2735 return all_of(Cond->users(), [](const Value *V) {
2736 return isa<SelectInst>(V);
2737 });
2738}
2739
2740void SelectionDAGBuilder::visitSelect(const User &I) {
2741 SmallVector<EVT, 4> ValueVTs;
2742 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2743 ValueVTs);
2744 unsigned NumValues = ValueVTs.size();
2745 if (NumValues == 0) return;
2746
2747 SmallVector<SDValue, 4> Values(NumValues);
2748 SDValue Cond = getValue(I.getOperand(0));
2749 SDValue LHSVal = getValue(I.getOperand(1));
2750 SDValue RHSVal = getValue(I.getOperand(2));
2751 auto BaseOps = {Cond};
2752 ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2753 ISD::VSELECT : ISD::SELECT;
2754
2755 // Min/max matching is only viable if all output VTs are the same.
2756 if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2757 EVT VT = ValueVTs[0];
2758 LLVMContext &Ctx = *DAG.getContext();
2759 auto &TLI = DAG.getTargetLoweringInfo();
2760
2761 // We care about the legality of the operation after it has been type
2762 // legalized.
2763 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2764 VT != TLI.getTypeToTransformTo(Ctx, VT))
2765 VT = TLI.getTypeToTransformTo(Ctx, VT);
2766
2767 // If the vselect is legal, assume we want to leave this as a vector setcc +
2768 // vselect. Otherwise, if this is going to be scalarized, we want to see if
2769 // min/max is legal on the scalar type.
2770 bool UseScalarMinMax = VT.isVector() &&
2771 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2772
2773 Value *LHS, *RHS;
2774 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2775 ISD::NodeType Opc = ISD::DELETED_NODE;
2776 switch (SPR.Flavor) {
2777 case SPF_UMAX: Opc = ISD::UMAX; break;
2778 case SPF_UMIN: Opc = ISD::UMIN; break;
2779 case SPF_SMAX: Opc = ISD::SMAX; break;
2780 case SPF_SMIN: Opc = ISD::SMIN; break;
2781 case SPF_FMINNUM:
2782 switch (SPR.NaNBehavior) {
2783 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2783)
;
2784 case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2785 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2786 case SPNB_RETURNS_ANY: {
2787 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2788 Opc = ISD::FMINNUM;
2789 else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2790 Opc = ISD::FMINNAN;
2791 else if (UseScalarMinMax)
2792 Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2793 ISD::FMINNUM : ISD::FMINNAN;
2794 break;
2795 }
2796 }
2797 break;
2798 case SPF_FMAXNUM:
2799 switch (SPR.NaNBehavior) {
2800 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2800)
;
2801 case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2802 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2803 case SPNB_RETURNS_ANY:
2804
2805 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2806 Opc = ISD::FMAXNUM;
2807 else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2808 Opc = ISD::FMAXNAN;
2809 else if (UseScalarMinMax)
2810 Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
2811 ISD::FMAXNUM : ISD::FMAXNAN;
2812 break;
2813 }
2814 break;
2815 default: break;
2816 }
2817
2818 if (Opc != ISD::DELETED_NODE &&
2819 (TLI.isOperationLegalOrCustom(Opc, VT) ||
2820 (UseScalarMinMax &&
2821 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
2822 // If the underlying comparison instruction is used by any other
2823 // instruction, the consumed instructions won't be destroyed, so it is
2824 // not profitable to convert to a min/max.
2825 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
2826 OpCode = Opc;
2827 LHSVal = getValue(LHS);
2828 RHSVal = getValue(RHS);
2829 BaseOps = {};
2830 }
2831 }
2832
2833 for (unsigned i = 0; i != NumValues; ++i) {
2834 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2835 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2836 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2837 Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2838 LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2839 Ops);
2840 }
2841
2842 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
2843 DAG.getVTList(ValueVTs), Values));
2844}
2845
2846void SelectionDAGBuilder::visitTrunc(const User &I) {
2847 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
2848 SDValue N = getValue(I.getOperand(0));
2849 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2850 I.getType());
2851 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
2852}
2853
2854void SelectionDAGBuilder::visitZExt(const User &I) {
2855 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2856 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
2857 SDValue N = getValue(I.getOperand(0));
2858 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2859 I.getType());
2860 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
2861}
2862
2863void SelectionDAGBuilder::visitSExt(const User &I) {
2864 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2865 // SExt also can't be a cast to bool for same reason. So, nothing much to do
2866 SDValue N = getValue(I.getOperand(0));
2867 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2868 I.getType());
2869 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
2870}
2871
2872void SelectionDAGBuilder::visitFPTrunc(const User &I) {
2873 // FPTrunc is never a no-op cast, no need to check
2874 SDValue N = getValue(I.getOperand(0));
2875 SDLoc dl = getCurSDLoc();
2876 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2877 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2878 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
2879 DAG.getTargetConstant(
2880 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
2881}
2882
2883void SelectionDAGBuilder::visitFPExt(const User &I) {
2884 // FPExt is never a no-op cast, no need to check
2885 SDValue N = getValue(I.getOperand(0));
2886 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2887 I.getType());
2888 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
2889}
2890
2891void SelectionDAGBuilder::visitFPToUI(const User &I) {
2892 // FPToUI is never a no-op cast, no need to check
2893 SDValue N = getValue(I.getOperand(0));
2894 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2895 I.getType());
2896 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
2897}
2898
2899void SelectionDAGBuilder::visitFPToSI(const User &I) {
2900 // FPToSI is never a no-op cast, no need to check
2901 SDValue N = getValue(I.getOperand(0));
2902 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2903 I.getType());
2904 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
2905}
2906
2907void SelectionDAGBuilder::visitUIToFP(const User &I) {
2908 // UIToFP is never a no-op cast, no need to check
2909 SDValue N = getValue(I.getOperand(0));
2910 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2911 I.getType());
2912 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
2913}
2914
2915void SelectionDAGBuilder::visitSIToFP(const User &I) {
2916 // SIToFP is never a no-op cast, no need to check
2917 SDValue N = getValue(I.getOperand(0));
2918 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2919 I.getType());
2920 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
2921}
2922
2923void SelectionDAGBuilder::visitPtrToInt(const User &I) {
2924 // What to do depends on the size of the integer and the size of the pointer.
2925 // We can either truncate, zero extend, or no-op, accordingly.
2926 SDValue N = getValue(I.getOperand(0));
2927 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2928 I.getType());
2929 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2930}
2931
2932void SelectionDAGBuilder::visitIntToPtr(const User &I) {
2933 // What to do depends on the size of the integer and the size of the pointer.
2934 // We can either truncate, zero extend, or no-op, accordingly.
2935 SDValue N = getValue(I.getOperand(0));
2936 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2937 I.getType());
2938 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2939}
2940
2941void SelectionDAGBuilder::visitBitCast(const User &I) {
2942 SDValue N = getValue(I.getOperand(0));
2943 SDLoc dl = getCurSDLoc();
2944 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2945 I.getType());
2946
2947 // BitCast assures us that source and destination are the same size so this is
2948 // either a BITCAST or a no-op.
2949 if (DestVT != N.getValueType())
2950 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
2951 DestVT, N)); // convert types.
2952 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
2953 // might fold any kind of constant expression to an integer constant and that
2954 // is not what we are looking for. Only regcognize a bitcast of a genuine
2955 // constant integer as an opaque constant.
2956 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
2957 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
2958 /*isOpaque*/true));
2959 else
2960 setValue(&I, N); // noop cast.
2961}
2962
2963void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
2964 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2965 const Value *SV = I.getOperand(0);
2966 SDValue N = getValue(SV);
2967 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2968
2969 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
2970 unsigned DestAS = I.getType()->getPointerAddressSpace();
2971
2972 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
2973 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
2974
2975 setValue(&I, N);
2976}
2977
2978void SelectionDAGBuilder::visitInsertElement(const User &I) {
2979 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2980 SDValue InVec = getValue(I.getOperand(0));
2981 SDValue InVal = getValue(I.getOperand(1));
2982 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
2983 TLI.getVectorIdxTy(DAG.getDataLayout()));
2984 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
2985 TLI.getValueType(DAG.getDataLayout(), I.getType()),
2986 InVec, InVal, InIdx));
2987}
2988
2989void SelectionDAGBuilder::visitExtractElement(const User &I) {
2990 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2991 SDValue InVec = getValue(I.getOperand(0));
2992 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
2993 TLI.getVectorIdxTy(DAG.getDataLayout()));
2994 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
2995 TLI.getValueType(DAG.getDataLayout(), I.getType()),
2996 InVec, InIdx));
2997}
2998
2999void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3000 SDValue Src1 = getValue(I.getOperand(0));
3001 SDValue Src2 = getValue(I.getOperand(1));
3002 SDLoc DL = getCurSDLoc();
3003
3004 SmallVector<int, 8> Mask;
3005 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3006 unsigned MaskNumElts = Mask.size();
3007
3008 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3009 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3010 EVT SrcVT = Src1.getValueType();
3011 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3012
3013 if (SrcNumElts == MaskNumElts) {
3014 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3015 return;
3016 }
3017
3018 // Normalize the shuffle vector since mask and vector length don't match.
3019 if (SrcNumElts < MaskNumElts) {
3020 // Mask is longer than the source vectors. We can use concatenate vector to
3021 // make the mask and vectors lengths match.
3022
3023 if (MaskNumElts % SrcNumElts == 0) {
3024 // Mask length is a multiple of the source vector length.
3025 // Check if the shuffle is some kind of concatenation of the input
3026 // vectors.
3027 unsigned NumConcat = MaskNumElts / SrcNumElts;
3028 bool IsConcat = true;
3029 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3030 for (unsigned i = 0; i != MaskNumElts; ++i) {
3031 int Idx = Mask[i];
3032 if (Idx < 0)
3033 continue;
3034 // Ensure the indices in each SrcVT sized piece are sequential and that
3035 // the same source is used for the whole piece.
3036 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3037 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3038 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3039 IsConcat = false;
3040 break;
3041 }
3042 // Remember which source this index came from.
3043 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3044 }
3045
3046 // The shuffle is concatenating multiple vectors together. Just emit
3047 // a CONCAT_VECTORS operation.
3048 if (IsConcat) {
3049 SmallVector<SDValue, 8> ConcatOps;
3050 for (auto Src : ConcatSrcs) {
3051 if (Src < 0)
3052 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3053 else if (Src == 0)
3054 ConcatOps.push_back(Src1);
3055 else
3056 ConcatOps.push_back(Src2);
3057 }
3058 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3059 return;
3060 }
3061 }
3062
3063 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3064 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3065 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3066 PaddedMaskNumElts);
3067
3068 // Pad both vectors with undefs to make them the same length as the mask.
3069 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3070
3071 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3072 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3073 MOps1[0] = Src1;
3074 MOps2[0] = Src2;
3075
3076 Src1 = Src1.isUndef()
3077 ? DAG.getUNDEF(PaddedVT)
3078 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3079 Src2 = Src2.isUndef()
3080 ? DAG.getUNDEF(PaddedVT)
3081 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3082
3083 // Readjust mask for new input vector length.
3084 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3085 for (unsigned i = 0; i != MaskNumElts; ++i) {
3086 int Idx = Mask[i];
3087 if (Idx >= (int)SrcNumElts)
3088 Idx -= SrcNumElts - PaddedMaskNumElts;
3089 MappedOps[i] = Idx;
3090 }
3091
3092 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3093
3094 // If the concatenated vector was padded, extract a subvector with the
3095 // correct number of elements.
3096 if (MaskNumElts != PaddedMaskNumElts)
3097 Result = DAG.getNode(
3098 ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3099 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3100
3101 setValue(&I, Result);
3102 return;
3103 }
3104
3105 if (SrcNumElts > MaskNumElts) {
3106 // Analyze the access pattern of the vector to see if we can extract
3107 // two subvectors and do the shuffle.
3108 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3109 bool CanExtract = true;
3110 for (int Idx : Mask) {
3111 unsigned Input = 0;
3112 if (Idx < 0)
3113 continue;
3114
3115 if (Idx >= (int)SrcNumElts) {
3116 Input = 1;
3117 Idx -= SrcNumElts;
3118 }
3119
3120 // If all the indices come from the same MaskNumElts sized portion of
3121 // the sources we can use extract. Also make sure the extract wouldn't
3122 // extract past the end of the source.
3123 int NewStartIdx = alignDown(Idx, MaskNumElts);
3124 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3125 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3126 CanExtract = false;
3127 // Make sure we always update StartIdx as we use it to track if all
3128 // elements are undef.
3129 StartIdx[Input] = NewStartIdx;
3130 }
3131
3132 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3133 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3134 return;
3135 }
3136 if (CanExtract) {
3137 // Extract appropriate subvector and generate a vector shuffle
3138 for (unsigned Input = 0; Input < 2; ++Input) {
3139 SDValue &Src = Input == 0 ? Src1 : Src2;
3140 if (StartIdx[Input] < 0)
3141 Src = DAG.getUNDEF(VT);
3142 else {
3143 Src = DAG.getNode(
3144 ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3145 DAG.getConstant(StartIdx[Input], DL,
3146 TLI.getVectorIdxTy(DAG.getDataLayout())));
3147 }
3148 }
3149
3150 // Calculate new mask.
3151 SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3152 for (int &Idx : MappedOps) {
3153 if (Idx >= (int)SrcNumElts)
3154 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3155 else if (Idx >= 0)
3156 Idx -= StartIdx[0];
3157 }
3158
3159 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3160 return;
3161 }
3162 }
3163
3164 // We can't use either concat vectors or extract subvectors so fall back to
3165 // replacing the shuffle with extract and build vector.
3166 // to insert and build vector.
3167 EVT EltVT = VT.getVectorElementType();
3168 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3169 SmallVector<SDValue,8> Ops;
3170 for (int Idx : Mask) {
3171 SDValue Res;
3172
3173 if (Idx < 0) {
3174 Res = DAG.getUNDEF(EltVT);
3175 } else {
3176 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3177 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3178
3179 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3180 EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3181 }
3182
3183 Ops.push_back(Res);
3184 }
3185
3186 setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops));
3187}
3188
3189void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3190 const Value *Op0 = I.getOperand(0);
3191 const Value *Op1 = I.getOperand(1);
3192 Type *AggTy = I.getType();
3193 Type *ValTy = Op1->getType();
3194 bool IntoUndef = isa<UndefValue>(Op0);
3195 bool FromUndef = isa<UndefValue>(Op1);
3196
3197 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3198
3199 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3200 SmallVector<EVT, 4> AggValueVTs;
3201 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3202 SmallVector<EVT, 4> ValValueVTs;
3203 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3204
3205 unsigned NumAggValues = AggValueVTs.size();
3206 unsigned NumValValues = ValValueVTs.size();
3207 SmallVector<SDValue, 4> Values(NumAggValues);
3208
3209 // Ignore an insertvalue that produces an empty object
3210 if (!NumAggValues) {
3211 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3212 return;
3213 }
3214
3215 SDValue Agg = getValue(Op0);
3216 unsigned i = 0;
3217 // Copy the beginning value(s) from the original aggregate.
3218 for (; i != LinearIndex; ++i)
3219 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3220 SDValue(Agg.getNode(), Agg.getResNo() + i);
3221 // Copy values from the inserted value(s).
3222 if (NumValValues) {
3223 SDValue Val = getValue(Op1);
3224 for (; i != LinearIndex + NumValValues; ++i)
3225 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3226 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3227 }
3228 // Copy remaining value(s) from the original aggregate.
3229 for (; i != NumAggValues; ++i)
3230 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3231 SDValue(Agg.getNode(), Agg.getResNo() + i);
3232
3233 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3234 DAG.getVTList(AggValueVTs), Values));
3235}
3236
3237void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3238 const Value *Op0 = I.getOperand(0);
3239 Type *AggTy = Op0->getType();
3240 Type *ValTy = I.getType();
3241 bool OutOfUndef = isa<UndefValue>(Op0);
3242
3243 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3244
3245 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3246 SmallVector<EVT, 4> ValValueVTs;
3247 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3248
3249 unsigned NumValValues = ValValueVTs.size();
3250
3251 // Ignore a extractvalue that produces an empty object
3252 if (!NumValValues) {
3253 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3254 return;
3255 }
3256
3257 SmallVector<SDValue, 4> Values(NumValValues);
3258
3259 SDValue Agg = getValue(Op0);
3260 // Copy out the selected value(s).
3261 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3262 Values[i - LinearIndex] =
3263 OutOfUndef ?
3264 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3265 SDValue(Agg.getNode(), Agg.getResNo() + i);
3266
3267 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3268 DAG.getVTList(ValValueVTs), Values));
3269}
3270
3271void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3272 Value *Op0 = I.getOperand(0);
3273 // Note that the pointer operand may be a vector of pointers. Take the scalar
3274 // element which holds a pointer.
3275 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3276 SDValue N = getValue(Op0);
3277 SDLoc dl = getCurSDLoc();
3278
3279 // Normalize Vector GEP - all scalar operands should be converted to the
3280 // splat vector.
3281 unsigned VectorWidth = I.getType()->isVectorTy() ?
3282 cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3283
3284 if (VectorWidth && !N.getValueType().isVector()) {
3285 LLVMContext &Context = *DAG.getContext();
3286 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3287 N = DAG.getSplatBuildVector(VT, dl, N);
3288 }
3289
3290 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3291 GTI != E; ++GTI) {
3292 const Value *Idx = GTI.getOperand();
3293 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3294 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3295 if (Field) {
3296 // N = N + Offset
3297 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3298
3299 // In an inbouds GEP with an offset that is nonnegative even when
3300 // interpreted as signed, assume there is no unsigned overflow.
3301 SDNodeFlags Flags;
3302 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3303 Flags.setNoUnsignedWrap(true);
3304
3305 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3306 DAG.getConstant(Offset, dl, N.getValueType()), &Flags);
3307 }
3308 } else {
3309 MVT PtrTy =
3310 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout(), AS);
3311 unsigned PtrSize = PtrTy.getSizeInBits();
3312 APInt ElementSize(PtrSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3313
3314 // If this is a scalar constant or a splat vector of constants,
3315 // handle it quickly.
3316 const auto *CI = dyn_cast<ConstantInt>(Idx);
3317 if (!CI && isa<ConstantDataVector>(Idx) &&
3318 cast<ConstantDataVector>(Idx)->getSplatValue())
3319 CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3320
3321 if (CI) {
3322 if (CI->isZero())
3323 continue;
3324 APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
3325 LLVMContext &Context = *DAG.getContext();
3326 SDValue OffsVal = VectorWidth ?
3327 DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, PtrTy, VectorWidth)) :
3328 DAG.getConstant(Offs, dl, PtrTy);
3329
3330 // In an inbouds GEP with an offset that is nonnegative even when
3331 // interpreted as signed, assume there is no unsigned overflow.
3332 SDNodeFlags Flags;
3333 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3334 Flags.setNoUnsignedWrap(true);
3335
3336 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, &Flags);
3337 continue;
3338 }
3339
3340 // N = N + Idx * ElementSize;
3341 SDValue IdxN = getValue(Idx);
3342
3343 if (!IdxN.getValueType().isVector() && VectorWidth) {
3344 MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
3345 IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3346 }
3347
3348 // If the index is smaller or larger than intptr_t, truncate or extend
3349 // it.
3350 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3351
3352 // If this is a multiply by a power of two, turn it into a shl
3353 // immediately. This is a very common case.
3354 if (ElementSize != 1) {
3355 if (ElementSize.isPowerOf2()) {
3356 unsigned Amt = ElementSize.logBase2();
3357 IdxN = DAG.getNode(ISD::SHL, dl,
3358 N.getValueType(), IdxN,
3359 DAG.getConstant(Amt, dl, IdxN.getValueType()));
3360 } else {
3361 SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3362 IdxN = DAG.getNode(ISD::MUL, dl,
3363 N.getValueType(), IdxN, Scale);
3364 }
3365 }
3366
3367 N = DAG.getNode(ISD::ADD, dl,
3368 N.getValueType(), N, IdxN);
3369 }
3370 }
3371
3372 setValue(&I, N);
3373}
3374
3375void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3376 // If this is a fixed sized alloca in the entry block of the function,
3377 // allocate it statically on the stack.
3378 if (FuncInfo.StaticAllocaMap.count(&I))
3379 return; // getValue will auto-populate this.
3380
3381 SDLoc dl = getCurSDLoc();
3382 Type *Ty = I.getAllocatedType();
3383 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3384 auto &DL = DAG.getDataLayout();
3385 uint64_t TySize = DL.getTypeAllocSize(Ty);
3386 unsigned Align =
3387 std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3388
3389 SDValue AllocSize = getValue(I.getArraySize());
3390
3391 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
3392 if (AllocSize.getValueType() != IntPtr)
3393 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3394
3395 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3396 AllocSize,
3397 DAG.getConstant(TySize, dl, IntPtr));
3398
3399 // Handle alignment. If the requested alignment is less than or equal to
3400 // the stack alignment, ignore it. If the size is greater than or equal to
3401 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3402 unsigned StackAlign =
3403 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3404 if (Align <= StackAlign)
3405 Align = 0;
3406
3407 // Round the size of the allocation up to the stack alignment size
3408 // by add SA-1 to the size. This doesn't overflow because we're computing
3409 // an address inside an alloca.
3410 SDNodeFlags Flags;
3411 Flags.setNoUnsignedWrap(true);
3412 AllocSize = DAG.getNode(ISD::ADD, dl,
3413 AllocSize.getValueType(), AllocSize,
3414 DAG.getIntPtrConstant(StackAlign - 1, dl), &Flags);
3415
3416 // Mask out the low bits for alignment purposes.
3417 AllocSize = DAG.getNode(ISD::AND, dl,
3418 AllocSize.getValueType(), AllocSize,
3419 DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
3420 dl));
3421
3422 SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
3423 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3424 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3425 setValue(&I, DSA);
3426 DAG.setRoot(DSA.getValue(1));
3427
3428 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())((FuncInfo.MF->getFrameInfo().hasVarSizedObjects()) ? static_cast
<void> (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3428, __PRETTY_FUNCTION__))
;
3429}
3430
3431void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3432 if (I.isAtomic())
3433 return visitAtomicLoad(I);
3434
3435 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3436 const Value *SV = I.getOperand(0);
3437 if (TLI.supportSwiftError()) {
3438 // Swifterror values can come from either a function parameter with
3439 // swifterror attribute or an alloca with swifterror attribute.
3440 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3441 if (Arg->hasSwiftErrorAttr())
3442 return visitLoadFromSwiftError(I);
3443 }
3444
3445 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3446 if (Alloca->isSwiftError())
3447 return visitLoadFromSwiftError(I);
3448 }
3449 }
3450
3451 SDValue Ptr = getValue(SV);
3452
3453 Type *Ty = I.getType();
3454
3455 bool isVolatile = I.isVolatile();
3456 bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3457 bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3458 bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3459 unsigned Alignment = I.getAlignment();
3460
3461 AAMDNodes AAInfo;
3462 I.getAAMetadata(AAInfo);
3463 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3464
3465 SmallVector<EVT, 4> ValueVTs;
3466 SmallVector<uint64_t, 4> Offsets;
3467 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3468 unsigned NumValues = ValueVTs.size();
3469 if (NumValues == 0)
3470 return;
3471
3472 SDValue Root;
3473 bool ConstantMemory = false;
3474 if (isVolatile || NumValues > MaxParallelChains)
3475 // Serialize volatile loads with other side effects.
3476 Root = getRoot();
3477 else if (AA->pointsToConstantMemory(MemoryLocation(
3478 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3479 // Do not serialize (non-volatile) loads of constant memory with anything.
3480 Root = DAG.getEntryNode();
3481 ConstantMemory = true;
3482 } else {
3483 // Do not serialize non-volatile loads against each other.
3484 Root = DAG.getRoot();
3485 }
3486
3487 SDLoc dl = getCurSDLoc();
3488
3489 if (isVolatile)
3490 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3491
3492 // An aggregate load cannot wrap around the address space, so offsets to its
3493 // parts don't wrap either.
3494 SDNodeFlags Flags;
3495 Flags.setNoUnsignedWrap(true);
3496
3497 SmallVector<SDValue, 4> Values(NumValues);
3498 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3499 EVT PtrVT = Ptr.getValueType();
3500 unsigned ChainI = 0;
3501 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3502 // Serializing loads here may result in excessive register pressure, and
3503 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3504 // could recover a bit by hoisting nodes upward in the chain by recognizing
3505 // they are side-effect free or do not alias. The optimizer should really
3506 // avoid this case by converting large object/array copies to llvm.memcpy
3507 // (MaxParallelChains should always remain as failsafe).
3508 if (ChainI == MaxParallelChains) {
3509 assert(PendingLoads.empty() && "PendingLoads must be serialized first")((PendingLoads.empty() && "PendingLoads must be serialized first"
) ? static_cast<void> (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3509, __PRETTY_FUNCTION__))
;
3510 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3511 makeArrayRef(Chains.data(), ChainI));
3512 Root = Chain;
3513 ChainI = 0;
3514 }
3515 SDValue A = DAG.getNode(ISD::ADD, dl,
3516 PtrVT, Ptr,
3517 DAG.getConstant(Offsets[i], dl, PtrVT),
3518 &Flags);
3519 auto MMOFlags = MachineMemOperand::MONone;
3520 if (isVolatile)
3521 MMOFlags |= MachineMemOperand::MOVolatile;
3522 if (isNonTemporal)
3523 MMOFlags |= MachineMemOperand::MONonTemporal;
3524 if (isInvariant)
3525 MMOFlags |= MachineMemOperand::MOInvariant;
3526 if (isDereferenceable)
3527 MMOFlags |= MachineMemOperand::MODereferenceable;
3528
3529 SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3530 MachinePointerInfo(SV, Offsets[i]), Alignment,
3531 MMOFlags, AAInfo, Ranges);
3532
3533 Values[i] = L;
3534 Chains[ChainI] = L.getValue(1);
3535 }
3536
3537 if (!ConstantMemory) {
3538 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3539 makeArrayRef(Chains.data(), ChainI));
3540 if (isVolatile)
3541 DAG.setRoot(Chain);
3542 else
3543 PendingLoads.push_back(Chain);
3544 }
3545
3546 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3547 DAG.getVTList(ValueVTs), Values));
3548}
3549
3550void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3551 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3552 assert(TLI.supportSwiftError() &&((TLI.supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("TLI.supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3553, __PRETTY_FUNCTION__))
3553 "call visitStoreToSwiftError when backend supports swifterror")((TLI.supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("TLI.supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3553, __PRETTY_FUNCTION__))
;
3554
3555 SmallVector<EVT, 4> ValueVTs;
3556 SmallVector<uint64_t, 4> Offsets;
3557 const Value *SrcV = I.getOperand(0);
3558 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3559 SrcV->getType(), ValueVTs, &Offsets);
3560 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3561, __PRETTY_FUNCTION__))
3561 "expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3561, __PRETTY_FUNCTION__))
;
3562
3563 SDValue Src = getValue(SrcV);
3564 // Create a virtual register, then update the virtual register.
3565 auto &DL = DAG.getDataLayout();
3566 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
3567 unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
3568 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3569 // Chain can be getRoot or getControlRoot.
3570 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3571 SDValue(Src.getNode(), Src.getResNo()));
3572 DAG.setRoot(CopyNode);
3573 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3574}
3575
3576void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3577 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3578, __PRETTY_FUNCTION__))
3578 "call visitLoadFromSwiftError when backend supports swifterror")((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3578, __PRETTY_FUNCTION__))
;
3579
3580 assert(!I.isVolatile() &&((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3583, __PRETTY_FUNCTION__))
3581 I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3583, __PRETTY_FUNCTION__))
3582 I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3583, __PRETTY_FUNCTION__))
3583 "Support volatile, non temporal, invariant for load_from_swift_error")((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3583, __PRETTY_FUNCTION__))
;
3584
3585 const Value *SV = I.getOperand(0);
3586 Type *Ty = I.getType();
3587 AAMDNodes AAInfo;
3588 I.getAAMetadata(AAInfo);
3589 assert(!AA->pointsToConstantMemory(MemoryLocation(((!AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout
().getTypeStoreSize(Ty), AAInfo)) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("!AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo)) && \"load_from_swift_error should not be constant memory\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3591, __PRETTY_FUNCTION__))
3590 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo)) &&((!AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout
().getTypeStoreSize(Ty), AAInfo)) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("!AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo)) && \"load_from_swift_error should not be constant memory\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3591, __PRETTY_FUNCTION__))
3591 "load_from_swift_error should not be constant memory")((!AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout
().getTypeStoreSize(Ty), AAInfo)) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("!AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo)) && \"load_from_swift_error should not be constant memory\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3591, __PRETTY_FUNCTION__))
;
3592
3593 SmallVector<EVT, 4> ValueVTs;
3594 SmallVector<uint64_t, 4> Offsets;
3595 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3596 ValueVTs, &Offsets);
3597 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3598, __PRETTY_FUNCTION__))
3598 "expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3598, __PRETTY_FUNCTION__))
;
3599
3600 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3601 SDValue L = DAG.getCopyFromReg(
3602 getRoot(), getCurSDLoc(),
3603 FuncInfo.getOrCreateSwiftErrorVReg(FuncInfo.MBB, SV), ValueVTs[0]);
3604
3605 setValue(&I, L);
3606}
3607
3608void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3609 if (I.isAtomic())
3610 return visitAtomicStore(I);
3611
3612 const Value *SrcV = I.getOperand(0);
3613 const Value *PtrV = I.getOperand(1);
3614
3615 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3616 if (TLI.supportSwiftError()) {
3617 // Swifterror values can come from either a function parameter with
3618 // swifterror attribute or an alloca with swifterror attribute.
3619 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3620 if (Arg->hasSwiftErrorAttr())
3621 return visitStoreToSwiftError(I);
3622 }
3623
3624 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3625 if (Alloca->isSwiftError())
3626 return visitStoreToSwiftError(I);
3627 }
3628 }
3629
3630 SmallVector<EVT, 4> ValueVTs;
3631 SmallVector<uint64_t, 4> Offsets;
3632 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3633 SrcV->getType(), ValueVTs, &Offsets);
3634 unsigned NumValues = ValueVTs.size();
3635 if (NumValues == 0)
3636 return;
3637
3638 // Get the lowered operands. Note that we do this after
3639 // checking if NumResults is zero, because with zero results
3640 // the operands won't have values in the map.
3641 SDValue Src = getValue(SrcV);
3642 SDValue Ptr = getValue(PtrV);
3643
3644 SDValue Root = getRoot();
3645 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3646 SDLoc dl = getCurSDLoc();
3647 EVT PtrVT = Ptr.getValueType();
3648 unsigned Alignment = I.getAlignment();
3649 AAMDNodes AAInfo;
3650 I.getAAMetadata(AAInfo);
3651
3652 auto MMOFlags = MachineMemOperand::MONone;
3653 if (I.isVolatile())
3654 MMOFlags |= MachineMemOperand::MOVolatile;
3655 if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3656 MMOFlags |= MachineMemOperand::MONonTemporal;
3657
3658 // An aggregate load cannot wrap around the address space, so offsets to its
3659 // parts don't wrap either.
3660 SDNodeFlags Flags;
3661 Flags.setNoUnsignedWrap(true);
3662
3663 unsigned ChainI = 0;
3664 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3665 // See visitLoad comments.
3666 if (ChainI == MaxParallelChains) {
3667 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3668 makeArrayRef(Chains.data(), ChainI));
3669 Root = Chain;
3670 ChainI = 0;
3671 }
3672 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3673 DAG.getConstant(Offsets[i], dl, PtrVT), &Flags);
3674 SDValue St = DAG.getStore(
3675 Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3676 MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3677 Chains[ChainI] = St;
3678 }
3679
3680 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3681 makeArrayRef(Chains.data(), ChainI));
3682 DAG.setRoot(StoreNode);
3683}
3684
3685void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3686 bool IsCompressing) {
3687 SDLoc sdl = getCurSDLoc();
3688
3689 auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3690 unsigned& Alignment) {
3691 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3692 Src0 = I.getArgOperand(0);
3693 Ptr = I.getArgOperand(1);
3694 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3695 Mask = I.getArgOperand(3);
3696 };
3697 auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3698 unsigned& Alignment) {
3699 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3700 Src0 = I.getArgOperand(0);
3701 Ptr = I.getArgOperand(1);
3702 Mask = I.getArgOperand(2);
3703 Alignment = 0;
3704 };
3705
3706 Value *PtrOperand, *MaskOperand, *Src0Operand;
3707 unsigned Alignment;
3708 if (IsCompressing)
3709 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3710 else
3711 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3712
3713 SDValue Ptr = getValue(PtrOperand);
3714 SDValue Src0 = getValue(Src0Operand);
3715 SDValue Mask = getValue(MaskOperand);
3716
3717 EVT VT = Src0.getValueType();
3718 if (!Alignment)
3719 Alignment = DAG.getEVTAlignment(VT);
3720
3721 AAMDNodes AAInfo;
3722 I.getAAMetadata(AAInfo);
3723
3724 MachineMemOperand *MMO =
3725 DAG.getMachineFunction().
3726 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3727 MachineMemOperand::MOStore, VT.getStoreSize(),
3728 Alignment, AAInfo);
3729 SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3730 MMO, false /* Truncating */,
3731 IsCompressing);
3732 DAG.setRoot(StoreNode);
3733 setValue(&I, StoreNode);
3734}
3735
3736// Get a uniform base for the Gather/Scatter intrinsic.
3737// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3738// We try to represent it as a base pointer + vector of indices.
3739// Usually, the vector of pointers comes from a 'getelementptr' instruction.
3740// The first operand of the GEP may be a single pointer or a vector of pointers
3741// Example:
3742// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3743// or
3744// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3745// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3746//
3747// When the first GEP operand is a single pointer - it is the uniform base we
3748// are looking for. If first operand of the GEP is a splat vector - we
3749// extract the spalt value and use it as a uniform base.
3750// In all other cases the function returns 'false'.
3751//
3752static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3753 SelectionDAGBuilder* SDB) {
3754
3755 SelectionDAG& DAG = SDB->DAG;
3756 LLVMContext &Context = *DAG.getContext();
3757
3758 assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")((Ptr->getType()->isVectorTy() && "Uexpected pointer type"
) ? static_cast<void> (0) : __assert_fail ("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3758, __PRETTY_FUNCTION__))
;
3759 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3760 if (!GEP || GEP->getNumOperands() > 2)
3761 return false;
3762
3763 const Value *GEPPtr = GEP->getPointerOperand();
3764 if (!GEPPtr->getType()->isVectorTy())
3765 Ptr = GEPPtr;
3766 else if (!(Ptr = getSplatValue(GEPPtr)))
3767 return false;
3768
3769 Value *IndexVal = GEP->getOperand(1);
3770
3771 // The operands of the GEP may be defined in another basic block.
3772 // In this case we'll not find nodes for the operands.
3773 if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3774 return false;
3775
3776 Base = SDB->getValue(Ptr);
3777 Index = SDB->getValue(IndexVal);
3778
3779 // Suppress sign extension.
3780 if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
3781 if (SDB->findValue(Sext->getOperand(0))) {
3782 IndexVal = Sext->getOperand(0);
3783 Index = SDB->getValue(IndexVal);
3784 }
3785 }
3786 if (!Index.getValueType().isVector()) {
3787 unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3788 EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3789 Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3790 }
3791 return true;
3792}
3793
3794void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3795 SDLoc sdl = getCurSDLoc();
3796
3797 // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3798 const Value *Ptr = I.getArgOperand(1);
3799 SDValue Src0 = getValue(I.getArgOperand(0));
3800 SDValue Mask = getValue(I.getArgOperand(3));
3801 EVT VT = Src0.getValueType();
3802 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3803 if (!Alignment)
3804 Alignment = DAG.getEVTAlignment(VT);
3805 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3806
3807 AAMDNodes AAInfo;
3808 I.getAAMetadata(AAInfo);
3809
3810 SDValue Base;
3811 SDValue Index;
3812 const Value *BasePtr = Ptr;
3813 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3814
3815 const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3816 MachineMemOperand *MMO = DAG.getMachineFunction().
3817 getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3818 MachineMemOperand::MOStore, VT.getStoreSize(),
3819 Alignment, AAInfo);
3820 if (!UniformBase) {
3821 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3822 Index = getValue(Ptr);
3823 }
3824 SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
3825 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
3826 Ops, MMO);
3827 DAG.setRoot(Scatter);
3828 setValue(&I, Scatter);
3829}
3830
3831void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
3832 SDLoc sdl = getCurSDLoc();
3833
3834 auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3835 unsigned& Alignment) {
3836 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
3837 Ptr = I.getArgOperand(0);
3838 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
3839 Mask = I.getArgOperand(2);
3840 Src0 = I.getArgOperand(3);
3841 };
3842 auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3843 unsigned& Alignment) {
3844 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
3845 Ptr = I.getArgOperand(0);
3846 Alignment = 0;
3847 Mask = I.getArgOperand(1);
3848 Src0 = I.getArgOperand(2);
3849 };
3850
3851 Value *PtrOperand, *MaskOperand, *Src0Operand;
3852 unsigned Alignment;
3853 if (IsExpanding)
3854 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3855 else
3856 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3857
3858 SDValue Ptr = getValue(PtrOperand);
3859 SDValue Src0 = getValue(Src0Operand);
3860 SDValue Mask = getValue(MaskOperand);
3861
3862 EVT VT = Src0.getValueType();
3863 if (!Alignment)
3864 Alignment = DAG.getEVTAlignment(VT);
3865
3866 AAMDNodes AAInfo;
3867 I.getAAMetadata(AAInfo);
3868 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3869
3870 // Do not serialize masked loads of constant memory with anything.
3871 bool AddToChain = !AA->pointsToConstantMemory(MemoryLocation(
3872 PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
3873 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
3874
3875 MachineMemOperand *MMO =
3876 DAG.getMachineFunction().
3877 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3878 MachineMemOperand::MOLoad, VT.getStoreSize(),
3879 Alignment, AAInfo, Ranges);
3880
3881 SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
3882 ISD::NON_EXTLOAD, IsExpanding);
3883 if (AddToChain) {
3884 SDValue OutChain = Load.getValue(1);
3885 DAG.setRoot(OutChain);
3886 }
3887 setValue(&I, Load);
3888}
3889
3890void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
3891 SDLoc sdl = getCurSDLoc();
3892
3893 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
3894 const Value *Ptr = I.getArgOperand(0);
3895 SDValue Src0 = getValue(I.getArgOperand(3));
3896 SDValue Mask = getValue(I.getArgOperand(2));
3897
3898 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3899 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3900 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3901 if (!Alignment)
3902 Alignment = DAG.getEVTAlignment(VT);
3903
3904 AAMDNodes AAInfo;
3905 I.getAAMetadata(AAInfo);
3906 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3907
3908 SDValue Root = DAG.getRoot();
3909 SDValue Base;
3910 SDValue Index;
3911 const Value *BasePtr = Ptr;
3912 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3913 bool ConstantMemory = false;
3914 if (UniformBase &&
3915 AA->pointsToConstantMemory(MemoryLocation(
3916 BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
3917 AAInfo))) {
3918 // Do not serialize (non-volatile) loads of constant memory with anything.
3919 Root = DAG.getEntryNode();
3920 ConstantMemory = true;
3921 }
3922
3923 MachineMemOperand *MMO =
3924 DAG.getMachineFunction().
3925 getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
3926 MachineMemOperand::MOLoad, VT.getStoreSize(),
3927 Alignment, AAInfo, Ranges);
3928
3929 if (!UniformBase) {
3930 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3931 Index = getValue(Ptr);
3932 }
3933 SDValue Ops[] = { Root, Src0, Mask, Base, Index };
3934 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
3935 Ops, MMO);
3936
3937 SDValue OutChain = Gather.getValue(1);
3938 if (!ConstantMemory)
3939 PendingLoads.push_back(OutChain);
3940 setValue(&I, Gather);
3941}
3942
3943void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
3944 SDLoc dl = getCurSDLoc();
3945 AtomicOrdering SuccessOrder = I.getSuccessOrdering();
3946 AtomicOrdering FailureOrder = I.getFailureOrdering();
3947 SynchronizationScope Scope = I.getSynchScope();
3948
3949 SDValue InChain = getRoot();
3950
3951 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
3952 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
3953 SDValue L = DAG.getAtomicCmpSwap(
3954 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
3955 getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
3956 getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
3957 /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
3958
3959 SDValue OutChain = L.getValue(2);
3960
3961 setValue(&I, L);
3962 DAG.setRoot(OutChain);
3963}
3964
3965void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
3966 SDLoc dl = getCurSDLoc();
3967 ISD::NodeType NT;
3968 switch (I.getOperation()) {
3969 default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3969)
;
3970 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
3971 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
3972 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
3973 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
3974 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
3975 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
3976 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
3977 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
3978 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
3979 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
3980 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
3981 }
3982 AtomicOrdering Order = I.getOrdering();
3983 SynchronizationScope Scope = I.getSynchScope();
3984
3985 SDValue InChain = getRoot();
3986
3987 SDValue L =
3988 DAG.getAtomic(NT, dl,
3989 getValue(I.getValOperand()).getSimpleValueType(),
3990 InChain,
3991 getValue(I.getPointerOperand()),
3992 getValue(I.getValOperand()),
3993 I.getPointerOperand(),
3994 /* Alignment=*/ 0, Order, Scope);
3995
3996 SDValue OutChain = L.getValue(1);
3997
3998 setValue(&I, L);
3999 DAG.setRoot(OutChain);
4000}
4001
4002void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4003 SDLoc dl = getCurSDLoc();
4004 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4005 SDValue Ops[3];
4006 Ops[0] = getRoot();
4007 Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4008 TLI.getPointerTy(DAG.getDataLayout()));
4009 Ops[2] = DAG.getConstant(I.getSynchScope(), dl,
4010 TLI.getPointerTy(DAG.getDataLayout()));
4011 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4012}
4013
4014void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4015 SDLoc dl = getCurSDLoc();
4016 AtomicOrdering Order = I.getOrdering();
4017 SynchronizationScope Scope = I.getSynchScope();
4018
4019 SDValue InChain = getRoot();
4020
4021 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4022 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4023
4024 if (I.getAlignment() < VT.getSizeInBits() / 8)
4025 report_fatal_error("Cannot generate unaligned atomic load");
4026
4027 MachineMemOperand *MMO =
4028 DAG.getMachineFunction().
4029 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4030 MachineMemOperand::MOVolatile |
4031 MachineMemOperand::MOLoad,
4032 VT.getStoreSize(),
4033 I.getAlignment() ? I.getAlignment() :
4034 DAG.getEVTAlignment(VT),
4035 AAMDNodes(), nullptr, Scope, Order);
4036
4037 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4038 SDValue L =
4039 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4040 getValue(I.getPointerOperand()), MMO);
4041
4042 SDValue OutChain = L.getValue(1);
4043
4044 setValue(&I, L);
4045 DAG.setRoot(OutChain);
4046}
4047
4048void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4049 SDLoc dl = getCurSDLoc();
4050
4051 AtomicOrdering Order = I.getOrdering();
4052 SynchronizationScope Scope = I.getSynchScope();
4053
4054 SDValue InChain = getRoot();
4055
4056 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4057 EVT VT =
4058 TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4059
4060 if (I.getAlignment() < VT.getSizeInBits() / 8)
4061 report_fatal_error("Cannot generate unaligned atomic store");
4062
4063 SDValue OutChain =
4064 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4065 InChain,
4066 getValue(I.getPointerOperand()),
4067 getValue(I.getValueOperand()),
4068 I.getPointerOperand(), I.getAlignment(),
4069 Order, Scope);
4070
4071 DAG.setRoot(OutChain);
4072}
4073
4074/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4075/// node.
4076void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4077 unsigned Intrinsic) {
4078 // Ignore the callsite's attributes. A specific call site may be marked with
4079 // readnone, but the lowering code will expect the chain based on the
4080 // definition.
4081 const Function *F = I.getCalledFunction();
4082 bool HasChain = !F->doesNotAccessMemory();
4083 bool OnlyLoad = HasChain && F->onlyReadsMemory();
4084
4085 // Build the operand list.
4086 SmallVector<SDValue, 8> Ops;
4087 if (HasChain) { // If this intrinsic has side-effects, chainify it.
4088 if (OnlyLoad) {
4089 // We don't need to serialize loads against other loads.
4090 Ops.push_back(DAG.getRoot());
4091 } else {
4092 Ops.push_back(getRoot());
4093 }
4094 }
4095
4096 // Info is set by getTgtMemInstrinsic
4097 TargetLowering::IntrinsicInfo Info;
4098 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4099 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
4100
4101 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4102 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4103 Info.opc == ISD::INTRINSIC_W_CHAIN)
4104 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4105 TLI.getPointerTy(DAG.getDataLayout())));
4106
4107 // Add all operands of the call to the operand list.
4108 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4109 SDValue Op = getValue(I.getArgOperand(i));
4110 Ops.push_back(Op);
4111 }
4112
4113 SmallVector<EVT, 4> ValueVTs;
4114 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4115
4116 if (HasChain)
4117 ValueVTs.push_back(MVT::Other);
4118
4119 SDVTList VTs = DAG.getVTList(ValueVTs);
4120
4121 // Create the node.
4122 SDValue Result;
4123 if (IsTgtIntrinsic) {
4124 // This is target intrinsic that touches memory
4125 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
4126 VTs, Ops, Info.memVT,
4127 MachinePointerInfo(Info.ptrVal, Info.offset),
4128 Info.align, Info.vol,
4129 Info.readMem, Info.writeMem, Info.size);
4130 } else if (!HasChain) {
4131 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4132 } else if (!I.getType()->isVoidTy()) {
4133 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4134 } else {
4135 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4136 }
4137
4138 if (HasChain) {
4139 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4140 if (OnlyLoad)
4141 PendingLoads.push_back(Chain);
4142 else
4143 DAG.setRoot(Chain);
4144 }
4145
4146 if (!I.getType()->isVoidTy()) {
4147 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4148 EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4149 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4150 } else
4151 Result = lowerRangeToAssertZExt(DAG, I, Result);
4152
4153 setValue(&I, Result);
4154 }
4155}
4156
4157/// GetSignificand - Get the significand and build it into a floating-point
4158/// number with exponent of 1:
4159///
4160/// Op = (Op & 0x007fffff) | 0x3f800000;
4161///
4162/// where Op is the hexadecimal representation of floating point value.
4163static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4164 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4165 DAG.getConstant(0x007fffff, dl, MVT::i32));
4166 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4167 DAG.getConstant(0x3f800000, dl, MVT::i32));
4168 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4169}
4170
4171/// GetExponent - Get the exponent:
4172///
4173/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4174///
4175/// where Op is the hexadecimal representation of floating point value.
4176static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4177 const TargetLowering &TLI, const SDLoc &dl) {
4178 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4179 DAG.getConstant(0x7f800000, dl, MVT::i32));
4180 SDValue t1 = DAG.getNode(
4181 ISD::SRL, dl, MVT::i32, t0,
4182 DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4183 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4184 DAG.getConstant(127, dl, MVT::i32));
4185 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4186}
4187
4188/// getF32Constant - Get 32-bit floating point constant.
4189static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4190 const SDLoc &dl) {
4191 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4192 MVT::f32);
4193}
4194
4195static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4196 SelectionDAG &DAG) {
4197 // TODO: What fast-math-flags should be set on the floating-point nodes?
4198
4199 // IntegerPartOfX = ((int32_t)(t0);
4200 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4201
4202 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4203 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4204 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4205
4206 // IntegerPartOfX <<= 23;
4207 IntegerPartOfX = DAG.getNode(
4208 ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4209 DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4210 DAG.getDataLayout())));
4211
4212 SDValue TwoToFractionalPartOfX;
4213 if (LimitFloatPrecision <= 6) {
4214 // For floating-point precision of 6:
4215 //
4216 // TwoToFractionalPartOfX =
4217 // 0.997535578f +
4218 // (0.735607626f + 0.252464424f * x) * x;
4219 //
4220 // error 0.0144103317, which is 6 bits
4221 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4222 getF32Constant(DAG, 0x3e814304, dl));
4223 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4224 getF32Constant(DAG, 0x3f3c50c8, dl));
4225 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4226 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4227 getF32Constant(DAG, 0x3f7f5e7e, dl));
4228 } else if (LimitFloatPrecision <= 12) {
4229 // For floating-point precision of 12:
4230 //
4231 // TwoToFractionalPartOfX =
4232 // 0.999892986f +
4233 // (0.696457318f +
4234 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4235 //
4236 // error 0.000107046256, which is 13 to 14 bits
4237 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4238 getF32Constant(DAG, 0x3da235e3, dl));
4239 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4240 getF32Constant(DAG, 0x3e65b8f3, dl));
4241 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4242 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4243 getF32Constant(DAG, 0x3f324b07, dl));
4244 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4245 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4246 getF32Constant(DAG, 0x3f7ff8fd, dl));
4247 } else { // LimitFloatPrecision <= 18
4248 // For floating-point precision of 18:
4249 //
4250 // TwoToFractionalPartOfX =
4251 // 0.999999982f +
4252 // (0.693148872f +
4253 // (0.240227044f +
4254 // (0.554906021e-1f +
4255 // (0.961591928e-2f +
4256 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4257 // error 2.47208000*10^(-7), which is better than 18 bits
4258 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4259 getF32Constant(DAG, 0x3924b03e, dl));
4260 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4261 getF32Constant(DAG, 0x3ab24b87, dl));
4262 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4263 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4264 getF32Constant(DAG, 0x3c1d8c17, dl));
4265 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4266 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4267 getF32Constant(DAG, 0x3d634a1d, dl));
4268 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4269 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4270 getF32Constant(DAG, 0x3e75fe14, dl));
4271 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4272 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4273 getF32Constant(DAG, 0x3f317234, dl));
4274 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4275 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4276 getF32Constant(DAG, 0x3f800000, dl));
4277 }
4278
4279 // Add the exponent into the result in integer domain.
4280 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4281 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4282 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4283}
4284
4285/// expandExp - Lower an exp intrinsic. Handles the special sequences for
4286/// limited-precision mode.
4287static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4288 const TargetLowering &TLI) {
4289 if (Op.getValueType() == MVT::f32 &&
4290 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4291
4292 // Put the exponent in the right bit position for later addition to the
4293 // final result:
4294 //
4295 // #define LOG2OFe 1.4426950f
4296 // t0 = Op * LOG2OFe
4297
4298 // TODO: What fast-math-flags should be set here?
4299 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4300 getF32Constant(DAG, 0x3fb8aa3b, dl));
4301 return getLimitedPrecisionExp2(t0, dl, DAG);
4302 }
4303
4304 // No special expansion.
4305 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4306}
4307
4308/// expandLog - Lower a log intrinsic. Handles the special sequences for
4309/// limited-precision mode.
4310static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4311 const TargetLowering &TLI) {
4312
4313 // TODO: What fast-math-flags should be set on the floating-point nodes?
4314
4315 if (Op.getValueType() == MVT::f32 &&
4316 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4317 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4318
4319 // Scale the exponent by log(2) [0.69314718f].
4320 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4321 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4322 getF32Constant(DAG, 0x3f317218, dl));
4323
4324 // Get the significand and build it into a floating-point number with
4325 // exponent of 1.
4326 SDValue X = GetSignificand(DAG, Op1, dl);
4327
4328 SDValue LogOfMantissa;
4329 if (LimitFloatPrecision <= 6) {
4330 // For floating-point precision of 6:
4331 //
4332 // LogofMantissa =
4333 // -1.1609546f +
4334 // (1.4034025f - 0.23903021f * x) * x;
4335 //
4336 // error 0.0034276066, which is better than 8 bits
4337 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4338 getF32Constant(DAG, 0xbe74c456, dl));
4339 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4340 getF32Constant(DAG, 0x3fb3a2b1, dl));
4341 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4342 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4343 getF32Constant(DAG, 0x3f949a29, dl));
4344 } else if (LimitFloatPrecision <= 12) {
4345 // For floating-point precision of 12:
4346 //
4347 // LogOfMantissa =
4348 // -1.7417939f +
4349 // (2.8212026f +
4350 // (-1.4699568f +
4351 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4352 //
4353 // error 0.000061011436, which is 14 bits
4354 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4355 getF32Constant(DAG, 0xbd67b6d6, dl));
4356 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4357 getF32Constant(DAG, 0x3ee4f4b8, dl));
4358 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4359 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4360 getF32Constant(DAG, 0x3fbc278b, dl));
4361 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4362 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4363 getF32Constant(DAG, 0x40348e95, dl));
4364 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4365 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4366 getF32Constant(DAG, 0x3fdef31a, dl));
4367 } else { // LimitFloatPrecision <= 18
4368 // For floating-point precision of 18:
4369 //
4370 // LogOfMantissa =
4371 // -2.1072184f +
4372 // (4.2372794f +
4373 // (-3.7029485f +
4374 // (2.2781945f +
4375 // (-0.87823314f +
4376 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4377 //
4378 // error 0.0000023660568, which is better than 18 bits
4379 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4380 getF32Constant(DAG, 0xbc91e5ac, dl));
4381 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4382 getF32Constant(DAG, 0x3e4350aa, dl));
4383 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4384 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4385 getF32Constant(DAG, 0x3f60d3e3, dl));
4386 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4387 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4388 getF32Constant(DAG, 0x4011cdf0, dl));
4389 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4390 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4391 getF32Constant(DAG, 0x406cfd1c, dl));
4392 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4393 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4394 getF32Constant(DAG, 0x408797cb, dl));
4395 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4396 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4397 getF32Constant(DAG, 0x4006dcab, dl));
4398 }
4399
4400 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4401 }
4402
4403 // No special expansion.
4404 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4405}
4406
4407/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4408/// limited-precision mode.
4409static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4410 const TargetLowering &TLI) {
4411
4412 // TODO: What fast-math-flags should be set on the floating-point nodes?
4413
4414 if (Op.getValueType() == MVT::f32 &&
4415 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4416 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4417
4418 // Get the exponent.
4419 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4420
4421 // Get the significand and build it into a floating-point number with
4422 // exponent of 1.
4423 SDValue X = GetSignificand(DAG, Op1, dl);
4424
4425 // Different possible minimax approximations of significand in
4426 // floating-point for various degrees of accuracy over [1,2].
4427 SDValue Log2ofMantissa;
4428 if (LimitFloatPrecision <= 6) {
4429 // For floating-point precision of 6:
4430 //
4431 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4432 //
4433 // error 0.0049451742, which is more than 7 bits
4434 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4435 getF32Constant(DAG, 0xbeb08fe0, dl));
4436 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4437 getF32Constant(DAG, 0x40019463, dl));
4438 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4439 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4440 getF32Constant(DAG, 0x3fd6633d, dl));
4441 } else if (LimitFloatPrecision <= 12) {
4442 // For floating-point precision of 12:
4443 //
4444 // Log2ofMantissa =
4445 // -2.51285454f +
4446 // (4.07009056f +
4447 // (-2.12067489f +
4448 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4449 //
4450 // error 0.0000876136000, which is better than 13 bits
4451 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4452 getF32Constant(DAG, 0xbda7262e, dl));
4453 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4454 getF32Constant(DAG, 0x3f25280b, dl));
4455 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4456 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4457 getF32Constant(DAG, 0x4007b923, dl));
4458 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4459 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4460 getF32Constant(DAG, 0x40823e2f, dl));
4461 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4462 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4463 getF32Constant(DAG, 0x4020d29c, dl));
4464 } else { // LimitFloatPrecision <= 18
4465 // For floating-point precision of 18:
4466 //
4467 // Log2ofMantissa =
4468 // -3.0400495f +
4469 // (6.1129976f +
4470 // (-5.3420409f +
4471 // (3.2865683f +
4472 // (-1.2669343f +
4473 // (0.27515199f -
4474 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4475 //
4476 // error 0.0000018516, which is better than 18 bits
4477 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4478 getF32Constant(DAG, 0xbcd2769e, dl));
4479 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4480 getF32Constant(DAG, 0x3e8ce0b9, dl));
4481 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4482 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4483 getF32Constant(DAG, 0x3fa22ae7, dl));
4484 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4485 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4486 getF32Constant(DAG, 0x40525723, dl));
4487 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4488 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4489 getF32Constant(DAG, 0x40aaf200, dl));
4490 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4491 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4492 getF32Constant(DAG, 0x40c39dad, dl));
4493 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4494 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4495 getF32Constant(DAG, 0x4042902c, dl));
4496 }
4497
4498 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4499 }
4500
4501 // No special expansion.
4502 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4503}
4504
4505/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4506/// limited-precision mode.
4507static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4508 const TargetLowering &TLI) {
4509
4510 // TODO: What fast-math-flags should be set on the floating-point nodes?
4511
4512 if (Op.getValueType() == MVT::f32 &&
4513 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4514 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4515
4516 // Scale the exponent by log10(2) [0.30102999f].
4517 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4518 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4519 getF32Constant(DAG, 0x3e9a209a, dl));
4520
4521 // Get the significand and build it into a floating-point number with
4522 // exponent of 1.
4523 SDValue X = GetSignificand(DAG, Op1, dl);
4524
4525 SDValue Log10ofMantissa;
4526 if (LimitFloatPrecision <= 6) {
4527 // For floating-point precision of 6:
4528 //
4529 // Log10ofMantissa =
4530 // -0.50419619f +
4531 // (0.60948995f - 0.10380950f * x) * x;
4532 //
4533 // error 0.0014886165, which is 6 bits
4534 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4535 getF32Constant(DAG, 0xbdd49a13, dl));
4536 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4537 getF32Constant(DAG, 0x3f1c0789, dl));
4538 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4539 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4540 getF32Constant(DAG, 0x3f011300, dl));
4541 } else if (LimitFloatPrecision <= 12) {
4542 // For floating-point precision of 12:
4543 //
4544 // Log10ofMantissa =
4545 // -0.64831180f +
4546 // (0.91751397f +
4547 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4548 //
4549 // error 0.00019228036, which is better than 12 bits
4550 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4551 getF32Constant(DAG, 0x3d431f31, dl));
4552 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4553 getF32Constant(DAG, 0x3ea21fb2, dl));
4554 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4555 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4556 getF32Constant(DAG, 0x3f6ae232, dl));
4557 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4558 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4559 getF32Constant(DAG, 0x3f25f7c3, dl));
4560 } else { // LimitFloatPrecision <= 18
4561 // For floating-point precision of 18:
4562 //
4563 // Log10ofMantissa =
4564 // -0.84299375f +
4565 // (1.5327582f +
4566 // (-1.0688956f +
4567 // (0.49102474f +
4568 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4569 //
4570 // error 0.0000037995730, which is better than 18 bits
4571 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4572 getF32Constant(DAG, 0x3c5d51ce, dl));
4573 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4574 getF32Constant(DAG, 0x3e00685a, dl));
4575 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4576 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4577 getF32Constant(DAG, 0x3efb6798, dl));
4578 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4579 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4580 getF32Constant(DAG, 0x3f88d192, dl));
4581 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4582 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4583 getF32Constant(DAG, 0x3fc4316c, dl));
4584 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4585 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4586 getF32Constant(DAG, 0x3f57ce70, dl));
4587 }
4588
4589 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4590 }
4591
4592 // No special expansion.
4593 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4594}
4595
4596/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4597/// limited-precision mode.
4598static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4599 const TargetLowering &TLI) {
4600 if (Op.getValueType() == MVT::f32 &&
4601 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4602 return getLimitedPrecisionExp2(Op, dl, DAG);
4603
4604 // No special expansion.
4605 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4606}
4607
4608/// visitPow - Lower a pow intrinsic. Handles the special sequences for
4609/// limited-precision mode with x == 10.0f.
4610static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4611 SelectionDAG &DAG, const TargetLowering &TLI) {
4612 bool IsExp10 = false;
4613 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4614 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4615 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4616 APFloat Ten(10.0f);
4617 IsExp10 = LHSC->isExactlyValue(Ten);
4618 }
4619 }
4620
4621 // TODO: What fast-math-flags should be set on the FMUL node?
4622 if (IsExp10) {
4623 // Put the exponent in the right bit position for later addition to the
4624 // final result:
4625 //
4626 // #define LOG2OF10 3.3219281f
4627 // t0 = Op * LOG2OF10;
4628 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4629 getF32Constant(DAG, 0x40549a78, dl));
4630 return getLimitedPrecisionExp2(t0, dl, DAG);
4631 }
4632
4633 // No special expansion.
4634 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4635}
4636
4637
4638/// ExpandPowI - Expand a llvm.powi intrinsic.
4639static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4640 SelectionDAG &DAG) {
4641 // If RHS is a constant, we can expand this out to a multiplication tree,
4642 // otherwise we end up lowering to a call to __powidf2 (for example). When
4643 // optimizing for size, we only want to do this if the expansion would produce
4644 // a small number of multiplies, otherwise we do the full expansion.
4645 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4646 // Get the exponent as a positive value.
4647 unsigned Val = RHSC->getSExtValue();
4648 if ((int)Val < 0) Val = -Val;
4649
4650 // powi(x, 0) -> 1.0
4651 if (Val == 0)
4652 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4653
4654 const Function *F = DAG.getMachineFunction().getFunction();
4655 if (!F->optForSize() ||
4656 // If optimizing for size, don't insert too many multiplies.
4657 // This inserts up to 5 multiplies.
4658 countPopulation(Val) + Log2_32(Val) < 7) {
4659 // We use the simple binary decomposition method to generate the multiply
4660 // sequence. There are more optimal ways to do this (for example,
4661 // powi(x,15) generates one more multiply than it should), but this has
4662 // the benefit of being both really simple and much better than a libcall.
4663 SDValue Res; // Logically starts equal to 1.0
4664 SDValue CurSquare = LHS;
4665 // TODO: Intrinsics should have fast-math-flags that propagate to these
4666 // nodes.
4667 while (Val) {
4668 if (Val & 1) {
4669 if (Res.getNode())
4670 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4671 else
4672 Res = CurSquare; // 1.0*CurSquare.
4673 }
4674
4675 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4676 CurSquare, CurSquare);
4677 Val >>= 1;
4678 }
4679
4680 // If the original was negative, invert the result, producing 1/(x*x*x).
4681 if (RHSC->getSExtValue() < 0)
4682 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4683 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4684 return Res;
4685 }
4686 }
4687
4688 // Otherwise, expand to a libcall.
4689 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4690}
4691
4692// getUnderlyingArgReg - Find underlying register used for a truncated or
4693// bitcasted argument.
4694static unsigned getUnderlyingArgReg(const SDValue &N) {
4695 switch (N.getOpcode()) {
4696 case ISD::CopyFromReg:
4697 return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4698 case ISD::BITCAST:
4699 case ISD::AssertZext:
4700 case ISD::AssertSext:
4701 case ISD::TRUNCATE:
4702 return getUnderlyingArgReg(N.getOperand(0));
4703 default:
4704 return 0;
4705 }
4706}
4707
4708/// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
4709/// argument, create the corresponding DBG_VALUE machine instruction for it now.
4710/// At the end of instruction selection, they will be inserted to the entry BB.
4711bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4712 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4713 DILocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
4714 const Argument *Arg = dyn_cast<Argument>(V);
4715 if (!Arg)
4716 return false;
4717
4718 MachineFunction &MF = DAG.getMachineFunction();
4719 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4720
4721 // Ignore inlined function arguments here.
4722 //
4723 // FIXME: Should we be checking DL->inlinedAt() to determine this?
4724 if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
4725 return false;
4726
4727 Optional<MachineOperand> Op;
4728 // Some arguments' frame index is recorded during argument lowering.
4729 if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
4730 Op = MachineOperand::CreateFI(FI);
4731
4732 if (!Op && N.getNode()) {
4733 unsigned Reg = getUnderlyingArgReg(N);
4734 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4735 MachineRegisterInfo &RegInfo = MF.getRegInfo();
4736 unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4737 if (PR)
4738 Reg = PR;
4739 }
4740 if (Reg)
4741 Op = MachineOperand::CreateReg(Reg, false);
4742 }
4743
4744 if (!Op) {
4745 // Check if ValueMap has reg number.
4746 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4747 if (VMI != FuncInfo.ValueMap.end())
4748 Op = MachineOperand::CreateReg(VMI->second, false);
4749 }
4750
4751 if (!Op && N.getNode())
4752 // Check if frame index is available.
4753 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4754 if (FrameIndexSDNode *FINode =
4755 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4756 Op = MachineOperand::CreateFI(FINode->getIndex());
4757
4758 if (!Op)
4759 return false;
4760
4761 assert(Variable->isValidLocationForIntrinsic(DL) &&((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4762, __PRETTY_FUNCTION__))
4762 "Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4762, __PRETTY_FUNCTION__))
;
4763 if (Op->isReg())
4764 FuncInfo.ArgDbgValues.push_back(
4765 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4766 Op->getReg(), Offset, Variable, Expr));
4767 else
4768 FuncInfo.ArgDbgValues.push_back(
4769 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
4770 .add(*Op)
4771 .addImm(Offset)
4772 .addMetadata(Variable)
4773 .addMetadata(Expr));
4774
4775 return true;
4776}
4777
4778/// Return the appropriate SDDbgValue based on N.
4779SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4780 DILocalVariable *Variable,
4781 DIExpression *Expr, int64_t Offset,
4782 const DebugLoc &dl,
4783 unsigned DbgSDNodeOrder) {
4784 SDDbgValue *SDV;
4785 auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode());
4786 if (FISDN && Expr->startsWithDeref()) {
4787 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4788 // stack slot locations as such instead of as indirectly addressed
4789 // locations.
4790 ArrayRef<uint64_t> TrailingElements(Expr->elements_begin() + 1,
4791 Expr->elements_end());
4792 DIExpression *DerefedDIExpr =
4793 DIExpression::get(*DAG.getContext(), TrailingElements);
4794 int FI = FISDN->getIndex();
4795 SDV = DAG.getFrameIndexDbgValue(Variable, DerefedDIExpr, FI, 0, dl,
4796 DbgSDNodeOrder);
4797 } else {
4798 SDV = DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), false,
4799 Offset, dl, DbgSDNodeOrder);
4800 }
4801 return SDV;
4802}
4803
4804// VisualStudio defines setjmp as _setjmp
4805#if defined(_MSC_VER) && defined(setjmp) && \
4806 !defined(setjmp_undefined_for_msvc)
4807# pragma push_macro("setjmp")
4808# undef setjmp
4809# define setjmp_undefined_for_msvc
4810#endif
4811
4812/// visitIntrinsicCall - Lower the call to the specified intrinsic function. If
4813/// we want to emit this as a call to a named external function, return the name
4814/// otherwise lower it and return null.
4815const char *
4816SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4817 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4818 SDLoc sdl = getCurSDLoc();
4819 DebugLoc dl = getCurDebugLoc();
4820 SDValue Res;
4821
4822 switch (Intrinsic) {
4823 default:
4824 // By default, turn this into a target intrinsic node.
4825 visitTargetIntrinsic(I, Intrinsic);
4826 return nullptr;
4827 case Intrinsic::vastart: visitVAStart(I); return nullptr;
4828 case Intrinsic::vaend: visitVAEnd(I); return nullptr;
4829 case Intrinsic::vacopy: visitVACopy(I); return nullptr;
4830 case Intrinsic::returnaddress:
4831 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
4832 TLI.getPointerTy(DAG.getDataLayout()),
4833 getValue(I.getArgOperand(0))));
4834 return nullptr;
4835 case Intrinsic::addressofreturnaddress:
4836 setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
4837 TLI.getPointerTy(DAG.getDataLayout())));
4838 return nullptr;
4839 case Intrinsic::frameaddress:
4840 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
4841 TLI.getPointerTy(DAG.getDataLayout()),
4842 getValue(I.getArgOperand(0))));
4843 return nullptr;
4844 case Intrinsic::read_register: {
4845 Value *Reg = I.getArgOperand(0);
4846 SDValue Chain = getRoot();
4847 SDValue RegName =
4848 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4849 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4850 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
4851 DAG.getVTList(VT, MVT::Other), Chain, RegName);
4852 setValue(&I, Res);
4853 DAG.setRoot(Res.getValue(1));
4854 return nullptr;
4855 }
4856 case Intrinsic::write_register: {
4857 Value *Reg = I.getArgOperand(0);
4858 Value *RegValue = I.getArgOperand(1);
4859 SDValue Chain = getRoot();
4860 SDValue RegName =
4861 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4862 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
4863 RegName, getValue(RegValue)));
4864 return nullptr;
4865 }
4866 case Intrinsic::setjmp:
4867 return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
4868 case Intrinsic::longjmp:
4869 return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
4870 case Intrinsic::memcpy: {
4871 SDValue Op1 = getValue(I.getArgOperand(0));
4872 SDValue Op2 = getValue(I.getArgOperand(1));
4873 SDValue Op3 = getValue(I.getArgOperand(2));
4874 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4875 if (!Align)
4876 Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
4877 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4878 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4879 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4880 false, isTC,
4881 MachinePointerInfo(I.getArgOperand(0)),
4882 MachinePointerInfo(I.getArgOperand(1)));
4883 updateDAGForMaybeTailCall(MC);
4884 return nullptr;
4885 }
4886 case Intrinsic::memset: {
4887 SDValue Op1 = getValue(I.getArgOperand(0));
4888 SDValue Op2 = getValue(I.getArgOperand(1));
4889 SDValue Op3 = getValue(I.getArgOperand(2));
4890 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4891 if (!Align)
4892 Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
4893 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4894 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4895 SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4896 isTC, MachinePointerInfo(I.getArgOperand(0)));
4897 updateDAGForMaybeTailCall(MS);
4898 return nullptr;
4899 }
4900 case Intrinsic::memmove: {
4901 SDValue Op1 = getValue(I.getArgOperand(0));
4902 SDValue Op2 = getValue(I.getArgOperand(1));
4903 SDValue Op3 = getValue(I.getArgOperand(2));
4904 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4905 if (!Align)
4906 Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
4907 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4908 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4909 SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4910 isTC, MachinePointerInfo(I.getArgOperand(0)),
4911 MachinePointerInfo(I.getArgOperand(1)));
4912 updateDAGForMaybeTailCall(MM);
4913 return nullptr;
4914 }
4915 case Intrinsic::memcpy_element_atomic: {
4916 SDValue Dst = getValue(I.getArgOperand(0));
4917 SDValue Src = getValue(I.getArgOperand(1));
4918 SDValue NumElements = getValue(I.getArgOperand(2));
4919 SDValue ElementSize = getValue(I.getArgOperand(3));
4920
4921 // Emit a library call.
4922 TargetLowering::ArgListTy Args;
4923 TargetLowering::ArgListEntry Entry;
4924 Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
4925 Entry.Node = Dst;
4926 Args.push_back(Entry);
4927
4928 Entry.Node = Src;
4929 Args.push_back(Entry);
4930
4931 Entry.Ty = I.getArgOperand(2)->getType();
4932 Entry.Node = NumElements;
4933 Args.push_back(Entry);
4934
4935 Entry.Ty = Type::getInt32Ty(*DAG.getContext());
4936 Entry.Node = ElementSize;
4937 Args.push_back(Entry);
4938
4939 uint64_t ElementSizeConstant =
4940 cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4941 RTLIB::Libcall LibraryCall =
4942 RTLIB::getMEMCPY_ELEMENT_ATOMIC(ElementSizeConstant);
4943 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
4944 report_fatal_error("Unsupported element size");
4945
4946 TargetLowering::CallLoweringInfo CLI(DAG);
4947 CLI.setDebugLoc(sdl)
4948 .setChain(getRoot())
4949 .setCallee(TLI.getLibcallCallingConv(LibraryCall),
4950 Type::getVoidTy(*DAG.getContext()),
4951 DAG.getExternalSymbol(
4952 TLI.getLibcallName(LibraryCall),
4953 TLI.getPointerTy(DAG.getDataLayout())),
4954 std::move(Args));
4955
4956 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4957 DAG.setRoot(CallResult.second);
4958 return nullptr;
4959 }
4960 case Intrinsic::dbg_declare: {
4961 const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
4962 DILocalVariable *Variable = DI.getVariable();
4963 DIExpression *Expression = DI.getExpression();
4964 const Value *Address = DI.getAddress();
4965 assert(Variable && "Missing variable")((Variable && "Missing variable") ? static_cast<void
> (0) : __assert_fail ("Variable && \"Missing variable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4965, __PRETTY_FUNCTION__))
;
4966 if (!Address) {
4967 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
4968 return nullptr;
4969 }
4970
4971 // Check if address has undef value.
4972 if (isa<UndefValue>(Address) ||
4973 (Address->use_empty() && !isa<Argument>(Address))) {
4974 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
4975 return nullptr;
4976 }
4977
4978 SDValue &N = NodeMap[Address];
4979 if (!N.getNode() && isa<Argument>(Address))
4980 // Check unused arguments map.
4981 N = UnusedArgNodeMap[Address];
4982 SDDbgValue *SDV;
4983 if (N.getNode()) {
4984 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
4985 Address = BCI->getOperand(0);
4986 // Parameters are handled specially.
4987 bool isParameter = Variable->isParameter() || isa<Argument>(Address);
4988 auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
4989 if (isParameter && FINode) {
4990 // Byval parameter. We have a frame index at this point.
4991 SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
4992 FINode->getIndex(), 0, dl, SDNodeOrder);
4993 } else if (isa<Argument>(Address)) {
4994 // Address is an argument, so try to emit its dbg value using
4995 // virtual register info from the FuncInfo.ValueMap.
4996 EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
4997 N);
4998 return nullptr;
4999 } else {
5000 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
5001 true, 0, dl, SDNodeOrder);
5002 }
5003 DAG.AddDbgValue(SDV, N.getNode(), isParameter);
5004 } else {
5005 // If Address is an argument then try to emit its dbg value using
5006 // virtual register info from the FuncInfo.ValueMap.
5007 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
5008 N)) {
5009 // If variable is pinned by a alloca in dominating bb then
5010 // use StaticAllocaMap.
5011 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
5012 if (AI->getParent() != DI.getParent()) {
5013 DenseMap<const AllocaInst*, int>::iterator SI =
5014 FuncInfo.StaticAllocaMap.find(AI);
5015 if (SI != FuncInfo.StaticAllocaMap.end()) {
5016 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
5017 0, dl, SDNodeOrder);
5018 DAG.AddDbgValue(SDV, nullptr, false);
5019 return nullptr;
5020 }
5021 }
5022 }
5023 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
5024 }
5025 }
5026 return nullptr;
5027 }
5028 case Intrinsic::dbg_value: {
5029 const DbgValueInst &DI = cast<DbgValueInst>(I);
5030 assert(DI.getVariable() && "Missing variable")((DI.getVariable() && "Missing variable") ? static_cast
<void> (0) : __assert_fail ("DI.getVariable() && \"Missing variable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5030, __PRETTY_FUNCTION__))
;
5031
5032 DILocalVariable *Variable = DI.getVariable();
5033 DIExpression *Expression = DI.getExpression();
5034 uint64_t Offset = DI.getOffset();
5035 const Value *V = DI.getValue();
5036 if (!V)
5037 return nullptr;
5038
5039 SDDbgValue *SDV;
5040 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
5041 SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
5042 SDNodeOrder);
5043 DAG.AddDbgValue(SDV, nullptr, false);
5044 } else {
5045 // Do not use getValue() in here; we don't want to generate code at
5046 // this point if it hasn't been done yet.
5047 SDValue N = NodeMap[V];
5048 if (!N.getNode() && isa<Argument>(V))
5049 // Check unused arguments map.
5050 N = UnusedArgNodeMap[V];
5051 if (N.getNode()) {
5052 if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset,
5053 false, N)) {
5054 SDV = getDbgValue(N, Variable, Expression, Offset, dl, SDNodeOrder);
5055 DAG.AddDbgValue(SDV, N.getNode(), false);
5056 }
5057 } else if (!V->use_empty() ) {
5058 // Do not call getValue(V) yet, as we don't want to generate code.
5059 // Remember it for later.
5060 DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
5061 DanglingDebugInfoMap[V] = DDI;
5062 } else {
5063 // We may expand this to cover more cases. One case where we have no
5064 // data available is an unreferenced parameter.
5065 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
5066 }
5067 }
5068
5069 // Build a debug info table entry.
5070 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
5071 V = BCI->getOperand(0);
5072 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
5073 // Don't handle byval struct arguments or VLAs, for example.
5074 if (!AI) {
5075 DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug location info for:\n "
<< DI << "\n"; } } while (false)
;
5076 DEBUG(dbgs() << " Last seen at:\n " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
V << "\n"; } } while (false)
;
5077 return nullptr;
5078 }
5079 DenseMap<const AllocaInst*, int>::iterator SI =
5080 FuncInfo.StaticAllocaMap.find(AI);
5081 if (SI == FuncInfo.StaticAllocaMap.end())
5082 return nullptr; // VLAs.
5083 return nullptr;
5084 }
5085
5086 case Intrinsic::eh_typeid_for: {
5087 // Find the type id for the given typeinfo.
5088 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5089 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5090 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5091 setValue(&I, Res);
5092 return nullptr;
5093 }
5094
5095 case Intrinsic::eh_return_i32:
5096 case Intrinsic::eh_return_i64:
5097 DAG.getMachineFunction().setCallsEHReturn(true);
5098 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5099 MVT::Other,
5100 getControlRoot(),
5101 getValue(I.getArgOperand(0)),
5102 getValue(I.getArgOperand(1))));
5103 return nullptr;
5104 case Intrinsic::eh_unwind_init:
5105 DAG.getMachineFunction().setCallsUnwindInit(true);
5106 return nullptr;
5107 case Intrinsic::eh_dwarf_cfa: {
5108 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5109 TLI.getPointerTy(DAG.getDataLayout()),
5110 getValue(I.getArgOperand(0))));
5111 return nullptr;
5112 }
5113 case Intrinsic::eh_sjlj_callsite: {
5114 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5115 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5116 assert(CI && "Non-constant call site value in eh.sjlj.callsite!")((CI && "Non-constant call site value in eh.sjlj.callsite!"
) ? static_cast<void> (0) : __assert_fail ("CI && \"Non-constant call site value in eh.sjlj.callsite!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5116, __PRETTY_FUNCTION__))
;
5117 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!")((MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"
) ? static_cast<void> (0) : __assert_fail ("MMI.getCurrentCallSite() == 0 && \"Overlapping call sites!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5117, __PRETTY_FUNCTION__))
;
5118
5119 MMI.setCurrentCallSite(CI->getZExtValue());
5120 return nullptr;
5121 }
5122 case Intrinsic::eh_sjlj_functioncontext: {
5123 // Get and store the index of the function context.
5124 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5125 AllocaInst *FnCtx =
5126 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5127 int FI = FuncInfo.StaticAllocaMap[FnCtx];
5128 MFI.setFunctionContextIndex(FI);
5129 return nullptr;
5130 }
5131 case Intrinsic::eh_sjlj_setjmp: {
5132 SDValue Ops[2];
5133 Ops[0] = getRoot();
5134 Ops[1] = getValue(I.getArgOperand(0));
5135 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5136 DAG.getVTList(MVT::i32, MVT::Other), Ops);
5137 setValue(&I, Op.getValue(0));
5138 DAG.setRoot(Op.getValue(1));
5139 return nullptr;
5140 }
5141 case Intrinsic::eh_sjlj_longjmp: {
5142 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5143 getRoot(), getValue(I.getArgOperand(0))));
5144 return nullptr;
5145 }
5146 case Intrinsic::eh_sjlj_setup_dispatch: {
5147 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5148 getRoot()));
5149 return nullptr;
5150 }
5151
5152 case Intrinsic::masked_gather:
5153 visitMaskedGather(I);
5154 return nullptr;
5155 case Intrinsic::masked_load:
5156 visitMaskedLoad(I);
5157 return nullptr;
5158 case Intrinsic::masked_scatter:
5159 visitMaskedScatter(I);
5160 return nullptr;
5161 case Intrinsic::masked_store:
5162 visitMaskedStore(I);
5163 return nullptr;
5164 case Intrinsic::masked_expandload:
5165 visitMaskedLoad(I, true /* IsExpanding */);
5166 return nullptr;
5167 case Intrinsic::masked_compressstore:
5168 visitMaskedStore(I, true /* IsCompressing */);
5169 return nullptr;
5170 case Intrinsic::x86_mmx_pslli_w:
5171 case Intrinsic::x86_mmx_pslli_d:
5172 case Intrinsic::x86_mmx_pslli_q:
5173 case Intrinsic::x86_mmx_psrli_w:
5174 case Intrinsic::x86_mmx_psrli_d:
5175 case Intrinsic::x86_mmx_psrli_q:
5176 case Intrinsic::x86_mmx_psrai_w:
5177 case Intrinsic::x86_mmx_psrai_d: {
5178 SDValue ShAmt = getValue(I.getArgOperand(1));
5179 if (isa<ConstantSDNode>(ShAmt)) {
5180 visitTargetIntrinsic(I, Intrinsic);
5181 return nullptr;
5182 }
5183 unsigned NewIntrinsic = 0;
5184 EVT ShAmtVT = MVT::v2i32;
5185 switch (Intrinsic) {
5186 case Intrinsic::x86_mmx_pslli_w:
5187 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5188 break;
5189 case Intrinsic::x86_mmx_pslli_d:
5190 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5191 break;
5192 case Intrinsic::x86_mmx_pslli_q:
5193 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5194 break;
5195 case Intrinsic::x86_mmx_psrli_w:
5196 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5197 break;
5198 case Intrinsic::x86_mmx_psrli_d:
5199 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5200 break;
5201 case Intrinsic::x86_mmx_psrli_q:
5202 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5203 break;
5204 case Intrinsic::x86_mmx_psrai_w:
5205 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5206 break;
5207 case Intrinsic::x86_mmx_psrai_d:
5208 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5209 break;
5210 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5210)
; // Can't reach here.
5211 }
5212
5213 // The vector shift intrinsics with scalars uses 32b shift amounts but
5214 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5215 // to be zero.
5216 // We must do this early because v2i32 is not a legal type.
5217 SDValue ShOps[2];
5218 ShOps[0] = ShAmt;
5219 ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5220 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
5221 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5222 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5223 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5224 DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5225 getValue(I.getArgOperand(0)), ShAmt);
5226 setValue(&I, Res);
5227 return nullptr;
5228 }
5229 case Intrinsic::powi:
5230 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5231 getValue(I.getArgOperand(1)), DAG));
5232 return nullptr;
5233 case Intrinsic::log:
5234 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5235 return nullptr;
5236 case Intrinsic::log2:
5237 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5238 return nullptr;
5239 case Intrinsic::log10:
5240 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5241 return nullptr;
5242 case Intrinsic::exp:
5243 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5244 return nullptr;
5245 case Intrinsic::exp2:
5246 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5247 return nullptr;
5248 case Intrinsic::pow:
5249 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5250 getValue(I.getArgOperand(1)), DAG, TLI));
5251 return nullptr;
5252 case Intrinsic::sqrt:
5253 case Intrinsic::fabs:
5254 case Intrinsic::sin:
5255 case Intrinsic::cos:
5256 case Intrinsic::floor:
5257 case Intrinsic::ceil:
5258 case Intrinsic::trunc:
5259 case Intrinsic::rint:
5260 case Intrinsic::nearbyint:
5261 case Intrinsic::round:
5262 case Intrinsic::canonicalize: {
5263 unsigned Opcode;
5264 switch (Intrinsic) {
5265 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5265)
; // Can't reach here.
5266 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
5267 case Intrinsic::fabs: Opcode = ISD::FABS; break;
5268 case Intrinsic::sin: Opcode = ISD::FSIN; break;
5269 case Intrinsic::cos: Opcode = ISD::FCOS; break;
5270 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
5271 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
5272 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
5273 case Intrinsic::rint: Opcode = ISD::FRINT; break;
5274 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5275 case Intrinsic::round: Opcode = ISD::FROUND; break;
5276 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5277 }
5278
5279 setValue(&I, DAG.getNode(Opcode, sdl,
5280 getValue(I.getArgOperand(0)).getValueType(),
5281 getValue(I.getArgOperand(0))));
5282 return nullptr;
5283 }
5284 case Intrinsic::minnum: {
5285 auto VT = getValue(I.getArgOperand(0)).getValueType();
5286 unsigned Opc =
5287 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT)
5288 ? ISD::FMINNAN
5289 : ISD::FMINNUM;
5290 setValue(&I, DAG.getNode(Opc, sdl, VT,
5291 getValue(I.getArgOperand(0)),
5292 getValue(I.getArgOperand(1))));
5293 return nullptr;
5294 }
5295 case Intrinsic::maxnum: {
5296 auto VT = getValue(I.getArgOperand(0)).getValueType();
5297 unsigned Opc =
5298 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT)
5299 ? ISD::FMAXNAN
5300 : ISD::FMAXNUM;
5301 setValue(&I, DAG.getNode(Opc, sdl, VT,
5302 getValue(I.getArgOperand(0)),
5303 getValue(I.getArgOperand(1))));
5304 return nullptr;
5305 }
5306 case Intrinsic::copysign:
5307 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5308 getValue(I.getArgOperand(0)).getValueType(),
5309 getValue(I.getArgOperand(0)),
5310 getValue(I.getArgOperand(1))));
5311 return nullptr;
5312 case Intrinsic::fma:
5313 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5314 getValue(I.getArgOperand(0)).getValueType(),
5315 getValue(I.getArgOperand(0)),
5316 getValue(I.getArgOperand(1)),
5317 getValue(I.getArgOperand(2))));
5318 return nullptr;
5319 case Intrinsic::experimental_constrained_fadd:
5320 case Intrinsic::experimental_constrained_fsub:
5321 case Intrinsic::experimental_constrained_fmul:
5322 case Intrinsic::experimental_constrained_fdiv:
5323 case Intrinsic::experimental_constrained_frem:
5324 visitConstrainedFPIntrinsic(I, Intrinsic);
5325 return nullptr;
5326 case Intrinsic::fmuladd: {
5327 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5328 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5329 TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5330 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5331 getValue(I.getArgOperand(0)).getValueType(),
5332 getValue(I.getArgOperand(0)),
5333 getValue(I.getArgOperand(1)),
5334 getValue(I.getArgOperand(2))));
5335 } else {
5336 // TODO: Intrinsic calls should have fast-math-flags.
5337 SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5338 getValue(I.getArgOperand(0)).getValueType(),
5339 getValue(I.getArgOperand(0)),
5340 getValue(I.getArgOperand(1)));
5341 SDValue Add = DAG.getNode(ISD::FADD, sdl,
5342 getValue(I.getArgOperand(0)).getValueType(),
5343 Mul,
5344 getValue(I.getArgOperand(2)));
5345 setValue(&I, Add);
5346 }
5347 return nullptr;
5348 }
5349 case Intrinsic::convert_to_fp16:
5350 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5351 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5352 getValue(I.getArgOperand(0)),
5353 DAG.getTargetConstant(0, sdl,
5354 MVT::i32))));
5355 return nullptr;
5356 case Intrinsic::convert_from_fp16:
5357 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5358 TLI.getValueType(DAG.getDataLayout(), I.getType()),
5359 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5360 getValue(I.getArgOperand(0)))));
5361 return nullptr;
5362 case Intrinsic::pcmarker: {
5363 SDValue Tmp = getValue(I.getArgOperand(0));
5364 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5365 return nullptr;
5366 }
5367 case Intrinsic::readcyclecounter: {
5368 SDValue Op = getRoot();
5369 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5370 DAG.getVTList(MVT::i64, MVT::Other), Op);
5371 setValue(&I, Res);
5372 DAG.setRoot(Res.getValue(1));
5373 return nullptr;
5374 }
5375 case Intrinsic::bitreverse:
5376 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5377 getValue(I.getArgOperand(0)).getValueType(),
5378 getValue(I.getArgOperand(0))));
5379 return nullptr;
5380 case Intrinsic::bswap:
5381 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5382 getValue(I.getArgOperand(0)).getValueType(),
5383 getValue(I.getArgOperand(0))));
5384 return nullptr;
5385 case Intrinsic::cttz: {
5386 SDValue Arg = getValue(I.getArgOperand(0));
5387 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5388 EVT Ty = Arg.getValueType();
5389 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5390 sdl, Ty, Arg));
5391 return nullptr;
5392 }
5393 case Intrinsic::ctlz: {
5394 SDValue Arg = getValue(I.getArgOperand(0));
5395 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5396 EVT Ty = Arg.getValueType();
5397 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5398 sdl, Ty, Arg));
5399 return nullptr;
5400 }
5401 case Intrinsic::ctpop: {
5402 SDValue Arg = getValue(I.getArgOperand(0));
5403 EVT Ty = Arg.getValueType();
5404 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5405 return nullptr;
5406 }
5407 case Intrinsic::stacksave: {
5408 SDValue Op = getRoot();
5409 Res = DAG.getNode(
5410 ISD::STACKSAVE, sdl,
5411 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5412 setValue(&I, Res);
5413 DAG.setRoot(Res.getValue(1));
5414 return nullptr;
5415 }
5416 case Intrinsic::stackrestore: {
5417 Res = getValue(I.getArgOperand(0));
5418 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5419 return nullptr;
5420 }
5421 case Intrinsic::get_dynamic_area_offset: {
5422 SDValue Op = getRoot();
5423 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5424 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5425 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5426 // target.
5427 if (PtrTy != ResTy)
5428 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5429 " intrinsic!");
5430 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5431 Op);
5432 DAG.setRoot(Op);
5433 setValue(&I, Res);
5434 return nullptr;
5435 }
5436 case Intrinsic::stackguard: {
5437 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5438 MachineFunction &MF = DAG.getMachineFunction();
5439 const Module &M = *MF.getFunction()->getParent();
5440 SDValue Chain = getRoot();
5441 if (TLI.useLoadStackGuardNode()) {
5442 Res = getLoadStackGuard(DAG, sdl, Chain);
5443 } else {
5444 const Value *Global = TLI.getSDagStackGuard(M);
5445 unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5446 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5447 MachinePointerInfo(Global, 0), Align,
5448 MachineMemOperand::MOVolatile);
5449 }
5450 DAG.setRoot(Chain);
5451 setValue(&I, Res);
5452 return nullptr;
5453 }
5454 case Intrinsic::stackprotector: {
5455 // Emit code into the DAG to store the stack guard onto the stack.
5456 MachineFunction &MF = DAG.getMachineFunction();
5457 MachineFrameInfo &MFI = MF.getFrameInfo();
5458 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5459 SDValue Src, Chain = getRoot();
5460
5461 if (TLI.useLoadStackGuardNode())
5462 Src = getLoadStackGuard(DAG, sdl, Chain);
5463 else
5464 Src = getValue(I.getArgOperand(0)); // The guard's value.
5465
5466 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5467
5468 int FI = FuncInfo.StaticAllocaMap[Slot];
5469 MFI.setStackProtectorIndex(FI);
5470
5471 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5472
5473 // Store the stack protector onto the stack.
5474 Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5475 DAG.getMachineFunction(), FI),
5476 /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5477 setValue(&I, Res);
5478 DAG.setRoot(Res);
5479 return nullptr;
5480 }
5481 case Intrinsic::objectsize: {
5482 // If we don't know by now, we're never going to know.
5483 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5484
5485 assert(CI && "Non-constant type in __builtin_object_size?")((CI && "Non-constant type in __builtin_object_size?"
) ? static_cast<void> (0) : __assert_fail ("CI && \"Non-constant type in __builtin_object_size?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5485, __PRETTY_FUNCTION__))
;
5486
5487 SDValue Arg = getValue(I.getCalledValue());
5488 EVT Ty = Arg.getValueType();
5489
5490 if (CI->isZero())
5491 Res = DAG.getConstant(-1ULL, sdl, Ty);
5492 else
5493 Res = DAG.getConstant(0, sdl, Ty);
5494
5495 setValue(&I, Res);
5496 return nullptr;
5497 }
5498 case Intrinsic::annotation:
5499 case Intrinsic::ptr_annotation:
5500 case Intrinsic::invariant_group_barrier:
5501 // Drop the intrinsic, but forward the value
5502 setValue(&I, getValue(I.getOperand(0)));
5503 return nullptr;
5504 case Intrinsic::assume:
5505 case Intrinsic::var_annotation:
5506 // Discard annotate attributes and assumptions
5507 return nullptr;
5508
5509 case Intrinsic::init_trampoline: {
5510 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5511
5512 SDValue Ops[6];
5513 Ops[0] = getRoot();
5514 Ops[1] = getValue(I.getArgOperand(0));
5515 Ops[2] = getValue(I.getArgOperand(1));
5516 Ops[3] = getValue(I.getArgOperand(2));
5517 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5518 Ops[5] = DAG.getSrcValue(F);
5519
5520 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5521
5522 DAG.setRoot(Res);
5523 return nullptr;
5524 }
5525 case Intrinsic::adjust_trampoline: {
5526 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5527 TLI.getPointerTy(DAG.getDataLayout()),
5528 getValue(I.getArgOperand(0))));
5529 return nullptr;
5530 }
5531 case Intrinsic::gcroot: {
5532 MachineFunction &MF = DAG.getMachineFunction();
5533 const Function *F = MF.getFunction();
5534 (void)F;
5535 assert(F->hasGC() &&((F->hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? static_cast<void> (0) : __assert_fail ("F->hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5536, __PRETTY_FUNCTION__))
5536 "only valid in functions with gc specified, enforced by Verifier")((F->hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? static_cast<void> (0) : __assert_fail ("F->hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5536, __PRETTY_FUNCTION__))
;
5537 assert(GFI && "implied by previous")((GFI && "implied by previous") ? static_cast<void
> (0) : __assert_fail ("GFI && \"implied by previous\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5537, __PRETTY_FUNCTION__))
;
5538 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5539 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5540
5541 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5542 GFI->addStackRoot(FI->getIndex(), TypeMap);
5543 return nullptr;
5544 }
5545 case Intrinsic::gcread:
5546 case Intrinsic::gcwrite:
5547 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!")::llvm::llvm_unreachable_internal("GC failed to lower gcread/gcwrite intrinsics!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5547)
;
5548 case Intrinsic::flt_rounds:
5549 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
5550 return nullptr;
5551
5552 case Intrinsic::expect: {
5553 // Just replace __builtin_expect(exp, c) with EXP.
5554 setValue(&I, getValue(I.getArgOperand(0)));
5555 return nullptr;
5556 }
5557
5558 case Intrinsic::debugtrap:
5559 case Intrinsic::trap: {
5560 StringRef TrapFuncName =
5561 I.getAttributes()
5562 .getAttribute(AttributeSet::FunctionIndex, "trap-func-name")
5563 .getValueAsString();
5564 if (TrapFuncName.empty()) {
5565 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
5566 ISD::TRAP : ISD::DEBUGTRAP;
5567 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
5568 return nullptr;
5569 }
5570 TargetLowering::ArgListTy Args;
5571
5572 TargetLowering::CallLoweringInfo CLI(DAG);
5573 CLI.setDebugLoc(sdl).setChain(getRoot()).setCallee(
5574 CallingConv::C, I.getType(),
5575 DAG.getExternalSymbol(TrapFuncName.data(),
5576 TLI.getPointerTy(DAG.getDataLayout())),
5577 std::move(Args));
5578
5579 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5580 DAG.setRoot(Result.second);
5581 return nullptr;
5582 }
5583
5584 case Intrinsic::uadd_with_overflow:
5585 case Intrinsic::sadd_with_overflow:
5586 case Intrinsic::usub_with_overflow:
5587 case Intrinsic::ssub_with_overflow:
5588 case Intrinsic::umul_with_overflow:
5589 case Intrinsic::smul_with_overflow: {
5590 ISD::NodeType Op;
5591 switch (Intrinsic) {
5592 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5592)
; // Can't reach here.
5593 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
5594 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
5595 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
5596 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
5597 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
5598 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
5599 }
5600 SDValue Op1 = getValue(I.getArgOperand(0));
5601 SDValue Op2 = getValue(I.getArgOperand(1));
5602
5603 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
5604 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
5605 return nullptr;
5606 }
5607 case Intrinsic::prefetch: {
5608 SDValue Ops[5];
5609 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
5610 Ops[0] = getRoot();
5611 Ops[1] = getValue(I.getArgOperand(0));
5612 Ops[2] = getValue(I.getArgOperand(1));
5613 Ops[3] = getValue(I.getArgOperand(2));
5614 Ops[4] = getValue(I.getArgOperand(3));
5615 DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
5616 DAG.getVTList(MVT::Other), Ops,
5617 EVT::getIntegerVT(*Context, 8),
5618 MachinePointerInfo(I.getArgOperand(0)),
5619 0, /* align */
5620 false, /* volatile */
5621 rw==0, /* read */
5622 rw==1)); /* write */
5623 return nullptr;
5624 }
5625 case Intrinsic::lifetime_start:
5626 case Intrinsic::lifetime_end: {
5627 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
5628 // Stack coloring is not enabled in O0, discard region information.
5629 if (TM.getOptLevel() == CodeGenOpt::None)
5630 return nullptr;
5631
5632 SmallVector<Value *, 4> Allocas;
5633 GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
5634
5635 for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
5636 E = Allocas.end(); Object != E; ++Object) {
5637 AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
5638
5639 // Could not find an Alloca.
5640 if (!LifetimeObject)
5641 continue;
5642
5643 // First check that the Alloca is static, otherwise it won't have a
5644 // valid frame index.
5645 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
5646 if (SI == FuncInfo.StaticAllocaMap.end())
5647 return nullptr;
5648
5649 int FI = SI->second;
5650
5651 SDValue Ops[2];
5652 Ops[0] = getRoot();
5653 Ops[1] =
5654 DAG.getFrameIndex(FI, TLI.getPointerTy(DAG.getDataLayout()), true);
5655 unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
5656
5657 Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
5658 DAG.setRoot(Res);
5659 }
5660 return nullptr;
5661 }
5662 case Intrinsic::invariant_start:
5663 // Discard region information.
5664 setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
5665 return nullptr;
5666 case Intrinsic::invariant_end:
5667 // Discard region information.
5668 return nullptr;
5669 case Intrinsic::clear_cache:
5670 return TLI.getClearCacheBuiltinName();
5671 case Intrinsic::donothing:
5672 // ignore
5673 return nullptr;
5674 case Intrinsic::experimental_stackmap: {
5675 visitStackmap(I);
5676 return nullptr;
5677 }
5678 case Intrinsic::experimental_patchpoint_void:
5679 case Intrinsic::experimental_patchpoint_i64: {
5680 visitPatchpoint(&I);
5681 return nullptr;
5682 }
5683 case Intrinsic::experimental_gc_statepoint: {
5684 LowerStatepoint(ImmutableStatepoint(&I));
5685 return nullptr;
5686 }
5687 case Intrinsic::experimental_gc_result: {
5688 visitGCResult(cast<GCResultInst>(I));
5689 return nullptr;
5690 }
5691 case Intrinsic::experimental_gc_relocate: {
5692 visitGCRelocate(cast<GCRelocateInst>(I));
5693 return nullptr;
5694 }
5695 case Intrinsic::instrprof_increment:
5696 llvm_unreachable("instrprof failed to lower an increment")::llvm::llvm_unreachable_internal("instrprof failed to lower an increment"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5696)
;
5697 case Intrinsic::instrprof_value_profile:
5698 llvm_unreachable("instrprof failed to lower a value profiling call")::llvm::llvm_unreachable_internal("instrprof failed to lower a value profiling call"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5698)
;
5699 case Intrinsic::localescape: {
5700 MachineFunction &MF = DAG.getMachineFunction();
5701 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5702
5703 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
5704 // is the same on all targets.
5705 for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
5706 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
5707 if (isa<ConstantPointerNull>(Arg))
5708 continue; // Skip null pointers. They represent a hole in index space.
5709 AllocaInst *Slot = cast<AllocaInst>(Arg);
5710 assert(FuncInfo.StaticAllocaMap.count(Slot) &&((FuncInfo.StaticAllocaMap.count(Slot) && "can only escape static allocas"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5711, __PRETTY_FUNCTION__))
5711 "can only escape static allocas")((FuncInfo.StaticAllocaMap.count(Slot) && "can only escape static allocas"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5711, __PRETTY_FUNCTION__))
;
5712 int FI = FuncInfo.StaticAllocaMap[Slot];
5713 MCSymbol *FrameAllocSym =
5714 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5715 GlobalValue::getRealLinkageName(MF.getName()), Idx);
5716 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
5717 TII->get(TargetOpcode::LOCAL_ESCAPE))
5718 .addSym(FrameAllocSym)
5719 .addFrameIndex(FI);
5720 }
5721
5722 return nullptr;
5723 }
5724
5725 case Intrinsic::localrecover: {
5726 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
5727 MachineFunction &MF = DAG.getMachineFunction();
5728 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
5729
5730 // Get the symbol that defines the frame offset.
5731 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
5732 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
5733 unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX2147483647));
5734 MCSymbol *FrameAllocSym =
5735 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5736 GlobalValue::getRealLinkageName(Fn->getName()), IdxVal);
5737
5738 // Create a MCSymbol for the label to avoid any target lowering
5739 // that would make this PC relative.
5740 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
5741 SDValue OffsetVal =
5742 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
5743
5744 // Add the offset to the FP.
5745 Value *FP = I.getArgOperand(1);
5746 SDValue FPVal = getValue(FP);
5747 SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
5748 setValue(&I, Add);
5749
5750 return nullptr;
5751 }
5752
5753 case Intrinsic::eh_exceptionpointer:
5754 case Intrinsic::eh_exceptioncode: {
5755 // Get the exception pointer vreg, copy from it, and resize it to fit.
5756 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
5757 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5758 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
5759 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
5760 SDValue N =
5761 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
5762 if (Intrinsic == Intrinsic::eh_exceptioncode)
5763 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
5764 setValue(&I, N);
5765 return nullptr;
5766 }
5767
5768 case Intrinsic::experimental_deoptimize:
5769 LowerDeoptimizeCall(&I);
5770 return nullptr;
5771 }
5772}
5773
5774void SelectionDAGBuilder::visitConstrainedFPIntrinsic(const CallInst &I,
5775 unsigned Intrinsic) {
5776 SDLoc sdl = getCurSDLoc();
5777 unsigned Opcode;
5778 switch (Intrinsic) {
5779 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5779)
; // Can't reach here.
5780 case Intrinsic::experimental_constrained_fadd:
5781 Opcode = ISD::STRICT_FADD;
5782 break;
5783 case Intrinsic::experimental_constrained_fsub:
5784 Opcode = ISD::STRICT_FSUB;
5785 break;
5786 case Intrinsic::experimental_constrained_fmul:
5787 Opcode = ISD::STRICT_FMUL;
5788 break;
5789 case Intrinsic::experimental_constrained_fdiv:
5790 Opcode = ISD::STRICT_FDIV;
5791 break;
5792 case Intrinsic::experimental_constrained_frem:
5793 Opcode = ISD::STRICT_FREM;
5794 break;
5795 }
5796 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5797 SDValue Chain = getRoot();
5798 SDValue Ops[3] = { Chain, getValue(I.getArgOperand(0)),
5799 getValue(I.getArgOperand(1)) };
5800 SmallVector<EVT, 4> ValueVTs;
5801 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5802 ValueVTs.push_back(MVT::Other); // Out chain
5803
5804 SDVTList VTs = DAG.getVTList(ValueVTs);
5805 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Ops);
5806
5807 assert(Result.getNode()->getNumValues() == 2)((Result.getNode()->getNumValues() == 2) ? static_cast<
void> (0) : __assert_fail ("Result.getNode()->getNumValues() == 2"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5807, __PRETTY_FUNCTION__))
;
5808 SDValue OutChain = Result.getValue(1);
5809 DAG.setRoot(OutChain);
5810 SDValue FPResult = Result.getValue(0);
5811 setValue(&I, FPResult);
5812}
5813
5814std::pair<SDValue, SDValue>
5815SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
5816 const BasicBlock *EHPadBB) {
5817 MachineFunction &MF = DAG.getMachineFunction();
5818 MachineModuleInfo &MMI = MF.getMMI();
5819 MCSymbol *BeginLabel = nullptr;
5820
5821 if (EHPadBB) {
5822 // Insert a label before the invoke call to mark the try range. This can be
5823 // used to detect deletion of the invoke via the MachineModuleInfo.
5824 BeginLabel = MMI.getContext().createTempSymbol();
5825
5826 // For SjLj, keep track of which landing pads go with which invokes
5827 // so as to maintain the ordering of pads in the LSDA.
5828 unsigned CallSiteIndex = MMI.getCurrentCallSite();
5829 if (CallSiteIndex) {
5830 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
5831 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
5832
5833 // Now that the call site is handled, stop tracking it.
5834 MMI.setCurrentCallSite(0);
5835 }
5836
5837 // Both PendingLoads and PendingExports must be flushed here;
5838 // this call might not return.
5839 (void)getRoot();
5840 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
5841
5842 CLI.setChain(getRoot());
5843 }
5844 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5845 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5846
5847 assert((CLI.IsTailCall || Result.second.getNode()) &&(((CLI.IsTailCall || Result.second.getNode()) && "Non-null chain expected with non-tail call!"
) ? static_cast<void> (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5848, __PRETTY_FUNCTION__))
5848 "Non-null chain expected with non-tail call!")(((CLI.IsTailCall || Result.second.getNode()) && "Non-null chain expected with non-tail call!"
) ? static_cast<void> (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5848, __PRETTY_FUNCTION__))
;
5849 assert((Result.second.getNode() || !Result.first.getNode()) &&(((Result.second.getNode() || !Result.first.getNode()) &&
"Null value expected with tail call!") ? static_cast<void
> (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5850, __PRETTY_FUNCTION__))
5850 "Null value expected with tail call!")(((Result.second.getNode() || !Result.first.getNode()) &&
"Null value expected with tail call!") ? static_cast<void
> (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5850, __PRETTY_FUNCTION__))
;
5851
5852 if (!Result.second.getNode()) {
5853 // As a special case, a null chain means that a tail call has been emitted
5854 // and the DAG root is already updated.
5855 HasTailCall = true;
5856
5857 // Since there's no actual continuation from this block, nothing can be
5858 // relying on us setting vregs for them.
5859 PendingExports.clear();
5860 } else {
5861 DAG.setRoot(Result.second);
5862 }
5863
5864 if (EHPadBB) {
5865 // Insert a label at the end of the invoke call to mark the try range. This
5866 // can be used to detect deletion of the invoke via the MachineModuleInfo.
5867 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
5868 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
5869
5870 // Inform MachineModuleInfo of range.
5871 if (MF.hasEHFunclets()) {
5872 assert(CLI.CS)((CLI.CS) ? static_cast<void> (0) : __assert_fail ("CLI.CS"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5872, __PRETTY_FUNCTION__))
;
5873 WinEHFuncInfo *EHInfo = DAG.getMachineFunction().getWinEHFuncInfo();
5874 EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS->getInstruction()),
5875 BeginLabel, EndLabel);
5876 } else {
5877 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
5878 }
5879 }
5880
5881 return Result;
5882}
5883
5884void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
5885 bool isTailCall,
5886 const BasicBlock *EHPadBB) {
5887 auto &DL = DAG.getDataLayout();
5888 FunctionType *FTy = CS.getFunctionType();
5889 Type *RetTy = CS.getType();
5890
5891 TargetLowering::ArgListTy Args;
5892 TargetLowering::ArgListEntry Entry;
5893 Args.reserve(CS.arg_size());
5894
5895 const Value *SwiftErrorVal = nullptr;
5896 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5897
5898 // We can't tail call inside a function with a swifterror argument. Lowering
5899 // does not support this yet. It would have to move into the swifterror
5900 // register before the call.
5901 auto *Caller = CS.getInstruction()->getParent()->getParent();
5902 if (TLI.supportSwiftError() &&
5903 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
5904 isTailCall = false;
5905
5906 for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
5907 i != e; ++i) {
5908 const Value *V = *i;
5909
5910 // Skip empty types
5911 if (V->getType()->isEmptyTy())
5912 continue;
5913
5914 SDValue ArgNode = getValue(V);
5915 Entry.Node = ArgNode; Entry.Ty = V->getType();
5916
5917 // Skip the first return-type Attribute to get to params.
5918 Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
5919
5920 // Use swifterror virtual register as input to the call.
5921 if (Entry.isSwiftError && TLI.supportSwiftError()) {
5922 SwiftErrorVal = V;
5923 // We find the virtual register for the actual swifterror argument.
5924 // Instead of using the Value, we use the virtual register instead.
5925 Entry.Node =
5926 DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVReg(FuncInfo.MBB, V),
5927 EVT(TLI.getPointerTy(DL)));
5928 }
5929
5930 Args.push_back(Entry);
5931
5932 // If we have an explicit sret argument that is an Instruction, (i.e., it
5933 // might point to function-local memory), we can't meaningfully tail-call.
5934 if (Entry.isSRet && isa<Instruction>(V))
5935 isTailCall = false;
5936 }
5937
5938 // Check if target-independent constraints permit a tail call here.
5939 // Target-dependent constraints are checked within TLI->LowerCallTo.
5940 if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
5941 isTailCall = false;
5942
5943 // Disable tail calls if there is an swifterror argument. Targets have not
5944 // been updated to support tail calls.
5945 if (TLI.supportSwiftError() && SwiftErrorVal)
5946 isTailCall = false;
5947
5948 TargetLowering::CallLoweringInfo CLI(DAG);
5949 CLI.setDebugLoc(getCurSDLoc())
5950 .setChain(getRoot())
5951 .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
5952 .setTailCall(isTailCall)
5953 .setConvergent(CS.isConvergent());
5954 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
5955
5956 if (Result.first.getNode()) {
5957 const Instruction *Inst = CS.getInstruction();
5958 Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
5959 setValue(Inst, Result.first);
5960 }
5961
5962 // The last element of CLI.InVals has the SDValue for swifterror return.
5963 // Here we copy it to a virtual register and update SwiftErrorMap for
5964 // book-keeping.
5965 if (SwiftErrorVal && TLI.supportSwiftError()) {
5966 // Get the last element of InVals.
5967 SDValue Src = CLI.InVals.back();
5968 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
5969 unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
5970 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
5971 // We update the virtual register for the actual swifterror argument.
5972 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
5973 DAG.setRoot(CopyNode);
5974 }
5975}
5976
5977/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
5978/// value is equal or not-equal to zero.
5979static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
5980 for (const User *U : V->users()) {
5981 if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
5982 if (IC->isEquality())
5983 if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
5984 if (C->isNullValue())
5985 continue;
5986 // Unknown instruction.
5987 return false;
5988 }
5989 return true;
5990}
5991
5992static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
5993 Type *LoadTy,
5994 SelectionDAGBuilder &Builder) {
5995
5996 // Check to see if this load can be trivially constant folded, e.g. if the
5997 // input is from a string literal.
5998 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
5999 // Cast pointer to the type we really want to load.
6000 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
6001 PointerType::getUnqual(LoadTy));
6002
6003 if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
6004 const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
6005 return Builder.getValue(LoadCst);
6006 }
6007
6008 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
6009 // still constant memory, the input chain can be the entry node.
6010 SDValue Root;
6011 bool ConstantMemory = false;
6012
6013 // Do not serialize (non-volatile) loads of constant memory with anything.
6014 if (Builder.AA->pointsToConstantMemory(PtrVal)) {
6015 Root = Builder.DAG.getEntryNode();
6016 ConstantMemory = true;
6017 } else {
6018 // Do not serialize non-volatile loads against each other.
6019 Root = Builder.DAG.getRoot();
6020 }
6021
6022 SDValue Ptr = Builder.getValue(PtrVal);
6023 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
6024 Ptr, MachinePointerInfo(PtrVal),
6025 /* Alignment = */ 1);
6026
6027 if (!ConstantMemory)
6028 Builder.PendingLoads.push_back(LoadVal.getValue(1));
6029 return LoadVal;
6030}
6031
6032/// processIntegerCallValue - Record the value for an instruction that
6033/// produces an integer result, converting the type where necessary.
6034void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
6035 SDValue Value,
6036 bool IsSigned) {
6037 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6038 I.getType(), true);
6039 if (IsSigned)
6040 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
6041 else
6042 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
6043 setValue(&I, Value);
6044}
6045
6046/// See if we can lower a memcmp call into an optimized form. If so, return
6047/// true and lower it, otherwise return false and it will be lowered like a
6048/// normal call.
6049/// The caller already checked that \p I calls the appropriate LibFunc with a
6050/// correct prototype.
6051bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
6052 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
6053 const Value *Size = I.getArgOperand(2);
6054 const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6055 if (CSize && CSize->getZExtValue() == 0) {
6056 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6057 I.getType(), true);
6058 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6059 return true;
6060 }
6061
6062 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6063 std::pair<SDValue, SDValue> Res =
6064 TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6065 getValue(LHS), getValue(RHS), getValue(Size),
6066 MachinePointerInfo(LHS),
6067 MachinePointerInfo(RHS));
6068 if (Res.first.getNode()) {
6069 processIntegerCallValue(I, Res.first, true);
6070 PendingLoads.push_back(Res.second);
6071 return true;
6072 }
6073
6074 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
6075 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
6076 if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
6077 bool ActuallyDoIt = true;
6078 MVT LoadVT;
6079 Type *LoadTy;
6080 switch (CSize->getZExtValue()) {
6081 default:
6082 LoadVT = MVT::Other;
6083 LoadTy = nullptr;
6084 ActuallyDoIt = false;
6085 break;
6086 case 2:
6087 LoadVT = MVT::i16;
6088 LoadTy = Type::getInt16Ty(CSize->getContext());
6089 break;
6090 case 4:
6091 LoadVT = MVT::i32;
6092 LoadTy = Type::getInt32Ty(CSize->getContext());
6093 break;
6094 case 8:
6095 LoadVT = MVT::i64;
6096 LoadTy = Type::getInt64Ty(CSize->getContext());
6097 break;
6098 /*
6099 case 16:
6100 LoadVT = MVT::v4i32;
6101 LoadTy = Type::getInt32Ty(CSize->getContext());
6102 LoadTy = VectorType::get(LoadTy, 4);
6103 break;
6104 */
6105 }
6106
6107 // This turns into unaligned loads. We only do this if the target natively
6108 // supports the MVT we'll be loading or if it is small enough (<= 4) that
6109 // we'll only produce a small number of byte loads.
6110
6111 // Require that we can find a legal MVT, and only do this if the target
6112 // supports unaligned loads of that type. Expanding into byte loads would
6113 // bloat the code.
6114 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6115 if (ActuallyDoIt && CSize->getZExtValue() > 4) {
6116 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6117 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6118 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6119 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6120 // TODO: Check alignment of src and dest ptrs.
6121 if (!TLI.isTypeLegal(LoadVT) ||
6122 !TLI.allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
6123 !TLI.allowsMisalignedMemoryAccesses(LoadVT, DstAS))
6124 ActuallyDoIt = false;
6125 }
6126
6127 if (ActuallyDoIt) {
6128 SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
6129 SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
6130
6131 SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
6132 ISD::SETNE);
6133 processIntegerCallValue(I, Res, false);
6134 return true;
6135 }
6136 }
6137
6138
6139 return false;
6140}
6141
6142/// See if we can lower a memchr call into an optimized form. If so, return
6143/// true and lower it, otherwise return false and it will be lowered like a
6144/// normal call.
6145/// The caller already checked that \p I calls the appropriate LibFunc with a
6146/// correct prototype.
6147bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6148 const Value *Src = I.getArgOperand(0);
6149 const Value *Char = I.getArgOperand(1);
6150 const Value *Length = I.getArgOperand(2);
6151
6152 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6153 std::pair<SDValue, SDValue> Res =
6154 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
6155 getValue(Src), getValue(Char), getValue(Length),
6156 MachinePointerInfo(Src));
6157 if (Res.first.getNode()) {
6158 setValue(&I, Res.first);
6159 PendingLoads.push_back(Res.second);
6160 return true;
6161 }
6162
6163 return false;
6164}
6165
6166/// See if we can lower a mempcpy call into an optimized form. If so, return
6167/// true and lower it, otherwise return false and it will be lowered like a
6168/// normal call.
6169/// The caller already checked that \p I calls the appropriate LibFunc with a
6170/// correct prototype.
6171bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6172 SDValue Dst = getValue(I.getArgOperand(0));
6173 SDValue Src = getValue(I.getArgOperand(1));
6174 SDValue Size = getValue(I.getArgOperand(2));
6175
6176 unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6177 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6178 unsigned Align = std::min(DstAlign, SrcAlign);
6179 if (Align == 0) // Alignment of one or both could not be inferred.
6180 Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6181
6182 bool isVol = false;
6183 SDLoc sdl = getCurSDLoc();
6184
6185 // In the mempcpy context we need to pass in a false value for isTailCall
6186 // because the return pointer needs to be adjusted by the size of
6187 // the copied memory.
6188 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6189 false, /*isTailCall=*/false,
6190 MachinePointerInfo(I.getArgOperand(0)),
6191 MachinePointerInfo(I.getArgOperand(1)));
6192 assert(MC.getNode() != nullptr &&((MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **"
) ? static_cast<void> (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6193, __PRETTY_FUNCTION__))
6193 "** memcpy should not be lowered as TailCall in mempcpy context **")((MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **"
) ? static_cast<void> (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6193, __PRETTY_FUNCTION__))
;
6194 DAG.setRoot(MC);
6195
6196 // Check if Size needs to be truncated or extended.
6197 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6198
6199 // Adjust return pointer to point just past the last dst byte.
6200 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6201 Dst, Size);
6202 setValue(&I, DstPlusSize);
6203 return true;
6204}
6205
6206/// See if we can lower a strcpy call into an optimized form. If so, return
6207/// true and lower it, otherwise return false and it will be lowered like a
6208/// normal call.
6209/// The caller already checked that \p I calls the appropriate LibFunc with a
6210/// correct prototype.
6211bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6212 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6213
6214 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6215 std::pair<SDValue, SDValue> Res =
6216 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
6217 getValue(Arg0), getValue(Arg1),
6218 MachinePointerInfo(Arg0),
6219 MachinePointerInfo(Arg1), isStpcpy);
6220 if (Res.first.getNode()) {
6221 setValue(&I, Res.first);
6222 DAG.setRoot(Res.second);
6223 return true;
6224 }
6225
6226 return false;
6227}
6228
6229/// See if we can lower a strcmp call into an optimized form. If so, return
6230/// true and lower it, otherwise return false and it will be lowered like a
6231/// normal call.
6232/// The caller already checked that \p I calls the appropriate LibFunc with a
6233/// correct prototype.
6234bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6235 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6236
6237 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6238 std::pair<SDValue, SDValue> Res =
6239 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6240 getValue(Arg0), getValue(Arg1),
6241 MachinePointerInfo(Arg0),
6242 MachinePointerInfo(Arg1));
6243 if (Res.first.getNode()) {
6244 processIntegerCallValue(I, Res.first, true);
6245 PendingLoads.push_back(Res.second);
6246 return true;
6247 }
6248
6249 return false;
6250}
6251
6252/// See if we can lower a strlen call into an optimized form. If so, return
6253/// true and lower it, otherwise return false and it will be lowered like a
6254/// normal call.
6255/// The caller already checked that \p I calls the appropriate LibFunc with a
6256/// correct prototype.
6257bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6258 const Value *Arg0 = I.getArgOperand(0);
6259
6260 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6261 std::pair<SDValue, SDValue> Res =
6262 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
6263 getValue(Arg0), MachinePointerInfo(Arg0));
6264 if (Res.first.getNode()) {
6265 processIntegerCallValue(I, Res.first, false);
6266 PendingLoads.push_back(Res.second);
6267 return true;
6268 }
6269
6270 return false;
6271}
6272
6273/// See if we can lower a strnlen call into an optimized form. If so, return
6274/// true and lower it, otherwise return false and it will be lowered like a
6275/// normal call.
6276/// The caller already checked that \p I calls the appropriate LibFunc with a
6277/// correct prototype.
6278bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6279 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6280
6281 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6282 std::pair<SDValue, SDValue> Res =
6283 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
6284 getValue(Arg0), getValue(Arg1),
6285 MachinePointerInfo(Arg0));
6286 if (Res.first.getNode()) {
6287 processIntegerCallValue(I, Res.first, false);
6288 PendingLoads.push_back(Res.second);
6289 return true;
6290 }
6291
6292 return false;
6293}
6294
6295/// See if we can lower a unary floating-point operation into an SDNode with
6296/// the specified Opcode. If so, return true and lower it, otherwise return
6297/// false and it will be lowered like a normal call.
6298/// The caller already checked that \p I calls the appropriate LibFunc with a
6299/// correct prototype.
6300bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6301 unsigned Opcode) {
6302 // We already checked this call's prototype; verify it doesn't modify errno.
6303 if (!I.onlyReadsMemory())
6304 return false;
6305
6306 SDValue Tmp = getValue(I.getArgOperand(0));
6307 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
6308 return true;
6309}
6310
6311/// See if we can lower a binary floating-point operation into an SDNode with
6312/// the specified Opcode. If so, return true and lower it, otherwise return
6313/// false and it will be lowered like a normal call.
6314/// The caller already checked that \p I calls the appropriate LibFunc with a
6315/// correct prototype.
6316bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
6317 unsigned Opcode) {
6318 // We already checked this call's prototype; verify it doesn't modify errno.
6319 if (!I.onlyReadsMemory())
6320 return false;
6321
6322 SDValue Tmp0 = getValue(I.getArgOperand(0));
6323 SDValue Tmp1 = getValue(I.getArgOperand(1));
6324 EVT VT = Tmp0.getValueType();
6325 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
6326 return true;
6327}
6328
6329void SelectionDAGBuilder::visitCall(const CallInst &I) {
6330 // Handle inline assembly differently.
6331 if (isa<InlineAsm>(I.getCalledValue())) {
6332 visitInlineAsm(&I);
6333 return;
6334 }
6335
6336 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6337 computeUsesVAFloatArgument(I, MMI);
6338
6339 const char *RenameFn = nullptr;
6340 if (Function *F = I.getCalledFunction()) {
6341 if (F->isDeclaration()) {
6342 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
6343 if (unsigned IID = II->getIntrinsicID(F)) {
6344 RenameFn = visitIntrinsicCall(I, IID);
6345 if (!RenameFn)
6346 return;
6347 }
6348 }
6349 if (Intrinsic::ID IID = F->getIntrinsicID()) {
6350 RenameFn = visitIntrinsicCall(I, IID);
6351 if (!RenameFn)
6352 return;
6353 }
6354 }
6355
6356 // Check for well-known libc/libm calls. If the function is internal, it
6357 // can't be a library call. Don't do the check if marked as nobuiltin for
6358 // some reason.
6359 LibFunc Func;
6360 if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
6361 LibInfo->getLibFunc(*F, Func) &&
6362 LibInfo->hasOptimizedCodeGen(Func)) {
6363 switch (Func) {
6364 default: break;
6365 case LibFunc_copysign:
6366 case LibFunc_copysignf:
6367 case LibFunc_copysignl:
6368 // We already checked this call's prototype; verify it doesn't modify
6369 // errno.
6370 if (I.onlyReadsMemory()) {
6371 SDValue LHS = getValue(I.getArgOperand(0));
6372 SDValue RHS = getValue(I.getArgOperand(1));
6373 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
6374 LHS.getValueType(), LHS, RHS));
6375 return;
6376 }
6377 break;
6378 case LibFunc_fabs:
6379 case LibFunc_fabsf:
6380 case LibFunc_fabsl:
6381 if (visitUnaryFloatCall(I, ISD::FABS))
6382 return;
6383 break;
6384 case LibFunc_fmin:
6385 case LibFunc_fminf:
6386 case LibFunc_fminl:
6387 if (visitBinaryFloatCall(I, ISD::FMINNUM))
6388 return;
6389 break;
6390 case LibFunc_fmax:
6391 case LibFunc_fmaxf:
6392 case LibFunc_fmaxl:
6393 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
6394 return;
6395 break;
6396 case LibFunc_sin:
6397 case LibFunc_sinf:
6398 case LibFunc_sinl:
6399 if (visitUnaryFloatCall(I, ISD::FSIN))
6400 return;
6401 break;
6402 case LibFunc_cos:
6403 case LibFunc_cosf:
6404 case LibFunc_cosl:
6405 if (visitUnaryFloatCall(I, ISD::FCOS))
6406 return;
6407 break;
6408 case LibFunc_sqrt:
6409 case LibFunc_sqrtf:
6410 case LibFunc_sqrtl:
6411 case LibFunc_sqrt_finite:
6412 case LibFunc_sqrtf_finite:
6413 case LibFunc_sqrtl_finite:
6414 if (visitUnaryFloatCall(I, ISD::FSQRT))
6415 return;
6416 break;
6417 case LibFunc_floor:
6418 case LibFunc_floorf:
6419 case LibFunc_floorl:
6420 if (visitUnaryFloatCall(I, ISD::FFLOOR))
6421 return;
6422 break;
6423 case LibFunc_nearbyint:
6424 case LibFunc_nearbyintf:
6425 case LibFunc_nearbyintl:
6426 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
6427 return;
6428 break;
6429 case LibFunc_ceil:
6430 case LibFunc_ceilf:
6431 case LibFunc_ceill:
6432 if (visitUnaryFloatCall(I, ISD::FCEIL))
6433 return;
6434 break;
6435 case LibFunc_rint:
6436 case LibFunc_rintf:
6437 case LibFunc_rintl:
6438 if (visitUnaryFloatCall(I, ISD::FRINT))
6439 return;
6440 break;
6441 case LibFunc_round:
6442 case LibFunc_roundf:
6443 case LibFunc_roundl:
6444 if (visitUnaryFloatCall(I, ISD::FROUND))
6445 return;
6446 break;
6447 case LibFunc_trunc:
6448 case LibFunc_truncf:
6449 case LibFunc_truncl:
6450 if (visitUnaryFloatCall(I, ISD::FTRUNC))
6451 return;
6452 break;
6453 case LibFunc_log2:
6454 case LibFunc_log2f:
6455 case LibFunc_log2l:
6456 if (visitUnaryFloatCall(I, ISD::FLOG2))
6457 return;
6458 break;
6459 case LibFunc_exp2:
6460 case LibFunc_exp2f:
6461 case LibFunc_exp2l:
6462 if (visitUnaryFloatCall(I, ISD::FEXP2))
6463 return;
6464 break;
6465 case LibFunc_memcmp:
6466 if (visitMemCmpCall(I))
6467 return;
6468 break;
6469 case LibFunc_mempcpy:
6470 if (visitMemPCpyCall(I))
6471 return;
6472 break;
6473 case LibFunc_memchr:
6474 if (visitMemChrCall(I))
6475 return;
6476 break;
6477 case LibFunc_strcpy:
6478 if (visitStrCpyCall(I, false))
6479 return;
6480 break;
6481 case LibFunc_stpcpy:
6482 if (visitStrCpyCall(I, true))
6483 return;
6484 break;
6485 case LibFunc_strcmp:
6486 if (visitStrCmpCall(I))
6487 return;
6488 break;
6489 case LibFunc_strlen:
6490 if (visitStrLenCall(I))
6491 return;
6492 break;
6493 case LibFunc_strnlen:
6494 if (visitStrNLenCall(I))
6495 return;
6496 break;
6497 }
6498 }
6499 }
6500
6501 SDValue Callee;
6502 if (!RenameFn)
6503 Callee = getValue(I.getCalledValue());
6504 else
6505 Callee = DAG.getExternalSymbol(
6506 RenameFn,
6507 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
6508
6509 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
6510 // have to do anything here to lower funclet bundles.
6511 assert(!I.hasOperandBundlesOtherThan(((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6513, __PRETTY_FUNCTION__))
6512 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6513, __PRETTY_FUNCTION__))
6513 "Cannot lower calls with arbitrary operand bundles!")((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6513, __PRETTY_FUNCTION__))
;
6514
6515 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
6516 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
6517 else
6518 // Check if we can potentially perform a tail call. More detailed checking
6519 // is be done within LowerCallTo, after more information about the call is
6520 // known.
6521 LowerCallTo(&I, Callee, I.isTailCall());
6522}
6523
6524namespace {
6525
6526/// AsmOperandInfo - This contains information for each constraint that we are
6527/// lowering.
6528class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
6529public:
6530 /// CallOperand - If this is the result output operand or a clobber
6531 /// this is null, otherwise it is the incoming operand to the CallInst.
6532 /// This gets modified as the asm is processed.
6533 SDValue CallOperand;
6534
6535 /// AssignedRegs - If this is a register or register class operand, this
6536 /// contains the set of register corresponding to the operand.
6537 RegsForValue AssignedRegs;
6538
6539 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
6540 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
6541 }
6542
6543 /// Whether or not this operand accesses memory
6544 bool hasMemory(const TargetLowering &TLI) const {
6545 // Indirect operand accesses access memory.
6546 if (isIndirect)
6547 return true;
6548
6549 for (const auto &Code : Codes)
6550 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
6551 return true;
6552
6553 return false;
6554 }
6555
6556 /// getCallOperandValEVT - Return the EVT of the Value* that this operand
6557 /// corresponds to. If there is no Value* for this operand, it returns
6558 /// MVT::Other.
6559 EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
6560 const DataLayout &DL) const {
6561 if (!CallOperandVal) return MVT::Other;
6562
6563 if (isa<BasicBlock>(CallOperandVal))
6564 return TLI.getPointerTy(DL);
6565
6566 llvm::Type *OpTy = CallOperandVal->getType();
6567
6568 // FIXME: code duplicated from TargetLowering::ParseConstraints().
6569 // If this is an indirect operand, the operand is a pointer to the
6570 // accessed type.
6571 if (isIndirect) {
6572 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
6573 if (!PtrTy)
6574 report_fatal_error("Indirect operand for inline asm not a pointer!");
6575 OpTy = PtrTy->getElementType();
6576 }
6577
6578 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6579 if (StructType *STy = dyn_cast<StructType>(OpTy))
6580 if (STy->getNumElements() == 1)
6581 OpTy = STy->getElementType(0);
6582
6583 // If OpTy is not a single value, it may be a struct/union that we
6584 // can tile with integers.
6585 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6586 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
6587 switch (BitSize) {
6588 default: break;
6589 case 1:
6590 case 8:
6591 case 16:
6592 case 32:
6593 case 64:
6594 case 128:
6595 OpTy = IntegerType::get(Context, BitSize);
6596 break;
6597 }
6598 }
6599
6600 return TLI.getValueType(DL, OpTy, true);
6601 }
6602};
6603
6604typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
6605
6606} // end anonymous namespace
6607
6608/// Make sure that the output operand \p OpInfo and its corresponding input
6609/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
6610/// out).
6611static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
6612 SDISelAsmOperandInfo &MatchingOpInfo,
6613 SelectionDAG &DAG) {
6614 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
6615 return;
6616
6617 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
6618 const auto &TLI = DAG.getTargetLoweringInfo();
6619
6620 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6621 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
6622 OpInfo.ConstraintVT);
6623 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6624 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
6625 MatchingOpInfo.ConstraintVT);
6626 if ((OpInfo.ConstraintVT.isInteger() !=
6627 MatchingOpInfo.ConstraintVT.isInteger()) ||
6628 (MatchRC.second != InputRC.second)) {
6629 // FIXME: error out in a more elegant fashion
6630 report_fatal_error("Unsupported asm: input constraint"
6631 " with a matching output constraint of"
6632 " incompatible type!");
6633 }
6634 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
6635}
6636
6637/// Get a direct memory input to behave well as an indirect operand.
6638/// This may introduce stores, hence the need for a \p Chain.
6639/// \return The (possibly updated) chain.
6640static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
6641 SDISelAsmOperandInfo &OpInfo,
6642 SelectionDAG &DAG) {
6643 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6644
6645 // If we don't have an indirect input, put it in the constpool if we can,
6646 // otherwise spill it to a stack slot.
6647 // TODO: This isn't quite right. We need to handle these according to
6648 // the addressing mode that the constraint wants. Also, this may take
6649 // an additional register for the computation and we don't want that
6650 // either.
6651
6652 // If the operand is a float, integer, or vector constant, spill to a
6653 // constant pool entry to get its address.
6654 const Value *OpVal = OpInfo.CallOperandVal;
6655 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
6656 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
6657 OpInfo.CallOperand = DAG.getConstantPool(
6658 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
6659 return Chain;
6660 }
6661
6662 // Otherwise, create a stack slot and emit a store to it before the asm.
6663 Type *Ty = OpVal->getType();
6664 auto &DL = DAG.getDataLayout();
6665 uint64_t TySize = DL.getTypeAllocSize(Ty);
6666 unsigned Align = DL.getPrefTypeAlignment(Ty);
6667 MachineFunction &MF = DAG.getMachineFunction();
6668 int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
6669 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy(DL));
6670 Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
6671 MachinePointerInfo::getFixedStack(MF, SSFI));
6672 OpInfo.CallOperand = StackSlot;
6673
6674 return Chain;
6675}
6676
6677/// GetRegistersForValue - Assign registers (virtual or physical) for the
6678/// specified operand. We prefer to assign virtual registers, to allow the
6679/// register allocator to handle the assignment process. However, if the asm
6680/// uses features that we can't model on machineinstrs, we have SDISel do the
6681/// allocation. This produces generally horrible, but correct, code.
6682///
6683/// OpInfo describes the operand.
6684///
6685static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
6686 const SDLoc &DL,
6687 SDISelAsmOperandInfo &OpInfo) {
6688 LLVMContext &Context = *DAG.getContext();
6689
6690 MachineFunction &MF = DAG.getMachineFunction();
6691 SmallVector<unsigned, 4> Regs;
6692
6693 // If this is a constraint for a single physreg, or a constraint for a
6694 // register class, find it.
6695 std::pair<unsigned, const TargetRegisterClass *> PhysReg =
6696 TLI.getRegForInlineAsmConstraint(MF.getSubtarget().getRegisterInfo(),
1
Value assigned to 'PhysReg.second'
6697 OpInfo.ConstraintCode,
6698 OpInfo.ConstraintVT);
6699
6700 unsigned NumRegs = 1;
6701 if (OpInfo.ConstraintVT != MVT::Other) {
2
Taking true branch
6702 // If this is a FP input in an integer register (or visa versa) insert a bit
6703 // cast of the input value. More generally, handle any case where the input
6704 // value disagrees with the register class we plan to stick this in.
6705 if (OpInfo.Type == InlineAsm::isInput &&
3
Assuming the condition is true
4
Assuming pointer value is null
6706 PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) {
5
Assuming the condition is false
6707 // Try to convert to the first EVT that the reg class contains. If the
6708 // types are identical size, use a bitcast to convert (e.g. two differing
6709 // vector types).
6710 MVT RegVT = *PhysReg.second->vt_begin();
6711 if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
6712 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6713 RegVT, OpInfo.CallOperand);
6714 OpInfo.ConstraintVT = RegVT;
6715 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
6716 // If the input is a FP value and we want it in FP registers, do a
6717 // bitcast to the corresponding integer type. This turns an f64 value
6718 // into i64, which can be passed with two i32 values on a 32-bit
6719 // machine.
6720 RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
6721 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6722 RegVT, OpInfo.CallOperand);
6723 OpInfo.ConstraintVT = RegVT;
6724 }
6725 }
6726
6727 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
6728 }
6729
6730 MVT RegVT;
6731 EVT ValueVT = OpInfo.ConstraintVT;
6732
6733 // If this is a constraint for a specific physical register, like {r17},
6734 // assign it now.
6735 if (unsigned AssignedReg = PhysReg.first) {
6
Assuming 'AssignedReg' is not equal to 0
7
Taking true branch
6736 const TargetRegisterClass *RC = PhysReg.second;
8
'RC' initialized to a null pointer value
6737 if (OpInfo.ConstraintVT == MVT::Other)
9
Taking false branch
6738 ValueVT = *RC->vt_begin();
6739
6740 // Get the actual register value type. This is important, because the user
6741 // may have asked for (e.g.) the AX register in i32 type. We need to
6742 // remember that AX is actually i16 to get the right extension.
6743 RegVT = *RC->vt_begin();
10
Called C++ object pointer is null
6744
6745 // This is a explicit reference to a physical register.
6746 Regs.push_back(AssignedReg);
6747
6748 // If this is an expanded reference, add the rest of the regs to Regs.
6749 if (NumRegs != 1) {
6750 TargetRegisterClass::iterator I = RC->begin();
6751 for (; *I != AssignedReg; ++I)
6752 assert(I != RC->end() && "Didn't find reg!")((I != RC->end() && "Didn't find reg!") ? static_cast
<void> (0) : __assert_fail ("I != RC->end() && \"Didn't find reg!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6752, __PRETTY_FUNCTION__))
;
6753
6754 // Already added the first reg.
6755 --NumRegs; ++I;
6756 for (; NumRegs; --NumRegs, ++I) {
6757 assert(I != RC->end() && "Ran out of registers to allocate!")((I != RC->end() && "Ran out of registers to allocate!"
) ? static_cast<void> (0) : __assert_fail ("I != RC->end() && \"Ran out of registers to allocate!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6757, __PRETTY_FUNCTION__))
;
6758 Regs.push_back(*I);
6759 }
6760 }
6761
6762 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6763 return;
6764 }
6765
6766 // Otherwise, if this was a reference to an LLVM register class, create vregs
6767 // for this reference.
6768 if (const TargetRegisterClass *RC = PhysReg.second) {
6769 RegVT = *RC->vt_begin();
6770 if (OpInfo.ConstraintVT == MVT::Other)
6771 ValueVT = RegVT;
6772
6773 // Create the appropriate number of virtual registers.
6774 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6775 for (; NumRegs; --NumRegs)
6776 Regs.push_back(RegInfo.createVirtualRegister(RC));
6777
6778 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6779 return;
6780 }
6781
6782 // Otherwise, we couldn't allocate enough registers for this.
6783}
6784
6785static unsigned
6786findMatchingInlineAsmOperand(unsigned OperandNo,
6787 const std::vector<SDValue> &AsmNodeOperands) {
6788 // Scan until we find the definition we already emitted of this operand.
6789 unsigned CurOp = InlineAsm::Op_FirstOperand;
6790 for (; OperandNo; --OperandNo) {
6791 // Advance to the next operand.
6792 unsigned OpFlag =
6793 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
6794 assert((InlineAsm::isRegDefKind(OpFlag) ||(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6797, __PRETTY_FUNCTION__))
6795 InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6797, __PRETTY_FUNCTION__))
6796 InlineAsm::isMemKind(OpFlag)) &&(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6797, __PRETTY_FUNCTION__))
6797 "Skipped past definitions?")(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6797, __PRETTY_FUNCTION__))
;
6798 CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
6799 }
6800 return CurOp;
6801}
6802
6803/// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
6804/// \return true if it has succeeded, false otherwise
6805static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
6806 MVT RegVT, SelectionDAG &DAG) {
6807 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6808 MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
6809 for (unsigned i = 0, e = NumRegs; i != e; ++i) {
6810 if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
6811 Regs.push_back(RegInfo.createVirtualRegister(RC));
6812 else
6813 return false;
6814 }
6815 return true;
6816}
6817
6818class ExtraFlags {
6819 unsigned Flags = 0;
6820
6821public:
6822 explicit ExtraFlags(ImmutableCallSite CS) {
6823 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6824 if (IA->hasSideEffects())
6825 Flags |= InlineAsm::Extra_HasSideEffects;
6826 if (IA->isAlignStack())
6827 Flags |= InlineAsm::Extra_IsAlignStack;
6828 if (CS.isConvergent())
6829 Flags |= InlineAsm::Extra_IsConvergent;
6830 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
6831 }
6832
6833 void update(const llvm::TargetLowering::AsmOperandInfo &OpInfo) {
6834 // Ideally, we would only check against memory constraints. However, the
6835 // meaning of an Other constraint can be target-specific and we can't easily
6836 // reason about it. Therefore, be conservative and set MayLoad/MayStore
6837 // for Other constraints as well.
6838 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
6839 OpInfo.ConstraintType == TargetLowering::C_Other) {
6840 if (OpInfo.Type == InlineAsm::isInput)
6841 Flags |= InlineAsm::Extra_MayLoad;
6842 else if (OpInfo.Type == InlineAsm::isOutput)
6843 Flags |= InlineAsm::Extra_MayStore;
6844 else if (OpInfo.Type == InlineAsm::isClobber)
6845 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
6846 }
6847 }
6848
6849 unsigned get() const { return Flags; }
6850};
6851
6852/// visitInlineAsm - Handle a call to an InlineAsm object.
6853///
6854void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
6855 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6856
6857 /// ConstraintOperands - Information about all of the constraints.
6858 SDISelAsmOperandInfoVector ConstraintOperands;
6859
6860 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6861 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
6862 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
6863
6864 bool hasMemory = false;
6865
6866 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6867 ExtraFlags ExtraInfo(CS);
6868
6869 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6870 unsigned ResNo = 0; // ResNo - The result number of the next output.
6871 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
6872 ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
6873 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
6874
6875 MVT OpVT = MVT::Other;
6876
6877 // Compute the value type for each operand.
6878 if (OpInfo.Type == InlineAsm::isInput ||
6879 (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
6880 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
6881
6882 // Process the call argument. BasicBlocks are labels, currently appearing
6883 // only in asm's.
6884 if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
6885 OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
6886 } else {
6887 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
6888 }
6889
6890 OpVT =
6891 OpInfo
6892 .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
6893 .getSimpleVT();
6894 }
6895
6896 if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
6897 // The return value of the call is this value. As such, there is no
6898 // corresponding argument.
6899 assert(!CS.getType()->isVoidTy() && "Bad inline asm!")((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6899, __PRETTY_FUNCTION__))
;
6900 if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
6901 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
6902 STy->getElementType(ResNo));
6903 } else {
6904 assert(ResNo == 0 && "Asm only has one result!")((ResNo == 0 && "Asm only has one result!") ? static_cast
<void> (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6904, __PRETTY_FUNCTION__))
;
6905 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
6906 }
6907 ++ResNo;
6908 }
6909
6910 OpInfo.ConstraintVT = OpVT;
6911
6912 if (!hasMemory)
6913 hasMemory = OpInfo.hasMemory(TLI);
6914
6915 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
6916 // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
6917 auto TargetConstraint = TargetConstraints[i];
6918
6919 // Compute the constraint code and ConstraintType to use.
6920 TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
6921
6922 ExtraInfo.update(TargetConstraint);
6923 }
6924
6925 SDValue Chain, Flag;
6926
6927 // We won't need to flush pending loads if this asm doesn't touch
6928 // memory and is nonvolatile.
6929 if (hasMemory || IA->hasSideEffects())
6930 Chain = getRoot();
6931 else
6932 Chain = DAG.getRoot();
6933
6934 // Second pass over the constraints: compute which constraint option to use
6935 // and assign registers to constraints that want a specific physreg.
6936 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6937 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6938
6939 // If this is an output operand with a matching input operand, look up the
6940 // matching input. If their types mismatch, e.g. one is an integer, the
6941 // other is floating point, or their sizes are different, flag it as an
6942 // error.
6943 if (OpInfo.hasMatchingInput()) {
6944 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6945 patchMatchingInput(OpInfo, Input, DAG);
6946 }
6947
6948 // Compute the constraint code and ConstraintType to use.
6949 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
6950
6951 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6952 OpInfo.Type == InlineAsm::isClobber)
6953 continue;
6954
6955 // If this is a memory input, and if the operand is not indirect, do what we
6956 // need to to provide an address for the memory input.
6957 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6958 !OpInfo.isIndirect) {
6959 assert((OpInfo.isMultipleAlternative ||(((OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::
isInput)) && "Can only indirectify direct input operands!"
) ? static_cast<void> (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6961, __PRETTY_FUNCTION__))
6960 (OpInfo.Type == InlineAsm::isInput)) &&(((OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::
isInput)) && "Can only indirectify direct input operands!"
) ? static_cast<void> (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6961, __PRETTY_FUNCTION__))
6961 "Can only indirectify direct input operands!")(((OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::
isInput)) && "Can only indirectify direct input operands!"
) ? static_cast<void> (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6961, __PRETTY_FUNCTION__))
;
6962
6963 // Memory operands really want the address of the value.
6964 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
6965
6966 // There is no longer a Value* corresponding to this operand.
6967 OpInfo.CallOperandVal = nullptr;
6968
6969 // It is now an indirect operand.
6970 OpInfo.isIndirect = true;
6971 }
6972
6973 // If this constraint is for a specific register, allocate it before
6974 // anything else.
6975 if (OpInfo.ConstraintType == TargetLowering::C_Register)
6976 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6977 }
6978
6979 // Third pass - Loop over all of the operands, assigning virtual or physregs
6980 // to register class operands.
6981 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6982 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6983
6984 // C_Register operands have already been allocated, Other/Memory don't need
6985 // to be.
6986 if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
6987 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6988 }
6989
6990 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
6991 std::vector<SDValue> AsmNodeOperands;
6992 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
6993 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
6994 IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
6995
6996 // If we have a !srcloc metadata node associated with it, we want to attach
6997 // this to the ultimately generated inline asm machineinstr. To do this, we
6998 // pass in the third operand as this (potentially null) inline asm MDNode.
6999 const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
7000 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
7001
7002 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7003 // bits as operand 3.
7004 AsmNodeOperands.push_back(DAG.getTargetConstant(
7005 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7006
7007 // Loop over all of the inputs, copying the operand values into the
7008 // appropriate registers and processing the output regs.
7009 RegsForValue RetValRegs;
7010
7011 // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
7012 std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
7013
7014 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7015 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7016
7017 switch (OpInfo.Type) {
7018 case InlineAsm::isOutput: {
7019 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
7020 OpInfo.ConstraintType != TargetLowering::C_Register) {
7021 // Memory output, or 'other' output (e.g. 'X' constraint).
7022 assert(OpInfo.isIndirect && "Memory output must be indirect operand")((OpInfo.isIndirect && "Memory output must be indirect operand"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.isIndirect && \"Memory output must be indirect operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7022, __PRETTY_FUNCTION__))
;
7023
7024 unsigned ConstraintID =
7025 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7026 assert(ConstraintID != InlineAsm::Constraint_Unknown &&((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7027, __PRETTY_FUNCTION__))
7027 "Failed to convert memory constraint code to constraint id.")((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7027, __PRETTY_FUNCTION__))
;
7028
7029 // Add information to the INLINEASM node to know about this output.
7030 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7031 OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
7032 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
7033 MVT::i32));
7034 AsmNodeOperands.push_back(OpInfo.CallOperand);
7035 break;
7036 }
7037
7038 // Otherwise, this is a register or register class output.
7039
7040 // Copy the output from the appropriate register. Find a register that
7041 // we can use.
7042 if (OpInfo.AssignedRegs.Regs.empty()) {
7043 emitInlineAsmError(
7044 CS, "couldn't allocate output register for constraint '" +
7045 Twine(OpInfo.ConstraintCode) + "'");
7046 return;
7047 }
7048
7049 // If this is an indirect operand, store through the pointer after the
7050 // asm.
7051 if (OpInfo.isIndirect) {
7052 IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7053 OpInfo.CallOperandVal));
7054 } else {
7055 // This is the result value of the call.
7056 assert(!CS.getType()->isVoidTy() && "Bad inline asm!")((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7056, __PRETTY_FUNCTION__))
;
7057 // Concatenate this output onto the outputs list.
7058 RetValRegs.append(OpInfo.AssignedRegs);
7059 }
7060
7061 // Add information to the INLINEASM node to know that this register is
7062 // set.
7063 OpInfo.AssignedRegs
7064 .AddInlineAsmOperands(OpInfo.isEarlyClobber
7065 ? InlineAsm::Kind_RegDefEarlyClobber
7066 : InlineAsm::Kind_RegDef,
7067 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7068 break;
7069 }
7070 case InlineAsm::isInput: {
7071 SDValue InOperandVal = OpInfo.CallOperand;
7072
7073 if (OpInfo.isMatchingInputConstraint()) {
7074 // If this is required to match an output register we have already set,
7075 // just use its register.
7076 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7077 AsmNodeOperands);
7078 unsigned OpFlag =
7079 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7080 if (InlineAsm::isRegDefKind(OpFlag) ||
7081 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
7082 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7083 if (OpInfo.isIndirect) {
7084 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7085 emitInlineAsmError(CS, "inline asm not supported yet:"
7086 " don't know how to handle tied "
7087 "indirect register inputs");
7088 return;
7089 }
7090
7091 MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7092 SmallVector<unsigned, 4> Regs;
7093
7094 if (!createVirtualRegs(Regs,
7095 InlineAsm::getNumOperandRegisters(OpFlag),
7096 RegVT, DAG)) {
7097 emitInlineAsmError(CS, "inline asm error: This value type register "
7098 "class is not natively supported!");
7099 return;
7100 }
7101
7102 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7103
7104 SDLoc dl = getCurSDLoc();
7105 // Use the produced MatchedRegs object to
7106 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7107 Chain, &Flag, CS.getInstruction());
7108 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7109 true, OpInfo.getMatchedOperand(), dl,
7110 DAG, AsmNodeOperands);
7111 break;
7112 }
7113
7114 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!")((InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::isMemKind(OpFlag) && \"Unknown matching constraint!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7114, __PRETTY_FUNCTION__))
;
7115 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&((InlineAsm::getNumOperandRegisters(OpFlag) == 1 && "Unexpected number of operands"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7116, __PRETTY_FUNCTION__))
7116 "Unexpected number of operands")((InlineAsm::getNumOperandRegisters(OpFlag) == 1 && "Unexpected number of operands"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7116, __PRETTY_FUNCTION__))
;
7117 // Add information to the INLINEASM node to know about this input.
7118 // See InlineAsm.h isUseOperandTiedToDef.
7119 OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
7120 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7121 OpInfo.getMatchedOperand());
7122 AsmNodeOperands.push_back(DAG.getTargetConstant(
7123 OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7124 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7125 break;
7126 }
7127
7128 // Treat indirect 'X' constraint as memory.
7129 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7130 OpInfo.isIndirect)
7131 OpInfo.ConstraintType = TargetLowering::C_Memory;
7132
7133 if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7134 std::vector<SDValue> Ops;
7135 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7136 Ops, DAG);
7137 if (Ops.empty()) {
7138 emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7139 Twine(OpInfo.ConstraintCode) + "'");
7140 return;
7141 }
7142
7143 // Add information to the INLINEASM node to know about this input.
7144 unsigned ResOpType =
7145 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
7146 AsmNodeOperands.push_back(DAG.getTargetConstant(
7147 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7148 AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7149 break;
7150 }
7151
7152 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7153 assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!")((OpInfo.isIndirect && "Operand must be indirect to be a mem!"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.isIndirect && \"Operand must be indirect to be a mem!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7153, __PRETTY_FUNCTION__))
;
7154 assert(InOperandVal.getValueType() ==((InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout
()) && "Memory operands expect pointer values") ? static_cast
<void> (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7156, __PRETTY_FUNCTION__))
7155 TLI.getPointerTy(DAG.getDataLayout()) &&((InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout
()) && "Memory operands expect pointer values") ? static_cast
<void> (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7156, __PRETTY_FUNCTION__))
7156 "Memory operands expect pointer values")((InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout
()) && "Memory operands expect pointer values") ? static_cast
<void> (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7156, __PRETTY_FUNCTION__))
;
7157
7158 unsigned ConstraintID =
7159 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7160 assert(ConstraintID != InlineAsm::Constraint_Unknown &&((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7161, __PRETTY_FUNCTION__))
7161 "Failed to convert memory constraint code to constraint id.")((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7161, __PRETTY_FUNCTION__))
;
7162
7163 // Add information to the INLINEASM node to know about this input.
7164 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7165 ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7166 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7167 getCurSDLoc(),
7168 MVT::i32));
7169 AsmNodeOperands.push_back(InOperandVal);
7170 break;
7171 }
7172
7173 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||(((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
OpInfo.ConstraintType == TargetLowering::C_Register) &&
"Unknown constraint type!") ? static_cast<void> (0) : __assert_fail
("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7175, __PRETTY_FUNCTION__))
7174 OpInfo.ConstraintType == TargetLowering::C_Register) &&(((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
OpInfo.ConstraintType == TargetLowering::C_Register) &&
"Unknown constraint type!") ? static_cast<void> (0) : __assert_fail
("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7175, __PRETTY_FUNCTION__))
7175 "Unknown constraint type!")(((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
OpInfo.ConstraintType == TargetLowering::C_Register) &&
"Unknown constraint type!") ? static_cast<void> (0) : __assert_fail
("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7175, __PRETTY_FUNCTION__))
;
7176
7177 // TODO: Support this.
7178 if (OpInfo.isIndirect) {
7179 emitInlineAsmError(
7180 CS, "Don't know how to handle indirect register inputs yet "
7181 "for constraint '" +
7182 Twine(OpInfo.ConstraintCode) + "'");
7183 return;
7184 }
7185
7186 // Copy the input into the appropriate registers.
7187 if (OpInfo.AssignedRegs.Regs.empty()) {
7188 emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7189 Twine(OpInfo.ConstraintCode) + "'");
7190 return;
7191 }
7192
7193 SDLoc dl = getCurSDLoc();
7194
7195 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7196 Chain, &Flag, CS.getInstruction());
7197
7198 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7199 dl, DAG, AsmNodeOperands);
7200 break;
7201 }
7202 case InlineAsm::isClobber: {
7203 // Add the clobbered value to the operand list, so that the register
7204 // allocator is aware that the physreg got clobbered.
7205 if (!OpInfo.AssignedRegs.Regs.empty())
7206 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7207 false, 0, getCurSDLoc(), DAG,
7208 AsmNodeOperands);
7209 break;
7210 }
7211 }
7212 }
7213
7214 // Finish up input operands. Set the input chain and add the flag last.
7215 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7216 if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7217
7218 Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
7219 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7220 Flag = Chain.getValue(1);
7221
7222 // If this asm returns a register value, copy the result from that register
7223 // and set it as the value of the call.
7224 if (!RetValRegs.Regs.empty()) {
7225 SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7226 Chain, &Flag, CS.getInstruction());
7227
7228 // FIXME: Why don't we do this for inline asms with MRVs?
7229 if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
7230 EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7231
7232 // If any of the results of the inline asm is a vector, it may have the
7233 // wrong width/num elts. This can happen for register classes that can
7234 // contain multiple different value types. The preg or vreg allocated may
7235 // not have the same VT as was expected. Convert it to the right type
7236 // with bit_convert.
7237 if (ResultType != Val.getValueType() && Val.getValueType().isVector()) {
7238 Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(),
7239 ResultType, Val);
7240
7241 } else if (ResultType != Val.getValueType() &&
7242 ResultType.isInteger() && Val.getValueType().isInteger()) {
7243 // If a result value was tied to an input value, the computed result may
7244 // have a wider width than the expected result. Extract the relevant
7245 // portion.
7246 Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val);
7247 }
7248
7249 assert(ResultType == Val.getValueType() && "Asm result value mismatch!")((ResultType == Val.getValueType() && "Asm result value mismatch!"
) ? static_cast<void> (0) : __assert_fail ("ResultType == Val.getValueType() && \"Asm result value mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7249, __PRETTY_FUNCTION__))
;
7250 }
7251
7252 setValue(CS.getInstruction(), Val);
7253 // Don't need to use this as a chain in this case.
7254 if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
7255 return;
7256 }
7257
7258 std::vector<std::pair<SDValue, const Value *> > StoresToEmit;
7259
7260 // Process indirect outputs, first output all of the flagged copies out of
7261 // physregs.
7262 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
7263 RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
7264 const Value *Ptr = IndirectStoresToEmit[i].second;
7265 SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7266 Chain, &Flag, IA);
7267 StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
7268 }
7269
7270 // Emit the non-flagged stores from the physregs.
7271 SmallVector<SDValue, 8> OutChains;
7272 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
7273 SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
7274 getValue(StoresToEmit[i].second),
7275 MachinePointerInfo(StoresToEmit[i].second));
7276 OutChains.push_back(Val);
7277 }
7278
7279 if (!OutChains.empty())
7280 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
7281
7282 DAG.setRoot(Chain);
7283}
7284
7285void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
7286 const Twine &Message) {
7287 LLVMContext &Ctx = *DAG.getContext();
7288 Ctx.emitError(CS.getInstruction(), Message);
7289
7290 // Make sure we leave the DAG in a valid state
7291 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7292 auto VT = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7293 setValue(CS.getInstruction(), DAG.getUNDEF(VT));
7294}
7295
7296void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
7297 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
7298 MVT::Other, getRoot(),
7299 getValue(I.getArgOperand(0)),
7300 DAG.getSrcValue(I.getArgOperand(0))));
7301}
7302
7303void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
7304 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7305 const DataLayout &DL = DAG.getDataLayout();
7306 SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7307 getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
7308 DAG.getSrcValue(I.getOperand(0)),
7309 DL.getABITypeAlignment(I.getType()));
7310 setValue(&I, V);
7311 DAG.setRoot(V.getValue(1));
7312}
7313
7314void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
7315 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
7316 MVT::Other, getRoot(),
7317 getValue(I.getArgOperand(0)),
7318 DAG.getSrcValue(I.getArgOperand(0))));
7319}
7320
7321void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
7322 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
7323 MVT::Other, getRoot(),
7324 getValue(I.getArgOperand(0)),
7325 getValue(I.getArgOperand(1)),
7326 DAG.getSrcValue(I.getArgOperand(0)),
7327 DAG.getSrcValue(I.getArgOperand(1))));
7328}
7329
7330SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
7331 const Instruction &I,
7332 SDValue Op) {
7333 const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
7334 if (!Range)
7335 return Op;
7336
7337 ConstantRange CR = getConstantRangeFromMetadata(*Range);
7338 if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
7339 return Op;
7340
7341 APInt Lo = CR.getUnsignedMin();
7342 if (!Lo.isMinValue())
7343 return Op;
7344
7345 APInt Hi = CR.getUnsignedMax();
7346 unsigned Bits = Hi.getActiveBits();
7347
7348 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
7349
7350 SDLoc SL = getCurSDLoc();
7351
7352 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
7353 DAG.getValueType(SmallVT));
7354 unsigned NumVals = Op.getNode()->getNumValues();
7355 if (NumVals == 1)
7356 return ZExt;
7357
7358 SmallVector<SDValue, 4> Ops;
7359
7360 Ops.push_back(ZExt);
7361 for (unsigned I = 1; I != NumVals; ++I)
7362 Ops.push_back(Op.getValue(I));
7363
7364 return DAG.getMergeValues(Ops, SL);
7365}
7366
7367/// \brief Populate a CallLowerinInfo (into \p CLI) based on the properties of
7368/// the call being lowered.
7369///
7370/// This is a helper for lowering intrinsics that follow a target calling
7371/// convention or require stack pointer adjustment. Only a subset of the
7372/// intrinsic's operands need to participate in the calling convention.
7373void SelectionDAGBuilder::populateCallLoweringInfo(
7374 TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
7375 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
7376 bool IsPatchPoint) {
7377 TargetLowering::ArgListTy Args;
7378 Args.reserve(NumArgs);
7379
7380 // Populate the argument list.
7381 // Attributes for args start at offset 1, after the return attribute.
7382 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs, AttrI = ArgIdx + 1;
7383 ArgI != ArgE; ++ArgI) {
7384 const Value *V = CS->getOperand(ArgI);
7385
7386 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.")((!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isEmptyTy() && \"Empty type passed to intrinsic.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7386, __PRETTY_FUNCTION__))
;
7387
7388 TargetLowering::ArgListEntry Entry;
7389 Entry.Node = getValue(V);
7390 Entry.Ty = V->getType();
7391 Entry.setAttributes(&CS, AttrI);
7392 Args.push_back(Entry);
7393 }
7394
7395 CLI.setDebugLoc(getCurSDLoc())
7396 .setChain(getRoot())
7397 .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
7398 .setDiscardResult(CS->use_empty())
7399 .setIsPatchPoint(IsPatchPoint);
7400}
7401
7402/// \brief Add a stack map intrinsic call's live variable operands to a stackmap
7403/// or patchpoint target node's operand list.
7404///
7405/// Constants are converted to TargetConstants purely as an optimization to
7406/// avoid constant materialization and register allocation.
7407///
7408/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
7409/// generate addess computation nodes, and so ExpandISelPseudo can convert the
7410/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
7411/// address materialization and register allocation, but may also be required
7412/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
7413/// alloca in the entry block, then the runtime may assume that the alloca's
7414/// StackMap location can be read immediately after compilation and that the
7415/// location is valid at any point during execution (this is similar to the
7416/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
7417/// only available in a register, then the runtime would need to trap when
7418/// execution reaches the StackMap in order to read the alloca's location.
7419static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
7420 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
7421 SelectionDAGBuilder &Builder) {
7422 for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
7423 SDValue OpVal = Builder.getValue(CS.getArgument(i));
7424 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
7425 Ops.push_back(
7426 Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
7427 Ops.push_back(
7428 Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
7429 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
7430 const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
7431 Ops.push_back(Builder.DAG.getTargetFrameIndex(
7432 FI->getIndex(), TLI.getPointerTy(Builder.DAG.getDataLayout())));
7433 } else
7434 Ops.push_back(OpVal);
7435 }
7436}
7437
7438/// \brief Lower llvm.experimental.stackmap directly to its target opcode.
7439void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
7440 // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
7441 // [live variables...])
7442
7443 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.")((CI.getType()->isVoidTy() && "Stackmap cannot return a value."
) ? static_cast<void> (0) : __assert_fail ("CI.getType()->isVoidTy() && \"Stackmap cannot return a value.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7443, __PRETTY_FUNCTION__))
;
7444
7445 SDValue Chain, InFlag, Callee, NullPtr;
7446 SmallVector<SDValue, 32> Ops;
7447
7448 SDLoc DL = getCurSDLoc();
7449 Callee = getValue(CI.getCalledValue());
7450 NullPtr = DAG.getIntPtrConstant(0, DL, true);
7451
7452 // The stackmap intrinsic only records the live variables (the arguemnts
7453 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
7454 // intrinsic, this won't be lowered to a function call. This means we don't
7455 // have to worry about calling conventions and target specific lowering code.
7456 // Instead we perform the call lowering right here.
7457 //
7458 // chain, flag = CALLSEQ_START(chain, 0)
7459 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
7460 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
7461 //
7462 Chain = DAG.getCALLSEQ_START(getRoot(), NullPtr, DL);
7463 InFlag = Chain.getValue(1);
7464
7465 // Add the <id> and <numBytes> constants.
7466 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
7467 Ops.push_back(DAG.getTargetConstant(
7468 cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
7469 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
7470 Ops.push_back(DAG.getTargetConstant(
7471 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
7472 MVT::i32));
7473
7474 // Push live variables for the stack map.
7475 addStackMapLiveVars(&CI, 2, DL, Ops, *this);
7476
7477 // We are not pushing any register mask info here on the operands list,
7478 // because the stackmap doesn't clobber anything.
7479
7480 // Push the chain and the glue flag.
7481 Ops.push_back(Chain);
7482 Ops.push_back(InFlag);
7483
7484 // Create the STACKMAP node.
7485 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7486 SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
7487 Chain = SDValue(SM, 0);
7488 InFlag = Chain.getValue(1);
7489
7490 Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
7491
7492 // Stackmaps don't generate values, so nothing goes into the NodeMap.
7493
7494 // Set the root to the target-lowered call chain.
7495 DAG.setRoot(Chain);
7496
7497 // Inform the Frame Information that we have a stackmap in this function.
7498 FuncInfo.MF->getFrameInfo().setHasStackMap();
7499}
7500
7501/// \brief Lower llvm.experimental.patchpoint directly to its target opcode.
7502void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
7503 const BasicBlock *EHPadBB) {
7504 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
7505 // i32 <numBytes>,
7506 // i8* <target>,
7507 // i32 <numArgs>,
7508 // [Args...],
7509 // [live variables...])
7510
7511 CallingConv::ID CC = CS.getCallingConv();
7512 bool IsAnyRegCC = CC == CallingConv::AnyReg;
7513 bool HasDef = !CS->getType()->isVoidTy();
7514 SDLoc dl = getCurSDLoc();
7515 SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
7516
7517 // Handle immediate and symbolic callees.
7518 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
7519 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
7520 /*isTarget=*/true);
7521 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
7522 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
7523 SDLoc(SymbolicCallee),
7524 SymbolicCallee->getValueType(0));
7525
7526 // Get the real number of arguments participating in the call <numArgs>
7527 SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
7528 unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
7529
7530 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
7531 // Intrinsics include all meta-operands up to but not including CC.
7532 unsigned NumMetaOpers = PatchPointOpers::CCPos;
7533 assert(CS.arg_size() >= NumMetaOpers + NumArgs &&((CS.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? static_cast<void> (0) : __assert_fail ("CS.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7534, __PRETTY_FUNCTION__))
7534 "Not enough arguments provided to the patchpoint intrinsic")((CS.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? static_cast<void> (0) : __assert_fail ("CS.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7534, __PRETTY_FUNCTION__))
;
7535
7536 // For AnyRegCC the arguments are lowered later on manually.
7537 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
7538 Type *ReturnTy =
7539 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
7540
7541 TargetLowering::CallLoweringInfo CLI(DAG);
7542 populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
7543 true);
7544 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7545
7546 SDNode *CallEnd = Result.second.getNode();
7547 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
7548 CallEnd = CallEnd->getOperand(0).getNode();
7549
7550 /// Get a call instruction from the call sequence chain.
7551 /// Tail calls are not allowed.
7552 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&((CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node."
) ? static_cast<void> (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7553, __PRETTY_FUNCTION__))
7553 "Expected a callseq node.")((CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node."
) ? static_cast<void> (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7553, __PRETTY_FUNCTION__))
;
7554 SDNode *Call = CallEnd->getOperand(0).getNode();
7555 bool HasGlue = Call->getGluedNode();
7556
7557 // Replace the target specific call node with the patchable intrinsic.
7558 SmallVector<SDValue, 8> Ops;
7559
7560 // Add the <id> and <numBytes> constants.
7561 SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
7562 Ops.push_back(DAG.getTargetConstant(
7563 cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
7564 SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
7565 Ops.push_back(DAG.getTargetConstant(
7566 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
7567 MVT::i32));
7568
7569 // Add the callee.
7570 Ops.push_back(Callee);
7571
7572 // Adjust <numArgs> to account for any arguments that have been passed on the
7573 // stack instead.
7574 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
7575 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
7576 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
7577 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
7578
7579 // Add the calling convention
7580 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
7581
7582 // Add the arguments we omitted previously. The register allocator should
7583 // place these in any free register.
7584 if (IsAnyRegCC)
7585 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
7586 Ops.push_back(getValue(CS.getArgument(i)));
7587
7588 // Push the arguments from the call instruction up to the register mask.
7589 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
7590 Ops.append(Call->op_begin() + 2, e);
7591
7592 // Push live variables for the stack map.
7593 addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
7594
7595 // Push the register mask info.
7596 if (HasGlue)
7597 Ops.push_back(*(Call->op_end()-2));
7598 else
7599 Ops.push_back(*(Call->op_end()-1));
7600
7601 // Push the chain (this is originally the first operand of the call, but
7602 // becomes now the last or second to last operand).
7603 Ops.push_back(*(Call->op_begin()));
7604
7605 // Push the glue flag (last operand).
7606 if (HasGlue)
7607 Ops.push_back(*(Call->op_end()-1));
7608
7609 SDVTList NodeTys;
7610 if (IsAnyRegCC && HasDef) {
7611 // Create the return types based on the intrinsic definition
7612 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7613 SmallVector<EVT, 3> ValueVTs;
7614 ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
7615 assert(ValueVTs.size() == 1 && "Expected only one return value type.")((ValueVTs.size() == 1 && "Expected only one return value type."
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && \"Expected only one return value type.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7615, __PRETTY_FUNCTION__))
;
7616
7617 // There is always a chain and a glue type at the end
7618 ValueVTs.push_back(MVT::Other);
7619 ValueVTs.push_back(MVT::Glue);
7620 NodeTys = DAG.getVTList(ValueVTs);
7621 } else
7622 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7623
7624 // Replace the target specific call node with a PATCHPOINT node.
7625 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
7626 dl, NodeTys, Ops);
7627
7628 // Update the NodeMap.
7629 if (HasDef) {
7630 if (IsAnyRegCC)
7631 setValue(CS.getInstruction(), SDValue(MN, 0));
7632 else
7633 setValue(CS.getInstruction(), Result.first);
7634 }
7635
7636 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
7637 // call sequence. Furthermore the location of the chain and glue can change
7638 // when the AnyReg calling convention is used and the intrinsic returns a
7639 // value.
7640 if (IsAnyRegCC && HasDef) {
7641 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
7642 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
7643 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
7644 } else
7645 DAG.ReplaceAllUsesWith(Call, MN);
7646 DAG.DeleteNode(Call);
7647
7648 // Inform the Frame Information that we have a patchpoint in this function.
7649 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
7650}
7651
7652/// Returns an AttributeSet representing the attributes applied to the return
7653/// value of the given call.
7654static AttributeSet getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
7655 SmallVector<Attribute::AttrKind, 2> Attrs;
7656 if (CLI.RetSExt)
7657 Attrs.push_back(Attribute::SExt);
7658 if (CLI.RetZExt)
7659 Attrs.push_back(Attribute::ZExt);
7660 if (CLI.IsInReg)
7661 Attrs.push_back(Attribute::InReg);
7662
7663 return AttributeSet::get(CLI.RetTy->getContext(), AttributeSet::ReturnIndex,
7664 Attrs);
7665}
7666
7667/// TargetLowering::LowerCallTo - This is the default LowerCallTo
7668/// implementation, which just calls LowerCall.
7669/// FIXME: When all targets are
7670/// migrated to using LowerCall, this hook should be integrated into SDISel.
7671std::pair<SDValue, SDValue>
7672TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
7673 // Handle the incoming return values from the call.
7674 CLI.Ins.clear();
7675 Type *OrigRetTy = CLI.RetTy;
7676 SmallVector<EVT, 4> RetTys;
7677 SmallVector<uint64_t, 4> Offsets;
7678 auto &DL = CLI.DAG.getDataLayout();
7679 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
7680
7681 SmallVector<ISD::OutputArg, 4> Outs;
7682 GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
7683
7684 bool CanLowerReturn =
7685 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
7686 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
7687
7688 SDValue DemoteStackSlot;
7689 int DemoteStackIdx = -100;
7690 if (!CanLowerReturn) {
7691 // FIXME: equivalent assert?
7692 // assert(!CS.hasInAllocaArgument() &&
7693 // "sret demotion is incompatible with inalloca");
7694 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
7695 unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
7696 MachineFunction &MF = CLI.DAG.getMachineFunction();
7697 DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7698 Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
7699
7700 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getPointerTy(DL));
7701 ArgListEntry Entry;
7702 Entry.Node = DemoteStackSlot;
7703 Entry.Ty = StackSlotPtrType;
7704 Entry.isSExt = false;
7705 Entry.isZExt = false;
7706 Entry.isInReg = false;
7707 Entry.isSRet = true;
7708 Entry.isNest = false;
7709 Entry.isByVal = false;
7710 Entry.isReturned = false;
7711 Entry.isSwiftSelf = false;
7712 Entry.isSwiftError = false;
7713 Entry.Alignment = Align;
7714 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
7715 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
7716
7717 // sret demotion isn't compatible with tail-calls, since the sret argument
7718 // points into the callers stack frame.
7719 CLI.IsTailCall = false;
7720 } else {
7721 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
7722 EVT VT = RetTys[I];
7723 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
7724 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
7725 for (unsigned i = 0; i != NumRegs; ++i) {
7726 ISD::InputArg MyFlags;
7727 MyFlags.VT = RegisterVT;
7728 MyFlags.ArgVT = VT;
7729 MyFlags.Used = CLI.IsReturnValueUsed;
7730 if (CLI.RetSExt)
7731 MyFlags.Flags.setSExt();
7732 if (CLI.RetZExt)
7733 MyFlags.Flags.setZExt();
7734 if (CLI.IsInReg)
7735 MyFlags.Flags.setInReg();
7736 CLI.Ins.push_back(MyFlags);
7737 }
7738 }
7739 }
7740
7741 // We push in swifterror return as the last element of CLI.Ins.
7742 ArgListTy &Args = CLI.getArgs();
7743 if (supportSwiftError()) {
7744 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
7745 if (Args[i].isSwiftError) {
7746 ISD::InputArg MyFlags;
7747 MyFlags.VT = getPointerTy(DL);
7748 MyFlags.ArgVT = EVT(getPointerTy(DL));
7749 MyFlags.Flags.setSwiftError();
7750 CLI.Ins.push_back(MyFlags);
7751 }
7752 }